src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/nat_inlines.h>
  29 #include <nat/nat_affinity.h>
  30 #include <nat/nat_syslog.h>
  31 #include <nat/nat_ha.h>
  32 #include <vnet/fib/fib_table.h>
  33 #include <vnet/fib/ip4_fib.h>
  34 #include <vnet/ip/reass/ip4_sv_reass.h>
  35
  36 #include <vpp/app/version.h>
  37
  38 snat_main_t snat_main;
  39
  40 fib_source_t nat_fib_src_hi;
  41 fib_source_t nat_fib_src_low;
  42
  43 /* *INDENT-OFF* */
  44 /* Hook up input features */
  45 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  46   .arc_name = "ip4-unicast",
  47   .node_name = "nat-pre-in2out",
  48   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  49                                "ip4-sv-reassembly-feature"),
  50 };
  51 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  52   .arc_name = "ip4-unicast",
  53   .node_name = "nat-pre-out2in",
  54   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  55                                "ip4-dhcp-client-detect",
  56                                "ip4-sv-reassembly-feature"),
  57 };
  58 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  59   .arc_name = "ip4-unicast",
  60   .node_name = "nat44-in2out-worker-handoff",
  61   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  62 };
  63 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  64   .arc_name = "ip4-unicast",
  65   .node_name = "nat44-out2in-worker-handoff",
  66   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  67                                "ip4-dhcp-client-detect"),
  68 };
  69 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  70   .arc_name = "ip4-unicast",
  71   .node_name = "nat44-in2out",
  72   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  73 };
  74 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  75   .arc_name = "ip4-unicast",
  76   .node_name = "nat44-out2in",
  77   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  78                                "ip4-dhcp-client-detect"),
  79 };
  80 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat44-classify",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat44-det-in2out",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
  89 };
  90 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  91   .arc_name = "ip4-unicast",
  92   .node_name = "nat44-det-out2in",
  93   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
  94                                "ip4-dhcp-client-detect"),
  95 };
  96 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-det-classify",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 100 };
 101 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 102   .arc_name = "ip4-unicast",
 103   .node_name = "nat44-ed-in2out",
 104   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 105 };
 106 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 107   .arc_name = "ip4-unicast",
 108   .node_name = "nat44-ed-out2in",
 109   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 110                                "ip4-dhcp-client-detect"),
 111 };
 112 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 113   .arc_name = "ip4-unicast",
 114   .node_name = "nat44-ed-classify",
 115   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 116 };
 117 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 118   .arc_name = "ip4-unicast",
 119   .node_name = "nat44-handoff-classify",
 120   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 121 };
 122 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 123   .arc_name = "ip4-unicast",
 124   .node_name = "nat44-in2out-fast",
 125   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 126 };
 127 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 128   .arc_name = "ip4-unicast",
 129   .node_name = "nat44-out2in-fast",
 130   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 131                                "ip4-dhcp-client-detect"),
 132 };
 133 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 134   .arc_name = "ip4-unicast",
 135   .node_name = "nat44-hairpin-dst",
 136   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 139   .arc_name = "ip4-unicast",
 140   .node_name = "nat44-ed-hairpin-dst",
 141   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 142 };
 143
 144 /* Hook up output features */
 145 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 146   .arc_name = "ip4-output",
 147   .node_name = "nat44-in2out-output",
 148   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 149 };
 150 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 151   .arc_name = "ip4-output",
 152   .node_name = "nat44-in2out-output-worker-handoff",
 153   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 154 };
 155 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 156   .arc_name = "ip4-output",
 157   .node_name = "nat44-hairpin-src",
 158   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa","ip4-sv-reassembly-output-feature"),
 159 };
 160 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 161   .arc_name = "ip4-output",
 162   .node_name = "nat44-ed-in2out-output",
 163   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 164   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 165 };
 166 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 167   .arc_name = "ip4-output",
 168   .node_name = "nat44-ed-hairpin-src",
 169   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 170   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 171 };
 172
 173 /* Hook up ip4-local features */
 174 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 175 {
 176   .arc_name = "ip4-local",
 177   .node_name = "nat44-hairpinning",
 178   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 179 };
 180 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 181 {
 182   .arc_name = "ip4-local",
 183   .node_name = "nat44-ed-hairpinning",
 184   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 185 };
 186
 187
 188 VLIB_PLUGIN_REGISTER () = {
 189     .version = VPP_BUILD_VER,
 190     .description = "Network Address Translation (NAT)",
 191 };
 192 /* *INDENT-ON* */
 193
 194 void
 195 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 196                        u8 is_ha)
 197 {
 198   snat_session_key_t key;
 199   clib_bihash_kv_8_8_t kv;
 200   nat_ed_ses_key_t ed_key;
 201   clib_bihash_kv_16_8_t ed_kv;
 202   snat_main_per_thread_data_t *tsm =
 203     vec_elt_at_index (sm->per_thread_data, thread_index);
 204
 205   if (is_fwd_bypass_session (s))
 206     {
 207       if (snat_is_unk_proto_session (s))
 208         {
 209           ed_key.proto = s->in2out.port;
 210           ed_key.r_port = 0;
 211           ed_key.l_port = 0;
 212         }
 213       else
 214         {
 215           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 216           ed_key.l_port = s->in2out.port;
 217           ed_key.r_port = s->ext_host_port;
 218         }
 219       ed_key.l_addr = s->in2out.addr;
 220       ed_key.r_addr = s->ext_host_addr;
 221       ed_key.fib_index = 0;
 222       ed_kv.key[0] = ed_key.as_u64[0];
 223       ed_kv.key[1] = ed_key.as_u64[1];
 224       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 225         nat_elog_warn ("in2out_ed key del failed");
 226       return;
 227     }
 228
 229   /* session lookup tables */
 230   if (is_ed_session (s))
 231     {
 232       if (is_affinity_sessions (s))
 233         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 234                              s->in2out.protocol, s->out2in.port);
 235       ed_key.l_addr = s->out2in.addr;
 236       ed_key.r_addr = s->ext_host_addr;
 237       ed_key.fib_index = s->out2in.fib_index;
 238       if (snat_is_unk_proto_session (s))
 239         {
 240           ed_key.proto = s->in2out.port;
 241           ed_key.r_port = 0;
 242           ed_key.l_port = 0;
 243         }
 244       else
 245         {
 246           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 247           ed_key.l_port = s->out2in.port;
 248           ed_key.r_port = s->ext_host_port;
 249         }
 250       ed_kv.key[0] = ed_key.as_u64[0];
 251       ed_kv.key[1] = ed_key.as_u64[1];
 252       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 253         nat_elog_warn ("out2in_ed key del failed");
 254       ed_key.l_addr = s->in2out.addr;
 255       ed_key.fib_index = s->in2out.fib_index;
 256       if (!snat_is_unk_proto_session (s))
 257         ed_key.l_port = s->in2out.port;
 258       if (is_twice_nat_session (s))
 259         {
 260           ed_key.r_addr = s->ext_host_nat_addr;
 261           ed_key.r_port = s->ext_host_nat_port;
 262         }
 263       ed_kv.key[0] = ed_key.as_u64[0];
 264       ed_kv.key[1] = ed_key.as_u64[1];
 265       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 266         nat_elog_warn ("in2out_ed key del failed");
 267
 268       if (!is_ha)
 269         nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 270                                &s->in2out.addr, s->in2out.port,
 271                                &s->ext_host_nat_addr, s->ext_host_nat_port,
 272                                &s->out2in.addr, s->out2in.port,
 273                                &s->ext_host_addr, s->ext_host_port,
 274                                s->in2out.protocol, is_twice_nat_session (s));
 275     }
 276   else
 277     {
 278       kv.key = s->in2out.as_u64;
 279       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 280         nat_elog_warn ("in2out key del failed");
 281       kv.key = s->out2in.as_u64;
 282       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 283         nat_elog_warn ("out2in key del failed");
 284
 285       if (!is_ha)
 286         nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 287                                  &s->in2out.addr, s->in2out.port,
 288                                  &s->out2in.addr, s->out2in.port,
 289                                  s->in2out.protocol);
 290     }
 291
 292   if (snat_is_unk_proto_session (s))
 293     return;
 294
 295   if (!is_ha)
 296     {
 297       /* log NAT event */
 298       snat_ipfix_logging_nat44_ses_delete (thread_index,
 299                                            s->in2out.addr.as_u32,
 300                                            s->out2in.addr.as_u32,
 301                                            s->in2out.protocol,
 302                                            s->in2out.port,
 303                                            s->out2in.port,
 304                                            s->in2out.fib_index);
 305
 306       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 307                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 308                    thread_index);
 309     }
 310
 311   /* Twice NAT address and port for external host */
 312   if (is_twice_nat_session (s))
 313     {
 314       key.protocol = s->in2out.protocol;
 315       key.port = s->ext_host_nat_port;
 316       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 317       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 318                                           thread_index, &key);
 319     }
 320
 321   if (snat_is_session_static (s))
 322     return;
 323
 324   snat_free_outside_address_and_port (sm->addresses, thread_index,
 325                                       &s->out2in);
 326 }
 327
 328 snat_user_t *
 329 nat_user_get_or_create (snat_main_t * sm, ip4_address_t * addr, u32 fib_index,
 330                         u32 thread_index)
 331 {
 332   snat_user_t *u = 0;
 333   snat_user_key_t user_key;
 334   clib_bihash_kv_8_8_t kv, value;
 335   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 336   dlist_elt_t *per_user_list_head_elt;
 337
 338   user_key.addr.as_u32 = addr->as_u32;
 339   user_key.fib_index = fib_index;
 340   kv.key = user_key.as_u64;
 341
 342   /* Ever heard of the "user" = src ip4 address before? */
 343   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 344     {
 345       /* no, make a new one */
 346       pool_get (tsm->users, u);
 347       clib_memset (u, 0, sizeof (*u));
 348       u->addr.as_u32 = addr->as_u32;
 349       u->fib_index = fib_index;
 350
 351       pool_get (tsm->list_pool, per_user_list_head_elt);
 352
 353       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 354         tsm->list_pool;
 355
 356       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 357
 358       kv.value = u - tsm->users;
 359
 360       /* add user */
 361       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 362         nat_elog_warn ("user_hash keay add failed");
 363
 364       vlib_set_simple_counter (&sm->total_users, thread_index, 0,
 365                                pool_elts (tsm->users));
 366     }
 367   else
 368     {
 369       u = pool_elt_at_index (tsm->users, value.value);
 370     }
 371
 372   return u;
 373 }
 374
 375 snat_session_t *
 376 nat_session_alloc_or_recycle (snat_main_t * sm, snat_user_t * u,
 377                               u32 thread_index, f64 now)
 378 {
 379   snat_session_t *s;
 380   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 381   u32 oldest_per_user_translation_list_index, session_index;
 382   dlist_elt_t *oldest_per_user_translation_list_elt;
 383   dlist_elt_t *per_user_translation_list_elt;
 384
 385   /* Over quota? Recycle the least recently used translation */
 386   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 387     {
 388       oldest_per_user_translation_list_index =
 389         clib_dlist_remove_head (tsm->list_pool,
 390                                 u->sessions_per_user_list_head_index);
 391
 392       ASSERT (oldest_per_user_translation_list_index != ~0);
 393
 394       /* Add it back to the end of the LRU list */
 395       clib_dlist_addtail (tsm->list_pool,
 396                           u->sessions_per_user_list_head_index,
 397                           oldest_per_user_translation_list_index);
 398       /* Get the list element */
 399       oldest_per_user_translation_list_elt =
 400         pool_elt_at_index (tsm->list_pool,
 401                            oldest_per_user_translation_list_index);
 402
 403       /* Get the session index from the list element */
 404       session_index = oldest_per_user_translation_list_elt->value;
 405
 406       /* Get the session */
 407       s = pool_elt_at_index (tsm->sessions, session_index);
 408       nat_free_session_data (sm, s, thread_index, 0);
 409       if (snat_is_session_static (s))
 410         u->nstaticsessions--;
 411       else
 412         u->nsessions--;
 413       s->flags = 0;
 414       s->total_bytes = 0;
 415       s->total_pkts = 0;
 416       s->state = 0;
 417       s->ext_host_addr.as_u32 = 0;
 418       s->ext_host_port = 0;
 419       s->ext_host_nat_addr.as_u32 = 0;
 420       s->ext_host_nat_port = 0;
 421     }
 422   else
 423     {
 424       pool_get (tsm->sessions, s);
 425       clib_memset (s, 0, sizeof (*s));
 426
 427       /* Create list elts */
 428       pool_get (tsm->list_pool, per_user_translation_list_elt);
 429       clib_dlist_init (tsm->list_pool,
 430                        per_user_translation_list_elt - tsm->list_pool);
 431
 432       per_user_translation_list_elt->value = s - tsm->sessions;
 433       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 434       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 435
 436       clib_dlist_addtail (tsm->list_pool,
 437                           s->per_user_list_head_index,
 438                           per_user_translation_list_elt - tsm->list_pool);
 439
 440       s->user_index = u - tsm->users;
 441       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 442                                pool_elts (tsm->sessions));
 443     }
 444
 445   s->ha_last_refreshed = now;
 446
 447   return s;
 448 }
 449
 450 snat_session_t *
 451 nat_ed_session_alloc (snat_main_t * sm, snat_user_t * u, u32 thread_index,
 452                       f64 now)
 453 {
 454   snat_session_t *s;
 455   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 456   dlist_elt_t *per_user_translation_list_elt, *oldest_elt;
 457   u32 oldest_index;
 458   u64 sess_timeout_time;
 459
 460   if (PREDICT_FALSE (!(u->nsessions) && !(u->nstaticsessions)))
 461     goto alloc_new;
 462
 463   oldest_index =
 464     clib_dlist_remove_head (tsm->list_pool,
 465                             u->sessions_per_user_list_head_index);
 466   oldest_elt = pool_elt_at_index (tsm->list_pool, oldest_index);
 467   s = pool_elt_at_index (tsm->sessions, oldest_elt->value);
 468
 469   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
 470   if (now >= sess_timeout_time)
 471     {
 472       clib_dlist_addtail (tsm->list_pool,
 473                           u->sessions_per_user_list_head_index, oldest_index);
 474       nat_free_session_data (sm, s, thread_index, 0);
 475       if (snat_is_session_static (s))
 476         u->nstaticsessions--;
 477       else
 478         u->nsessions--;
 479       s->flags = 0;
 480       s->total_bytes = 0;
 481       s->total_pkts = 0;
 482       s->state = 0;
 483       s->ext_host_addr.as_u32 = 0;
 484       s->ext_host_port = 0;
 485       s->ext_host_nat_addr.as_u32 = 0;
 486       s->ext_host_nat_port = 0;
 487     }
 488   else
 489     {
 490       clib_dlist_addhead (tsm->list_pool,
 491                           u->sessions_per_user_list_head_index, oldest_index);
 492       if ((u->nsessions + u->nstaticsessions) >=
 493           sm->max_translations_per_user)
 494         {
 495           nat_elog_addr (SNAT_LOG_WARNING, "[warn] max translations per user",
 496                          clib_net_to_host_u32 (u->addr.as_u32));
 497           snat_ipfix_logging_max_entries_per_user
 498             (thread_index, sm->max_translations_per_user, u->addr.as_u32);
 499           return 0;
 500         }
 501       else
 502         {
 503         alloc_new:
 504           pool_get (tsm->sessions, s);
 505           clib_memset (s, 0, sizeof (*s));
 506
 507           /* Create list elts */
 508           pool_get (tsm->list_pool, per_user_translation_list_elt);
 509           clib_dlist_init (tsm->list_pool,
 510                            per_user_translation_list_elt - tsm->list_pool);
 511
 512           per_user_translation_list_elt->value = s - tsm->sessions;
 513           s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 514           s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 515
 516           clib_dlist_addtail (tsm->list_pool,
 517                               s->per_user_list_head_index,
 518                               per_user_translation_list_elt - tsm->list_pool);
 519         }
 520
 521       vlib_set_simple_counter (&sm->total_sessions, thread_index, 0,
 522                                pool_elts (tsm->sessions));
 523     }
 524
 525   s->ha_last_refreshed = now;
 526
 527   return s;
 528 }
 529
 530 void
 531 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 532                           int is_add)
 533 {
 534   fib_prefix_t prefix = {
 535     .fp_len = p_len,
 536     .fp_proto = FIB_PROTOCOL_IP4,
 537     .fp_addr = {
 538                 .ip4.as_u32 = addr->as_u32,
 539                 },
 540   };
 541   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 542
 543   if (is_add)
 544     fib_table_entry_update_one_path (fib_index,
 545                                      &prefix,
 546                                      nat_fib_src_low,
 547                                      (FIB_ENTRY_FLAG_CONNECTED |
 548                                       FIB_ENTRY_FLAG_LOCAL |
 549                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 550                                      DPO_PROTO_IP4,
 551                                      NULL,
 552                                      sw_if_index,
 553                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 554   else
 555     fib_table_entry_delete (fib_index, &prefix, nat_fib_src_low);
 556 }
 557
 558 int
 559 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 560                   u8 twice_nat)
 561 {
 562   snat_address_t *ap;
 563   snat_interface_t *i;
 564   vlib_thread_main_t *tm = vlib_get_thread_main ();
 565
 566   if (twice_nat && !sm->endpoint_dependent)
 567     return VNET_API_ERROR_FEATURE_DISABLED;
 568
 569   /* Check if address already exists */
 570   /* *INDENT-OFF* */
 571   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 572     {
 573       if (ap->addr.as_u32 == addr->as_u32)
 574         return VNET_API_ERROR_VALUE_EXIST;
 575     }
 576   /* *INDENT-ON* */
 577
 578   if (twice_nat)
 579     vec_add2 (sm->twice_nat_addresses, ap, 1);
 580   else
 581     vec_add2 (sm->addresses, ap, 1);
 582
 583   ap->addr = *addr;
 584   if (vrf_id != ~0)
 585     ap->fib_index =
 586       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 587                                          nat_fib_src_low);
 588   else
 589     ap->fib_index = ~0;
 590 #define _(N, i, n, s) \
 591   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 592   ap->busy_##n##_ports = 0; \
 593   ap->busy_##n##_ports_per_thread = 0;\
 594   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 595   foreach_snat_protocol
 596 #undef _
 597     if (twice_nat)
 598     return 0;
 599
 600   /* Add external address to FIB */
 601   /* *INDENT-OFF* */
 602   pool_foreach (i, sm->interfaces,
 603   ({
 604     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 605       continue;
 606
 607     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 608     break;
 609   }));
 610   pool_foreach (i, sm->output_feature_interfaces,
 611   ({
 612     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 613       continue;
 614
 615     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 616     break;
 617   }));
 618   /* *INDENT-ON* */
 619
 620   return 0;
 621 }
 622
 623 static int
 624 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 625 {
 626   snat_static_mapping_t *m;
 627   /* *INDENT-OFF* */
 628   pool_foreach (m, sm->static_mappings,
 629   ({
 630       if (is_addr_only_static_mapping (m) ||
 631           is_out2in_only_static_mapping (m) ||
 632           is_identity_static_mapping (m))
 633         continue;
 634       if (m->external_addr.as_u32 == addr.as_u32)
 635         return 1;
 636   }));
 637   /* *INDENT-ON* */
 638
 639   return 0;
 640 }
 641
 642 static void
 643 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 644                                        ip4_address_t l_addr,
 645                                        u16 l_port,
 646                                        u32 sw_if_index,
 647                                        u16 e_port,
 648                                        u32 vrf_id,
 649                                        snat_protocol_t proto,
 650                                        int addr_only, int is_add, u8 * tag,
 651                                        int twice_nat, int out2in_only,
 652                                        int identity_nat)
 653 {
 654   snat_static_map_resolve_t *rp;
 655
 656   vec_add2 (sm->to_resolve, rp, 1);
 657   rp->l_addr.as_u32 = l_addr.as_u32;
 658   rp->l_port = l_port;
 659   rp->sw_if_index = sw_if_index;
 660   rp->e_port = e_port;
 661   rp->vrf_id = vrf_id;
 662   rp->proto = proto;
 663   rp->addr_only = addr_only;
 664   rp->is_add = is_add;
 665   rp->twice_nat = twice_nat;
 666   rp->out2in_only = out2in_only;
 667   rp->identity_nat = identity_nat;
 668   rp->tag = vec_dup (tag);
 669 }
 670
 671 static u32
 672 get_thread_idx_by_port (u16 e_port)
 673 {
 674   snat_main_t *sm = &snat_main;
 675   u32 thread_idx = sm->num_workers;
 676   if (sm->num_workers > 1)
 677     {
 678       thread_idx =
 679         sm->first_worker_index +
 680         sm->workers[(e_port - 1024) / sm->port_per_thread];
 681     }
 682   return thread_idx;
 683 }
 684
 685 int
 686 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 687                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 688                          u32 sw_if_index, snat_protocol_t proto, int is_add,
 689                          twice_nat_type_t twice_nat, u8 out2in_only, u8 * tag,
 690                          u8 identity_nat)
 691 {
 692   snat_main_t *sm = &snat_main;
 693   snat_static_mapping_t *m;
 694   snat_session_key_t m_key;
 695   clib_bihash_kv_8_8_t kv, value;
 696   snat_address_t *a = 0;
 697   u32 fib_index = ~0;
 698   snat_interface_t *interface;
 699   int i;
 700   snat_main_per_thread_data_t *tsm;
 701   snat_user_key_t u_key;
 702   snat_user_t *u;
 703   dlist_elt_t *head, *elt;
 704   u32 elt_index, head_index;
 705   u32 ses_index;
 706   u64 user_index;
 707   snat_session_t *s;
 708   snat_static_map_resolve_t *rp, *rp_match = 0;
 709   nat44_lb_addr_port_t *local;
 710   u32 find = ~0;
 711
 712   if (!sm->endpoint_dependent)
 713     {
 714       if (twice_nat || out2in_only)
 715         return VNET_API_ERROR_FEATURE_DISABLED;
 716     }
 717
 718   /* If the external address is a specific interface address */
 719   if (sw_if_index != ~0)
 720     {
 721       ip4_address_t *first_int_addr;
 722
 723       for (i = 0; i < vec_len (sm->to_resolve); i++)
 724         {
 725           rp = sm->to_resolve + i;
 726           if (rp->sw_if_index != sw_if_index ||
 727               rp->l_addr.as_u32 != l_addr.as_u32 ||
 728               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 729             continue;
 730
 731           if (!addr_only)
 732             {
 733               if ((rp->l_port != l_port && rp->e_port != e_port)
 734                   || rp->proto != proto)
 735                 continue;
 736             }
 737
 738           rp_match = rp;
 739           break;
 740         }
 741
 742       /* Might be already set... */
 743       first_int_addr = ip4_interface_first_address
 744         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 745
 746       if (is_add)
 747         {
 748           if (rp_match)
 749             return VNET_API_ERROR_VALUE_EXIST;
 750
 751           snat_add_static_mapping_when_resolved
 752             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 753              addr_only, is_add, tag, twice_nat, out2in_only, identity_nat);
 754
 755           /* DHCP resolution required? */
 756           if (first_int_addr == 0)
 757             {
 758               return 0;
 759             }
 760           else
 761             {
 762               e_addr.as_u32 = first_int_addr->as_u32;
 763               /* Identity mapping? */
 764               if (l_addr.as_u32 == 0)
 765                 l_addr.as_u32 = e_addr.as_u32;
 766             }
 767         }
 768       else
 769         {
 770           if (!rp_match)
 771             return VNET_API_ERROR_NO_SUCH_ENTRY;
 772
 773           vec_del1 (sm->to_resolve, i);
 774
 775           if (first_int_addr)
 776             {
 777               e_addr.as_u32 = first_int_addr->as_u32;
 778               /* Identity mapping? */
 779               if (l_addr.as_u32 == 0)
 780                 l_addr.as_u32 = e_addr.as_u32;
 781             }
 782           else
 783             return 0;
 784         }
 785     }
 786
 787   m_key.addr = e_addr;
 788   m_key.port = addr_only ? 0 : e_port;
 789   m_key.protocol = addr_only ? 0 : proto;
 790   m_key.fib_index = 0;
 791   kv.key = m_key.as_u64;
 792   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 793     m = 0;
 794   else
 795     m = pool_elt_at_index (sm->static_mappings, value.value);
 796
 797   if (is_add)
 798     {
 799       if (m)
 800         {
 801           if (is_identity_static_mapping (m))
 802             {
 803               /* *INDENT-OFF* */
 804               pool_foreach (local, m->locals,
 805               ({
 806                 if (local->vrf_id == vrf_id)
 807                   return VNET_API_ERROR_VALUE_EXIST;
 808               }));
 809               /* *INDENT-ON* */
 810               pool_get (m->locals, local);
 811               local->vrf_id = vrf_id;
 812               local->fib_index =
 813                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 814                                                    nat_fib_src_low);
 815               m_key.addr = m->local_addr;
 816               m_key.port = m->local_port;
 817               m_key.protocol = m->proto;
 818               m_key.fib_index = local->fib_index;
 819               kv.key = m_key.as_u64;
 820               kv.value = m - sm->static_mappings;
 821               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 822               return 0;
 823             }
 824           else
 825             return VNET_API_ERROR_VALUE_EXIST;
 826         }
 827
 828       if (twice_nat && addr_only)
 829         return VNET_API_ERROR_UNSUPPORTED;
 830
 831       /* Convert VRF id to FIB index */
 832       if (vrf_id != ~0)
 833         fib_index =
 834           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 835                                              nat_fib_src_low);
 836       /* If not specified use inside VRF id from SNAT plugin startup config */
 837       else
 838         {
 839           fib_index = sm->inside_fib_index;
 840           vrf_id = sm->inside_vrf_id;
 841           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
 842         }
 843
 844       if (!(out2in_only || identity_nat))
 845         {
 846           m_key.addr = l_addr;
 847           m_key.port = addr_only ? 0 : l_port;
 848           m_key.protocol = addr_only ? 0 : proto;
 849           m_key.fib_index = fib_index;
 850           kv.key = m_key.as_u64;
 851           if (!clib_bihash_search_8_8
 852               (&sm->static_mapping_by_local, &kv, &value))
 853             return VNET_API_ERROR_VALUE_EXIST;
 854         }
 855
 856       /* Find external address in allocated addresses and reserve port for
 857          address and port pair mapping when dynamic translations enabled */
 858       if (!(addr_only || sm->static_mapping_only || out2in_only))
 859         {
 860           for (i = 0; i < vec_len (sm->addresses); i++)
 861             {
 862               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 863                 {
 864                   a = sm->addresses + i;
 865                   /* External port must be unused */
 866                   switch (proto)
 867                     {
 868 #define _(N, j, n, s) \
 869                     case SNAT_PROTOCOL_##N: \
 870                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 871                         return VNET_API_ERROR_INVALID_VALUE; \
 872                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 873                       if (e_port > 1024) \
 874                         { \
 875                           a->busy_##n##_ports++; \
 876                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 877                         } \
 878                       break;
 879                       foreach_snat_protocol
 880 #undef _
 881                     default:
 882                       nat_elog_info ("unknown protocol");
 883                       return VNET_API_ERROR_INVALID_VALUE_2;
 884                     }
 885                   break;
 886                 }
 887             }
 888           /* External address must be allocated */
 889           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 890             {
 891               if (sw_if_index != ~0)
 892                 {
 893                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 894                     {
 895                       rp = sm->to_resolve + i;
 896                       if (rp->addr_only)
 897                         continue;
 898                       if (rp->sw_if_index != sw_if_index &&
 899                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 900                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 901                           rp->e_port != e_port && rp->proto != proto)
 902                         continue;
 903
 904                       vec_del1 (sm->to_resolve, i);
 905                       break;
 906                     }
 907                 }
 908               return VNET_API_ERROR_NO_SUCH_ENTRY;
 909             }
 910         }
 911
 912       pool_get (sm->static_mappings, m);
 913       clib_memset (m, 0, sizeof (*m));
 914       m->tag = vec_dup (tag);
 915       m->local_addr = l_addr;
 916       m->external_addr = e_addr;
 917       m->twice_nat = twice_nat;
 918       if (out2in_only)
 919         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 920       if (addr_only)
 921         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 922       if (identity_nat)
 923         {
 924           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 925           pool_get (m->locals, local);
 926           local->vrf_id = vrf_id;
 927           local->fib_index = fib_index;
 928         }
 929       else
 930         {
 931           m->vrf_id = vrf_id;
 932           m->fib_index = fib_index;
 933         }
 934       if (!addr_only)
 935         {
 936           m->local_port = l_port;
 937           m->external_port = e_port;
 938           m->proto = proto;
 939         }
 940
 941       if (sm->num_workers > 1)
 942         {
 943           ip4_header_t ip = {
 944             .src_address = m->local_addr,
 945           };
 946           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index, 0));
 947           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 948         }
 949       else
 950         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 951
 952       m_key.addr = m->local_addr;
 953       m_key.port = m->local_port;
 954       m_key.protocol = m->proto;
 955       m_key.fib_index = fib_index;
 956       kv.key = m_key.as_u64;
 957       kv.value = m - sm->static_mappings;
 958       if (!out2in_only)
 959         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 960
 961       m_key.addr = m->external_addr;
 962       m_key.port = m->external_port;
 963       m_key.fib_index = 0;
 964       kv.key = m_key.as_u64;
 965       kv.value = m - sm->static_mappings;
 966       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 967
 968       /* Delete dynamic sessions matching local address (+ local port) */
 969       if (!(sm->static_mapping_only))
 970         {
 971           u_key.addr = m->local_addr;
 972           u_key.fib_index = m->fib_index;
 973           kv.key = u_key.as_u64;
 974           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 975             {
 976               user_index = value.value;
 977               u = pool_elt_at_index (tsm->users, user_index);
 978               if (u->nsessions)
 979                 {
 980                   head_index = u->sessions_per_user_list_head_index;
 981                   head = pool_elt_at_index (tsm->list_pool, head_index);
 982                   elt_index = head->next;
 983                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 984                   ses_index = elt->value;
 985                   while (ses_index != ~0)
 986                     {
 987                       s = pool_elt_at_index (tsm->sessions, ses_index);
 988                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 989                       ses_index = elt->value;
 990
 991                       if (snat_is_session_static (s))
 992                         continue;
 993
 994                       if (!addr_only
 995                           && (clib_net_to_host_u16 (s->in2out.port) !=
 996                               m->local_port))
 997                         continue;
 998
 999                       nat_free_session_data (sm, s,
1000                                              tsm - sm->per_thread_data, 0);
1001                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1002
1003                       if (!addr_only && !sm->endpoint_dependent)
1004                         break;
1005                     }
1006                 }
1007             }
1008         }
1009     }
1010   else
1011     {
1012       if (!m)
1013         {
1014           if (sw_if_index != ~0)
1015             return 0;
1016           else
1017             return VNET_API_ERROR_NO_SUCH_ENTRY;
1018         }
1019
1020       if (identity_nat)
1021         {
1022           if (vrf_id == ~0)
1023             vrf_id = sm->inside_vrf_id;
1024
1025           /* *INDENT-OFF* */
1026           pool_foreach (local, m->locals,
1027           ({
1028             if (local->vrf_id == vrf_id)
1029               find = local - m->locals;
1030           }));
1031           /* *INDENT-ON* */
1032           if (find == ~0)
1033             return VNET_API_ERROR_NO_SUCH_ENTRY;
1034
1035           local = pool_elt_at_index (m->locals, find);
1036           fib_index = local->fib_index;
1037           pool_put (m->locals, local);
1038         }
1039       else
1040         fib_index = m->fib_index;
1041
1042       /* Free external address port */
1043       if (!(addr_only || sm->static_mapping_only || out2in_only))
1044         {
1045           for (i = 0; i < vec_len (sm->addresses); i++)
1046             {
1047               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1048                 {
1049                   a = sm->addresses + i;
1050                   switch (proto)
1051                     {
1052 #define _(N, j, n, s) \
1053                     case SNAT_PROTOCOL_##N: \
1054                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1055                       if (e_port > 1024) \
1056                         { \
1057                           a->busy_##n##_ports--; \
1058                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1059                         } \
1060                       break;
1061                       foreach_snat_protocol
1062 #undef _
1063                     default:
1064                       nat_elog_info ("unknown protocol");
1065                       return VNET_API_ERROR_INVALID_VALUE_2;
1066                     }
1067                   break;
1068                 }
1069             }
1070         }
1071
1072       if (sm->num_workers > 1)
1073         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1074       else
1075         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1076
1077       m_key.addr = m->local_addr;
1078       m_key.port = m->local_port;
1079       m_key.protocol = m->proto;
1080       m_key.fib_index = fib_index;
1081       kv.key = m_key.as_u64;
1082       if (!out2in_only)
1083         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1084
1085       /* Delete session(s) for static mapping if exist */
1086       if (!(sm->static_mapping_only) ||
1087           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1088         {
1089           u_key.addr = m->local_addr;
1090           u_key.fib_index = fib_index;
1091           kv.key = u_key.as_u64;
1092           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1093             {
1094               user_index = value.value;
1095               u = pool_elt_at_index (tsm->users, user_index);
1096               if (u->nstaticsessions)
1097                 {
1098                   head_index = u->sessions_per_user_list_head_index;
1099                   head = pool_elt_at_index (tsm->list_pool, head_index);
1100                   elt_index = head->next;
1101                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1102                   ses_index = elt->value;
1103                   while (ses_index != ~0)
1104                     {
1105                       s = pool_elt_at_index (tsm->sessions, ses_index);
1106                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1107                       ses_index = elt->value;
1108
1109                       if (!addr_only)
1110                         {
1111                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1112                               (clib_net_to_host_u16 (s->out2in.port) !=
1113                                e_port))
1114                             continue;
1115                         }
1116
1117                       if (is_lb_session (s))
1118                         continue;
1119
1120                       if (!snat_is_session_static (s))
1121                         continue;
1122
1123                       nat_free_session_data (sm, s,
1124                                              tsm - sm->per_thread_data, 0);
1125                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1126
1127                       if (!addr_only && !sm->endpoint_dependent)
1128                         break;
1129                     }
1130                 }
1131             }
1132         }
1133
1134       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1135       if (pool_elts (m->locals))
1136         return 0;
1137
1138       m_key.addr = m->external_addr;
1139       m_key.port = m->external_port;
1140       m_key.fib_index = 0;
1141       kv.key = m_key.as_u64;
1142       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1143
1144       vec_free (m->tag);
1145       vec_free (m->workers);
1146       /* Delete static mapping from pool */
1147       pool_put (sm->static_mappings, m);
1148     }
1149
1150   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1151     return 0;
1152
1153   /* Add/delete external address to FIB */
1154   /* *INDENT-OFF* */
1155   pool_foreach (interface, sm->interfaces,
1156   ({
1157     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1158       continue;
1159
1160     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1161     break;
1162   }));
1163   pool_foreach (interface, sm->output_feature_interfaces,
1164   ({
1165     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1166       continue;
1167
1168     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1169     break;
1170   }));
1171   /* *INDENT-ON* */
1172
1173   return 0;
1174 }
1175
1176 int
1177 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1178                                  snat_protocol_t proto,
1179                                  nat44_lb_addr_port_t * locals, u8 is_add,
1180                                  twice_nat_type_t twice_nat, u8 out2in_only,
1181                                  u8 * tag, u32 affinity)
1182 {
1183   snat_main_t *sm = &snat_main;
1184   snat_static_mapping_t *m;
1185   snat_session_key_t m_key;
1186   clib_bihash_kv_8_8_t kv, value;
1187   snat_address_t *a = 0;
1188   int i;
1189   nat44_lb_addr_port_t *local;
1190   u32 elt_index, head_index, ses_index;
1191   snat_main_per_thread_data_t *tsm;
1192   snat_user_key_t u_key;
1193   snat_user_t *u;
1194   snat_session_t *s;
1195   dlist_elt_t *head, *elt;
1196   uword *bitmap = 0;
1197
1198   if (!sm->endpoint_dependent)
1199     return VNET_API_ERROR_FEATURE_DISABLED;
1200
1201   m_key.addr = e_addr;
1202   m_key.port = e_port;
1203   m_key.protocol = proto;
1204   m_key.fib_index = 0;
1205   kv.key = m_key.as_u64;
1206   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1207     m = 0;
1208   else
1209     m = pool_elt_at_index (sm->static_mappings, value.value);
1210
1211   if (is_add)
1212     {
1213       if (m)
1214         return VNET_API_ERROR_VALUE_EXIST;
1215
1216       if (vec_len (locals) < 2)
1217         return VNET_API_ERROR_INVALID_VALUE;
1218
1219       /* Find external address in allocated addresses and reserve port for
1220          address and port pair mapping when dynamic translations enabled */
1221       if (!(sm->static_mapping_only || out2in_only))
1222         {
1223           for (i = 0; i < vec_len (sm->addresses); i++)
1224             {
1225               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1226                 {
1227                   a = sm->addresses + i;
1228                   /* External port must be unused */
1229                   switch (proto)
1230                     {
1231 #define _(N, j, n, s) \
1232                     case SNAT_PROTOCOL_##N: \
1233                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1234                         return VNET_API_ERROR_INVALID_VALUE; \
1235                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1236                       if (e_port > 1024) \
1237                         { \
1238                           a->busy_##n##_ports++; \
1239                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1240                         } \
1241                       break;
1242                       foreach_snat_protocol
1243 #undef _
1244                     default:
1245                       nat_elog_info ("unknown protocol");
1246                       return VNET_API_ERROR_INVALID_VALUE_2;
1247                     }
1248                   break;
1249                 }
1250             }
1251           /* External address must be allocated */
1252           if (!a)
1253             return VNET_API_ERROR_NO_SUCH_ENTRY;
1254         }
1255
1256       pool_get (sm->static_mappings, m);
1257       clib_memset (m, 0, sizeof (*m));
1258       m->tag = vec_dup (tag);
1259       m->external_addr = e_addr;
1260       m->external_port = e_port;
1261       m->proto = proto;
1262       m->twice_nat = twice_nat;
1263       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
1264       if (out2in_only)
1265         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
1266       m->affinity = affinity;
1267
1268       if (affinity)
1269         m->affinity_per_service_list_head_index =
1270           nat_affinity_get_per_service_list_head_index ();
1271       else
1272         m->affinity_per_service_list_head_index = ~0;
1273
1274       m_key.addr = m->external_addr;
1275       m_key.port = m->external_port;
1276       m_key.protocol = m->proto;
1277       m_key.fib_index = 0;
1278       kv.key = m_key.as_u64;
1279       kv.value = m - sm->static_mappings;
1280       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1281         {
1282           nat_elog_err ("static_mapping_by_external key add failed");
1283           return VNET_API_ERROR_UNSPECIFIED;
1284         }
1285
1286       m_key.fib_index = m->fib_index;
1287       for (i = 0; i < vec_len (locals); i++)
1288         {
1289           locals[i].fib_index =
1290             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1291                                                locals[i].vrf_id,
1292                                                nat_fib_src_low);
1293           m_key.addr = locals[i].addr;
1294           m_key.fib_index = locals[i].fib_index;
1295           if (!out2in_only)
1296             {
1297               m_key.port = locals[i].port;
1298               kv.key = m_key.as_u64;
1299               kv.value = m - sm->static_mappings;
1300               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1301             }
1302           locals[i].prefix = (i == 0) ? locals[i].probability :
1303             (locals[i - 1].prefix + locals[i].probability);
1304           pool_get (m->locals, local);
1305           *local = locals[i];
1306           if (sm->num_workers > 1)
1307             {
1308               ip4_header_t ip = {
1309                 .src_address = locals[i].addr,
1310               };
1311               bitmap =
1312                 clib_bitmap_set (bitmap,
1313                                  sm->worker_in2out_cb (&ip, m->fib_index, 0),
1314                                  1);
1315             }
1316         }
1317
1318       /* Assign workers */
1319       if (sm->num_workers > 1)
1320         {
1321           /* *INDENT-OFF* */
1322           clib_bitmap_foreach (i, bitmap,
1323             ({
1324                vec_add1(m->workers, i);
1325             }));
1326           /* *INDENT-ON* */
1327         }
1328     }
1329   else
1330     {
1331       if (!m)
1332         return VNET_API_ERROR_NO_SUCH_ENTRY;
1333
1334       if (!is_lb_static_mapping (m))
1335         return VNET_API_ERROR_INVALID_VALUE;
1336
1337       /* Free external address port */
1338       if (!(sm->static_mapping_only || out2in_only))
1339         {
1340           for (i = 0; i < vec_len (sm->addresses); i++)
1341             {
1342               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1343                 {
1344                   a = sm->addresses + i;
1345                   switch (proto)
1346                     {
1347 #define _(N, j, n, s) \
1348                     case SNAT_PROTOCOL_##N: \
1349                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1350                       if (e_port > 1024) \
1351                         { \
1352                           a->busy_##n##_ports--; \
1353                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1354                         } \
1355                       break;
1356                       foreach_snat_protocol
1357 #undef _
1358                     default:
1359                       nat_elog_info ("unknown protocol");
1360                       return VNET_API_ERROR_INVALID_VALUE_2;
1361                     }
1362                   break;
1363                 }
1364             }
1365         }
1366
1367       m_key.addr = m->external_addr;
1368       m_key.port = m->external_port;
1369       m_key.protocol = m->proto;
1370       m_key.fib_index = 0;
1371       kv.key = m_key.as_u64;
1372       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1373         {
1374           nat_elog_err ("static_mapping_by_external key del failed");
1375           return VNET_API_ERROR_UNSPECIFIED;
1376         }
1377
1378       /* *INDENT-OFF* */
1379       pool_foreach (local, m->locals,
1380       ({
1381           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1382                             nat_fib_src_low);
1383           m_key.addr = local->addr;
1384           if (!out2in_only)
1385             {
1386               m_key.port = local->port;
1387               m_key.fib_index = local->fib_index;
1388               kv.key = m_key.as_u64;
1389               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1390                 {
1391                   nat_elog_err ("static_mapping_by_local key del failed");
1392                   return VNET_API_ERROR_UNSPECIFIED;
1393                 }
1394             }
1395
1396           if (sm->num_workers > 1)
1397             {
1398               ip4_header_t ip = {
1399                 .src_address = local->addr,
1400               };
1401               tsm = vec_elt_at_index (sm->per_thread_data,
1402                                       sm->worker_in2out_cb (&ip, m->fib_index, 0));
1403             }
1404           else
1405             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1406
1407           /* Delete sessions */
1408           u_key.addr = local->addr;
1409           u_key.fib_index = local->fib_index;
1410           kv.key = u_key.as_u64;
1411           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1412             {
1413               u = pool_elt_at_index (tsm->users, value.value);
1414               if (u->nstaticsessions)
1415                 {
1416                   head_index = u->sessions_per_user_list_head_index;
1417                   head = pool_elt_at_index (tsm->list_pool, head_index);
1418                   elt_index = head->next;
1419                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1420                   ses_index = elt->value;
1421                   while (ses_index != ~0)
1422                     {
1423                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1424                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1425                       ses_index = elt->value;
1426
1427                       if (!(is_lb_session (s)))
1428                         continue;
1429
1430                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1431                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1432                         continue;
1433
1434                       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1435                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1436                     }
1437                 }
1438             }
1439       }));
1440       /* *INDENT-ON* */
1441       if (m->affinity)
1442         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1443       pool_free (m->locals);
1444       vec_free (m->tag);
1445       vec_free (m->workers);
1446
1447       pool_put (sm->static_mappings, m);
1448     }
1449
1450   return 0;
1451 }
1452
1453 int
1454 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1455                                        ip4_address_t l_addr, u16 l_port,
1456                                        snat_protocol_t proto, u32 vrf_id,
1457                                        u8 probability, u8 is_add)
1458 {
1459   snat_main_t *sm = &snat_main;
1460   snat_static_mapping_t *m = 0;
1461   snat_session_key_t m_key;
1462   clib_bihash_kv_8_8_t kv, value;
1463   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1464   snat_main_per_thread_data_t *tsm;
1465   snat_user_key_t u_key;
1466   snat_user_t *u;
1467   snat_session_t *s;
1468   dlist_elt_t *head, *elt;
1469   u32 elt_index, head_index, ses_index, *locals = 0;
1470   uword *bitmap = 0;
1471   int i;
1472
1473   if (!sm->endpoint_dependent)
1474     return VNET_API_ERROR_FEATURE_DISABLED;
1475
1476   m_key.addr = e_addr;
1477   m_key.port = e_port;
1478   m_key.protocol = proto;
1479   m_key.fib_index = 0;
1480   kv.key = m_key.as_u64;
1481   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1482     m = pool_elt_at_index (sm->static_mappings, value.value);
1483
1484   if (!m)
1485     return VNET_API_ERROR_NO_SUCH_ENTRY;
1486
1487   if (!is_lb_static_mapping (m))
1488     return VNET_API_ERROR_INVALID_VALUE;
1489
1490   /* *INDENT-OFF* */
1491   pool_foreach (local, m->locals,
1492   ({
1493     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1494         (local->vrf_id == vrf_id))
1495       {
1496         match_local = local;
1497         break;
1498       }
1499   }));
1500   /* *INDENT-ON* */
1501
1502   if (is_add)
1503     {
1504       if (match_local)
1505         return VNET_API_ERROR_VALUE_EXIST;
1506
1507       pool_get (m->locals, local);
1508       clib_memset (local, 0, sizeof (*local));
1509       local->addr.as_u32 = l_addr.as_u32;
1510       local->port = l_port;
1511       local->probability = probability;
1512       local->vrf_id = vrf_id;
1513       local->fib_index =
1514         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1515                                            nat_fib_src_low);
1516
1517       if (!is_out2in_only_static_mapping (m))
1518         {
1519           m_key.addr = l_addr;
1520           m_key.port = l_port;
1521           m_key.fib_index = local->fib_index;
1522           kv.key = m_key.as_u64;
1523           kv.value = m - sm->static_mappings;
1524           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1525             nat_elog_err ("static_mapping_by_local key add failed");
1526         }
1527     }
1528   else
1529     {
1530       if (!match_local)
1531         return VNET_API_ERROR_NO_SUCH_ENTRY;
1532
1533       if (pool_elts (m->locals) < 3)
1534         return VNET_API_ERROR_UNSPECIFIED;
1535
1536       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1537                         nat_fib_src_low);
1538
1539       if (!is_out2in_only_static_mapping (m))
1540         {
1541           m_key.addr = l_addr;
1542           m_key.port = l_port;
1543           m_key.fib_index = match_local->fib_index;
1544           kv.key = m_key.as_u64;
1545           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1546             nat_elog_err ("static_mapping_by_local key del failed");
1547         }
1548
1549       if (sm->num_workers > 1)
1550         {
1551           ip4_header_t ip = {
1552             .src_address = local->addr,
1553           };
1554           tsm = vec_elt_at_index (sm->per_thread_data,
1555                                   sm->worker_in2out_cb (&ip, m->fib_index,
1556                                                         0));
1557         }
1558       else
1559         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1560
1561       /* Delete sessions */
1562       u_key.addr = match_local->addr;
1563       u_key.fib_index = match_local->fib_index;
1564       kv.key = u_key.as_u64;
1565       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1566         {
1567           u = pool_elt_at_index (tsm->users, value.value);
1568           if (u->nstaticsessions)
1569             {
1570               head_index = u->sessions_per_user_list_head_index;
1571               head = pool_elt_at_index (tsm->list_pool, head_index);
1572               elt_index = head->next;
1573               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1574               ses_index = elt->value;
1575               while (ses_index != ~0)
1576                 {
1577                   s = pool_elt_at_index (tsm->sessions, ses_index);
1578                   elt = pool_elt_at_index (tsm->list_pool, elt->next);
1579                   ses_index = elt->value;
1580
1581                   if (!(is_lb_session (s)))
1582                     continue;
1583
1584                   if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1585                       (clib_net_to_host_u16 (s->in2out.port) !=
1586                        match_local->port))
1587                     continue;
1588
1589                   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1590                   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1591                 }
1592             }
1593         }
1594
1595       pool_put (m->locals, match_local);
1596     }
1597
1598   vec_free (m->workers);
1599
1600   /* *INDENT-OFF* */
1601   pool_foreach (local, m->locals,
1602   ({
1603     vec_add1 (locals, local - m->locals);
1604     if (sm->num_workers > 1)
1605       {
1606         ip4_header_t ip;
1607         ip.src_address.as_u32 = local->addr.as_u32,
1608         bitmap = clib_bitmap_set (bitmap,
1609                                   sm->worker_in2out_cb (&ip, local->fib_index, 0),
1610                                   1);
1611       }
1612   }));
1613   /* *INDENT-ON* */
1614
1615   ASSERT (vec_len (locals) > 1);
1616
1617   local = pool_elt_at_index (m->locals, locals[0]);
1618   local->prefix = local->probability;
1619   for (i = 1; i < vec_len (locals); i++)
1620     {
1621       local = pool_elt_at_index (m->locals, locals[i]);
1622       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1623       local->prefix = local->probability + prev_local->prefix;
1624     }
1625
1626   /* Assign workers */
1627   if (sm->num_workers > 1)
1628     {
1629       /* *INDENT-OFF* */
1630       clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
1631       /* *INDENT-ON* */
1632     }
1633
1634   return 0;
1635 }
1636
1637 int
1638 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1639                   u8 twice_nat)
1640 {
1641   snat_address_t *a = 0;
1642   snat_session_t *ses;
1643   u32 *ses_to_be_removed = 0, *ses_index;
1644   snat_main_per_thread_data_t *tsm;
1645   snat_static_mapping_t *m;
1646   snat_interface_t *interface;
1647   int i;
1648   snat_address_t *addresses =
1649     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1650
1651   /* Find SNAT address */
1652   for (i = 0; i < vec_len (addresses); i++)
1653     {
1654       if (addresses[i].addr.as_u32 == addr.as_u32)
1655         {
1656           a = addresses + i;
1657           break;
1658         }
1659     }
1660   if (!a)
1661     return VNET_API_ERROR_NO_SUCH_ENTRY;
1662
1663   if (delete_sm)
1664     {
1665       /* *INDENT-OFF* */
1666       pool_foreach (m, sm->static_mappings,
1667       ({
1668           if (m->external_addr.as_u32 == addr.as_u32)
1669             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1670                                             m->local_port, m->external_port,
1671                                             m->vrf_id, is_addr_only_static_mapping(m), ~0,
1672                                             m->proto, 0, m->twice_nat,
1673                                             is_out2in_only_static_mapping(m), m->tag, is_identity_static_mapping(m));
1674       }));
1675       /* *INDENT-ON* */
1676     }
1677   else
1678     {
1679       /* Check if address is used in some static mapping */
1680       if (is_snat_address_used_in_static_mapping (sm, addr))
1681         {
1682           nat_elog_notice ("address used in static mapping");
1683           return VNET_API_ERROR_UNSPECIFIED;
1684         }
1685     }
1686
1687   if (a->fib_index != ~0)
1688     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, nat_fib_src_low);
1689
1690   /* Delete sessions using address */
1691   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1692     {
1693       /* *INDENT-OFF* */
1694       vec_foreach (tsm, sm->per_thread_data)
1695         {
1696           pool_foreach (ses, tsm->sessions, ({
1697             if (ses->out2in.addr.as_u32 == addr.as_u32)
1698               {
1699                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1700                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1701               }
1702           }));
1703
1704           vec_foreach (ses_index, ses_to_be_removed)
1705             {
1706               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1707               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1708             }
1709
1710           vec_free (ses_to_be_removed);
1711         }
1712       /* *INDENT-ON* */
1713     }
1714
1715 #define _(N, i, n, s) \
1716   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1717   vec_free (a->busy_##n##_ports_per_thread);
1718   foreach_snat_protocol
1719 #undef _
1720     if (twice_nat)
1721     {
1722       vec_del1 (sm->twice_nat_addresses, i);
1723       return 0;
1724     }
1725   else
1726     vec_del1 (sm->addresses, i);
1727
1728   /* Delete external address from FIB */
1729   /* *INDENT-OFF* */
1730   pool_foreach (interface, sm->interfaces,
1731   ({
1732     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1733       continue;
1734
1735     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1736     break;
1737   }));
1738   pool_foreach (interface, sm->output_feature_interfaces,
1739   ({
1740     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1741       continue;
1742
1743     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1744     break;
1745   }));
1746   /* *INDENT-ON* */
1747
1748   return 0;
1749 }
1750
1751 int
1752 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1753 {
1754   snat_main_t *sm = &snat_main;
1755   snat_interface_t *i;
1756   const char *feature_name, *del_feature_name;
1757   snat_address_t *ap;
1758   snat_static_mapping_t *m;
1759   snat_det_map_t *dm;
1760   nat_outside_fib_t *outside_fib;
1761   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1762                                                        sw_if_index);
1763
1764   if (sm->out2in_dpo && !is_inside)
1765     return VNET_API_ERROR_UNSUPPORTED;
1766
1767   /* *INDENT-OFF* */
1768   pool_foreach (i, sm->output_feature_interfaces,
1769   ({
1770     if (i->sw_if_index == sw_if_index)
1771       return VNET_API_ERROR_VALUE_EXIST;
1772   }));
1773   /* *INDENT-ON* */
1774
1775   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1776     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1777   else
1778     {
1779       if (sm->num_workers > 1 && !sm->deterministic)
1780         feature_name =
1781           is_inside ? "nat44-in2out-worker-handoff" :
1782           "nat44-out2in-worker-handoff";
1783       else if (sm->deterministic)
1784         feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
1785       else if (sm->endpoint_dependent)
1786         {
1787           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1788         }
1789       else
1790         feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
1791     }
1792
1793   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1794     sm->fq_in2out_index =
1795       vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
1796
1797   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1798     sm->fq_out2in_index =
1799       vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
1800
1801   if (!is_inside)
1802     {
1803       /* *INDENT-OFF* */
1804       vec_foreach (outside_fib, sm->outside_fibs)
1805         {
1806           if (outside_fib->fib_index == fib_index)
1807             {
1808               if (is_del)
1809                 {
1810                   outside_fib->refcount--;
1811                   if (!outside_fib->refcount)
1812                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1813                 }
1814               else
1815                 outside_fib->refcount++;
1816               goto feature_set;
1817             }
1818         }
1819       /* *INDENT-ON* */
1820       if (!is_del)
1821         {
1822           vec_add2 (sm->outside_fibs, outside_fib, 1);
1823           outside_fib->refcount = 1;
1824           outside_fib->fib_index = fib_index;
1825         }
1826     }
1827 feature_set:
1828   /* *INDENT-OFF* */
1829   pool_foreach (i, sm->interfaces,
1830   ({
1831     if (i->sw_if_index == sw_if_index)
1832       {
1833         if (is_del)
1834           {
1835             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1836               {
1837                 if (is_inside)
1838                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1839                 else
1840                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1841
1842                 if (sm->num_workers > 1 && !sm->deterministic)
1843                   {
1844                     del_feature_name = "nat44-handoff-classify";
1845                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1846                                                  "nat44-out2in-worker-handoff";
1847                   }
1848                 else if (sm->deterministic)
1849                   {
1850                     del_feature_name = "nat44-det-classify";
1851                     feature_name = !is_inside ?  "nat44-det-in2out" :
1852                                                  "nat44-det-out2in";
1853                   }
1854                 else if (sm->endpoint_dependent)
1855                   {
1856                     del_feature_name = "nat44-ed-classify";
1857                     feature_name = !is_inside ?  "nat-pre-in2out" :
1858                                                  "nat-pre-out2in";
1859                   }
1860                 else
1861                   {
1862                     del_feature_name = "nat44-classify";
1863                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1864                   }
1865
1866                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1867                 if (rv)
1868                   return rv;
1869                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1870                                              sw_if_index, 0, 0, 0);
1871                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1872                                              sw_if_index, 1, 0, 0);
1873                 if (!is_inside)
1874                   {
1875                     if (sm->endpoint_dependent)
1876                       vnet_feature_enable_disable ("ip4-local",
1877                                                    "nat44-ed-hairpinning",
1878                                                    sw_if_index, 1, 0, 0);
1879                     else if (!sm->deterministic)
1880                       vnet_feature_enable_disable ("ip4-local",
1881                                                    "nat44-hairpinning",
1882                                                    sw_if_index, 1, 0, 0);
1883                   }
1884               }
1885             else
1886               {
1887                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1888                 if (rv)
1889                   return rv;
1890                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1891                                              sw_if_index, 0, 0, 0);
1892                 pool_put (sm->interfaces, i);
1893                 if (is_inside)
1894                   {
1895                     if (sm->endpoint_dependent)
1896                       vnet_feature_enable_disable ("ip4-local",
1897                                                    "nat44-ed-hairpinning",
1898                                                    sw_if_index, 0, 0, 0);
1899                     else if (!sm->deterministic)
1900                       vnet_feature_enable_disable ("ip4-local",
1901                                                    "nat44-hairpinning",
1902                                                    sw_if_index, 0, 0, 0);
1903                   }
1904               }
1905           }
1906         else
1907           {
1908             if ((nat_interface_is_inside(i) && is_inside) ||
1909                 (nat_interface_is_outside(i) && !is_inside))
1910               return 0;
1911
1912             if (sm->num_workers > 1 && !sm->deterministic)
1913               {
1914                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1915                                                  "nat44-out2in-worker-handoff";
1916                 feature_name = "nat44-handoff-classify";
1917               }
1918             else if (sm->deterministic)
1919               {
1920                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1921                                                  "nat44-det-out2in";
1922                 feature_name = "nat44-det-classify";
1923               }
1924             else if (sm->endpoint_dependent)
1925               {
1926                 del_feature_name = !is_inside ?  "nat-pre-in2out" :
1927                                                  "nat-pre-out2in";
1928
1929                 feature_name = "nat44-ed-classify";
1930               }
1931             else
1932               {
1933                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1934                 feature_name = "nat44-classify";
1935               }
1936
1937             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1938             if (rv)
1939               return rv;
1940             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1941                                          sw_if_index, 0, 0, 0);
1942             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1943                                          sw_if_index, 1, 0, 0);
1944             if (!is_inside)
1945               {
1946                 if (sm->endpoint_dependent)
1947                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1948                                                sw_if_index, 0, 0, 0);
1949                 else if (!sm->deterministic)
1950                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1951                                                sw_if_index, 0, 0, 0);
1952               }
1953             goto set_flags;
1954           }
1955
1956         goto fib;
1957       }
1958   }));
1959   /* *INDENT-ON* */
1960
1961   if (is_del)
1962     return VNET_API_ERROR_NO_SUCH_ENTRY;
1963
1964   pool_get (sm->interfaces, i);
1965   i->sw_if_index = sw_if_index;
1966   i->flags = 0;
1967   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1968                                0);
1969
1970   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1971   if (rv)
1972     return rv;
1973
1974   if (is_inside && !sm->out2in_dpo)
1975     {
1976       if (sm->endpoint_dependent)
1977         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1978                                      sw_if_index, 1, 0, 0);
1979       else if (!sm->deterministic)
1980         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1981                                      sw_if_index, 1, 0, 0);
1982     }
1983
1984 set_flags:
1985   if (is_inside)
1986     {
1987       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1988       return 0;
1989     }
1990   else
1991     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1992
1993   /* Add/delete external addresses to FIB */
1994 fib:
1995   /* *INDENT-OFF* */
1996   vec_foreach (ap, sm->addresses)
1997     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1998
1999   pool_foreach (m, sm->static_mappings,
2000   ({
2001     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
2002       continue;
2003
2004     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2005   }));
2006
2007   pool_foreach (dm, sm->det_maps,
2008   ({
2009     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
2010   }));
2011   /* *INDENT-ON* */
2012
2013   return 0;
2014 }
2015
2016 int
2017 snat_interface_add_del_output_feature (u32 sw_if_index,
2018                                        u8 is_inside, int is_del)
2019 {
2020   snat_main_t *sm = &snat_main;
2021   snat_interface_t *i;
2022   snat_address_t *ap;
2023   snat_static_mapping_t *m;
2024   nat_outside_fib_t *outside_fib;
2025   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2026                                                        sw_if_index);
2027
2028
2029   if (sm->deterministic ||
2030       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
2031     return VNET_API_ERROR_UNSUPPORTED;
2032
2033   /* *INDENT-OFF* */
2034   pool_foreach (i, sm->interfaces,
2035   ({
2036     if (i->sw_if_index == sw_if_index)
2037       return VNET_API_ERROR_VALUE_EXIST;
2038   }));
2039   /* *INDENT-ON* */
2040
2041   if (!is_inside)
2042     {
2043       /* *INDENT-OFF* */
2044       vec_foreach (outside_fib, sm->outside_fibs)
2045         {
2046           if (outside_fib->fib_index == fib_index)
2047             {
2048               if (is_del)
2049                 {
2050                   outside_fib->refcount--;
2051                   if (!outside_fib->refcount)
2052                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2053                 }
2054               else
2055                 outside_fib->refcount++;
2056               goto feature_set;
2057             }
2058         }
2059       /* *INDENT-ON* */
2060       if (!is_del)
2061         {
2062           vec_add2 (sm->outside_fibs, outside_fib, 1);
2063           outside_fib->refcount = 1;
2064           outside_fib->fib_index = fib_index;
2065         }
2066     }
2067
2068 feature_set:
2069   if (is_inside)
2070     {
2071       if (sm->endpoint_dependent)
2072         {
2073           int rv =
2074             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2075           if (rv)
2076             return rv;
2077           rv =
2078             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2079                                                             !is_del);
2080           if (rv)
2081             return rv;
2082           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
2083                                        sw_if_index, !is_del, 0, 0);
2084           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
2085                                        sw_if_index, !is_del, 0, 0);
2086         }
2087       else
2088         {
2089           int rv =
2090             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2091           if (rv)
2092             return rv;
2093           rv =
2094             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2095                                                             !is_del);
2096           if (rv)
2097             return rv;
2098           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
2099                                        sw_if_index, !is_del, 0, 0);
2100           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
2101                                        sw_if_index, !is_del, 0, 0);
2102         }
2103       goto fq;
2104     }
2105
2106   if (sm->num_workers > 1)
2107     {
2108       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2109       if (rv)
2110         return rv;
2111       rv =
2112         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
2113       if (rv)
2114         return rv;
2115       vnet_feature_enable_disable ("ip4-unicast",
2116                                    "nat44-out2in-worker-handoff",
2117                                    sw_if_index, !is_del, 0, 0);
2118       vnet_feature_enable_disable ("ip4-output",
2119                                    "nat44-in2out-output-worker-handoff",
2120                                    sw_if_index, !is_del, 0, 0);
2121     }
2122   else
2123     {
2124       if (sm->endpoint_dependent)
2125         {
2126           int rv =
2127             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2128           if (rv)
2129             return rv;
2130           rv =
2131             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2132                                                             !is_del);
2133           if (rv)
2134             return rv;
2135           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2136                                        sw_if_index, !is_del, 0, 0);
2137           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
2138                                        sw_if_index, !is_del, 0, 0);
2139         }
2140       else
2141         {
2142           int rv =
2143             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
2144           if (rv)
2145             return rv;
2146           rv =
2147             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
2148                                                             !is_del);
2149           if (rv)
2150             return rv;
2151           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
2152                                        sw_if_index, !is_del, 0, 0);
2153           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
2154                                        sw_if_index, !is_del, 0, 0);
2155         }
2156     }
2157
2158 fq:
2159   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
2160     sm->fq_in2out_output_index =
2161       vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
2162
2163   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
2164     sm->fq_out2in_index =
2165       vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
2166
2167   /* *INDENT-OFF* */
2168   pool_foreach (i, sm->output_feature_interfaces,
2169   ({
2170     if (i->sw_if_index == sw_if_index)
2171       {
2172         if (is_del)
2173           pool_put (sm->output_feature_interfaces, i);
2174         else
2175           return VNET_API_ERROR_VALUE_EXIST;
2176
2177         goto fib;
2178       }
2179   }));
2180   /* *INDENT-ON* */
2181
2182   if (is_del)
2183     return VNET_API_ERROR_NO_SUCH_ENTRY;
2184
2185   pool_get (sm->output_feature_interfaces, i);
2186   i->sw_if_index = sw_if_index;
2187   i->flags = 0;
2188   if (is_inside)
2189     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2190   else
2191     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2192
2193   /* Add/delete external addresses to FIB */
2194 fib:
2195   if (is_inside)
2196     return 0;
2197
2198   /* *INDENT-OFF* */
2199   vec_foreach (ap, sm->addresses)
2200     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
2201
2202   pool_foreach (m, sm->static_mappings,
2203   ({
2204     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
2205       continue;
2206
2207     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
2208   }));
2209   /* *INDENT-ON* */
2210
2211   return 0;
2212 }
2213
2214 int
2215 snat_set_workers (uword * bitmap)
2216 {
2217   snat_main_t *sm = &snat_main;
2218   int i, j = 0;
2219
2220   if (sm->num_workers < 2)
2221     return VNET_API_ERROR_FEATURE_DISABLED;
2222
2223   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2224     return VNET_API_ERROR_INVALID_WORKER;
2225
2226   vec_free (sm->workers);
2227   /* *INDENT-OFF* */
2228   clib_bitmap_foreach (i, bitmap,
2229     ({
2230       vec_add1(sm->workers, i);
2231       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2232       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2233       j++;
2234     }));
2235   /* *INDENT-ON* */
2236
2237   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2238   sm->num_snat_thread = _vec_len (sm->workers);
2239
2240   return 0;
2241 }
2242
2243 static void
2244 snat_update_outside_fib (u32 sw_if_index, u32 new_fib_index,
2245                          u32 old_fib_index)
2246 {
2247   snat_main_t *sm = &snat_main;
2248   nat_outside_fib_t *outside_fib;
2249   snat_interface_t *i;
2250   u8 is_add = 1;
2251   u8 match = 0;
2252
2253   if (new_fib_index == old_fib_index)
2254     return;
2255
2256   if (!vec_len (sm->outside_fibs))
2257     return;
2258
2259   /* *INDENT-OFF* */
2260   pool_foreach (i, sm->interfaces,
2261     ({
2262       if (i->sw_if_index == sw_if_index)
2263         {
2264           if (!(nat_interface_is_outside (i)))
2265             return;
2266           match = 1;
2267         }
2268     }));
2269
2270   pool_foreach (i, sm->output_feature_interfaces,
2271     ({
2272       if (i->sw_if_index == sw_if_index)
2273         {
2274           if (!(nat_interface_is_outside (i)))
2275             return;
2276           match = 1;
2277         }
2278     }));
2279   /* *INDENT-ON* */
2280
2281   if (!match)
2282     return;
2283
2284   vec_foreach (outside_fib, sm->outside_fibs)
2285   {
2286     if (outside_fib->fib_index == old_fib_index)
2287       {
2288         outside_fib->refcount--;
2289         if (!outside_fib->refcount)
2290           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2291         break;
2292       }
2293   }
2294
2295   vec_foreach (outside_fib, sm->outside_fibs)
2296   {
2297     if (outside_fib->fib_index == new_fib_index)
2298       {
2299         outside_fib->refcount++;
2300         is_add = 0;
2301         break;
2302       }
2303   }
2304
2305   if (is_add)
2306     {
2307       vec_add2 (sm->outside_fibs, outside_fib, 1);
2308       outside_fib->refcount = 1;
2309       outside_fib->fib_index = new_fib_index;
2310     }
2311 }
2312
2313 static void
2314 snat_ip4_table_bind (ip4_main_t * im,
2315                      uword opaque,
2316                      u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
2317 {
2318   snat_update_outside_fib (sw_if_index, new_fib_index, old_fib_index);
2319 }
2320
2321 static void
2322 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2323                                        uword opaque,
2324                                        u32 sw_if_index,
2325                                        ip4_address_t * address,
2326                                        u32 address_length,
2327                                        u32 if_address_index, u32 is_delete);
2328
2329 static void
2330 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2331                                  uword opaque,
2332                                  u32 sw_if_index,
2333                                  ip4_address_t * address,
2334                                  u32 address_length,
2335                                  u32 if_address_index, u32 is_delete);
2336
2337 static int
2338 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2339                                  u32 fib_index,
2340                                  u32 thread_index,
2341                                  snat_session_key_t * k,
2342                                  u16 port_per_thread, u32 snat_thread_index);
2343
2344 static clib_error_t *
2345 snat_init (vlib_main_t * vm)
2346 {
2347   snat_main_t *sm = &snat_main;
2348   clib_error_t *error = 0;
2349   ip4_main_t *im = &ip4_main;
2350   ip_lookup_main_t *lm = &im->lookup_main;
2351   uword *p;
2352   vlib_thread_registration_t *tr;
2353   vlib_thread_main_t *tm = vlib_get_thread_main ();
2354   uword *bitmap = 0;
2355   u32 i;
2356   ip4_add_del_interface_address_callback_t cb4;
2357   vlib_node_t *node;
2358
2359   sm->vlib_main = vm;
2360   sm->vnet_main = vnet_get_main ();
2361   sm->ip4_main = im;
2362   sm->ip4_lookup_main = lm;
2363   sm->api_main = vlibapi_get_main ();
2364   sm->first_worker_index = 0;
2365   sm->num_workers = 0;
2366   sm->num_snat_thread = 1;
2367   sm->workers = 0;
2368   sm->port_per_thread = 0xffff - 1024;
2369   sm->fq_in2out_index = ~0;
2370   sm->fq_in2out_output_index = ~0;
2371   sm->fq_out2in_index = ~0;
2372
2373
2374   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2375   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2376   sm->forwarding_enabled = 0;
2377   sm->log_class = vlib_log_register_class ("nat", 0);
2378   sm->log_level = SNAT_LOG_ERROR;
2379   sm->mss_clamping = 0;
2380
2381   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2382   sm->error_node_index = node->index;
2383
2384   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2385   sm->pre_in2out_node_index = node->index;
2386   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2387   sm->pre_out2in_node_index = node->index;
2388
2389   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
2390   sm->pre_in2out_node_index = node->index;
2391
2392   node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
2393   sm->pre_out2in_node_index = node->index;
2394
2395   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
2396   sm->in2out_node_index = node->index;
2397   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
2398   sm->in2out_output_node_index = node->index;
2399   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-fast");
2400   sm->in2out_fast_node_index = node->index;
2401   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-slowpath");
2402   sm->in2out_slowpath_node_index = node->index;
2403   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output-slowpath");
2404   sm->in2out_slowpath_output_node_index = node->index;
2405
2406   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2407   sm->ed_in2out_node_index = node->index;
2408   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-slowpath");
2409   sm->ed_in2out_slowpath_node_index = node->index;
2410
2411   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in");
2412   sm->out2in_node_index = node->index;
2413   node = vlib_get_node_by_name (vm, (u8 *) "nat44-out2in-fast");
2414   sm->out2in_fast_node_index = node->index;
2415
2416   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2417   sm->ed_out2in_node_index = node->index;
2418   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in-slowpath");
2419   sm->ed_out2in_slowpath_node_index = node->index;
2420
2421   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-in2out");
2422   sm->det_in2out_node_index = node->index;
2423   node = vlib_get_node_by_name (vm, (u8 *) "nat44-det-out2in");
2424   sm->det_out2in_node_index = node->index;
2425
2426   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpinning");
2427   sm->hairpinning_node_index = node->index;
2428   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-dst");
2429   sm->hairpin_dst_node_index = node->index;
2430   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
2431   sm->hairpin_src_node_index = node->index;
2432   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
2433   sm->ed_hairpinning_node_index = node->index;
2434   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
2435   sm->ed_hairpin_dst_node_index = node->index;
2436   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
2437   sm->ed_hairpin_src_node_index = node->index;
2438
2439   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2440   if (p)
2441     {
2442       tr = (vlib_thread_registration_t *) p[0];
2443       if (tr)
2444         {
2445           sm->num_workers = tr->count;
2446           sm->first_worker_index = tr->first_index;
2447         }
2448     }
2449
2450   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2451
2452   /* Use all available workers by default */
2453   if (sm->num_workers > 1)
2454     {
2455       for (i = 0; i < sm->num_workers; i++)
2456         bitmap = clib_bitmap_set (bitmap, i, 1);
2457       snat_set_workers (bitmap);
2458       clib_bitmap_free (bitmap);
2459     }
2460   else
2461     {
2462       sm->per_thread_data[0].snat_thread_index = 0;
2463     }
2464
2465   error = snat_api_init (vm, sm);
2466   if (error)
2467     return error;
2468
2469   /* Set up the interface address add/del callback */
2470   cb4.function = snat_ip4_add_del_interface_address_cb;
2471   cb4.function_opaque = 0;
2472
2473   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2474
2475   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2476   cb4.function_opaque = 0;
2477
2478   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2479
2480   nat_dpo_module_init ();
2481
2482   /* Init counters */
2483   sm->total_users.name = "total-users";
2484   sm->total_users.stat_segment_name = "/nat44/total-users";
2485   vlib_validate_simple_counter (&sm->total_users, 0);
2486   vlib_zero_simple_counter (&sm->total_users, 0);
2487   sm->total_sessions.name = "total-sessions";
2488   sm->total_sessions.stat_segment_name = "/nat44/total-sessions";
2489   vlib_validate_simple_counter (&sm->total_sessions, 0);
2490   vlib_zero_simple_counter (&sm->total_sessions, 0);
2491
2492   /* Init IPFIX logging */
2493   snat_ipfix_logging_init (vm);
2494
2495   /* Init NAT64 */
2496   error = nat64_init (vm);
2497   if (error)
2498     return error;
2499
2500   nat66_init (vm);
2501
2502   ip4_table_bind_callback_t cbt4 = {
2503     .function = snat_ip4_table_bind,
2504   };
2505   vec_add1 (ip4_main.table_bind_callbacks, cbt4);
2506
2507   nat_fib_src_hi = fib_source_allocate ("nat-hi",
2508                                         FIB_SOURCE_PRIORITY_HI,
2509                                         FIB_SOURCE_BH_SIMPLE);
2510   nat_fib_src_low = fib_source_allocate ("nat-low",
2511                                          FIB_SOURCE_PRIORITY_LOW,
2512                                          FIB_SOURCE_BH_SIMPLE);
2513
2514   return error;
2515 }
2516
2517 VLIB_INIT_FUNCTION (snat_init);
2518
2519 void
2520 snat_free_outside_address_and_port (snat_address_t * addresses,
2521                                     u32 thread_index, snat_session_key_t * k)
2522 {
2523   snat_address_t *a;
2524   u32 address_index;
2525   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2526
2527   for (address_index = 0; address_index < vec_len (addresses);
2528        address_index++)
2529     {
2530       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2531         break;
2532     }
2533
2534   ASSERT (address_index < vec_len (addresses));
2535
2536   a = addresses + address_index;
2537
2538   switch (k->protocol)
2539     {
2540 #define _(N, i, n, s) \
2541     case SNAT_PROTOCOL_##N: \
2542       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2543         port_host_byte_order) == 1); \
2544       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2545         port_host_byte_order, 0); \
2546       a->busy_##n##_ports--; \
2547       a->busy_##n##_ports_per_thread[thread_index]--; \
2548       break;
2549       foreach_snat_protocol
2550 #undef _
2551     default:
2552       nat_elog_info ("unknown protocol");
2553       return;
2554     }
2555 }
2556
2557 static int
2558 nat_set_outside_address_and_port (snat_address_t * addresses,
2559                                   u32 thread_index, snat_session_key_t * k)
2560 {
2561   snat_address_t *a = 0;
2562   u32 address_index;
2563   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2564
2565   for (address_index = 0; address_index < vec_len (addresses);
2566        address_index++)
2567     {
2568       if (addresses[address_index].addr.as_u32 != k->addr.as_u32)
2569         continue;
2570
2571       a = addresses + address_index;
2572       switch (k->protocol)
2573         {
2574 #define _(N, j, n, s) \
2575         case SNAT_PROTOCOL_##N: \
2576           if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, port_host_byte_order)) \
2577             return VNET_API_ERROR_INSTANCE_IN_USE; \
2578           clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port_host_byte_order, 1); \
2579           a->busy_##n##_ports_per_thread[thread_index]++; \
2580           a->busy_##n##_ports++; \
2581           return 0;
2582           foreach_snat_protocol
2583 #undef _
2584         default:
2585           nat_elog_info ("unknown protocol");
2586           return 1;
2587         }
2588     }
2589
2590   return VNET_API_ERROR_NO_SUCH_ENTRY;
2591 }
2592
2593 int
2594 snat_static_mapping_match (snat_main_t * sm,
2595                            snat_session_key_t match,
2596                            snat_session_key_t * mapping,
2597                            u8 by_external,
2598                            u8 * is_addr_only,
2599                            twice_nat_type_t * twice_nat,
2600                            lb_nat_type_t * lb, ip4_address_t * ext_host_addr,
2601                            u8 * is_identity_nat)
2602 {
2603   clib_bihash_kv_8_8_t kv, value;
2604   snat_static_mapping_t *m;
2605   snat_session_key_t m_key;
2606   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2607   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2608   u8 backend_index;
2609   nat44_lb_addr_port_t *local;
2610
2611   m_key.fib_index = match.fib_index;
2612   if (by_external)
2613     {
2614       mapping_hash = &sm->static_mapping_by_external;
2615       m_key.fib_index = 0;
2616     }
2617
2618   m_key.addr = match.addr;
2619   m_key.port = clib_net_to_host_u16 (match.port);
2620   m_key.protocol = match.protocol;
2621
2622   kv.key = m_key.as_u64;
2623
2624   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2625     {
2626       /* Try address only mapping */
2627       m_key.port = 0;
2628       m_key.protocol = 0;
2629       kv.key = m_key.as_u64;
2630       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2631         return 1;
2632     }
2633
2634   m = pool_elt_at_index (sm->static_mappings, value.value);
2635
2636   if (by_external)
2637     {
2638       if (is_lb_static_mapping (m))
2639         {
2640           if (PREDICT_FALSE (lb != 0))
2641             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2642           if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
2643                                                           match.addr,
2644                                                           match.protocol,
2645                                                           match.port,
2646                                                           &backend_index))
2647             {
2648               local = pool_elt_at_index (m->locals, backend_index);
2649               mapping->addr = local->addr;
2650               mapping->port = clib_host_to_net_u16 (local->port);
2651               mapping->fib_index = local->fib_index;
2652               goto end;
2653             }
2654           // pick locals matching this worker
2655           if (PREDICT_FALSE (sm->num_workers > 1))
2656             {
2657               u32 thread_index = vlib_get_thread_index ();
2658               /* *INDENT-OFF* */
2659               pool_foreach_index (i, m->locals,
2660               ({
2661                 local = pool_elt_at_index (m->locals, i);
2662
2663                 ip4_header_t ip = {
2664                   .src_address = local->addr,
2665                 };
2666
2667                 if (sm->worker_in2out_cb (&ip, m->fib_index, 0) ==
2668                     thread_index)
2669                   {
2670                     vec_add1 (tmp, i);
2671                   }
2672               }));
2673               /* *INDENT-ON* */
2674               ASSERT (vec_len (tmp) != 0);
2675             }
2676           else
2677             {
2678               /* *INDENT-OFF* */
2679               pool_foreach_index (i, m->locals,
2680               ({
2681                 vec_add1 (tmp, i);
2682               }));
2683               /* *INDENT-ON* */
2684             }
2685           hi = vec_len (tmp) - 1;
2686           local = pool_elt_at_index (m->locals, tmp[hi]);
2687           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2688           while (lo < hi)
2689             {
2690               mid = ((hi - lo) >> 1) + lo;
2691               local = pool_elt_at_index (m->locals, tmp[mid]);
2692               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2693             }
2694           local = pool_elt_at_index (m->locals, tmp[lo]);
2695           if (!(local->prefix >= rand))
2696             return 1;
2697           mapping->addr = local->addr;
2698           mapping->port = clib_host_to_net_u16 (local->port);
2699           mapping->fib_index = local->fib_index;
2700           if (m->affinity)
2701             {
2702               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2703                                                 match.protocol, match.port,
2704                                                 tmp[lo], m->affinity,
2705                                                 m->affinity_per_service_list_head_index))
2706                 nat_elog_info ("create affinity record failed");
2707             }
2708           vec_free (tmp);
2709         }
2710       else
2711         {
2712           if (PREDICT_FALSE (lb != 0))
2713             *lb = NO_LB_NAT;
2714           mapping->fib_index = m->fib_index;
2715           mapping->addr = m->local_addr;
2716           /* Address only mapping doesn't change port */
2717           mapping->port = is_addr_only_static_mapping (m) ? match.port
2718             : clib_host_to_net_u16 (m->local_port);
2719         }
2720       mapping->protocol = m->proto;
2721     }
2722   else
2723     {
2724       mapping->addr = m->external_addr;
2725       /* Address only mapping doesn't change port */
2726       mapping->port = is_addr_only_static_mapping (m) ? match.port
2727         : clib_host_to_net_u16 (m->external_port);
2728       mapping->fib_index = sm->outside_fib_index;
2729     }
2730
2731 end:
2732   if (PREDICT_FALSE (is_addr_only != 0))
2733     *is_addr_only = is_addr_only_static_mapping (m);
2734
2735   if (PREDICT_FALSE (twice_nat != 0))
2736     *twice_nat = m->twice_nat;
2737
2738   if (PREDICT_FALSE (is_identity_nat != 0))
2739     *is_identity_nat = is_identity_static_mapping (m);
2740
2741   return 0;
2742 }
2743
2744 static_always_inline u16
2745 snat_random_port (u16 min, u16 max)
2746 {
2747   snat_main_t *sm = &snat_main;
2748   return min + random_u32 (&sm->random_seed) /
2749     (random_u32_max () / (max - min + 1) + 1);
2750 }
2751
2752 int
2753 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2754                                      u32 fib_index,
2755                                      u32 thread_index,
2756                                      snat_session_key_t * k,
2757                                      u16 port_per_thread,
2758                                      u32 snat_thread_index)
2759 {
2760   snat_main_t *sm = &snat_main;
2761
2762   return sm->alloc_addr_and_port (addresses, fib_index, thread_index, k,
2763                                   port_per_thread, snat_thread_index);
2764 }
2765
2766 static int
2767 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2768                                  u32 fib_index,
2769                                  u32 thread_index,
2770                                  snat_session_key_t * k,
2771                                  u16 port_per_thread, u32 snat_thread_index)
2772 {
2773   int i;
2774   snat_address_t *a, *ga = 0;
2775   u32 portnum;
2776
2777   for (i = 0; i < vec_len (addresses); i++)
2778     {
2779       a = addresses + i;
2780       switch (k->protocol)
2781         {
2782 #define _(N, j, n, s) \
2783         case SNAT_PROTOCOL_##N: \
2784           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2785             { \
2786               if (a->fib_index == fib_index) \
2787                 { \
2788                   while (1) \
2789                     { \
2790                       portnum = (port_per_thread * \
2791                         snat_thread_index) + \
2792                         snat_random_port(1, port_per_thread) + 1024; \
2793                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2794                         continue; \
2795                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2796                       a->busy_##n##_ports_per_thread[thread_index]++; \
2797                       a->busy_##n##_ports++; \
2798                       k->addr = a->addr; \
2799                       k->port = clib_host_to_net_u16(portnum); \
2800                       return 0; \
2801                     } \
2802                 } \
2803               else if (a->fib_index == ~0) \
2804                 { \
2805                   ga = a; \
2806                 } \
2807             } \
2808           break;
2809           foreach_snat_protocol
2810 #undef _
2811         default:
2812           nat_elog_info ("unknown protocol");
2813           return 1;
2814         }
2815
2816     }
2817
2818   if (ga)
2819     {
2820       a = ga;
2821       switch (k->protocol)
2822         {
2823 #define _(N, j, n, s) \
2824         case SNAT_PROTOCOL_##N: \
2825           while (1) \
2826             { \
2827               portnum = (port_per_thread * \
2828                 snat_thread_index) + \
2829                 snat_random_port(1, port_per_thread) + 1024; \
2830               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2831                 continue; \
2832               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2833               a->busy_##n##_ports_per_thread[thread_index]++; \
2834               a->busy_##n##_ports++; \
2835               k->addr = a->addr; \
2836               k->port = clib_host_to_net_u16(portnum); \
2837               return 0; \
2838             }
2839           break;
2840           foreach_snat_protocol
2841 #undef _
2842         default:
2843           nat_elog_info ("unknown protocol");
2844           return 1;
2845         }
2846     }
2847
2848   /* Totally out of translations to use... */
2849   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2850   return 1;
2851 }
2852
2853 static int
2854 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2855                               u32 fib_index,
2856                               u32 thread_index,
2857                               snat_session_key_t * k,
2858                               u16 port_per_thread, u32 snat_thread_index)
2859 {
2860   snat_main_t *sm = &snat_main;
2861   snat_address_t *a = addresses;
2862   u16 m, ports, portnum, A, j;
2863   m = 16 - (sm->psid_offset + sm->psid_length);
2864   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2865
2866   if (!vec_len (addresses))
2867     goto exhausted;
2868
2869   switch (k->protocol)
2870     {
2871 #define _(N, i, n, s) \
2872     case SNAT_PROTOCOL_##N: \
2873       if (a->busy_##n##_ports < ports) \
2874         { \
2875           while (1) \
2876             { \
2877               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2878               j = snat_random_port(0, pow2_mask(m)); \
2879               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2880               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2881                 continue; \
2882               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2883               a->busy_##n##_ports++; \
2884               k->addr = a->addr; \
2885               k->port = clib_host_to_net_u16 (portnum); \
2886               return 0; \
2887             } \
2888         } \
2889       break;
2890       foreach_snat_protocol
2891 #undef _
2892     default:
2893       nat_elog_info ("unknown protocol");
2894       return 1;
2895     }
2896
2897 exhausted:
2898   /* Totally out of translations to use... */
2899   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2900   return 1;
2901 }
2902
2903 static int
2904 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2905                                u32 fib_index,
2906                                u32 thread_index,
2907                                snat_session_key_t * k,
2908                                u16 port_per_thread, u32 snat_thread_index)
2909 {
2910   snat_main_t *sm = &snat_main;
2911   snat_address_t *a = addresses;
2912   u16 portnum, ports;
2913
2914   ports = sm->end_port - sm->start_port + 1;
2915
2916   if (!vec_len (addresses))
2917     goto exhausted;
2918
2919   switch (k->protocol)
2920     {
2921 #define _(N, i, n, s) \
2922     case SNAT_PROTOCOL_##N: \
2923       if (a->busy_##n##_ports < ports) \
2924         { \
2925           while (1) \
2926             { \
2927               portnum = snat_random_port(sm->start_port, sm->end_port); \
2928               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2929                 continue; \
2930               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2931               a->busy_##n##_ports++; \
2932               k->addr = a->addr; \
2933               k->port = clib_host_to_net_u16 (portnum); \
2934               return 0; \
2935             } \
2936         } \
2937       break;
2938       foreach_snat_protocol
2939 #undef _
2940     default:
2941       nat_elog_info ("unknown protocol");
2942       return 1;
2943     }
2944
2945 exhausted:
2946   /* Totally out of translations to use... */
2947   snat_ipfix_logging_addresses_exhausted (thread_index, 0);
2948   return 1;
2949 }
2950
2951 void
2952 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2953 {
2954   dpo_id_t dpo_v4 = DPO_INVALID;
2955   fib_prefix_t pfx = {
2956     .fp_proto = FIB_PROTOCOL_IP4,
2957     .fp_len = 32,
2958     .fp_addr.ip4.as_u32 = addr.as_u32,
2959   };
2960
2961   if (is_add)
2962     {
2963       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2964       fib_table_entry_special_dpo_add (0, &pfx, nat_fib_src_hi,
2965                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2966       dpo_reset (&dpo_v4);
2967     }
2968   else
2969     {
2970       fib_table_entry_special_remove (0, &pfx, nat_fib_src_hi);
2971     }
2972 }
2973
2974 u8 *
2975 format_session_kvp (u8 * s, va_list * args)
2976 {
2977   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2978   snat_session_key_t k;
2979
2980   k.as_u64 = v->key;
2981
2982   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2983
2984   return s;
2985 }
2986
2987 u8 *
2988 format_static_mapping_kvp (u8 * s, va_list * args)
2989 {
2990   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2991   snat_session_key_t k;
2992
2993   k.as_u64 = v->key;
2994
2995   s = format (s, "%U static-mapping-index %llu",
2996               format_static_mapping_key, &k, v->value);
2997
2998   return s;
2999 }
3000
3001 u8 *
3002 format_user_kvp (u8 * s, va_list * args)
3003 {
3004   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
3005   snat_user_key_t k;
3006
3007   k.as_u64 = v->key;
3008
3009   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
3010               k.fib_index, v->value);
3011
3012   return s;
3013 }
3014
3015 u8 *
3016 format_ed_session_kvp (u8 * s, va_list * args)
3017 {
3018   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
3019   nat_ed_ses_key_t k;
3020
3021   k.as_u64[0] = v->key[0];
3022   k.as_u64[1] = v->key[1];
3023
3024   s =
3025     format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
3026             format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
3027             format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
3028             format_ip_protocol, k.proto, k.fib_index, v->value);
3029
3030   return s;
3031 }
3032
3033 static u32
3034 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0,
3035                            u8 is_output)
3036 {
3037   snat_main_t *sm = &snat_main;
3038   u32 next_worker_index = 0;
3039   u32 hash;
3040
3041   next_worker_index = sm->first_worker_index;
3042   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
3043     (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >> 24);
3044
3045   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3046     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3047   else
3048     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3049
3050   return next_worker_index;
3051 }
3052
3053 static u32
3054 snat_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip0,
3055                            u32 rx_fib_index0, u8 is_output)
3056 {
3057   snat_main_t *sm = &snat_main;
3058   udp_header_t *udp;
3059   u16 port;
3060   snat_session_key_t m_key;
3061   clib_bihash_kv_8_8_t kv, value;
3062   snat_static_mapping_t *m;
3063   u32 proto;
3064   u32 next_worker_index = 0;
3065
3066   /* first try static mappings without port */
3067   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3068     {
3069       m_key.addr = ip0->dst_address;
3070       m_key.port = 0;
3071       m_key.protocol = 0;
3072       m_key.fib_index = rx_fib_index0;
3073       kv.key = m_key.as_u64;
3074       if (!clib_bihash_search_8_8
3075           (&sm->static_mapping_by_external, &kv, &value))
3076         {
3077           m = pool_elt_at_index (sm->static_mappings, value.value);
3078           return m->workers[0];
3079         }
3080     }
3081
3082   proto = ip_proto_to_snat_proto (ip0->protocol);
3083   udp = ip4_next_header (ip0);
3084   port = udp->dst_port;
3085
3086   /* unknown protocol */
3087   if (PREDICT_FALSE (proto == ~0))
3088     {
3089       /* use current thread */
3090       return vlib_get_thread_index ();
3091     }
3092
3093   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
3094     {
3095       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3096       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3097       if (!icmp_type_is_error_message
3098           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3099         port = vnet_buffer (b)->ip.reass.l4_src_port;
3100       else
3101         {
3102           /* if error message, then it's not fragmented and we can access it */
3103           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3104           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3105           void *l4_header = ip4_next_header (inner_ip);
3106           switch (proto)
3107             {
3108             case SNAT_PROTOCOL_ICMP:
3109               icmp = (icmp46_header_t *) l4_header;
3110               echo = (icmp_echo_header_t *) (icmp + 1);
3111               port = echo->identifier;
3112               break;
3113             case SNAT_PROTOCOL_UDP:
3114             case SNAT_PROTOCOL_TCP:
3115               port = ((tcp_udp_header_t *) l4_header)->src_port;
3116               break;
3117             default:
3118               return vlib_get_thread_index ();
3119             }
3120         }
3121     }
3122
3123   /* try static mappings with port */
3124   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3125     {
3126       m_key.addr = ip0->dst_address;
3127       m_key.port = clib_net_to_host_u16 (port);
3128       m_key.protocol = proto;
3129       m_key.fib_index = rx_fib_index0;
3130       kv.key = m_key.as_u64;
3131       if (!clib_bihash_search_8_8
3132           (&sm->static_mapping_by_external, &kv, &value))
3133         {
3134           m = pool_elt_at_index (sm->static_mappings, value.value);
3135           return m->workers[0];
3136         }
3137     }
3138
3139   /* worker by outside port */
3140   next_worker_index = sm->first_worker_index;
3141   next_worker_index +=
3142     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3143   return next_worker_index;
3144 }
3145
3146 static u32
3147 nat44_ed_get_worker_in2out_cb (ip4_header_t * ip, u32 rx_fib_index,
3148                                u8 is_output)
3149 {
3150   snat_main_t *sm = &snat_main;
3151   u32 next_worker_index = sm->first_worker_index;
3152   u32 hash;
3153
3154   clib_bihash_kv_16_8_t kv16, value16;
3155   snat_main_per_thread_data_t *tsm;
3156   udp_header_t *udp;
3157
3158   if (PREDICT_FALSE (is_output))
3159     {
3160       u32 fib_index = sm->outside_fib_index;
3161       nat_outside_fib_t *outside_fib;
3162       fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3163       fib_prefix_t pfx = {
3164         .fp_proto = FIB_PROTOCOL_IP4,
3165         .fp_len = 32,
3166         .fp_addr = {
3167                     .ip4.as_u32 = ip->dst_address.as_u32,
3168                     }
3169         ,
3170       };
3171
3172       udp = ip4_next_header (ip);
3173
3174       switch (vec_len (sm->outside_fibs))
3175         {
3176         case 0:
3177           fib_index = sm->outside_fib_index;
3178           break;
3179         case 1:
3180           fib_index = sm->outside_fibs[0].fib_index;
3181           break;
3182         default:
3183             /* *INDENT-OFF* */
3184             vec_foreach (outside_fib, sm->outside_fibs)
3185               {
3186                 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3187                 if (FIB_NODE_INDEX_INVALID != fei)
3188                   {
3189                     if (fib_entry_get_resolving_interface (fei) != ~0)
3190                       {
3191                         fib_index = outside_fib->fib_index;
3192                         break;
3193                       }
3194                   }
3195               }
3196             /* *INDENT-ON* */
3197           break;
3198         }
3199
3200       make_ed_kv (&kv16, &ip->src_address, &ip->dst_address,
3201                   ip->protocol, fib_index, udp->src_port, udp->dst_port);
3202
3203       /* *INDENT-OFF* */
3204       vec_foreach (tsm, sm->per_thread_data)
3205         {
3206           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3207                                                       &kv16, &value16)))
3208             {
3209               next_worker_index += tsm->thread_index;
3210
3211               nat_elog_debug_handoff (
3212                 "HANDOFF IN2OUT-OUTPUT-FEATURE (session)",
3213                 next_worker_index, fib_index,
3214                 clib_net_to_host_u32 (ip->src_address.as_u32),
3215                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3216
3217               return next_worker_index;
3218             }
3219         }
3220       /* *INDENT-ON* */
3221     }
3222
3223   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3224     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3225
3226   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3227     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3228   else
3229     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3230
3231   if (PREDICT_TRUE (!is_output))
3232     {
3233       nat_elog_debug_handoff ("HANDOFF IN2OUT",
3234                               next_worker_index, rx_fib_index,
3235                               clib_net_to_host_u32 (ip->src_address.as_u32),
3236                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3237     }
3238   else
3239     {
3240       nat_elog_debug_handoff ("HANDOFF IN2OUT-OUTPUT-FEATURE",
3241                               next_worker_index, rx_fib_index,
3242                               clib_net_to_host_u32 (ip->src_address.as_u32),
3243                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3244     }
3245
3246   return next_worker_index;
3247 }
3248
3249 static u32
3250 nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
3251                                u32 rx_fib_index, u8 is_output)
3252 {
3253   snat_main_t *sm = &snat_main;
3254   clib_bihash_kv_8_8_t kv, value;
3255   clib_bihash_kv_16_8_t kv16, value16;
3256   snat_main_per_thread_data_t *tsm;
3257
3258   u32 proto, next_worker_index = 0;
3259   udp_header_t *udp;
3260   u16 port;
3261   snat_static_mapping_t *m;
3262   u32 hash;
3263
3264   proto = ip_proto_to_snat_proto (ip->protocol);
3265
3266   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_UDP || proto == SNAT_PROTOCOL_TCP))
3267     {
3268       udp = ip4_next_header (ip);
3269
3270       make_ed_kv (&kv16, &ip->dst_address, &ip->src_address,
3271                   ip->protocol, rx_fib_index, udp->dst_port, udp->src_port);
3272
3273       /* *INDENT-OFF* */
3274       vec_foreach (tsm, sm->per_thread_data)
3275         {
3276           if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3277                                                       &kv16, &value16)))
3278             {
3279               next_worker_index = sm->first_worker_index + tsm->thread_index;
3280               nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3281                           next_worker_index, rx_fib_index,
3282                           clib_net_to_host_u32 (ip->src_address.as_u32),
3283                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3284               return next_worker_index;
3285             }
3286           }
3287         /* *INDENT-ON* */
3288     }
3289   else if (proto == SNAT_PROTOCOL_ICMP)
3290     {
3291       nat_ed_ses_key_t key;
3292
3293       if (!get_icmp_o2i_ed_key (b, ip, &key))
3294         {
3295
3296           key.fib_index = rx_fib_index;
3297           kv16.key[0] = key.as_u64[0];
3298           kv16.key[1] = key.as_u64[1];
3299
3300           /* *INDENT-OFF* */
3301           vec_foreach (tsm, sm->per_thread_data)
3302             {
3303               if (PREDICT_TRUE (!clib_bihash_search_16_8 (&tsm->out2in_ed,
3304                                                           &kv16, &value16)))
3305                 {
3306                   next_worker_index = sm->first_worker_index +
3307                                       tsm->thread_index;
3308                   nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
3309                               next_worker_index, rx_fib_index,
3310                               clib_net_to_host_u32 (ip->src_address.as_u32),
3311                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3312                   return next_worker_index;
3313                 }
3314             }
3315           /* *INDENT-ON* */
3316         }
3317     }
3318
3319   /* first try static mappings without port */
3320   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3321     {
3322       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
3323       if (!clib_bihash_search_8_8
3324           (&sm->static_mapping_by_external, &kv, &value))
3325         {
3326           m = pool_elt_at_index (sm->static_mappings, value.value);
3327           next_worker_index = m->workers[0];
3328           goto done;
3329         }
3330     }
3331
3332   /* unknown protocol */
3333   if (PREDICT_FALSE (proto == ~0))
3334     {
3335       /* use current thread */
3336       next_worker_index = vlib_get_thread_index ();
3337       goto done;
3338     }
3339
3340   udp = ip4_next_header (ip);
3341   port = udp->dst_port;
3342
3343   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3344     {
3345       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3346       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3347       if (!icmp_type_is_error_message
3348           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3349         port = vnet_buffer (b)->ip.reass.l4_src_port;
3350       else
3351         {
3352           /* if error message, then it's not fragmented and we can access it */
3353           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3354           proto = ip_proto_to_snat_proto (inner_ip->protocol);
3355           void *l4_header = ip4_next_header (inner_ip);
3356           switch (proto)
3357             {
3358             case SNAT_PROTOCOL_ICMP:
3359               icmp = (icmp46_header_t *) l4_header;
3360               echo = (icmp_echo_header_t *) (icmp + 1);
3361               port = echo->identifier;
3362               break;
3363             case SNAT_PROTOCOL_UDP:
3364             case SNAT_PROTOCOL_TCP:
3365               port = ((tcp_udp_header_t *) l4_header)->src_port;
3366               break;
3367             default:
3368               next_worker_index = vlib_get_thread_index ();
3369               goto done;
3370             }
3371         }
3372     }
3373
3374   /* try static mappings with port */
3375   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3376     {
3377       make_sm_kv (&kv, &ip->dst_address, proto, 0,
3378                   clib_net_to_host_u16 (port));
3379       if (!clib_bihash_search_8_8
3380           (&sm->static_mapping_by_external, &kv, &value))
3381         {
3382           m = pool_elt_at_index (sm->static_mappings, value.value);
3383           if (!is_lb_static_mapping (m))
3384             {
3385               next_worker_index = m->workers[0];
3386               goto done;
3387             }
3388
3389           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3390             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3391
3392           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3393             next_worker_index =
3394               m->workers[hash & (_vec_len (m->workers) - 1)];
3395           else
3396             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3397           goto done;
3398         }
3399     }
3400
3401   /* worker by outside port */
3402   next_worker_index = sm->first_worker_index;
3403   next_worker_index +=
3404     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3405
3406 done:
3407   nat_elog_debug_handoff ("HANDOFF OUT2IN", next_worker_index, rx_fib_index,
3408                           clib_net_to_host_u32 (ip->src_address.as_u32),
3409                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3410   return next_worker_index;
3411 }
3412
3413 void
3414 nat_ha_sadd_cb (ip4_address_t * in_addr, u16 in_port,
3415                 ip4_address_t * out_addr, u16 out_port,
3416                 ip4_address_t * eh_addr, u16 eh_port,
3417                 ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3418                 u32 fib_index, u16 flags, u32 thread_index)
3419 {
3420   snat_main_t *sm = &snat_main;
3421   snat_session_key_t key;
3422   snat_user_t *u;
3423   snat_session_t *s;
3424   clib_bihash_kv_8_8_t kv;
3425   f64 now = vlib_time_now (sm->vlib_main);
3426   nat_outside_fib_t *outside_fib;
3427   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3428   snat_main_per_thread_data_t *tsm;
3429   fib_prefix_t pfx = {
3430     .fp_proto = FIB_PROTOCOL_IP4,
3431     .fp_len = 32,
3432     .fp_addr = {
3433                 .ip4.as_u32 = eh_addr->as_u32,
3434                 },
3435   };
3436
3437   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3438
3439   key.addr.as_u32 = out_addr->as_u32;
3440   key.port = out_port;
3441   key.protocol = proto;
3442
3443   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3444     {
3445       if (nat_set_outside_address_and_port
3446           (sm->addresses, thread_index, &key))
3447         return;
3448     }
3449
3450   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3451   if (!u)
3452     return;
3453
3454   s = nat_session_alloc_or_recycle (sm, u, thread_index, now);
3455   if (!s)
3456     return;
3457
3458   s->last_heard = now;
3459   s->flags = flags;
3460   s->ext_host_addr.as_u32 = eh_addr->as_u32;
3461   s->ext_host_port = eh_port;
3462   user_session_increment (sm, u, snat_is_session_static (s));
3463   switch (vec_len (sm->outside_fibs))
3464     {
3465     case 0:
3466       key.fib_index = sm->outside_fib_index;
3467       break;
3468     case 1:
3469       key.fib_index = sm->outside_fibs[0].fib_index;
3470       break;
3471     default:
3472       /* *INDENT-OFF* */
3473       vec_foreach (outside_fib, sm->outside_fibs)
3474         {
3475           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3476           if (FIB_NODE_INDEX_INVALID != fei)
3477             {
3478               if (fib_entry_get_resolving_interface (fei) != ~0)
3479                 {
3480                   key.fib_index = outside_fib->fib_index;
3481                   break;
3482                 }
3483             }
3484         }
3485       /* *INDENT-ON* */
3486       break;
3487     }
3488   s->out2in = key;
3489   kv.key = key.as_u64;
3490   kv.value = s - tsm->sessions;
3491   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 1))
3492     nat_elog_warn ("out2in key add failed");
3493
3494   key.addr.as_u32 = in_addr->as_u32;
3495   key.port = in_port;
3496   key.fib_index = fib_index;
3497   s->in2out = key;
3498   kv.key = key.as_u64;
3499   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 1))
3500     nat_elog_warn ("in2out key add failed");
3501 }
3502
3503 void
3504 nat_ha_sdel_cb (ip4_address_t * out_addr, u16 out_port,
3505                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3506                 u32 ti)
3507 {
3508   snat_main_t *sm = &snat_main;
3509   snat_session_key_t key;
3510   clib_bihash_kv_8_8_t kv, value;
3511   u32 thread_index;
3512   snat_session_t *s;
3513   snat_main_per_thread_data_t *tsm;
3514
3515   if (sm->num_workers > 1)
3516     thread_index =
3517       sm->first_worker_index +
3518       (sm->workers[(clib_net_to_host_u16 (out_port) -
3519                     1024) / sm->port_per_thread]);
3520   else
3521     thread_index = sm->num_workers;
3522   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3523
3524   key.addr.as_u32 = out_addr->as_u32;
3525   key.port = out_port;
3526   key.protocol = proto;
3527   key.fib_index = fib_index;
3528   kv.key = key.as_u64;
3529   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3530     return;
3531
3532   s = pool_elt_at_index (tsm->sessions, value.value);
3533   nat_free_session_data (sm, s, thread_index, 1);
3534   nat44_delete_session (sm, s, thread_index);
3535 }
3536
3537 void
3538 nat_ha_sref_cb (ip4_address_t * out_addr, u16 out_port,
3539                 ip4_address_t * eh_addr, u16 eh_port, u8 proto, u32 fib_index,
3540                 u32 total_pkts, u64 total_bytes, u32 thread_index)
3541 {
3542   snat_main_t *sm = &snat_main;
3543   snat_session_key_t key;
3544   clib_bihash_kv_8_8_t kv, value;
3545   snat_session_t *s;
3546   snat_main_per_thread_data_t *tsm;
3547
3548   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3549
3550   key.addr.as_u32 = out_addr->as_u32;
3551   key.port = out_port;
3552   key.protocol = proto;
3553   key.fib_index = fib_index;
3554   kv.key = key.as_u64;
3555   if (clib_bihash_search_8_8 (&tsm->out2in, &kv, &value))
3556     return;
3557
3558   s = pool_elt_at_index (tsm->sessions, value.value);
3559   s->total_pkts = total_pkts;
3560   s->total_bytes = total_bytes;
3561 }
3562
3563 void
3564 nat_ha_sadd_ed_cb (ip4_address_t * in_addr, u16 in_port,
3565                    ip4_address_t * out_addr, u16 out_port,
3566                    ip4_address_t * eh_addr, u16 eh_port,
3567                    ip4_address_t * ehn_addr, u16 ehn_port, u8 proto,
3568                    u32 fib_index, u16 flags, u32 thread_index)
3569 {
3570   snat_main_t *sm = &snat_main;
3571   snat_session_key_t key;
3572   snat_user_t *u;
3573   snat_session_t *s;
3574   clib_bihash_kv_16_8_t kv;
3575   f64 now = vlib_time_now (sm->vlib_main);
3576   nat_outside_fib_t *outside_fib;
3577   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
3578   snat_main_per_thread_data_t *tsm;
3579   fib_prefix_t pfx = {
3580     .fp_proto = FIB_PROTOCOL_IP4,
3581     .fp_len = 32,
3582     .fp_addr = {
3583                 .ip4.as_u32 = eh_addr->as_u32,
3584                 },
3585   };
3586
3587   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3588
3589   key.addr.as_u32 = out_addr->as_u32;
3590   key.port = out_port;
3591   key.protocol = proto;
3592
3593   if (!(flags & SNAT_SESSION_FLAG_STATIC_MAPPING))
3594     {
3595       if (nat_set_outside_address_and_port
3596           (sm->addresses, thread_index, &key))
3597         return;
3598     }
3599
3600   key.addr.as_u32 = ehn_addr->as_u32;
3601   key.port = ehn_port;
3602   if (flags & SNAT_SESSION_FLAG_TWICE_NAT)
3603     {
3604       if (nat_set_outside_address_and_port
3605           (sm->twice_nat_addresses, thread_index, &key))
3606         return;
3607     }
3608
3609   u = nat_user_get_or_create (sm, in_addr, fib_index, thread_index);
3610   if (!u)
3611     return;
3612
3613   s = nat_ed_session_alloc (sm, u, thread_index, now);
3614   if (!s)
3615     return;
3616
3617   s->last_heard = now;
3618   s->flags = flags;
3619   s->ext_host_nat_addr.as_u32 = s->ext_host_addr.as_u32 = eh_addr->as_u32;
3620   s->ext_host_nat_port = s->ext_host_port = eh_port;
3621   if (is_twice_nat_session (s))
3622     {
3623       s->ext_host_nat_addr.as_u32 = ehn_addr->as_u32;
3624       s->ext_host_nat_port = ehn_port;
3625     }
3626   user_session_increment (sm, u, snat_is_session_static (s));
3627   switch (vec_len (sm->outside_fibs))
3628     {
3629     case 0:
3630       key.fib_index = sm->outside_fib_index;
3631       break;
3632     case 1:
3633       key.fib_index = sm->outside_fibs[0].fib_index;
3634       break;
3635     default:
3636       /* *INDENT-OFF* */
3637       vec_foreach (outside_fib, sm->outside_fibs)
3638         {
3639           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
3640           if (FIB_NODE_INDEX_INVALID != fei)
3641             {
3642               if (fib_entry_get_resolving_interface (fei) != ~0)
3643                 {
3644                   key.fib_index = outside_fib->fib_index;
3645                   break;
3646                 }
3647             }
3648         }
3649       /* *INDENT-ON* */
3650       break;
3651     }
3652   key.addr.as_u32 = out_addr->as_u32;
3653   key.port = out_port;
3654   s->out2in = key;
3655   kv.value = s - tsm->sessions;
3656
3657   key.addr.as_u32 = in_addr->as_u32;
3658   key.port = in_port;
3659   key.fib_index = fib_index;
3660   s->in2out = key;
3661
3662   make_ed_kv (&kv, in_addr, &s->ext_host_nat_addr,
3663               snat_proto_to_ip_proto (proto), fib_index, in_port,
3664               s->ext_host_nat_port);
3665   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
3666     nat_elog_warn ("in2out key add failed");
3667
3668   make_ed_kv (&kv, out_addr, eh_addr, snat_proto_to_ip_proto (proto),
3669               s->out2in.fib_index, out_port, eh_port);
3670   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
3671     nat_elog_warn ("out2in key add failed");
3672 }
3673
3674 void
3675 nat_ha_sdel_ed_cb (ip4_address_t * out_addr, u16 out_port,
3676                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3677                    u32 fib_index, u32 ti)
3678 {
3679   snat_main_t *sm = &snat_main;
3680   nat_ed_ses_key_t key;
3681   clib_bihash_kv_16_8_t kv, value;
3682   u32 thread_index;
3683   snat_session_t *s;
3684   snat_main_per_thread_data_t *tsm;
3685
3686   if (sm->num_workers > 1)
3687     thread_index =
3688       sm->first_worker_index +
3689       (sm->workers[(clib_net_to_host_u16 (out_port) -
3690                     1024) / sm->port_per_thread]);
3691   else
3692     thread_index = sm->num_workers;
3693   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3694
3695   key.l_addr.as_u32 = out_addr->as_u32;
3696   key.l_port = out_port;
3697   key.r_addr.as_u32 = eh_addr->as_u32;
3698   key.r_port = eh_port;
3699   key.proto = proto;
3700   key.fib_index = fib_index;
3701   kv.key[0] = key.as_u64[0];
3702   kv.key[1] = key.as_u64[1];
3703   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3704     return;
3705
3706   s = pool_elt_at_index (tsm->sessions, value.value);
3707   nat_free_session_data (sm, s, thread_index, 1);
3708   nat44_delete_session (sm, s, thread_index);
3709 }
3710
3711 void
3712 nat_ha_sref_ed_cb (ip4_address_t * out_addr, u16 out_port,
3713                    ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3714                    u32 fib_index, u32 total_pkts, u64 total_bytes,
3715                    u32 thread_index)
3716 {
3717   snat_main_t *sm = &snat_main;
3718   nat_ed_ses_key_t key;
3719   clib_bihash_kv_16_8_t kv, value;
3720   snat_session_t *s;
3721   snat_main_per_thread_data_t *tsm;
3722
3723   tsm = vec_elt_at_index (sm->per_thread_data, thread_index);
3724
3725   key.l_addr.as_u32 = out_addr->as_u32;
3726   key.l_port = out_port;
3727   key.r_addr.as_u32 = eh_addr->as_u32;
3728   key.r_port = eh_port;
3729   key.proto = proto;
3730   key.fib_index = fib_index;
3731   kv.key[0] = key.as_u64[0];
3732   kv.key[1] = key.as_u64[1];
3733   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
3734     return;
3735
3736   s = pool_elt_at_index (tsm->sessions, value.value);
3737   s->total_pkts = total_pkts;
3738   s->total_bytes = total_bytes;
3739 }
3740
3741 static clib_error_t *
3742 snat_config (vlib_main_t * vm, unformat_input_t * input)
3743 {
3744   snat_main_t *sm = &snat_main;
3745   nat66_main_t *nm = &nat66_main;
3746   //dslite_main_t *dm = &dslite_main;
3747   snat_main_per_thread_data_t *tsm;
3748
3749   u32 static_mapping_buckets = 1024;
3750   u32 static_mapping_memory_size = 64 << 20;
3751
3752   u32 nat64_bib_buckets = 1024;
3753   u32 nat64_bib_memory_size = 128 << 20;
3754
3755   u32 nat64_st_buckets = 2048;
3756   u32 nat64_st_memory_size = 256 << 20;
3757
3758   u32 user_buckets = 128;
3759   u32 user_memory_size = 64 << 20;
3760   u32 translation_buckets = 1024;
3761   u32 translation_memory_size = 128 << 20;
3762
3763   u32 max_translations_per_user = ~0;
3764
3765   u32 outside_vrf_id = 0;
3766   u32 outside_ip6_vrf_id = 0;
3767   u32 inside_vrf_id = 0;
3768   u8 static_mapping_only = 0;
3769   u8 static_mapping_connection_tracking = 0;
3770
3771   u32 udp_timeout = SNAT_UDP_TIMEOUT;
3772   u32 icmp_timeout = SNAT_ICMP_TIMEOUT;
3773
3774   u32 tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
3775   u32 tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
3776
3777   sm->deterministic = 0;
3778   sm->out2in_dpo = 0;
3779   sm->endpoint_dependent = 0;
3780
3781   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3782     {
3783       if (unformat
3784           (input, "translation hash buckets %d", &translation_buckets))
3785         ;
3786       else if (unformat (input, "udp timeout %d", &udp_timeout))
3787         ;
3788       else if (unformat (input, "icmp timeout %d", &icmp_timeout))
3789         ;
3790       else if (unformat (input, "tcp transitory timeout %d",
3791                          &tcp_transitory_timeout));
3792       else if (unformat (input, "tcp established timeout %d",
3793                          &tcp_established_timeout));
3794       else if (unformat (input, "translation hash memory %d",
3795                          &translation_memory_size));
3796       else if (unformat (input, "user hash buckets %d", &user_buckets))
3797         ;
3798       else if (unformat (input, "user hash memory %d", &user_memory_size))
3799         ;
3800       else if (unformat (input, "max translations per user %d",
3801                          &max_translations_per_user))
3802         ;
3803       else if (unformat (input, "outside VRF id %d", &outside_vrf_id))
3804         ;
3805       else if (unformat (input, "outside ip6 VRF id %d", &outside_ip6_vrf_id))
3806         ;
3807       else if (unformat (input, "inside VRF id %d", &inside_vrf_id))
3808         ;
3809       else if (unformat (input, "static mapping only"))
3810         {
3811           static_mapping_only = 1;
3812           if (unformat (input, "connection tracking"))
3813             static_mapping_connection_tracking = 1;
3814         }
3815       else if (unformat (input, "deterministic"))
3816         sm->deterministic = 1;
3817       else if (unformat (input, "nat64 bib hash buckets %d",
3818                          &nat64_bib_buckets))
3819         ;
3820       else if (unformat (input, "nat64 bib hash memory %d",
3821                          &nat64_bib_memory_size))
3822         ;
3823       else
3824         if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
3825         ;
3826       else if (unformat (input, "nat64 st hash memory %d",
3827                          &nat64_st_memory_size))
3828         ;
3829       else if (unformat (input, "out2in dpo"))
3830         sm->out2in_dpo = 1;
3831       //else if (unformat (input, "dslite ce"))
3832       //dslite_set_ce (dm, 1);
3833       else if (unformat (input, "endpoint-dependent"))
3834         sm->endpoint_dependent = 1;
3835       else
3836         return clib_error_return (0, "unknown input '%U'",
3837                                   format_unformat_error, input);
3838     }
3839
3840   if (sm->deterministic && sm->endpoint_dependent)
3841     return clib_error_return (0,
3842                               "deterministic and endpoint-dependent modes are mutually exclusive");
3843
3844   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
3845     return clib_error_return (0,
3846                               "static mapping only mode available only for simple nat");
3847
3848   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
3849     return clib_error_return (0,
3850                               "out2in dpo mode available only for simple nat");
3851
3852   /* optionally configurable timeouts for testing purposes */
3853   sm->udp_timeout = udp_timeout;
3854   sm->icmp_timeout = icmp_timeout;
3855   sm->tcp_transitory_timeout = tcp_transitory_timeout;
3856   sm->tcp_established_timeout = tcp_established_timeout;
3857
3858   sm->user_buckets = user_buckets;
3859   sm->user_memory_size = user_memory_size;
3860
3861   sm->translation_buckets = translation_buckets;
3862   sm->translation_memory_size = translation_memory_size;
3863
3864   /* do not exceed load factor 10 */
3865   sm->max_translations = 10 * translation_buckets;
3866   sm->max_translations_per_user = max_translations_per_user == ~0 ?
3867     sm->max_translations : max_translations_per_user;
3868
3869   sm->outside_vrf_id = outside_vrf_id;
3870   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3871                                                              outside_vrf_id,
3872                                                              nat_fib_src_hi);
3873   nm->outside_vrf_id = outside_ip6_vrf_id;
3874   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3875                                                              outside_ip6_vrf_id,
3876                                                              nat_fib_src_hi);
3877   sm->inside_vrf_id = inside_vrf_id;
3878   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3879                                                             inside_vrf_id,
3880                                                             nat_fib_src_hi);
3881   sm->static_mapping_only = static_mapping_only;
3882   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3883
3884   nat64_set_hash (nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3885                   nat64_st_memory_size);
3886
3887   if (sm->deterministic)
3888     {
3889       sm->in2out_node_index = snat_det_in2out_node.index;
3890       sm->in2out_output_node_index = ~0;
3891       sm->out2in_node_index = snat_det_out2in_node.index;
3892       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3893       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3894     }
3895   else
3896     {
3897       if (sm->endpoint_dependent)
3898         {
3899           sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
3900           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3901
3902           sm->handoff_out2in_index = nat_pre_out2in_node.index;
3903           sm->handoff_in2out_index = nat_pre_in2out_node.index;
3904           sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
3905
3906           sm->in2out_node_index = nat44_ed_in2out_node.index;
3907           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3908           sm->out2in_node_index = nat44_ed_out2in_node.index;
3909
3910           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3911           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3912           nat_affinity_init (vm);
3913           nat_ha_init (vm, nat_ha_sadd_ed_cb, nat_ha_sdel_ed_cb,
3914                        nat_ha_sref_ed_cb);
3915         }
3916       else
3917         {
3918           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3919           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3920
3921           sm->handoff_out2in_index = snat_in2out_node.index;
3922           sm->handoff_in2out_index = snat_out2in_node.index;
3923           sm->handoff_in2out_output_index = snat_in2out_output_node.index;
3924
3925           sm->in2out_node_index = snat_in2out_node.index;
3926           sm->in2out_output_node_index = snat_in2out_output_node.index;
3927           sm->out2in_node_index = snat_out2in_node.index;
3928           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3929           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3930           nat_ha_init (vm, nat_ha_sadd_cb, nat_ha_sdel_cb, nat_ha_sref_cb);
3931         }
3932       if (!static_mapping_only ||
3933           (static_mapping_only && static_mapping_connection_tracking))
3934         {
3935           /* *INDENT-OFF* */
3936           vec_foreach (tsm, sm->per_thread_data)
3937             {
3938               if (sm->endpoint_dependent)
3939                 {
3940                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3941                                          translation_buckets,
3942                                          translation_memory_size);
3943                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3944                                                       format_ed_session_kvp);
3945
3946                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3947                                          translation_buckets,
3948                                          translation_memory_size);
3949                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3950                                                       format_ed_session_kvp);
3951                 }
3952               else
3953                 {
3954                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3955                                         translation_buckets,
3956                                         translation_memory_size);
3957                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3958                                                      format_session_kvp);
3959
3960                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3961                                         translation_buckets,
3962                                         translation_memory_size);
3963                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3964                                                      format_session_kvp);
3965                 }
3966
3967               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3968                                     user_memory_size);
3969               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3970                                                  format_user_kvp);
3971             }
3972           /* *INDENT-ON* */
3973
3974         }
3975       else
3976         {
3977           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3978           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3979         }
3980       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3981                             "static_mapping_by_local", static_mapping_buckets,
3982                             static_mapping_memory_size);
3983       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3984                                          format_static_mapping_kvp);
3985
3986       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3987                             "static_mapping_by_external",
3988                             static_mapping_buckets,
3989                             static_mapping_memory_size);
3990       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3991                                          format_static_mapping_kvp);
3992     }
3993
3994   return 0;
3995 }
3996
3997 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3998
3999 static void
4000 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
4001                                  uword opaque,
4002                                  u32 sw_if_index,
4003                                  ip4_address_t * address,
4004                                  u32 address_length,
4005                                  u32 if_address_index, u32 is_delete)
4006 {
4007   snat_main_t *sm = &snat_main;
4008   snat_static_map_resolve_t *rp;
4009   snat_static_mapping_t *m;
4010   snat_session_key_t m_key;
4011   clib_bihash_kv_8_8_t kv, value;
4012   int i, rv;
4013   ip4_address_t l_addr;
4014
4015   for (i = 0; i < vec_len (sm->to_resolve); i++)
4016     {
4017       rp = sm->to_resolve + i;
4018       if (rp->addr_only == 0)
4019         continue;
4020       if (rp->sw_if_index == sw_if_index)
4021         goto match;
4022     }
4023
4024   return;
4025
4026 match:
4027   m_key.addr.as_u32 = address->as_u32;
4028   m_key.port = rp->addr_only ? 0 : rp->e_port;
4029   m_key.protocol = rp->addr_only ? 0 : rp->proto;
4030   m_key.fib_index = sm->outside_fib_index;
4031   kv.key = m_key.as_u64;
4032   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4033     m = 0;
4034   else
4035     m = pool_elt_at_index (sm->static_mappings, value.value);
4036
4037   if (!is_delete)
4038     {
4039       /* Don't trip over lease renewal, static config */
4040       if (m)
4041         return;
4042     }
4043   else
4044     {
4045       if (!m)
4046         return;
4047     }
4048
4049   /* Indetity mapping? */
4050   if (rp->l_addr.as_u32 == 0)
4051     l_addr.as_u32 = address[0].as_u32;
4052   else
4053     l_addr.as_u32 = rp->l_addr.as_u32;
4054   /* Add the static mapping */
4055   rv = snat_add_static_mapping (l_addr,
4056                                 address[0],
4057                                 rp->l_port,
4058                                 rp->e_port,
4059                                 rp->vrf_id,
4060                                 rp->addr_only, ~0 /* sw_if_index */ ,
4061                                 rp->proto, !is_delete, rp->twice_nat,
4062                                 rp->out2in_only, rp->tag, rp->identity_nat);
4063   if (rv)
4064     nat_elog_notice_X1 ("snat_add_static_mapping returned %d", "i4", rv);
4065 }
4066
4067 static void
4068 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
4069                                        uword opaque,
4070                                        u32 sw_if_index,
4071                                        ip4_address_t * address,
4072                                        u32 address_length,
4073                                        u32 if_address_index, u32 is_delete)
4074 {
4075   snat_main_t *sm = &snat_main;
4076   snat_static_map_resolve_t *rp;
4077   ip4_address_t l_addr;
4078   int i, j;
4079   int rv;
4080   u8 twice_nat = 0;
4081   snat_address_t *addresses = sm->addresses;
4082
4083   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
4084     {
4085       if (sw_if_index == sm->auto_add_sw_if_indices[i])
4086         goto match;
4087     }
4088
4089   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
4090     {
4091       twice_nat = 1;
4092       addresses = sm->twice_nat_addresses;
4093       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
4094         goto match;
4095     }
4096
4097   return;
4098
4099 match:
4100   if (!is_delete)
4101     {
4102       /* Don't trip over lease renewal, static config */
4103       for (j = 0; j < vec_len (addresses); j++)
4104         if (addresses[j].addr.as_u32 == address->as_u32)
4105           return;
4106
4107       (void) snat_add_address (sm, address, ~0, twice_nat);
4108       /* Scan static map resolution vector */
4109       for (j = 0; j < vec_len (sm->to_resolve); j++)
4110         {
4111           rp = sm->to_resolve + j;
4112           if (rp->addr_only)
4113             continue;
4114           /* On this interface? */
4115           if (rp->sw_if_index == sw_if_index)
4116             {
4117               /* Indetity mapping? */
4118               if (rp->l_addr.as_u32 == 0)
4119                 l_addr.as_u32 = address[0].as_u32;
4120               else
4121                 l_addr.as_u32 = rp->l_addr.as_u32;
4122               /* Add the static mapping */
4123               rv = snat_add_static_mapping (l_addr,
4124                                             address[0],
4125                                             rp->l_port,
4126                                             rp->e_port,
4127                                             rp->vrf_id,
4128                                             rp->addr_only,
4129                                             ~0 /* sw_if_index */ ,
4130                                             rp->proto,
4131                                             rp->is_add, rp->twice_nat,
4132                                             rp->out2in_only, rp->tag,
4133                                             rp->identity_nat);
4134               if (rv)
4135                 nat_elog_notice_X1 ("snat_add_static_mapping returned %d",
4136                                     "i4", rv);
4137             }
4138         }
4139       return;
4140     }
4141   else
4142     {
4143       (void) snat_del_address (sm, address[0], 1, twice_nat);
4144       return;
4145     }
4146 }
4147
4148
4149 int
4150 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
4151                             u8 twice_nat)
4152 {
4153   ip4_main_t *ip4_main = sm->ip4_main;
4154   ip4_address_t *first_int_addr;
4155   snat_static_map_resolve_t *rp;
4156   u32 *indices_to_delete = 0;
4157   int i, j;
4158   u32 *auto_add_sw_if_indices =
4159     twice_nat ? sm->
4160     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
4161
4162   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
4163     );
4164
4165   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
4166     {
4167       if (auto_add_sw_if_indices[i] == sw_if_index)
4168         {
4169           if (is_del)
4170             {
4171               /* if have address remove it */
4172               if (first_int_addr)
4173                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
4174               else
4175                 {
4176                   for (j = 0; j < vec_len (sm->to_resolve); j++)
4177                     {
4178                       rp = sm->to_resolve + j;
4179                       if (rp->sw_if_index == sw_if_index)
4180                         vec_add1 (indices_to_delete, j);
4181                     }
4182                   if (vec_len (indices_to_delete))
4183                     {
4184                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
4185                         vec_del1 (sm->to_resolve, j);
4186                       vec_free (indices_to_delete);
4187                     }
4188                 }
4189               if (twice_nat)
4190                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
4191               else
4192                 vec_del1 (sm->auto_add_sw_if_indices, i);
4193             }
4194           else
4195             return VNET_API_ERROR_VALUE_EXIST;
4196
4197           return 0;
4198         }
4199     }
4200
4201   if (is_del)
4202     return VNET_API_ERROR_NO_SUCH_ENTRY;
4203
4204   /* add to the auto-address list */
4205   if (twice_nat)
4206     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
4207   else
4208     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
4209
4210   /* If the address is already bound - or static - add it now */
4211   if (first_int_addr)
4212     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
4213
4214   return 0;
4215 }
4216
4217 int
4218 nat44_del_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4219                    snat_protocol_t proto, u32 vrf_id, int is_in)
4220 {
4221   snat_main_per_thread_data_t *tsm;
4222   clib_bihash_kv_8_8_t kv, value;
4223   ip4_header_t ip;
4224   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4225   snat_session_key_t key;
4226   snat_session_t *s;
4227   clib_bihash_8_8_t *t;
4228
4229   if (sm->endpoint_dependent)
4230     return VNET_API_ERROR_UNSUPPORTED;
4231
4232   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4233   if (sm->num_workers > 1)
4234     tsm =
4235       vec_elt_at_index (sm->per_thread_data,
4236                         sm->worker_in2out_cb (&ip, fib_index, 0));
4237   else
4238     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4239
4240   key.addr.as_u32 = addr->as_u32;
4241   key.port = clib_host_to_net_u16 (port);
4242   key.protocol = proto;
4243   key.fib_index = fib_index;
4244   kv.key = key.as_u64;
4245   t = is_in ? &tsm->in2out : &tsm->out2in;
4246   if (!clib_bihash_search_8_8 (t, &kv, &value))
4247     {
4248       if (pool_is_free_index (tsm->sessions, value.value))
4249         return VNET_API_ERROR_UNSPECIFIED;
4250
4251       s = pool_elt_at_index (tsm->sessions, value.value);
4252       nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4253       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4254       return 0;
4255     }
4256
4257   return VNET_API_ERROR_NO_SUCH_ENTRY;
4258 }
4259
4260 int
4261 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
4262                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
4263                       u32 vrf_id, int is_in)
4264 {
4265   ip4_header_t ip;
4266   clib_bihash_16_8_t *t;
4267   nat_ed_ses_key_t key;
4268   clib_bihash_kv_16_8_t kv, value;
4269   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
4270   snat_session_t *s;
4271   snat_main_per_thread_data_t *tsm;
4272
4273   if (!sm->endpoint_dependent)
4274     return VNET_API_ERROR_FEATURE_DISABLED;
4275
4276   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
4277   if (sm->num_workers > 1)
4278     tsm =
4279       vec_elt_at_index (sm->per_thread_data,
4280                         sm->worker_in2out_cb (&ip, fib_index, 0));
4281   else
4282     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4283
4284   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
4285   key.l_addr.as_u32 = addr->as_u32;
4286   key.r_addr.as_u32 = eh_addr->as_u32;
4287   key.l_port = clib_host_to_net_u16 (port);
4288   key.r_port = clib_host_to_net_u16 (eh_port);
4289   key.proto = proto;
4290   key.fib_index = fib_index;
4291   kv.key[0] = key.as_u64[0];
4292   kv.key[1] = key.as_u64[1];
4293   if (clib_bihash_search_16_8 (t, &kv, &value))
4294     return VNET_API_ERROR_NO_SUCH_ENTRY;
4295
4296   if (pool_is_free_index (tsm->sessions, value.value))
4297     return VNET_API_ERROR_UNSPECIFIED;
4298   s = pool_elt_at_index (tsm->sessions, value.value);
4299   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
4300   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
4301   return 0;
4302 }
4303
4304 void
4305 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
4306 {
4307   snat_main_t *sm = &snat_main;
4308
4309   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
4310   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
4311   sm->psid = psid;
4312   sm->psid_offset = psid_offset;
4313   sm->psid_length = psid_length;
4314 }
4315
4316 void
4317 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
4318 {
4319   snat_main_t *sm = &snat_main;
4320
4321   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
4322   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
4323   sm->start_port = start_port;
4324   sm->end_port = end_port;
4325 }
4326
4327 void
4328 nat_set_alloc_addr_and_port_default (void)
4329 {
4330   snat_main_t *sm = &snat_main;
4331
4332   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
4333   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
4334 }
4335
4336 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
4337                                  vlib_node_runtime_t * node,
4338                                  vlib_frame_t * frame)
4339 {
4340   return 0;
4341 }
4342
4343 /* *INDENT-OFF* */
4344 VLIB_REGISTER_NODE (nat_default_node) = {
4345   .name = "nat-default",
4346   .vector_size = sizeof (u32),
4347   .format_trace = 0,
4348   .type = VLIB_NODE_TYPE_INTERNAL,
4349   .n_errors = 0,
4350   .n_next_nodes = NAT_N_NEXT,
4351   .next_nodes = {
4352     [NAT_NEXT_DROP] = "error-drop",
4353     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4354     [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
4355     [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
4356     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
4357     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
4358     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
4359     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
4360     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
4361   },
4362 };
4363 /* *INDENT-ON* */
4364
4365 /*
4366  * fd.io coding-style-patch-verification: ON
4367  *
4368  * Local Variables:
4369  * eval: (c-set-style "gnu")
4370  * End:
4371  */