src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <vnet/fib/fib_table.h>
  33 #include <vnet/fib/ip4_fib.h>
  34
  35 #include <vpp/app/version.h>
  36
  37 snat_main_t snat_main;
  38
  39
  40 /* Hook up input features */
  41 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  42   .arc_name = "ip4-unicast",
  43   .node_name = "nat44-in2out",
  44   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  45 };
  46 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat44-out2in",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-dhcp-client-detect"),
  51 };
  52 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat44-classify",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  56 };
  57 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  58   .arc_name = "ip4-unicast",
  59   .node_name = "nat44-det-in2out",
  60   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  61 };
  62 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  63   .arc_name = "ip4-unicast",
  64   .node_name = "nat44-det-out2in",
  65   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  66                                "ip4-dhcp-client-detect"),
  67 };
  68 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  69   .arc_name = "ip4-unicast",
  70   .node_name = "nat44-det-classify",
  71   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  72 };
  73 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  74   .arc_name = "ip4-unicast",
  75   .node_name = "nat44-ed-in2out",
  76   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  77 };
  78 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  79   .arc_name = "ip4-unicast",
  80   .node_name = "nat44-ed-out2in",
  81   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  82                                "ip4-dhcp-client-detect"),
  83 };
  84 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-ed-classify",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-in2out-worker-handoff",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  93 };
  94 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-out2in-worker-handoff",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-dhcp-client-detect"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-handoff-classify",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 104 };
 105 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 106   .arc_name = "ip4-unicast",
 107   .node_name = "nat44-in2out-fast",
 108   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 109 };
 110 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 111   .arc_name = "ip4-unicast",
 112   .node_name = "nat44-out2in-fast",
 113   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 114                                "ip4-dhcp-client-detect"),
 115 };
 116 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 117   .arc_name = "ip4-unicast",
 118   .node_name = "nat44-hairpin-dst",
 119   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 120 };
 121 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 122   .arc_name = "ip4-unicast",
 123   .node_name = "nat44-ed-hairpin-dst",
 124   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 125 };
 126
 127 /* Hook up output features */
 128 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 129   .arc_name = "ip4-output",
 130   .node_name = "nat44-in2out-output",
 131   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 132 };
 133 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 134   .arc_name = "ip4-output",
 135   .node_name = "nat44-in2out-output-worker-handoff",
 136   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-hairpin-src",
 141   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 142 };
 143 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 144   .arc_name = "ip4-output",
 145   .node_name = "nat44-ed-in2out-output",
 146   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 147 };
 148 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-ed-hairpin-src",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 152 };
 153
 154 /* Hook up ip4-local features */
 155 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 156 {
 157   .arc_name = "ip4-local",
 158   .node_name = "nat44-hairpinning",
 159   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 162 {
 163   .arc_name = "ip4-local",
 164   .node_name = "nat44-ed-hairpinning",
 165   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 166 };
 167
 168
 169 /* *INDENT-OFF* */
 170 VLIB_PLUGIN_REGISTER () = {
 171     .version = VPP_BUILD_VER,
 172     .description = "Network Address Translation",
 173 };
 174 /* *INDENT-ON* */
 175
 176 vlib_node_registration_t nat44_classify_node;
 177 vlib_node_registration_t nat44_ed_classify_node;
 178 vlib_node_registration_t nat44_det_classify_node;
 179 vlib_node_registration_t nat44_handoff_classify_node;
 180
 181 typedef enum {
 182   NAT44_CLASSIFY_NEXT_IN2OUT,
 183   NAT44_CLASSIFY_NEXT_OUT2IN,
 184   NAT44_CLASSIFY_N_NEXT,
 185 } nat44_classify_next_t;
 186
 187 void
 188 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 189 {
 190   snat_session_key_t key;
 191   clib_bihash_kv_8_8_t kv;
 192   nat_ed_ses_key_t ed_key;
 193   clib_bihash_kv_16_8_t ed_kv;
 194   snat_main_per_thread_data_t *tsm =
 195     vec_elt_at_index (sm->per_thread_data, thread_index);
 196
 197   if (is_fwd_bypass_session (s))
 198     {
 199       ed_key.l_addr = s->in2out.addr;
 200       ed_key.r_addr = s->ext_host_addr;
 201       ed_key.l_port = s->in2out.port;
 202       ed_key.r_port = s->ext_host_port;
 203       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 204       ed_key.fib_index = 0;
 205       ed_kv.key[0] = ed_key.as_u64[0];
 206       ed_kv.key[1] = ed_key.as_u64[1];
 207       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 208         nat_log_warn ("in2out_ed key del failed");
 209       return;
 210     }
 211
 212   /* session lookup tables */
 213   if (is_ed_session (s))
 214     {
 215       if (is_affinity_sessions (s))
 216         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 217                              s->in2out.protocol, s->out2in.port);
 218       ed_key.l_addr = s->out2in.addr;
 219       ed_key.r_addr = s->ext_host_addr;
 220       ed_key.fib_index = s->out2in.fib_index;
 221       if (snat_is_unk_proto_session (s))
 222         {
 223           ed_key.proto = s->in2out.port;
 224           ed_key.r_port = 0;
 225           ed_key.l_port = 0;
 226         }
 227       else
 228         {
 229           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 230           ed_key.l_port = s->out2in.port;
 231           ed_key.r_port = s->ext_host_port;
 232         }
 233       ed_kv.key[0] = ed_key.as_u64[0];
 234       ed_kv.key[1] = ed_key.as_u64[1];
 235       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 236         nat_log_warn ("out2in_ed key del failed");
 237       ed_key.l_addr = s->in2out.addr;
 238       ed_key.fib_index = s->in2out.fib_index;
 239       if (!snat_is_unk_proto_session (s))
 240         ed_key.l_port = s->in2out.port;
 241       if (is_twice_nat_session (s))
 242         {
 243           ed_key.r_addr = s->ext_host_nat_addr;
 244           ed_key.r_port = s->ext_host_nat_port;
 245         }
 246       ed_kv.key[0] = ed_key.as_u64[0];
 247       ed_kv.key[1] = ed_key.as_u64[1];
 248       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 249         nat_log_warn ("in2out_ed key del failed");
 250     }
 251   else
 252     {
 253       kv.key = s->in2out.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 255         nat_log_warn ("in2out key del failed");
 256       kv.key = s->out2in.as_u64;
 257       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 258         nat_log_warn ("out2in key del failed");
 259     }
 260
 261   if (snat_is_unk_proto_session (s))
 262     return;
 263
 264   /* log NAT event */
 265   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 266                                       s->out2in.addr.as_u32,
 267                                       s->in2out.protocol,
 268                                       s->in2out.port,
 269                                       s->out2in.port,
 270                                       s->in2out.fib_index);
 271
 272   /* Twice NAT address and port for external host */
 273   if (is_twice_nat_session (s))
 274     {
 275       key.protocol = s->in2out.protocol;
 276       key.port = s->ext_host_nat_port;
 277       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 278       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 279                                           thread_index, &key);
 280     }
 281
 282   if (snat_is_session_static (s))
 283     return;
 284
 285   if (s->outside_address_index != ~0)
 286     snat_free_outside_address_and_port (sm->addresses, thread_index,
 287                                         &s->out2in);
 288 }
 289
 290 snat_user_t *
 291 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 292                         u32 thread_index)
 293 {
 294   snat_user_t *u = 0;
 295   snat_user_key_t user_key;
 296   clib_bihash_kv_8_8_t kv, value;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   dlist_elt_t * per_user_list_head_elt;
 299
 300   user_key.addr.as_u32 = addr->as_u32;
 301   user_key.fib_index = fib_index;
 302   kv.key = user_key.as_u64;
 303
 304   /* Ever heard of the "user" = src ip4 address before? */
 305   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 306     {
 307       /* no, make a new one */
 308       pool_get (tsm->users, u);
 309       memset (u, 0, sizeof (*u));
 310       u->addr.as_u32 = addr->as_u32;
 311       u->fib_index = fib_index;
 312
 313       pool_get (tsm->list_pool, per_user_list_head_elt);
 314
 315       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 316         tsm->list_pool;
 317
 318       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 319
 320       kv.value = u - tsm->users;
 321
 322       /* add user */
 323       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 324         nat_log_warn ("user_hash keay add failed");
 325     }
 326   else
 327     {
 328       u = pool_elt_at_index (tsm->users, value.value);
 329     }
 330
 331   return u;
 332 }
 333
 334 snat_session_t *
 335 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 336 {
 337   snat_session_t *s;
 338   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 339   u32 oldest_per_user_translation_list_index, session_index;
 340   dlist_elt_t * oldest_per_user_translation_list_elt;
 341   dlist_elt_t * per_user_translation_list_elt;
 342
 343   /* Over quota? Recycle the least recently used translation */
 344   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 345     {
 346       oldest_per_user_translation_list_index =
 347         clib_dlist_remove_head (tsm->list_pool,
 348                                 u->sessions_per_user_list_head_index);
 349
 350       ASSERT (oldest_per_user_translation_list_index != ~0);
 351
 352       /* Add it back to the end of the LRU list */
 353       clib_dlist_addtail (tsm->list_pool,
 354                           u->sessions_per_user_list_head_index,
 355                           oldest_per_user_translation_list_index);
 356       /* Get the list element */
 357       oldest_per_user_translation_list_elt =
 358         pool_elt_at_index (tsm->list_pool,
 359                            oldest_per_user_translation_list_index);
 360
 361       /* Get the session index from the list element */
 362       session_index = oldest_per_user_translation_list_elt->value;
 363
 364       /* Get the session */
 365       s = pool_elt_at_index (tsm->sessions, session_index);
 366       nat_free_session_data (sm, s, thread_index);
 367       if (snat_is_session_static(s))
 368         u->nstaticsessions--;
 369       else
 370         u->nsessions--;
 371       s->outside_address_index = ~0;
 372       s->flags = 0;
 373       s->total_bytes = 0;
 374       s->total_pkts = 0;
 375       s->state = 0;
 376       s->ext_host_addr.as_u32 = 0;
 377       s->ext_host_port = 0;
 378       s->ext_host_nat_addr.as_u32 = 0;
 379       s->ext_host_nat_port = 0;
 380     }
 381   else
 382     {
 383       pool_get (tsm->sessions, s);
 384       memset (s, 0, sizeof (*s));
 385       s->outside_address_index = ~0;
 386
 387       /* Create list elts */
 388       pool_get (tsm->list_pool, per_user_translation_list_elt);
 389       clib_dlist_init (tsm->list_pool,
 390                        per_user_translation_list_elt - tsm->list_pool);
 391
 392       per_user_translation_list_elt->value = s - tsm->sessions;
 393       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 394       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 395
 396       clib_dlist_addtail (tsm->list_pool,
 397                           s->per_user_list_head_index,
 398                           per_user_translation_list_elt - tsm->list_pool);
 399     }
 400
 401   return s;
 402 }
 403
 404 snat_session_t *
 405 nat_ed_session_alloc (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 406 {
 407   snat_session_t *s;
 408   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 409   dlist_elt_t * per_user_translation_list_elt;
 410
 411   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 412     {
 413       nat_log_warn ("max translations per user %U", format_ip4_address, &u->addr);
 414       snat_ipfix_logging_max_entries_per_user (sm->max_translations_per_user,
 415                                                u->addr.as_u32);
 416       return 0;
 417     }
 418
 419   pool_get (tsm->sessions, s);
 420   memset (s, 0, sizeof (*s));
 421   s->outside_address_index = ~0;
 422
 423   /* Create list elts */
 424   pool_get (tsm->list_pool, per_user_translation_list_elt);
 425   clib_dlist_init (tsm->list_pool,
 426                    per_user_translation_list_elt - tsm->list_pool);
 427
 428   per_user_translation_list_elt->value = s - tsm->sessions;
 429   s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 430   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 431
 432   clib_dlist_addtail (tsm->list_pool,
 433                       s->per_user_list_head_index,
 434                       per_user_translation_list_elt - tsm->list_pool);
 435
 436   return s;
 437 }
 438
 439 typedef struct {
 440   u8 next_in2out;
 441 } nat44_classify_trace_t;
 442
 443 static u8 * format_nat44_classify_trace (u8 * s, va_list * args)
 444 {
 445   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 446   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 447   nat44_classify_trace_t *t = va_arg (*args, nat44_classify_trace_t *);
 448   char *next;
 449
 450   next = t->next_in2out ? "nat44-in2out" : "nat44-out2in";
 451
 452   s = format (s, "nat44-classify: next %s", next);
 453
 454   return s;
 455 }
 456
 457 static inline uword
 458 nat44_classify_node_fn_inline (vlib_main_t * vm,
 459                                vlib_node_runtime_t * node,
 460                                vlib_frame_t * frame,
 461                                int is_ed)
 462 {
 463   u32 n_left_from, * from, * to_next;
 464   nat44_classify_next_t next_index;
 465   snat_main_t *sm = &snat_main;
 466   snat_static_mapping_t *m;
 467   u32 thread_index = vm->thread_index;
 468   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 469
 470   from = vlib_frame_vector_args (frame);
 471   n_left_from = frame->n_vectors;
 472   next_index = node->cached_next_index;
 473
 474   while (n_left_from > 0)
 475     {
 476       u32 n_left_to_next;
 477
 478       vlib_get_next_frame (vm, node, next_index,
 479                            to_next, n_left_to_next);
 480
 481       while (n_left_from > 0 && n_left_to_next > 0)
 482         {
 483           u32 bi0;
 484           vlib_buffer_t *b0;
 485           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0;
 486           ip4_header_t *ip0;
 487           snat_address_t *ap;
 488           snat_session_key_t m_key0;
 489           clib_bihash_kv_8_8_t kv0, value0;
 490           clib_bihash_kv_16_8_t ed_kv0, ed_value0;
 491           udp_header_t *udp0;
 492
 493           /* speculatively enqueue b0 to the current next frame */
 494           bi0 = from[0];
 495           to_next[0] = bi0;
 496           from += 1;
 497           to_next += 1;
 498           n_left_from -= 1;
 499           n_left_to_next -= 1;
 500
 501           b0 = vlib_get_buffer (vm, bi0);
 502           ip0 = vlib_buffer_get_current (b0);
 503           udp0 = ip4_next_header (ip0);
 504
 505           if (is_ed)
 506             {
 507               sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 508               rx_fib_index0 =
 509                 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 510                                                      sw_if_index0);
 511               make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address,
 512                           ip0->protocol, rx_fib_index0, udp0->src_port,
 513                           udp0->dst_port);
 514               if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, &ed_value0))
 515                 goto enqueue0;
 516             }
 517
 518           vec_foreach (ap, sm->addresses)
 519             {
 520               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 521                 {
 522                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 523                   goto enqueue0;
 524                 }
 525             }
 526
 527           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 528             {
 529               m_key0.addr = ip0->dst_address;
 530               m_key0.port = 0;
 531               m_key0.protocol = 0;
 532               m_key0.fib_index = 0;
 533               kv0.key = m_key0.as_u64;
 534               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 535                 {
 536                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 537                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 538                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 539                   goto enqueue0;
 540                 }
 541               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 542               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 543               kv0.key = m_key0.as_u64;
 544               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 545                 {
 546                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 547                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 548                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 549                 }
 550             }
 551
 552         enqueue0:
 553           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 554                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 555             {
 556               nat44_classify_trace_t *t =
 557                   vlib_add_trace (vm, node, b0, sizeof (*t));
 558               t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
 559             }
 560
 561           /* verify speculative enqueue, maybe switch current next frame */
 562           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 563                                            to_next, n_left_to_next,
 564                                            bi0, next0);
 565         }
 566
 567       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 568     }
 569
 570   return frame->n_vectors;
 571 }
 572
 573 static uword
 574 nat44_classify_node_fn (vlib_main_t * vm,
 575                         vlib_node_runtime_t * node,
 576                         vlib_frame_t * frame)
 577 {
 578   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 579 };
 580
 581 VLIB_REGISTER_NODE (nat44_classify_node) = {
 582   .function = nat44_classify_node_fn,
 583   .name = "nat44-classify",
 584   .vector_size = sizeof (u32),
 585   .format_trace = format_nat44_classify_trace,
 586   .type = VLIB_NODE_TYPE_INTERNAL,
 587   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 588   .next_nodes = {
 589     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 590     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 591   },
 592 };
 593
 594 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 595                               nat44_classify_node_fn);
 596 static uword
 597 nat44_ed_classify_node_fn (vlib_main_t * vm,
 598                            vlib_node_runtime_t * node,
 599                            vlib_frame_t * frame)
 600 {
 601   return nat44_classify_node_fn_inline (vm, node, frame, 1);
 602 };
 603
 604 VLIB_REGISTER_NODE (nat44_ed_classify_node) = {
 605   .function = nat44_ed_classify_node_fn,
 606   .name = "nat44-ed-classify",
 607   .vector_size = sizeof (u32),
 608   .format_trace = format_nat44_classify_trace,
 609   .type = VLIB_NODE_TYPE_INTERNAL,
 610   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 611   .next_nodes = {
 612     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out",
 613     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in",
 614   },
 615 };
 616
 617 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_classify_node,
 618                               nat44_ed_classify_node_fn);
 619
 620 static uword
 621 nat44_det_classify_node_fn (vlib_main_t * vm,
 622                             vlib_node_runtime_t * node,
 623                             vlib_frame_t * frame)
 624 {
 625   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 626 };
 627
 628 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 629   .function = nat44_det_classify_node_fn,
 630   .name = "nat44-det-classify",
 631   .vector_size = sizeof (u32),
 632   .format_trace = format_nat44_classify_trace,
 633   .type = VLIB_NODE_TYPE_INTERNAL,
 634   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 635   .next_nodes = {
 636     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 637     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 638   },
 639 };
 640
 641 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 642                               nat44_det_classify_node_fn);
 643
 644 static uword
 645 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 646                                 vlib_node_runtime_t * node,
 647                                 vlib_frame_t * frame)
 648 {
 649   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 650 };
 651
 652 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 653   .function = nat44_handoff_classify_node_fn,
 654   .name = "nat44-handoff-classify",
 655   .vector_size = sizeof (u32),
 656   .format_trace = format_nat44_classify_trace,
 657   .type = VLIB_NODE_TYPE_INTERNAL,
 658   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 659   .next_nodes = {
 660     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 661     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 662   },
 663 };
 664
 665 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 666                               nat44_handoff_classify_node_fn);
 667
 668 /**
 669  * @brief Add/del NAT address to FIB.
 670  *
 671  * Add the external NAT address to the FIB as receive entries. This ensures
 672  * that VPP will reply to ARP for this address and we don't need to enable
 673  * proxy ARP on the outside interface.
 674  *
 675  * @param addr IPv4 address.
 676  * @param plen address prefix length
 677  * @param sw_if_index Interface.
 678  * @param is_add If 0 delete, otherwise add.
 679  */
 680 void
 681 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 682                           int is_add)
 683 {
 684   fib_prefix_t prefix = {
 685     .fp_len = p_len,
 686     .fp_proto = FIB_PROTOCOL_IP4,
 687     .fp_addr = {
 688         .ip4.as_u32 = addr->as_u32,
 689     },
 690   };
 691   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 692
 693   if (is_add)
 694     fib_table_entry_update_one_path(fib_index,
 695                                     &prefix,
 696                                     FIB_SOURCE_PLUGIN_LOW,
 697                                     (FIB_ENTRY_FLAG_CONNECTED |
 698                                      FIB_ENTRY_FLAG_LOCAL |
 699                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 700                                     DPO_PROTO_IP4,
 701                                     NULL,
 702                                     sw_if_index,
 703                                     ~0,
 704                                     1,
 705                                     NULL,
 706                                     FIB_ROUTE_PATH_FLAG_NONE);
 707   else
 708     fib_table_entry_delete(fib_index,
 709                            &prefix,
 710                            FIB_SOURCE_PLUGIN_LOW);
 711 }
 712
 713 int snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 714                        u8 twice_nat)
 715 {
 716   snat_address_t * ap;
 717   snat_interface_t *i;
 718   vlib_thread_main_t *tm = vlib_get_thread_main ();
 719
 720   if (twice_nat && !sm->endpoint_dependent)
 721     return VNET_API_ERROR_FEATURE_DISABLED;
 722
 723   /* Check if address already exists */
 724   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 725     {
 726       if (ap->addr.as_u32 == addr->as_u32)
 727         return VNET_API_ERROR_VALUE_EXIST;
 728     }
 729
 730   if (twice_nat)
 731     vec_add2 (sm->twice_nat_addresses, ap, 1);
 732   else
 733     vec_add2 (sm->addresses, ap, 1);
 734
 735   ap->addr = *addr;
 736   if (vrf_id != ~0)
 737     ap->fib_index =
 738       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 739                                          FIB_SOURCE_PLUGIN_LOW);
 740   else
 741     ap->fib_index = ~0;
 742 #define _(N, i, n, s) \
 743   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 744   ap->busy_##n##_ports = 0; \
 745   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 746   foreach_snat_protocol
 747 #undef _
 748
 749   if (twice_nat)
 750     return 0;
 751
 752   /* Add external address to FIB */
 753   pool_foreach (i, sm->interfaces,
 754   ({
 755     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 756       continue;
 757
 758     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 759     break;
 760   }));
 761   pool_foreach (i, sm->output_feature_interfaces,
 762   ({
 763     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 764       continue;
 765
 766     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 767     break;
 768   }));
 769
 770   return 0;
 771 }
 772
 773 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 774                                                    ip4_address_t addr)
 775 {
 776   snat_static_mapping_t *m;
 777   pool_foreach (m, sm->static_mappings,
 778   ({
 779       if (m->external_addr.as_u32 == addr.as_u32)
 780         return 1;
 781   }));
 782
 783   return 0;
 784 }
 785
 786 void increment_v4_address (ip4_address_t * a)
 787 {
 788   u32 v;
 789
 790   v = clib_net_to_host_u32(a->as_u32) + 1;
 791   a->as_u32 = clib_host_to_net_u32(v);
 792 }
 793
 794 static void
 795 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 796                                        ip4_address_t l_addr,
 797                                        u16 l_port,
 798                                        u32 sw_if_index,
 799                                        u16 e_port,
 800                                        u32 vrf_id,
 801                                        snat_protocol_t proto,
 802                                        int addr_only,
 803                                        int is_add,
 804                                        u8 * tag)
 805 {
 806   snat_static_map_resolve_t *rp;
 807
 808   vec_add2 (sm->to_resolve, rp, 1);
 809   rp->l_addr.as_u32 = l_addr.as_u32;
 810   rp->l_port = l_port;
 811   rp->sw_if_index = sw_if_index;
 812   rp->e_port = e_port;
 813   rp->vrf_id = vrf_id;
 814   rp->proto = proto;
 815   rp->addr_only = addr_only;
 816   rp->is_add = is_add;
 817   rp->tag = vec_dup (tag);
 818 }
 819
 820 /**
 821  * @brief Add static mapping.
 822  *
 823  * Create static mapping between local addr+port and external addr+port.
 824  *
 825  * @param l_addr Local IPv4 address.
 826  * @param e_addr External IPv4 address.
 827  * @param l_port Local port number.
 828  * @param e_port External port number.
 829  * @param vrf_id VRF ID.
 830  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 831  * @param sw_if_index External port instead of specific IP address.
 832  * @param is_add If 0 delete static mapping, otherwise add.
 833  * @param twice_nat If value is TWICE_NAT then translate external host address
 834  *                  and port.
 835  *                  If value is TWICE_NAT_SELF then translate external host
 836  *                  address and port whenever external host address equals
 837  *                  local address of internal host.
 838  * @param out2in_only If 1 rule match only out2in direction
 839  * @param tag - opaque string tag
 840  *
 841  * @returns
 842  */
 843 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 844                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 845                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 846                             twice_nat_type_t twice_nat, u8 out2in_only,
 847                             u8 * tag)
 848 {
 849   snat_main_t * sm = &snat_main;
 850   snat_static_mapping_t *m;
 851   snat_session_key_t m_key;
 852   clib_bihash_kv_8_8_t kv, value;
 853   snat_address_t *a = 0;
 854   u32 fib_index = ~0;
 855   uword * p;
 856   snat_interface_t *interface;
 857   int i;
 858   snat_main_per_thread_data_t *tsm;
 859   snat_user_key_t u_key;
 860   snat_user_t *u;
 861   dlist_elt_t * head, * elt;
 862   u32 elt_index, head_index;
 863   u32 ses_index;
 864   u64 user_index;
 865   snat_session_t * s;
 866   snat_static_map_resolve_t *rp, *rp_match = 0;
 867
 868   if (!sm->endpoint_dependent)
 869     {
 870       if (twice_nat || out2in_only)
 871         return VNET_API_ERROR_FEATURE_DISABLED;
 872     }
 873
 874   /* If the external address is a specific interface address */
 875   if (sw_if_index != ~0)
 876     {
 877       ip4_address_t * first_int_addr;
 878
 879       for (i = 0; i < vec_len (sm->to_resolve); i++)
 880         {
 881           rp = sm->to_resolve + i;
 882           if (rp->sw_if_index != sw_if_index ||
 883               rp->l_addr.as_u32 != l_addr.as_u32 ||
 884               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 885             continue;
 886
 887           if (!addr_only)
 888             {
 889               if (rp->l_port != l_port || rp->e_port != e_port || rp->proto != proto)
 890                 continue;
 891             }
 892
 893           rp_match = rp;
 894           break;
 895         }
 896
 897       /* Might be already set... */
 898       first_int_addr = ip4_interface_first_address
 899         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 900
 901       if (is_add)
 902         {
 903           if (rp_match)
 904             return VNET_API_ERROR_VALUE_EXIST;
 905
 906           snat_add_static_mapping_when_resolved
 907             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 908              addr_only,  is_add, tag);
 909
 910           /* DHCP resolution required? */
 911           if (first_int_addr == 0)
 912             {
 913               return 0;
 914             }
 915           else
 916             {
 917               e_addr.as_u32 = first_int_addr->as_u32;
 918               /* Identity mapping? */
 919               if (l_addr.as_u32 == 0)
 920                 l_addr.as_u32 = e_addr.as_u32;
 921             }
 922         }
 923       else
 924         {
 925           if (!rp_match)
 926             return VNET_API_ERROR_NO_SUCH_ENTRY;
 927
 928           vec_del1 (sm->to_resolve, i);
 929
 930           if (first_int_addr)
 931             {
 932               e_addr.as_u32 = first_int_addr->as_u32;
 933               /* Identity mapping? */
 934               if (l_addr.as_u32 == 0)
 935                 l_addr.as_u32 = e_addr.as_u32;
 936             }
 937           else
 938             return 0;
 939         }
 940     }
 941
 942   m_key.addr = e_addr;
 943   m_key.port = addr_only ? 0 : e_port;
 944   m_key.protocol = addr_only ? 0 : proto;
 945   m_key.fib_index = 0;
 946   kv.key = m_key.as_u64;
 947   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 948     m = 0;
 949   else
 950     m = pool_elt_at_index (sm->static_mappings, value.value);
 951
 952   if (is_add)
 953     {
 954       if (m)
 955         return VNET_API_ERROR_VALUE_EXIST;
 956
 957       if (twice_nat && addr_only)
 958         return VNET_API_ERROR_UNSUPPORTED;
 959
 960       /* Convert VRF id to FIB index */
 961       if (vrf_id != ~0)
 962         {
 963           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 964           if (!p)
 965             return VNET_API_ERROR_NO_SUCH_FIB;
 966           fib_index = p[0];
 967         }
 968       /* If not specified use inside VRF id from SNAT plugin startup config */
 969       else
 970         {
 971           fib_index = sm->inside_fib_index;
 972           vrf_id = sm->inside_vrf_id;
 973         }
 974
 975       if (!out2in_only)
 976         {
 977           m_key.addr = l_addr;
 978           m_key.port = addr_only ? 0 : l_port;
 979           m_key.protocol = addr_only ? 0 : proto;
 980           m_key.fib_index = fib_index;
 981           kv.key = m_key.as_u64;
 982           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 983             return VNET_API_ERROR_VALUE_EXIST;
 984         }
 985
 986       /* Find external address in allocated addresses and reserve port for
 987          address and port pair mapping when dynamic translations enabled */
 988       if (!(addr_only || sm->static_mapping_only || out2in_only))
 989         {
 990           for (i = 0; i < vec_len (sm->addresses); i++)
 991             {
 992               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 993                 {
 994                   a = sm->addresses + i;
 995                   /* External port must be unused */
 996                   switch (proto)
 997                     {
 998 #define _(N, j, n, s) \
 999                     case SNAT_PROTOCOL_##N: \
1000                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1001                         return VNET_API_ERROR_INVALID_VALUE; \
1002                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1003                       if (e_port > 1024) \
1004                         { \
1005                           a->busy_##n##_ports++; \
1006                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1007                         } \
1008                       break;
1009                       foreach_snat_protocol
1010 #undef _
1011                     default:
1012                       nat_log_info ("unknown protocol");
1013                       return VNET_API_ERROR_INVALID_VALUE_2;
1014                     }
1015                   break;
1016                 }
1017             }
1018           /* External address must be allocated */
1019           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1020             {
1021               if (sw_if_index != ~0)
1022                 {
1023                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1024                     {
1025                       rp = sm->to_resolve + i;
1026                       if (rp->addr_only)
1027                          continue;
1028                       if (rp->sw_if_index != sw_if_index &&
1029                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1030                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1031                           rp->e_port != e_port && rp->proto != proto)
1032                         continue;
1033
1034                       vec_del1 (sm->to_resolve, i);
1035                       break;
1036                     }
1037                 }
1038               return VNET_API_ERROR_NO_SUCH_ENTRY;
1039             }
1040         }
1041
1042       pool_get (sm->static_mappings, m);
1043       memset (m, 0, sizeof (*m));
1044       m->tag = vec_dup (tag);
1045       m->local_addr = l_addr;
1046       m->external_addr = e_addr;
1047       m->addr_only = addr_only;
1048       m->vrf_id = vrf_id;
1049       m->fib_index = fib_index;
1050       m->twice_nat = twice_nat;
1051       m->out2in_only = out2in_only;
1052       if (!addr_only)
1053         {
1054           m->local_port = l_port;
1055           m->external_port = e_port;
1056           m->proto = proto;
1057         }
1058
1059       if (sm->num_workers > 1)
1060         {
1061           ip4_header_t ip = {
1062             .src_address = m->local_addr,
1063           };
1064           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
1065           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1066         }
1067       else
1068         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1069
1070       m_key.addr = m->local_addr;
1071       m_key.port = m->local_port;
1072       m_key.protocol = m->proto;
1073       m_key.fib_index = m->fib_index;
1074       kv.key = m_key.as_u64;
1075       kv.value = m - sm->static_mappings;
1076       if (!out2in_only)
1077         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1078
1079       m_key.addr = m->external_addr;
1080       m_key.port = m->external_port;
1081       m_key.fib_index = 0;
1082       kv.key = m_key.as_u64;
1083       kv.value = m - sm->static_mappings;
1084       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
1085
1086       /* Delete dynamic sessions matching local address (+ local port) */
1087       if (!(sm->static_mapping_only))
1088         {
1089           u_key.addr = m->local_addr;
1090           u_key.fib_index = m->fib_index;
1091           kv.key = u_key.as_u64;
1092           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1093             {
1094               user_index = value.value;
1095               u = pool_elt_at_index (tsm->users, user_index);
1096               if (u->nsessions)
1097                 {
1098                   head_index = u->sessions_per_user_list_head_index;
1099                   head = pool_elt_at_index (tsm->list_pool, head_index);
1100                   elt_index = head->next;
1101                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1102                   ses_index = elt->value;
1103                   while (ses_index != ~0)
1104                     {
1105                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1106                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1107                       ses_index = elt->value;
1108
1109                       if (snat_is_session_static (s))
1110                         continue;
1111
1112                       if (!addr_only && (clib_net_to_host_u16 (s->in2out.port) != m->local_port))
1113                         continue;
1114
1115                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1116                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1117
1118                       if (!addr_only && !sm->endpoint_dependent)
1119                         break;
1120                     }
1121                 }
1122             }
1123         }
1124     }
1125   else
1126     {
1127       if (!m)
1128         {
1129           if (sw_if_index != ~0)
1130             return 0;
1131           else
1132             return VNET_API_ERROR_NO_SUCH_ENTRY;
1133         }
1134
1135       /* Free external address port */
1136       if (!(addr_only || sm->static_mapping_only || out2in_only))
1137         {
1138           for (i = 0; i < vec_len (sm->addresses); i++)
1139             {
1140               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1141                 {
1142                   a = sm->addresses + i;
1143                   switch (proto)
1144                     {
1145 #define _(N, j, n, s) \
1146                     case SNAT_PROTOCOL_##N: \
1147                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1148                       if (e_port > 1024) \
1149                         { \
1150                           a->busy_##n##_ports--; \
1151                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1152                         } \
1153                       break;
1154                       foreach_snat_protocol
1155 #undef _
1156                     default:
1157                       nat_log_info ("unknown protocol");
1158                       return VNET_API_ERROR_INVALID_VALUE_2;
1159                     }
1160                   break;
1161                 }
1162             }
1163         }
1164
1165       if (sm->num_workers > 1)
1166         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1167       else
1168         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1169
1170       m_key.addr = m->local_addr;
1171       m_key.port = m->local_port;
1172       m_key.protocol = m->proto;
1173       m_key.fib_index = m->fib_index;
1174       kv.key = m_key.as_u64;
1175       if (!out2in_only)
1176         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1177
1178       m_key.addr = m->external_addr;
1179       m_key.port = m->external_port;
1180       m_key.fib_index = 0;
1181       kv.key = m_key.as_u64;
1182       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1183
1184       /* Delete session(s) for static mapping if exist */
1185       if (!(sm->static_mapping_only) ||
1186           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1187         {
1188           u_key.addr = m->local_addr;
1189           u_key.fib_index = m->fib_index;
1190           kv.key = u_key.as_u64;
1191           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1192             {
1193               user_index = value.value;
1194               u = pool_elt_at_index (tsm->users, user_index);
1195               if (u->nstaticsessions)
1196                 {
1197                   head_index = u->sessions_per_user_list_head_index;
1198                   head = pool_elt_at_index (tsm->list_pool, head_index);
1199                   elt_index = head->next;
1200                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1201                   ses_index = elt->value;
1202                   while (ses_index != ~0)
1203                     {
1204                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1205                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1206                       ses_index = elt->value;
1207
1208                       if (!addr_only)
1209                         {
1210                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1211                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1212                             continue;
1213                         }
1214
1215                       if (is_lb_session (s))
1216                         continue;
1217
1218                       if (!snat_is_session_static (s))
1219                         continue;
1220
1221                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1222                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1223
1224                       if (!addr_only && !sm->endpoint_dependent)
1225                         break;
1226                     }
1227                 }
1228             }
1229         }
1230
1231       vec_free (m->tag);
1232       vec_free (m->workers);
1233       /* Delete static mapping from pool */
1234       pool_put (sm->static_mappings, m);
1235     }
1236
1237   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1238     return 0;
1239
1240   /* Add/delete external address to FIB */
1241   pool_foreach (interface, sm->interfaces,
1242   ({
1243     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1244       continue;
1245
1246     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1247     break;
1248   }));
1249   pool_foreach (interface, sm->output_feature_interfaces,
1250   ({
1251     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1252       continue;
1253
1254     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1255     break;
1256   }));
1257
1258   return 0;
1259 }
1260
1261 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1262                                      snat_protocol_t proto,
1263                                      nat44_lb_addr_port_t *locals, u8 is_add,
1264                                      twice_nat_type_t twice_nat, u8 out2in_only,
1265                                      u8 *tag, u32 affinity)
1266 {
1267   snat_main_t * sm = &snat_main;
1268   snat_static_mapping_t *m;
1269   snat_session_key_t m_key;
1270   clib_bihash_kv_8_8_t kv, value;
1271   snat_address_t *a = 0;
1272   int i;
1273   nat44_lb_addr_port_t *local;
1274   u32 elt_index, head_index, ses_index;
1275   snat_main_per_thread_data_t *tsm;
1276   snat_user_key_t u_key;
1277   snat_user_t *u;
1278   snat_session_t * s;
1279   dlist_elt_t * head, * elt;
1280   uword *bitmap = 0;
1281
1282   if (!sm->endpoint_dependent)
1283     return VNET_API_ERROR_FEATURE_DISABLED;
1284
1285   m_key.addr = e_addr;
1286   m_key.port = e_port;
1287   m_key.protocol = proto;
1288   m_key.fib_index = 0;
1289   kv.key = m_key.as_u64;
1290   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1291     m = 0;
1292   else
1293     m = pool_elt_at_index (sm->static_mappings, value.value);
1294
1295   if (is_add)
1296     {
1297       if (m)
1298         return VNET_API_ERROR_VALUE_EXIST;
1299
1300       if (vec_len (locals) < 2)
1301         return VNET_API_ERROR_INVALID_VALUE;
1302
1303       /* Find external address in allocated addresses and reserve port for
1304          address and port pair mapping when dynamic translations enabled */
1305       if (!(sm->static_mapping_only || out2in_only))
1306         {
1307           for (i = 0; i < vec_len (sm->addresses); i++)
1308             {
1309               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1310                 {
1311                   a = sm->addresses + i;
1312                   /* External port must be unused */
1313                   switch (proto)
1314                     {
1315 #define _(N, j, n, s) \
1316                     case SNAT_PROTOCOL_##N: \
1317                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1318                         return VNET_API_ERROR_INVALID_VALUE; \
1319                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1320                       if (e_port > 1024) \
1321                         { \
1322                           a->busy_##n##_ports++; \
1323                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1324                         } \
1325                       break;
1326                       foreach_snat_protocol
1327 #undef _
1328                     default:
1329                       nat_log_info ("unknown protocol");
1330                       return VNET_API_ERROR_INVALID_VALUE_2;
1331                     }
1332                   break;
1333                 }
1334             }
1335           /* External address must be allocated */
1336           if (!a)
1337             return VNET_API_ERROR_NO_SUCH_ENTRY;
1338         }
1339
1340       pool_get (sm->static_mappings, m);
1341       memset (m, 0, sizeof (*m));
1342       m->tag = vec_dup (tag);
1343       m->external_addr = e_addr;
1344       m->addr_only = 0;
1345       m->external_port = e_port;
1346       m->proto = proto;
1347       m->twice_nat = twice_nat;
1348       m->out2in_only = out2in_only;
1349       m->affinity = affinity;
1350
1351       if (affinity)
1352         m->affinity_per_service_list_head_index =
1353           nat_affinity_get_per_service_list_head_index();
1354       else
1355         m->affinity_per_service_list_head_index = ~0;
1356
1357       m_key.addr = m->external_addr;
1358       m_key.port = m->external_port;
1359       m_key.protocol = m->proto;
1360       m_key.fib_index = 0;
1361       kv.key = m_key.as_u64;
1362       kv.value = m - sm->static_mappings;
1363       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1364         {
1365           nat_log_err ("static_mapping_by_external key add failed");
1366           return VNET_API_ERROR_UNSPECIFIED;
1367         }
1368
1369       m_key.fib_index = m->fib_index;
1370       for (i = 0; i < vec_len (locals); i++)
1371         {
1372           locals[i].fib_index = fib_table_find_or_create_and_lock (
1373             FIB_PROTOCOL_IP4, locals[i].vrf_id, FIB_SOURCE_PLUGIN_LOW);
1374           m_key.addr = locals[i].addr;
1375           m_key.fib_index = locals[i].fib_index;
1376           if (!out2in_only)
1377             {
1378               m_key.port = locals[i].port;
1379               kv.key = m_key.as_u64;
1380               kv.value = m - sm->static_mappings;
1381               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1382             }
1383           locals[i].prefix = (i == 0) ? locals[i].probability :\
1384             (locals[i - 1].prefix + locals[i].probability);
1385           vec_add1 (m->locals, locals[i]);
1386           if (sm->num_workers > 1)
1387             {
1388               ip4_header_t ip = {
1389                 .src_address = locals[i].addr,
1390               };
1391               bitmap = clib_bitmap_set (
1392                 bitmap, sm->worker_in2out_cb (&ip, m->fib_index), 1);
1393             }
1394         }
1395
1396       /* Assign workers */
1397       if (sm->num_workers > 1)
1398         {
1399           clib_bitmap_foreach (i, bitmap,
1400             ({
1401                vec_add1(m->workers, i);
1402             }));
1403         }
1404     }
1405   else
1406     {
1407       if (!m)
1408         return VNET_API_ERROR_NO_SUCH_ENTRY;
1409
1410       /* Free external address port */
1411       if (!(sm->static_mapping_only || out2in_only))
1412         {
1413           for (i = 0; i < vec_len (sm->addresses); i++)
1414             {
1415               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1416                 {
1417                   a = sm->addresses + i;
1418                   switch (proto)
1419                     {
1420 #define _(N, j, n, s) \
1421                     case SNAT_PROTOCOL_##N: \
1422                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1423                       if (e_port > 1024) \
1424                         { \
1425                           a->busy_##n##_ports--; \
1426                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1427                         } \
1428                       break;
1429                       foreach_snat_protocol
1430 #undef _
1431                     default:
1432                       nat_log_info ("unknown protocol");
1433                       return VNET_API_ERROR_INVALID_VALUE_2;
1434                     }
1435                   break;
1436                 }
1437             }
1438         }
1439
1440       m_key.addr = m->external_addr;
1441       m_key.port = m->external_port;
1442       m_key.protocol = m->proto;
1443       m_key.fib_index = 0;
1444       kv.key = m_key.as_u64;
1445       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1446         {
1447           nat_log_err ("static_mapping_by_external key del failed");
1448           return VNET_API_ERROR_UNSPECIFIED;
1449         }
1450
1451       vec_foreach (local, m->locals)
1452         {
1453           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1454                             FIB_SOURCE_PLUGIN_LOW);
1455           m_key.addr = local->addr;
1456           if (!out2in_only)
1457             {
1458               m_key.port = local->port;
1459               m_key.fib_index = local->fib_index;
1460               kv.key = m_key.as_u64;
1461               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1462                 {
1463                   nat_log_err ("static_mapping_by_local key del failed");
1464                   return VNET_API_ERROR_UNSPECIFIED;
1465                 }
1466             }
1467
1468           if (sm->num_workers > 1)
1469             {
1470               ip4_header_t ip = {
1471                 .src_address = local->addr,
1472               };
1473               tsm = vec_elt_at_index (sm->per_thread_data,
1474                                       sm->worker_in2out_cb (&ip, m->fib_index));
1475             }
1476           else
1477             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1478
1479           /* Delete sessions */
1480           u_key.addr = local->addr;
1481           u_key.fib_index = m->fib_index;
1482           kv.key = u_key.as_u64;
1483           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1484             {
1485               u = pool_elt_at_index (tsm->users, value.value);
1486               if (u->nstaticsessions)
1487                 {
1488                   head_index = u->sessions_per_user_list_head_index;
1489                   head = pool_elt_at_index (tsm->list_pool, head_index);
1490                   elt_index = head->next;
1491                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1492                   ses_index = elt->value;
1493                   while (ses_index != ~0)
1494                     {
1495                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1496                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1497                       ses_index = elt->value;
1498
1499                       if (!(is_lb_session (s)))
1500                         continue;
1501
1502                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1503                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1504                         continue;
1505
1506                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1507                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1508                     }
1509                 }
1510             }
1511         }
1512       if (m->affinity)
1513         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1514       vec_free(m->locals);
1515       vec_free(m->tag);
1516       vec_free(m->workers);
1517
1518       pool_put (sm->static_mappings, m);
1519     }
1520
1521   return 0;
1522 }
1523
1524 int
1525 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1526                   u8 twice_nat)
1527 {
1528   snat_address_t *a = 0;
1529   snat_session_t *ses;
1530   u32 *ses_to_be_removed = 0, *ses_index;
1531   snat_main_per_thread_data_t *tsm;
1532   snat_static_mapping_t *m;
1533   snat_interface_t *interface;
1534   int i;
1535   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1536
1537   /* Find SNAT address */
1538   for (i=0; i < vec_len (addresses); i++)
1539     {
1540       if (addresses[i].addr.as_u32 == addr.as_u32)
1541         {
1542           a = addresses + i;
1543           break;
1544         }
1545     }
1546   if (!a)
1547     return VNET_API_ERROR_NO_SUCH_ENTRY;
1548
1549   if (delete_sm)
1550     {
1551       pool_foreach (m, sm->static_mappings,
1552       ({
1553           if (m->external_addr.as_u32 == addr.as_u32)
1554             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1555                                             m->local_port, m->external_port,
1556                                             m->vrf_id, m->addr_only, ~0,
1557                                             m->proto, 0, m->twice_nat,
1558                                             m->out2in_only, m->tag);
1559       }));
1560     }
1561   else
1562     {
1563       /* Check if address is used in some static mapping */
1564       if (is_snat_address_used_in_static_mapping(sm, addr))
1565         {
1566           nat_log_notice ("address used in static mapping");
1567           return VNET_API_ERROR_UNSPECIFIED;
1568         }
1569     }
1570
1571   if (a->fib_index != ~0)
1572     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1573                      FIB_SOURCE_PLUGIN_LOW);
1574
1575   /* Delete sessions using address */
1576   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1577     {
1578       vec_foreach (tsm, sm->per_thread_data)
1579         {
1580           pool_foreach (ses, tsm->sessions, ({
1581             if (ses->out2in.addr.as_u32 == addr.as_u32)
1582               {
1583                 ses->outside_address_index = ~0;
1584                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1585                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1586               }
1587           }));
1588
1589           vec_foreach (ses_index, ses_to_be_removed)
1590             {
1591               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1592               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1593             }
1594
1595           vec_free (ses_to_be_removed);
1596        }
1597     }
1598
1599   if (twice_nat)
1600     {
1601       vec_del1 (sm->twice_nat_addresses, i);
1602       return 0;
1603     }
1604   else
1605     vec_del1 (sm->addresses, i);
1606
1607   /* Delete external address from FIB */
1608   pool_foreach (interface, sm->interfaces,
1609   ({
1610     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1611       continue;
1612
1613     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1614     break;
1615   }));
1616   pool_foreach (interface, sm->output_feature_interfaces,
1617   ({
1618     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1619       continue;
1620
1621     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1622     break;
1623   }));
1624
1625   return 0;
1626 }
1627
1628 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1629 {
1630   snat_main_t *sm = &snat_main;
1631   snat_interface_t *i;
1632   const char * feature_name, *del_feature_name;
1633   snat_address_t * ap;
1634   snat_static_mapping_t * m;
1635   snat_det_map_t * dm;
1636   nat_outside_fib_t *outside_fib;
1637   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1638                                                        sw_if_index);
1639
1640   if (sm->out2in_dpo && !is_inside)
1641     return VNET_API_ERROR_UNSUPPORTED;
1642
1643   pool_foreach (i, sm->output_feature_interfaces,
1644   ({
1645     if (i->sw_if_index == sw_if_index)
1646       return VNET_API_ERROR_VALUE_EXIST;
1647   }));
1648
1649   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1650     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1651   else
1652     {
1653       if (sm->num_workers > 1 && !sm->deterministic)
1654         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1655       else if (sm->deterministic)
1656         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1657       else if (sm->endpoint_dependent)
1658         feature_name = is_inside ?  "nat44-ed-in2out" : "nat44-ed-out2in";
1659       else
1660         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1661     }
1662
1663   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1664     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1665                                                       NAT_FQ_NELTS);
1666
1667   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1668     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1669                                                       NAT_FQ_NELTS);
1670
1671   if (!is_inside)
1672     {
1673       vec_foreach (outside_fib, sm->outside_fibs)
1674         {
1675           if (outside_fib->fib_index == fib_index)
1676             {
1677               if (is_del)
1678                 {
1679                   outside_fib->refcount--;
1680                   if (!outside_fib->refcount)
1681                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1682                 }
1683               else
1684                 outside_fib->refcount++;
1685               goto feature_set;
1686             }
1687         }
1688       if (!is_del)
1689         {
1690           vec_add2 (sm->outside_fibs, outside_fib, 1);
1691           outside_fib->refcount = 1;
1692           outside_fib->fib_index = fib_index;
1693         }
1694     }
1695 feature_set:
1696   pool_foreach (i, sm->interfaces,
1697   ({
1698     if (i->sw_if_index == sw_if_index)
1699       {
1700         if (is_del)
1701           {
1702             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1703               {
1704                 if (is_inside)
1705                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1706                 else
1707                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1708
1709                 if (sm->num_workers > 1 && !sm->deterministic)
1710                   {
1711                     del_feature_name = "nat44-handoff-classify";
1712                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1713                                                  "nat44-out2in-worker-handoff";
1714                   }
1715                 else if (sm->deterministic)
1716                   {
1717                     del_feature_name = "nat44-det-classify";
1718                     feature_name = !is_inside ?  "nat44-det-in2out" :
1719                                                  "nat44-det-out2in";
1720                   }
1721                 else if (sm->endpoint_dependent)
1722                   {
1723                     del_feature_name = "nat44-ed-classify";
1724                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1725                                                  "nat44-ed-out2in";
1726                   }
1727                 else
1728                   {
1729                     del_feature_name = "nat44-classify";
1730                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1731                   }
1732
1733                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1734                                              sw_if_index, 0, 0, 0);
1735                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1736                                              sw_if_index, 1, 0, 0);
1737                 if (!is_inside)
1738                   {
1739                     if (sm->endpoint_dependent)
1740                       vnet_feature_enable_disable ("ip4-local",
1741                                                    "nat44-ed-hairpinning",
1742                                                    sw_if_index, 1, 0, 0);
1743                     else if (!sm->deterministic)
1744                       vnet_feature_enable_disable ("ip4-local",
1745                                                    "nat44-hairpinning",
1746                                                    sw_if_index, 1, 0, 0);
1747                   }
1748               }
1749             else
1750               {
1751                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1752                                              sw_if_index, 0, 0, 0);
1753                 pool_put (sm->interfaces, i);
1754                 if (is_inside)
1755                   {
1756                     if (sm->endpoint_dependent)
1757                       vnet_feature_enable_disable ("ip4-local",
1758                                                    "nat44-ed-hairpinning",
1759                                                    sw_if_index, 0, 0, 0);
1760                     else if (!sm->deterministic)
1761                       vnet_feature_enable_disable ("ip4-local",
1762                                                    "nat44-hairpinning",
1763                                                    sw_if_index, 0, 0, 0);
1764                   }
1765               }
1766           }
1767         else
1768           {
1769             if ((nat_interface_is_inside(i) && is_inside) ||
1770                 (nat_interface_is_outside(i) && !is_inside))
1771               return 0;
1772
1773             if (sm->num_workers > 1 && !sm->deterministic)
1774               {
1775                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1776                                                  "nat44-out2in-worker-handoff";
1777                 feature_name = "nat44-handoff-classify";
1778               }
1779             else if (sm->deterministic)
1780               {
1781                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1782                                                  "nat44-det-out2in";
1783                 feature_name = "nat44-det-classify";
1784               }
1785             else if (sm->endpoint_dependent)
1786               {
1787                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1788                                                  "nat44-ed-out2in";
1789                 feature_name = "nat44-ed-classify";
1790               }
1791             else
1792               {
1793                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1794                 feature_name = "nat44-classify";
1795               }
1796
1797             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1798                                          sw_if_index, 0, 0, 0);
1799             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1800                                          sw_if_index, 1, 0, 0);
1801             if (!is_inside)
1802               {
1803                 if (sm->endpoint_dependent)
1804                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1805                                                sw_if_index, 0, 0, 0);
1806                 else if (!sm->deterministic)
1807                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1808                                                sw_if_index, 0, 0, 0);
1809               }
1810             goto set_flags;
1811           }
1812
1813         goto fib;
1814       }
1815   }));
1816
1817   if (is_del)
1818     return VNET_API_ERROR_NO_SUCH_ENTRY;
1819
1820   pool_get (sm->interfaces, i);
1821   i->sw_if_index = sw_if_index;
1822   i->flags = 0;
1823   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1824
1825   if (is_inside && !sm->out2in_dpo)
1826     {
1827       if (sm->endpoint_dependent)
1828         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1829                                      sw_if_index, 1, 0, 0);
1830       else if (!sm->deterministic)
1831         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1832                                      sw_if_index, 1, 0, 0);
1833     }
1834
1835 set_flags:
1836   if (is_inside)
1837     {
1838       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1839       return 0;
1840     }
1841   else
1842     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1843
1844   /* Add/delete external addresses to FIB */
1845 fib:
1846   vec_foreach (ap, sm->addresses)
1847     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1848
1849   pool_foreach (m, sm->static_mappings,
1850   ({
1851     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1852       continue;
1853
1854     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1855   }));
1856
1857   pool_foreach (dm, sm->det_maps,
1858   ({
1859     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1860   }));
1861
1862   return 0;
1863 }
1864
1865 int snat_interface_add_del_output_feature (u32 sw_if_index,
1866                                            u8 is_inside,
1867                                            int is_del)
1868 {
1869   snat_main_t *sm = &snat_main;
1870   snat_interface_t *i;
1871   snat_address_t * ap;
1872   snat_static_mapping_t * m;
1873
1874   if (sm->deterministic ||
1875       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1876     return VNET_API_ERROR_UNSUPPORTED;
1877
1878   pool_foreach (i, sm->interfaces,
1879   ({
1880     if (i->sw_if_index == sw_if_index)
1881       return VNET_API_ERROR_VALUE_EXIST;
1882   }));
1883
1884   if (is_inside)
1885     {
1886       if (sm->endpoint_dependent)
1887         {
1888           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1889                                        sw_if_index, !is_del, 0, 0);
1890           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1891                                        sw_if_index, !is_del, 0, 0);
1892         }
1893       else
1894         {
1895           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1896                                        sw_if_index, !is_del, 0, 0);
1897           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1898                                        sw_if_index, !is_del, 0, 0);
1899         }
1900       goto fq;
1901     }
1902
1903   if (sm->num_workers > 1)
1904     {
1905       vnet_feature_enable_disable ("ip4-unicast",
1906                                    "nat44-out2in-worker-handoff",
1907                                    sw_if_index, !is_del, 0, 0);
1908       vnet_feature_enable_disable ("ip4-output",
1909                                    "nat44-in2out-output-worker-handoff",
1910                                    sw_if_index, !is_del, 0, 0);
1911     }
1912   else
1913     {
1914       if (sm->endpoint_dependent)
1915         {
1916           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
1917                                        sw_if_index, !is_del, 0, 0);
1918           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
1919                                        sw_if_index, !is_del, 0, 0);
1920         }
1921       else
1922         {
1923           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
1924                                        sw_if_index, !is_del, 0, 0);
1925           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1926                                        sw_if_index, !is_del, 0, 0);
1927         }
1928     }
1929
1930 fq:
1931   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1932     sm->fq_in2out_output_index =
1933       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1934
1935   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1936     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1937
1938   pool_foreach (i, sm->output_feature_interfaces,
1939   ({
1940     if (i->sw_if_index == sw_if_index)
1941       {
1942         if (is_del)
1943           pool_put (sm->output_feature_interfaces, i);
1944         else
1945           return VNET_API_ERROR_VALUE_EXIST;
1946
1947         goto fib;
1948       }
1949   }));
1950
1951   if (is_del)
1952     return VNET_API_ERROR_NO_SUCH_ENTRY;
1953
1954   pool_get (sm->output_feature_interfaces, i);
1955   i->sw_if_index = sw_if_index;
1956   i->flags = 0;
1957   if (is_inside)
1958     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1959   else
1960     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1961
1962   /* Add/delete external addresses to FIB */
1963 fib:
1964   if (is_inside)
1965     return 0;
1966
1967   vec_foreach (ap, sm->addresses)
1968     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1969
1970   pool_foreach (m, sm->static_mappings,
1971   ({
1972     if (!(m->addr_only)  || (m->local_addr.as_u32 == m->external_addr.as_u32))
1973       continue;
1974
1975     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1976   }));
1977
1978   return 0;
1979 }
1980
1981 int snat_set_workers (uword * bitmap)
1982 {
1983   snat_main_t *sm = &snat_main;
1984   int i, j = 0;
1985
1986   if (sm->num_workers < 2)
1987     return VNET_API_ERROR_FEATURE_DISABLED;
1988
1989   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1990     return VNET_API_ERROR_INVALID_WORKER;
1991
1992   vec_free (sm->workers);
1993   clib_bitmap_foreach (i, bitmap,
1994     ({
1995       vec_add1(sm->workers, i);
1996       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1997       j++;
1998     }));
1999
2000   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2001   sm->num_snat_thread = _vec_len (sm->workers);
2002
2003   return 0;
2004 }
2005
2006
2007 static void
2008 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2009                                        uword opaque,
2010                                        u32 sw_if_index,
2011                                        ip4_address_t * address,
2012                                        u32 address_length,
2013                                        u32 if_address_index,
2014                                        u32 is_delete);
2015
2016 static void
2017 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2018                                  uword opaque,
2019                                  u32 sw_if_index,
2020                                  ip4_address_t * address,
2021                                  u32 address_length,
2022                                  u32 if_address_index,
2023                                  u32 is_delete);
2024
2025 static int
2026 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2027                                  u32 fib_index,
2028                                  u32 thread_index,
2029                                  snat_session_key_t * k,
2030                                  u32 * address_indexp,
2031                                  u16 port_per_thread,
2032                                  u32 snat_thread_index);
2033
2034 static clib_error_t * snat_init (vlib_main_t * vm)
2035 {
2036   snat_main_t * sm = &snat_main;
2037   clib_error_t * error = 0;
2038   ip4_main_t * im = &ip4_main;
2039   ip_lookup_main_t * lm = &im->lookup_main;
2040   uword *p;
2041   vlib_thread_registration_t *tr;
2042   vlib_thread_main_t *tm = vlib_get_thread_main ();
2043   uword *bitmap = 0;
2044   u32 i;
2045   ip4_add_del_interface_address_callback_t cb4;
2046   vlib_node_t * error_drop_node;
2047
2048   sm->vlib_main = vm;
2049   sm->vnet_main = vnet_get_main();
2050   sm->ip4_main = im;
2051   sm->ip4_lookup_main = lm;
2052   sm->api_main = &api_main;
2053   sm->first_worker_index = 0;
2054   sm->num_workers = 0;
2055   sm->num_snat_thread = 1;
2056   sm->workers = 0;
2057   sm->port_per_thread = 0xffff - 1024;
2058   sm->fq_in2out_index = ~0;
2059   sm->fq_out2in_index = ~0;
2060   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2061   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2062   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2063   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2064   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2065   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2066   sm->forwarding_enabled = 0;
2067   sm->log_class = vlib_log_register_class ("nat", 0);
2068   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2069   sm->error_node_index = error_drop_node->index;
2070
2071   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2072   if (p)
2073     {
2074       tr = (vlib_thread_registration_t *) p[0];
2075       if (tr)
2076         {
2077           sm->num_workers = tr->count;
2078           sm->first_worker_index = tr->first_index;
2079         }
2080     }
2081
2082   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2083
2084   /* Use all available workers by default */
2085   if (sm->num_workers > 1)
2086     {
2087       for (i=0; i < sm->num_workers; i++)
2088         bitmap = clib_bitmap_set (bitmap, i, 1);
2089       snat_set_workers(bitmap);
2090       clib_bitmap_free (bitmap);
2091     }
2092   else
2093     {
2094       sm->per_thread_data[0].snat_thread_index = 0;
2095     }
2096
2097   error = snat_api_init(vm, sm);
2098   if (error)
2099     return error;
2100
2101   /* Set up the interface address add/del callback */
2102   cb4.function = snat_ip4_add_del_interface_address_cb;
2103   cb4.function_opaque = 0;
2104
2105   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2106
2107   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2108   cb4.function_opaque = 0;
2109
2110   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2111
2112   nat_dpo_module_init ();
2113
2114   /* Init IPFIX logging */
2115   snat_ipfix_logging_init(vm);
2116
2117   /* Init NAT64 */
2118   error = nat64_init(vm);
2119   if (error)
2120     return error;
2121
2122   dslite_init(vm);
2123
2124   nat66_init();
2125
2126   /* Init virtual fragmenentation reassembly */
2127   return nat_reass_init(vm);
2128 }
2129
2130 VLIB_INIT_FUNCTION (snat_init);
2131
2132 void snat_free_outside_address_and_port (snat_address_t * addresses,
2133                                          u32 thread_index,
2134                                          snat_session_key_t * k)
2135 {
2136   snat_address_t *a;
2137   u32 address_index;
2138   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2139
2140   for (address_index = 0; address_index < vec_len (addresses); address_index++)
2141     {
2142       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2143         break;
2144     }
2145
2146   ASSERT (address_index < vec_len (addresses));
2147
2148   a = addresses + address_index;
2149
2150   switch (k->protocol)
2151     {
2152 #define _(N, i, n, s) \
2153     case SNAT_PROTOCOL_##N: \
2154       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2155         port_host_byte_order) == 1); \
2156       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2157         port_host_byte_order, 0); \
2158       a->busy_##n##_ports--; \
2159       a->busy_##n##_ports_per_thread[thread_index]--; \
2160       break;
2161       foreach_snat_protocol
2162 #undef _
2163     default:
2164       nat_log_info ("unknown protocol");
2165       return;
2166     }
2167 }
2168
2169 /**
2170  * @brief Match NAT44 static mapping.
2171  *
2172  * @param sm          NAT main.
2173  * @param match       Address and port to match.
2174  * @param mapping     External or local address and port of the matched mapping.
2175  * @param by_external If 0 match by local address otherwise match by external
2176  *                    address.
2177  * @param is_addr_only If matched mapping is address only
2178  * @param twice_nat If matched mapping is twice NAT.
2179  * @param lb If matched mapping is load-balanced.
2180  *
2181  * @returns 0 if match found otherwise 1.
2182  */
2183 int snat_static_mapping_match (snat_main_t * sm,
2184                                snat_session_key_t match,
2185                                snat_session_key_t * mapping,
2186                                u8 by_external,
2187                                u8 *is_addr_only,
2188                                twice_nat_type_t *twice_nat,
2189                                lb_nat_type_t *lb,
2190                                ip4_address_t * ext_host_addr)
2191 {
2192   clib_bihash_kv_8_8_t kv, value;
2193   snat_static_mapping_t *m;
2194   snat_session_key_t m_key;
2195   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2196   u32 rand, lo = 0, hi, mid;
2197   u8 backend_index;
2198
2199   m_key.fib_index = match.fib_index;
2200   if (by_external)
2201     {
2202       mapping_hash = &sm->static_mapping_by_external;
2203       m_key.fib_index = 0;
2204     }
2205
2206   m_key.addr = match.addr;
2207   m_key.port = clib_net_to_host_u16 (match.port);
2208   m_key.protocol = match.protocol;
2209
2210   kv.key = m_key.as_u64;
2211
2212   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2213     {
2214       /* Try address only mapping */
2215       m_key.port = 0;
2216       m_key.protocol = 0;
2217       kv.key = m_key.as_u64;
2218       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2219         return 1;
2220     }
2221
2222   m = pool_elt_at_index (sm->static_mappings, value.value);
2223
2224   if (by_external)
2225     {
2226       if (vec_len (m->locals))
2227         {
2228           if (PREDICT_FALSE(lb != 0))
2229             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2230           if (m->affinity)
2231             {
2232               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2233                   match.protocol, match.port, &backend_index))
2234                 goto get_local;
2235
2236               mapping->addr = m->locals[backend_index].addr;
2237               mapping->port = clib_host_to_net_u16 (m->locals[backend_index].port);
2238               mapping->fib_index = m->locals[backend_index].fib_index;
2239               goto end;
2240             }
2241 get_local:
2242           hi = vec_len (m->locals) - 1;
2243           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2244           while (lo < hi)
2245             {
2246               mid = ((hi - lo) >> 1) + lo;
2247               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2248             }
2249           if (!(m->locals[lo].prefix >= rand))
2250             return 1;
2251           if (PREDICT_FALSE (sm->num_workers > 1))
2252             {
2253               ip4_header_t ip = {
2254                 .src_address = m->locals[lo].addr,
2255               };
2256               if (sm->worker_in2out_cb (&ip, m->fib_index) != vlib_get_thread_index ())
2257                 goto get_local;
2258             }
2259           mapping->addr = m->locals[lo].addr;
2260           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2261           mapping->fib_index = m->locals[lo].fib_index;
2262           if (m->affinity)
2263             {
2264               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2265                   match.protocol, match.port, lo, m->affinity,
2266                   m->affinity_per_service_list_head_index))
2267                 nat_log_info ("create affinity record failed");
2268             }
2269         }
2270       else
2271         {
2272           if (PREDICT_FALSE(lb != 0))
2273             *lb = NO_LB_NAT;
2274           mapping->fib_index = m->fib_index;
2275           mapping->addr = m->local_addr;
2276           /* Address only mapping doesn't change port */
2277           mapping->port = m->addr_only ? match.port
2278             : clib_host_to_net_u16 (m->local_port);
2279         }
2280       mapping->protocol = m->proto;
2281     }
2282   else
2283     {
2284       mapping->addr = m->external_addr;
2285       /* Address only mapping doesn't change port */
2286       mapping->port = m->addr_only ? match.port
2287         : clib_host_to_net_u16 (m->external_port);
2288       mapping->fib_index = sm->outside_fib_index;
2289     }
2290
2291 end:
2292   if (PREDICT_FALSE(is_addr_only != 0))
2293     *is_addr_only = m->addr_only;
2294
2295   if (PREDICT_FALSE(twice_nat != 0))
2296     *twice_nat = m->twice_nat;
2297
2298   return 0;
2299 }
2300
2301 static_always_inline u16
2302 snat_random_port (u16 min, u16 max)
2303 {
2304   snat_main_t *sm = &snat_main;
2305   return min + random_u32 (&sm->random_seed) /
2306     (random_u32_max() / (max - min + 1) + 1);
2307 }
2308
2309 int
2310 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2311                                      u32 fib_index,
2312                                      u32 thread_index,
2313                                      snat_session_key_t * k,
2314                                      u32 * address_indexp,
2315                                      u16 port_per_thread,
2316                                      u32 snat_thread_index)
2317 {
2318   snat_main_t *sm = &snat_main;
2319
2320   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2321                                  address_indexp, port_per_thread,
2322                                  snat_thread_index);
2323 }
2324
2325 static int
2326 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2327                                  u32 fib_index,
2328                                  u32 thread_index,
2329                                  snat_session_key_t * k,
2330                                  u32 * address_indexp,
2331                                  u16 port_per_thread,
2332                                  u32 snat_thread_index)
2333 {
2334   int i, gi = 0;
2335   snat_address_t *a, *ga = 0;
2336   u32 portnum;
2337
2338   for (i = 0; i < vec_len (addresses); i++)
2339     {
2340       a = addresses + i;
2341       switch (k->protocol)
2342         {
2343 #define _(N, j, n, s) \
2344         case SNAT_PROTOCOL_##N: \
2345           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2346             { \
2347               if (a->fib_index == fib_index) \
2348                 { \
2349                   while (1) \
2350                     { \
2351                       portnum = (port_per_thread * \
2352                         snat_thread_index) + \
2353                         snat_random_port(1, port_per_thread) + 1024; \
2354                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2355                         continue; \
2356                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2357                       a->busy_##n##_ports_per_thread[thread_index]++; \
2358                       a->busy_##n##_ports++; \
2359                       k->addr = a->addr; \
2360                       k->port = clib_host_to_net_u16(portnum); \
2361                       *address_indexp = i; \
2362                       return 0; \
2363                     } \
2364                 } \
2365               else if (a->fib_index == ~0) \
2366                 { \
2367                   ga = a; \
2368                   gi = i; \
2369                 } \
2370             } \
2371           break;
2372           foreach_snat_protocol
2373 #undef _
2374         default:
2375           nat_log_info ("unknown protocol");
2376           return 1;
2377         }
2378
2379     }
2380
2381   if (ga)
2382     {
2383       a = ga;
2384       switch (k->protocol)
2385         {
2386 #define _(N, j, n, s) \
2387         case SNAT_PROTOCOL_##N: \
2388           while (1) \
2389             { \
2390               portnum = (port_per_thread * \
2391                 snat_thread_index) + \
2392                 snat_random_port(1, port_per_thread) + 1024; \
2393               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2394                 continue; \
2395               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2396               a->busy_##n##_ports_per_thread[thread_index]++; \
2397               a->busy_##n##_ports++; \
2398               k->addr = a->addr; \
2399               k->port = clib_host_to_net_u16(portnum); \
2400               *address_indexp = gi; \
2401               return 0; \
2402             }
2403           break;
2404           foreach_snat_protocol
2405 #undef _
2406         default:
2407           nat_log_info ("unknown protocol");
2408           return 1;
2409         }
2410     }
2411
2412   /* Totally out of translations to use... */
2413   snat_ipfix_logging_addresses_exhausted(0);
2414   return 1;
2415 }
2416
2417 static int
2418 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2419                               u32 fib_index,
2420                               u32 thread_index,
2421                               snat_session_key_t * k,
2422                               u32 * address_indexp,
2423                               u16 port_per_thread,
2424                               u32 snat_thread_index)
2425 {
2426   snat_main_t *sm = &snat_main;
2427   snat_address_t *a = addresses;
2428   u16 m, ports, portnum, A, j;
2429   m = 16 - (sm->psid_offset + sm->psid_length);
2430   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2431
2432   if (!vec_len (addresses))
2433     goto exhausted;
2434
2435   switch (k->protocol)
2436     {
2437 #define _(N, i, n, s) \
2438     case SNAT_PROTOCOL_##N: \
2439       if (a->busy_##n##_ports < ports) \
2440         { \
2441           while (1) \
2442             { \
2443               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2444               j = snat_random_port(0, pow2_mask(m)); \
2445               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2446               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2447                 continue; \
2448               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2449               a->busy_##n##_ports++; \
2450               k->addr = a->addr; \
2451               k->port = clib_host_to_net_u16 (portnum); \
2452               *address_indexp = i; \
2453               return 0; \
2454             } \
2455         } \
2456       break;
2457       foreach_snat_protocol
2458 #undef _
2459     default:
2460       nat_log_info ("unknown protocol");
2461       return 1;
2462     }
2463
2464 exhausted:
2465   /* Totally out of translations to use... */
2466   snat_ipfix_logging_addresses_exhausted(0);
2467   return 1;
2468 }
2469
2470 static int
2471 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2472                                u32 fib_index,
2473                                u32 thread_index,
2474                                snat_session_key_t * k,
2475                                u32 * address_indexp,
2476                                u16 port_per_thread,
2477                                u32 snat_thread_index)
2478 {
2479   snat_main_t *sm = &snat_main;
2480   snat_address_t *a = addresses;
2481   u16 portnum, ports;
2482
2483   ports = sm->end_port - sm->start_port + 1;
2484
2485   if (!vec_len (addresses))
2486     goto exhausted;
2487
2488   switch (k->protocol)
2489     {
2490 #define _(N, i, n, s) \
2491     case SNAT_PROTOCOL_##N: \
2492       if (a->busy_##n##_ports < ports) \
2493         { \
2494           while (1) \
2495             { \
2496               portnum = snat_random_port(sm->start_port, sm->end_port); \
2497               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2498                 continue; \
2499               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2500               a->busy_##n##_ports++; \
2501               k->addr = a->addr; \
2502               k->port = clib_host_to_net_u16 (portnum); \
2503               *address_indexp = i; \
2504               return 0; \
2505             } \
2506         } \
2507       break;
2508       foreach_snat_protocol
2509 #undef _
2510     default:
2511       nat_log_info ("unknown protocol");
2512       return 1;
2513     }
2514
2515 exhausted:
2516   /* Totally out of translations to use... */
2517   snat_ipfix_logging_addresses_exhausted(0);
2518   return 1;
2519 }
2520
2521 void
2522 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2523 {
2524   dpo_id_t dpo_v4 = DPO_INVALID;
2525   fib_prefix_t pfx = {
2526     .fp_proto = FIB_PROTOCOL_IP4,
2527     .fp_len = 32,
2528     .fp_addr.ip4.as_u32 = addr.as_u32,
2529   };
2530
2531   if (is_add)
2532     {
2533       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2534       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2535                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2536       dpo_reset (&dpo_v4);
2537     }
2538   else
2539     {
2540       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2541     }
2542 }
2543
2544 uword
2545 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2546 {
2547   u32 *r = va_arg (*args, u32 *);
2548
2549   if (0);
2550 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2551   foreach_snat_protocol
2552 #undef _
2553   else
2554     return 0;
2555   return 1;
2556 }
2557
2558 u8 *
2559 format_snat_protocol (u8 * s, va_list * args)
2560 {
2561   u32 i = va_arg (*args, u32);
2562   u8 *t = 0;
2563
2564   switch (i)
2565     {
2566 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2567       foreach_snat_protocol
2568 #undef _
2569     default:
2570       s = format (s, "unknown");
2571       return s;
2572     }
2573   s = format (s, "%s", t);
2574   return s;
2575 }
2576
2577 u8 *
2578 format_nat_addr_and_port_alloc_alg (u8 * s, va_list * args)
2579 {
2580   u32 i = va_arg (*args, u32);
2581   u8 *t = 0;
2582
2583   switch (i)
2584     {
2585 #define _(v, N, s) case NAT_ADDR_AND_PORT_ALLOC_ALG_##N: t = (u8 *) s; break;
2586       foreach_nat_addr_and_port_alloc_alg
2587 #undef _
2588     default:
2589       s = format (s, "unknown");
2590       return s;
2591     }
2592   s = format (s, "%s", t);
2593   return s;
2594 }
2595
2596 u8 * format_snat_key (u8 * s, va_list * args);
2597 u8 * format_static_mapping_key (u8 * s, va_list * args);
2598
2599 u8 *
2600 format_session_kvp (u8 * s, va_list * args)
2601 {
2602   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2603   snat_session_key_t k;
2604
2605   k.as_u64 = v->key;
2606
2607   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2608
2609   return s;
2610 }
2611
2612 u8 *
2613 format_static_mapping_kvp (u8 * s, va_list * args)
2614 {
2615   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2616   snat_session_key_t k;
2617
2618   k.as_u64 = v->key;
2619
2620   s = format (s, "%U static-mapping-index %llu",
2621               format_static_mapping_key, &k, v->value);
2622
2623   return s;
2624 }
2625
2626 u8 *
2627 format_user_kvp (u8 * s, va_list * args)
2628 {
2629   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2630   snat_user_key_t k;
2631
2632   k.as_u64 = v->key;
2633
2634   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2635               k.fib_index, v->value);
2636
2637   return s;
2638 }
2639
2640 u8 *
2641 format_ed_session_kvp (u8 * s, va_list * args)
2642 {
2643   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2644   nat_ed_ses_key_t k;
2645
2646   k.as_u64[0] = v->key[0];
2647   k.as_u64[1] = v->key[1];
2648
2649   s = format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2650               format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2651               format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2652               format_ip_protocol, k.proto, k.fib_index, v->value);
2653
2654   return s;
2655 }
2656
2657 static u32
2658 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2659 {
2660   snat_main_t *sm = &snat_main;
2661   u32 next_worker_index = 0;
2662   u32 hash;
2663
2664   next_worker_index = sm->first_worker_index;
2665   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2666          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2667
2668   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2669     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2670   else
2671     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2672
2673   return next_worker_index;
2674 }
2675
2676 static u32
2677 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2678 {
2679   snat_main_t *sm = &snat_main;
2680   udp_header_t *udp;
2681   u16 port;
2682   snat_session_key_t m_key;
2683   clib_bihash_kv_8_8_t kv, value;
2684   snat_static_mapping_t *m;
2685   u32 proto;
2686   u32 next_worker_index = 0;
2687
2688   /* first try static mappings without port */
2689   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2690     {
2691       m_key.addr = ip0->dst_address;
2692       m_key.port = 0;
2693       m_key.protocol = 0;
2694       m_key.fib_index = rx_fib_index0;
2695       kv.key = m_key.as_u64;
2696       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2697         {
2698           m = pool_elt_at_index (sm->static_mappings, value.value);
2699           return m->workers[0];
2700         }
2701     }
2702
2703   proto = ip_proto_to_snat_proto (ip0->protocol);
2704   udp = ip4_next_header (ip0);
2705   port = udp->dst_port;
2706
2707   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2708     {
2709       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2710         return vlib_get_thread_index ();
2711
2712       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2713         {
2714           nat_reass_ip4_t *reass;
2715
2716           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2717                                       ip0->fragment_id, ip0->protocol);
2718
2719           if (reass && (reass->thread_index != (u32) ~ 0))
2720             return reass->thread_index;
2721           else
2722             return vlib_get_thread_index ();
2723         }
2724     }
2725
2726   /* unknown protocol */
2727   if (PREDICT_FALSE (proto == ~0))
2728     {
2729       /* use current thread */
2730       return vlib_get_thread_index ();
2731     }
2732
2733   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2734     {
2735       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2736       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2737       if (!icmp_is_error_message (icmp))
2738         port = echo->identifier;
2739       else
2740         {
2741           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2742           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2743           void *l4_header = ip4_next_header (inner_ip);
2744           switch (proto)
2745             {
2746             case SNAT_PROTOCOL_ICMP:
2747               icmp = (icmp46_header_t*)l4_header;
2748               echo = (icmp_echo_header_t *)(icmp + 1);
2749               port = echo->identifier;
2750               break;
2751             case SNAT_PROTOCOL_UDP:
2752             case SNAT_PROTOCOL_TCP:
2753               port = ((tcp_udp_header_t*)l4_header)->src_port;
2754               break;
2755             default:
2756               return vlib_get_thread_index ();
2757             }
2758         }
2759     }
2760
2761   /* try static mappings with port */
2762   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2763     {
2764       m_key.addr = ip0->dst_address;
2765       m_key.port = clib_net_to_host_u16 (port);
2766       m_key.protocol = proto;
2767       m_key.fib_index = rx_fib_index0;
2768       kv.key = m_key.as_u64;
2769       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2770         {
2771           m = pool_elt_at_index (sm->static_mappings, value.value);
2772           return m->workers[0];
2773         }
2774     }
2775
2776   /* worker by outside port */
2777   next_worker_index = sm->first_worker_index;
2778   next_worker_index +=
2779     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2780   return next_worker_index;
2781 }
2782
2783 static u32
2784 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2785 {
2786   snat_main_t *sm = &snat_main;
2787   clib_bihash_kv_8_8_t kv, value;
2788   u32 proto, next_worker_index = 0;
2789   udp_header_t *udp;
2790   u16 port;
2791   snat_static_mapping_t *m;
2792   u32 hash;
2793
2794   /* first try static mappings without port */
2795   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2796     {
2797       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2798       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2799         {
2800           m = pool_elt_at_index (sm->static_mappings, value.value);
2801           return m->workers[0];
2802         }
2803     }
2804
2805   proto = ip_proto_to_snat_proto (ip->protocol);
2806
2807   /* unknown protocol */
2808   if (PREDICT_FALSE (proto == ~0))
2809     {
2810       /* use current thread */
2811       return vlib_get_thread_index ();
2812     }
2813
2814   udp = ip4_next_header (ip);
2815   port = udp->dst_port;
2816
2817   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2818     {
2819       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2820       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2821       if (!icmp_is_error_message (icmp))
2822         port = echo->identifier;
2823       else
2824         {
2825           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2826           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2827           void *l4_header = ip4_next_header (inner_ip);
2828           switch (proto)
2829             {
2830             case SNAT_PROTOCOL_ICMP:
2831               icmp = (icmp46_header_t*)l4_header;
2832               echo = (icmp_echo_header_t *)(icmp + 1);
2833               port = echo->identifier;
2834               break;
2835             case SNAT_PROTOCOL_UDP:
2836             case SNAT_PROTOCOL_TCP:
2837               port = ((tcp_udp_header_t*)l4_header)->src_port;
2838               break;
2839             default:
2840               return vlib_get_thread_index ();
2841             }
2842         }
2843     }
2844
2845   /* try static mappings with port */
2846   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2847     {
2848       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2849                   clib_net_to_host_u16 (port));
2850       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2851         {
2852           m = pool_elt_at_index (sm->static_mappings, value.value);
2853           if (!vec_len(m->locals))
2854             return m->workers[0];
2855
2856           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2857                  (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >>24);
2858
2859           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2860             return m->workers[hash & (_vec_len (m->workers) - 1)];
2861           else
2862             return m->workers[hash % _vec_len (m->workers)];
2863         }
2864     }
2865
2866   /* worker by outside port */
2867   next_worker_index = sm->first_worker_index;
2868   next_worker_index +=
2869     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2870
2871   return next_worker_index;
2872 }
2873
2874 static clib_error_t *
2875 snat_config (vlib_main_t * vm, unformat_input_t * input)
2876 {
2877   snat_main_t * sm = &snat_main;
2878   nat66_main_t * nm = &nat66_main;
2879   u32 translation_buckets = 1024;
2880   u32 translation_memory_size = 128<<20;
2881   u32 user_buckets = 128;
2882   u32 user_memory_size = 64<<20;
2883   u32 max_translations_per_user = 100;
2884   u32 outside_vrf_id = 0;
2885   u32 outside_ip6_vrf_id = 0;
2886   u32 inside_vrf_id = 0;
2887   u32 static_mapping_buckets = 1024;
2888   u32 static_mapping_memory_size = 64<<20;
2889   u32 nat64_bib_buckets = 1024;
2890   u32 nat64_bib_memory_size = 128 << 20;
2891   u32 nat64_st_buckets = 2048;
2892   u32 nat64_st_memory_size = 256 << 20;
2893   u8 static_mapping_only = 0;
2894   u8 static_mapping_connection_tracking = 0;
2895   snat_main_per_thread_data_t *tsm;
2896   dslite_main_t * dm = &dslite_main;
2897
2898   sm->deterministic = 0;
2899   sm->out2in_dpo = 0;
2900   sm->endpoint_dependent = 0;
2901
2902   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2903     {
2904       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2905         ;
2906       else if (unformat (input, "translation hash memory %d",
2907                          &translation_memory_size));
2908       else if (unformat (input, "user hash buckets %d", &user_buckets))
2909         ;
2910       else if (unformat (input, "user hash memory %d",
2911                          &user_memory_size))
2912         ;
2913       else if (unformat (input, "max translations per user %d",
2914                          &max_translations_per_user))
2915         ;
2916       else if (unformat (input, "outside VRF id %d",
2917                          &outside_vrf_id))
2918         ;
2919       else if (unformat (input, "outside ip6 VRF id %d",
2920                          &outside_ip6_vrf_id))
2921         ;
2922       else if (unformat (input, "inside VRF id %d",
2923                          &inside_vrf_id))
2924         ;
2925       else if (unformat (input, "static mapping only"))
2926         {
2927           static_mapping_only = 1;
2928           if (unformat (input, "connection tracking"))
2929             static_mapping_connection_tracking = 1;
2930         }
2931       else if (unformat (input, "deterministic"))
2932         sm->deterministic = 1;
2933       else if (unformat (input, "nat64 bib hash buckets %d",
2934                          &nat64_bib_buckets))
2935         ;
2936       else if (unformat (input, "nat64 bib hash memory %d",
2937                          &nat64_bib_memory_size))
2938         ;
2939       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2940         ;
2941       else if (unformat (input, "nat64 st hash memory %d",
2942                          &nat64_st_memory_size))
2943         ;
2944       else if (unformat (input, "out2in dpo"))
2945         sm->out2in_dpo = 1;
2946       else if (unformat (input, "dslite ce"))
2947         dslite_set_ce(dm, 1);
2948       else if (unformat (input, "endpoint-dependent"))
2949         sm->endpoint_dependent = 1;
2950       else
2951         return clib_error_return (0, "unknown input '%U'",
2952                                   format_unformat_error, input);
2953     }
2954
2955   if (sm->deterministic && sm->endpoint_dependent)
2956     return clib_error_return (
2957       0, "deterministic and endpoint-dependent modes are mutually exclusive");
2958
2959   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
2960     return clib_error_return (
2961       0, "static mapping only mode available only for simple nat");
2962
2963   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
2964     return clib_error_return (
2965       0, "out2in dpo mode available only for simple nat");
2966
2967   /* for show commands, etc. */
2968   sm->translation_buckets = translation_buckets;
2969   sm->translation_memory_size = translation_memory_size;
2970   /* do not exceed load factor 10 */
2971   sm->max_translations = 10 * translation_buckets;
2972   sm->user_buckets = user_buckets;
2973   sm->user_memory_size = user_memory_size;
2974   sm->max_translations_per_user = max_translations_per_user;
2975   sm->outside_vrf_id = outside_vrf_id;
2976   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2977                                                              outside_vrf_id,
2978                                                              FIB_SOURCE_PLUGIN_HI);
2979   nm->outside_vrf_id = outside_ip6_vrf_id;
2980   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2981                                                              outside_ip6_vrf_id,
2982                                                              FIB_SOURCE_PLUGIN_HI);
2983   sm->inside_vrf_id = inside_vrf_id;
2984   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2985                                                             inside_vrf_id,
2986                                                             FIB_SOURCE_PLUGIN_HI);
2987   sm->static_mapping_only = static_mapping_only;
2988   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
2989
2990   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
2991                  nat64_st_memory_size);
2992
2993   if (sm->deterministic)
2994     {
2995       sm->in2out_node_index = snat_det_in2out_node.index;
2996       sm->in2out_output_node_index = ~0;
2997       sm->out2in_node_index = snat_det_out2in_node.index;
2998       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
2999       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3000     }
3001   else
3002     {
3003       if (sm->endpoint_dependent)
3004         {
3005           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3006           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3007           sm->in2out_node_index = nat44_ed_in2out_node.index;
3008           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3009           sm->out2in_node_index = nat44_ed_out2in_node.index;
3010           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3011           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3012           nat_affinity_init (vm);
3013         }
3014       else
3015         {
3016           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3017           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3018           sm->in2out_node_index = snat_in2out_node.index;
3019           sm->in2out_output_node_index = snat_in2out_output_node.index;
3020           sm->out2in_node_index = snat_out2in_node.index;
3021           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3022           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3023         }
3024       if (!static_mapping_only ||
3025           (static_mapping_only && static_mapping_connection_tracking))
3026         {
3027           vec_foreach (tsm, sm->per_thread_data)
3028             {
3029               if (sm->endpoint_dependent)
3030                 {
3031                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3032                                          translation_buckets,
3033                                          translation_memory_size);
3034                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3035                                                       format_ed_session_kvp);
3036
3037                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3038                                          translation_buckets,
3039                                          translation_memory_size);
3040                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3041                                                       format_ed_session_kvp);
3042                 }
3043               else
3044                 {
3045                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3046                                         translation_buckets,
3047                                         translation_memory_size);
3048                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3049                                                      format_session_kvp);
3050
3051                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3052                                         translation_buckets,
3053                                         translation_memory_size);
3054                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3055                                                      format_session_kvp);
3056                 }
3057
3058               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3059                                     user_memory_size);
3060               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3061                                                  format_user_kvp);
3062             }
3063
3064         }
3065       else
3066         {
3067           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3068           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3069         }
3070       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3071                             "static_mapping_by_local", static_mapping_buckets,
3072                             static_mapping_memory_size);
3073       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3074                                          format_static_mapping_kvp);
3075
3076       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3077                             "static_mapping_by_external", static_mapping_buckets,
3078                             static_mapping_memory_size);
3079       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3080                                          format_static_mapping_kvp);
3081     }
3082
3083   return 0;
3084 }
3085
3086 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3087
3088 u8 * format_snat_session_state (u8 * s, va_list * args)
3089 {
3090   u32 i = va_arg (*args, u32);
3091   u8 *t = 0;
3092
3093   switch (i)
3094     {
3095 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
3096     foreach_snat_session_state
3097 #undef _
3098     default:
3099       t = format (t, "unknown");
3100     }
3101   s = format (s, "%s", t);
3102   return s;
3103 }
3104
3105 u8 * format_snat_key (u8 * s, va_list * args)
3106 {
3107   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3108
3109   s = format (s, "%U proto %U port %d fib %d",
3110               format_ip4_address, &key->addr,
3111               format_snat_protocol, key->protocol,
3112               clib_net_to_host_u16 (key->port), key->fib_index);
3113   return s;
3114 }
3115
3116 u8 * format_static_mapping_key (u8 * s, va_list * args)
3117 {
3118   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3119
3120   s = format (s, "%U proto %U port %d fib %d",
3121               format_ip4_address, &key->addr,
3122               format_snat_protocol, key->protocol,
3123               key->port, key->fib_index);
3124   return s;
3125 }
3126
3127 u8 * format_snat_session (u8 * s, va_list * args)
3128 {
3129   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3130   snat_session_t * sess = va_arg (*args, snat_session_t *);
3131
3132   if (snat_is_unk_proto_session (sess))
3133     {
3134       s = format (s, "  i2o %U proto %u fib %u\n",
3135                   format_ip4_address, &sess->in2out.addr,
3136                   clib_net_to_host_u16 (sess->in2out.port),
3137                   sess->in2out.fib_index);
3138       s = format (s, "    o2i %U proto %u fib %u\n",
3139                   format_ip4_address, &sess->out2in.addr,
3140                   clib_net_to_host_u16 (sess->out2in.port),
3141                   sess->out2in.fib_index);
3142     }
3143   else
3144     {
3145       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
3146       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
3147     }
3148   if (is_ed_session (sess) || is_fwd_bypass_session (sess))
3149     {
3150       if (is_twice_nat_session (sess))
3151         {
3152           s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
3153                       format_ip4_address, &sess->ext_host_addr,
3154                       clib_net_to_host_u16 (sess->ext_host_port),
3155                       format_ip4_address, &sess->ext_host_nat_addr,
3156                       clib_net_to_host_u16 (sess->ext_host_nat_port));
3157         }
3158       else
3159         {
3160           if (sess->ext_host_addr.as_u32)
3161               s = format (s, "       external host %U:%u\n",
3162                           format_ip4_address, &sess->ext_host_addr,
3163                           clib_net_to_host_u16 (sess->ext_host_port));
3164         }
3165     }
3166   s = format (s, "       index %llu\n", sess - sm->sessions);
3167   s = format (s, "       last heard %.2f\n", sess->last_heard);
3168   s = format (s, "       total pkts %d, total bytes %lld\n",
3169               sess->total_pkts, sess->total_bytes);
3170   if (snat_is_session_static (sess))
3171     s = format (s, "       static translation\n");
3172   else
3173     s = format (s, "       dynamic translation\n");
3174   if (is_fwd_bypass_session (sess))
3175     s = format (s, "       forwarding-bypass\n");
3176   if (is_lb_session (sess))
3177     s = format (s, "       load-balancing\n");
3178   if (is_twice_nat_session (sess))
3179     s = format (s, "       twice-nat\n");
3180
3181   return s;
3182 }
3183
3184 u8 * format_snat_user (u8 * s, va_list * args)
3185 {
3186   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3187   snat_user_t * u = va_arg (*args, snat_user_t *);
3188   int verbose = va_arg (*args, int);
3189   dlist_elt_t * head, * elt;
3190   u32 elt_index, head_index;
3191   u32 session_index;
3192   snat_session_t * sess;
3193
3194   s = format (s, "%U: %d dynamic translations, %d static translations\n",
3195               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
3196
3197   if (verbose == 0)
3198     return s;
3199
3200   if (u->nsessions || u->nstaticsessions)
3201     {
3202       head_index = u->sessions_per_user_list_head_index;
3203       head = pool_elt_at_index (sm->list_pool, head_index);
3204
3205       elt_index = head->next;
3206       elt = pool_elt_at_index (sm->list_pool, elt_index);
3207       session_index = elt->value;
3208
3209       while (session_index != ~0)
3210         {
3211           sess = pool_elt_at_index (sm->sessions, session_index);
3212
3213           s = format (s, "  %U\n", format_snat_session, sm, sess);
3214
3215           elt_index = elt->next;
3216           elt = pool_elt_at_index (sm->list_pool, elt_index);
3217           session_index = elt->value;
3218         }
3219     }
3220
3221   return s;
3222 }
3223
3224 u8 * format_snat_static_mapping (u8 * s, va_list * args)
3225 {
3226   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
3227   nat44_lb_addr_port_t *local;
3228
3229   if (m->addr_only)
3230       s = format (s, "local %U external %U vrf %d %s %s",
3231                   format_ip4_address, &m->local_addr,
3232                   format_ip4_address, &m->external_addr,
3233                   m->vrf_id,
3234                   m->twice_nat == TWICE_NAT ? "twice-nat" :
3235                   m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3236                   m->out2in_only ? "out2in-only" : "");
3237   else
3238    {
3239       if (vec_len (m->locals))
3240         {
3241           s = format (s, "%U external %U:%d %s %s",
3242                       format_snat_protocol, m->proto,
3243                       format_ip4_address, &m->external_addr, m->external_port,
3244                       m->twice_nat == TWICE_NAT ? "twice-nat" :
3245                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3246                       m->out2in_only ? "out2in-only" : "");
3247           vec_foreach (local, m->locals)
3248             s = format (s, "\n  local %U:%d vrf %d probability %d\%",
3249                         format_ip4_address, &local->addr, local->port,
3250                         local->vrf_id, local->probability);
3251         }
3252       else
3253         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
3254                     format_snat_protocol, m->proto,
3255                     format_ip4_address, &m->local_addr, m->local_port,
3256                     format_ip4_address, &m->external_addr, m->external_port,
3257                     m->vrf_id,
3258                     m->twice_nat == TWICE_NAT ? "twice-nat" :
3259                     m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3260                     m->out2in_only ? "out2in-only" : "");
3261    }
3262   return s;
3263 }
3264
3265 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
3266 {
3267   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
3268   vnet_main_t *vnm = vnet_get_main();
3269
3270   if (m->addr_only)
3271       s = format (s, "local %U external %U vrf %d",
3272                   format_ip4_address, &m->l_addr,
3273                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3274                   m->vrf_id);
3275   else
3276       s = format (s, "%U local %U:%d external %U:%d vrf %d",
3277                   format_snat_protocol, m->proto,
3278                   format_ip4_address, &m->l_addr, m->l_port,
3279                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3280                   m->e_port, m->vrf_id);
3281
3282   return s;
3283 }
3284
3285 u8 * format_det_map_ses (u8 * s, va_list * args)
3286 {
3287   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
3288   ip4_address_t in_addr, out_addr;
3289   u32 in_offset, out_offset;
3290   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
3291   u32 * i = va_arg (*args, u32 *);
3292
3293   u32 user_index = *i / SNAT_DET_SES_PER_USER;
3294   in_addr.as_u32 = clib_host_to_net_u32 (
3295     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
3296   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
3297     clib_net_to_host_u32(det_map->in_addr.as_u32);
3298   out_offset = in_offset / det_map->sharing_ratio;
3299   out_addr.as_u32 = clib_host_to_net_u32(
3300     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
3301   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
3302               format_ip4_address, &in_addr,
3303               clib_net_to_host_u16 (ses->in_port),
3304               format_ip4_address, &out_addr,
3305               clib_net_to_host_u16 (ses->out.out_port),
3306               format_ip4_address, &ses->out.ext_host_addr,
3307               clib_net_to_host_u16 (ses->out.ext_host_port),
3308               format_snat_session_state, ses->state,
3309               ses->expire);
3310
3311   return s;
3312 }
3313
3314 static void
3315 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3316                                  uword opaque,
3317                                  u32 sw_if_index,
3318                                  ip4_address_t * address,
3319                                  u32 address_length,
3320                                  u32 if_address_index,
3321                                  u32 is_delete)
3322 {
3323   snat_main_t *sm = &snat_main;
3324   snat_static_map_resolve_t *rp;
3325   snat_static_mapping_t *m;
3326   snat_session_key_t m_key;
3327   clib_bihash_kv_8_8_t kv, value;
3328   int i, rv;
3329   ip4_address_t l_addr;
3330
3331   for (i = 0; i < vec_len (sm->to_resolve); i++)
3332     {
3333       rp = sm->to_resolve + i;
3334       if (rp->addr_only == 0)
3335         continue;
3336       if (rp->sw_if_index == sw_if_index)
3337         goto match;
3338     }
3339
3340   return;
3341
3342 match:
3343   m_key.addr.as_u32 = address->as_u32;
3344   m_key.port = rp->addr_only ? 0 : rp->e_port;
3345   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3346   m_key.fib_index = sm->outside_fib_index;
3347   kv.key = m_key.as_u64;
3348   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3349     m = 0;
3350   else
3351     m = pool_elt_at_index (sm->static_mappings, value.value);
3352
3353   if (!is_delete)
3354     {
3355       /* Don't trip over lease renewal, static config */
3356       if (m)
3357         return;
3358     }
3359   else
3360     {
3361       if (!m)
3362         return;
3363     }
3364
3365   /* Indetity mapping? */
3366   if (rp->l_addr.as_u32 == 0)
3367     l_addr.as_u32 = address[0].as_u32;
3368   else
3369     l_addr.as_u32 = rp->l_addr.as_u32;
3370   /* Add the static mapping */
3371   rv = snat_add_static_mapping (l_addr,
3372                                 address[0],
3373                                 rp->l_port,
3374                                 rp->e_port,
3375                                 rp->vrf_id,
3376                                 rp->addr_only,
3377                                 ~0 /* sw_if_index */,
3378                                 rp->proto,
3379                                 !is_delete,
3380                                 0, 0, rp->tag);
3381   if (rv)
3382     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3383 }
3384
3385 static void
3386 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3387                                        uword opaque,
3388                                        u32 sw_if_index,
3389                                        ip4_address_t * address,
3390                                        u32 address_length,
3391                                        u32 if_address_index,
3392                                        u32 is_delete)
3393 {
3394   snat_main_t *sm = &snat_main;
3395   snat_static_map_resolve_t *rp;
3396   ip4_address_t l_addr;
3397   int i, j;
3398   int rv;
3399   u8 twice_nat = 0;
3400   snat_address_t *addresses = sm->addresses;
3401
3402   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
3403     {
3404       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3405           goto match;
3406     }
3407
3408   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
3409     {
3410       twice_nat = 1;
3411       addresses = sm->twice_nat_addresses;
3412       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3413           goto match;
3414     }
3415
3416   return;
3417
3418 match:
3419   if (!is_delete)
3420     {
3421       /* Don't trip over lease renewal, static config */
3422       for (j = 0; j < vec_len(addresses); j++)
3423         if (addresses[j].addr.as_u32 == address->as_u32)
3424           return;
3425
3426       (void) snat_add_address (sm, address, ~0, twice_nat);
3427       /* Scan static map resolution vector */
3428       for (j = 0; j < vec_len (sm->to_resolve); j++)
3429         {
3430           rp = sm->to_resolve + j;
3431           if (rp->addr_only)
3432             continue;
3433           /* On this interface? */
3434           if (rp->sw_if_index == sw_if_index)
3435             {
3436               /* Indetity mapping? */
3437               if (rp->l_addr.as_u32 == 0)
3438                 l_addr.as_u32 = address[0].as_u32;
3439               else
3440                 l_addr.as_u32 = rp->l_addr.as_u32;
3441               /* Add the static mapping */
3442               rv = snat_add_static_mapping (l_addr,
3443                                             address[0],
3444                                             rp->l_port,
3445                                             rp->e_port,
3446                                             rp->vrf_id,
3447                                             rp->addr_only,
3448                                             ~0 /* sw_if_index */,
3449                                             rp->proto,
3450                                             rp->is_add,
3451                                             0, 0, rp->tag);
3452               if (rv)
3453                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3454             }
3455         }
3456       return;
3457     }
3458   else
3459     {
3460       (void) snat_del_address(sm, address[0], 1, twice_nat);
3461       return;
3462     }
3463 }
3464
3465
3466 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
3467                                 u8 twice_nat)
3468 {
3469   ip4_main_t * ip4_main = sm->ip4_main;
3470   ip4_address_t * first_int_addr;
3471   snat_static_map_resolve_t *rp;
3472   u32 *indices_to_delete = 0;
3473   int i, j;
3474   u32 *auto_add_sw_if_indices =
3475     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3476
3477   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
3478                                                 0 /* just want the address*/);
3479
3480   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
3481     {
3482       if (auto_add_sw_if_indices[i] == sw_if_index)
3483         {
3484           if (is_del)
3485             {
3486               /* if have address remove it */
3487               if (first_int_addr)
3488                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3489               else
3490                 {
3491                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3492                     {
3493                       rp = sm->to_resolve + j;
3494                       if (rp->sw_if_index == sw_if_index)
3495                         vec_add1 (indices_to_delete, j);
3496                     }
3497                   if (vec_len(indices_to_delete))
3498                     {
3499                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3500                         vec_del1(sm->to_resolve, j);
3501                       vec_free(indices_to_delete);
3502                     }
3503                 }
3504               if (twice_nat)
3505                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3506               else
3507                 vec_del1(sm->auto_add_sw_if_indices, i);
3508             }
3509           else
3510             return VNET_API_ERROR_VALUE_EXIST;
3511
3512           return 0;
3513         }
3514     }
3515
3516   if (is_del)
3517     return VNET_API_ERROR_NO_SUCH_ENTRY;
3518
3519   /* add to the auto-address list */
3520   if (twice_nat)
3521     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3522   else
3523     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3524
3525   /* If the address is already bound - or static - add it now */
3526   if (first_int_addr)
3527       (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3528
3529   return 0;
3530 }
3531
3532 int
3533 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3534                    snat_protocol_t proto, u32 vrf_id, int is_in)
3535 {
3536   snat_main_per_thread_data_t *tsm;
3537   clib_bihash_kv_8_8_t kv, value;
3538   ip4_header_t ip;
3539   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3540   snat_session_key_t key;
3541   snat_session_t *s;
3542   clib_bihash_8_8_t *t;
3543
3544   if (sm->endpoint_dependent)
3545     return VNET_API_ERROR_UNSUPPORTED;
3546
3547   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3548   if (sm->num_workers > 1)
3549     tsm =
3550       vec_elt_at_index (sm->per_thread_data,
3551                         sm->worker_in2out_cb (&ip, fib_index));
3552   else
3553     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3554
3555   key.addr.as_u32 = addr->as_u32;
3556   key.port = clib_host_to_net_u16 (port);
3557   key.protocol = proto;
3558   key.fib_index = fib_index;
3559   kv.key = key.as_u64;
3560   t = is_in ? &tsm->in2out : &tsm->out2in;
3561   if (!clib_bihash_search_8_8 (t, &kv, &value))
3562     {
3563       if (pool_is_free_index (tsm->sessions, value.value))
3564         return VNET_API_ERROR_UNSPECIFIED;
3565
3566       s = pool_elt_at_index (tsm->sessions, value.value);
3567       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3568       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3569       return 0;
3570     }
3571
3572   return VNET_API_ERROR_NO_SUCH_ENTRY;
3573 }
3574
3575 int
3576 nat44_del_ed_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3577                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3578                       u32 vrf_id, int is_in)
3579 {
3580   ip4_header_t ip;
3581   clib_bihash_16_8_t *t;
3582   nat_ed_ses_key_t key;
3583   clib_bihash_kv_16_8_t kv, value;
3584   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3585   snat_session_t *s;
3586   snat_main_per_thread_data_t *tsm;
3587
3588   if (!sm->endpoint_dependent)
3589     return VNET_API_ERROR_FEATURE_DISABLED;
3590
3591   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3592   if (sm->num_workers > 1)
3593     tsm =
3594       vec_elt_at_index (sm->per_thread_data,
3595                         sm->worker_in2out_cb (&ip, fib_index));
3596   else
3597     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3598
3599   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3600   key.l_addr.as_u32 = addr->as_u32;
3601   key.r_addr.as_u32 = eh_addr->as_u32;
3602   key.l_port = clib_host_to_net_u16 (port);
3603   key.r_port = clib_host_to_net_u16 (eh_port);
3604   key.proto = proto;
3605   key.fib_index = clib_host_to_net_u32 (fib_index);
3606   kv.key[0] = key.as_u64[0];
3607   kv.key[1] = key.as_u64[1];
3608   if (clib_bihash_search_16_8 (t, &kv, &value))
3609     return VNET_API_ERROR_NO_SUCH_ENTRY;
3610
3611   if (pool_is_free_index (tsm->sessions, value.value))
3612     return VNET_API_ERROR_UNSPECIFIED;
3613   s = pool_elt_at_index (tsm->sessions, value.value);
3614   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3615   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3616   return 0;
3617 }
3618
3619 void
3620 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3621 {
3622   snat_main_t *sm = &snat_main;
3623
3624   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3625   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3626   sm->psid = psid;
3627   sm->psid_offset = psid_offset;
3628   sm->psid_length = psid_length;
3629 }
3630
3631 void
3632 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3633 {
3634   snat_main_t *sm = &snat_main;
3635
3636   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3637   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3638   sm->start_port = start_port;
3639   sm->end_port = end_port;
3640 }
3641
3642 void
3643 nat_set_alloc_addr_and_port_default (void)
3644 {
3645   snat_main_t *sm = &snat_main;
3646
3647   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3648   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3649 }
3650