src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <vnet/fib/fib_table.h>
  33 #include <vnet/fib/ip4_fib.h>
  34
  35 #include <vpp/app/version.h>
  36
  37 snat_main_t snat_main;
  38
  39
  40 /* Hook up input features */
  41 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  42   .arc_name = "ip4-unicast",
  43   .node_name = "nat44-in2out",
  44   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  45 };
  46 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat44-out2in",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-dhcp-client-detect"),
  51 };
  52 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat44-classify",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  56 };
  57 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  58   .arc_name = "ip4-unicast",
  59   .node_name = "nat44-det-in2out",
  60   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  61 };
  62 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  63   .arc_name = "ip4-unicast",
  64   .node_name = "nat44-det-out2in",
  65   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  66                                "ip4-dhcp-client-detect"),
  67 };
  68 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  69   .arc_name = "ip4-unicast",
  70   .node_name = "nat44-det-classify",
  71   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  72 };
  73 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  74   .arc_name = "ip4-unicast",
  75   .node_name = "nat44-ed-in2out",
  76   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  77 };
  78 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  79   .arc_name = "ip4-unicast",
  80   .node_name = "nat44-ed-out2in",
  81   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  82                                "ip4-dhcp-client-detect"),
  83 };
  84 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-ed-classify",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-in2out-worker-handoff",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  93 };
  94 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-out2in-worker-handoff",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-dhcp-client-detect"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-handoff-classify",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 104 };
 105 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 106   .arc_name = "ip4-unicast",
 107   .node_name = "nat44-in2out-fast",
 108   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 109 };
 110 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 111   .arc_name = "ip4-unicast",
 112   .node_name = "nat44-out2in-fast",
 113   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 114                                "ip4-dhcp-client-detect"),
 115 };
 116 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 117   .arc_name = "ip4-unicast",
 118   .node_name = "nat44-hairpin-dst",
 119   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 120 };
 121 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 122   .arc_name = "ip4-unicast",
 123   .node_name = "nat44-ed-hairpin-dst",
 124   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 125 };
 126
 127 /* Hook up output features */
 128 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 129   .arc_name = "ip4-output",
 130   .node_name = "nat44-in2out-output",
 131   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 132 };
 133 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 134   .arc_name = "ip4-output",
 135   .node_name = "nat44-in2out-output-worker-handoff",
 136   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-hairpin-src",
 141   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 142 };
 143 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 144   .arc_name = "ip4-output",
 145   .node_name = "nat44-ed-in2out-output",
 146   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 147 };
 148 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-ed-hairpin-src",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 152 };
 153
 154 /* Hook up ip4-local features */
 155 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 156 {
 157   .arc_name = "ip4-local",
 158   .node_name = "nat44-hairpinning",
 159   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 162 {
 163   .arc_name = "ip4-local",
 164   .node_name = "nat44-ed-hairpinning",
 165   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 166 };
 167
 168
 169 /* *INDENT-OFF* */
 170 VLIB_PLUGIN_REGISTER () = {
 171     .version = VPP_BUILD_VER,
 172     .description = "Network Address Translation",
 173 };
 174 /* *INDENT-ON* */
 175
 176 vlib_node_registration_t nat44_classify_node;
 177 vlib_node_registration_t nat44_ed_classify_node;
 178 vlib_node_registration_t nat44_det_classify_node;
 179 vlib_node_registration_t nat44_handoff_classify_node;
 180
 181 typedef enum {
 182   NAT44_CLASSIFY_NEXT_IN2OUT,
 183   NAT44_CLASSIFY_NEXT_OUT2IN,
 184   NAT44_CLASSIFY_N_NEXT,
 185 } nat44_classify_next_t;
 186
 187 void
 188 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 189 {
 190   snat_session_key_t key;
 191   clib_bihash_kv_8_8_t kv;
 192   nat_ed_ses_key_t ed_key;
 193   clib_bihash_kv_16_8_t ed_kv;
 194   snat_main_per_thread_data_t *tsm =
 195     vec_elt_at_index (sm->per_thread_data, thread_index);
 196
 197   if (is_fwd_bypass_session (s))
 198     {
 199       ed_key.l_addr = s->in2out.addr;
 200       ed_key.r_addr = s->ext_host_addr;
 201       ed_key.l_port = s->in2out.port;
 202       ed_key.r_port = s->ext_host_port;
 203       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 204       ed_key.fib_index = 0;
 205       ed_kv.key[0] = ed_key.as_u64[0];
 206       ed_kv.key[1] = ed_key.as_u64[1];
 207       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 208         nat_log_warn ("in2out_ed key del failed");
 209       return;
 210     }
 211
 212   /* session lookup tables */
 213   if (is_ed_session (s))
 214     {
 215       if (is_affinity_sessions (s))
 216         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 217                              s->in2out.protocol, s->out2in.port);
 218       ed_key.l_addr = s->out2in.addr;
 219       ed_key.r_addr = s->ext_host_addr;
 220       ed_key.fib_index = s->out2in.fib_index;
 221       if (snat_is_unk_proto_session (s))
 222         {
 223           ed_key.proto = s->in2out.port;
 224           ed_key.r_port = 0;
 225           ed_key.l_port = 0;
 226         }
 227       else
 228         {
 229           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 230           ed_key.l_port = s->out2in.port;
 231           ed_key.r_port = s->ext_host_port;
 232         }
 233       ed_kv.key[0] = ed_key.as_u64[0];
 234       ed_kv.key[1] = ed_key.as_u64[1];
 235       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 236         nat_log_warn ("out2in_ed key del failed");
 237       ed_key.l_addr = s->in2out.addr;
 238       ed_key.fib_index = s->in2out.fib_index;
 239       if (!snat_is_unk_proto_session (s))
 240         ed_key.l_port = s->in2out.port;
 241       if (is_twice_nat_session (s))
 242         {
 243           ed_key.r_addr = s->ext_host_nat_addr;
 244           ed_key.r_port = s->ext_host_nat_port;
 245         }
 246       ed_kv.key[0] = ed_key.as_u64[0];
 247       ed_kv.key[1] = ed_key.as_u64[1];
 248       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 249         nat_log_warn ("in2out_ed key del failed");
 250     }
 251   else
 252     {
 253       kv.key = s->in2out.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 255         nat_log_warn ("in2out key del failed");
 256       kv.key = s->out2in.as_u64;
 257       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 258         nat_log_warn ("out2in key del failed");
 259     }
 260
 261   if (snat_is_unk_proto_session (s))
 262     return;
 263
 264   /* log NAT event */
 265   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 266                                       s->out2in.addr.as_u32,
 267                                       s->in2out.protocol,
 268                                       s->in2out.port,
 269                                       s->out2in.port,
 270                                       s->in2out.fib_index);
 271
 272   /* Twice NAT address and port for external host */
 273   if (is_twice_nat_session (s))
 274     {
 275       key.protocol = s->in2out.protocol;
 276       key.port = s->ext_host_nat_port;
 277       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 278       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 279                                           thread_index, &key);
 280     }
 281
 282   if (snat_is_session_static (s))
 283     return;
 284
 285   if (s->outside_address_index != ~0)
 286     snat_free_outside_address_and_port (sm->addresses, thread_index,
 287                                         &s->out2in);
 288 }
 289
 290 snat_user_t *
 291 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 292                         u32 thread_index)
 293 {
 294   snat_user_t *u = 0;
 295   snat_user_key_t user_key;
 296   clib_bihash_kv_8_8_t kv, value;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   dlist_elt_t * per_user_list_head_elt;
 299
 300   user_key.addr.as_u32 = addr->as_u32;
 301   user_key.fib_index = fib_index;
 302   kv.key = user_key.as_u64;
 303
 304   /* Ever heard of the "user" = src ip4 address before? */
 305   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 306     {
 307       /* no, make a new one */
 308       pool_get (tsm->users, u);
 309       memset (u, 0, sizeof (*u));
 310       u->addr.as_u32 = addr->as_u32;
 311       u->fib_index = fib_index;
 312
 313       pool_get (tsm->list_pool, per_user_list_head_elt);
 314
 315       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 316         tsm->list_pool;
 317
 318       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 319
 320       kv.value = u - tsm->users;
 321
 322       /* add user */
 323       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 324         nat_log_warn ("user_hash keay add failed");
 325     }
 326   else
 327     {
 328       u = pool_elt_at_index (tsm->users, value.value);
 329     }
 330
 331   return u;
 332 }
 333
 334 snat_session_t *
 335 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 336 {
 337   snat_session_t *s;
 338   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 339   u32 oldest_per_user_translation_list_index, session_index;
 340   dlist_elt_t * oldest_per_user_translation_list_elt;
 341   dlist_elt_t * per_user_translation_list_elt;
 342
 343   /* Over quota? Recycle the least recently used translation */
 344   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 345     {
 346       oldest_per_user_translation_list_index =
 347         clib_dlist_remove_head (tsm->list_pool,
 348                                 u->sessions_per_user_list_head_index);
 349
 350       ASSERT (oldest_per_user_translation_list_index != ~0);
 351
 352       /* Add it back to the end of the LRU list */
 353       clib_dlist_addtail (tsm->list_pool,
 354                           u->sessions_per_user_list_head_index,
 355                           oldest_per_user_translation_list_index);
 356       /* Get the list element */
 357       oldest_per_user_translation_list_elt =
 358         pool_elt_at_index (tsm->list_pool,
 359                            oldest_per_user_translation_list_index);
 360
 361       /* Get the session index from the list element */
 362       session_index = oldest_per_user_translation_list_elt->value;
 363
 364       /* Get the session */
 365       s = pool_elt_at_index (tsm->sessions, session_index);
 366       nat_free_session_data (sm, s, thread_index);
 367       if (snat_is_session_static(s))
 368         u->nstaticsessions--;
 369       else
 370         u->nsessions--;
 371       s->outside_address_index = ~0;
 372       s->flags = 0;
 373       s->total_bytes = 0;
 374       s->total_pkts = 0;
 375       s->state = 0;
 376       s->ext_host_addr.as_u32 = 0;
 377       s->ext_host_port = 0;
 378       s->ext_host_nat_addr.as_u32 = 0;
 379       s->ext_host_nat_port = 0;
 380     }
 381   else
 382     {
 383       pool_get (tsm->sessions, s);
 384       memset (s, 0, sizeof (*s));
 385       s->outside_address_index = ~0;
 386
 387       /* Create list elts */
 388       pool_get (tsm->list_pool, per_user_translation_list_elt);
 389       clib_dlist_init (tsm->list_pool,
 390                        per_user_translation_list_elt - tsm->list_pool);
 391
 392       per_user_translation_list_elt->value = s - tsm->sessions;
 393       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 394       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 395
 396       clib_dlist_addtail (tsm->list_pool,
 397                           s->per_user_list_head_index,
 398                           per_user_translation_list_elt - tsm->list_pool);
 399     }
 400
 401   return s;
 402 }
 403
 404 snat_session_t *
 405 nat_ed_session_alloc (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 406 {
 407   snat_session_t *s;
 408   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 409   dlist_elt_t * per_user_translation_list_elt;
 410
 411   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 412     {
 413       nat_log_warn ("max translations per user %U", format_ip4_address, &u->addr);
 414       snat_ipfix_logging_max_entries_per_user (sm->max_translations_per_user,
 415                                                u->addr.as_u32);
 416       return 0;
 417     }
 418
 419   pool_get (tsm->sessions, s);
 420   memset (s, 0, sizeof (*s));
 421   s->outside_address_index = ~0;
 422
 423   /* Create list elts */
 424   pool_get (tsm->list_pool, per_user_translation_list_elt);
 425   clib_dlist_init (tsm->list_pool,
 426                    per_user_translation_list_elt - tsm->list_pool);
 427
 428   per_user_translation_list_elt->value = s - tsm->sessions;
 429   s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 430   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 431
 432   clib_dlist_addtail (tsm->list_pool,
 433                       s->per_user_list_head_index,
 434                       per_user_translation_list_elt - tsm->list_pool);
 435
 436   return s;
 437 }
 438
 439 typedef struct {
 440   u8 next_in2out;
 441 } nat44_classify_trace_t;
 442
 443 static u8 * format_nat44_classify_trace (u8 * s, va_list * args)
 444 {
 445   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 446   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 447   nat44_classify_trace_t *t = va_arg (*args, nat44_classify_trace_t *);
 448   char *next;
 449
 450   next = t->next_in2out ? "nat44-in2out" : "nat44-out2in";
 451
 452   s = format (s, "nat44-classify: next %s", next);
 453
 454   return s;
 455 }
 456
 457 static inline uword
 458 nat44_classify_node_fn_inline (vlib_main_t * vm,
 459                                vlib_node_runtime_t * node,
 460                                vlib_frame_t * frame,
 461                                int is_ed)
 462 {
 463   u32 n_left_from, * from, * to_next;
 464   nat44_classify_next_t next_index;
 465   snat_main_t *sm = &snat_main;
 466   snat_static_mapping_t *m;
 467   u32 thread_index = vm->thread_index;
 468   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 469
 470   from = vlib_frame_vector_args (frame);
 471   n_left_from = frame->n_vectors;
 472   next_index = node->cached_next_index;
 473
 474   while (n_left_from > 0)
 475     {
 476       u32 n_left_to_next;
 477
 478       vlib_get_next_frame (vm, node, next_index,
 479                            to_next, n_left_to_next);
 480
 481       while (n_left_from > 0 && n_left_to_next > 0)
 482         {
 483           u32 bi0;
 484           vlib_buffer_t *b0;
 485           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0;
 486           ip4_header_t *ip0;
 487           snat_address_t *ap;
 488           snat_session_key_t m_key0;
 489           clib_bihash_kv_8_8_t kv0, value0;
 490           clib_bihash_kv_16_8_t ed_kv0, ed_value0;
 491           udp_header_t *udp0;
 492
 493           /* speculatively enqueue b0 to the current next frame */
 494           bi0 = from[0];
 495           to_next[0] = bi0;
 496           from += 1;
 497           to_next += 1;
 498           n_left_from -= 1;
 499           n_left_to_next -= 1;
 500
 501           b0 = vlib_get_buffer (vm, bi0);
 502           ip0 = vlib_buffer_get_current (b0);
 503           udp0 = ip4_next_header (ip0);
 504
 505           if (is_ed)
 506             {
 507               sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 508               rx_fib_index0 =
 509                 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 510                                                      sw_if_index0);
 511               make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address,
 512                           ip0->protocol, rx_fib_index0, udp0->src_port,
 513                           udp0->dst_port);
 514               if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, &ed_value0))
 515                 goto enqueue0;
 516             }
 517
 518           vec_foreach (ap, sm->addresses)
 519             {
 520               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 521                 {
 522                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 523                   goto enqueue0;
 524                 }
 525             }
 526
 527           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 528             {
 529               m_key0.addr = ip0->dst_address;
 530               m_key0.port = 0;
 531               m_key0.protocol = 0;
 532               m_key0.fib_index = 0;
 533               kv0.key = m_key0.as_u64;
 534               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 535                 {
 536                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 537                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 538                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 539                   goto enqueue0;
 540                 }
 541               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 542               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 543               kv0.key = m_key0.as_u64;
 544               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 545                 {
 546                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 547                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 548                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 549                 }
 550             }
 551
 552         enqueue0:
 553           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 554                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 555             {
 556               nat44_classify_trace_t *t =
 557                   vlib_add_trace (vm, node, b0, sizeof (*t));
 558               t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
 559             }
 560
 561           /* verify speculative enqueue, maybe switch current next frame */
 562           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 563                                            to_next, n_left_to_next,
 564                                            bi0, next0);
 565         }
 566
 567       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 568     }
 569
 570   return frame->n_vectors;
 571 }
 572
 573 static uword
 574 nat44_classify_node_fn (vlib_main_t * vm,
 575                         vlib_node_runtime_t * node,
 576                         vlib_frame_t * frame)
 577 {
 578   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 579 };
 580
 581 VLIB_REGISTER_NODE (nat44_classify_node) = {
 582   .function = nat44_classify_node_fn,
 583   .name = "nat44-classify",
 584   .vector_size = sizeof (u32),
 585   .format_trace = format_nat44_classify_trace,
 586   .type = VLIB_NODE_TYPE_INTERNAL,
 587   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 588   .next_nodes = {
 589     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 590     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 591   },
 592 };
 593
 594 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 595                               nat44_classify_node_fn);
 596 static uword
 597 nat44_ed_classify_node_fn (vlib_main_t * vm,
 598                            vlib_node_runtime_t * node,
 599                            vlib_frame_t * frame)
 600 {
 601   return nat44_classify_node_fn_inline (vm, node, frame, 1);
 602 };
 603
 604 VLIB_REGISTER_NODE (nat44_ed_classify_node) = {
 605   .function = nat44_ed_classify_node_fn,
 606   .name = "nat44-ed-classify",
 607   .vector_size = sizeof (u32),
 608   .format_trace = format_nat44_classify_trace,
 609   .type = VLIB_NODE_TYPE_INTERNAL,
 610   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 611   .next_nodes = {
 612     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out",
 613     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in",
 614   },
 615 };
 616
 617 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_classify_node,
 618                               nat44_ed_classify_node_fn);
 619
 620 static uword
 621 nat44_det_classify_node_fn (vlib_main_t * vm,
 622                             vlib_node_runtime_t * node,
 623                             vlib_frame_t * frame)
 624 {
 625   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 626 };
 627
 628 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 629   .function = nat44_det_classify_node_fn,
 630   .name = "nat44-det-classify",
 631   .vector_size = sizeof (u32),
 632   .format_trace = format_nat44_classify_trace,
 633   .type = VLIB_NODE_TYPE_INTERNAL,
 634   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 635   .next_nodes = {
 636     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 637     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 638   },
 639 };
 640
 641 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 642                               nat44_det_classify_node_fn);
 643
 644 static uword
 645 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 646                                 vlib_node_runtime_t * node,
 647                                 vlib_frame_t * frame)
 648 {
 649   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 650 };
 651
 652 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 653   .function = nat44_handoff_classify_node_fn,
 654   .name = "nat44-handoff-classify",
 655   .vector_size = sizeof (u32),
 656   .format_trace = format_nat44_classify_trace,
 657   .type = VLIB_NODE_TYPE_INTERNAL,
 658   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 659   .next_nodes = {
 660     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 661     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 662   },
 663 };
 664
 665 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 666                               nat44_handoff_classify_node_fn);
 667
 668 /**
 669  * @brief Add/del NAT address to FIB.
 670  *
 671  * Add the external NAT address to the FIB as receive entries. This ensures
 672  * that VPP will reply to ARP for this address and we don't need to enable
 673  * proxy ARP on the outside interface.
 674  *
 675  * @param addr IPv4 address.
 676  * @param plen address prefix length
 677  * @param sw_if_index Interface.
 678  * @param is_add If 0 delete, otherwise add.
 679  */
 680 void
 681 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 682                           int is_add)
 683 {
 684   fib_prefix_t prefix = {
 685     .fp_len = p_len,
 686     .fp_proto = FIB_PROTOCOL_IP4,
 687     .fp_addr = {
 688         .ip4.as_u32 = addr->as_u32,
 689     },
 690   };
 691   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 692
 693   if (is_add)
 694     fib_table_entry_update_one_path(fib_index,
 695                                     &prefix,
 696                                     FIB_SOURCE_PLUGIN_LOW,
 697                                     (FIB_ENTRY_FLAG_CONNECTED |
 698                                      FIB_ENTRY_FLAG_LOCAL |
 699                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 700                                     DPO_PROTO_IP4,
 701                                     NULL,
 702                                     sw_if_index,
 703                                     ~0,
 704                                     1,
 705                                     NULL,
 706                                     FIB_ROUTE_PATH_FLAG_NONE);
 707   else
 708     fib_table_entry_delete(fib_index,
 709                            &prefix,
 710                            FIB_SOURCE_PLUGIN_LOW);
 711 }
 712
 713 int snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 714                        u8 twice_nat)
 715 {
 716   snat_address_t * ap;
 717   snat_interface_t *i;
 718   vlib_thread_main_t *tm = vlib_get_thread_main ();
 719
 720   if (twice_nat && !sm->endpoint_dependent)
 721     return VNET_API_ERROR_FEATURE_DISABLED;
 722
 723   /* Check if address already exists */
 724   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 725     {
 726       if (ap->addr.as_u32 == addr->as_u32)
 727         return VNET_API_ERROR_VALUE_EXIST;
 728     }
 729
 730   if (twice_nat)
 731     vec_add2 (sm->twice_nat_addresses, ap, 1);
 732   else
 733     vec_add2 (sm->addresses, ap, 1);
 734
 735   ap->addr = *addr;
 736   if (vrf_id != ~0)
 737     ap->fib_index =
 738       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 739                                          FIB_SOURCE_PLUGIN_LOW);
 740   else
 741     ap->fib_index = ~0;
 742 #define _(N, i, n, s) \
 743   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 744   ap->busy_##n##_ports = 0; \
 745   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 746   foreach_snat_protocol
 747 #undef _
 748
 749   if (twice_nat)
 750     return 0;
 751
 752   /* Add external address to FIB */
 753   pool_foreach (i, sm->interfaces,
 754   ({
 755     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 756       continue;
 757
 758     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 759     break;
 760   }));
 761   pool_foreach (i, sm->output_feature_interfaces,
 762   ({
 763     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 764       continue;
 765
 766     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 767     break;
 768   }));
 769
 770   return 0;
 771 }
 772
 773 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 774                                                    ip4_address_t addr)
 775 {
 776   snat_static_mapping_t *m;
 777   pool_foreach (m, sm->static_mappings,
 778   ({
 779       if (m->external_addr.as_u32 == addr.as_u32)
 780         return 1;
 781   }));
 782
 783   return 0;
 784 }
 785
 786 void increment_v4_address (ip4_address_t * a)
 787 {
 788   u32 v;
 789
 790   v = clib_net_to_host_u32(a->as_u32) + 1;
 791   a->as_u32 = clib_host_to_net_u32(v);
 792 }
 793
 794 static void
 795 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 796                                        ip4_address_t l_addr,
 797                                        u16 l_port,
 798                                        u32 sw_if_index,
 799                                        u16 e_port,
 800                                        u32 vrf_id,
 801                                        snat_protocol_t proto,
 802                                        int addr_only,
 803                                        int is_add,
 804                                        u8 * tag)
 805 {
 806   snat_static_map_resolve_t *rp;
 807
 808   vec_add2 (sm->to_resolve, rp, 1);
 809   rp->l_addr.as_u32 = l_addr.as_u32;
 810   rp->l_port = l_port;
 811   rp->sw_if_index = sw_if_index;
 812   rp->e_port = e_port;
 813   rp->vrf_id = vrf_id;
 814   rp->proto = proto;
 815   rp->addr_only = addr_only;
 816   rp->is_add = is_add;
 817   rp->tag = vec_dup (tag);
 818 }
 819
 820 /**
 821  * @brief Add static mapping.
 822  *
 823  * Create static mapping between local addr+port and external addr+port.
 824  *
 825  * @param l_addr Local IPv4 address.
 826  * @param e_addr External IPv4 address.
 827  * @param l_port Local port number.
 828  * @param e_port External port number.
 829  * @param vrf_id VRF ID.
 830  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 831  * @param sw_if_index External port instead of specific IP address.
 832  * @param is_add If 0 delete static mapping, otherwise add.
 833  * @param twice_nat If value is TWICE_NAT then translate external host address
 834  *                  and port.
 835  *                  If value is TWICE_NAT_SELF then translate external host
 836  *                  address and port whenever external host address equals
 837  *                  local address of internal host.
 838  * @param out2in_only If 1 rule match only out2in direction
 839  * @param tag - opaque string tag
 840  *
 841  * @returns
 842  */
 843 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 844                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 845                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 846                             twice_nat_type_t twice_nat, u8 out2in_only,
 847                             u8 * tag)
 848 {
 849   snat_main_t * sm = &snat_main;
 850   snat_static_mapping_t *m;
 851   snat_session_key_t m_key;
 852   clib_bihash_kv_8_8_t kv, value;
 853   snat_address_t *a = 0;
 854   u32 fib_index = ~0;
 855   uword * p;
 856   snat_interface_t *interface;
 857   int i;
 858   snat_main_per_thread_data_t *tsm;
 859   snat_user_key_t u_key;
 860   snat_user_t *u;
 861   dlist_elt_t * head, * elt;
 862   u32 elt_index, head_index;
 863   u32 ses_index;
 864   u64 user_index;
 865   snat_session_t * s;
 866   snat_static_map_resolve_t *rp, *rp_match = 0;
 867
 868   if (!sm->endpoint_dependent)
 869     {
 870       if (twice_nat || out2in_only)
 871         return VNET_API_ERROR_FEATURE_DISABLED;
 872     }
 873
 874   /* If the external address is a specific interface address */
 875   if (sw_if_index != ~0)
 876     {
 877       ip4_address_t * first_int_addr;
 878
 879       for (i = 0; i < vec_len (sm->to_resolve); i++)
 880         {
 881           rp = sm->to_resolve + i;
 882           if (rp->sw_if_index != sw_if_index ||
 883               rp->l_addr.as_u32 != l_addr.as_u32 ||
 884               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 885             continue;
 886
 887           if (!addr_only)
 888             {
 889               if (rp->l_port != l_port || rp->e_port != e_port || rp->proto != proto)
 890                 continue;
 891             }
 892
 893           rp_match = rp;
 894           break;
 895         }
 896
 897       /* Might be already set... */
 898       first_int_addr = ip4_interface_first_address
 899         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 900
 901       if (is_add)
 902         {
 903           if (rp_match)
 904             return VNET_API_ERROR_VALUE_EXIST;
 905
 906           snat_add_static_mapping_when_resolved
 907             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 908              addr_only,  is_add, tag);
 909
 910           /* DHCP resolution required? */
 911           if (first_int_addr == 0)
 912             {
 913               return 0;
 914             }
 915           else
 916             {
 917               e_addr.as_u32 = first_int_addr->as_u32;
 918               /* Identity mapping? */
 919               if (l_addr.as_u32 == 0)
 920                 l_addr.as_u32 = e_addr.as_u32;
 921             }
 922         }
 923       else
 924         {
 925           if (!rp_match)
 926             return VNET_API_ERROR_NO_SUCH_ENTRY;
 927
 928           vec_del1 (sm->to_resolve, i);
 929
 930           if (first_int_addr)
 931             {
 932               e_addr.as_u32 = first_int_addr->as_u32;
 933               /* Identity mapping? */
 934               if (l_addr.as_u32 == 0)
 935                 l_addr.as_u32 = e_addr.as_u32;
 936             }
 937           else
 938             return 0;
 939         }
 940     }
 941
 942   m_key.addr = e_addr;
 943   m_key.port = addr_only ? 0 : e_port;
 944   m_key.protocol = addr_only ? 0 : proto;
 945   m_key.fib_index = 0;
 946   kv.key = m_key.as_u64;
 947   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 948     m = 0;
 949   else
 950     m = pool_elt_at_index (sm->static_mappings, value.value);
 951
 952   if (is_add)
 953     {
 954       if (m)
 955         return VNET_API_ERROR_VALUE_EXIST;
 956
 957       if (twice_nat && addr_only)
 958         return VNET_API_ERROR_UNSUPPORTED;
 959
 960       /* Convert VRF id to FIB index */
 961       if (vrf_id != ~0)
 962         {
 963           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 964           if (!p)
 965             return VNET_API_ERROR_NO_SUCH_FIB;
 966           fib_index = p[0];
 967         }
 968       /* If not specified use inside VRF id from SNAT plugin startup config */
 969       else
 970         {
 971           fib_index = sm->inside_fib_index;
 972           vrf_id = sm->inside_vrf_id;
 973         }
 974
 975       if (!out2in_only)
 976         {
 977           m_key.addr = l_addr;
 978           m_key.port = addr_only ? 0 : l_port;
 979           m_key.protocol = addr_only ? 0 : proto;
 980           m_key.fib_index = fib_index;
 981           kv.key = m_key.as_u64;
 982           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 983             return VNET_API_ERROR_VALUE_EXIST;
 984         }
 985
 986       /* Find external address in allocated addresses and reserve port for
 987          address and port pair mapping when dynamic translations enabled */
 988       if (!(addr_only || sm->static_mapping_only || out2in_only))
 989         {
 990           for (i = 0; i < vec_len (sm->addresses); i++)
 991             {
 992               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 993                 {
 994                   a = sm->addresses + i;
 995                   /* External port must be unused */
 996                   switch (proto)
 997                     {
 998 #define _(N, j, n, s) \
 999                     case SNAT_PROTOCOL_##N: \
1000                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1001                         return VNET_API_ERROR_INVALID_VALUE; \
1002                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1003                       if (e_port > 1024) \
1004                         { \
1005                           a->busy_##n##_ports++; \
1006                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1007                         } \
1008                       break;
1009                       foreach_snat_protocol
1010 #undef _
1011                     default:
1012                       nat_log_info ("unknown protocol");
1013                       return VNET_API_ERROR_INVALID_VALUE_2;
1014                     }
1015                   break;
1016                 }
1017             }
1018           /* External address must be allocated */
1019           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1020             {
1021               if (sw_if_index != ~0)
1022                 {
1023                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1024                     {
1025                       rp = sm->to_resolve + i;
1026                       if (rp->addr_only)
1027                          continue;
1028                       if (rp->sw_if_index != sw_if_index &&
1029                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1030                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1031                           rp->e_port != e_port && rp->proto != proto)
1032                         continue;
1033
1034                       vec_del1 (sm->to_resolve, i);
1035                       break;
1036                     }
1037                 }
1038               return VNET_API_ERROR_NO_SUCH_ENTRY;
1039             }
1040         }
1041
1042       pool_get (sm->static_mappings, m);
1043       memset (m, 0, sizeof (*m));
1044       m->tag = vec_dup (tag);
1045       m->local_addr = l_addr;
1046       m->external_addr = e_addr;
1047       m->addr_only = addr_only;
1048       m->vrf_id = vrf_id;
1049       m->fib_index = fib_index;
1050       m->twice_nat = twice_nat;
1051       m->out2in_only = out2in_only;
1052       if (!addr_only)
1053         {
1054           m->local_port = l_port;
1055           m->external_port = e_port;
1056           m->proto = proto;
1057         }
1058
1059       if (sm->num_workers > 1)
1060         {
1061           ip4_header_t ip = {
1062             .src_address = m->local_addr,
1063           };
1064           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
1065           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1066         }
1067       else
1068         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1069
1070       m_key.addr = m->local_addr;
1071       m_key.port = m->local_port;
1072       m_key.protocol = m->proto;
1073       m_key.fib_index = m->fib_index;
1074       kv.key = m_key.as_u64;
1075       kv.value = m - sm->static_mappings;
1076       if (!out2in_only)
1077         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1078
1079       m_key.addr = m->external_addr;
1080       m_key.port = m->external_port;
1081       m_key.fib_index = 0;
1082       kv.key = m_key.as_u64;
1083       kv.value = m - sm->static_mappings;
1084       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
1085
1086       /* Delete dynamic sessions matching local address (+ local port) */
1087       if (!(sm->static_mapping_only))
1088         {
1089           u_key.addr = m->local_addr;
1090           u_key.fib_index = m->fib_index;
1091           kv.key = u_key.as_u64;
1092           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1093             {
1094               user_index = value.value;
1095               u = pool_elt_at_index (tsm->users, user_index);
1096               if (u->nsessions)
1097                 {
1098                   head_index = u->sessions_per_user_list_head_index;
1099                   head = pool_elt_at_index (tsm->list_pool, head_index);
1100                   elt_index = head->next;
1101                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1102                   ses_index = elt->value;
1103                   while (ses_index != ~0)
1104                     {
1105                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1106                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1107                       ses_index = elt->value;
1108
1109                       if (snat_is_session_static (s))
1110                         continue;
1111
1112                       if (!addr_only && (clib_net_to_host_u16 (s->in2out.port) != m->local_port))
1113                         continue;
1114
1115                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1116                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1117
1118                       if (!addr_only && !sm->endpoint_dependent)
1119                         break;
1120                     }
1121                 }
1122             }
1123         }
1124     }
1125   else
1126     {
1127       if (!m)
1128         {
1129           if (sw_if_index != ~0)
1130             return 0;
1131           else
1132             return VNET_API_ERROR_NO_SUCH_ENTRY;
1133         }
1134
1135       /* Free external address port */
1136       if (!(addr_only || sm->static_mapping_only || out2in_only))
1137         {
1138           for (i = 0; i < vec_len (sm->addresses); i++)
1139             {
1140               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1141                 {
1142                   a = sm->addresses + i;
1143                   switch (proto)
1144                     {
1145 #define _(N, j, n, s) \
1146                     case SNAT_PROTOCOL_##N: \
1147                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1148                       if (e_port > 1024) \
1149                         { \
1150                           a->busy_##n##_ports--; \
1151                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1152                         } \
1153                       break;
1154                       foreach_snat_protocol
1155 #undef _
1156                     default:
1157                       nat_log_info ("unknown protocol");
1158                       return VNET_API_ERROR_INVALID_VALUE_2;
1159                     }
1160                   break;
1161                 }
1162             }
1163         }
1164
1165       if (sm->num_workers > 1)
1166         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1167       else
1168         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1169
1170       m_key.addr = m->local_addr;
1171       m_key.port = m->local_port;
1172       m_key.protocol = m->proto;
1173       m_key.fib_index = m->fib_index;
1174       kv.key = m_key.as_u64;
1175       if (!out2in_only)
1176         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1177
1178       m_key.addr = m->external_addr;
1179       m_key.port = m->external_port;
1180       m_key.fib_index = 0;
1181       kv.key = m_key.as_u64;
1182       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1183
1184       /* Delete session(s) for static mapping if exist */
1185       if (!(sm->static_mapping_only) ||
1186           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1187         {
1188           u_key.addr = m->local_addr;
1189           u_key.fib_index = m->fib_index;
1190           kv.key = u_key.as_u64;
1191           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1192             {
1193               user_index = value.value;
1194               u = pool_elt_at_index (tsm->users, user_index);
1195               if (u->nstaticsessions)
1196                 {
1197                   head_index = u->sessions_per_user_list_head_index;
1198                   head = pool_elt_at_index (tsm->list_pool, head_index);
1199                   elt_index = head->next;
1200                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1201                   ses_index = elt->value;
1202                   while (ses_index != ~0)
1203                     {
1204                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1205                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1206                       ses_index = elt->value;
1207
1208                       if (!addr_only)
1209                         {
1210                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1211                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1212                             continue;
1213                         }
1214
1215                       if (is_lb_session (s))
1216                         continue;
1217
1218                       if (!snat_is_session_static (s))
1219                         continue;
1220
1221                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1222                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1223
1224                       if (!addr_only && !sm->endpoint_dependent)
1225                         break;
1226                     }
1227                 }
1228             }
1229         }
1230
1231       vec_free (m->tag);
1232       vec_free (m->workers);
1233       /* Delete static mapping from pool */
1234       pool_put (sm->static_mappings, m);
1235     }
1236
1237   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1238     return 0;
1239
1240   /* Add/delete external address to FIB */
1241   pool_foreach (interface, sm->interfaces,
1242   ({
1243     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1244       continue;
1245
1246     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1247     break;
1248   }));
1249   pool_foreach (interface, sm->output_feature_interfaces,
1250   ({
1251     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1252       continue;
1253
1254     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1255     break;
1256   }));
1257
1258   return 0;
1259 }
1260
1261 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1262                                      snat_protocol_t proto,
1263                                      nat44_lb_addr_port_t *locals, u8 is_add,
1264                                      twice_nat_type_t twice_nat, u8 out2in_only,
1265                                      u8 *tag, u32 affinity)
1266 {
1267   snat_main_t * sm = &snat_main;
1268   snat_static_mapping_t *m;
1269   snat_session_key_t m_key;
1270   clib_bihash_kv_8_8_t kv, value;
1271   snat_address_t *a = 0;
1272   int i;
1273   nat44_lb_addr_port_t *local;
1274   u32 elt_index, head_index, ses_index;
1275   snat_main_per_thread_data_t *tsm;
1276   snat_user_key_t u_key;
1277   snat_user_t *u;
1278   snat_session_t * s;
1279   dlist_elt_t * head, * elt;
1280   uword *bitmap = 0;
1281
1282   if (!sm->endpoint_dependent)
1283     return VNET_API_ERROR_FEATURE_DISABLED;
1284
1285   m_key.addr = e_addr;
1286   m_key.port = e_port;
1287   m_key.protocol = proto;
1288   m_key.fib_index = 0;
1289   kv.key = m_key.as_u64;
1290   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1291     m = 0;
1292   else
1293     m = pool_elt_at_index (sm->static_mappings, value.value);
1294
1295   if (is_add)
1296     {
1297       if (m)
1298         return VNET_API_ERROR_VALUE_EXIST;
1299
1300       if (vec_len (locals) < 2)
1301         return VNET_API_ERROR_INVALID_VALUE;
1302
1303       /* Find external address in allocated addresses and reserve port for
1304          address and port pair mapping when dynamic translations enabled */
1305       if (!(sm->static_mapping_only || out2in_only))
1306         {
1307           for (i = 0; i < vec_len (sm->addresses); i++)
1308             {
1309               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1310                 {
1311                   a = sm->addresses + i;
1312                   /* External port must be unused */
1313                   switch (proto)
1314                     {
1315 #define _(N, j, n, s) \
1316                     case SNAT_PROTOCOL_##N: \
1317                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1318                         return VNET_API_ERROR_INVALID_VALUE; \
1319                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1320                       if (e_port > 1024) \
1321                         { \
1322                           a->busy_##n##_ports++; \
1323                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1324                         } \
1325                       break;
1326                       foreach_snat_protocol
1327 #undef _
1328                     default:
1329                       nat_log_info ("unknown protocol");
1330                       return VNET_API_ERROR_INVALID_VALUE_2;
1331                     }
1332                   break;
1333                 }
1334             }
1335           /* External address must be allocated */
1336           if (!a)
1337             return VNET_API_ERROR_NO_SUCH_ENTRY;
1338         }
1339
1340       pool_get (sm->static_mappings, m);
1341       memset (m, 0, sizeof (*m));
1342       m->tag = vec_dup (tag);
1343       m->external_addr = e_addr;
1344       m->addr_only = 0;
1345       m->external_port = e_port;
1346       m->proto = proto;
1347       m->twice_nat = twice_nat;
1348       m->out2in_only = out2in_only;
1349       m->affinity = affinity;
1350
1351       if (affinity)
1352         m->affinity_per_service_list_head_index =
1353           nat_affinity_get_per_service_list_head_index();
1354       else
1355         m->affinity_per_service_list_head_index = ~0;
1356
1357       m_key.addr = m->external_addr;
1358       m_key.port = m->external_port;
1359       m_key.protocol = m->proto;
1360       m_key.fib_index = 0;
1361       kv.key = m_key.as_u64;
1362       kv.value = m - sm->static_mappings;
1363       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1364         {
1365           nat_log_err ("static_mapping_by_external key add failed");
1366           return VNET_API_ERROR_UNSPECIFIED;
1367         }
1368
1369       m_key.fib_index = m->fib_index;
1370       for (i = 0; i < vec_len (locals); i++)
1371         {
1372           locals[i].fib_index = fib_table_find_or_create_and_lock (
1373             FIB_PROTOCOL_IP4, locals[i].vrf_id, FIB_SOURCE_PLUGIN_LOW);
1374           m_key.addr = locals[i].addr;
1375           m_key.fib_index = locals[i].fib_index;
1376           if (!out2in_only)
1377             {
1378               m_key.port = locals[i].port;
1379               kv.key = m_key.as_u64;
1380               kv.value = m - sm->static_mappings;
1381               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1382             }
1383           locals[i].prefix = (i == 0) ? locals[i].probability :\
1384             (locals[i - 1].prefix + locals[i].probability);
1385           vec_add1 (m->locals, locals[i]);
1386           if (sm->num_workers > 1)
1387             {
1388               ip4_header_t ip = {
1389                 .src_address = locals[i].addr,
1390               };
1391               bitmap = clib_bitmap_set (
1392                 bitmap, sm->worker_in2out_cb (&ip, m->fib_index), 1);
1393             }
1394         }
1395
1396       /* Assign workers */
1397       if (sm->num_workers > 1)
1398         {
1399           clib_bitmap_foreach (i, bitmap,
1400             ({
1401                vec_add1(m->workers, i);
1402             }));
1403         }
1404     }
1405   else
1406     {
1407       if (!m)
1408         return VNET_API_ERROR_NO_SUCH_ENTRY;
1409
1410       /* Free external address port */
1411       if (!(sm->static_mapping_only || out2in_only))
1412         {
1413           for (i = 0; i < vec_len (sm->addresses); i++)
1414             {
1415               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1416                 {
1417                   a = sm->addresses + i;
1418                   switch (proto)
1419                     {
1420 #define _(N, j, n, s) \
1421                     case SNAT_PROTOCOL_##N: \
1422                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1423                       if (e_port > 1024) \
1424                         { \
1425                           a->busy_##n##_ports--; \
1426                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1427                         } \
1428                       break;
1429                       foreach_snat_protocol
1430 #undef _
1431                     default:
1432                       nat_log_info ("unknown protocol");
1433                       return VNET_API_ERROR_INVALID_VALUE_2;
1434                     }
1435                   break;
1436                 }
1437             }
1438         }
1439
1440       m_key.addr = m->external_addr;
1441       m_key.port = m->external_port;
1442       m_key.protocol = m->proto;
1443       m_key.fib_index = 0;
1444       kv.key = m_key.as_u64;
1445       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1446         {
1447           nat_log_err ("static_mapping_by_external key del failed");
1448           return VNET_API_ERROR_UNSPECIFIED;
1449         }
1450
1451       vec_foreach (local, m->locals)
1452         {
1453           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1454                             FIB_SOURCE_PLUGIN_LOW);
1455           m_key.addr = local->addr;
1456           if (!out2in_only)
1457             {
1458               m_key.port = local->port;
1459               m_key.fib_index = local->fib_index;
1460               kv.key = m_key.as_u64;
1461               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1462                 {
1463                   nat_log_err ("static_mapping_by_local key del failed");
1464                   return VNET_API_ERROR_UNSPECIFIED;
1465                 }
1466             }
1467
1468           if (sm->num_workers > 1)
1469             {
1470               ip4_header_t ip = {
1471                 .src_address = local->addr,
1472               };
1473               tsm = vec_elt_at_index (sm->per_thread_data,
1474                                       sm->worker_in2out_cb (&ip, m->fib_index));
1475             }
1476           else
1477             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1478
1479           /* Delete sessions */
1480           u_key.addr = local->addr;
1481           u_key.fib_index = m->fib_index;
1482           kv.key = u_key.as_u64;
1483           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1484             {
1485               u = pool_elt_at_index (tsm->users, value.value);
1486               if (u->nstaticsessions)
1487                 {
1488                   head_index = u->sessions_per_user_list_head_index;
1489                   head = pool_elt_at_index (tsm->list_pool, head_index);
1490                   elt_index = head->next;
1491                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1492                   ses_index = elt->value;
1493                   while (ses_index != ~0)
1494                     {
1495                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1496                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1497                       ses_index = elt->value;
1498
1499                       if (!(is_lb_session (s)))
1500                         continue;
1501
1502                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1503                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1504                         continue;
1505
1506                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1507                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1508                     }
1509                 }
1510             }
1511         }
1512       if (m->affinity)
1513         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1514       vec_free(m->locals);
1515       vec_free(m->tag);
1516       vec_free(m->workers);
1517
1518       pool_put (sm->static_mappings, m);
1519     }
1520
1521   return 0;
1522 }
1523
1524 int
1525 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1526                   u8 twice_nat)
1527 {
1528   snat_address_t *a = 0;
1529   snat_session_t *ses;
1530   u32 *ses_to_be_removed = 0, *ses_index;
1531   snat_main_per_thread_data_t *tsm;
1532   snat_static_mapping_t *m;
1533   snat_interface_t *interface;
1534   int i;
1535   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1536
1537   /* Find SNAT address */
1538   for (i=0; i < vec_len (addresses); i++)
1539     {
1540       if (addresses[i].addr.as_u32 == addr.as_u32)
1541         {
1542           a = addresses + i;
1543           break;
1544         }
1545     }
1546   if (!a)
1547     return VNET_API_ERROR_NO_SUCH_ENTRY;
1548
1549   if (delete_sm)
1550     {
1551       pool_foreach (m, sm->static_mappings,
1552       ({
1553           if (m->external_addr.as_u32 == addr.as_u32)
1554             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1555                                             m->local_port, m->external_port,
1556                                             m->vrf_id, m->addr_only, ~0,
1557                                             m->proto, 0, m->twice_nat,
1558                                             m->out2in_only, m->tag);
1559       }));
1560     }
1561   else
1562     {
1563       /* Check if address is used in some static mapping */
1564       if (is_snat_address_used_in_static_mapping(sm, addr))
1565         {
1566           nat_log_notice ("address used in static mapping");
1567           return VNET_API_ERROR_UNSPECIFIED;
1568         }
1569     }
1570
1571   if (a->fib_index != ~0)
1572     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1573                      FIB_SOURCE_PLUGIN_LOW);
1574
1575   /* Delete sessions using address */
1576   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1577     {
1578       vec_foreach (tsm, sm->per_thread_data)
1579         {
1580           pool_foreach (ses, tsm->sessions, ({
1581             if (ses->out2in.addr.as_u32 == addr.as_u32)
1582               {
1583                 ses->outside_address_index = ~0;
1584                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1585                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1586               }
1587           }));
1588
1589           vec_foreach (ses_index, ses_to_be_removed)
1590             {
1591               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1592               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1593             }
1594
1595           vec_free (ses_to_be_removed);
1596        }
1597     }
1598
1599   if (twice_nat)
1600     {
1601       vec_del1 (sm->twice_nat_addresses, i);
1602       return 0;
1603     }
1604   else
1605     vec_del1 (sm->addresses, i);
1606
1607   /* Delete external address from FIB */
1608   pool_foreach (interface, sm->interfaces,
1609   ({
1610     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1611       continue;
1612
1613     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1614     break;
1615   }));
1616   pool_foreach (interface, sm->output_feature_interfaces,
1617   ({
1618     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1619       continue;
1620
1621     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1622     break;
1623   }));
1624
1625   return 0;
1626 }
1627
1628 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1629 {
1630   snat_main_t *sm = &snat_main;
1631   snat_interface_t *i;
1632   const char * feature_name, *del_feature_name;
1633   snat_address_t * ap;
1634   snat_static_mapping_t * m;
1635   snat_det_map_t * dm;
1636   nat_outside_fib_t *outside_fib;
1637   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1638                                                        sw_if_index);
1639
1640   if (sm->out2in_dpo && !is_inside)
1641     return VNET_API_ERROR_UNSUPPORTED;
1642
1643   pool_foreach (i, sm->output_feature_interfaces,
1644   ({
1645     if (i->sw_if_index == sw_if_index)
1646       return VNET_API_ERROR_VALUE_EXIST;
1647   }));
1648
1649   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1650     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1651   else
1652     {
1653       if (sm->num_workers > 1 && !sm->deterministic)
1654         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1655       else if (sm->deterministic)
1656         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1657       else if (sm->endpoint_dependent)
1658         feature_name = is_inside ?  "nat44-ed-in2out" : "nat44-ed-out2in";
1659       else
1660         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1661     }
1662
1663   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1664     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1665                                                       NAT_FQ_NELTS);
1666
1667   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1668     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1669                                                       NAT_FQ_NELTS);
1670
1671   if (!is_inside)
1672     {
1673       vec_foreach (outside_fib, sm->outside_fibs)
1674         {
1675           if (outside_fib->fib_index == fib_index)
1676             {
1677               if (is_del)
1678                 {
1679                   outside_fib->refcount--;
1680                   if (!outside_fib->refcount)
1681                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1682                 }
1683               else
1684                 outside_fib->refcount++;
1685               goto feature_set;
1686             }
1687         }
1688       if (!is_del)
1689         {
1690           vec_add2 (sm->outside_fibs, outside_fib, 1);
1691           outside_fib->refcount = 1;
1692           outside_fib->fib_index = fib_index;
1693         }
1694     }
1695 feature_set:
1696   pool_foreach (i, sm->interfaces,
1697   ({
1698     if (i->sw_if_index == sw_if_index)
1699       {
1700         if (is_del)
1701           {
1702             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1703               {
1704                 if (is_inside)
1705                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1706                 else
1707                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1708
1709                 if (sm->num_workers > 1 && !sm->deterministic)
1710                   {
1711                     del_feature_name = "nat44-handoff-classify";
1712                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1713                                                  "nat44-out2in-worker-handoff";
1714                   }
1715                 else if (sm->deterministic)
1716                   {
1717                     del_feature_name = "nat44-det-classify";
1718                     feature_name = !is_inside ?  "nat44-det-in2out" :
1719                                                  "nat44-det-out2in";
1720                   }
1721                 else if (sm->endpoint_dependent)
1722                   {
1723                     del_feature_name = "nat44-ed-classify";
1724                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1725                                                  "nat44-ed-out2in";
1726                   }
1727                 else
1728                   {
1729                     del_feature_name = "nat44-classify";
1730                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1731                   }
1732
1733                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1734                                              sw_if_index, 0, 0, 0);
1735                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1736                                              sw_if_index, 1, 0, 0);
1737                 if (!is_inside)
1738                   {
1739                     if (sm->endpoint_dependent)
1740                       vnet_feature_enable_disable ("ip4-local",
1741                                                    "nat44-ed-hairpinning",
1742                                                    sw_if_index, 1, 0, 0);
1743                     else if (!sm->deterministic)
1744                       vnet_feature_enable_disable ("ip4-local",
1745                                                    "nat44-hairpinning",
1746                                                    sw_if_index, 1, 0, 0);
1747                   }
1748               }
1749             else
1750               {
1751                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1752                                              sw_if_index, 0, 0, 0);
1753                 pool_put (sm->interfaces, i);
1754                 if (is_inside)
1755                   {
1756                     if (sm->endpoint_dependent)
1757                       vnet_feature_enable_disable ("ip4-local",
1758                                                    "nat44-ed-hairpinning",
1759                                                    sw_if_index, 0, 0, 0);
1760                     else if (!sm->deterministic)
1761                       vnet_feature_enable_disable ("ip4-local",
1762                                                    "nat44-hairpinning",
1763                                                    sw_if_index, 0, 0, 0);
1764                   }
1765               }
1766           }
1767         else
1768           {
1769             if ((nat_interface_is_inside(i) && is_inside) ||
1770                 (nat_interface_is_outside(i) && !is_inside))
1771               return 0;
1772
1773             if (sm->num_workers > 1 && !sm->deterministic)
1774               {
1775                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1776                                                  "nat44-out2in-worker-handoff";
1777                 feature_name = "nat44-handoff-classify";
1778               }
1779             else if (sm->deterministic)
1780               {
1781                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1782                                                  "nat44-det-out2in";
1783                 feature_name = "nat44-det-classify";
1784               }
1785             else if (sm->endpoint_dependent)
1786               {
1787                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1788                                                  "nat44-ed-out2in";
1789                 feature_name = "nat44-ed-classify";
1790               }
1791             else
1792               {
1793                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1794                 feature_name = "nat44-classify";
1795               }
1796
1797             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1798                                          sw_if_index, 0, 0, 0);
1799             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1800                                          sw_if_index, 1, 0, 0);
1801             if (!is_inside)
1802               {
1803                 if (sm->endpoint_dependent)
1804                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1805                                                sw_if_index, 0, 0, 0);
1806                 else if (!sm->deterministic)
1807                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1808                                                sw_if_index, 0, 0, 0);
1809               }
1810             goto set_flags;
1811           }
1812
1813         goto fib;
1814       }
1815   }));
1816
1817   if (is_del)
1818     return VNET_API_ERROR_NO_SUCH_ENTRY;
1819
1820   pool_get (sm->interfaces, i);
1821   i->sw_if_index = sw_if_index;
1822   i->flags = 0;
1823   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1824
1825   if (is_inside && !sm->out2in_dpo)
1826     {
1827       if (sm->endpoint_dependent)
1828         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1829                                      sw_if_index, 1, 0, 0);
1830       else if (!sm->deterministic)
1831         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1832                                      sw_if_index, 1, 0, 0);
1833     }
1834
1835 set_flags:
1836   if (is_inside)
1837     {
1838       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1839       return 0;
1840     }
1841   else
1842     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1843
1844   /* Add/delete external addresses to FIB */
1845 fib:
1846   vec_foreach (ap, sm->addresses)
1847     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1848
1849   pool_foreach (m, sm->static_mappings,
1850   ({
1851     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1852       continue;
1853
1854     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1855   }));
1856
1857   pool_foreach (dm, sm->det_maps,
1858   ({
1859     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1860   }));
1861
1862   return 0;
1863 }
1864
1865 int snat_interface_add_del_output_feature (u32 sw_if_index,
1866                                            u8 is_inside,
1867                                            int is_del)
1868 {
1869   snat_main_t *sm = &snat_main;
1870   snat_interface_t *i;
1871   snat_address_t * ap;
1872   snat_static_mapping_t * m;
1873
1874   if (sm->deterministic ||
1875       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1876     return VNET_API_ERROR_UNSUPPORTED;
1877
1878   pool_foreach (i, sm->interfaces,
1879   ({
1880     if (i->sw_if_index == sw_if_index)
1881       return VNET_API_ERROR_VALUE_EXIST;
1882   }));
1883
1884   if (is_inside)
1885     {
1886       if (sm->endpoint_dependent)
1887         {
1888           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1889                                        sw_if_index, !is_del, 0, 0);
1890           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1891                                        sw_if_index, !is_del, 0, 0);
1892         }
1893       else
1894         {
1895           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1896                                        sw_if_index, !is_del, 0, 0);
1897           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1898                                        sw_if_index, !is_del, 0, 0);
1899         }
1900       goto fq;
1901     }
1902
1903   if (sm->num_workers > 1)
1904     {
1905       vnet_feature_enable_disable ("ip4-unicast",
1906                                    "nat44-out2in-worker-handoff",
1907                                    sw_if_index, !is_del, 0, 0);
1908       vnet_feature_enable_disable ("ip4-output",
1909                                    "nat44-in2out-output-worker-handoff",
1910                                    sw_if_index, !is_del, 0, 0);
1911     }
1912   else
1913     {
1914       if (sm->endpoint_dependent)
1915         {
1916           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
1917                                        sw_if_index, !is_del, 0, 0);
1918           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
1919                                        sw_if_index, !is_del, 0, 0);
1920         }
1921       else
1922         {
1923           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
1924                                        sw_if_index, !is_del, 0, 0);
1925           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1926                                        sw_if_index, !is_del, 0, 0);
1927         }
1928     }
1929
1930 fq:
1931   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1932     sm->fq_in2out_output_index =
1933       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1934
1935   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1936     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1937
1938   pool_foreach (i, sm->output_feature_interfaces,
1939   ({
1940     if (i->sw_if_index == sw_if_index)
1941       {
1942         if (is_del)
1943           pool_put (sm->output_feature_interfaces, i);
1944         else
1945           return VNET_API_ERROR_VALUE_EXIST;
1946
1947         goto fib;
1948       }
1949   }));
1950
1951   if (is_del)
1952     return VNET_API_ERROR_NO_SUCH_ENTRY;
1953
1954   pool_get (sm->output_feature_interfaces, i);
1955   i->sw_if_index = sw_if_index;
1956   i->flags = 0;
1957   if (is_inside)
1958     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1959   else
1960     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1961
1962   /* Add/delete external addresses to FIB */
1963 fib:
1964   if (is_inside)
1965     return 0;
1966
1967   vec_foreach (ap, sm->addresses)
1968     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1969
1970   pool_foreach (m, sm->static_mappings,
1971   ({
1972     if (!(m->addr_only)  || (m->local_addr.as_u32 == m->external_addr.as_u32))
1973       continue;
1974
1975     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1976   }));
1977
1978   return 0;
1979 }
1980
1981 int snat_set_workers (uword * bitmap)
1982 {
1983   snat_main_t *sm = &snat_main;
1984   int i, j = 0;
1985
1986   if (sm->num_workers < 2)
1987     return VNET_API_ERROR_FEATURE_DISABLED;
1988
1989   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1990     return VNET_API_ERROR_INVALID_WORKER;
1991
1992   vec_free (sm->workers);
1993   clib_bitmap_foreach (i, bitmap,
1994     ({
1995       vec_add1(sm->workers, i);
1996       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1997       j++;
1998     }));
1999
2000   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2001   sm->num_snat_thread = _vec_len (sm->workers);
2002
2003   return 0;
2004 }
2005
2006
2007 static void
2008 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2009                                        uword opaque,
2010                                        u32 sw_if_index,
2011                                        ip4_address_t * address,
2012                                        u32 address_length,
2013                                        u32 if_address_index,
2014                                        u32 is_delete);
2015
2016 static void
2017 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2018                                  uword opaque,
2019                                  u32 sw_if_index,
2020                                  ip4_address_t * address,
2021                                  u32 address_length,
2022                                  u32 if_address_index,
2023                                  u32 is_delete);
2024
2025 static int
2026 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2027                                  u32 fib_index,
2028                                  u32 thread_index,
2029                                  snat_session_key_t * k,
2030                                  u32 * address_indexp,
2031                                  u16 port_per_thread,
2032                                  u32 snat_thread_index);
2033
2034 static clib_error_t * snat_init (vlib_main_t * vm)
2035 {
2036   snat_main_t * sm = &snat_main;
2037   clib_error_t * error = 0;
2038   ip4_main_t * im = &ip4_main;
2039   ip_lookup_main_t * lm = &im->lookup_main;
2040   uword *p;
2041   vlib_thread_registration_t *tr;
2042   vlib_thread_main_t *tm = vlib_get_thread_main ();
2043   uword *bitmap = 0;
2044   u32 i;
2045   ip4_add_del_interface_address_callback_t cb4;
2046   vlib_node_t * error_drop_node;
2047
2048   sm->vlib_main = vm;
2049   sm->vnet_main = vnet_get_main();
2050   sm->ip4_main = im;
2051   sm->ip4_lookup_main = lm;
2052   sm->api_main = &api_main;
2053   sm->first_worker_index = 0;
2054   sm->num_workers = 0;
2055   sm->num_snat_thread = 1;
2056   sm->workers = 0;
2057   sm->port_per_thread = 0xffff - 1024;
2058   sm->fq_in2out_index = ~0;
2059   sm->fq_out2in_index = ~0;
2060   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2061   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2062   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2063   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2064   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2065   sm->forwarding_enabled = 0;
2066   sm->log_class = vlib_log_register_class ("nat", 0);
2067   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2068   sm->error_node_index = error_drop_node->index;
2069
2070   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2071   if (p)
2072     {
2073       tr = (vlib_thread_registration_t *) p[0];
2074       if (tr)
2075         {
2076           sm->num_workers = tr->count;
2077           sm->first_worker_index = tr->first_index;
2078         }
2079     }
2080
2081   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2082
2083   /* Use all available workers by default */
2084   if (sm->num_workers > 1)
2085     {
2086       for (i=0; i < sm->num_workers; i++)
2087         bitmap = clib_bitmap_set (bitmap, i, 1);
2088       snat_set_workers(bitmap);
2089       clib_bitmap_free (bitmap);
2090     }
2091   else
2092     {
2093       sm->per_thread_data[0].snat_thread_index = 0;
2094     }
2095
2096   error = snat_api_init(vm, sm);
2097   if (error)
2098     return error;
2099
2100   /* Set up the interface address add/del callback */
2101   cb4.function = snat_ip4_add_del_interface_address_cb;
2102   cb4.function_opaque = 0;
2103
2104   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2105
2106   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2107   cb4.function_opaque = 0;
2108
2109   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2110
2111   nat_dpo_module_init ();
2112
2113   /* Init IPFIX logging */
2114   snat_ipfix_logging_init(vm);
2115
2116   /* Init NAT64 */
2117   error = nat64_init(vm);
2118   if (error)
2119     return error;
2120
2121   dslite_init(vm);
2122
2123   nat66_init();
2124
2125   /* Init virtual fragmenentation reassembly */
2126   return nat_reass_init(vm);
2127 }
2128
2129 VLIB_INIT_FUNCTION (snat_init);
2130
2131 void snat_free_outside_address_and_port (snat_address_t * addresses,
2132                                          u32 thread_index,
2133                                          snat_session_key_t * k)
2134 {
2135   snat_address_t *a;
2136   u32 address_index;
2137   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2138
2139   for (address_index = 0; address_index < vec_len (addresses); address_index++)
2140     {
2141       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2142         break;
2143     }
2144
2145   ASSERT (address_index < vec_len (addresses));
2146
2147   a = addresses + address_index;
2148
2149   switch (k->protocol)
2150     {
2151 #define _(N, i, n, s) \
2152     case SNAT_PROTOCOL_##N: \
2153       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2154         port_host_byte_order) == 1); \
2155       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2156         port_host_byte_order, 0); \
2157       a->busy_##n##_ports--; \
2158       a->busy_##n##_ports_per_thread[thread_index]--; \
2159       break;
2160       foreach_snat_protocol
2161 #undef _
2162     default:
2163       nat_log_info ("unknown protocol");
2164       return;
2165     }
2166 }
2167
2168 /**
2169  * @brief Match NAT44 static mapping.
2170  *
2171  * @param sm          NAT main.
2172  * @param match       Address and port to match.
2173  * @param mapping     External or local address and port of the matched mapping.
2174  * @param by_external If 0 match by local address otherwise match by external
2175  *                    address.
2176  * @param is_addr_only If matched mapping is address only
2177  * @param twice_nat If matched mapping is twice NAT.
2178  * @param lb If matched mapping is load-balanced.
2179  *
2180  * @returns 0 if match found otherwise 1.
2181  */
2182 int snat_static_mapping_match (snat_main_t * sm,
2183                                snat_session_key_t match,
2184                                snat_session_key_t * mapping,
2185                                u8 by_external,
2186                                u8 *is_addr_only,
2187                                twice_nat_type_t *twice_nat,
2188                                lb_nat_type_t *lb,
2189                                ip4_address_t * ext_host_addr)
2190 {
2191   clib_bihash_kv_8_8_t kv, value;
2192   snat_static_mapping_t *m;
2193   snat_session_key_t m_key;
2194   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2195   u32 rand, lo = 0, hi, mid;
2196   u8 backend_index;
2197
2198   m_key.fib_index = match.fib_index;
2199   if (by_external)
2200     {
2201       mapping_hash = &sm->static_mapping_by_external;
2202       m_key.fib_index = 0;
2203     }
2204
2205   m_key.addr = match.addr;
2206   m_key.port = clib_net_to_host_u16 (match.port);
2207   m_key.protocol = match.protocol;
2208
2209   kv.key = m_key.as_u64;
2210
2211   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2212     {
2213       /* Try address only mapping */
2214       m_key.port = 0;
2215       m_key.protocol = 0;
2216       kv.key = m_key.as_u64;
2217       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2218         return 1;
2219     }
2220
2221   m = pool_elt_at_index (sm->static_mappings, value.value);
2222
2223   if (by_external)
2224     {
2225       if (vec_len (m->locals))
2226         {
2227           if (PREDICT_FALSE(lb != 0))
2228             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2229           if (m->affinity)
2230             {
2231               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2232                   match.protocol, match.port, &backend_index))
2233                 goto get_local;
2234
2235               mapping->addr = m->locals[backend_index].addr;
2236               mapping->port = clib_host_to_net_u16 (m->locals[backend_index].port);
2237               mapping->fib_index = m->locals[backend_index].fib_index;
2238               goto end;
2239             }
2240 get_local:
2241           hi = vec_len (m->locals) - 1;
2242           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2243           while (lo < hi)
2244             {
2245               mid = ((hi - lo) >> 1) + lo;
2246               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2247             }
2248           if (!(m->locals[lo].prefix >= rand))
2249             return 1;
2250           if (PREDICT_FALSE (sm->num_workers > 1))
2251             {
2252               ip4_header_t ip = {
2253                 .src_address = m->locals[lo].addr,
2254               };
2255               if (sm->worker_in2out_cb (&ip, m->fib_index) != vlib_get_thread_index ())
2256                 goto get_local;
2257             }
2258           mapping->addr = m->locals[lo].addr;
2259           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2260           mapping->fib_index = m->locals[lo].fib_index;
2261           if (m->affinity)
2262             {
2263               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2264                   match.protocol, match.port, lo, m->affinity,
2265                   m->affinity_per_service_list_head_index))
2266                 nat_log_info ("create affinity record failed");
2267             }
2268         }
2269       else
2270         {
2271           if (PREDICT_FALSE(lb != 0))
2272             *lb = NO_LB_NAT;
2273           mapping->fib_index = m->fib_index;
2274           mapping->addr = m->local_addr;
2275           /* Address only mapping doesn't change port */
2276           mapping->port = m->addr_only ? match.port
2277             : clib_host_to_net_u16 (m->local_port);
2278         }
2279       mapping->protocol = m->proto;
2280     }
2281   else
2282     {
2283       mapping->addr = m->external_addr;
2284       /* Address only mapping doesn't change port */
2285       mapping->port = m->addr_only ? match.port
2286         : clib_host_to_net_u16 (m->external_port);
2287       mapping->fib_index = sm->outside_fib_index;
2288     }
2289
2290 end:
2291   if (PREDICT_FALSE(is_addr_only != 0))
2292     *is_addr_only = m->addr_only;
2293
2294   if (PREDICT_FALSE(twice_nat != 0))
2295     *twice_nat = m->twice_nat;
2296
2297   return 0;
2298 }
2299
2300 static_always_inline u16
2301 snat_random_port (u16 min, u16 max)
2302 {
2303   snat_main_t *sm = &snat_main;
2304   return min + random_u32 (&sm->random_seed) /
2305     (random_u32_max() / (max - min + 1) + 1);
2306 }
2307
2308 int
2309 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2310                                      u32 fib_index,
2311                                      u32 thread_index,
2312                                      snat_session_key_t * k,
2313                                      u32 * address_indexp,
2314                                      u16 port_per_thread,
2315                                      u32 snat_thread_index)
2316 {
2317   snat_main_t *sm = &snat_main;
2318
2319   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2320                                  address_indexp, port_per_thread,
2321                                  snat_thread_index);
2322 }
2323
2324 static int
2325 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2326                                  u32 fib_index,
2327                                  u32 thread_index,
2328                                  snat_session_key_t * k,
2329                                  u32 * address_indexp,
2330                                  u16 port_per_thread,
2331                                  u32 snat_thread_index)
2332 {
2333   int i, gi = 0;
2334   snat_address_t *a, *ga = 0;
2335   u32 portnum;
2336
2337   for (i = 0; i < vec_len (addresses); i++)
2338     {
2339       a = addresses + i;
2340       switch (k->protocol)
2341         {
2342 #define _(N, j, n, s) \
2343         case SNAT_PROTOCOL_##N: \
2344           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2345             { \
2346               if (a->fib_index == fib_index) \
2347                 { \
2348                   while (1) \
2349                     { \
2350                       portnum = (port_per_thread * \
2351                         snat_thread_index) + \
2352                         snat_random_port(1, port_per_thread) + 1024; \
2353                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2354                         continue; \
2355                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2356                       a->busy_##n##_ports_per_thread[thread_index]++; \
2357                       a->busy_##n##_ports++; \
2358                       k->addr = a->addr; \
2359                       k->port = clib_host_to_net_u16(portnum); \
2360                       *address_indexp = i; \
2361                       return 0; \
2362                     } \
2363                 } \
2364               else if (a->fib_index == ~0) \
2365                 { \
2366                   ga = a; \
2367                   gi = i; \
2368                 } \
2369             } \
2370           break;
2371           foreach_snat_protocol
2372 #undef _
2373         default:
2374           nat_log_info ("unknown protocol");
2375           return 1;
2376         }
2377
2378     }
2379
2380   if (ga)
2381     {
2382       a = ga;
2383       switch (k->protocol)
2384         {
2385 #define _(N, j, n, s) \
2386         case SNAT_PROTOCOL_##N: \
2387           while (1) \
2388             { \
2389               portnum = (port_per_thread * \
2390                 snat_thread_index) + \
2391                 snat_random_port(1, port_per_thread) + 1024; \
2392               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2393                 continue; \
2394               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2395               a->busy_##n##_ports_per_thread[thread_index]++; \
2396               a->busy_##n##_ports++; \
2397               k->addr = a->addr; \
2398               k->port = clib_host_to_net_u16(portnum); \
2399               *address_indexp = gi; \
2400               return 0; \
2401             }
2402           break;
2403           foreach_snat_protocol
2404 #undef _
2405         default:
2406           nat_log_info ("unknown protocol");
2407           return 1;
2408         }
2409     }
2410
2411   /* Totally out of translations to use... */
2412   snat_ipfix_logging_addresses_exhausted(0);
2413   return 1;
2414 }
2415
2416 static int
2417 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2418                               u32 fib_index,
2419                               u32 thread_index,
2420                               snat_session_key_t * k,
2421                               u32 * address_indexp,
2422                               u16 port_per_thread,
2423                               u32 snat_thread_index)
2424 {
2425   snat_main_t *sm = &snat_main;
2426   snat_address_t *a = addresses;
2427   u16 m, ports, portnum, A, j;
2428   m = 16 - (sm->psid_offset + sm->psid_length);
2429   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2430
2431   if (!vec_len (addresses))
2432     goto exhausted;
2433
2434   switch (k->protocol)
2435     {
2436 #define _(N, i, n, s) \
2437     case SNAT_PROTOCOL_##N: \
2438       if (a->busy_##n##_ports < ports) \
2439         { \
2440           while (1) \
2441             { \
2442               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2443               j = snat_random_port(0, pow2_mask(m)); \
2444               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2445               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2446                 continue; \
2447               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2448               a->busy_##n##_ports++; \
2449               k->addr = a->addr; \
2450               k->port = clib_host_to_net_u16 (portnum); \
2451               *address_indexp = i; \
2452               return 0; \
2453             } \
2454         } \
2455       break;
2456       foreach_snat_protocol
2457 #undef _
2458     default:
2459       nat_log_info ("unknown protocol");
2460       return 1;
2461     }
2462
2463 exhausted:
2464   /* Totally out of translations to use... */
2465   snat_ipfix_logging_addresses_exhausted(0);
2466   return 1;
2467 }
2468
2469 void
2470 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2471 {
2472   dpo_id_t dpo_v4 = DPO_INVALID;
2473   fib_prefix_t pfx = {
2474     .fp_proto = FIB_PROTOCOL_IP4,
2475     .fp_len = 32,
2476     .fp_addr.ip4.as_u32 = addr.as_u32,
2477   };
2478
2479   if (is_add)
2480     {
2481       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2482       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2483                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2484       dpo_reset (&dpo_v4);
2485     }
2486   else
2487     {
2488       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2489     }
2490 }
2491
2492 uword
2493 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2494 {
2495   u32 *r = va_arg (*args, u32 *);
2496
2497   if (0);
2498 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2499   foreach_snat_protocol
2500 #undef _
2501   else
2502     return 0;
2503   return 1;
2504 }
2505
2506 u8 *
2507 format_snat_protocol (u8 * s, va_list * args)
2508 {
2509   u32 i = va_arg (*args, u32);
2510   u8 *t = 0;
2511
2512   switch (i)
2513     {
2514 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2515       foreach_snat_protocol
2516 #undef _
2517     default:
2518       s = format (s, "unknown");
2519       return s;
2520     }
2521   s = format (s, "%s", t);
2522   return s;
2523 }
2524
2525 u8 * format_snat_key (u8 * s, va_list * args);
2526 u8 * format_static_mapping_key (u8 * s, va_list * args);
2527
2528 u8 *
2529 format_session_kvp (u8 * s, va_list * args)
2530 {
2531   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2532   snat_session_key_t k;
2533
2534   k.as_u64 = v->key;
2535
2536   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2537
2538   return s;
2539 }
2540
2541 u8 *
2542 format_static_mapping_kvp (u8 * s, va_list * args)
2543 {
2544   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2545   snat_session_key_t k;
2546
2547   k.as_u64 = v->key;
2548
2549   s = format (s, "%U static-mapping-index %llu",
2550               format_static_mapping_key, &k, v->value);
2551
2552   return s;
2553 }
2554
2555 u8 *
2556 format_user_kvp (u8 * s, va_list * args)
2557 {
2558   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2559   snat_user_key_t k;
2560
2561   k.as_u64 = v->key;
2562
2563   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2564               k.fib_index, v->value);
2565
2566   return s;
2567 }
2568
2569 u8 *
2570 format_ed_session_kvp (u8 * s, va_list * args)
2571 {
2572   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2573   nat_ed_ses_key_t k;
2574
2575   k.as_u64[0] = v->key[0];
2576   k.as_u64[1] = v->key[1];
2577
2578   s = format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2579               format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2580               format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2581               format_ip_protocol, k.proto, k.fib_index, v->value);
2582
2583   return s;
2584 }
2585
2586 static u32
2587 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2588 {
2589   snat_main_t *sm = &snat_main;
2590   u32 next_worker_index = 0;
2591   u32 hash;
2592
2593   next_worker_index = sm->first_worker_index;
2594   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2595          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2596
2597   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2598     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2599   else
2600     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2601
2602   return next_worker_index;
2603 }
2604
2605 static u32
2606 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2607 {
2608   snat_main_t *sm = &snat_main;
2609   udp_header_t *udp;
2610   u16 port;
2611   snat_session_key_t m_key;
2612   clib_bihash_kv_8_8_t kv, value;
2613   snat_static_mapping_t *m;
2614   u32 proto;
2615   u32 next_worker_index = 0;
2616
2617   /* first try static mappings without port */
2618   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2619     {
2620       m_key.addr = ip0->dst_address;
2621       m_key.port = 0;
2622       m_key.protocol = 0;
2623       m_key.fib_index = rx_fib_index0;
2624       kv.key = m_key.as_u64;
2625       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2626         {
2627           m = pool_elt_at_index (sm->static_mappings, value.value);
2628           return m->workers[0];
2629         }
2630     }
2631
2632   proto = ip_proto_to_snat_proto (ip0->protocol);
2633   udp = ip4_next_header (ip0);
2634   port = udp->dst_port;
2635
2636   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2637     {
2638       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2639         return vlib_get_thread_index ();
2640
2641       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2642         {
2643           nat_reass_ip4_t *reass;
2644
2645           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2646                                       ip0->fragment_id, ip0->protocol);
2647
2648           if (reass && (reass->thread_index != (u32) ~ 0))
2649             return reass->thread_index;
2650           else
2651             return vlib_get_thread_index ();
2652         }
2653     }
2654
2655   /* unknown protocol */
2656   if (PREDICT_FALSE (proto == ~0))
2657     {
2658       /* use current thread */
2659       return vlib_get_thread_index ();
2660     }
2661
2662   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2663     {
2664       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2665       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2666       if (!icmp_is_error_message (icmp))
2667         port = echo->identifier;
2668       else
2669         {
2670           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2671           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2672           void *l4_header = ip4_next_header (inner_ip);
2673           switch (proto)
2674             {
2675             case SNAT_PROTOCOL_ICMP:
2676               icmp = (icmp46_header_t*)l4_header;
2677               echo = (icmp_echo_header_t *)(icmp + 1);
2678               port = echo->identifier;
2679               break;
2680             case SNAT_PROTOCOL_UDP:
2681             case SNAT_PROTOCOL_TCP:
2682               port = ((tcp_udp_header_t*)l4_header)->src_port;
2683               break;
2684             default:
2685               return vlib_get_thread_index ();
2686             }
2687         }
2688     }
2689
2690   /* try static mappings with port */
2691   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2692     {
2693       m_key.addr = ip0->dst_address;
2694       m_key.port = clib_net_to_host_u16 (port);
2695       m_key.protocol = proto;
2696       m_key.fib_index = rx_fib_index0;
2697       kv.key = m_key.as_u64;
2698       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2699         {
2700           m = pool_elt_at_index (sm->static_mappings, value.value);
2701           return m->workers[0];
2702         }
2703     }
2704
2705   /* worker by outside port */
2706   next_worker_index = sm->first_worker_index;
2707   next_worker_index +=
2708     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2709   return next_worker_index;
2710 }
2711
2712 static u32
2713 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2714 {
2715   snat_main_t *sm = &snat_main;
2716   clib_bihash_kv_8_8_t kv, value;
2717   u32 proto, next_worker_index = 0;
2718   udp_header_t *udp;
2719   u16 port;
2720   snat_static_mapping_t *m;
2721   u32 hash;
2722
2723   /* first try static mappings without port */
2724   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2725     {
2726       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2727       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2728         {
2729           m = pool_elt_at_index (sm->static_mappings, value.value);
2730           return m->workers[0];
2731         }
2732     }
2733
2734   proto = ip_proto_to_snat_proto (ip->protocol);
2735
2736   /* unknown protocol */
2737   if (PREDICT_FALSE (proto == ~0))
2738     {
2739       /* use current thread */
2740       return vlib_get_thread_index ();
2741     }
2742
2743   udp = ip4_next_header (ip);
2744   port = udp->dst_port;
2745
2746   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2747     {
2748       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2749       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2750       if (!icmp_is_error_message (icmp))
2751         port = echo->identifier;
2752       else
2753         {
2754           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2755           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2756           void *l4_header = ip4_next_header (inner_ip);
2757           switch (proto)
2758             {
2759             case SNAT_PROTOCOL_ICMP:
2760               icmp = (icmp46_header_t*)l4_header;
2761               echo = (icmp_echo_header_t *)(icmp + 1);
2762               port = echo->identifier;
2763               break;
2764             case SNAT_PROTOCOL_UDP:
2765             case SNAT_PROTOCOL_TCP:
2766               port = ((tcp_udp_header_t*)l4_header)->src_port;
2767               break;
2768             default:
2769               return vlib_get_thread_index ();
2770             }
2771         }
2772     }
2773
2774   /* try static mappings with port */
2775   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2776     {
2777       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2778                   clib_net_to_host_u16 (port));
2779       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2780         {
2781           m = pool_elt_at_index (sm->static_mappings, value.value);
2782           if (!vec_len(m->locals))
2783             return m->workers[0];
2784
2785           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2786                  (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >>24);
2787
2788           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2789             return m->workers[hash & (_vec_len (m->workers) - 1)];
2790           else
2791             return m->workers[hash % _vec_len (m->workers)];
2792         }
2793     }
2794
2795   /* worker by outside port */
2796   next_worker_index = sm->first_worker_index;
2797   next_worker_index +=
2798     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2799
2800   return next_worker_index;
2801 }
2802
2803 static clib_error_t *
2804 snat_config (vlib_main_t * vm, unformat_input_t * input)
2805 {
2806   snat_main_t * sm = &snat_main;
2807   nat66_main_t * nm = &nat66_main;
2808   u32 translation_buckets = 1024;
2809   u32 translation_memory_size = 128<<20;
2810   u32 user_buckets = 128;
2811   u32 user_memory_size = 64<<20;
2812   u32 max_translations_per_user = 100;
2813   u32 outside_vrf_id = 0;
2814   u32 outside_ip6_vrf_id = 0;
2815   u32 inside_vrf_id = 0;
2816   u32 static_mapping_buckets = 1024;
2817   u32 static_mapping_memory_size = 64<<20;
2818   u32 nat64_bib_buckets = 1024;
2819   u32 nat64_bib_memory_size = 128 << 20;
2820   u32 nat64_st_buckets = 2048;
2821   u32 nat64_st_memory_size = 256 << 20;
2822   u8 static_mapping_only = 0;
2823   u8 static_mapping_connection_tracking = 0;
2824   snat_main_per_thread_data_t *tsm;
2825   dslite_main_t * dm = &dslite_main;
2826
2827   sm->deterministic = 0;
2828   sm->out2in_dpo = 0;
2829   sm->endpoint_dependent = 0;
2830
2831   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2832     {
2833       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2834         ;
2835       else if (unformat (input, "translation hash memory %d",
2836                          &translation_memory_size));
2837       else if (unformat (input, "user hash buckets %d", &user_buckets))
2838         ;
2839       else if (unformat (input, "user hash memory %d",
2840                          &user_memory_size))
2841         ;
2842       else if (unformat (input, "max translations per user %d",
2843                          &max_translations_per_user))
2844         ;
2845       else if (unformat (input, "outside VRF id %d",
2846                          &outside_vrf_id))
2847         ;
2848       else if (unformat (input, "outside ip6 VRF id %d",
2849                          &outside_ip6_vrf_id))
2850         ;
2851       else if (unformat (input, "inside VRF id %d",
2852                          &inside_vrf_id))
2853         ;
2854       else if (unformat (input, "static mapping only"))
2855         {
2856           static_mapping_only = 1;
2857           if (unformat (input, "connection tracking"))
2858             static_mapping_connection_tracking = 1;
2859         }
2860       else if (unformat (input, "deterministic"))
2861         sm->deterministic = 1;
2862       else if (unformat (input, "nat64 bib hash buckets %d",
2863                          &nat64_bib_buckets))
2864         ;
2865       else if (unformat (input, "nat64 bib hash memory %d",
2866                          &nat64_bib_memory_size))
2867         ;
2868       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2869         ;
2870       else if (unformat (input, "nat64 st hash memory %d",
2871                          &nat64_st_memory_size))
2872         ;
2873       else if (unformat (input, "out2in dpo"))
2874         sm->out2in_dpo = 1;
2875       else if (unformat (input, "dslite ce"))
2876         dslite_set_ce(dm, 1);
2877       else if (unformat (input, "endpoint-dependent"))
2878         sm->endpoint_dependent = 1;
2879       else
2880         return clib_error_return (0, "unknown input '%U'",
2881                                   format_unformat_error, input);
2882     }
2883
2884   if (sm->deterministic && sm->endpoint_dependent)
2885     return clib_error_return (
2886       0, "deterministic and endpoint-dependent modes are mutually exclusive");
2887
2888   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
2889     return clib_error_return (
2890       0, "static mapping only mode available only for simple nat");
2891
2892   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
2893     return clib_error_return (
2894       0, "out2in dpo mode available only for simple nat");
2895
2896   /* for show commands, etc. */
2897   sm->translation_buckets = translation_buckets;
2898   sm->translation_memory_size = translation_memory_size;
2899   /* do not exceed load factor 10 */
2900   sm->max_translations = 10 * translation_buckets;
2901   sm->user_buckets = user_buckets;
2902   sm->user_memory_size = user_memory_size;
2903   sm->max_translations_per_user = max_translations_per_user;
2904   sm->outside_vrf_id = outside_vrf_id;
2905   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2906                                                              outside_vrf_id,
2907                                                              FIB_SOURCE_PLUGIN_HI);
2908   nm->outside_vrf_id = outside_ip6_vrf_id;
2909   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2910                                                              outside_ip6_vrf_id,
2911                                                              FIB_SOURCE_PLUGIN_HI);
2912   sm->inside_vrf_id = inside_vrf_id;
2913   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2914                                                             inside_vrf_id,
2915                                                             FIB_SOURCE_PLUGIN_HI);
2916   sm->static_mapping_only = static_mapping_only;
2917   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
2918
2919   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
2920                  nat64_st_memory_size);
2921
2922   if (sm->deterministic)
2923     {
2924       sm->in2out_node_index = snat_det_in2out_node.index;
2925       sm->in2out_output_node_index = ~0;
2926       sm->out2in_node_index = snat_det_out2in_node.index;
2927       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
2928       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
2929     }
2930   else
2931     {
2932       if (sm->endpoint_dependent)
2933         {
2934           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2935           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2936           sm->in2out_node_index = nat44_ed_in2out_node.index;
2937           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
2938           sm->out2in_node_index = nat44_ed_out2in_node.index;
2939           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
2940           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
2941           nat_affinity_init (vm);
2942         }
2943       else
2944         {
2945           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2946           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2947           sm->in2out_node_index = snat_in2out_node.index;
2948           sm->in2out_output_node_index = snat_in2out_output_node.index;
2949           sm->out2in_node_index = snat_out2in_node.index;
2950           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2951           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2952         }
2953       if (!static_mapping_only ||
2954           (static_mapping_only && static_mapping_connection_tracking))
2955         {
2956           vec_foreach (tsm, sm->per_thread_data)
2957             {
2958               if (sm->endpoint_dependent)
2959                 {
2960                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
2961                                          translation_buckets,
2962                                          translation_memory_size);
2963                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
2964                                                       format_ed_session_kvp);
2965
2966                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
2967                                          translation_buckets,
2968                                          translation_memory_size);
2969                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
2970                                                       format_ed_session_kvp);
2971                 }
2972               else
2973                 {
2974                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
2975                                         translation_buckets,
2976                                         translation_memory_size);
2977                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
2978                                                      format_session_kvp);
2979
2980                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
2981                                         translation_buckets,
2982                                         translation_memory_size);
2983                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
2984                                                      format_session_kvp);
2985                 }
2986
2987               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
2988                                     user_memory_size);
2989               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
2990                                                  format_user_kvp);
2991             }
2992
2993         }
2994       else
2995         {
2996           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2997           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2998         }
2999       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3000                             "static_mapping_by_local", static_mapping_buckets,
3001                             static_mapping_memory_size);
3002       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3003                                          format_static_mapping_kvp);
3004
3005       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3006                             "static_mapping_by_external", static_mapping_buckets,
3007                             static_mapping_memory_size);
3008       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3009                                          format_static_mapping_kvp);
3010     }
3011
3012   return 0;
3013 }
3014
3015 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3016
3017 u8 * format_snat_session_state (u8 * s, va_list * args)
3018 {
3019   u32 i = va_arg (*args, u32);
3020   u8 *t = 0;
3021
3022   switch (i)
3023     {
3024 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
3025     foreach_snat_session_state
3026 #undef _
3027     default:
3028       t = format (t, "unknown");
3029     }
3030   s = format (s, "%s", t);
3031   return s;
3032 }
3033
3034 u8 * format_snat_key (u8 * s, va_list * args)
3035 {
3036   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3037
3038   s = format (s, "%U proto %U port %d fib %d",
3039               format_ip4_address, &key->addr,
3040               format_snat_protocol, key->protocol,
3041               clib_net_to_host_u16 (key->port), key->fib_index);
3042   return s;
3043 }
3044
3045 u8 * format_static_mapping_key (u8 * s, va_list * args)
3046 {
3047   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3048
3049   s = format (s, "%U proto %U port %d fib %d",
3050               format_ip4_address, &key->addr,
3051               format_snat_protocol, key->protocol,
3052               key->port, key->fib_index);
3053   return s;
3054 }
3055
3056 u8 * format_snat_session (u8 * s, va_list * args)
3057 {
3058   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3059   snat_session_t * sess = va_arg (*args, snat_session_t *);
3060
3061   if (snat_is_unk_proto_session (sess))
3062     {
3063       s = format (s, "  i2o %U proto %u fib %u\n",
3064                   format_ip4_address, &sess->in2out.addr,
3065                   clib_net_to_host_u16 (sess->in2out.port),
3066                   sess->in2out.fib_index);
3067       s = format (s, "    o2i %U proto %u fib %u\n",
3068                   format_ip4_address, &sess->out2in.addr,
3069                   clib_net_to_host_u16 (sess->out2in.port),
3070                   sess->out2in.fib_index);
3071     }
3072   else
3073     {
3074       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
3075       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
3076     }
3077   if (is_ed_session (sess) || is_fwd_bypass_session (sess))
3078     {
3079       if (is_twice_nat_session (sess))
3080         {
3081           s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
3082                       format_ip4_address, &sess->ext_host_addr,
3083                       clib_net_to_host_u16 (sess->ext_host_port),
3084                       format_ip4_address, &sess->ext_host_nat_addr,
3085                       clib_net_to_host_u16 (sess->ext_host_nat_port));
3086         }
3087       else
3088         {
3089           if (sess->ext_host_addr.as_u32)
3090               s = format (s, "       external host %U:%u\n",
3091                           format_ip4_address, &sess->ext_host_addr,
3092                           clib_net_to_host_u16 (sess->ext_host_port));
3093         }
3094     }
3095   s = format (s, "       index %llu\n", sess - sm->sessions);
3096   s = format (s, "       last heard %.2f\n", sess->last_heard);
3097   s = format (s, "       total pkts %d, total bytes %lld\n",
3098               sess->total_pkts, sess->total_bytes);
3099   if (snat_is_session_static (sess))
3100     s = format (s, "       static translation\n");
3101   else
3102     s = format (s, "       dynamic translation\n");
3103   if (is_fwd_bypass_session (sess))
3104     s = format (s, "       forwarding-bypass\n");
3105   if (is_lb_session (sess))
3106     s = format (s, "       load-balancing\n");
3107   if (is_twice_nat_session (sess))
3108     s = format (s, "       twice-nat\n");
3109
3110   return s;
3111 }
3112
3113 u8 * format_snat_user (u8 * s, va_list * args)
3114 {
3115   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3116   snat_user_t * u = va_arg (*args, snat_user_t *);
3117   int verbose = va_arg (*args, int);
3118   dlist_elt_t * head, * elt;
3119   u32 elt_index, head_index;
3120   u32 session_index;
3121   snat_session_t * sess;
3122
3123   s = format (s, "%U: %d dynamic translations, %d static translations\n",
3124               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
3125
3126   if (verbose == 0)
3127     return s;
3128
3129   if (u->nsessions || u->nstaticsessions)
3130     {
3131       head_index = u->sessions_per_user_list_head_index;
3132       head = pool_elt_at_index (sm->list_pool, head_index);
3133
3134       elt_index = head->next;
3135       elt = pool_elt_at_index (sm->list_pool, elt_index);
3136       session_index = elt->value;
3137
3138       while (session_index != ~0)
3139         {
3140           sess = pool_elt_at_index (sm->sessions, session_index);
3141
3142           s = format (s, "  %U\n", format_snat_session, sm, sess);
3143
3144           elt_index = elt->next;
3145           elt = pool_elt_at_index (sm->list_pool, elt_index);
3146           session_index = elt->value;
3147         }
3148     }
3149
3150   return s;
3151 }
3152
3153 u8 * format_snat_static_mapping (u8 * s, va_list * args)
3154 {
3155   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
3156   nat44_lb_addr_port_t *local;
3157
3158   if (m->addr_only)
3159       s = format (s, "local %U external %U vrf %d %s %s",
3160                   format_ip4_address, &m->local_addr,
3161                   format_ip4_address, &m->external_addr,
3162                   m->vrf_id,
3163                   m->twice_nat == TWICE_NAT ? "twice-nat" :
3164                   m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3165                   m->out2in_only ? "out2in-only" : "");
3166   else
3167    {
3168       if (vec_len (m->locals))
3169         {
3170           s = format (s, "%U external %U:%d %s %s",
3171                       format_snat_protocol, m->proto,
3172                       format_ip4_address, &m->external_addr, m->external_port,
3173                       m->twice_nat == TWICE_NAT ? "twice-nat" :
3174                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3175                       m->out2in_only ? "out2in-only" : "");
3176           vec_foreach (local, m->locals)
3177             s = format (s, "\n  local %U:%d vrf %d probability %d\%",
3178                         format_ip4_address, &local->addr, local->port,
3179                         local->vrf_id, local->probability);
3180         }
3181       else
3182         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
3183                     format_snat_protocol, m->proto,
3184                     format_ip4_address, &m->local_addr, m->local_port,
3185                     format_ip4_address, &m->external_addr, m->external_port,
3186                     m->vrf_id,
3187                     m->twice_nat == TWICE_NAT ? "twice-nat" :
3188                     m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3189                     m->out2in_only ? "out2in-only" : "");
3190    }
3191   return s;
3192 }
3193
3194 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
3195 {
3196   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
3197   vnet_main_t *vnm = vnet_get_main();
3198
3199   if (m->addr_only)
3200       s = format (s, "local %U external %U vrf %d",
3201                   format_ip4_address, &m->l_addr,
3202                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3203                   m->vrf_id);
3204   else
3205       s = format (s, "%U local %U:%d external %U:%d vrf %d",
3206                   format_snat_protocol, m->proto,
3207                   format_ip4_address, &m->l_addr, m->l_port,
3208                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3209                   m->e_port, m->vrf_id);
3210
3211   return s;
3212 }
3213
3214 u8 * format_det_map_ses (u8 * s, va_list * args)
3215 {
3216   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
3217   ip4_address_t in_addr, out_addr;
3218   u32 in_offset, out_offset;
3219   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
3220   u32 * i = va_arg (*args, u32 *);
3221
3222   u32 user_index = *i / SNAT_DET_SES_PER_USER;
3223   in_addr.as_u32 = clib_host_to_net_u32 (
3224     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
3225   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
3226     clib_net_to_host_u32(det_map->in_addr.as_u32);
3227   out_offset = in_offset / det_map->sharing_ratio;
3228   out_addr.as_u32 = clib_host_to_net_u32(
3229     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
3230   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
3231               format_ip4_address, &in_addr,
3232               clib_net_to_host_u16 (ses->in_port),
3233               format_ip4_address, &out_addr,
3234               clib_net_to_host_u16 (ses->out.out_port),
3235               format_ip4_address, &ses->out.ext_host_addr,
3236               clib_net_to_host_u16 (ses->out.ext_host_port),
3237               format_snat_session_state, ses->state,
3238               ses->expire);
3239
3240   return s;
3241 }
3242
3243 static void
3244 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3245                                  uword opaque,
3246                                  u32 sw_if_index,
3247                                  ip4_address_t * address,
3248                                  u32 address_length,
3249                                  u32 if_address_index,
3250                                  u32 is_delete)
3251 {
3252   snat_main_t *sm = &snat_main;
3253   snat_static_map_resolve_t *rp;
3254   snat_static_mapping_t *m;
3255   snat_session_key_t m_key;
3256   clib_bihash_kv_8_8_t kv, value;
3257   int i, rv;
3258   ip4_address_t l_addr;
3259
3260   for (i = 0; i < vec_len (sm->to_resolve); i++)
3261     {
3262       rp = sm->to_resolve + i;
3263       if (rp->addr_only == 0)
3264         continue;
3265       if (rp->sw_if_index == sw_if_index)
3266         goto match;
3267     }
3268
3269   return;
3270
3271 match:
3272   m_key.addr.as_u32 = address->as_u32;
3273   m_key.port = rp->addr_only ? 0 : rp->e_port;
3274   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3275   m_key.fib_index = sm->outside_fib_index;
3276   kv.key = m_key.as_u64;
3277   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3278     m = 0;
3279   else
3280     m = pool_elt_at_index (sm->static_mappings, value.value);
3281
3282   if (!is_delete)
3283     {
3284       /* Don't trip over lease renewal, static config */
3285       if (m)
3286         return;
3287     }
3288   else
3289     {
3290       if (!m)
3291         return;
3292     }
3293
3294   /* Indetity mapping? */
3295   if (rp->l_addr.as_u32 == 0)
3296     l_addr.as_u32 = address[0].as_u32;
3297   else
3298     l_addr.as_u32 = rp->l_addr.as_u32;
3299   /* Add the static mapping */
3300   rv = snat_add_static_mapping (l_addr,
3301                                 address[0],
3302                                 rp->l_port,
3303                                 rp->e_port,
3304                                 rp->vrf_id,
3305                                 rp->addr_only,
3306                                 ~0 /* sw_if_index */,
3307                                 rp->proto,
3308                                 !is_delete,
3309                                 0, 0, rp->tag);
3310   if (rv)
3311     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3312 }
3313
3314 static void
3315 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3316                                        uword opaque,
3317                                        u32 sw_if_index,
3318                                        ip4_address_t * address,
3319                                        u32 address_length,
3320                                        u32 if_address_index,
3321                                        u32 is_delete)
3322 {
3323   snat_main_t *sm = &snat_main;
3324   snat_static_map_resolve_t *rp;
3325   ip4_address_t l_addr;
3326   int i, j;
3327   int rv;
3328   u8 twice_nat = 0;
3329   snat_address_t *addresses = sm->addresses;
3330
3331   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
3332     {
3333       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3334           goto match;
3335     }
3336
3337   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
3338     {
3339       twice_nat = 1;
3340       addresses = sm->twice_nat_addresses;
3341       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3342           goto match;
3343     }
3344
3345   return;
3346
3347 match:
3348   if (!is_delete)
3349     {
3350       /* Don't trip over lease renewal, static config */
3351       for (j = 0; j < vec_len(addresses); j++)
3352         if (addresses[j].addr.as_u32 == address->as_u32)
3353           return;
3354
3355       (void) snat_add_address (sm, address, ~0, twice_nat);
3356       /* Scan static map resolution vector */
3357       for (j = 0; j < vec_len (sm->to_resolve); j++)
3358         {
3359           rp = sm->to_resolve + j;
3360           if (rp->addr_only)
3361             continue;
3362           /* On this interface? */
3363           if (rp->sw_if_index == sw_if_index)
3364             {
3365               /* Indetity mapping? */
3366               if (rp->l_addr.as_u32 == 0)
3367                 l_addr.as_u32 = address[0].as_u32;
3368               else
3369                 l_addr.as_u32 = rp->l_addr.as_u32;
3370               /* Add the static mapping */
3371               rv = snat_add_static_mapping (l_addr,
3372                                             address[0],
3373                                             rp->l_port,
3374                                             rp->e_port,
3375                                             rp->vrf_id,
3376                                             rp->addr_only,
3377                                             ~0 /* sw_if_index */,
3378                                             rp->proto,
3379                                             rp->is_add,
3380                                             0, 0, rp->tag);
3381               if (rv)
3382                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3383             }
3384         }
3385       return;
3386     }
3387   else
3388     {
3389       (void) snat_del_address(sm, address[0], 1, twice_nat);
3390       return;
3391     }
3392 }
3393
3394
3395 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
3396                                 u8 twice_nat)
3397 {
3398   ip4_main_t * ip4_main = sm->ip4_main;
3399   ip4_address_t * first_int_addr;
3400   snat_static_map_resolve_t *rp;
3401   u32 *indices_to_delete = 0;
3402   int i, j;
3403   u32 *auto_add_sw_if_indices =
3404     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3405
3406   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
3407                                                 0 /* just want the address*/);
3408
3409   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
3410     {
3411       if (auto_add_sw_if_indices[i] == sw_if_index)
3412         {
3413           if (is_del)
3414             {
3415               /* if have address remove it */
3416               if (first_int_addr)
3417                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3418               else
3419                 {
3420                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3421                     {
3422                       rp = sm->to_resolve + j;
3423                       if (rp->sw_if_index == sw_if_index)
3424                         vec_add1 (indices_to_delete, j);
3425                     }
3426                   if (vec_len(indices_to_delete))
3427                     {
3428                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3429                         vec_del1(sm->to_resolve, j);
3430                       vec_free(indices_to_delete);
3431                     }
3432                 }
3433               if (twice_nat)
3434                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3435               else
3436                 vec_del1(sm->auto_add_sw_if_indices, i);
3437             }
3438           else
3439             return VNET_API_ERROR_VALUE_EXIST;
3440
3441           return 0;
3442         }
3443     }
3444
3445   if (is_del)
3446     return VNET_API_ERROR_NO_SUCH_ENTRY;
3447
3448   /* add to the auto-address list */
3449   if (twice_nat)
3450     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3451   else
3452     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3453
3454   /* If the address is already bound - or static - add it now */
3455   if (first_int_addr)
3456       (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3457
3458   return 0;
3459 }
3460
3461 int
3462 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3463                    snat_protocol_t proto, u32 vrf_id, int is_in)
3464 {
3465   snat_main_per_thread_data_t *tsm;
3466   clib_bihash_kv_8_8_t kv, value;
3467   ip4_header_t ip;
3468   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3469   snat_session_key_t key;
3470   snat_session_t *s;
3471   clib_bihash_8_8_t *t;
3472
3473   if (sm->endpoint_dependent)
3474     return VNET_API_ERROR_UNSUPPORTED;
3475
3476   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3477   if (sm->num_workers > 1)
3478     tsm =
3479       vec_elt_at_index (sm->per_thread_data,
3480                         sm->worker_in2out_cb (&ip, fib_index));
3481   else
3482     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3483
3484   key.addr.as_u32 = addr->as_u32;
3485   key.port = clib_host_to_net_u16 (port);
3486   key.protocol = proto;
3487   key.fib_index = fib_index;
3488   kv.key = key.as_u64;
3489   t = is_in ? &tsm->in2out : &tsm->out2in;
3490   if (!clib_bihash_search_8_8 (t, &kv, &value))
3491     {
3492       if (pool_is_free_index (tsm->sessions, value.value))
3493         return VNET_API_ERROR_UNSPECIFIED;
3494
3495       s = pool_elt_at_index (tsm->sessions, value.value);
3496       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3497       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3498       return 0;
3499     }
3500
3501   return VNET_API_ERROR_NO_SUCH_ENTRY;
3502 }
3503
3504 int
3505 nat44_del_ed_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3506                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3507                       u32 vrf_id, int is_in)
3508 {
3509   ip4_header_t ip;
3510   clib_bihash_16_8_t *t;
3511   nat_ed_ses_key_t key;
3512   clib_bihash_kv_16_8_t kv, value;
3513   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3514   snat_session_t *s;
3515   snat_main_per_thread_data_t *tsm;
3516
3517   if (!sm->endpoint_dependent)
3518     return VNET_API_ERROR_FEATURE_DISABLED;
3519
3520   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3521   if (sm->num_workers > 1)
3522     tsm =
3523       vec_elt_at_index (sm->per_thread_data,
3524                         sm->worker_in2out_cb (&ip, fib_index));
3525   else
3526     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3527
3528   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3529   key.l_addr.as_u32 = addr->as_u32;
3530   key.r_addr.as_u32 = eh_addr->as_u32;
3531   key.l_port = clib_host_to_net_u16 (port);
3532   key.r_port = clib_host_to_net_u16 (eh_port);
3533   key.proto = proto;
3534   key.fib_index = clib_host_to_net_u32 (fib_index);
3535   kv.key[0] = key.as_u64[0];
3536   kv.key[1] = key.as_u64[1];
3537   if (clib_bihash_search_16_8 (t, &kv, &value))
3538     return VNET_API_ERROR_NO_SUCH_ENTRY;
3539
3540   if (pool_is_free_index (tsm->sessions, value.value))
3541     return VNET_API_ERROR_UNSPECIFIED;
3542   s = pool_elt_at_index (tsm->sessions, value.value);
3543   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3544   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3545   return 0;
3546 }
3547
3548 void
3549 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3550 {
3551   snat_main_t *sm = &snat_main;
3552
3553   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3554   sm->psid = psid;
3555   sm->psid_offset = psid_offset;
3556   sm->psid_length = psid_length;
3557 }
3558
3559 void
3560 nat_set_alloc_addr_and_port_default (void)
3561 {
3562   snat_main_t *sm = &snat_main;
3563
3564   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3565 }
3566