src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <vnet/fib/fib_table.h>
  33 #include <vnet/fib/ip4_fib.h>
  34
  35 #include <vpp/app/version.h>
  36
  37 snat_main_t snat_main;
  38
  39
  40 /* Hook up input features */
  41 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  42   .arc_name = "ip4-unicast",
  43   .node_name = "nat44-in2out",
  44   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  45 };
  46 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat44-out2in",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-dhcp-client-detect"),
  51 };
  52 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat44-classify",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  56 };
  57 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  58   .arc_name = "ip4-unicast",
  59   .node_name = "nat44-det-in2out",
  60   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  61 };
  62 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  63   .arc_name = "ip4-unicast",
  64   .node_name = "nat44-det-out2in",
  65   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  66                                "ip4-dhcp-client-detect"),
  67 };
  68 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  69   .arc_name = "ip4-unicast",
  70   .node_name = "nat44-det-classify",
  71   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  72 };
  73 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  74   .arc_name = "ip4-unicast",
  75   .node_name = "nat44-ed-in2out",
  76   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  77 };
  78 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  79   .arc_name = "ip4-unicast",
  80   .node_name = "nat44-ed-out2in",
  81   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  82                                "ip4-dhcp-client-detect"),
  83 };
  84 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-ed-classify",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-in2out-worker-handoff",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  93 };
  94 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-out2in-worker-handoff",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-dhcp-client-detect"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-handoff-classify",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 104 };
 105 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 106   .arc_name = "ip4-unicast",
 107   .node_name = "nat44-in2out-fast",
 108   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 109 };
 110 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 111   .arc_name = "ip4-unicast",
 112   .node_name = "nat44-out2in-fast",
 113   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 114                                "ip4-dhcp-client-detect"),
 115 };
 116 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 117   .arc_name = "ip4-unicast",
 118   .node_name = "nat44-hairpin-dst",
 119   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 120 };
 121 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 122   .arc_name = "ip4-unicast",
 123   .node_name = "nat44-ed-hairpin-dst",
 124   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 125 };
 126
 127 /* Hook up output features */
 128 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 129   .arc_name = "ip4-output",
 130   .node_name = "nat44-in2out-output",
 131   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 132 };
 133 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 134   .arc_name = "ip4-output",
 135   .node_name = "nat44-in2out-output-worker-handoff",
 136   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-hairpin-src",
 141   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 142 };
 143 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 144   .arc_name = "ip4-output",
 145   .node_name = "nat44-ed-in2out-output",
 146   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 147 };
 148 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-ed-hairpin-src",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 152 };
 153
 154 /* Hook up ip4-local features */
 155 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 156 {
 157   .arc_name = "ip4-local",
 158   .node_name = "nat44-hairpinning",
 159   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 162 {
 163   .arc_name = "ip4-local",
 164   .node_name = "nat44-ed-hairpinning",
 165   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 166 };
 167
 168
 169 /* *INDENT-OFF* */
 170 VLIB_PLUGIN_REGISTER () = {
 171     .version = VPP_BUILD_VER,
 172     .description = "Network Address Translation",
 173 };
 174 /* *INDENT-ON* */
 175
 176 vlib_node_registration_t nat44_classify_node;
 177 vlib_node_registration_t nat44_ed_classify_node;
 178 vlib_node_registration_t nat44_det_classify_node;
 179 vlib_node_registration_t nat44_handoff_classify_node;
 180
 181 typedef enum {
 182   NAT44_CLASSIFY_NEXT_IN2OUT,
 183   NAT44_CLASSIFY_NEXT_OUT2IN,
 184   NAT44_CLASSIFY_N_NEXT,
 185 } nat44_classify_next_t;
 186
 187 void
 188 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 189 {
 190   snat_session_key_t key;
 191   clib_bihash_kv_8_8_t kv;
 192   nat_ed_ses_key_t ed_key;
 193   clib_bihash_kv_16_8_t ed_kv;
 194   snat_main_per_thread_data_t *tsm =
 195     vec_elt_at_index (sm->per_thread_data, thread_index);
 196
 197   if (is_fwd_bypass_session (s))
 198     {
 199       ed_key.l_addr = s->in2out.addr;
 200       ed_key.r_addr = s->ext_host_addr;
 201       ed_key.l_port = s->in2out.port;
 202       ed_key.r_port = s->ext_host_port;
 203       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 204       ed_key.fib_index = 0;
 205       ed_kv.key[0] = ed_key.as_u64[0];
 206       ed_kv.key[1] = ed_key.as_u64[1];
 207       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 208         nat_log_warn ("in2out_ed key del failed");
 209       return;
 210     }
 211
 212   /* session lookup tables */
 213   if (is_ed_session (s))
 214     {
 215       if (is_affinity_sessions (s))
 216         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 217                              s->in2out.protocol, s->out2in.port);
 218       ed_key.l_addr = s->out2in.addr;
 219       ed_key.r_addr = s->ext_host_addr;
 220       ed_key.fib_index = s->out2in.fib_index;
 221       if (snat_is_unk_proto_session (s))
 222         {
 223           ed_key.proto = s->in2out.port;
 224           ed_key.r_port = 0;
 225           ed_key.l_port = 0;
 226         }
 227       else
 228         {
 229           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 230           ed_key.l_port = s->out2in.port;
 231           ed_key.r_port = s->ext_host_port;
 232         }
 233       ed_kv.key[0] = ed_key.as_u64[0];
 234       ed_kv.key[1] = ed_key.as_u64[1];
 235       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 236         nat_log_warn ("out2in_ed key del failed");
 237       ed_key.l_addr = s->in2out.addr;
 238       ed_key.fib_index = s->in2out.fib_index;
 239       if (!snat_is_unk_proto_session (s))
 240         ed_key.l_port = s->in2out.port;
 241       if (is_twice_nat_session (s))
 242         {
 243           ed_key.r_addr = s->ext_host_nat_addr;
 244           ed_key.r_port = s->ext_host_nat_port;
 245         }
 246       ed_kv.key[0] = ed_key.as_u64[0];
 247       ed_kv.key[1] = ed_key.as_u64[1];
 248       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 249         nat_log_warn ("in2out_ed key del failed");
 250     }
 251   else
 252     {
 253       kv.key = s->in2out.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 255         nat_log_warn ("in2out key del failed");
 256       kv.key = s->out2in.as_u64;
 257       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 258         nat_log_warn ("out2in key del failed");
 259     }
 260
 261   if (snat_is_unk_proto_session (s))
 262     return;
 263
 264   /* log NAT event */
 265   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 266                                       s->out2in.addr.as_u32,
 267                                       s->in2out.protocol,
 268                                       s->in2out.port,
 269                                       s->out2in.port,
 270                                       s->in2out.fib_index);
 271
 272   /* Twice NAT address and port for external host */
 273   if (is_twice_nat_session (s))
 274     {
 275       key.protocol = s->in2out.protocol;
 276       key.port = s->ext_host_nat_port;
 277       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 278       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 279                                           thread_index, &key);
 280     }
 281
 282   if (snat_is_session_static (s))
 283     return;
 284
 285   if (s->outside_address_index != ~0)
 286     snat_free_outside_address_and_port (sm->addresses, thread_index,
 287                                         &s->out2in);
 288 }
 289
 290 snat_user_t *
 291 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 292                         u32 thread_index)
 293 {
 294   snat_user_t *u = 0;
 295   snat_user_key_t user_key;
 296   clib_bihash_kv_8_8_t kv, value;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   dlist_elt_t * per_user_list_head_elt;
 299
 300   user_key.addr.as_u32 = addr->as_u32;
 301   user_key.fib_index = fib_index;
 302   kv.key = user_key.as_u64;
 303
 304   /* Ever heard of the "user" = src ip4 address before? */
 305   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 306     {
 307       /* no, make a new one */
 308       pool_get (tsm->users, u);
 309       memset (u, 0, sizeof (*u));
 310       u->addr.as_u32 = addr->as_u32;
 311       u->fib_index = fib_index;
 312
 313       pool_get (tsm->list_pool, per_user_list_head_elt);
 314
 315       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 316         tsm->list_pool;
 317
 318       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 319
 320       kv.value = u - tsm->users;
 321
 322       /* add user */
 323       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 324         nat_log_warn ("user_hash keay add failed");
 325     }
 326   else
 327     {
 328       u = pool_elt_at_index (tsm->users, value.value);
 329     }
 330
 331   return u;
 332 }
 333
 334 snat_session_t *
 335 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 336 {
 337   snat_session_t *s;
 338   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 339   u32 oldest_per_user_translation_list_index, session_index;
 340   dlist_elt_t * oldest_per_user_translation_list_elt;
 341   dlist_elt_t * per_user_translation_list_elt;
 342
 343   /* Over quota? Recycle the least recently used translation */
 344   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 345     {
 346       oldest_per_user_translation_list_index =
 347         clib_dlist_remove_head (tsm->list_pool,
 348                                 u->sessions_per_user_list_head_index);
 349
 350       ASSERT (oldest_per_user_translation_list_index != ~0);
 351
 352       /* Add it back to the end of the LRU list */
 353       clib_dlist_addtail (tsm->list_pool,
 354                           u->sessions_per_user_list_head_index,
 355                           oldest_per_user_translation_list_index);
 356       /* Get the list element */
 357       oldest_per_user_translation_list_elt =
 358         pool_elt_at_index (tsm->list_pool,
 359                            oldest_per_user_translation_list_index);
 360
 361       /* Get the session index from the list element */
 362       session_index = oldest_per_user_translation_list_elt->value;
 363
 364       /* Get the session */
 365       s = pool_elt_at_index (tsm->sessions, session_index);
 366       nat_free_session_data (sm, s, thread_index);
 367       if (snat_is_session_static(s))
 368         u->nstaticsessions--;
 369       else
 370         u->nsessions--;
 371       s->outside_address_index = ~0;
 372       s->flags = 0;
 373       s->total_bytes = 0;
 374       s->total_pkts = 0;
 375       s->state = 0;
 376       s->ext_host_addr.as_u32 = 0;
 377       s->ext_host_port = 0;
 378       s->ext_host_nat_addr.as_u32 = 0;
 379       s->ext_host_nat_port = 0;
 380     }
 381   else
 382     {
 383       pool_get (tsm->sessions, s);
 384       memset (s, 0, sizeof (*s));
 385       s->outside_address_index = ~0;
 386
 387       /* Create list elts */
 388       pool_get (tsm->list_pool, per_user_translation_list_elt);
 389       clib_dlist_init (tsm->list_pool,
 390                        per_user_translation_list_elt - tsm->list_pool);
 391
 392       per_user_translation_list_elt->value = s - tsm->sessions;
 393       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 394       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 395
 396       clib_dlist_addtail (tsm->list_pool,
 397                           s->per_user_list_head_index,
 398                           per_user_translation_list_elt - tsm->list_pool);
 399     }
 400
 401   return s;
 402 }
 403
 404 snat_session_t *
 405 nat_ed_session_alloc (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 406 {
 407   snat_session_t *s;
 408   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 409   dlist_elt_t * per_user_translation_list_elt;
 410
 411   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 412     {
 413       nat_log_warn ("max translations per user %U", format_ip4_address, &u->addr);
 414       snat_ipfix_logging_max_entries_per_user (sm->max_translations_per_user,
 415                                                u->addr.as_u32);
 416       return 0;
 417     }
 418
 419   pool_get (tsm->sessions, s);
 420   memset (s, 0, sizeof (*s));
 421   s->outside_address_index = ~0;
 422
 423   /* Create list elts */
 424   pool_get (tsm->list_pool, per_user_translation_list_elt);
 425   clib_dlist_init (tsm->list_pool,
 426                    per_user_translation_list_elt - tsm->list_pool);
 427
 428   per_user_translation_list_elt->value = s - tsm->sessions;
 429   s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 430   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 431
 432   clib_dlist_addtail (tsm->list_pool,
 433                       s->per_user_list_head_index,
 434                       per_user_translation_list_elt - tsm->list_pool);
 435
 436   return s;
 437 }
 438
 439 typedef struct {
 440   u8 next_in2out;
 441 } nat44_classify_trace_t;
 442
 443 static u8 * format_nat44_classify_trace (u8 * s, va_list * args)
 444 {
 445   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 446   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 447   nat44_classify_trace_t *t = va_arg (*args, nat44_classify_trace_t *);
 448   char *next;
 449
 450   next = t->next_in2out ? "nat44-in2out" : "nat44-out2in";
 451
 452   s = format (s, "nat44-classify: next %s", next);
 453
 454   return s;
 455 }
 456
 457 static inline uword
 458 nat44_classify_node_fn_inline (vlib_main_t * vm,
 459                                vlib_node_runtime_t * node,
 460                                vlib_frame_t * frame,
 461                                int is_ed)
 462 {
 463   u32 n_left_from, * from, * to_next;
 464   nat44_classify_next_t next_index;
 465   snat_main_t *sm = &snat_main;
 466   snat_static_mapping_t *m;
 467   u32 thread_index = vm->thread_index;
 468   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 469
 470   from = vlib_frame_vector_args (frame);
 471   n_left_from = frame->n_vectors;
 472   next_index = node->cached_next_index;
 473
 474   while (n_left_from > 0)
 475     {
 476       u32 n_left_to_next;
 477
 478       vlib_get_next_frame (vm, node, next_index,
 479                            to_next, n_left_to_next);
 480
 481       while (n_left_from > 0 && n_left_to_next > 0)
 482         {
 483           u32 bi0;
 484           vlib_buffer_t *b0;
 485           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0;
 486           ip4_header_t *ip0;
 487           snat_address_t *ap;
 488           snat_session_key_t m_key0;
 489           clib_bihash_kv_8_8_t kv0, value0;
 490           clib_bihash_kv_16_8_t ed_kv0, ed_value0;
 491           udp_header_t *udp0;
 492
 493           /* speculatively enqueue b0 to the current next frame */
 494           bi0 = from[0];
 495           to_next[0] = bi0;
 496           from += 1;
 497           to_next += 1;
 498           n_left_from -= 1;
 499           n_left_to_next -= 1;
 500
 501           b0 = vlib_get_buffer (vm, bi0);
 502           ip0 = vlib_buffer_get_current (b0);
 503           udp0 = ip4_next_header (ip0);
 504
 505           if (is_ed)
 506             {
 507               sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 508               rx_fib_index0 =
 509                 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 510                                                      sw_if_index0);
 511               make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address,
 512                           ip0->protocol, rx_fib_index0, udp0->src_port,
 513                           udp0->dst_port);
 514               if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, &ed_value0))
 515                 goto enqueue0;
 516             }
 517
 518           vec_foreach (ap, sm->addresses)
 519             {
 520               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 521                 {
 522                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 523                   goto enqueue0;
 524                 }
 525             }
 526
 527           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 528             {
 529               m_key0.addr = ip0->dst_address;
 530               m_key0.port = 0;
 531               m_key0.protocol = 0;
 532               m_key0.fib_index = 0;
 533               kv0.key = m_key0.as_u64;
 534               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 535                 {
 536                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 537                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 538                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 539                   goto enqueue0;
 540                 }
 541               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 542               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 543               kv0.key = m_key0.as_u64;
 544               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 545                 {
 546                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 547                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 548                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 549                 }
 550             }
 551
 552         enqueue0:
 553           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 554                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 555             {
 556               nat44_classify_trace_t *t =
 557                   vlib_add_trace (vm, node, b0, sizeof (*t));
 558               t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
 559             }
 560
 561           /* verify speculative enqueue, maybe switch current next frame */
 562           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 563                                            to_next, n_left_to_next,
 564                                            bi0, next0);
 565         }
 566
 567       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 568     }
 569
 570   return frame->n_vectors;
 571 }
 572
 573 static uword
 574 nat44_classify_node_fn (vlib_main_t * vm,
 575                         vlib_node_runtime_t * node,
 576                         vlib_frame_t * frame)
 577 {
 578   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 579 };
 580
 581 VLIB_REGISTER_NODE (nat44_classify_node) = {
 582   .function = nat44_classify_node_fn,
 583   .name = "nat44-classify",
 584   .vector_size = sizeof (u32),
 585   .format_trace = format_nat44_classify_trace,
 586   .type = VLIB_NODE_TYPE_INTERNAL,
 587   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 588   .next_nodes = {
 589     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 590     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 591   },
 592 };
 593
 594 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 595                               nat44_classify_node_fn);
 596 static uword
 597 nat44_ed_classify_node_fn (vlib_main_t * vm,
 598                            vlib_node_runtime_t * node,
 599                            vlib_frame_t * frame)
 600 {
 601   return nat44_classify_node_fn_inline (vm, node, frame, 1);
 602 };
 603
 604 VLIB_REGISTER_NODE (nat44_ed_classify_node) = {
 605   .function = nat44_ed_classify_node_fn,
 606   .name = "nat44-ed-classify",
 607   .vector_size = sizeof (u32),
 608   .format_trace = format_nat44_classify_trace,
 609   .type = VLIB_NODE_TYPE_INTERNAL,
 610   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 611   .next_nodes = {
 612     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out",
 613     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in",
 614   },
 615 };
 616
 617 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_classify_node,
 618                               nat44_ed_classify_node_fn);
 619
 620 static uword
 621 nat44_det_classify_node_fn (vlib_main_t * vm,
 622                             vlib_node_runtime_t * node,
 623                             vlib_frame_t * frame)
 624 {
 625   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 626 };
 627
 628 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 629   .function = nat44_det_classify_node_fn,
 630   .name = "nat44-det-classify",
 631   .vector_size = sizeof (u32),
 632   .format_trace = format_nat44_classify_trace,
 633   .type = VLIB_NODE_TYPE_INTERNAL,
 634   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 635   .next_nodes = {
 636     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 637     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 638   },
 639 };
 640
 641 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 642                               nat44_det_classify_node_fn);
 643
 644 static uword
 645 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 646                                 vlib_node_runtime_t * node,
 647                                 vlib_frame_t * frame)
 648 {
 649   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 650 };
 651
 652 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 653   .function = nat44_handoff_classify_node_fn,
 654   .name = "nat44-handoff-classify",
 655   .vector_size = sizeof (u32),
 656   .format_trace = format_nat44_classify_trace,
 657   .type = VLIB_NODE_TYPE_INTERNAL,
 658   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 659   .next_nodes = {
 660     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 661     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 662   },
 663 };
 664
 665 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 666                               nat44_handoff_classify_node_fn);
 667
 668 /**
 669  * @brief Add/del NAT address to FIB.
 670  *
 671  * Add the external NAT address to the FIB as receive entries. This ensures
 672  * that VPP will reply to ARP for this address and we don't need to enable
 673  * proxy ARP on the outside interface.
 674  *
 675  * @param addr IPv4 address.
 676  * @param plen address prefix length
 677  * @param sw_if_index Interface.
 678  * @param is_add If 0 delete, otherwise add.
 679  */
 680 void
 681 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 682                           int is_add)
 683 {
 684   fib_prefix_t prefix = {
 685     .fp_len = p_len,
 686     .fp_proto = FIB_PROTOCOL_IP4,
 687     .fp_addr = {
 688         .ip4.as_u32 = addr->as_u32,
 689     },
 690   };
 691   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 692
 693   if (is_add)
 694     fib_table_entry_update_one_path(fib_index,
 695                                     &prefix,
 696                                     FIB_SOURCE_PLUGIN_LOW,
 697                                     (FIB_ENTRY_FLAG_CONNECTED |
 698                                      FIB_ENTRY_FLAG_LOCAL |
 699                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 700                                     DPO_PROTO_IP4,
 701                                     NULL,
 702                                     sw_if_index,
 703                                     ~0,
 704                                     1,
 705                                     NULL,
 706                                     FIB_ROUTE_PATH_FLAG_NONE);
 707   else
 708     fib_table_entry_delete(fib_index,
 709                            &prefix,
 710                            FIB_SOURCE_PLUGIN_LOW);
 711 }
 712
 713 int snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 714                        u8 twice_nat)
 715 {
 716   snat_address_t * ap;
 717   snat_interface_t *i;
 718   vlib_thread_main_t *tm = vlib_get_thread_main ();
 719
 720   if (twice_nat && !sm->endpoint_dependent)
 721     return VNET_API_ERROR_FEATURE_DISABLED;
 722
 723   /* Check if address already exists */
 724   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 725     {
 726       if (ap->addr.as_u32 == addr->as_u32)
 727         return VNET_API_ERROR_VALUE_EXIST;
 728     }
 729
 730   if (twice_nat)
 731     vec_add2 (sm->twice_nat_addresses, ap, 1);
 732   else
 733     vec_add2 (sm->addresses, ap, 1);
 734
 735   ap->addr = *addr;
 736   if (vrf_id != ~0)
 737     ap->fib_index =
 738       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 739                                          FIB_SOURCE_PLUGIN_LOW);
 740   else
 741     ap->fib_index = ~0;
 742 #define _(N, i, n, s) \
 743   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 744   ap->busy_##n##_ports = 0; \
 745   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 746   foreach_snat_protocol
 747 #undef _
 748
 749   if (twice_nat)
 750     return 0;
 751
 752   /* Add external address to FIB */
 753   pool_foreach (i, sm->interfaces,
 754   ({
 755     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 756       continue;
 757
 758     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 759     break;
 760   }));
 761   pool_foreach (i, sm->output_feature_interfaces,
 762   ({
 763     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 764       continue;
 765
 766     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 767     break;
 768   }));
 769
 770   return 0;
 771 }
 772
 773 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 774                                                    ip4_address_t addr)
 775 {
 776   snat_static_mapping_t *m;
 777   pool_foreach (m, sm->static_mappings,
 778   ({
 779       if (m->external_addr.as_u32 == addr.as_u32)
 780         return 1;
 781   }));
 782
 783   return 0;
 784 }
 785
 786 void increment_v4_address (ip4_address_t * a)
 787 {
 788   u32 v;
 789
 790   v = clib_net_to_host_u32(a->as_u32) + 1;
 791   a->as_u32 = clib_host_to_net_u32(v);
 792 }
 793
 794 static void
 795 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 796                                        ip4_address_t l_addr,
 797                                        u16 l_port,
 798                                        u32 sw_if_index,
 799                                        u16 e_port,
 800                                        u32 vrf_id,
 801                                        snat_protocol_t proto,
 802                                        int addr_only,
 803                                        int is_add,
 804                                        u8 * tag)
 805 {
 806   snat_static_map_resolve_t *rp;
 807
 808   vec_add2 (sm->to_resolve, rp, 1);
 809   rp->l_addr.as_u32 = l_addr.as_u32;
 810   rp->l_port = l_port;
 811   rp->sw_if_index = sw_if_index;
 812   rp->e_port = e_port;
 813   rp->vrf_id = vrf_id;
 814   rp->proto = proto;
 815   rp->addr_only = addr_only;
 816   rp->is_add = is_add;
 817   rp->tag = vec_dup (tag);
 818 }
 819
 820 static u32 get_thread_idx_by_port(u16 e_port)
 821 {
 822     snat_main_t * sm = &snat_main;
 823     u32 thread_idx = sm->num_workers;
 824     if (sm->num_workers > 1)
 825     {
 826         thread_idx = sm->first_worker_index + sm->workers[(e_port - 1024) / sm->port_per_thread];
 827     }
 828     return thread_idx;
 829 }
 830
 831 /**
 832  * @brief Add static mapping.
 833  *
 834  * Create static mapping between local addr+port and external addr+port.
 835  *
 836  * @param l_addr Local IPv4 address.
 837  * @param e_addr External IPv4 address.
 838  * @param l_port Local port number.
 839  * @param e_port External port number.
 840  * @param vrf_id VRF ID.
 841  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 842  * @param sw_if_index External port instead of specific IP address.
 843  * @param is_add If 0 delete static mapping, otherwise add.
 844  * @param twice_nat If value is TWICE_NAT then translate external host address
 845  *                  and port.
 846  *                  If value is TWICE_NAT_SELF then translate external host
 847  *                  address and port whenever external host address equals
 848  *                  local address of internal host.
 849  * @param out2in_only If 1 rule match only out2in direction
 850  * @param tag - opaque string tag
 851  *
 852  * @returns
 853  */
 854 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 855                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 856                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 857                             twice_nat_type_t twice_nat, u8 out2in_only,
 858                             u8 * tag)
 859 {
 860   snat_main_t * sm = &snat_main;
 861   snat_static_mapping_t *m;
 862   snat_session_key_t m_key;
 863   clib_bihash_kv_8_8_t kv, value;
 864   snat_address_t *a = 0;
 865   u32 fib_index = ~0;
 866   uword * p;
 867   snat_interface_t *interface;
 868   int i;
 869   snat_main_per_thread_data_t *tsm;
 870   snat_user_key_t u_key;
 871   snat_user_t *u;
 872   dlist_elt_t * head, * elt;
 873   u32 elt_index, head_index;
 874   u32 ses_index;
 875   u64 user_index;
 876   snat_session_t * s;
 877   snat_static_map_resolve_t *rp, *rp_match = 0;
 878
 879   if (!sm->endpoint_dependent)
 880     {
 881       if (twice_nat || out2in_only)
 882         return VNET_API_ERROR_FEATURE_DISABLED;
 883     }
 884
 885   /* If the external address is a specific interface address */
 886   if (sw_if_index != ~0)
 887     {
 888       ip4_address_t * first_int_addr;
 889
 890       for (i = 0; i < vec_len (sm->to_resolve); i++)
 891         {
 892           rp = sm->to_resolve + i;
 893           if (rp->sw_if_index != sw_if_index ||
 894               rp->l_addr.as_u32 != l_addr.as_u32 ||
 895               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 896             continue;
 897
 898           if (!addr_only)
 899             {
 900               if (rp->l_port != l_port || rp->e_port != e_port || rp->proto != proto)
 901                 continue;
 902             }
 903
 904           rp_match = rp;
 905           break;
 906         }
 907
 908       /* Might be already set... */
 909       first_int_addr = ip4_interface_first_address
 910         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 911
 912       if (is_add)
 913         {
 914           if (rp_match)
 915             return VNET_API_ERROR_VALUE_EXIST;
 916
 917           snat_add_static_mapping_when_resolved
 918             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 919              addr_only,  is_add, tag);
 920
 921           /* DHCP resolution required? */
 922           if (first_int_addr == 0)
 923             {
 924               return 0;
 925             }
 926           else
 927             {
 928               e_addr.as_u32 = first_int_addr->as_u32;
 929               /* Identity mapping? */
 930               if (l_addr.as_u32 == 0)
 931                 l_addr.as_u32 = e_addr.as_u32;
 932             }
 933         }
 934       else
 935         {
 936           if (!rp_match)
 937             return VNET_API_ERROR_NO_SUCH_ENTRY;
 938
 939           vec_del1 (sm->to_resolve, i);
 940
 941           if (first_int_addr)
 942             {
 943               e_addr.as_u32 = first_int_addr->as_u32;
 944               /* Identity mapping? */
 945               if (l_addr.as_u32 == 0)
 946                 l_addr.as_u32 = e_addr.as_u32;
 947             }
 948           else
 949             return 0;
 950         }
 951     }
 952
 953   m_key.addr = e_addr;
 954   m_key.port = addr_only ? 0 : e_port;
 955   m_key.protocol = addr_only ? 0 : proto;
 956   m_key.fib_index = 0;
 957   kv.key = m_key.as_u64;
 958   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 959     m = 0;
 960   else
 961     m = pool_elt_at_index (sm->static_mappings, value.value);
 962
 963   if (is_add)
 964     {
 965       if (m)
 966         return VNET_API_ERROR_VALUE_EXIST;
 967
 968       if (twice_nat && addr_only)
 969         return VNET_API_ERROR_UNSUPPORTED;
 970
 971       /* Convert VRF id to FIB index */
 972       if (vrf_id != ~0)
 973         {
 974           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 975           if (!p)
 976             return VNET_API_ERROR_NO_SUCH_FIB;
 977           fib_index = p[0];
 978         }
 979       /* If not specified use inside VRF id from SNAT plugin startup config */
 980       else
 981         {
 982           fib_index = sm->inside_fib_index;
 983           vrf_id = sm->inside_vrf_id;
 984         }
 985
 986       if (!out2in_only)
 987         {
 988           m_key.addr = l_addr;
 989           m_key.port = addr_only ? 0 : l_port;
 990           m_key.protocol = addr_only ? 0 : proto;
 991           m_key.fib_index = fib_index;
 992           kv.key = m_key.as_u64;
 993           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 994             return VNET_API_ERROR_VALUE_EXIST;
 995         }
 996
 997       /* Find external address in allocated addresses and reserve port for
 998          address and port pair mapping when dynamic translations enabled */
 999       if (!(addr_only || sm->static_mapping_only || out2in_only))
1000         {
1001           for (i = 0; i < vec_len (sm->addresses); i++)
1002             {
1003               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1004                 {
1005                   a = sm->addresses + i;
1006                   /* External port must be unused */
1007                   switch (proto)
1008                     {
1009 #define _(N, j, n, s) \
1010                     case SNAT_PROTOCOL_##N: \
1011                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1012                         return VNET_API_ERROR_INVALID_VALUE; \
1013                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1014                       if (e_port > 1024) \
1015                         { \
1016                           a->busy_##n##_ports++; \
1017                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1018                         } \
1019                       break;
1020                       foreach_snat_protocol
1021 #undef _
1022                     default:
1023                       nat_log_info ("unknown protocol");
1024                       return VNET_API_ERROR_INVALID_VALUE_2;
1025                     }
1026                   break;
1027                 }
1028             }
1029           /* External address must be allocated */
1030           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1031             {
1032               if (sw_if_index != ~0)
1033                 {
1034                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1035                     {
1036                       rp = sm->to_resolve + i;
1037                       if (rp->addr_only)
1038                          continue;
1039                       if (rp->sw_if_index != sw_if_index &&
1040                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1041                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1042                           rp->e_port != e_port && rp->proto != proto)
1043                         continue;
1044
1045                       vec_del1 (sm->to_resolve, i);
1046                       break;
1047                     }
1048                 }
1049               return VNET_API_ERROR_NO_SUCH_ENTRY;
1050             }
1051         }
1052
1053       pool_get (sm->static_mappings, m);
1054       memset (m, 0, sizeof (*m));
1055       m->tag = vec_dup (tag);
1056       m->local_addr = l_addr;
1057       m->external_addr = e_addr;
1058       m->addr_only = addr_only;
1059       m->vrf_id = vrf_id;
1060       m->fib_index = fib_index;
1061       m->twice_nat = twice_nat;
1062       m->out2in_only = out2in_only;
1063       if (!addr_only)
1064         {
1065           m->local_port = l_port;
1066           m->external_port = e_port;
1067           m->proto = proto;
1068         }
1069
1070       if (sm->num_workers > 1)
1071         {
1072           ip4_header_t ip = {
1073             .src_address = m->local_addr,
1074           };
1075           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
1076           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1077         }
1078       else
1079         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1080
1081       m_key.addr = m->local_addr;
1082       m_key.port = m->local_port;
1083       m_key.protocol = m->proto;
1084       m_key.fib_index = m->fib_index;
1085       kv.key = m_key.as_u64;
1086       kv.value = m - sm->static_mappings;
1087       if (!out2in_only)
1088         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1089
1090       m_key.addr = m->external_addr;
1091       m_key.port = m->external_port;
1092       m_key.fib_index = 0;
1093       kv.key = m_key.as_u64;
1094       kv.value = m - sm->static_mappings;
1095       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
1096
1097       /* Delete dynamic sessions matching local address (+ local port) */
1098       if (!(sm->static_mapping_only))
1099         {
1100           u_key.addr = m->local_addr;
1101           u_key.fib_index = m->fib_index;
1102           kv.key = u_key.as_u64;
1103           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1104             {
1105               user_index = value.value;
1106               u = pool_elt_at_index (tsm->users, user_index);
1107               if (u->nsessions)
1108                 {
1109                   head_index = u->sessions_per_user_list_head_index;
1110                   head = pool_elt_at_index (tsm->list_pool, head_index);
1111                   elt_index = head->next;
1112                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1113                   ses_index = elt->value;
1114                   while (ses_index != ~0)
1115                     {
1116                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1117                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1118                       ses_index = elt->value;
1119
1120                       if (snat_is_session_static (s))
1121                         continue;
1122
1123                       if (!addr_only && (clib_net_to_host_u16 (s->in2out.port) != m->local_port))
1124                         continue;
1125
1126                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1127                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1128
1129                       if (!addr_only && !sm->endpoint_dependent)
1130                         break;
1131                     }
1132                 }
1133             }
1134         }
1135     }
1136   else
1137     {
1138       if (!m)
1139         {
1140           if (sw_if_index != ~0)
1141             return 0;
1142           else
1143             return VNET_API_ERROR_NO_SUCH_ENTRY;
1144         }
1145
1146       /* Free external address port */
1147       if (!(addr_only || sm->static_mapping_only || out2in_only))
1148         {
1149           for (i = 0; i < vec_len (sm->addresses); i++)
1150             {
1151               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1152                 {
1153                   a = sm->addresses + i;
1154                   switch (proto)
1155                     {
1156 #define _(N, j, n, s) \
1157                     case SNAT_PROTOCOL_##N: \
1158                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1159                       if (e_port > 1024) \
1160                         { \
1161                           a->busy_##n##_ports--; \
1162                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1163                         } \
1164                       break;
1165                       foreach_snat_protocol
1166 #undef _
1167                     default:
1168                       nat_log_info ("unknown protocol");
1169                       return VNET_API_ERROR_INVALID_VALUE_2;
1170                     }
1171                   break;
1172                 }
1173             }
1174         }
1175
1176       if (sm->num_workers > 1)
1177         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1178       else
1179         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1180
1181       m_key.addr = m->local_addr;
1182       m_key.port = m->local_port;
1183       m_key.protocol = m->proto;
1184       m_key.fib_index = m->fib_index;
1185       kv.key = m_key.as_u64;
1186       if (!out2in_only)
1187         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1188
1189       m_key.addr = m->external_addr;
1190       m_key.port = m->external_port;
1191       m_key.fib_index = 0;
1192       kv.key = m_key.as_u64;
1193       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1194
1195       /* Delete session(s) for static mapping if exist */
1196       if (!(sm->static_mapping_only) ||
1197           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1198         {
1199           u_key.addr = m->local_addr;
1200           u_key.fib_index = m->fib_index;
1201           kv.key = u_key.as_u64;
1202           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1203             {
1204               user_index = value.value;
1205               u = pool_elt_at_index (tsm->users, user_index);
1206               if (u->nstaticsessions)
1207                 {
1208                   head_index = u->sessions_per_user_list_head_index;
1209                   head = pool_elt_at_index (tsm->list_pool, head_index);
1210                   elt_index = head->next;
1211                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1212                   ses_index = elt->value;
1213                   while (ses_index != ~0)
1214                     {
1215                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1216                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1217                       ses_index = elt->value;
1218
1219                       if (!addr_only)
1220                         {
1221                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1222                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1223                             continue;
1224                         }
1225
1226                       if (is_lb_session (s))
1227                         continue;
1228
1229                       if (!snat_is_session_static (s))
1230                         continue;
1231
1232                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1233                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1234
1235                       if (!addr_only && !sm->endpoint_dependent)
1236                         break;
1237                     }
1238                 }
1239             }
1240         }
1241
1242       vec_free (m->tag);
1243       vec_free (m->workers);
1244       /* Delete static mapping from pool */
1245       pool_put (sm->static_mappings, m);
1246     }
1247
1248   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1249     return 0;
1250
1251   /* Add/delete external address to FIB */
1252   pool_foreach (interface, sm->interfaces,
1253   ({
1254     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1255       continue;
1256
1257     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1258     break;
1259   }));
1260   pool_foreach (interface, sm->output_feature_interfaces,
1261   ({
1262     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1263       continue;
1264
1265     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1266     break;
1267   }));
1268
1269   return 0;
1270 }
1271
1272 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1273                                      snat_protocol_t proto,
1274                                      nat44_lb_addr_port_t *locals, u8 is_add,
1275                                      twice_nat_type_t twice_nat, u8 out2in_only,
1276                                      u8 *tag, u32 affinity)
1277 {
1278   snat_main_t * sm = &snat_main;
1279   snat_static_mapping_t *m;
1280   snat_session_key_t m_key;
1281   clib_bihash_kv_8_8_t kv, value;
1282   snat_address_t *a = 0;
1283   int i;
1284   nat44_lb_addr_port_t *local;
1285   u32 elt_index, head_index, ses_index;
1286   snat_main_per_thread_data_t *tsm;
1287   snat_user_key_t u_key;
1288   snat_user_t *u;
1289   snat_session_t * s;
1290   dlist_elt_t * head, * elt;
1291   uword *bitmap = 0;
1292
1293   if (!sm->endpoint_dependent)
1294     return VNET_API_ERROR_FEATURE_DISABLED;
1295
1296   m_key.addr = e_addr;
1297   m_key.port = e_port;
1298   m_key.protocol = proto;
1299   m_key.fib_index = 0;
1300   kv.key = m_key.as_u64;
1301   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1302     m = 0;
1303   else
1304     m = pool_elt_at_index (sm->static_mappings, value.value);
1305
1306   if (is_add)
1307     {
1308       if (m)
1309         return VNET_API_ERROR_VALUE_EXIST;
1310
1311       if (vec_len (locals) < 2)
1312         return VNET_API_ERROR_INVALID_VALUE;
1313
1314       /* Find external address in allocated addresses and reserve port for
1315          address and port pair mapping when dynamic translations enabled */
1316       if (!(sm->static_mapping_only || out2in_only))
1317         {
1318           for (i = 0; i < vec_len (sm->addresses); i++)
1319             {
1320               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1321                 {
1322                   a = sm->addresses + i;
1323                   /* External port must be unused */
1324                   switch (proto)
1325                     {
1326 #define _(N, j, n, s) \
1327                     case SNAT_PROTOCOL_##N: \
1328                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1329                         return VNET_API_ERROR_INVALID_VALUE; \
1330                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1331                       if (e_port > 1024) \
1332                         { \
1333                           a->busy_##n##_ports++; \
1334                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1335                         } \
1336                       break;
1337                       foreach_snat_protocol
1338 #undef _
1339                     default:
1340                       nat_log_info ("unknown protocol");
1341                       return VNET_API_ERROR_INVALID_VALUE_2;
1342                     }
1343                   break;
1344                 }
1345             }
1346           /* External address must be allocated */
1347           if (!a)
1348             return VNET_API_ERROR_NO_SUCH_ENTRY;
1349         }
1350
1351       pool_get (sm->static_mappings, m);
1352       memset (m, 0, sizeof (*m));
1353       m->tag = vec_dup (tag);
1354       m->external_addr = e_addr;
1355       m->addr_only = 0;
1356       m->external_port = e_port;
1357       m->proto = proto;
1358       m->twice_nat = twice_nat;
1359       m->out2in_only = out2in_only;
1360       m->affinity = affinity;
1361
1362       if (affinity)
1363         m->affinity_per_service_list_head_index =
1364           nat_affinity_get_per_service_list_head_index();
1365       else
1366         m->affinity_per_service_list_head_index = ~0;
1367
1368       m_key.addr = m->external_addr;
1369       m_key.port = m->external_port;
1370       m_key.protocol = m->proto;
1371       m_key.fib_index = 0;
1372       kv.key = m_key.as_u64;
1373       kv.value = m - sm->static_mappings;
1374       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1375         {
1376           nat_log_err ("static_mapping_by_external key add failed");
1377           return VNET_API_ERROR_UNSPECIFIED;
1378         }
1379
1380       m_key.fib_index = m->fib_index;
1381       for (i = 0; i < vec_len (locals); i++)
1382         {
1383           locals[i].fib_index = fib_table_find_or_create_and_lock (
1384             FIB_PROTOCOL_IP4, locals[i].vrf_id, FIB_SOURCE_PLUGIN_LOW);
1385           m_key.addr = locals[i].addr;
1386           m_key.fib_index = locals[i].fib_index;
1387           if (!out2in_only)
1388             {
1389               m_key.port = locals[i].port;
1390               kv.key = m_key.as_u64;
1391               kv.value = m - sm->static_mappings;
1392               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1393             }
1394           locals[i].prefix = (i == 0) ? locals[i].probability :\
1395             (locals[i - 1].prefix + locals[i].probability);
1396           vec_add1 (m->locals, locals[i]);
1397           if (sm->num_workers > 1)
1398             {
1399               ip4_header_t ip = {
1400                 .src_address = locals[i].addr,
1401               };
1402               bitmap = clib_bitmap_set (
1403                 bitmap, sm->worker_in2out_cb (&ip, m->fib_index), 1);
1404             }
1405         }
1406
1407       /* Assign workers */
1408       if (sm->num_workers > 1)
1409         {
1410           clib_bitmap_foreach (i, bitmap,
1411             ({
1412                vec_add1(m->workers, i);
1413             }));
1414         }
1415     }
1416   else
1417     {
1418       if (!m)
1419         return VNET_API_ERROR_NO_SUCH_ENTRY;
1420
1421       /* Free external address port */
1422       if (!(sm->static_mapping_only || out2in_only))
1423         {
1424           for (i = 0; i < vec_len (sm->addresses); i++)
1425             {
1426               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1427                 {
1428                   a = sm->addresses + i;
1429                   switch (proto)
1430                     {
1431 #define _(N, j, n, s) \
1432                     case SNAT_PROTOCOL_##N: \
1433                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1434                       if (e_port > 1024) \
1435                         { \
1436                           a->busy_##n##_ports--; \
1437                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1438                         } \
1439                       break;
1440                       foreach_snat_protocol
1441 #undef _
1442                     default:
1443                       nat_log_info ("unknown protocol");
1444                       return VNET_API_ERROR_INVALID_VALUE_2;
1445                     }
1446                   break;
1447                 }
1448             }
1449         }
1450
1451       m_key.addr = m->external_addr;
1452       m_key.port = m->external_port;
1453       m_key.protocol = m->proto;
1454       m_key.fib_index = 0;
1455       kv.key = m_key.as_u64;
1456       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1457         {
1458           nat_log_err ("static_mapping_by_external key del failed");
1459           return VNET_API_ERROR_UNSPECIFIED;
1460         }
1461
1462       vec_foreach (local, m->locals)
1463         {
1464           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1465                             FIB_SOURCE_PLUGIN_LOW);
1466           m_key.addr = local->addr;
1467           if (!out2in_only)
1468             {
1469               m_key.port = local->port;
1470               m_key.fib_index = local->fib_index;
1471               kv.key = m_key.as_u64;
1472               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1473                 {
1474                   nat_log_err ("static_mapping_by_local key del failed");
1475                   return VNET_API_ERROR_UNSPECIFIED;
1476                 }
1477             }
1478
1479           if (sm->num_workers > 1)
1480             {
1481               ip4_header_t ip = {
1482                 .src_address = local->addr,
1483               };
1484               tsm = vec_elt_at_index (sm->per_thread_data,
1485                                       sm->worker_in2out_cb (&ip, m->fib_index));
1486             }
1487           else
1488             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1489
1490           /* Delete sessions */
1491           u_key.addr = local->addr;
1492           u_key.fib_index = m->fib_index;
1493           kv.key = u_key.as_u64;
1494           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1495             {
1496               u = pool_elt_at_index (tsm->users, value.value);
1497               if (u->nstaticsessions)
1498                 {
1499                   head_index = u->sessions_per_user_list_head_index;
1500                   head = pool_elt_at_index (tsm->list_pool, head_index);
1501                   elt_index = head->next;
1502                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1503                   ses_index = elt->value;
1504                   while (ses_index != ~0)
1505                     {
1506                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1507                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1508                       ses_index = elt->value;
1509
1510                       if (!(is_lb_session (s)))
1511                         continue;
1512
1513                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1514                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1515                         continue;
1516
1517                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1518                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1519                     }
1520                 }
1521             }
1522         }
1523       if (m->affinity)
1524         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1525       vec_free(m->locals);
1526       vec_free(m->tag);
1527       vec_free(m->workers);
1528
1529       pool_put (sm->static_mappings, m);
1530     }
1531
1532   return 0;
1533 }
1534
1535 int
1536 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1537                   u8 twice_nat)
1538 {
1539   snat_address_t *a = 0;
1540   snat_session_t *ses;
1541   u32 *ses_to_be_removed = 0, *ses_index;
1542   snat_main_per_thread_data_t *tsm;
1543   snat_static_mapping_t *m;
1544   snat_interface_t *interface;
1545   int i;
1546   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1547
1548   /* Find SNAT address */
1549   for (i=0; i < vec_len (addresses); i++)
1550     {
1551       if (addresses[i].addr.as_u32 == addr.as_u32)
1552         {
1553           a = addresses + i;
1554           break;
1555         }
1556     }
1557   if (!a)
1558     return VNET_API_ERROR_NO_SUCH_ENTRY;
1559
1560   if (delete_sm)
1561     {
1562       pool_foreach (m, sm->static_mappings,
1563       ({
1564           if (m->external_addr.as_u32 == addr.as_u32)
1565             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1566                                             m->local_port, m->external_port,
1567                                             m->vrf_id, m->addr_only, ~0,
1568                                             m->proto, 0, m->twice_nat,
1569                                             m->out2in_only, m->tag);
1570       }));
1571     }
1572   else
1573     {
1574       /* Check if address is used in some static mapping */
1575       if (is_snat_address_used_in_static_mapping(sm, addr))
1576         {
1577           nat_log_notice ("address used in static mapping");
1578           return VNET_API_ERROR_UNSPECIFIED;
1579         }
1580     }
1581
1582   if (a->fib_index != ~0)
1583     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1584                      FIB_SOURCE_PLUGIN_LOW);
1585
1586   /* Delete sessions using address */
1587   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1588     {
1589       vec_foreach (tsm, sm->per_thread_data)
1590         {
1591           pool_foreach (ses, tsm->sessions, ({
1592             if (ses->out2in.addr.as_u32 == addr.as_u32)
1593               {
1594                 ses->outside_address_index = ~0;
1595                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1596                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1597               }
1598           }));
1599
1600           vec_foreach (ses_index, ses_to_be_removed)
1601             {
1602               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1603               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1604             }
1605
1606           vec_free (ses_to_be_removed);
1607        }
1608     }
1609
1610   if (twice_nat)
1611     {
1612       vec_del1 (sm->twice_nat_addresses, i);
1613       return 0;
1614     }
1615   else
1616     vec_del1 (sm->addresses, i);
1617
1618   /* Delete external address from FIB */
1619   pool_foreach (interface, sm->interfaces,
1620   ({
1621     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1622       continue;
1623
1624     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1625     break;
1626   }));
1627   pool_foreach (interface, sm->output_feature_interfaces,
1628   ({
1629     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1630       continue;
1631
1632     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1633     break;
1634   }));
1635
1636   return 0;
1637 }
1638
1639 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1640 {
1641   snat_main_t *sm = &snat_main;
1642   snat_interface_t *i;
1643   const char * feature_name, *del_feature_name;
1644   snat_address_t * ap;
1645   snat_static_mapping_t * m;
1646   snat_det_map_t * dm;
1647   nat_outside_fib_t *outside_fib;
1648   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1649                                                        sw_if_index);
1650
1651   if (sm->out2in_dpo && !is_inside)
1652     return VNET_API_ERROR_UNSUPPORTED;
1653
1654   pool_foreach (i, sm->output_feature_interfaces,
1655   ({
1656     if (i->sw_if_index == sw_if_index)
1657       return VNET_API_ERROR_VALUE_EXIST;
1658   }));
1659
1660   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1661     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1662   else
1663     {
1664       if (sm->num_workers > 1 && !sm->deterministic)
1665         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1666       else if (sm->deterministic)
1667         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1668       else if (sm->endpoint_dependent)
1669         feature_name = is_inside ?  "nat44-ed-in2out" : "nat44-ed-out2in";
1670       else
1671         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1672     }
1673
1674   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1675     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1676                                                       NAT_FQ_NELTS);
1677
1678   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1679     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1680                                                       NAT_FQ_NELTS);
1681
1682   if (!is_inside)
1683     {
1684       vec_foreach (outside_fib, sm->outside_fibs)
1685         {
1686           if (outside_fib->fib_index == fib_index)
1687             {
1688               if (is_del)
1689                 {
1690                   outside_fib->refcount--;
1691                   if (!outside_fib->refcount)
1692                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1693                 }
1694               else
1695                 outside_fib->refcount++;
1696               goto feature_set;
1697             }
1698         }
1699       if (!is_del)
1700         {
1701           vec_add2 (sm->outside_fibs, outside_fib, 1);
1702           outside_fib->refcount = 1;
1703           outside_fib->fib_index = fib_index;
1704         }
1705     }
1706 feature_set:
1707   pool_foreach (i, sm->interfaces,
1708   ({
1709     if (i->sw_if_index == sw_if_index)
1710       {
1711         if (is_del)
1712           {
1713             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1714               {
1715                 if (is_inside)
1716                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1717                 else
1718                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1719
1720                 if (sm->num_workers > 1 && !sm->deterministic)
1721                   {
1722                     del_feature_name = "nat44-handoff-classify";
1723                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1724                                                  "nat44-out2in-worker-handoff";
1725                   }
1726                 else if (sm->deterministic)
1727                   {
1728                     del_feature_name = "nat44-det-classify";
1729                     feature_name = !is_inside ?  "nat44-det-in2out" :
1730                                                  "nat44-det-out2in";
1731                   }
1732                 else if (sm->endpoint_dependent)
1733                   {
1734                     del_feature_name = "nat44-ed-classify";
1735                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1736                                                  "nat44-ed-out2in";
1737                   }
1738                 else
1739                   {
1740                     del_feature_name = "nat44-classify";
1741                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1742                   }
1743
1744                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1745                                              sw_if_index, 0, 0, 0);
1746                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1747                                              sw_if_index, 1, 0, 0);
1748                 if (!is_inside)
1749                   {
1750                     if (sm->endpoint_dependent)
1751                       vnet_feature_enable_disable ("ip4-local",
1752                                                    "nat44-ed-hairpinning",
1753                                                    sw_if_index, 1, 0, 0);
1754                     else if (!sm->deterministic)
1755                       vnet_feature_enable_disable ("ip4-local",
1756                                                    "nat44-hairpinning",
1757                                                    sw_if_index, 1, 0, 0);
1758                   }
1759               }
1760             else
1761               {
1762                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1763                                              sw_if_index, 0, 0, 0);
1764                 pool_put (sm->interfaces, i);
1765                 if (is_inside)
1766                   {
1767                     if (sm->endpoint_dependent)
1768                       vnet_feature_enable_disable ("ip4-local",
1769                                                    "nat44-ed-hairpinning",
1770                                                    sw_if_index, 0, 0, 0);
1771                     else if (!sm->deterministic)
1772                       vnet_feature_enable_disable ("ip4-local",
1773                                                    "nat44-hairpinning",
1774                                                    sw_if_index, 0, 0, 0);
1775                   }
1776               }
1777           }
1778         else
1779           {
1780             if ((nat_interface_is_inside(i) && is_inside) ||
1781                 (nat_interface_is_outside(i) && !is_inside))
1782               return 0;
1783
1784             if (sm->num_workers > 1 && !sm->deterministic)
1785               {
1786                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1787                                                  "nat44-out2in-worker-handoff";
1788                 feature_name = "nat44-handoff-classify";
1789               }
1790             else if (sm->deterministic)
1791               {
1792                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1793                                                  "nat44-det-out2in";
1794                 feature_name = "nat44-det-classify";
1795               }
1796             else if (sm->endpoint_dependent)
1797               {
1798                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1799                                                  "nat44-ed-out2in";
1800                 feature_name = "nat44-ed-classify";
1801               }
1802             else
1803               {
1804                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1805                 feature_name = "nat44-classify";
1806               }
1807
1808             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1809                                          sw_if_index, 0, 0, 0);
1810             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1811                                          sw_if_index, 1, 0, 0);
1812             if (!is_inside)
1813               {
1814                 if (sm->endpoint_dependent)
1815                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1816                                                sw_if_index, 0, 0, 0);
1817                 else if (!sm->deterministic)
1818                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1819                                                sw_if_index, 0, 0, 0);
1820               }
1821             goto set_flags;
1822           }
1823
1824         goto fib;
1825       }
1826   }));
1827
1828   if (is_del)
1829     return VNET_API_ERROR_NO_SUCH_ENTRY;
1830
1831   pool_get (sm->interfaces, i);
1832   i->sw_if_index = sw_if_index;
1833   i->flags = 0;
1834   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1835
1836   if (is_inside && !sm->out2in_dpo)
1837     {
1838       if (sm->endpoint_dependent)
1839         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1840                                      sw_if_index, 1, 0, 0);
1841       else if (!sm->deterministic)
1842         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1843                                      sw_if_index, 1, 0, 0);
1844     }
1845
1846 set_flags:
1847   if (is_inside)
1848     {
1849       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1850       return 0;
1851     }
1852   else
1853     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1854
1855   /* Add/delete external addresses to FIB */
1856 fib:
1857   vec_foreach (ap, sm->addresses)
1858     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1859
1860   pool_foreach (m, sm->static_mappings,
1861   ({
1862     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1863       continue;
1864
1865     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1866   }));
1867
1868   pool_foreach (dm, sm->det_maps,
1869   ({
1870     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1871   }));
1872
1873   return 0;
1874 }
1875
1876 int snat_interface_add_del_output_feature (u32 sw_if_index,
1877                                            u8 is_inside,
1878                                            int is_del)
1879 {
1880   snat_main_t *sm = &snat_main;
1881   snat_interface_t *i;
1882   snat_address_t * ap;
1883   snat_static_mapping_t * m;
1884
1885   if (sm->deterministic ||
1886       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1887     return VNET_API_ERROR_UNSUPPORTED;
1888
1889   pool_foreach (i, sm->interfaces,
1890   ({
1891     if (i->sw_if_index == sw_if_index)
1892       return VNET_API_ERROR_VALUE_EXIST;
1893   }));
1894
1895   if (is_inside)
1896     {
1897       if (sm->endpoint_dependent)
1898         {
1899           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1900                                        sw_if_index, !is_del, 0, 0);
1901           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1902                                        sw_if_index, !is_del, 0, 0);
1903         }
1904       else
1905         {
1906           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1907                                        sw_if_index, !is_del, 0, 0);
1908           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1909                                        sw_if_index, !is_del, 0, 0);
1910         }
1911       goto fq;
1912     }
1913
1914   if (sm->num_workers > 1)
1915     {
1916       vnet_feature_enable_disable ("ip4-unicast",
1917                                    "nat44-out2in-worker-handoff",
1918                                    sw_if_index, !is_del, 0, 0);
1919       vnet_feature_enable_disable ("ip4-output",
1920                                    "nat44-in2out-output-worker-handoff",
1921                                    sw_if_index, !is_del, 0, 0);
1922     }
1923   else
1924     {
1925       if (sm->endpoint_dependent)
1926         {
1927           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
1928                                        sw_if_index, !is_del, 0, 0);
1929           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
1930                                        sw_if_index, !is_del, 0, 0);
1931         }
1932       else
1933         {
1934           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
1935                                        sw_if_index, !is_del, 0, 0);
1936           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1937                                        sw_if_index, !is_del, 0, 0);
1938         }
1939     }
1940
1941 fq:
1942   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1943     sm->fq_in2out_output_index =
1944       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1945
1946   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1947     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1948
1949   pool_foreach (i, sm->output_feature_interfaces,
1950   ({
1951     if (i->sw_if_index == sw_if_index)
1952       {
1953         if (is_del)
1954           pool_put (sm->output_feature_interfaces, i);
1955         else
1956           return VNET_API_ERROR_VALUE_EXIST;
1957
1958         goto fib;
1959       }
1960   }));
1961
1962   if (is_del)
1963     return VNET_API_ERROR_NO_SUCH_ENTRY;
1964
1965   pool_get (sm->output_feature_interfaces, i);
1966   i->sw_if_index = sw_if_index;
1967   i->flags = 0;
1968   if (is_inside)
1969     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1970   else
1971     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1972
1973   /* Add/delete external addresses to FIB */
1974 fib:
1975   if (is_inside)
1976     return 0;
1977
1978   vec_foreach (ap, sm->addresses)
1979     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1980
1981   pool_foreach (m, sm->static_mappings,
1982   ({
1983     if (!(m->addr_only)  || (m->local_addr.as_u32 == m->external_addr.as_u32))
1984       continue;
1985
1986     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1987   }));
1988
1989   return 0;
1990 }
1991
1992 int snat_set_workers (uword * bitmap)
1993 {
1994   snat_main_t *sm = &snat_main;
1995   int i, j = 0;
1996
1997   if (sm->num_workers < 2)
1998     return VNET_API_ERROR_FEATURE_DISABLED;
1999
2000   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2001     return VNET_API_ERROR_INVALID_WORKER;
2002
2003   vec_free (sm->workers);
2004   clib_bitmap_foreach (i, bitmap,
2005     ({
2006       vec_add1(sm->workers, i);
2007       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2008       j++;
2009     }));
2010
2011   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2012   sm->num_snat_thread = _vec_len (sm->workers);
2013
2014   return 0;
2015 }
2016
2017
2018 static void
2019 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2020                                        uword opaque,
2021                                        u32 sw_if_index,
2022                                        ip4_address_t * address,
2023                                        u32 address_length,
2024                                        u32 if_address_index,
2025                                        u32 is_delete);
2026
2027 static void
2028 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2029                                  uword opaque,
2030                                  u32 sw_if_index,
2031                                  ip4_address_t * address,
2032                                  u32 address_length,
2033                                  u32 if_address_index,
2034                                  u32 is_delete);
2035
2036 static int
2037 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2038                                  u32 fib_index,
2039                                  u32 thread_index,
2040                                  snat_session_key_t * k,
2041                                  u32 * address_indexp,
2042                                  u16 port_per_thread,
2043                                  u32 snat_thread_index);
2044
2045 static clib_error_t * snat_init (vlib_main_t * vm)
2046 {
2047   snat_main_t * sm = &snat_main;
2048   clib_error_t * error = 0;
2049   ip4_main_t * im = &ip4_main;
2050   ip_lookup_main_t * lm = &im->lookup_main;
2051   uword *p;
2052   vlib_thread_registration_t *tr;
2053   vlib_thread_main_t *tm = vlib_get_thread_main ();
2054   uword *bitmap = 0;
2055   u32 i;
2056   ip4_add_del_interface_address_callback_t cb4;
2057   vlib_node_t * error_drop_node;
2058
2059   sm->vlib_main = vm;
2060   sm->vnet_main = vnet_get_main();
2061   sm->ip4_main = im;
2062   sm->ip4_lookup_main = lm;
2063   sm->api_main = &api_main;
2064   sm->first_worker_index = 0;
2065   sm->num_workers = 0;
2066   sm->num_snat_thread = 1;
2067   sm->workers = 0;
2068   sm->port_per_thread = 0xffff - 1024;
2069   sm->fq_in2out_index = ~0;
2070   sm->fq_out2in_index = ~0;
2071   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2072   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2073   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2074   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2075   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2076   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2077   sm->forwarding_enabled = 0;
2078   sm->log_class = vlib_log_register_class ("nat", 0);
2079   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2080   sm->error_node_index = error_drop_node->index;
2081
2082   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2083   if (p)
2084     {
2085       tr = (vlib_thread_registration_t *) p[0];
2086       if (tr)
2087         {
2088           sm->num_workers = tr->count;
2089           sm->first_worker_index = tr->first_index;
2090         }
2091     }
2092
2093   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2094
2095   /* Use all available workers by default */
2096   if (sm->num_workers > 1)
2097     {
2098       for (i=0; i < sm->num_workers; i++)
2099         bitmap = clib_bitmap_set (bitmap, i, 1);
2100       snat_set_workers(bitmap);
2101       clib_bitmap_free (bitmap);
2102     }
2103   else
2104     {
2105       sm->per_thread_data[0].snat_thread_index = 0;
2106     }
2107
2108   error = snat_api_init(vm, sm);
2109   if (error)
2110     return error;
2111
2112   /* Set up the interface address add/del callback */
2113   cb4.function = snat_ip4_add_del_interface_address_cb;
2114   cb4.function_opaque = 0;
2115
2116   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2117
2118   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2119   cb4.function_opaque = 0;
2120
2121   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2122
2123   nat_dpo_module_init ();
2124
2125   /* Init IPFIX logging */
2126   snat_ipfix_logging_init(vm);
2127
2128   /* Init NAT64 */
2129   error = nat64_init(vm);
2130   if (error)
2131     return error;
2132
2133   dslite_init(vm);
2134
2135   nat66_init();
2136
2137   /* Init virtual fragmenentation reassembly */
2138   return nat_reass_init(vm);
2139 }
2140
2141 VLIB_INIT_FUNCTION (snat_init);
2142
2143 void snat_free_outside_address_and_port (snat_address_t * addresses,
2144                                          u32 thread_index,
2145                                          snat_session_key_t * k)
2146 {
2147   snat_address_t *a;
2148   u32 address_index;
2149   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2150
2151   for (address_index = 0; address_index < vec_len (addresses); address_index++)
2152     {
2153       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2154         break;
2155     }
2156
2157   ASSERT (address_index < vec_len (addresses));
2158
2159   a = addresses + address_index;
2160
2161   switch (k->protocol)
2162     {
2163 #define _(N, i, n, s) \
2164     case SNAT_PROTOCOL_##N: \
2165       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2166         port_host_byte_order) == 1); \
2167       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2168         port_host_byte_order, 0); \
2169       a->busy_##n##_ports--; \
2170       a->busy_##n##_ports_per_thread[thread_index]--; \
2171       break;
2172       foreach_snat_protocol
2173 #undef _
2174     default:
2175       nat_log_info ("unknown protocol");
2176       return;
2177     }
2178 }
2179
2180 /**
2181  * @brief Match NAT44 static mapping.
2182  *
2183  * @param sm          NAT main.
2184  * @param match       Address and port to match.
2185  * @param mapping     External or local address and port of the matched mapping.
2186  * @param by_external If 0 match by local address otherwise match by external
2187  *                    address.
2188  * @param is_addr_only If matched mapping is address only
2189  * @param twice_nat If matched mapping is twice NAT.
2190  * @param lb If matched mapping is load-balanced.
2191  *
2192  * @returns 0 if match found otherwise 1.
2193  */
2194 int snat_static_mapping_match (snat_main_t * sm,
2195                                snat_session_key_t match,
2196                                snat_session_key_t * mapping,
2197                                u8 by_external,
2198                                u8 *is_addr_only,
2199                                twice_nat_type_t *twice_nat,
2200                                lb_nat_type_t *lb,
2201                                ip4_address_t * ext_host_addr)
2202 {
2203   clib_bihash_kv_8_8_t kv, value;
2204   snat_static_mapping_t *m;
2205   snat_session_key_t m_key;
2206   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2207   u32 rand, lo = 0, hi, mid;
2208   u8 backend_index;
2209
2210   m_key.fib_index = match.fib_index;
2211   if (by_external)
2212     {
2213       mapping_hash = &sm->static_mapping_by_external;
2214       m_key.fib_index = 0;
2215     }
2216
2217   m_key.addr = match.addr;
2218   m_key.port = clib_net_to_host_u16 (match.port);
2219   m_key.protocol = match.protocol;
2220
2221   kv.key = m_key.as_u64;
2222
2223   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2224     {
2225       /* Try address only mapping */
2226       m_key.port = 0;
2227       m_key.protocol = 0;
2228       kv.key = m_key.as_u64;
2229       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2230         return 1;
2231     }
2232
2233   m = pool_elt_at_index (sm->static_mappings, value.value);
2234
2235   if (by_external)
2236     {
2237       if (vec_len (m->locals))
2238         {
2239           if (PREDICT_FALSE(lb != 0))
2240             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2241           if (m->affinity)
2242             {
2243               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2244                   match.protocol, match.port, &backend_index))
2245                 goto get_local;
2246
2247               mapping->addr = m->locals[backend_index].addr;
2248               mapping->port = clib_host_to_net_u16 (m->locals[backend_index].port);
2249               mapping->fib_index = m->locals[backend_index].fib_index;
2250               goto end;
2251             }
2252 get_local:
2253           hi = vec_len (m->locals) - 1;
2254           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2255           while (lo < hi)
2256             {
2257               mid = ((hi - lo) >> 1) + lo;
2258               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2259             }
2260           if (!(m->locals[lo].prefix >= rand))
2261             return 1;
2262           if (PREDICT_FALSE (sm->num_workers > 1))
2263             {
2264               ip4_header_t ip = {
2265                 .src_address = m->locals[lo].addr,
2266               };
2267               if (sm->worker_in2out_cb (&ip, m->fib_index) != vlib_get_thread_index ())
2268                 goto get_local;
2269             }
2270           mapping->addr = m->locals[lo].addr;
2271           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2272           mapping->fib_index = m->locals[lo].fib_index;
2273           if (m->affinity)
2274             {
2275               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2276                   match.protocol, match.port, lo, m->affinity,
2277                   m->affinity_per_service_list_head_index))
2278                 nat_log_info ("create affinity record failed");
2279             }
2280         }
2281       else
2282         {
2283           if (PREDICT_FALSE(lb != 0))
2284             *lb = NO_LB_NAT;
2285           mapping->fib_index = m->fib_index;
2286           mapping->addr = m->local_addr;
2287           /* Address only mapping doesn't change port */
2288           mapping->port = m->addr_only ? match.port
2289             : clib_host_to_net_u16 (m->local_port);
2290         }
2291       mapping->protocol = m->proto;
2292     }
2293   else
2294     {
2295       mapping->addr = m->external_addr;
2296       /* Address only mapping doesn't change port */
2297       mapping->port = m->addr_only ? match.port
2298         : clib_host_to_net_u16 (m->external_port);
2299       mapping->fib_index = sm->outside_fib_index;
2300     }
2301
2302 end:
2303   if (PREDICT_FALSE(is_addr_only != 0))
2304     *is_addr_only = m->addr_only;
2305
2306   if (PREDICT_FALSE(twice_nat != 0))
2307     *twice_nat = m->twice_nat;
2308
2309   return 0;
2310 }
2311
2312 static_always_inline u16
2313 snat_random_port (u16 min, u16 max)
2314 {
2315   snat_main_t *sm = &snat_main;
2316   return min + random_u32 (&sm->random_seed) /
2317     (random_u32_max() / (max - min + 1) + 1);
2318 }
2319
2320 int
2321 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2322                                      u32 fib_index,
2323                                      u32 thread_index,
2324                                      snat_session_key_t * k,
2325                                      u32 * address_indexp,
2326                                      u16 port_per_thread,
2327                                      u32 snat_thread_index)
2328 {
2329   snat_main_t *sm = &snat_main;
2330
2331   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2332                                  address_indexp, port_per_thread,
2333                                  snat_thread_index);
2334 }
2335
2336 static int
2337 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2338                                  u32 fib_index,
2339                                  u32 thread_index,
2340                                  snat_session_key_t * k,
2341                                  u32 * address_indexp,
2342                                  u16 port_per_thread,
2343                                  u32 snat_thread_index)
2344 {
2345   int i, gi = 0;
2346   snat_address_t *a, *ga = 0;
2347   u32 portnum;
2348
2349   for (i = 0; i < vec_len (addresses); i++)
2350     {
2351       a = addresses + i;
2352       switch (k->protocol)
2353         {
2354 #define _(N, j, n, s) \
2355         case SNAT_PROTOCOL_##N: \
2356           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2357             { \
2358               if (a->fib_index == fib_index) \
2359                 { \
2360                   while (1) \
2361                     { \
2362                       portnum = (port_per_thread * \
2363                         snat_thread_index) + \
2364                         snat_random_port(1, port_per_thread) + 1024; \
2365                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2366                         continue; \
2367                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2368                       a->busy_##n##_ports_per_thread[thread_index]++; \
2369                       a->busy_##n##_ports++; \
2370                       k->addr = a->addr; \
2371                       k->port = clib_host_to_net_u16(portnum); \
2372                       *address_indexp = i; \
2373                       return 0; \
2374                     } \
2375                 } \
2376               else if (a->fib_index == ~0) \
2377                 { \
2378                   ga = a; \
2379                   gi = i; \
2380                 } \
2381             } \
2382           break;
2383           foreach_snat_protocol
2384 #undef _
2385         default:
2386           nat_log_info ("unknown protocol");
2387           return 1;
2388         }
2389
2390     }
2391
2392   if (ga)
2393     {
2394       a = ga;
2395       switch (k->protocol)
2396         {
2397 #define _(N, j, n, s) \
2398         case SNAT_PROTOCOL_##N: \
2399           while (1) \
2400             { \
2401               portnum = (port_per_thread * \
2402                 snat_thread_index) + \
2403                 snat_random_port(1, port_per_thread) + 1024; \
2404               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2405                 continue; \
2406               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2407               a->busy_##n##_ports_per_thread[thread_index]++; \
2408               a->busy_##n##_ports++; \
2409               k->addr = a->addr; \
2410               k->port = clib_host_to_net_u16(portnum); \
2411               *address_indexp = gi; \
2412               return 0; \
2413             }
2414           break;
2415           foreach_snat_protocol
2416 #undef _
2417         default:
2418           nat_log_info ("unknown protocol");
2419           return 1;
2420         }
2421     }
2422
2423   /* Totally out of translations to use... */
2424   snat_ipfix_logging_addresses_exhausted(0);
2425   return 1;
2426 }
2427
2428 static int
2429 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2430                               u32 fib_index,
2431                               u32 thread_index,
2432                               snat_session_key_t * k,
2433                               u32 * address_indexp,
2434                               u16 port_per_thread,
2435                               u32 snat_thread_index)
2436 {
2437   snat_main_t *sm = &snat_main;
2438   snat_address_t *a = addresses;
2439   u16 m, ports, portnum, A, j;
2440   m = 16 - (sm->psid_offset + sm->psid_length);
2441   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2442
2443   if (!vec_len (addresses))
2444     goto exhausted;
2445
2446   switch (k->protocol)
2447     {
2448 #define _(N, i, n, s) \
2449     case SNAT_PROTOCOL_##N: \
2450       if (a->busy_##n##_ports < ports) \
2451         { \
2452           while (1) \
2453             { \
2454               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2455               j = snat_random_port(0, pow2_mask(m)); \
2456               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2457               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2458                 continue; \
2459               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2460               a->busy_##n##_ports++; \
2461               k->addr = a->addr; \
2462               k->port = clib_host_to_net_u16 (portnum); \
2463               *address_indexp = i; \
2464               return 0; \
2465             } \
2466         } \
2467       break;
2468       foreach_snat_protocol
2469 #undef _
2470     default:
2471       nat_log_info ("unknown protocol");
2472       return 1;
2473     }
2474
2475 exhausted:
2476   /* Totally out of translations to use... */
2477   snat_ipfix_logging_addresses_exhausted(0);
2478   return 1;
2479 }
2480
2481 static int
2482 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2483                                u32 fib_index,
2484                                u32 thread_index,
2485                                snat_session_key_t * k,
2486                                u32 * address_indexp,
2487                                u16 port_per_thread,
2488                                u32 snat_thread_index)
2489 {
2490   snat_main_t *sm = &snat_main;
2491   snat_address_t *a = addresses;
2492   u16 portnum, ports;
2493
2494   ports = sm->end_port - sm->start_port + 1;
2495
2496   if (!vec_len (addresses))
2497     goto exhausted;
2498
2499   switch (k->protocol)
2500     {
2501 #define _(N, i, n, s) \
2502     case SNAT_PROTOCOL_##N: \
2503       if (a->busy_##n##_ports < ports) \
2504         { \
2505           while (1) \
2506             { \
2507               portnum = snat_random_port(sm->start_port, sm->end_port); \
2508               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2509                 continue; \
2510               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2511               a->busy_##n##_ports++; \
2512               k->addr = a->addr; \
2513               k->port = clib_host_to_net_u16 (portnum); \
2514               *address_indexp = i; \
2515               return 0; \
2516             } \
2517         } \
2518       break;
2519       foreach_snat_protocol
2520 #undef _
2521     default:
2522       nat_log_info ("unknown protocol");
2523       return 1;
2524     }
2525
2526 exhausted:
2527   /* Totally out of translations to use... */
2528   snat_ipfix_logging_addresses_exhausted(0);
2529   return 1;
2530 }
2531
2532 void
2533 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2534 {
2535   dpo_id_t dpo_v4 = DPO_INVALID;
2536   fib_prefix_t pfx = {
2537     .fp_proto = FIB_PROTOCOL_IP4,
2538     .fp_len = 32,
2539     .fp_addr.ip4.as_u32 = addr.as_u32,
2540   };
2541
2542   if (is_add)
2543     {
2544       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2545       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2546                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2547       dpo_reset (&dpo_v4);
2548     }
2549   else
2550     {
2551       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2552     }
2553 }
2554
2555 uword
2556 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2557 {
2558   u32 *r = va_arg (*args, u32 *);
2559
2560   if (0);
2561 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2562   foreach_snat_protocol
2563 #undef _
2564   else
2565     return 0;
2566   return 1;
2567 }
2568
2569 u8 *
2570 format_snat_protocol (u8 * s, va_list * args)
2571 {
2572   u32 i = va_arg (*args, u32);
2573   u8 *t = 0;
2574
2575   switch (i)
2576     {
2577 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2578       foreach_snat_protocol
2579 #undef _
2580     default:
2581       s = format (s, "unknown");
2582       return s;
2583     }
2584   s = format (s, "%s", t);
2585   return s;
2586 }
2587
2588 u8 *
2589 format_nat_addr_and_port_alloc_alg (u8 * s, va_list * args)
2590 {
2591   u32 i = va_arg (*args, u32);
2592   u8 *t = 0;
2593
2594   switch (i)
2595     {
2596 #define _(v, N, s) case NAT_ADDR_AND_PORT_ALLOC_ALG_##N: t = (u8 *) s; break;
2597       foreach_nat_addr_and_port_alloc_alg
2598 #undef _
2599     default:
2600       s = format (s, "unknown");
2601       return s;
2602     }
2603   s = format (s, "%s", t);
2604   return s;
2605 }
2606
2607 u8 * format_snat_key (u8 * s, va_list * args);
2608 u8 * format_static_mapping_key (u8 * s, va_list * args);
2609
2610 u8 *
2611 format_session_kvp (u8 * s, va_list * args)
2612 {
2613   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2614   snat_session_key_t k;
2615
2616   k.as_u64 = v->key;
2617
2618   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2619
2620   return s;
2621 }
2622
2623 u8 *
2624 format_static_mapping_kvp (u8 * s, va_list * args)
2625 {
2626   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2627   snat_session_key_t k;
2628
2629   k.as_u64 = v->key;
2630
2631   s = format (s, "%U static-mapping-index %llu",
2632               format_static_mapping_key, &k, v->value);
2633
2634   return s;
2635 }
2636
2637 u8 *
2638 format_user_kvp (u8 * s, va_list * args)
2639 {
2640   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2641   snat_user_key_t k;
2642
2643   k.as_u64 = v->key;
2644
2645   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2646               k.fib_index, v->value);
2647
2648   return s;
2649 }
2650
2651 u8 *
2652 format_ed_session_kvp (u8 * s, va_list * args)
2653 {
2654   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2655   nat_ed_ses_key_t k;
2656
2657   k.as_u64[0] = v->key[0];
2658   k.as_u64[1] = v->key[1];
2659
2660   s = format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2661               format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2662               format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2663               format_ip_protocol, k.proto, k.fib_index, v->value);
2664
2665   return s;
2666 }
2667
2668 static u32
2669 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2670 {
2671   snat_main_t *sm = &snat_main;
2672   u32 next_worker_index = 0;
2673   u32 hash;
2674
2675   next_worker_index = sm->first_worker_index;
2676   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2677          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2678
2679   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2680     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2681   else
2682     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2683
2684   return next_worker_index;
2685 }
2686
2687 static u32
2688 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2689 {
2690   snat_main_t *sm = &snat_main;
2691   udp_header_t *udp;
2692   u16 port;
2693   snat_session_key_t m_key;
2694   clib_bihash_kv_8_8_t kv, value;
2695   snat_static_mapping_t *m;
2696   u32 proto;
2697   u32 next_worker_index = 0;
2698
2699   /* first try static mappings without port */
2700   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2701     {
2702       m_key.addr = ip0->dst_address;
2703       m_key.port = 0;
2704       m_key.protocol = 0;
2705       m_key.fib_index = rx_fib_index0;
2706       kv.key = m_key.as_u64;
2707       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2708         {
2709           m = pool_elt_at_index (sm->static_mappings, value.value);
2710           return m->workers[0];
2711         }
2712     }
2713
2714   proto = ip_proto_to_snat_proto (ip0->protocol);
2715   udp = ip4_next_header (ip0);
2716   port = udp->dst_port;
2717
2718   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2719     {
2720       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2721         return vlib_get_thread_index ();
2722
2723       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2724         {
2725           nat_reass_ip4_t *reass;
2726
2727           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2728                                       ip0->fragment_id, ip0->protocol);
2729
2730           if (reass && (reass->thread_index != (u32) ~ 0))
2731             return reass->thread_index;
2732           else
2733             return vlib_get_thread_index ();
2734         }
2735     }
2736
2737   /* unknown protocol */
2738   if (PREDICT_FALSE (proto == ~0))
2739     {
2740       /* use current thread */
2741       return vlib_get_thread_index ();
2742     }
2743
2744   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2745     {
2746       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2747       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2748       if (!icmp_is_error_message (icmp))
2749         port = echo->identifier;
2750       else
2751         {
2752           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2753           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2754           void *l4_header = ip4_next_header (inner_ip);
2755           switch (proto)
2756             {
2757             case SNAT_PROTOCOL_ICMP:
2758               icmp = (icmp46_header_t*)l4_header;
2759               echo = (icmp_echo_header_t *)(icmp + 1);
2760               port = echo->identifier;
2761               break;
2762             case SNAT_PROTOCOL_UDP:
2763             case SNAT_PROTOCOL_TCP:
2764               port = ((tcp_udp_header_t*)l4_header)->src_port;
2765               break;
2766             default:
2767               return vlib_get_thread_index ();
2768             }
2769         }
2770     }
2771
2772   /* try static mappings with port */
2773   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2774     {
2775       m_key.addr = ip0->dst_address;
2776       m_key.port = clib_net_to_host_u16 (port);
2777       m_key.protocol = proto;
2778       m_key.fib_index = rx_fib_index0;
2779       kv.key = m_key.as_u64;
2780       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2781         {
2782           m = pool_elt_at_index (sm->static_mappings, value.value);
2783           return m->workers[0];
2784         }
2785     }
2786
2787   /* worker by outside port */
2788   next_worker_index = sm->first_worker_index;
2789   next_worker_index +=
2790     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2791   return next_worker_index;
2792 }
2793
2794 static u32
2795 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2796 {
2797   snat_main_t *sm = &snat_main;
2798   clib_bihash_kv_8_8_t kv, value;
2799   u32 proto, next_worker_index = 0;
2800   udp_header_t *udp;
2801   u16 port;
2802   snat_static_mapping_t *m;
2803   u32 hash;
2804
2805   /* first try static mappings without port */
2806   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2807     {
2808       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2809       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2810         {
2811           m = pool_elt_at_index (sm->static_mappings, value.value);
2812           return m->workers[0];
2813         }
2814     }
2815
2816   proto = ip_proto_to_snat_proto (ip->protocol);
2817
2818   /* unknown protocol */
2819   if (PREDICT_FALSE (proto == ~0))
2820     {
2821       /* use current thread */
2822       return vlib_get_thread_index ();
2823     }
2824
2825   udp = ip4_next_header (ip);
2826   port = udp->dst_port;
2827
2828   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2829     {
2830       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2831       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2832       if (!icmp_is_error_message (icmp))
2833         port = echo->identifier;
2834       else
2835         {
2836           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2837           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2838           void *l4_header = ip4_next_header (inner_ip);
2839           switch (proto)
2840             {
2841             case SNAT_PROTOCOL_ICMP:
2842               icmp = (icmp46_header_t*)l4_header;
2843               echo = (icmp_echo_header_t *)(icmp + 1);
2844               port = echo->identifier;
2845               break;
2846             case SNAT_PROTOCOL_UDP:
2847             case SNAT_PROTOCOL_TCP:
2848               port = ((tcp_udp_header_t*)l4_header)->src_port;
2849               break;
2850             default:
2851               return vlib_get_thread_index ();
2852             }
2853         }
2854     }
2855
2856   /* try static mappings with port */
2857   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2858     {
2859       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2860                   clib_net_to_host_u16 (port));
2861       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2862         {
2863           m = pool_elt_at_index (sm->static_mappings, value.value);
2864           if (!vec_len(m->locals))
2865             return m->workers[0];
2866
2867           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2868                  (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >>24);
2869
2870           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2871             return m->workers[hash & (_vec_len (m->workers) - 1)];
2872           else
2873             return m->workers[hash % _vec_len (m->workers)];
2874         }
2875     }
2876
2877   /* worker by outside port */
2878   next_worker_index = sm->first_worker_index;
2879   next_worker_index +=
2880     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2881
2882   return next_worker_index;
2883 }
2884
2885 static clib_error_t *
2886 snat_config (vlib_main_t * vm, unformat_input_t * input)
2887 {
2888   snat_main_t * sm = &snat_main;
2889   nat66_main_t * nm = &nat66_main;
2890   u32 translation_buckets = 1024;
2891   u32 translation_memory_size = 128<<20;
2892   u32 user_buckets = 128;
2893   u32 user_memory_size = 64<<20;
2894   u32 max_translations_per_user = 100;
2895   u32 outside_vrf_id = 0;
2896   u32 outside_ip6_vrf_id = 0;
2897   u32 inside_vrf_id = 0;
2898   u32 static_mapping_buckets = 1024;
2899   u32 static_mapping_memory_size = 64<<20;
2900   u32 nat64_bib_buckets = 1024;
2901   u32 nat64_bib_memory_size = 128 << 20;
2902   u32 nat64_st_buckets = 2048;
2903   u32 nat64_st_memory_size = 256 << 20;
2904   u8 static_mapping_only = 0;
2905   u8 static_mapping_connection_tracking = 0;
2906   snat_main_per_thread_data_t *tsm;
2907   dslite_main_t * dm = &dslite_main;
2908
2909   sm->deterministic = 0;
2910   sm->out2in_dpo = 0;
2911   sm->endpoint_dependent = 0;
2912
2913   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2914     {
2915       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2916         ;
2917       else if (unformat (input, "translation hash memory %d",
2918                          &translation_memory_size));
2919       else if (unformat (input, "user hash buckets %d", &user_buckets))
2920         ;
2921       else if (unformat (input, "user hash memory %d",
2922                          &user_memory_size))
2923         ;
2924       else if (unformat (input, "max translations per user %d",
2925                          &max_translations_per_user))
2926         ;
2927       else if (unformat (input, "outside VRF id %d",
2928                          &outside_vrf_id))
2929         ;
2930       else if (unformat (input, "outside ip6 VRF id %d",
2931                          &outside_ip6_vrf_id))
2932         ;
2933       else if (unformat (input, "inside VRF id %d",
2934                          &inside_vrf_id))
2935         ;
2936       else if (unformat (input, "static mapping only"))
2937         {
2938           static_mapping_only = 1;
2939           if (unformat (input, "connection tracking"))
2940             static_mapping_connection_tracking = 1;
2941         }
2942       else if (unformat (input, "deterministic"))
2943         sm->deterministic = 1;
2944       else if (unformat (input, "nat64 bib hash buckets %d",
2945                          &nat64_bib_buckets))
2946         ;
2947       else if (unformat (input, "nat64 bib hash memory %d",
2948                          &nat64_bib_memory_size))
2949         ;
2950       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2951         ;
2952       else if (unformat (input, "nat64 st hash memory %d",
2953                          &nat64_st_memory_size))
2954         ;
2955       else if (unformat (input, "out2in dpo"))
2956         sm->out2in_dpo = 1;
2957       else if (unformat (input, "dslite ce"))
2958         dslite_set_ce(dm, 1);
2959       else if (unformat (input, "endpoint-dependent"))
2960         sm->endpoint_dependent = 1;
2961       else
2962         return clib_error_return (0, "unknown input '%U'",
2963                                   format_unformat_error, input);
2964     }
2965
2966   if (sm->deterministic && sm->endpoint_dependent)
2967     return clib_error_return (
2968       0, "deterministic and endpoint-dependent modes are mutually exclusive");
2969
2970   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
2971     return clib_error_return (
2972       0, "static mapping only mode available only for simple nat");
2973
2974   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
2975     return clib_error_return (
2976       0, "out2in dpo mode available only for simple nat");
2977
2978   /* for show commands, etc. */
2979   sm->translation_buckets = translation_buckets;
2980   sm->translation_memory_size = translation_memory_size;
2981   /* do not exceed load factor 10 */
2982   sm->max_translations = 10 * translation_buckets;
2983   sm->user_buckets = user_buckets;
2984   sm->user_memory_size = user_memory_size;
2985   sm->max_translations_per_user = max_translations_per_user;
2986   sm->outside_vrf_id = outside_vrf_id;
2987   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2988                                                              outside_vrf_id,
2989                                                              FIB_SOURCE_PLUGIN_HI);
2990   nm->outside_vrf_id = outside_ip6_vrf_id;
2991   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2992                                                              outside_ip6_vrf_id,
2993                                                              FIB_SOURCE_PLUGIN_HI);
2994   sm->inside_vrf_id = inside_vrf_id;
2995   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2996                                                             inside_vrf_id,
2997                                                             FIB_SOURCE_PLUGIN_HI);
2998   sm->static_mapping_only = static_mapping_only;
2999   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3000
3001   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3002                  nat64_st_memory_size);
3003
3004   if (sm->deterministic)
3005     {
3006       sm->in2out_node_index = snat_det_in2out_node.index;
3007       sm->in2out_output_node_index = ~0;
3008       sm->out2in_node_index = snat_det_out2in_node.index;
3009       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3010       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3011     }
3012   else
3013     {
3014       if (sm->endpoint_dependent)
3015         {
3016           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3017           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3018           sm->in2out_node_index = nat44_ed_in2out_node.index;
3019           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3020           sm->out2in_node_index = nat44_ed_out2in_node.index;
3021           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3022           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3023           nat_affinity_init (vm);
3024         }
3025       else
3026         {
3027           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3028           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3029           sm->in2out_node_index = snat_in2out_node.index;
3030           sm->in2out_output_node_index = snat_in2out_output_node.index;
3031           sm->out2in_node_index = snat_out2in_node.index;
3032           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3033           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3034         }
3035       if (!static_mapping_only ||
3036           (static_mapping_only && static_mapping_connection_tracking))
3037         {
3038           vec_foreach (tsm, sm->per_thread_data)
3039             {
3040               if (sm->endpoint_dependent)
3041                 {
3042                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3043                                          translation_buckets,
3044                                          translation_memory_size);
3045                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3046                                                       format_ed_session_kvp);
3047
3048                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3049                                          translation_buckets,
3050                                          translation_memory_size);
3051                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3052                                                       format_ed_session_kvp);
3053                 }
3054               else
3055                 {
3056                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3057                                         translation_buckets,
3058                                         translation_memory_size);
3059                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3060                                                      format_session_kvp);
3061
3062                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3063                                         translation_buckets,
3064                                         translation_memory_size);
3065                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3066                                                      format_session_kvp);
3067                 }
3068
3069               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3070                                     user_memory_size);
3071               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3072                                                  format_user_kvp);
3073             }
3074
3075         }
3076       else
3077         {
3078           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3079           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3080         }
3081       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3082                             "static_mapping_by_local", static_mapping_buckets,
3083                             static_mapping_memory_size);
3084       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3085                                          format_static_mapping_kvp);
3086
3087       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3088                             "static_mapping_by_external", static_mapping_buckets,
3089                             static_mapping_memory_size);
3090       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3091                                          format_static_mapping_kvp);
3092     }
3093
3094   return 0;
3095 }
3096
3097 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3098
3099 u8 * format_snat_session_state (u8 * s, va_list * args)
3100 {
3101   u32 i = va_arg (*args, u32);
3102   u8 *t = 0;
3103
3104   switch (i)
3105     {
3106 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
3107     foreach_snat_session_state
3108 #undef _
3109     default:
3110       t = format (t, "unknown");
3111     }
3112   s = format (s, "%s", t);
3113   return s;
3114 }
3115
3116 u8 * format_snat_key (u8 * s, va_list * args)
3117 {
3118   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3119
3120   s = format (s, "%U proto %U port %d fib %d",
3121               format_ip4_address, &key->addr,
3122               format_snat_protocol, key->protocol,
3123               clib_net_to_host_u16 (key->port), key->fib_index);
3124   return s;
3125 }
3126
3127 u8 * format_static_mapping_key (u8 * s, va_list * args)
3128 {
3129   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3130
3131   s = format (s, "%U proto %U port %d fib %d",
3132               format_ip4_address, &key->addr,
3133               format_snat_protocol, key->protocol,
3134               key->port, key->fib_index);
3135   return s;
3136 }
3137
3138 u8 * format_snat_session (u8 * s, va_list * args)
3139 {
3140   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3141   snat_session_t * sess = va_arg (*args, snat_session_t *);
3142
3143   if (snat_is_unk_proto_session (sess))
3144     {
3145       s = format (s, "  i2o %U proto %u fib %u\n",
3146                   format_ip4_address, &sess->in2out.addr,
3147                   clib_net_to_host_u16 (sess->in2out.port),
3148                   sess->in2out.fib_index);
3149       s = format (s, "    o2i %U proto %u fib %u\n",
3150                   format_ip4_address, &sess->out2in.addr,
3151                   clib_net_to_host_u16 (sess->out2in.port),
3152                   sess->out2in.fib_index);
3153     }
3154   else
3155     {
3156       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
3157       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
3158     }
3159   if (is_ed_session (sess) || is_fwd_bypass_session (sess))
3160     {
3161       if (is_twice_nat_session (sess))
3162         {
3163           s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
3164                       format_ip4_address, &sess->ext_host_addr,
3165                       clib_net_to_host_u16 (sess->ext_host_port),
3166                       format_ip4_address, &sess->ext_host_nat_addr,
3167                       clib_net_to_host_u16 (sess->ext_host_nat_port));
3168         }
3169       else
3170         {
3171           if (sess->ext_host_addr.as_u32)
3172               s = format (s, "       external host %U:%u\n",
3173                           format_ip4_address, &sess->ext_host_addr,
3174                           clib_net_to_host_u16 (sess->ext_host_port));
3175         }
3176     }
3177   s = format (s, "       index %llu\n", sess - sm->sessions);
3178   s = format (s, "       last heard %.2f\n", sess->last_heard);
3179   s = format (s, "       total pkts %d, total bytes %lld\n",
3180               sess->total_pkts, sess->total_bytes);
3181   if (snat_is_session_static (sess))
3182     s = format (s, "       static translation\n");
3183   else
3184     s = format (s, "       dynamic translation\n");
3185   if (is_fwd_bypass_session (sess))
3186     s = format (s, "       forwarding-bypass\n");
3187   if (is_lb_session (sess))
3188     s = format (s, "       load-balancing\n");
3189   if (is_twice_nat_session (sess))
3190     s = format (s, "       twice-nat\n");
3191
3192   return s;
3193 }
3194
3195 u8 * format_snat_user (u8 * s, va_list * args)
3196 {
3197   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3198   snat_user_t * u = va_arg (*args, snat_user_t *);
3199   int verbose = va_arg (*args, int);
3200   dlist_elt_t * head, * elt;
3201   u32 elt_index, head_index;
3202   u32 session_index;
3203   snat_session_t * sess;
3204
3205   s = format (s, "%U: %d dynamic translations, %d static translations\n",
3206               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
3207
3208   if (verbose == 0)
3209     return s;
3210
3211   if (u->nsessions || u->nstaticsessions)
3212     {
3213       head_index = u->sessions_per_user_list_head_index;
3214       head = pool_elt_at_index (sm->list_pool, head_index);
3215
3216       elt_index = head->next;
3217       elt = pool_elt_at_index (sm->list_pool, elt_index);
3218       session_index = elt->value;
3219
3220       while (session_index != ~0)
3221         {
3222           sess = pool_elt_at_index (sm->sessions, session_index);
3223
3224           s = format (s, "  %U\n", format_snat_session, sm, sess);
3225
3226           elt_index = elt->next;
3227           elt = pool_elt_at_index (sm->list_pool, elt_index);
3228           session_index = elt->value;
3229         }
3230     }
3231
3232   return s;
3233 }
3234
3235 u8 * format_snat_static_mapping (u8 * s, va_list * args)
3236 {
3237   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
3238   nat44_lb_addr_port_t *local;
3239
3240   if (m->addr_only)
3241       s = format (s, "local %U external %U vrf %d %s %s",
3242                   format_ip4_address, &m->local_addr,
3243                   format_ip4_address, &m->external_addr,
3244                   m->vrf_id,
3245                   m->twice_nat == TWICE_NAT ? "twice-nat" :
3246                   m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3247                   m->out2in_only ? "out2in-only" : "");
3248   else
3249    {
3250       if (vec_len (m->locals))
3251         {
3252           s = format (s, "%U external %U:%d %s %s",
3253                       format_snat_protocol, m->proto,
3254                       format_ip4_address, &m->external_addr, m->external_port,
3255                       m->twice_nat == TWICE_NAT ? "twice-nat" :
3256                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3257                       m->out2in_only ? "out2in-only" : "");
3258           vec_foreach (local, m->locals)
3259             s = format (s, "\n  local %U:%d vrf %d probability %d\%",
3260                         format_ip4_address, &local->addr, local->port,
3261                         local->vrf_id, local->probability);
3262         }
3263       else
3264         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
3265                     format_snat_protocol, m->proto,
3266                     format_ip4_address, &m->local_addr, m->local_port,
3267                     format_ip4_address, &m->external_addr, m->external_port,
3268                     m->vrf_id,
3269                     m->twice_nat == TWICE_NAT ? "twice-nat" :
3270                     m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3271                     m->out2in_only ? "out2in-only" : "");
3272    }
3273   return s;
3274 }
3275
3276 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
3277 {
3278   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
3279   vnet_main_t *vnm = vnet_get_main();
3280
3281   if (m->addr_only)
3282       s = format (s, "local %U external %U vrf %d",
3283                   format_ip4_address, &m->l_addr,
3284                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3285                   m->vrf_id);
3286   else
3287       s = format (s, "%U local %U:%d external %U:%d vrf %d",
3288                   format_snat_protocol, m->proto,
3289                   format_ip4_address, &m->l_addr, m->l_port,
3290                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3291                   m->e_port, m->vrf_id);
3292
3293   return s;
3294 }
3295
3296 u8 * format_det_map_ses (u8 * s, va_list * args)
3297 {
3298   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
3299   ip4_address_t in_addr, out_addr;
3300   u32 in_offset, out_offset;
3301   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
3302   u32 * i = va_arg (*args, u32 *);
3303
3304   u32 user_index = *i / SNAT_DET_SES_PER_USER;
3305   in_addr.as_u32 = clib_host_to_net_u32 (
3306     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
3307   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
3308     clib_net_to_host_u32(det_map->in_addr.as_u32);
3309   out_offset = in_offset / det_map->sharing_ratio;
3310   out_addr.as_u32 = clib_host_to_net_u32(
3311     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
3312   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
3313               format_ip4_address, &in_addr,
3314               clib_net_to_host_u16 (ses->in_port),
3315               format_ip4_address, &out_addr,
3316               clib_net_to_host_u16 (ses->out.out_port),
3317               format_ip4_address, &ses->out.ext_host_addr,
3318               clib_net_to_host_u16 (ses->out.ext_host_port),
3319               format_snat_session_state, ses->state,
3320               ses->expire);
3321
3322   return s;
3323 }
3324
3325 static void
3326 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3327                                  uword opaque,
3328                                  u32 sw_if_index,
3329                                  ip4_address_t * address,
3330                                  u32 address_length,
3331                                  u32 if_address_index,
3332                                  u32 is_delete)
3333 {
3334   snat_main_t *sm = &snat_main;
3335   snat_static_map_resolve_t *rp;
3336   snat_static_mapping_t *m;
3337   snat_session_key_t m_key;
3338   clib_bihash_kv_8_8_t kv, value;
3339   int i, rv;
3340   ip4_address_t l_addr;
3341
3342   for (i = 0; i < vec_len (sm->to_resolve); i++)
3343     {
3344       rp = sm->to_resolve + i;
3345       if (rp->addr_only == 0)
3346         continue;
3347       if (rp->sw_if_index == sw_if_index)
3348         goto match;
3349     }
3350
3351   return;
3352
3353 match:
3354   m_key.addr.as_u32 = address->as_u32;
3355   m_key.port = rp->addr_only ? 0 : rp->e_port;
3356   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3357   m_key.fib_index = sm->outside_fib_index;
3358   kv.key = m_key.as_u64;
3359   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3360     m = 0;
3361   else
3362     m = pool_elt_at_index (sm->static_mappings, value.value);
3363
3364   if (!is_delete)
3365     {
3366       /* Don't trip over lease renewal, static config */
3367       if (m)
3368         return;
3369     }
3370   else
3371     {
3372       if (!m)
3373         return;
3374     }
3375
3376   /* Indetity mapping? */
3377   if (rp->l_addr.as_u32 == 0)
3378     l_addr.as_u32 = address[0].as_u32;
3379   else
3380     l_addr.as_u32 = rp->l_addr.as_u32;
3381   /* Add the static mapping */
3382   rv = snat_add_static_mapping (l_addr,
3383                                 address[0],
3384                                 rp->l_port,
3385                                 rp->e_port,
3386                                 rp->vrf_id,
3387                                 rp->addr_only,
3388                                 ~0 /* sw_if_index */,
3389                                 rp->proto,
3390                                 !is_delete,
3391                                 0, 0, rp->tag);
3392   if (rv)
3393     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3394 }
3395
3396 static void
3397 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3398                                        uword opaque,
3399                                        u32 sw_if_index,
3400                                        ip4_address_t * address,
3401                                        u32 address_length,
3402                                        u32 if_address_index,
3403                                        u32 is_delete)
3404 {
3405   snat_main_t *sm = &snat_main;
3406   snat_static_map_resolve_t *rp;
3407   ip4_address_t l_addr;
3408   int i, j;
3409   int rv;
3410   u8 twice_nat = 0;
3411   snat_address_t *addresses = sm->addresses;
3412
3413   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
3414     {
3415       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3416           goto match;
3417     }
3418
3419   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
3420     {
3421       twice_nat = 1;
3422       addresses = sm->twice_nat_addresses;
3423       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3424           goto match;
3425     }
3426
3427   return;
3428
3429 match:
3430   if (!is_delete)
3431     {
3432       /* Don't trip over lease renewal, static config */
3433       for (j = 0; j < vec_len(addresses); j++)
3434         if (addresses[j].addr.as_u32 == address->as_u32)
3435           return;
3436
3437       (void) snat_add_address (sm, address, ~0, twice_nat);
3438       /* Scan static map resolution vector */
3439       for (j = 0; j < vec_len (sm->to_resolve); j++)
3440         {
3441           rp = sm->to_resolve + j;
3442           if (rp->addr_only)
3443             continue;
3444           /* On this interface? */
3445           if (rp->sw_if_index == sw_if_index)
3446             {
3447               /* Indetity mapping? */
3448               if (rp->l_addr.as_u32 == 0)
3449                 l_addr.as_u32 = address[0].as_u32;
3450               else
3451                 l_addr.as_u32 = rp->l_addr.as_u32;
3452               /* Add the static mapping */
3453               rv = snat_add_static_mapping (l_addr,
3454                                             address[0],
3455                                             rp->l_port,
3456                                             rp->e_port,
3457                                             rp->vrf_id,
3458                                             rp->addr_only,
3459                                             ~0 /* sw_if_index */,
3460                                             rp->proto,
3461                                             rp->is_add,
3462                                             0, 0, rp->tag);
3463               if (rv)
3464                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3465             }
3466         }
3467       return;
3468     }
3469   else
3470     {
3471       (void) snat_del_address(sm, address[0], 1, twice_nat);
3472       return;
3473     }
3474 }
3475
3476
3477 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
3478                                 u8 twice_nat)
3479 {
3480   ip4_main_t * ip4_main = sm->ip4_main;
3481   ip4_address_t * first_int_addr;
3482   snat_static_map_resolve_t *rp;
3483   u32 *indices_to_delete = 0;
3484   int i, j;
3485   u32 *auto_add_sw_if_indices =
3486     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3487
3488   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
3489                                                 0 /* just want the address*/);
3490
3491   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
3492     {
3493       if (auto_add_sw_if_indices[i] == sw_if_index)
3494         {
3495           if (is_del)
3496             {
3497               /* if have address remove it */
3498               if (first_int_addr)
3499                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3500               else
3501                 {
3502                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3503                     {
3504                       rp = sm->to_resolve + j;
3505                       if (rp->sw_if_index == sw_if_index)
3506                         vec_add1 (indices_to_delete, j);
3507                     }
3508                   if (vec_len(indices_to_delete))
3509                     {
3510                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3511                         vec_del1(sm->to_resolve, j);
3512                       vec_free(indices_to_delete);
3513                     }
3514                 }
3515               if (twice_nat)
3516                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3517               else
3518                 vec_del1(sm->auto_add_sw_if_indices, i);
3519             }
3520           else
3521             return VNET_API_ERROR_VALUE_EXIST;
3522
3523           return 0;
3524         }
3525     }
3526
3527   if (is_del)
3528     return VNET_API_ERROR_NO_SUCH_ENTRY;
3529
3530   /* add to the auto-address list */
3531   if (twice_nat)
3532     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3533   else
3534     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3535
3536   /* If the address is already bound - or static - add it now */
3537   if (first_int_addr)
3538       (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3539
3540   return 0;
3541 }
3542
3543 int
3544 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3545                    snat_protocol_t proto, u32 vrf_id, int is_in)
3546 {
3547   snat_main_per_thread_data_t *tsm;
3548   clib_bihash_kv_8_8_t kv, value;
3549   ip4_header_t ip;
3550   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3551   snat_session_key_t key;
3552   snat_session_t *s;
3553   clib_bihash_8_8_t *t;
3554
3555   if (sm->endpoint_dependent)
3556     return VNET_API_ERROR_UNSUPPORTED;
3557
3558   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3559   if (sm->num_workers > 1)
3560     tsm =
3561       vec_elt_at_index (sm->per_thread_data,
3562                         sm->worker_in2out_cb (&ip, fib_index));
3563   else
3564     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3565
3566   key.addr.as_u32 = addr->as_u32;
3567   key.port = clib_host_to_net_u16 (port);
3568   key.protocol = proto;
3569   key.fib_index = fib_index;
3570   kv.key = key.as_u64;
3571   t = is_in ? &tsm->in2out : &tsm->out2in;
3572   if (!clib_bihash_search_8_8 (t, &kv, &value))
3573     {
3574       if (pool_is_free_index (tsm->sessions, value.value))
3575         return VNET_API_ERROR_UNSPECIFIED;
3576
3577       s = pool_elt_at_index (tsm->sessions, value.value);
3578       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3579       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3580       return 0;
3581     }
3582
3583   return VNET_API_ERROR_NO_SUCH_ENTRY;
3584 }
3585
3586 int
3587 nat44_del_ed_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3588                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3589                       u32 vrf_id, int is_in)
3590 {
3591   ip4_header_t ip;
3592   clib_bihash_16_8_t *t;
3593   nat_ed_ses_key_t key;
3594   clib_bihash_kv_16_8_t kv, value;
3595   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3596   snat_session_t *s;
3597   snat_main_per_thread_data_t *tsm;
3598
3599   if (!sm->endpoint_dependent)
3600     return VNET_API_ERROR_FEATURE_DISABLED;
3601
3602   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3603   if (sm->num_workers > 1)
3604     tsm =
3605       vec_elt_at_index (sm->per_thread_data,
3606                         sm->worker_in2out_cb (&ip, fib_index));
3607   else
3608     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3609
3610   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3611   key.l_addr.as_u32 = addr->as_u32;
3612   key.r_addr.as_u32 = eh_addr->as_u32;
3613   key.l_port = clib_host_to_net_u16 (port);
3614   key.r_port = clib_host_to_net_u16 (eh_port);
3615   key.proto = proto;
3616   key.fib_index = clib_host_to_net_u32 (fib_index);
3617   kv.key[0] = key.as_u64[0];
3618   kv.key[1] = key.as_u64[1];
3619   if (clib_bihash_search_16_8 (t, &kv, &value))
3620     return VNET_API_ERROR_NO_SUCH_ENTRY;
3621
3622   if (pool_is_free_index (tsm->sessions, value.value))
3623     return VNET_API_ERROR_UNSPECIFIED;
3624   s = pool_elt_at_index (tsm->sessions, value.value);
3625   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3626   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3627   return 0;
3628 }
3629
3630 void
3631 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3632 {
3633   snat_main_t *sm = &snat_main;
3634
3635   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3636   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3637   sm->psid = psid;
3638   sm->psid_offset = psid_offset;
3639   sm->psid_length = psid_length;
3640 }
3641
3642 void
3643 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3644 {
3645   snat_main_t *sm = &snat_main;
3646
3647   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3648   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3649   sm->start_port = start_port;
3650   sm->end_port = end_port;
3651 }
3652
3653 void
3654 nat_set_alloc_addr_and_port_default (void)
3655 {
3656   snat_main_t *sm = &snat_main;
3657
3658   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3659   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3660 }
3661