src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat_affinity.h>
  32 #include <vnet/fib/fib_table.h>
  33 #include <vnet/fib/ip4_fib.h>
  34
  35 #include <vpp/app/version.h>
  36
  37 snat_main_t snat_main;
  38
  39
  40 /* Hook up input features */
  41 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  42   .arc_name = "ip4-unicast",
  43   .node_name = "nat44-in2out",
  44   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  45 };
  46 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  47   .arc_name = "ip4-unicast",
  48   .node_name = "nat44-out2in",
  49   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  50                                "ip4-dhcp-client-detect"),
  51 };
  52 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  53   .arc_name = "ip4-unicast",
  54   .node_name = "nat44-classify",
  55   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  56 };
  57 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  58   .arc_name = "ip4-unicast",
  59   .node_name = "nat44-det-in2out",
  60   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  61 };
  62 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  63   .arc_name = "ip4-unicast",
  64   .node_name = "nat44-det-out2in",
  65   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  66                                "ip4-dhcp-client-detect"),
  67 };
  68 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  69   .arc_name = "ip4-unicast",
  70   .node_name = "nat44-det-classify",
  71   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  72 };
  73 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  74   .arc_name = "ip4-unicast",
  75   .node_name = "nat44-ed-in2out",
  76   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  77 };
  78 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  79   .arc_name = "ip4-unicast",
  80   .node_name = "nat44-ed-out2in",
  81   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  82                                "ip4-dhcp-client-detect"),
  83 };
  84 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-ed-classify",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-in2out-worker-handoff",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  93 };
  94 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-out2in-worker-handoff",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  98                                "ip4-dhcp-client-detect"),
  99 };
 100 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-handoff-classify",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 104 };
 105 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 106   .arc_name = "ip4-unicast",
 107   .node_name = "nat44-in2out-fast",
 108   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 109 };
 110 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 111   .arc_name = "ip4-unicast",
 112   .node_name = "nat44-out2in-fast",
 113   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 114                                "ip4-dhcp-client-detect"),
 115 };
 116 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 117   .arc_name = "ip4-unicast",
 118   .node_name = "nat44-hairpin-dst",
 119   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 120 };
 121 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 122   .arc_name = "ip4-unicast",
 123   .node_name = "nat44-ed-hairpin-dst",
 124   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 125 };
 126
 127 /* Hook up output features */
 128 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 129   .arc_name = "ip4-output",
 130   .node_name = "nat44-in2out-output",
 131   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 132 };
 133 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 134   .arc_name = "ip4-output",
 135   .node_name = "nat44-in2out-output-worker-handoff",
 136   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-hairpin-src",
 141   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 142 };
 143 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 144   .arc_name = "ip4-output",
 145   .node_name = "nat44-ed-in2out-output",
 146   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 147 };
 148 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 149   .arc_name = "ip4-output",
 150   .node_name = "nat44-ed-hairpin-src",
 151   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 152 };
 153
 154 /* Hook up ip4-local features */
 155 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 156 {
 157   .arc_name = "ip4-local",
 158   .node_name = "nat44-hairpinning",
 159   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 160 };
 161 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 162 {
 163   .arc_name = "ip4-local",
 164   .node_name = "nat44-ed-hairpinning",
 165   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 166 };
 167
 168
 169 /* *INDENT-OFF* */
 170 VLIB_PLUGIN_REGISTER () = {
 171     .version = VPP_BUILD_VER,
 172     .description = "Network Address Translation",
 173 };
 174 /* *INDENT-ON* */
 175
 176 vlib_node_registration_t nat44_classify_node;
 177 vlib_node_registration_t nat44_ed_classify_node;
 178 vlib_node_registration_t nat44_det_classify_node;
 179 vlib_node_registration_t nat44_handoff_classify_node;
 180
 181 typedef enum {
 182   NAT44_CLASSIFY_NEXT_IN2OUT,
 183   NAT44_CLASSIFY_NEXT_OUT2IN,
 184   NAT44_CLASSIFY_N_NEXT,
 185 } nat44_classify_next_t;
 186
 187 void
 188 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 189 {
 190   snat_session_key_t key;
 191   clib_bihash_kv_8_8_t kv;
 192   nat_ed_ses_key_t ed_key;
 193   clib_bihash_kv_16_8_t ed_kv;
 194   snat_main_per_thread_data_t *tsm =
 195     vec_elt_at_index (sm->per_thread_data, thread_index);
 196
 197   if (is_fwd_bypass_session (s))
 198     {
 199       ed_key.l_addr = s->in2out.addr;
 200       ed_key.r_addr = s->ext_host_addr;
 201       ed_key.l_port = s->in2out.port;
 202       ed_key.r_port = s->ext_host_port;
 203       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 204       ed_key.fib_index = 0;
 205       ed_kv.key[0] = ed_key.as_u64[0];
 206       ed_kv.key[1] = ed_key.as_u64[1];
 207       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 208         nat_log_warn ("in2out_ed key del failed");
 209       return;
 210     }
 211
 212   /* session lookup tables */
 213   if (is_ed_session (s))
 214     {
 215       if (is_affinity_sessions (s))
 216         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 217                              s->in2out.protocol, s->out2in.port);
 218       ed_key.l_addr = s->out2in.addr;
 219       ed_key.r_addr = s->ext_host_addr;
 220       ed_key.fib_index = s->out2in.fib_index;
 221       if (snat_is_unk_proto_session (s))
 222         {
 223           ed_key.proto = s->in2out.port;
 224           ed_key.r_port = 0;
 225           ed_key.l_port = 0;
 226         }
 227       else
 228         {
 229           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 230           ed_key.l_port = s->out2in.port;
 231           ed_key.r_port = s->ext_host_port;
 232         }
 233       ed_kv.key[0] = ed_key.as_u64[0];
 234       ed_kv.key[1] = ed_key.as_u64[1];
 235       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 236         nat_log_warn ("out2in_ed key del failed");
 237       ed_key.l_addr = s->in2out.addr;
 238       ed_key.fib_index = s->in2out.fib_index;
 239       if (!snat_is_unk_proto_session (s))
 240         ed_key.l_port = s->in2out.port;
 241       if (is_twice_nat_session (s))
 242         {
 243           ed_key.r_addr = s->ext_host_nat_addr;
 244           ed_key.r_port = s->ext_host_nat_port;
 245         }
 246       ed_kv.key[0] = ed_key.as_u64[0];
 247       ed_kv.key[1] = ed_key.as_u64[1];
 248       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 249         nat_log_warn ("in2out_ed key del failed");
 250     }
 251   else
 252     {
 253       kv.key = s->in2out.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 255         nat_log_warn ("in2out key del failed");
 256       kv.key = s->out2in.as_u64;
 257       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 258         nat_log_warn ("out2in key del failed");
 259     }
 260
 261   if (snat_is_unk_proto_session (s))
 262     return;
 263
 264   /* log NAT event */
 265   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 266                                       s->out2in.addr.as_u32,
 267                                       s->in2out.protocol,
 268                                       s->in2out.port,
 269                                       s->out2in.port,
 270                                       s->in2out.fib_index);
 271
 272   /* Twice NAT address and port for external host */
 273   if (is_twice_nat_session (s))
 274     {
 275       key.protocol = s->in2out.protocol;
 276       key.port = s->ext_host_nat_port;
 277       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 278       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 279                                           thread_index, &key);
 280     }
 281
 282   if (snat_is_session_static (s))
 283     return;
 284
 285   if (s->outside_address_index != ~0)
 286     snat_free_outside_address_and_port (sm->addresses, thread_index,
 287                                         &s->out2in);
 288 }
 289
 290 snat_user_t *
 291 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 292                         u32 thread_index)
 293 {
 294   snat_user_t *u = 0;
 295   snat_user_key_t user_key;
 296   clib_bihash_kv_8_8_t kv, value;
 297   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 298   dlist_elt_t * per_user_list_head_elt;
 299
 300   user_key.addr.as_u32 = addr->as_u32;
 301   user_key.fib_index = fib_index;
 302   kv.key = user_key.as_u64;
 303
 304   /* Ever heard of the "user" = src ip4 address before? */
 305   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 306     {
 307       /* no, make a new one */
 308       pool_get (tsm->users, u);
 309       memset (u, 0, sizeof (*u));
 310       u->addr.as_u32 = addr->as_u32;
 311       u->fib_index = fib_index;
 312
 313       pool_get (tsm->list_pool, per_user_list_head_elt);
 314
 315       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 316         tsm->list_pool;
 317
 318       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 319
 320       kv.value = u - tsm->users;
 321
 322       /* add user */
 323       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 324         nat_log_warn ("user_hash keay add failed");
 325     }
 326   else
 327     {
 328       u = pool_elt_at_index (tsm->users, value.value);
 329     }
 330
 331   return u;
 332 }
 333
 334 snat_session_t *
 335 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 336 {
 337   snat_session_t *s;
 338   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 339   u32 oldest_per_user_translation_list_index, session_index;
 340   dlist_elt_t * oldest_per_user_translation_list_elt;
 341   dlist_elt_t * per_user_translation_list_elt;
 342
 343   /* Over quota? Recycle the least recently used translation */
 344   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 345     {
 346       oldest_per_user_translation_list_index =
 347         clib_dlist_remove_head (tsm->list_pool,
 348                                 u->sessions_per_user_list_head_index);
 349
 350       ASSERT (oldest_per_user_translation_list_index != ~0);
 351
 352       /* Add it back to the end of the LRU list */
 353       clib_dlist_addtail (tsm->list_pool,
 354                           u->sessions_per_user_list_head_index,
 355                           oldest_per_user_translation_list_index);
 356       /* Get the list element */
 357       oldest_per_user_translation_list_elt =
 358         pool_elt_at_index (tsm->list_pool,
 359                            oldest_per_user_translation_list_index);
 360
 361       /* Get the session index from the list element */
 362       session_index = oldest_per_user_translation_list_elt->value;
 363
 364       /* Get the session */
 365       s = pool_elt_at_index (tsm->sessions, session_index);
 366       nat_free_session_data (sm, s, thread_index);
 367       if (snat_is_session_static(s))
 368         u->nstaticsessions--;
 369       else
 370         u->nsessions--;
 371       s->outside_address_index = ~0;
 372       s->flags = 0;
 373       s->total_bytes = 0;
 374       s->total_pkts = 0;
 375       s->state = 0;
 376       s->ext_host_addr.as_u32 = 0;
 377       s->ext_host_port = 0;
 378       s->ext_host_nat_addr.as_u32 = 0;
 379       s->ext_host_nat_port = 0;
 380     }
 381   else
 382     {
 383       pool_get (tsm->sessions, s);
 384       memset (s, 0, sizeof (*s));
 385       s->outside_address_index = ~0;
 386
 387       /* Create list elts */
 388       pool_get (tsm->list_pool, per_user_translation_list_elt);
 389       clib_dlist_init (tsm->list_pool,
 390                        per_user_translation_list_elt - tsm->list_pool);
 391
 392       per_user_translation_list_elt->value = s - tsm->sessions;
 393       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 394       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 395
 396       clib_dlist_addtail (tsm->list_pool,
 397                           s->per_user_list_head_index,
 398                           per_user_translation_list_elt - tsm->list_pool);
 399     }
 400
 401   return s;
 402 }
 403
 404 snat_session_t *
 405 nat_ed_session_alloc (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 406 {
 407   snat_session_t *s;
 408   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 409   dlist_elt_t * per_user_translation_list_elt;
 410
 411   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 412     {
 413       nat_log_warn ("max translations per user %U", format_ip4_address, &u->addr);
 414       snat_ipfix_logging_max_entries_per_user (sm->max_translations_per_user,
 415                                                u->addr.as_u32);
 416       return 0;
 417     }
 418
 419   pool_get (tsm->sessions, s);
 420   memset (s, 0, sizeof (*s));
 421   s->outside_address_index = ~0;
 422
 423   /* Create list elts */
 424   pool_get (tsm->list_pool, per_user_translation_list_elt);
 425   clib_dlist_init (tsm->list_pool,
 426                    per_user_translation_list_elt - tsm->list_pool);
 427
 428   per_user_translation_list_elt->value = s - tsm->sessions;
 429   s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 430   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 431
 432   clib_dlist_addtail (tsm->list_pool,
 433                       s->per_user_list_head_index,
 434                       per_user_translation_list_elt - tsm->list_pool);
 435
 436   return s;
 437 }
 438
 439 typedef struct {
 440   u8 next_in2out;
 441 } nat44_classify_trace_t;
 442
 443 static u8 * format_nat44_classify_trace (u8 * s, va_list * args)
 444 {
 445   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 446   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 447   nat44_classify_trace_t *t = va_arg (*args, nat44_classify_trace_t *);
 448   char *next;
 449
 450   next = t->next_in2out ? "nat44-in2out" : "nat44-out2in";
 451
 452   s = format (s, "nat44-classify: next %s", next);
 453
 454   return s;
 455 }
 456
 457 static inline uword
 458 nat44_classify_node_fn_inline (vlib_main_t * vm,
 459                                vlib_node_runtime_t * node,
 460                                vlib_frame_t * frame,
 461                                int is_ed)
 462 {
 463   u32 n_left_from, * from, * to_next;
 464   nat44_classify_next_t next_index;
 465   snat_main_t *sm = &snat_main;
 466   snat_static_mapping_t *m;
 467   u32 thread_index = vm->thread_index;
 468   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 469
 470   from = vlib_frame_vector_args (frame);
 471   n_left_from = frame->n_vectors;
 472   next_index = node->cached_next_index;
 473
 474   while (n_left_from > 0)
 475     {
 476       u32 n_left_to_next;
 477
 478       vlib_get_next_frame (vm, node, next_index,
 479                            to_next, n_left_to_next);
 480
 481       while (n_left_from > 0 && n_left_to_next > 0)
 482         {
 483           u32 bi0;
 484           vlib_buffer_t *b0;
 485           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0;
 486           ip4_header_t *ip0;
 487           snat_address_t *ap;
 488           snat_session_key_t m_key0;
 489           clib_bihash_kv_8_8_t kv0, value0;
 490           clib_bihash_kv_16_8_t ed_kv0, ed_value0;
 491           udp_header_t *udp0;
 492
 493           /* speculatively enqueue b0 to the current next frame */
 494           bi0 = from[0];
 495           to_next[0] = bi0;
 496           from += 1;
 497           to_next += 1;
 498           n_left_from -= 1;
 499           n_left_to_next -= 1;
 500
 501           b0 = vlib_get_buffer (vm, bi0);
 502           ip0 = vlib_buffer_get_current (b0);
 503           udp0 = ip4_next_header (ip0);
 504
 505           if (is_ed)
 506             {
 507               sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 508               rx_fib_index0 =
 509                 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 510                                                      sw_if_index0);
 511               make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address,
 512                           ip0->protocol, rx_fib_index0, udp0->src_port,
 513                           udp0->dst_port);
 514               if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, &ed_value0))
 515                 goto enqueue0;
 516             }
 517
 518           vec_foreach (ap, sm->addresses)
 519             {
 520               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 521                 {
 522                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 523                   goto enqueue0;
 524                 }
 525             }
 526
 527           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 528             {
 529               m_key0.addr = ip0->dst_address;
 530               m_key0.port = 0;
 531               m_key0.protocol = 0;
 532               m_key0.fib_index = 0;
 533               kv0.key = m_key0.as_u64;
 534               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 535                 {
 536                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 537                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 538                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 539                   goto enqueue0;
 540                 }
 541               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 542               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 543               kv0.key = m_key0.as_u64;
 544               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 545                 {
 546                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 547                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 548                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 549                 }
 550             }
 551
 552         enqueue0:
 553           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 554                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 555             {
 556               nat44_classify_trace_t *t =
 557                   vlib_add_trace (vm, node, b0, sizeof (*t));
 558               t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
 559             }
 560
 561           /* verify speculative enqueue, maybe switch current next frame */
 562           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 563                                            to_next, n_left_to_next,
 564                                            bi0, next0);
 565         }
 566
 567       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 568     }
 569
 570   return frame->n_vectors;
 571 }
 572
 573 static uword
 574 nat44_classify_node_fn (vlib_main_t * vm,
 575                         vlib_node_runtime_t * node,
 576                         vlib_frame_t * frame)
 577 {
 578   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 579 };
 580
 581 VLIB_REGISTER_NODE (nat44_classify_node) = {
 582   .function = nat44_classify_node_fn,
 583   .name = "nat44-classify",
 584   .vector_size = sizeof (u32),
 585   .format_trace = format_nat44_classify_trace,
 586   .type = VLIB_NODE_TYPE_INTERNAL,
 587   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 588   .next_nodes = {
 589     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 590     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 591   },
 592 };
 593
 594 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 595                               nat44_classify_node_fn);
 596 static uword
 597 nat44_ed_classify_node_fn (vlib_main_t * vm,
 598                            vlib_node_runtime_t * node,
 599                            vlib_frame_t * frame)
 600 {
 601   return nat44_classify_node_fn_inline (vm, node, frame, 1);
 602 };
 603
 604 VLIB_REGISTER_NODE (nat44_ed_classify_node) = {
 605   .function = nat44_ed_classify_node_fn,
 606   .name = "nat44-ed-classify",
 607   .vector_size = sizeof (u32),
 608   .format_trace = format_nat44_classify_trace,
 609   .type = VLIB_NODE_TYPE_INTERNAL,
 610   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 611   .next_nodes = {
 612     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out",
 613     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in",
 614   },
 615 };
 616
 617 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_classify_node,
 618                               nat44_ed_classify_node_fn);
 619
 620 static uword
 621 nat44_det_classify_node_fn (vlib_main_t * vm,
 622                             vlib_node_runtime_t * node,
 623                             vlib_frame_t * frame)
 624 {
 625   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 626 };
 627
 628 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 629   .function = nat44_det_classify_node_fn,
 630   .name = "nat44-det-classify",
 631   .vector_size = sizeof (u32),
 632   .format_trace = format_nat44_classify_trace,
 633   .type = VLIB_NODE_TYPE_INTERNAL,
 634   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 635   .next_nodes = {
 636     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 637     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 638   },
 639 };
 640
 641 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 642                               nat44_det_classify_node_fn);
 643
 644 static uword
 645 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 646                                 vlib_node_runtime_t * node,
 647                                 vlib_frame_t * frame)
 648 {
 649   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 650 };
 651
 652 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 653   .function = nat44_handoff_classify_node_fn,
 654   .name = "nat44-handoff-classify",
 655   .vector_size = sizeof (u32),
 656   .format_trace = format_nat44_classify_trace,
 657   .type = VLIB_NODE_TYPE_INTERNAL,
 658   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 659   .next_nodes = {
 660     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 661     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 662   },
 663 };
 664
 665 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 666                               nat44_handoff_classify_node_fn);
 667
 668 /**
 669  * @brief Add/del NAT address to FIB.
 670  *
 671  * Add the external NAT address to the FIB as receive entries. This ensures
 672  * that VPP will reply to ARP for this address and we don't need to enable
 673  * proxy ARP on the outside interface.
 674  *
 675  * @param addr IPv4 address.
 676  * @param plen address prefix length
 677  * @param sw_if_index Interface.
 678  * @param is_add If 0 delete, otherwise add.
 679  */
 680 void
 681 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 682                           int is_add)
 683 {
 684   fib_prefix_t prefix = {
 685     .fp_len = p_len,
 686     .fp_proto = FIB_PROTOCOL_IP4,
 687     .fp_addr = {
 688         .ip4.as_u32 = addr->as_u32,
 689     },
 690   };
 691   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 692
 693   if (is_add)
 694     fib_table_entry_update_one_path(fib_index,
 695                                     &prefix,
 696                                     FIB_SOURCE_PLUGIN_LOW,
 697                                     (FIB_ENTRY_FLAG_CONNECTED |
 698                                      FIB_ENTRY_FLAG_LOCAL |
 699                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 700                                     DPO_PROTO_IP4,
 701                                     NULL,
 702                                     sw_if_index,
 703                                     ~0,
 704                                     1,
 705                                     NULL,
 706                                     FIB_ROUTE_PATH_FLAG_NONE);
 707   else
 708     fib_table_entry_delete(fib_index,
 709                            &prefix,
 710                            FIB_SOURCE_PLUGIN_LOW);
 711 }
 712
 713 int snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 714                        u8 twice_nat)
 715 {
 716   snat_address_t * ap;
 717   snat_interface_t *i;
 718   vlib_thread_main_t *tm = vlib_get_thread_main ();
 719
 720   if (twice_nat && !sm->endpoint_dependent)
 721     return VNET_API_ERROR_FEATURE_DISABLED;
 722
 723   /* Check if address already exists */
 724   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 725     {
 726       if (ap->addr.as_u32 == addr->as_u32)
 727         return VNET_API_ERROR_VALUE_EXIST;
 728     }
 729
 730   if (twice_nat)
 731     vec_add2 (sm->twice_nat_addresses, ap, 1);
 732   else
 733     vec_add2 (sm->addresses, ap, 1);
 734
 735   ap->addr = *addr;
 736   if (vrf_id != ~0)
 737     ap->fib_index =
 738       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 739                                          FIB_SOURCE_PLUGIN_LOW);
 740   else
 741     ap->fib_index = ~0;
 742 #define _(N, i, n, s) \
 743   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 744   ap->busy_##n##_ports = 0; \
 745   ap->busy_##n##_ports_per_thread = 0;\
 746   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 747   foreach_snat_protocol
 748 #undef _
 749
 750   if (twice_nat)
 751     return 0;
 752
 753   /* Add external address to FIB */
 754   pool_foreach (i, sm->interfaces,
 755   ({
 756     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 757       continue;
 758
 759     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 760     break;
 761   }));
 762   pool_foreach (i, sm->output_feature_interfaces,
 763   ({
 764     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 765       continue;
 766
 767     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 768     break;
 769   }));
 770
 771   return 0;
 772 }
 773
 774 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 775                                                    ip4_address_t addr)
 776 {
 777   snat_static_mapping_t *m;
 778   pool_foreach (m, sm->static_mappings,
 779   ({
 780       if (m->external_addr.as_u32 == addr.as_u32)
 781         return 1;
 782   }));
 783
 784   return 0;
 785 }
 786
 787 void increment_v4_address (ip4_address_t * a)
 788 {
 789   u32 v;
 790
 791   v = clib_net_to_host_u32(a->as_u32) + 1;
 792   a->as_u32 = clib_host_to_net_u32(v);
 793 }
 794
 795 static void
 796 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 797                                        ip4_address_t l_addr,
 798                                        u16 l_port,
 799                                        u32 sw_if_index,
 800                                        u16 e_port,
 801                                        u32 vrf_id,
 802                                        snat_protocol_t proto,
 803                                        int addr_only,
 804                                        int is_add,
 805                                        u8 * tag)
 806 {
 807   snat_static_map_resolve_t *rp;
 808
 809   vec_add2 (sm->to_resolve, rp, 1);
 810   rp->l_addr.as_u32 = l_addr.as_u32;
 811   rp->l_port = l_port;
 812   rp->sw_if_index = sw_if_index;
 813   rp->e_port = e_port;
 814   rp->vrf_id = vrf_id;
 815   rp->proto = proto;
 816   rp->addr_only = addr_only;
 817   rp->is_add = is_add;
 818   rp->tag = vec_dup (tag);
 819 }
 820
 821 static u32 get_thread_idx_by_port(u16 e_port)
 822 {
 823     snat_main_t * sm = &snat_main;
 824     u32 thread_idx = sm->num_workers;
 825     if (sm->num_workers > 1)
 826     {
 827         thread_idx = sm->first_worker_index + sm->workers[(e_port - 1024) / sm->port_per_thread];
 828     }
 829     return thread_idx;
 830 }
 831
 832 /**
 833  * @brief Add static mapping.
 834  *
 835  * Create static mapping between local addr+port and external addr+port.
 836  *
 837  * @param l_addr Local IPv4 address.
 838  * @param e_addr External IPv4 address.
 839  * @param l_port Local port number.
 840  * @param e_port External port number.
 841  * @param vrf_id VRF ID.
 842  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 843  * @param sw_if_index External port instead of specific IP address.
 844  * @param is_add If 0 delete static mapping, otherwise add.
 845  * @param twice_nat If value is TWICE_NAT then translate external host address
 846  *                  and port.
 847  *                  If value is TWICE_NAT_SELF then translate external host
 848  *                  address and port whenever external host address equals
 849  *                  local address of internal host.
 850  * @param out2in_only If 1 rule match only out2in direction
 851  * @param tag - opaque string tag
 852  *
 853  * @returns
 854  */
 855 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 856                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 857                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 858                             twice_nat_type_t twice_nat, u8 out2in_only,
 859                             u8 * tag)
 860 {
 861   snat_main_t * sm = &snat_main;
 862   snat_static_mapping_t *m;
 863   snat_session_key_t m_key;
 864   clib_bihash_kv_8_8_t kv, value;
 865   snat_address_t *a = 0;
 866   u32 fib_index = ~0;
 867   uword * p;
 868   snat_interface_t *interface;
 869   int i;
 870   snat_main_per_thread_data_t *tsm;
 871   snat_user_key_t u_key;
 872   snat_user_t *u;
 873   dlist_elt_t * head, * elt;
 874   u32 elt_index, head_index;
 875   u32 ses_index;
 876   u64 user_index;
 877   snat_session_t * s;
 878   snat_static_map_resolve_t *rp, *rp_match = 0;
 879
 880   if (!sm->endpoint_dependent)
 881     {
 882       if (twice_nat || out2in_only)
 883         return VNET_API_ERROR_FEATURE_DISABLED;
 884     }
 885
 886   /* If the external address is a specific interface address */
 887   if (sw_if_index != ~0)
 888     {
 889       ip4_address_t * first_int_addr;
 890
 891       for (i = 0; i < vec_len (sm->to_resolve); i++)
 892         {
 893           rp = sm->to_resolve + i;
 894           if (rp->sw_if_index != sw_if_index ||
 895               rp->l_addr.as_u32 != l_addr.as_u32 ||
 896               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 897             continue;
 898
 899           if (!addr_only)
 900             {
 901               if (rp->l_port != l_port || rp->e_port != e_port || rp->proto != proto)
 902                 continue;
 903             }
 904
 905           rp_match = rp;
 906           break;
 907         }
 908
 909       /* Might be already set... */
 910       first_int_addr = ip4_interface_first_address
 911         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 912
 913       if (is_add)
 914         {
 915           if (rp_match)
 916             return VNET_API_ERROR_VALUE_EXIST;
 917
 918           snat_add_static_mapping_when_resolved
 919             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 920              addr_only,  is_add, tag);
 921
 922           /* DHCP resolution required? */
 923           if (first_int_addr == 0)
 924             {
 925               return 0;
 926             }
 927           else
 928             {
 929               e_addr.as_u32 = first_int_addr->as_u32;
 930               /* Identity mapping? */
 931               if (l_addr.as_u32 == 0)
 932                 l_addr.as_u32 = e_addr.as_u32;
 933             }
 934         }
 935       else
 936         {
 937           if (!rp_match)
 938             return VNET_API_ERROR_NO_SUCH_ENTRY;
 939
 940           vec_del1 (sm->to_resolve, i);
 941
 942           if (first_int_addr)
 943             {
 944               e_addr.as_u32 = first_int_addr->as_u32;
 945               /* Identity mapping? */
 946               if (l_addr.as_u32 == 0)
 947                 l_addr.as_u32 = e_addr.as_u32;
 948             }
 949           else
 950             return 0;
 951         }
 952     }
 953
 954   m_key.addr = e_addr;
 955   m_key.port = addr_only ? 0 : e_port;
 956   m_key.protocol = addr_only ? 0 : proto;
 957   m_key.fib_index = 0;
 958   kv.key = m_key.as_u64;
 959   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 960     m = 0;
 961   else
 962     m = pool_elt_at_index (sm->static_mappings, value.value);
 963
 964   if (is_add)
 965     {
 966       if (m)
 967         return VNET_API_ERROR_VALUE_EXIST;
 968
 969       if (twice_nat && addr_only)
 970         return VNET_API_ERROR_UNSUPPORTED;
 971
 972       /* Convert VRF id to FIB index */
 973       if (vrf_id != ~0)
 974         {
 975           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 976           if (!p)
 977             return VNET_API_ERROR_NO_SUCH_FIB;
 978           fib_index = p[0];
 979         }
 980       /* If not specified use inside VRF id from SNAT plugin startup config */
 981       else
 982         {
 983           fib_index = sm->inside_fib_index;
 984           vrf_id = sm->inside_vrf_id;
 985         }
 986
 987       if (!out2in_only)
 988         {
 989           m_key.addr = l_addr;
 990           m_key.port = addr_only ? 0 : l_port;
 991           m_key.protocol = addr_only ? 0 : proto;
 992           m_key.fib_index = fib_index;
 993           kv.key = m_key.as_u64;
 994           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 995             return VNET_API_ERROR_VALUE_EXIST;
 996         }
 997
 998       /* Find external address in allocated addresses and reserve port for
 999          address and port pair mapping when dynamic translations enabled */
1000       if (!(addr_only || sm->static_mapping_only || out2in_only))
1001         {
1002           for (i = 0; i < vec_len (sm->addresses); i++)
1003             {
1004               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1005                 {
1006                   a = sm->addresses + i;
1007                   /* External port must be unused */
1008                   switch (proto)
1009                     {
1010 #define _(N, j, n, s) \
1011                     case SNAT_PROTOCOL_##N: \
1012                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1013                         return VNET_API_ERROR_INVALID_VALUE; \
1014                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1015                       if (e_port > 1024) \
1016                         { \
1017                           a->busy_##n##_ports++; \
1018                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1019                         } \
1020                       break;
1021                       foreach_snat_protocol
1022 #undef _
1023                     default:
1024                       nat_log_info ("unknown protocol");
1025                       return VNET_API_ERROR_INVALID_VALUE_2;
1026                     }
1027                   break;
1028                 }
1029             }
1030           /* External address must be allocated */
1031           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1032             {
1033               if (sw_if_index != ~0)
1034                 {
1035                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1036                     {
1037                       rp = sm->to_resolve + i;
1038                       if (rp->addr_only)
1039                          continue;
1040                       if (rp->sw_if_index != sw_if_index &&
1041                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1042                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1043                           rp->e_port != e_port && rp->proto != proto)
1044                         continue;
1045
1046                       vec_del1 (sm->to_resolve, i);
1047                       break;
1048                     }
1049                 }
1050               return VNET_API_ERROR_NO_SUCH_ENTRY;
1051             }
1052         }
1053
1054       pool_get (sm->static_mappings, m);
1055       memset (m, 0, sizeof (*m));
1056       m->tag = vec_dup (tag);
1057       m->local_addr = l_addr;
1058       m->external_addr = e_addr;
1059       m->addr_only = addr_only;
1060       m->vrf_id = vrf_id;
1061       m->fib_index = fib_index;
1062       m->twice_nat = twice_nat;
1063       m->out2in_only = out2in_only;
1064       if (!addr_only)
1065         {
1066           m->local_port = l_port;
1067           m->external_port = e_port;
1068           m->proto = proto;
1069         }
1070
1071       if (sm->num_workers > 1)
1072         {
1073           ip4_header_t ip = {
1074             .src_address = m->local_addr,
1075           };
1076           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
1077           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1078         }
1079       else
1080         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1081
1082       m_key.addr = m->local_addr;
1083       m_key.port = m->local_port;
1084       m_key.protocol = m->proto;
1085       m_key.fib_index = m->fib_index;
1086       kv.key = m_key.as_u64;
1087       kv.value = m - sm->static_mappings;
1088       if (!out2in_only)
1089         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1090
1091       m_key.addr = m->external_addr;
1092       m_key.port = m->external_port;
1093       m_key.fib_index = 0;
1094       kv.key = m_key.as_u64;
1095       kv.value = m - sm->static_mappings;
1096       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
1097
1098       /* Delete dynamic sessions matching local address (+ local port) */
1099       if (!(sm->static_mapping_only))
1100         {
1101           u_key.addr = m->local_addr;
1102           u_key.fib_index = m->fib_index;
1103           kv.key = u_key.as_u64;
1104           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1105             {
1106               user_index = value.value;
1107               u = pool_elt_at_index (tsm->users, user_index);
1108               if (u->nsessions)
1109                 {
1110                   head_index = u->sessions_per_user_list_head_index;
1111                   head = pool_elt_at_index (tsm->list_pool, head_index);
1112                   elt_index = head->next;
1113                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1114                   ses_index = elt->value;
1115                   while (ses_index != ~0)
1116                     {
1117                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1118                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1119                       ses_index = elt->value;
1120
1121                       if (snat_is_session_static (s))
1122                         continue;
1123
1124                       if (!addr_only && (clib_net_to_host_u16 (s->in2out.port) != m->local_port))
1125                         continue;
1126
1127                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1128                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1129
1130                       if (!addr_only && !sm->endpoint_dependent)
1131                         break;
1132                     }
1133                 }
1134             }
1135         }
1136     }
1137   else
1138     {
1139       if (!m)
1140         {
1141           if (sw_if_index != ~0)
1142             return 0;
1143           else
1144             return VNET_API_ERROR_NO_SUCH_ENTRY;
1145         }
1146
1147       /* Free external address port */
1148       if (!(addr_only || sm->static_mapping_only || out2in_only))
1149         {
1150           for (i = 0; i < vec_len (sm->addresses); i++)
1151             {
1152               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1153                 {
1154                   a = sm->addresses + i;
1155                   switch (proto)
1156                     {
1157 #define _(N, j, n, s) \
1158                     case SNAT_PROTOCOL_##N: \
1159                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1160                       if (e_port > 1024) \
1161                         { \
1162                           a->busy_##n##_ports--; \
1163                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1164                         } \
1165                       break;
1166                       foreach_snat_protocol
1167 #undef _
1168                     default:
1169                       nat_log_info ("unknown protocol");
1170                       return VNET_API_ERROR_INVALID_VALUE_2;
1171                     }
1172                   break;
1173                 }
1174             }
1175         }
1176
1177       if (sm->num_workers > 1)
1178         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1179       else
1180         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1181
1182       m_key.addr = m->local_addr;
1183       m_key.port = m->local_port;
1184       m_key.protocol = m->proto;
1185       m_key.fib_index = m->fib_index;
1186       kv.key = m_key.as_u64;
1187       if (!out2in_only)
1188         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1189
1190       m_key.addr = m->external_addr;
1191       m_key.port = m->external_port;
1192       m_key.fib_index = 0;
1193       kv.key = m_key.as_u64;
1194       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1195
1196       /* Delete session(s) for static mapping if exist */
1197       if (!(sm->static_mapping_only) ||
1198           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1199         {
1200           u_key.addr = m->local_addr;
1201           u_key.fib_index = m->fib_index;
1202           kv.key = u_key.as_u64;
1203           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1204             {
1205               user_index = value.value;
1206               u = pool_elt_at_index (tsm->users, user_index);
1207               if (u->nstaticsessions)
1208                 {
1209                   head_index = u->sessions_per_user_list_head_index;
1210                   head = pool_elt_at_index (tsm->list_pool, head_index);
1211                   elt_index = head->next;
1212                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1213                   ses_index = elt->value;
1214                   while (ses_index != ~0)
1215                     {
1216                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1217                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1218                       ses_index = elt->value;
1219
1220                       if (!addr_only)
1221                         {
1222                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1223                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1224                             continue;
1225                         }
1226
1227                       if (is_lb_session (s))
1228                         continue;
1229
1230                       if (!snat_is_session_static (s))
1231                         continue;
1232
1233                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1234                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1235
1236                       if (!addr_only && !sm->endpoint_dependent)
1237                         break;
1238                     }
1239                 }
1240             }
1241         }
1242
1243       vec_free (m->tag);
1244       vec_free (m->workers);
1245       /* Delete static mapping from pool */
1246       pool_put (sm->static_mappings, m);
1247     }
1248
1249   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1250     return 0;
1251
1252   /* Add/delete external address to FIB */
1253   pool_foreach (interface, sm->interfaces,
1254   ({
1255     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1256       continue;
1257
1258     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1259     break;
1260   }));
1261   pool_foreach (interface, sm->output_feature_interfaces,
1262   ({
1263     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1264       continue;
1265
1266     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1267     break;
1268   }));
1269
1270   return 0;
1271 }
1272
1273 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1274                                      snat_protocol_t proto,
1275                                      nat44_lb_addr_port_t *locals, u8 is_add,
1276                                      twice_nat_type_t twice_nat, u8 out2in_only,
1277                                      u8 *tag, u32 affinity)
1278 {
1279   snat_main_t * sm = &snat_main;
1280   snat_static_mapping_t *m;
1281   snat_session_key_t m_key;
1282   clib_bihash_kv_8_8_t kv, value;
1283   snat_address_t *a = 0;
1284   int i;
1285   nat44_lb_addr_port_t *local;
1286   u32 elt_index, head_index, ses_index;
1287   snat_main_per_thread_data_t *tsm;
1288   snat_user_key_t u_key;
1289   snat_user_t *u;
1290   snat_session_t * s;
1291   dlist_elt_t * head, * elt;
1292   uword *bitmap = 0;
1293
1294   if (!sm->endpoint_dependent)
1295     return VNET_API_ERROR_FEATURE_DISABLED;
1296
1297   m_key.addr = e_addr;
1298   m_key.port = e_port;
1299   m_key.protocol = proto;
1300   m_key.fib_index = 0;
1301   kv.key = m_key.as_u64;
1302   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1303     m = 0;
1304   else
1305     m = pool_elt_at_index (sm->static_mappings, value.value);
1306
1307   if (is_add)
1308     {
1309       if (m)
1310         return VNET_API_ERROR_VALUE_EXIST;
1311
1312       if (vec_len (locals) < 2)
1313         return VNET_API_ERROR_INVALID_VALUE;
1314
1315       /* Find external address in allocated addresses and reserve port for
1316          address and port pair mapping when dynamic translations enabled */
1317       if (!(sm->static_mapping_only || out2in_only))
1318         {
1319           for (i = 0; i < vec_len (sm->addresses); i++)
1320             {
1321               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1322                 {
1323                   a = sm->addresses + i;
1324                   /* External port must be unused */
1325                   switch (proto)
1326                     {
1327 #define _(N, j, n, s) \
1328                     case SNAT_PROTOCOL_##N: \
1329                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1330                         return VNET_API_ERROR_INVALID_VALUE; \
1331                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1332                       if (e_port > 1024) \
1333                         { \
1334                           a->busy_##n##_ports++; \
1335                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
1336                         } \
1337                       break;
1338                       foreach_snat_protocol
1339 #undef _
1340                     default:
1341                       nat_log_info ("unknown protocol");
1342                       return VNET_API_ERROR_INVALID_VALUE_2;
1343                     }
1344                   break;
1345                 }
1346             }
1347           /* External address must be allocated */
1348           if (!a)
1349             return VNET_API_ERROR_NO_SUCH_ENTRY;
1350         }
1351
1352       pool_get (sm->static_mappings, m);
1353       memset (m, 0, sizeof (*m));
1354       m->tag = vec_dup (tag);
1355       m->external_addr = e_addr;
1356       m->addr_only = 0;
1357       m->external_port = e_port;
1358       m->proto = proto;
1359       m->twice_nat = twice_nat;
1360       m->out2in_only = out2in_only;
1361       m->affinity = affinity;
1362
1363       if (affinity)
1364         m->affinity_per_service_list_head_index =
1365           nat_affinity_get_per_service_list_head_index();
1366       else
1367         m->affinity_per_service_list_head_index = ~0;
1368
1369       m_key.addr = m->external_addr;
1370       m_key.port = m->external_port;
1371       m_key.protocol = m->proto;
1372       m_key.fib_index = 0;
1373       kv.key = m_key.as_u64;
1374       kv.value = m - sm->static_mappings;
1375       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1376         {
1377           nat_log_err ("static_mapping_by_external key add failed");
1378           return VNET_API_ERROR_UNSPECIFIED;
1379         }
1380
1381       m_key.fib_index = m->fib_index;
1382       for (i = 0; i < vec_len (locals); i++)
1383         {
1384           locals[i].fib_index = fib_table_find_or_create_and_lock (
1385             FIB_PROTOCOL_IP4, locals[i].vrf_id, FIB_SOURCE_PLUGIN_LOW);
1386           m_key.addr = locals[i].addr;
1387           m_key.fib_index = locals[i].fib_index;
1388           if (!out2in_only)
1389             {
1390               m_key.port = locals[i].port;
1391               kv.key = m_key.as_u64;
1392               kv.value = m - sm->static_mappings;
1393               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1394             }
1395           locals[i].prefix = (i == 0) ? locals[i].probability :\
1396             (locals[i - 1].prefix + locals[i].probability);
1397           vec_add1 (m->locals, locals[i]);
1398           if (sm->num_workers > 1)
1399             {
1400               ip4_header_t ip = {
1401                 .src_address = locals[i].addr,
1402               };
1403               bitmap = clib_bitmap_set (
1404                 bitmap, sm->worker_in2out_cb (&ip, m->fib_index), 1);
1405             }
1406         }
1407
1408       /* Assign workers */
1409       if (sm->num_workers > 1)
1410         {
1411           clib_bitmap_foreach (i, bitmap,
1412             ({
1413                vec_add1(m->workers, i);
1414             }));
1415         }
1416     }
1417   else
1418     {
1419       if (!m)
1420         return VNET_API_ERROR_NO_SUCH_ENTRY;
1421
1422       /* Free external address port */
1423       if (!(sm->static_mapping_only || out2in_only))
1424         {
1425           for (i = 0; i < vec_len (sm->addresses); i++)
1426             {
1427               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1428                 {
1429                   a = sm->addresses + i;
1430                   switch (proto)
1431                     {
1432 #define _(N, j, n, s) \
1433                     case SNAT_PROTOCOL_##N: \
1434                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1435                       if (e_port > 1024) \
1436                         { \
1437                           a->busy_##n##_ports--; \
1438                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1439                         } \
1440                       break;
1441                       foreach_snat_protocol
1442 #undef _
1443                     default:
1444                       nat_log_info ("unknown protocol");
1445                       return VNET_API_ERROR_INVALID_VALUE_2;
1446                     }
1447                   break;
1448                 }
1449             }
1450         }
1451
1452       m_key.addr = m->external_addr;
1453       m_key.port = m->external_port;
1454       m_key.protocol = m->proto;
1455       m_key.fib_index = 0;
1456       kv.key = m_key.as_u64;
1457       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1458         {
1459           nat_log_err ("static_mapping_by_external key del failed");
1460           return VNET_API_ERROR_UNSPECIFIED;
1461         }
1462
1463       vec_foreach (local, m->locals)
1464         {
1465           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1466                             FIB_SOURCE_PLUGIN_LOW);
1467           m_key.addr = local->addr;
1468           if (!out2in_only)
1469             {
1470               m_key.port = local->port;
1471               m_key.fib_index = local->fib_index;
1472               kv.key = m_key.as_u64;
1473               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1474                 {
1475                   nat_log_err ("static_mapping_by_local key del failed");
1476                   return VNET_API_ERROR_UNSPECIFIED;
1477                 }
1478             }
1479
1480           if (sm->num_workers > 1)
1481             {
1482               ip4_header_t ip = {
1483                 .src_address = local->addr,
1484               };
1485               tsm = vec_elt_at_index (sm->per_thread_data,
1486                                       sm->worker_in2out_cb (&ip, m->fib_index));
1487             }
1488           else
1489             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1490
1491           /* Delete sessions */
1492           u_key.addr = local->addr;
1493           u_key.fib_index = m->fib_index;
1494           kv.key = u_key.as_u64;
1495           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1496             {
1497               u = pool_elt_at_index (tsm->users, value.value);
1498               if (u->nstaticsessions)
1499                 {
1500                   head_index = u->sessions_per_user_list_head_index;
1501                   head = pool_elt_at_index (tsm->list_pool, head_index);
1502                   elt_index = head->next;
1503                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1504                   ses_index = elt->value;
1505                   while (ses_index != ~0)
1506                     {
1507                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1508                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1509                       ses_index = elt->value;
1510
1511                       if (!(is_lb_session (s)))
1512                         continue;
1513
1514                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1515                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1516                         continue;
1517
1518                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1519                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1520                     }
1521                 }
1522             }
1523         }
1524       if (m->affinity)
1525         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1526       vec_free(m->locals);
1527       vec_free(m->tag);
1528       vec_free(m->workers);
1529
1530       pool_put (sm->static_mappings, m);
1531     }
1532
1533   return 0;
1534 }
1535
1536 int
1537 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1538                   u8 twice_nat)
1539 {
1540   snat_address_t *a = 0;
1541   snat_session_t *ses;
1542   u32 *ses_to_be_removed = 0, *ses_index;
1543   snat_main_per_thread_data_t *tsm;
1544   snat_static_mapping_t *m;
1545   snat_interface_t *interface;
1546   int i;
1547   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1548
1549   /* Find SNAT address */
1550   for (i=0; i < vec_len (addresses); i++)
1551     {
1552       if (addresses[i].addr.as_u32 == addr.as_u32)
1553         {
1554           a = addresses + i;
1555           break;
1556         }
1557     }
1558   if (!a)
1559     return VNET_API_ERROR_NO_SUCH_ENTRY;
1560
1561   if (delete_sm)
1562     {
1563       pool_foreach (m, sm->static_mappings,
1564       ({
1565           if (m->external_addr.as_u32 == addr.as_u32)
1566             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1567                                             m->local_port, m->external_port,
1568                                             m->vrf_id, m->addr_only, ~0,
1569                                             m->proto, 0, m->twice_nat,
1570                                             m->out2in_only, m->tag);
1571       }));
1572     }
1573   else
1574     {
1575       /* Check if address is used in some static mapping */
1576       if (is_snat_address_used_in_static_mapping(sm, addr))
1577         {
1578           nat_log_notice ("address used in static mapping");
1579           return VNET_API_ERROR_UNSPECIFIED;
1580         }
1581     }
1582
1583   if (a->fib_index != ~0)
1584     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1585                      FIB_SOURCE_PLUGIN_LOW);
1586
1587   /* Delete sessions using address */
1588   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1589     {
1590       vec_foreach (tsm, sm->per_thread_data)
1591         {
1592           pool_foreach (ses, tsm->sessions, ({
1593             if (ses->out2in.addr.as_u32 == addr.as_u32)
1594               {
1595                 ses->outside_address_index = ~0;
1596                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1597                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1598               }
1599           }));
1600
1601           vec_foreach (ses_index, ses_to_be_removed)
1602             {
1603               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1604               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1605             }
1606
1607           vec_free (ses_to_be_removed);
1608        }
1609     }
1610
1611 #define _(N, i, n, s) \
1612   clib_bitmap_free (a->busy_##n##_port_bitmap); \
1613   vec_free (a->busy_##n##_ports_per_thread);
1614   foreach_snat_protocol
1615 #undef _
1616
1617   if (twice_nat)
1618     {
1619       vec_del1 (sm->twice_nat_addresses, i);
1620       return 0;
1621     }
1622   else
1623     vec_del1 (sm->addresses, i);
1624
1625   /* Delete external address from FIB */
1626   pool_foreach (interface, sm->interfaces,
1627   ({
1628     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1629       continue;
1630
1631     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1632     break;
1633   }));
1634   pool_foreach (interface, sm->output_feature_interfaces,
1635   ({
1636     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1637       continue;
1638
1639     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1640     break;
1641   }));
1642
1643   return 0;
1644 }
1645
1646 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1647 {
1648   snat_main_t *sm = &snat_main;
1649   snat_interface_t *i;
1650   const char * feature_name, *del_feature_name;
1651   snat_address_t * ap;
1652   snat_static_mapping_t * m;
1653   snat_det_map_t * dm;
1654   nat_outside_fib_t *outside_fib;
1655   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1656                                                        sw_if_index);
1657
1658   if (sm->out2in_dpo && !is_inside)
1659     return VNET_API_ERROR_UNSUPPORTED;
1660
1661   pool_foreach (i, sm->output_feature_interfaces,
1662   ({
1663     if (i->sw_if_index == sw_if_index)
1664       return VNET_API_ERROR_VALUE_EXIST;
1665   }));
1666
1667   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1668     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1669   else
1670     {
1671       if (sm->num_workers > 1 && !sm->deterministic)
1672         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1673       else if (sm->deterministic)
1674         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1675       else if (sm->endpoint_dependent)
1676         feature_name = is_inside ?  "nat44-ed-in2out" : "nat44-ed-out2in";
1677       else
1678         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1679     }
1680
1681   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1682     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1683                                                       NAT_FQ_NELTS);
1684
1685   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1686     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1687                                                       NAT_FQ_NELTS);
1688
1689   if (!is_inside)
1690     {
1691       vec_foreach (outside_fib, sm->outside_fibs)
1692         {
1693           if (outside_fib->fib_index == fib_index)
1694             {
1695               if (is_del)
1696                 {
1697                   outside_fib->refcount--;
1698                   if (!outside_fib->refcount)
1699                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1700                 }
1701               else
1702                 outside_fib->refcount++;
1703               goto feature_set;
1704             }
1705         }
1706       if (!is_del)
1707         {
1708           vec_add2 (sm->outside_fibs, outside_fib, 1);
1709           outside_fib->refcount = 1;
1710           outside_fib->fib_index = fib_index;
1711         }
1712     }
1713 feature_set:
1714   pool_foreach (i, sm->interfaces,
1715   ({
1716     if (i->sw_if_index == sw_if_index)
1717       {
1718         if (is_del)
1719           {
1720             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1721               {
1722                 if (is_inside)
1723                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1724                 else
1725                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1726
1727                 if (sm->num_workers > 1 && !sm->deterministic)
1728                   {
1729                     del_feature_name = "nat44-handoff-classify";
1730                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1731                                                  "nat44-out2in-worker-handoff";
1732                   }
1733                 else if (sm->deterministic)
1734                   {
1735                     del_feature_name = "nat44-det-classify";
1736                     feature_name = !is_inside ?  "nat44-det-in2out" :
1737                                                  "nat44-det-out2in";
1738                   }
1739                 else if (sm->endpoint_dependent)
1740                   {
1741                     del_feature_name = "nat44-ed-classify";
1742                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1743                                                  "nat44-ed-out2in";
1744                   }
1745                 else
1746                   {
1747                     del_feature_name = "nat44-classify";
1748                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1749                   }
1750
1751                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1752                                              sw_if_index, 0, 0, 0);
1753                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1754                                              sw_if_index, 1, 0, 0);
1755                 if (!is_inside)
1756                   {
1757                     if (sm->endpoint_dependent)
1758                       vnet_feature_enable_disable ("ip4-local",
1759                                                    "nat44-ed-hairpinning",
1760                                                    sw_if_index, 1, 0, 0);
1761                     else if (!sm->deterministic)
1762                       vnet_feature_enable_disable ("ip4-local",
1763                                                    "nat44-hairpinning",
1764                                                    sw_if_index, 1, 0, 0);
1765                   }
1766               }
1767             else
1768               {
1769                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1770                                              sw_if_index, 0, 0, 0);
1771                 pool_put (sm->interfaces, i);
1772                 if (is_inside)
1773                   {
1774                     if (sm->endpoint_dependent)
1775                       vnet_feature_enable_disable ("ip4-local",
1776                                                    "nat44-ed-hairpinning",
1777                                                    sw_if_index, 0, 0, 0);
1778                     else if (!sm->deterministic)
1779                       vnet_feature_enable_disable ("ip4-local",
1780                                                    "nat44-hairpinning",
1781                                                    sw_if_index, 0, 0, 0);
1782                   }
1783               }
1784           }
1785         else
1786           {
1787             if ((nat_interface_is_inside(i) && is_inside) ||
1788                 (nat_interface_is_outside(i) && !is_inside))
1789               return 0;
1790
1791             if (sm->num_workers > 1 && !sm->deterministic)
1792               {
1793                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1794                                                  "nat44-out2in-worker-handoff";
1795                 feature_name = "nat44-handoff-classify";
1796               }
1797             else if (sm->deterministic)
1798               {
1799                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1800                                                  "nat44-det-out2in";
1801                 feature_name = "nat44-det-classify";
1802               }
1803             else if (sm->endpoint_dependent)
1804               {
1805                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1806                                                  "nat44-ed-out2in";
1807                 feature_name = "nat44-ed-classify";
1808               }
1809             else
1810               {
1811                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1812                 feature_name = "nat44-classify";
1813               }
1814
1815             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1816                                          sw_if_index, 0, 0, 0);
1817             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1818                                          sw_if_index, 1, 0, 0);
1819             if (!is_inside)
1820               {
1821                 if (sm->endpoint_dependent)
1822                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1823                                                sw_if_index, 0, 0, 0);
1824                 else if (!sm->deterministic)
1825                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1826                                                sw_if_index, 0, 0, 0);
1827               }
1828             goto set_flags;
1829           }
1830
1831         goto fib;
1832       }
1833   }));
1834
1835   if (is_del)
1836     return VNET_API_ERROR_NO_SUCH_ENTRY;
1837
1838   pool_get (sm->interfaces, i);
1839   i->sw_if_index = sw_if_index;
1840   i->flags = 0;
1841   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1842
1843   if (is_inside && !sm->out2in_dpo)
1844     {
1845       if (sm->endpoint_dependent)
1846         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1847                                      sw_if_index, 1, 0, 0);
1848       else if (!sm->deterministic)
1849         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1850                                      sw_if_index, 1, 0, 0);
1851     }
1852
1853 set_flags:
1854   if (is_inside)
1855     {
1856       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1857       return 0;
1858     }
1859   else
1860     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1861
1862   /* Add/delete external addresses to FIB */
1863 fib:
1864   vec_foreach (ap, sm->addresses)
1865     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1866
1867   pool_foreach (m, sm->static_mappings,
1868   ({
1869     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1870       continue;
1871
1872     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1873   }));
1874
1875   pool_foreach (dm, sm->det_maps,
1876   ({
1877     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1878   }));
1879
1880   return 0;
1881 }
1882
1883 int snat_interface_add_del_output_feature (u32 sw_if_index,
1884                                            u8 is_inside,
1885                                            int is_del)
1886 {
1887   snat_main_t *sm = &snat_main;
1888   snat_interface_t *i;
1889   snat_address_t * ap;
1890   snat_static_mapping_t * m;
1891
1892   if (sm->deterministic ||
1893       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1894     return VNET_API_ERROR_UNSUPPORTED;
1895
1896   pool_foreach (i, sm->interfaces,
1897   ({
1898     if (i->sw_if_index == sw_if_index)
1899       return VNET_API_ERROR_VALUE_EXIST;
1900   }));
1901
1902   if (is_inside)
1903     {
1904       if (sm->endpoint_dependent)
1905         {
1906           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1907                                        sw_if_index, !is_del, 0, 0);
1908           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1909                                        sw_if_index, !is_del, 0, 0);
1910         }
1911       else
1912         {
1913           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1914                                        sw_if_index, !is_del, 0, 0);
1915           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1916                                        sw_if_index, !is_del, 0, 0);
1917         }
1918       goto fq;
1919     }
1920
1921   if (sm->num_workers > 1)
1922     {
1923       vnet_feature_enable_disable ("ip4-unicast",
1924                                    "nat44-out2in-worker-handoff",
1925                                    sw_if_index, !is_del, 0, 0);
1926       vnet_feature_enable_disable ("ip4-output",
1927                                    "nat44-in2out-output-worker-handoff",
1928                                    sw_if_index, !is_del, 0, 0);
1929     }
1930   else
1931     {
1932       if (sm->endpoint_dependent)
1933         {
1934           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
1935                                        sw_if_index, !is_del, 0, 0);
1936           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
1937                                        sw_if_index, !is_del, 0, 0);
1938         }
1939       else
1940         {
1941           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
1942                                        sw_if_index, !is_del, 0, 0);
1943           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1944                                        sw_if_index, !is_del, 0, 0);
1945         }
1946     }
1947
1948 fq:
1949   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1950     sm->fq_in2out_output_index =
1951       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1952
1953   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1954     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1955
1956   pool_foreach (i, sm->output_feature_interfaces,
1957   ({
1958     if (i->sw_if_index == sw_if_index)
1959       {
1960         if (is_del)
1961           pool_put (sm->output_feature_interfaces, i);
1962         else
1963           return VNET_API_ERROR_VALUE_EXIST;
1964
1965         goto fib;
1966       }
1967   }));
1968
1969   if (is_del)
1970     return VNET_API_ERROR_NO_SUCH_ENTRY;
1971
1972   pool_get (sm->output_feature_interfaces, i);
1973   i->sw_if_index = sw_if_index;
1974   i->flags = 0;
1975   if (is_inside)
1976     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1977   else
1978     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1979
1980   /* Add/delete external addresses to FIB */
1981 fib:
1982   if (is_inside)
1983     return 0;
1984
1985   vec_foreach (ap, sm->addresses)
1986     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1987
1988   pool_foreach (m, sm->static_mappings,
1989   ({
1990     if (!(m->addr_only)  || (m->local_addr.as_u32 == m->external_addr.as_u32))
1991       continue;
1992
1993     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1994   }));
1995
1996   return 0;
1997 }
1998
1999 int snat_set_workers (uword * bitmap)
2000 {
2001   snat_main_t *sm = &snat_main;
2002   int i, j = 0;
2003
2004   if (sm->num_workers < 2)
2005     return VNET_API_ERROR_FEATURE_DISABLED;
2006
2007   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2008     return VNET_API_ERROR_INVALID_WORKER;
2009
2010   vec_free (sm->workers);
2011   clib_bitmap_foreach (i, bitmap,
2012     ({
2013       vec_add1(sm->workers, i);
2014       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2015       j++;
2016     }));
2017
2018   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2019   sm->num_snat_thread = _vec_len (sm->workers);
2020
2021   return 0;
2022 }
2023
2024
2025 static void
2026 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2027                                        uword opaque,
2028                                        u32 sw_if_index,
2029                                        ip4_address_t * address,
2030                                        u32 address_length,
2031                                        u32 if_address_index,
2032                                        u32 is_delete);
2033
2034 static void
2035 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2036                                  uword opaque,
2037                                  u32 sw_if_index,
2038                                  ip4_address_t * address,
2039                                  u32 address_length,
2040                                  u32 if_address_index,
2041                                  u32 is_delete);
2042
2043 static int
2044 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2045                                  u32 fib_index,
2046                                  u32 thread_index,
2047                                  snat_session_key_t * k,
2048                                  u32 * address_indexp,
2049                                  u16 port_per_thread,
2050                                  u32 snat_thread_index);
2051
2052 static clib_error_t * snat_init (vlib_main_t * vm)
2053 {
2054   snat_main_t * sm = &snat_main;
2055   clib_error_t * error = 0;
2056   ip4_main_t * im = &ip4_main;
2057   ip_lookup_main_t * lm = &im->lookup_main;
2058   uword *p;
2059   vlib_thread_registration_t *tr;
2060   vlib_thread_main_t *tm = vlib_get_thread_main ();
2061   uword *bitmap = 0;
2062   u32 i;
2063   ip4_add_del_interface_address_callback_t cb4;
2064   vlib_node_t * error_drop_node;
2065
2066   sm->vlib_main = vm;
2067   sm->vnet_main = vnet_get_main();
2068   sm->ip4_main = im;
2069   sm->ip4_lookup_main = lm;
2070   sm->api_main = &api_main;
2071   sm->first_worker_index = 0;
2072   sm->num_workers = 0;
2073   sm->num_snat_thread = 1;
2074   sm->workers = 0;
2075   sm->port_per_thread = 0xffff - 1024;
2076   sm->fq_in2out_index = ~0;
2077   sm->fq_out2in_index = ~0;
2078   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2079   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2080   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2081   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2082   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2083   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
2084   sm->forwarding_enabled = 0;
2085   sm->log_class = vlib_log_register_class ("nat", 0);
2086   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2087   sm->error_node_index = error_drop_node->index;
2088   sm->mss_clamping = 0;
2089
2090   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2091   if (p)
2092     {
2093       tr = (vlib_thread_registration_t *) p[0];
2094       if (tr)
2095         {
2096           sm->num_workers = tr->count;
2097           sm->first_worker_index = tr->first_index;
2098         }
2099     }
2100
2101   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2102
2103   /* Use all available workers by default */
2104   if (sm->num_workers > 1)
2105     {
2106       for (i=0; i < sm->num_workers; i++)
2107         bitmap = clib_bitmap_set (bitmap, i, 1);
2108       snat_set_workers(bitmap);
2109       clib_bitmap_free (bitmap);
2110     }
2111   else
2112     {
2113       sm->per_thread_data[0].snat_thread_index = 0;
2114     }
2115
2116   error = snat_api_init(vm, sm);
2117   if (error)
2118     return error;
2119
2120   /* Set up the interface address add/del callback */
2121   cb4.function = snat_ip4_add_del_interface_address_cb;
2122   cb4.function_opaque = 0;
2123
2124   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2125
2126   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2127   cb4.function_opaque = 0;
2128
2129   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2130
2131   nat_dpo_module_init ();
2132
2133   /* Init IPFIX logging */
2134   snat_ipfix_logging_init(vm);
2135
2136   /* Init NAT64 */
2137   error = nat64_init(vm);
2138   if (error)
2139     return error;
2140
2141   dslite_init(vm);
2142
2143   nat66_init();
2144
2145   /* Init virtual fragmenentation reassembly */
2146   return nat_reass_init(vm);
2147 }
2148
2149 VLIB_INIT_FUNCTION (snat_init);
2150
2151 void snat_free_outside_address_and_port (snat_address_t * addresses,
2152                                          u32 thread_index,
2153                                          snat_session_key_t * k)
2154 {
2155   snat_address_t *a;
2156   u32 address_index;
2157   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2158
2159   for (address_index = 0; address_index < vec_len (addresses); address_index++)
2160     {
2161       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2162         break;
2163     }
2164
2165   ASSERT (address_index < vec_len (addresses));
2166
2167   a = addresses + address_index;
2168
2169   switch (k->protocol)
2170     {
2171 #define _(N, i, n, s) \
2172     case SNAT_PROTOCOL_##N: \
2173       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2174         port_host_byte_order) == 1); \
2175       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2176         port_host_byte_order, 0); \
2177       a->busy_##n##_ports--; \
2178       a->busy_##n##_ports_per_thread[thread_index]--; \
2179       break;
2180       foreach_snat_protocol
2181 #undef _
2182     default:
2183       nat_log_info ("unknown protocol");
2184       return;
2185     }
2186 }
2187
2188 /**
2189  * @brief Match NAT44 static mapping.
2190  *
2191  * @param sm          NAT main.
2192  * @param match       Address and port to match.
2193  * @param mapping     External or local address and port of the matched mapping.
2194  * @param by_external If 0 match by local address otherwise match by external
2195  *                    address.
2196  * @param is_addr_only If matched mapping is address only
2197  * @param twice_nat If matched mapping is twice NAT.
2198  * @param lb If matched mapping is load-balanced.
2199  *
2200  * @returns 0 if match found otherwise 1.
2201  */
2202 int snat_static_mapping_match (snat_main_t * sm,
2203                                snat_session_key_t match,
2204                                snat_session_key_t * mapping,
2205                                u8 by_external,
2206                                u8 *is_addr_only,
2207                                twice_nat_type_t *twice_nat,
2208                                lb_nat_type_t *lb,
2209                                ip4_address_t * ext_host_addr)
2210 {
2211   clib_bihash_kv_8_8_t kv, value;
2212   snat_static_mapping_t *m;
2213   snat_session_key_t m_key;
2214   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2215   u32 rand, lo = 0, hi, mid;
2216   u8 backend_index;
2217
2218   m_key.fib_index = match.fib_index;
2219   if (by_external)
2220     {
2221       mapping_hash = &sm->static_mapping_by_external;
2222       m_key.fib_index = 0;
2223     }
2224
2225   m_key.addr = match.addr;
2226   m_key.port = clib_net_to_host_u16 (match.port);
2227   m_key.protocol = match.protocol;
2228
2229   kv.key = m_key.as_u64;
2230
2231   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2232     {
2233       /* Try address only mapping */
2234       m_key.port = 0;
2235       m_key.protocol = 0;
2236       kv.key = m_key.as_u64;
2237       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2238         return 1;
2239     }
2240
2241   m = pool_elt_at_index (sm->static_mappings, value.value);
2242
2243   if (by_external)
2244     {
2245       if (vec_len (m->locals))
2246         {
2247           if (PREDICT_FALSE(lb != 0))
2248             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2249           if (m->affinity)
2250             {
2251               if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
2252                   match.protocol, match.port, &backend_index))
2253                 goto get_local;
2254
2255               mapping->addr = m->locals[backend_index].addr;
2256               mapping->port = clib_host_to_net_u16 (m->locals[backend_index].port);
2257               mapping->fib_index = m->locals[backend_index].fib_index;
2258               goto end;
2259             }
2260 get_local:
2261           hi = vec_len (m->locals) - 1;
2262           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2263           while (lo < hi)
2264             {
2265               mid = ((hi - lo) >> 1) + lo;
2266               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2267             }
2268           if (!(m->locals[lo].prefix >= rand))
2269             return 1;
2270           if (PREDICT_FALSE (sm->num_workers > 1))
2271             {
2272               ip4_header_t ip = {
2273                 .src_address = m->locals[lo].addr,
2274               };
2275               if (sm->worker_in2out_cb (&ip, m->fib_index) != vlib_get_thread_index ())
2276                 goto get_local;
2277             }
2278           mapping->addr = m->locals[lo].addr;
2279           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2280           mapping->fib_index = m->locals[lo].fib_index;
2281           if (m->affinity)
2282             {
2283               if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
2284                   match.protocol, match.port, lo, m->affinity,
2285                   m->affinity_per_service_list_head_index))
2286                 nat_log_info ("create affinity record failed");
2287             }
2288         }
2289       else
2290         {
2291           if (PREDICT_FALSE(lb != 0))
2292             *lb = NO_LB_NAT;
2293           mapping->fib_index = m->fib_index;
2294           mapping->addr = m->local_addr;
2295           /* Address only mapping doesn't change port */
2296           mapping->port = m->addr_only ? match.port
2297             : clib_host_to_net_u16 (m->local_port);
2298         }
2299       mapping->protocol = m->proto;
2300     }
2301   else
2302     {
2303       mapping->addr = m->external_addr;
2304       /* Address only mapping doesn't change port */
2305       mapping->port = m->addr_only ? match.port
2306         : clib_host_to_net_u16 (m->external_port);
2307       mapping->fib_index = sm->outside_fib_index;
2308     }
2309
2310 end:
2311   if (PREDICT_FALSE(is_addr_only != 0))
2312     *is_addr_only = m->addr_only;
2313
2314   if (PREDICT_FALSE(twice_nat != 0))
2315     *twice_nat = m->twice_nat;
2316
2317   return 0;
2318 }
2319
2320 static_always_inline u16
2321 snat_random_port (u16 min, u16 max)
2322 {
2323   snat_main_t *sm = &snat_main;
2324   return min + random_u32 (&sm->random_seed) /
2325     (random_u32_max() / (max - min + 1) + 1);
2326 }
2327
2328 int
2329 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2330                                      u32 fib_index,
2331                                      u32 thread_index,
2332                                      snat_session_key_t * k,
2333                                      u32 * address_indexp,
2334                                      u16 port_per_thread,
2335                                      u32 snat_thread_index)
2336 {
2337   snat_main_t *sm = &snat_main;
2338
2339   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2340                                  address_indexp, port_per_thread,
2341                                  snat_thread_index);
2342 }
2343
2344 static int
2345 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2346                                  u32 fib_index,
2347                                  u32 thread_index,
2348                                  snat_session_key_t * k,
2349                                  u32 * address_indexp,
2350                                  u16 port_per_thread,
2351                                  u32 snat_thread_index)
2352 {
2353   int i, gi = 0;
2354   snat_address_t *a, *ga = 0;
2355   u32 portnum;
2356
2357   for (i = 0; i < vec_len (addresses); i++)
2358     {
2359       a = addresses + i;
2360       switch (k->protocol)
2361         {
2362 #define _(N, j, n, s) \
2363         case SNAT_PROTOCOL_##N: \
2364           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2365             { \
2366               if (a->fib_index == fib_index) \
2367                 { \
2368                   while (1) \
2369                     { \
2370                       portnum = (port_per_thread * \
2371                         snat_thread_index) + \
2372                         snat_random_port(1, port_per_thread) + 1024; \
2373                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2374                         continue; \
2375                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2376                       a->busy_##n##_ports_per_thread[thread_index]++; \
2377                       a->busy_##n##_ports++; \
2378                       k->addr = a->addr; \
2379                       k->port = clib_host_to_net_u16(portnum); \
2380                       *address_indexp = i; \
2381                       return 0; \
2382                     } \
2383                 } \
2384               else if (a->fib_index == ~0) \
2385                 { \
2386                   ga = a; \
2387                   gi = i; \
2388                 } \
2389             } \
2390           break;
2391           foreach_snat_protocol
2392 #undef _
2393         default:
2394           nat_log_info ("unknown protocol");
2395           return 1;
2396         }
2397
2398     }
2399
2400   if (ga)
2401     {
2402       a = ga;
2403       switch (k->protocol)
2404         {
2405 #define _(N, j, n, s) \
2406         case SNAT_PROTOCOL_##N: \
2407           while (1) \
2408             { \
2409               portnum = (port_per_thread * \
2410                 snat_thread_index) + \
2411                 snat_random_port(1, port_per_thread) + 1024; \
2412               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2413                 continue; \
2414               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2415               a->busy_##n##_ports_per_thread[thread_index]++; \
2416               a->busy_##n##_ports++; \
2417               k->addr = a->addr; \
2418               k->port = clib_host_to_net_u16(portnum); \
2419               *address_indexp = gi; \
2420               return 0; \
2421             }
2422           break;
2423           foreach_snat_protocol
2424 #undef _
2425         default:
2426           nat_log_info ("unknown protocol");
2427           return 1;
2428         }
2429     }
2430
2431   /* Totally out of translations to use... */
2432   snat_ipfix_logging_addresses_exhausted(0);
2433   return 1;
2434 }
2435
2436 static int
2437 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2438                               u32 fib_index,
2439                               u32 thread_index,
2440                               snat_session_key_t * k,
2441                               u32 * address_indexp,
2442                               u16 port_per_thread,
2443                               u32 snat_thread_index)
2444 {
2445   snat_main_t *sm = &snat_main;
2446   snat_address_t *a = addresses;
2447   u16 m, ports, portnum, A, j;
2448   m = 16 - (sm->psid_offset + sm->psid_length);
2449   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2450
2451   if (!vec_len (addresses))
2452     goto exhausted;
2453
2454   switch (k->protocol)
2455     {
2456 #define _(N, i, n, s) \
2457     case SNAT_PROTOCOL_##N: \
2458       if (a->busy_##n##_ports < ports) \
2459         { \
2460           while (1) \
2461             { \
2462               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2463               j = snat_random_port(0, pow2_mask(m)); \
2464               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2465               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2466                 continue; \
2467               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2468               a->busy_##n##_ports++; \
2469               k->addr = a->addr; \
2470               k->port = clib_host_to_net_u16 (portnum); \
2471               *address_indexp = i; \
2472               return 0; \
2473             } \
2474         } \
2475       break;
2476       foreach_snat_protocol
2477 #undef _
2478     default:
2479       nat_log_info ("unknown protocol");
2480       return 1;
2481     }
2482
2483 exhausted:
2484   /* Totally out of translations to use... */
2485   snat_ipfix_logging_addresses_exhausted(0);
2486   return 1;
2487 }
2488
2489 static int
2490 nat_alloc_addr_and_port_range (snat_address_t * addresses,
2491                                u32 fib_index,
2492                                u32 thread_index,
2493                                snat_session_key_t * k,
2494                                u32 * address_indexp,
2495                                u16 port_per_thread,
2496                                u32 snat_thread_index)
2497 {
2498   snat_main_t *sm = &snat_main;
2499   snat_address_t *a = addresses;
2500   u16 portnum, ports;
2501
2502   ports = sm->end_port - sm->start_port + 1;
2503
2504   if (!vec_len (addresses))
2505     goto exhausted;
2506
2507   switch (k->protocol)
2508     {
2509 #define _(N, i, n, s) \
2510     case SNAT_PROTOCOL_##N: \
2511       if (a->busy_##n##_ports < ports) \
2512         { \
2513           while (1) \
2514             { \
2515               portnum = snat_random_port(sm->start_port, sm->end_port); \
2516               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2517                 continue; \
2518               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2519               a->busy_##n##_ports++; \
2520               k->addr = a->addr; \
2521               k->port = clib_host_to_net_u16 (portnum); \
2522               *address_indexp = i; \
2523               return 0; \
2524             } \
2525         } \
2526       break;
2527       foreach_snat_protocol
2528 #undef _
2529     default:
2530       nat_log_info ("unknown protocol");
2531       return 1;
2532     }
2533
2534 exhausted:
2535   /* Totally out of translations to use... */
2536   snat_ipfix_logging_addresses_exhausted(0);
2537   return 1;
2538 }
2539
2540 void
2541 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2542 {
2543   dpo_id_t dpo_v4 = DPO_INVALID;
2544   fib_prefix_t pfx = {
2545     .fp_proto = FIB_PROTOCOL_IP4,
2546     .fp_len = 32,
2547     .fp_addr.ip4.as_u32 = addr.as_u32,
2548   };
2549
2550   if (is_add)
2551     {
2552       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2553       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2554                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2555       dpo_reset (&dpo_v4);
2556     }
2557   else
2558     {
2559       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2560     }
2561 }
2562
2563 uword
2564 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2565 {
2566   u32 *r = va_arg (*args, u32 *);
2567
2568   if (0);
2569 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2570   foreach_snat_protocol
2571 #undef _
2572   else
2573     return 0;
2574   return 1;
2575 }
2576
2577 u8 *
2578 format_snat_protocol (u8 * s, va_list * args)
2579 {
2580   u32 i = va_arg (*args, u32);
2581   u8 *t = 0;
2582
2583   switch (i)
2584     {
2585 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2586       foreach_snat_protocol
2587 #undef _
2588     default:
2589       s = format (s, "unknown");
2590       return s;
2591     }
2592   s = format (s, "%s", t);
2593   return s;
2594 }
2595
2596 u8 *
2597 format_nat_addr_and_port_alloc_alg (u8 * s, va_list * args)
2598 {
2599   u32 i = va_arg (*args, u32);
2600   u8 *t = 0;
2601
2602   switch (i)
2603     {
2604 #define _(v, N, s) case NAT_ADDR_AND_PORT_ALLOC_ALG_##N: t = (u8 *) s; break;
2605       foreach_nat_addr_and_port_alloc_alg
2606 #undef _
2607     default:
2608       s = format (s, "unknown");
2609       return s;
2610     }
2611   s = format (s, "%s", t);
2612   return s;
2613 }
2614
2615 u8 * format_snat_key (u8 * s, va_list * args);
2616 u8 * format_static_mapping_key (u8 * s, va_list * args);
2617
2618 u8 *
2619 format_session_kvp (u8 * s, va_list * args)
2620 {
2621   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2622   snat_session_key_t k;
2623
2624   k.as_u64 = v->key;
2625
2626   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2627
2628   return s;
2629 }
2630
2631 u8 *
2632 format_static_mapping_kvp (u8 * s, va_list * args)
2633 {
2634   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2635   snat_session_key_t k;
2636
2637   k.as_u64 = v->key;
2638
2639   s = format (s, "%U static-mapping-index %llu",
2640               format_static_mapping_key, &k, v->value);
2641
2642   return s;
2643 }
2644
2645 u8 *
2646 format_user_kvp (u8 * s, va_list * args)
2647 {
2648   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2649   snat_user_key_t k;
2650
2651   k.as_u64 = v->key;
2652
2653   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2654               k.fib_index, v->value);
2655
2656   return s;
2657 }
2658
2659 u8 *
2660 format_ed_session_kvp (u8 * s, va_list * args)
2661 {
2662   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2663   nat_ed_ses_key_t k;
2664
2665   k.as_u64[0] = v->key[0];
2666   k.as_u64[1] = v->key[1];
2667
2668   s = format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2669               format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2670               format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2671               format_ip_protocol, k.proto, k.fib_index, v->value);
2672
2673   return s;
2674 }
2675
2676 static u32
2677 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2678 {
2679   snat_main_t *sm = &snat_main;
2680   u32 next_worker_index = 0;
2681   u32 hash;
2682
2683   next_worker_index = sm->first_worker_index;
2684   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2685          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2686
2687   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2688     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2689   else
2690     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2691
2692   return next_worker_index;
2693 }
2694
2695 static u32
2696 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2697 {
2698   snat_main_t *sm = &snat_main;
2699   udp_header_t *udp;
2700   u16 port;
2701   snat_session_key_t m_key;
2702   clib_bihash_kv_8_8_t kv, value;
2703   snat_static_mapping_t *m;
2704   u32 proto;
2705   u32 next_worker_index = 0;
2706
2707   /* first try static mappings without port */
2708   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2709     {
2710       m_key.addr = ip0->dst_address;
2711       m_key.port = 0;
2712       m_key.protocol = 0;
2713       m_key.fib_index = rx_fib_index0;
2714       kv.key = m_key.as_u64;
2715       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2716         {
2717           m = pool_elt_at_index (sm->static_mappings, value.value);
2718           return m->workers[0];
2719         }
2720     }
2721
2722   proto = ip_proto_to_snat_proto (ip0->protocol);
2723   udp = ip4_next_header (ip0);
2724   port = udp->dst_port;
2725
2726   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2727     {
2728       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2729         return vlib_get_thread_index ();
2730
2731       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2732         {
2733           nat_reass_ip4_t *reass;
2734
2735           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2736                                       ip0->fragment_id, ip0->protocol);
2737
2738           if (reass && (reass->thread_index != (u32) ~ 0))
2739             return reass->thread_index;
2740           else
2741             return vlib_get_thread_index ();
2742         }
2743     }
2744
2745   /* unknown protocol */
2746   if (PREDICT_FALSE (proto == ~0))
2747     {
2748       /* use current thread */
2749       return vlib_get_thread_index ();
2750     }
2751
2752   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2753     {
2754       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2755       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2756       if (!icmp_is_error_message (icmp))
2757         port = echo->identifier;
2758       else
2759         {
2760           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2761           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2762           void *l4_header = ip4_next_header (inner_ip);
2763           switch (proto)
2764             {
2765             case SNAT_PROTOCOL_ICMP:
2766               icmp = (icmp46_header_t*)l4_header;
2767               echo = (icmp_echo_header_t *)(icmp + 1);
2768               port = echo->identifier;
2769               break;
2770             case SNAT_PROTOCOL_UDP:
2771             case SNAT_PROTOCOL_TCP:
2772               port = ((tcp_udp_header_t*)l4_header)->src_port;
2773               break;
2774             default:
2775               return vlib_get_thread_index ();
2776             }
2777         }
2778     }
2779
2780   /* try static mappings with port */
2781   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2782     {
2783       m_key.addr = ip0->dst_address;
2784       m_key.port = clib_net_to_host_u16 (port);
2785       m_key.protocol = proto;
2786       m_key.fib_index = rx_fib_index0;
2787       kv.key = m_key.as_u64;
2788       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2789         {
2790           m = pool_elt_at_index (sm->static_mappings, value.value);
2791           return m->workers[0];
2792         }
2793     }
2794
2795   /* worker by outside port */
2796   next_worker_index = sm->first_worker_index;
2797   next_worker_index +=
2798     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2799   return next_worker_index;
2800 }
2801
2802 static u32
2803 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2804 {
2805   snat_main_t *sm = &snat_main;
2806   clib_bihash_kv_8_8_t kv, value;
2807   u32 proto, next_worker_index = 0;
2808   udp_header_t *udp;
2809   u16 port;
2810   snat_static_mapping_t *m;
2811   u32 hash;
2812
2813   /* first try static mappings without port */
2814   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2815     {
2816       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2817       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2818         {
2819           m = pool_elt_at_index (sm->static_mappings, value.value);
2820           return m->workers[0];
2821         }
2822     }
2823
2824   proto = ip_proto_to_snat_proto (ip->protocol);
2825
2826   /* unknown protocol */
2827   if (PREDICT_FALSE (proto == ~0))
2828     {
2829       /* use current thread */
2830       return vlib_get_thread_index ();
2831     }
2832
2833   udp = ip4_next_header (ip);
2834   port = udp->dst_port;
2835
2836   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2837     {
2838       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2839       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2840       if (!icmp_is_error_message (icmp))
2841         port = echo->identifier;
2842       else
2843         {
2844           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2845           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2846           void *l4_header = ip4_next_header (inner_ip);
2847           switch (proto)
2848             {
2849             case SNAT_PROTOCOL_ICMP:
2850               icmp = (icmp46_header_t*)l4_header;
2851               echo = (icmp_echo_header_t *)(icmp + 1);
2852               port = echo->identifier;
2853               break;
2854             case SNAT_PROTOCOL_UDP:
2855             case SNAT_PROTOCOL_TCP:
2856               port = ((tcp_udp_header_t*)l4_header)->src_port;
2857               break;
2858             default:
2859               return vlib_get_thread_index ();
2860             }
2861         }
2862     }
2863
2864   /* try static mappings with port */
2865   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2866     {
2867       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2868                   clib_net_to_host_u16 (port));
2869       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2870         {
2871           m = pool_elt_at_index (sm->static_mappings, value.value);
2872           if (!vec_len(m->locals))
2873             return m->workers[0];
2874
2875           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2876                  (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >>24);
2877
2878           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2879             return m->workers[hash & (_vec_len (m->workers) - 1)];
2880           else
2881             return m->workers[hash % _vec_len (m->workers)];
2882         }
2883     }
2884
2885   /* worker by outside port */
2886   next_worker_index = sm->first_worker_index;
2887   next_worker_index +=
2888     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2889
2890   return next_worker_index;
2891 }
2892
2893 static clib_error_t *
2894 snat_config (vlib_main_t * vm, unformat_input_t * input)
2895 {
2896   snat_main_t * sm = &snat_main;
2897   nat66_main_t * nm = &nat66_main;
2898   u32 translation_buckets = 1024;
2899   u32 translation_memory_size = 128<<20;
2900   u32 user_buckets = 128;
2901   u32 user_memory_size = 64<<20;
2902   u32 max_translations_per_user = 100;
2903   u32 outside_vrf_id = 0;
2904   u32 outside_ip6_vrf_id = 0;
2905   u32 inside_vrf_id = 0;
2906   u32 static_mapping_buckets = 1024;
2907   u32 static_mapping_memory_size = 64<<20;
2908   u32 nat64_bib_buckets = 1024;
2909   u32 nat64_bib_memory_size = 128 << 20;
2910   u32 nat64_st_buckets = 2048;
2911   u32 nat64_st_memory_size = 256 << 20;
2912   u8 static_mapping_only = 0;
2913   u8 static_mapping_connection_tracking = 0;
2914   snat_main_per_thread_data_t *tsm;
2915   dslite_main_t * dm = &dslite_main;
2916
2917   sm->deterministic = 0;
2918   sm->out2in_dpo = 0;
2919   sm->endpoint_dependent = 0;
2920
2921   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2922     {
2923       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2924         ;
2925       else if (unformat (input, "translation hash memory %d",
2926                          &translation_memory_size));
2927       else if (unformat (input, "user hash buckets %d", &user_buckets))
2928         ;
2929       else if (unformat (input, "user hash memory %d",
2930                          &user_memory_size))
2931         ;
2932       else if (unformat (input, "max translations per user %d",
2933                          &max_translations_per_user))
2934         ;
2935       else if (unformat (input, "outside VRF id %d",
2936                          &outside_vrf_id))
2937         ;
2938       else if (unformat (input, "outside ip6 VRF id %d",
2939                          &outside_ip6_vrf_id))
2940         ;
2941       else if (unformat (input, "inside VRF id %d",
2942                          &inside_vrf_id))
2943         ;
2944       else if (unformat (input, "static mapping only"))
2945         {
2946           static_mapping_only = 1;
2947           if (unformat (input, "connection tracking"))
2948             static_mapping_connection_tracking = 1;
2949         }
2950       else if (unformat (input, "deterministic"))
2951         sm->deterministic = 1;
2952       else if (unformat (input, "nat64 bib hash buckets %d",
2953                          &nat64_bib_buckets))
2954         ;
2955       else if (unformat (input, "nat64 bib hash memory %d",
2956                          &nat64_bib_memory_size))
2957         ;
2958       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2959         ;
2960       else if (unformat (input, "nat64 st hash memory %d",
2961                          &nat64_st_memory_size))
2962         ;
2963       else if (unformat (input, "out2in dpo"))
2964         sm->out2in_dpo = 1;
2965       else if (unformat (input, "dslite ce"))
2966         dslite_set_ce(dm, 1);
2967       else if (unformat (input, "endpoint-dependent"))
2968         sm->endpoint_dependent = 1;
2969       else
2970         return clib_error_return (0, "unknown input '%U'",
2971                                   format_unformat_error, input);
2972     }
2973
2974   if (sm->deterministic && sm->endpoint_dependent)
2975     return clib_error_return (
2976       0, "deterministic and endpoint-dependent modes are mutually exclusive");
2977
2978   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
2979     return clib_error_return (
2980       0, "static mapping only mode available only for simple nat");
2981
2982   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
2983     return clib_error_return (
2984       0, "out2in dpo mode available only for simple nat");
2985
2986   /* for show commands, etc. */
2987   sm->translation_buckets = translation_buckets;
2988   sm->translation_memory_size = translation_memory_size;
2989   /* do not exceed load factor 10 */
2990   sm->max_translations = 10 * translation_buckets;
2991   sm->user_buckets = user_buckets;
2992   sm->user_memory_size = user_memory_size;
2993   sm->max_translations_per_user = max_translations_per_user;
2994   sm->outside_vrf_id = outside_vrf_id;
2995   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2996                                                              outside_vrf_id,
2997                                                              FIB_SOURCE_PLUGIN_HI);
2998   nm->outside_vrf_id = outside_ip6_vrf_id;
2999   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
3000                                                              outside_ip6_vrf_id,
3001                                                              FIB_SOURCE_PLUGIN_HI);
3002   sm->inside_vrf_id = inside_vrf_id;
3003   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
3004                                                             inside_vrf_id,
3005                                                             FIB_SOURCE_PLUGIN_HI);
3006   sm->static_mapping_only = static_mapping_only;
3007   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
3008
3009   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
3010                  nat64_st_memory_size);
3011
3012   if (sm->deterministic)
3013     {
3014       sm->in2out_node_index = snat_det_in2out_node.index;
3015       sm->in2out_output_node_index = ~0;
3016       sm->out2in_node_index = snat_det_out2in_node.index;
3017       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
3018       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
3019     }
3020   else
3021     {
3022       if (sm->endpoint_dependent)
3023         {
3024           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3025           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
3026           sm->in2out_node_index = nat44_ed_in2out_node.index;
3027           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
3028           sm->out2in_node_index = nat44_ed_out2in_node.index;
3029           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
3030           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
3031           nat_affinity_init (vm);
3032         }
3033       else
3034         {
3035           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
3036           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
3037           sm->in2out_node_index = snat_in2out_node.index;
3038           sm->in2out_output_node_index = snat_in2out_output_node.index;
3039           sm->out2in_node_index = snat_out2in_node.index;
3040           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
3041           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
3042         }
3043       if (!static_mapping_only ||
3044           (static_mapping_only && static_mapping_connection_tracking))
3045         {
3046           vec_foreach (tsm, sm->per_thread_data)
3047             {
3048               if (sm->endpoint_dependent)
3049                 {
3050                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
3051                                          translation_buckets,
3052                                          translation_memory_size);
3053                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
3054                                                       format_ed_session_kvp);
3055
3056                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
3057                                          translation_buckets,
3058                                          translation_memory_size);
3059                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
3060                                                       format_ed_session_kvp);
3061                 }
3062               else
3063                 {
3064                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
3065                                         translation_buckets,
3066                                         translation_memory_size);
3067                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
3068                                                      format_session_kvp);
3069
3070                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
3071                                         translation_buckets,
3072                                         translation_memory_size);
3073                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
3074                                                      format_session_kvp);
3075                 }
3076
3077               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
3078                                     user_memory_size);
3079               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
3080                                                  format_user_kvp);
3081             }
3082
3083         }
3084       else
3085         {
3086           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
3087           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
3088         }
3089       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3090                             "static_mapping_by_local", static_mapping_buckets,
3091                             static_mapping_memory_size);
3092       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3093                                          format_static_mapping_kvp);
3094
3095       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3096                             "static_mapping_by_external", static_mapping_buckets,
3097                             static_mapping_memory_size);
3098       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3099                                          format_static_mapping_kvp);
3100     }
3101
3102   return 0;
3103 }
3104
3105 VLIB_CONFIG_FUNCTION (snat_config, "nat");
3106
3107 u8 * format_snat_session_state (u8 * s, va_list * args)
3108 {
3109   u32 i = va_arg (*args, u32);
3110   u8 *t = 0;
3111
3112   switch (i)
3113     {
3114 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
3115     foreach_snat_session_state
3116 #undef _
3117     default:
3118       t = format (t, "unknown");
3119     }
3120   s = format (s, "%s", t);
3121   return s;
3122 }
3123
3124 u8 * format_snat_key (u8 * s, va_list * args)
3125 {
3126   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3127
3128   s = format (s, "%U proto %U port %d fib %d",
3129               format_ip4_address, &key->addr,
3130               format_snat_protocol, key->protocol,
3131               clib_net_to_host_u16 (key->port), key->fib_index);
3132   return s;
3133 }
3134
3135 u8 * format_static_mapping_key (u8 * s, va_list * args)
3136 {
3137   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3138
3139   s = format (s, "%U proto %U port %d fib %d",
3140               format_ip4_address, &key->addr,
3141               format_snat_protocol, key->protocol,
3142               key->port, key->fib_index);
3143   return s;
3144 }
3145
3146 u8 * format_snat_session (u8 * s, va_list * args)
3147 {
3148   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3149   snat_session_t * sess = va_arg (*args, snat_session_t *);
3150
3151   if (snat_is_unk_proto_session (sess))
3152     {
3153       s = format (s, "  i2o %U proto %u fib %u\n",
3154                   format_ip4_address, &sess->in2out.addr,
3155                   clib_net_to_host_u16 (sess->in2out.port),
3156                   sess->in2out.fib_index);
3157       s = format (s, "    o2i %U proto %u fib %u\n",
3158                   format_ip4_address, &sess->out2in.addr,
3159                   clib_net_to_host_u16 (sess->out2in.port),
3160                   sess->out2in.fib_index);
3161     }
3162   else
3163     {
3164       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
3165       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
3166     }
3167   if (is_ed_session (sess) || is_fwd_bypass_session (sess))
3168     {
3169       if (is_twice_nat_session (sess))
3170         {
3171           s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
3172                       format_ip4_address, &sess->ext_host_addr,
3173                       clib_net_to_host_u16 (sess->ext_host_port),
3174                       format_ip4_address, &sess->ext_host_nat_addr,
3175                       clib_net_to_host_u16 (sess->ext_host_nat_port));
3176         }
3177       else
3178         {
3179           if (sess->ext_host_addr.as_u32)
3180               s = format (s, "       external host %U:%u\n",
3181                           format_ip4_address, &sess->ext_host_addr,
3182                           clib_net_to_host_u16 (sess->ext_host_port));
3183         }
3184     }
3185   s = format (s, "       index %llu\n", sess - sm->sessions);
3186   s = format (s, "       last heard %.2f\n", sess->last_heard);
3187   s = format (s, "       total pkts %d, total bytes %lld\n",
3188               sess->total_pkts, sess->total_bytes);
3189   if (snat_is_session_static (sess))
3190     s = format (s, "       static translation\n");
3191   else
3192     s = format (s, "       dynamic translation\n");
3193   if (is_fwd_bypass_session (sess))
3194     s = format (s, "       forwarding-bypass\n");
3195   if (is_lb_session (sess))
3196     s = format (s, "       load-balancing\n");
3197   if (is_twice_nat_session (sess))
3198     s = format (s, "       twice-nat\n");
3199
3200   return s;
3201 }
3202
3203 u8 * format_snat_user (u8 * s, va_list * args)
3204 {
3205   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3206   snat_user_t * u = va_arg (*args, snat_user_t *);
3207   int verbose = va_arg (*args, int);
3208   dlist_elt_t * head, * elt;
3209   u32 elt_index, head_index;
3210   u32 session_index;
3211   snat_session_t * sess;
3212
3213   s = format (s, "%U: %d dynamic translations, %d static translations\n",
3214               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
3215
3216   if (verbose == 0)
3217     return s;
3218
3219   if (u->nsessions || u->nstaticsessions)
3220     {
3221       head_index = u->sessions_per_user_list_head_index;
3222       head = pool_elt_at_index (sm->list_pool, head_index);
3223
3224       elt_index = head->next;
3225       elt = pool_elt_at_index (sm->list_pool, elt_index);
3226       session_index = elt->value;
3227
3228       while (session_index != ~0)
3229         {
3230           sess = pool_elt_at_index (sm->sessions, session_index);
3231
3232           s = format (s, "  %U\n", format_snat_session, sm, sess);
3233
3234           elt_index = elt->next;
3235           elt = pool_elt_at_index (sm->list_pool, elt_index);
3236           session_index = elt->value;
3237         }
3238     }
3239
3240   return s;
3241 }
3242
3243 u8 * format_snat_static_mapping (u8 * s, va_list * args)
3244 {
3245   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
3246   nat44_lb_addr_port_t *local;
3247
3248   if (m->addr_only)
3249       s = format (s, "local %U external %U vrf %d %s %s",
3250                   format_ip4_address, &m->local_addr,
3251                   format_ip4_address, &m->external_addr,
3252                   m->vrf_id,
3253                   m->twice_nat == TWICE_NAT ? "twice-nat" :
3254                   m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3255                   m->out2in_only ? "out2in-only" : "");
3256   else
3257    {
3258       if (vec_len (m->locals))
3259         {
3260           s = format (s, "%U external %U:%d %s %s",
3261                       format_snat_protocol, m->proto,
3262                       format_ip4_address, &m->external_addr, m->external_port,
3263                       m->twice_nat == TWICE_NAT ? "twice-nat" :
3264                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3265                       m->out2in_only ? "out2in-only" : "");
3266           vec_foreach (local, m->locals)
3267             s = format (s, "\n  local %U:%d vrf %d probability %d\%",
3268                         format_ip4_address, &local->addr, local->port,
3269                         local->vrf_id, local->probability);
3270         }
3271       else
3272         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
3273                     format_snat_protocol, m->proto,
3274                     format_ip4_address, &m->local_addr, m->local_port,
3275                     format_ip4_address, &m->external_addr, m->external_port,
3276                     m->vrf_id,
3277                     m->twice_nat == TWICE_NAT ? "twice-nat" :
3278                     m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3279                     m->out2in_only ? "out2in-only" : "");
3280    }
3281   return s;
3282 }
3283
3284 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
3285 {
3286   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
3287   vnet_main_t *vnm = vnet_get_main();
3288
3289   if (m->addr_only)
3290       s = format (s, "local %U external %U vrf %d",
3291                   format_ip4_address, &m->l_addr,
3292                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3293                   m->vrf_id);
3294   else
3295       s = format (s, "%U local %U:%d external %U:%d vrf %d",
3296                   format_snat_protocol, m->proto,
3297                   format_ip4_address, &m->l_addr, m->l_port,
3298                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3299                   m->e_port, m->vrf_id);
3300
3301   return s;
3302 }
3303
3304 u8 * format_det_map_ses (u8 * s, va_list * args)
3305 {
3306   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
3307   ip4_address_t in_addr, out_addr;
3308   u32 in_offset, out_offset;
3309   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
3310   u32 * i = va_arg (*args, u32 *);
3311
3312   u32 user_index = *i / SNAT_DET_SES_PER_USER;
3313   in_addr.as_u32 = clib_host_to_net_u32 (
3314     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
3315   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
3316     clib_net_to_host_u32(det_map->in_addr.as_u32);
3317   out_offset = in_offset / det_map->sharing_ratio;
3318   out_addr.as_u32 = clib_host_to_net_u32(
3319     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
3320   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
3321               format_ip4_address, &in_addr,
3322               clib_net_to_host_u16 (ses->in_port),
3323               format_ip4_address, &out_addr,
3324               clib_net_to_host_u16 (ses->out.out_port),
3325               format_ip4_address, &ses->out.ext_host_addr,
3326               clib_net_to_host_u16 (ses->out.ext_host_port),
3327               format_snat_session_state, ses->state,
3328               ses->expire);
3329
3330   return s;
3331 }
3332
3333 static void
3334 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3335                                  uword opaque,
3336                                  u32 sw_if_index,
3337                                  ip4_address_t * address,
3338                                  u32 address_length,
3339                                  u32 if_address_index,
3340                                  u32 is_delete)
3341 {
3342   snat_main_t *sm = &snat_main;
3343   snat_static_map_resolve_t *rp;
3344   snat_static_mapping_t *m;
3345   snat_session_key_t m_key;
3346   clib_bihash_kv_8_8_t kv, value;
3347   int i, rv;
3348   ip4_address_t l_addr;
3349
3350   for (i = 0; i < vec_len (sm->to_resolve); i++)
3351     {
3352       rp = sm->to_resolve + i;
3353       if (rp->addr_only == 0)
3354         continue;
3355       if (rp->sw_if_index == sw_if_index)
3356         goto match;
3357     }
3358
3359   return;
3360
3361 match:
3362   m_key.addr.as_u32 = address->as_u32;
3363   m_key.port = rp->addr_only ? 0 : rp->e_port;
3364   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3365   m_key.fib_index = sm->outside_fib_index;
3366   kv.key = m_key.as_u64;
3367   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3368     m = 0;
3369   else
3370     m = pool_elt_at_index (sm->static_mappings, value.value);
3371
3372   if (!is_delete)
3373     {
3374       /* Don't trip over lease renewal, static config */
3375       if (m)
3376         return;
3377     }
3378   else
3379     {
3380       if (!m)
3381         return;
3382     }
3383
3384   /* Indetity mapping? */
3385   if (rp->l_addr.as_u32 == 0)
3386     l_addr.as_u32 = address[0].as_u32;
3387   else
3388     l_addr.as_u32 = rp->l_addr.as_u32;
3389   /* Add the static mapping */
3390   rv = snat_add_static_mapping (l_addr,
3391                                 address[0],
3392                                 rp->l_port,
3393                                 rp->e_port,
3394                                 rp->vrf_id,
3395                                 rp->addr_only,
3396                                 ~0 /* sw_if_index */,
3397                                 rp->proto,
3398                                 !is_delete,
3399                                 0, 0, rp->tag);
3400   if (rv)
3401     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3402 }
3403
3404 static void
3405 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3406                                        uword opaque,
3407                                        u32 sw_if_index,
3408                                        ip4_address_t * address,
3409                                        u32 address_length,
3410                                        u32 if_address_index,
3411                                        u32 is_delete)
3412 {
3413   snat_main_t *sm = &snat_main;
3414   snat_static_map_resolve_t *rp;
3415   ip4_address_t l_addr;
3416   int i, j;
3417   int rv;
3418   u8 twice_nat = 0;
3419   snat_address_t *addresses = sm->addresses;
3420
3421   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
3422     {
3423       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3424           goto match;
3425     }
3426
3427   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
3428     {
3429       twice_nat = 1;
3430       addresses = sm->twice_nat_addresses;
3431       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3432           goto match;
3433     }
3434
3435   return;
3436
3437 match:
3438   if (!is_delete)
3439     {
3440       /* Don't trip over lease renewal, static config */
3441       for (j = 0; j < vec_len(addresses); j++)
3442         if (addresses[j].addr.as_u32 == address->as_u32)
3443           return;
3444
3445       (void) snat_add_address (sm, address, ~0, twice_nat);
3446       /* Scan static map resolution vector */
3447       for (j = 0; j < vec_len (sm->to_resolve); j++)
3448         {
3449           rp = sm->to_resolve + j;
3450           if (rp->addr_only)
3451             continue;
3452           /* On this interface? */
3453           if (rp->sw_if_index == sw_if_index)
3454             {
3455               /* Indetity mapping? */
3456               if (rp->l_addr.as_u32 == 0)
3457                 l_addr.as_u32 = address[0].as_u32;
3458               else
3459                 l_addr.as_u32 = rp->l_addr.as_u32;
3460               /* Add the static mapping */
3461               rv = snat_add_static_mapping (l_addr,
3462                                             address[0],
3463                                             rp->l_port,
3464                                             rp->e_port,
3465                                             rp->vrf_id,
3466                                             rp->addr_only,
3467                                             ~0 /* sw_if_index */,
3468                                             rp->proto,
3469                                             rp->is_add,
3470                                             0, 0, rp->tag);
3471               if (rv)
3472                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3473             }
3474         }
3475       return;
3476     }
3477   else
3478     {
3479       (void) snat_del_address(sm, address[0], 1, twice_nat);
3480       return;
3481     }
3482 }
3483
3484
3485 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
3486                                 u8 twice_nat)
3487 {
3488   ip4_main_t * ip4_main = sm->ip4_main;
3489   ip4_address_t * first_int_addr;
3490   snat_static_map_resolve_t *rp;
3491   u32 *indices_to_delete = 0;
3492   int i, j;
3493   u32 *auto_add_sw_if_indices =
3494     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3495
3496   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
3497                                                 0 /* just want the address*/);
3498
3499   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
3500     {
3501       if (auto_add_sw_if_indices[i] == sw_if_index)
3502         {
3503           if (is_del)
3504             {
3505               /* if have address remove it */
3506               if (first_int_addr)
3507                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3508               else
3509                 {
3510                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3511                     {
3512                       rp = sm->to_resolve + j;
3513                       if (rp->sw_if_index == sw_if_index)
3514                         vec_add1 (indices_to_delete, j);
3515                     }
3516                   if (vec_len(indices_to_delete))
3517                     {
3518                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3519                         vec_del1(sm->to_resolve, j);
3520                       vec_free(indices_to_delete);
3521                     }
3522                 }
3523               if (twice_nat)
3524                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3525               else
3526                 vec_del1(sm->auto_add_sw_if_indices, i);
3527             }
3528           else
3529             return VNET_API_ERROR_VALUE_EXIST;
3530
3531           return 0;
3532         }
3533     }
3534
3535   if (is_del)
3536     return VNET_API_ERROR_NO_SUCH_ENTRY;
3537
3538   /* add to the auto-address list */
3539   if (twice_nat)
3540     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3541   else
3542     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3543
3544   /* If the address is already bound - or static - add it now */
3545   if (first_int_addr)
3546       (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3547
3548   return 0;
3549 }
3550
3551 int
3552 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3553                    snat_protocol_t proto, u32 vrf_id, int is_in)
3554 {
3555   snat_main_per_thread_data_t *tsm;
3556   clib_bihash_kv_8_8_t kv, value;
3557   ip4_header_t ip;
3558   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3559   snat_session_key_t key;
3560   snat_session_t *s;
3561   clib_bihash_8_8_t *t;
3562
3563   if (sm->endpoint_dependent)
3564     return VNET_API_ERROR_UNSUPPORTED;
3565
3566   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3567   if (sm->num_workers > 1)
3568     tsm =
3569       vec_elt_at_index (sm->per_thread_data,
3570                         sm->worker_in2out_cb (&ip, fib_index));
3571   else
3572     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3573
3574   key.addr.as_u32 = addr->as_u32;
3575   key.port = clib_host_to_net_u16 (port);
3576   key.protocol = proto;
3577   key.fib_index = fib_index;
3578   kv.key = key.as_u64;
3579   t = is_in ? &tsm->in2out : &tsm->out2in;
3580   if (!clib_bihash_search_8_8 (t, &kv, &value))
3581     {
3582       if (pool_is_free_index (tsm->sessions, value.value))
3583         return VNET_API_ERROR_UNSPECIFIED;
3584
3585       s = pool_elt_at_index (tsm->sessions, value.value);
3586       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3587       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3588       return 0;
3589     }
3590
3591   return VNET_API_ERROR_NO_SUCH_ENTRY;
3592 }
3593
3594 int
3595 nat44_del_ed_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3596                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3597                       u32 vrf_id, int is_in)
3598 {
3599   ip4_header_t ip;
3600   clib_bihash_16_8_t *t;
3601   nat_ed_ses_key_t key;
3602   clib_bihash_kv_16_8_t kv, value;
3603   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3604   snat_session_t *s;
3605   snat_main_per_thread_data_t *tsm;
3606
3607   if (!sm->endpoint_dependent)
3608     return VNET_API_ERROR_FEATURE_DISABLED;
3609
3610   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3611   if (sm->num_workers > 1)
3612     tsm =
3613       vec_elt_at_index (sm->per_thread_data,
3614                         sm->worker_in2out_cb (&ip, fib_index));
3615   else
3616     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3617
3618   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3619   key.l_addr.as_u32 = addr->as_u32;
3620   key.r_addr.as_u32 = eh_addr->as_u32;
3621   key.l_port = clib_host_to_net_u16 (port);
3622   key.r_port = clib_host_to_net_u16 (eh_port);
3623   key.proto = proto;
3624   key.fib_index = clib_host_to_net_u32 (fib_index);
3625   kv.key[0] = key.as_u64[0];
3626   kv.key[1] = key.as_u64[1];
3627   if (clib_bihash_search_16_8 (t, &kv, &value))
3628     return VNET_API_ERROR_NO_SUCH_ENTRY;
3629
3630   if (pool_is_free_index (tsm->sessions, value.value))
3631     return VNET_API_ERROR_UNSPECIFIED;
3632   s = pool_elt_at_index (tsm->sessions, value.value);
3633   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3634   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3635   return 0;
3636 }
3637
3638 void
3639 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3640 {
3641   snat_main_t *sm = &snat_main;
3642
3643   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_MAPE;
3644   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3645   sm->psid = psid;
3646   sm->psid_offset = psid_offset;
3647   sm->psid_length = psid_length;
3648 }
3649
3650 void
3651 nat_set_alloc_addr_and_port_range (u16 start_port, u16 end_port)
3652 {
3653   snat_main_t *sm = &snat_main;
3654
3655   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_RANGE;
3656   sm->alloc_addr_and_port = nat_alloc_addr_and_port_range;
3657   sm->start_port = start_port;
3658   sm->end_port = end_port;
3659 }
3660
3661 void
3662 nat_set_alloc_addr_and_port_default (void)
3663 {
3664   snat_main_t *sm = &snat_main;
3665
3666   sm->addr_and_port_alloc_alg = NAT_ADDR_AND_PORT_ALLOC_ALG_DEFAULT;
3667   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3668 }
3669