src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <vnet/fib/fib_table.h>
  32 #include <vnet/fib/ip4_fib.h>
  33
  34 #include <vpp/app/version.h>
  35
  36 snat_main_t snat_main;
  37
  38
  39 /* Hook up input features */
  40 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  41   .arc_name = "ip4-unicast",
  42   .node_name = "nat44-in2out",
  43   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  44 };
  45 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  46   .arc_name = "ip4-unicast",
  47   .node_name = "nat44-out2in",
  48   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  49                                "ip4-dhcp-client-detect"),
  50 };
  51 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  52   .arc_name = "ip4-unicast",
  53   .node_name = "nat44-classify",
  54   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  55 };
  56 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  57   .arc_name = "ip4-unicast",
  58   .node_name = "nat44-det-in2out",
  59   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  60 };
  61 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-det-out2in",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  65                                "ip4-dhcp-client-detect"),
  66 };
  67 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  68   .arc_name = "ip4-unicast",
  69   .node_name = "nat44-det-classify",
  70   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  71 };
  72 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-ed-in2out",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  76 };
  77 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  78   .arc_name = "ip4-unicast",
  79   .node_name = "nat44-ed-out2in",
  80   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  81                                "ip4-dhcp-client-detect"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ed-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  87 };
  88 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-in2out-worker-handoff",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  92 };
  93 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-out2in-worker-handoff",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  97                                "ip4-dhcp-client-detect"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-handoff-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 103 };
 104 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-fast",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 108 };
 109 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-fast",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-hairpin-dst",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-hairpin-dst",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 124 };
 125
 126 /* Hook up output features */
 127 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 128   .arc_name = "ip4-output",
 129   .node_name = "nat44-in2out-output",
 130   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 136 };
 137 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 138   .arc_name = "ip4-output",
 139   .node_name = "nat44-hairpin-src",
 140   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 141 };
 142 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 143   .arc_name = "ip4-output",
 144   .node_name = "nat44-ed-in2out-output",
 145   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 146 };
 147 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 148   .arc_name = "ip4-output",
 149   .node_name = "nat44-ed-hairpin-src",
 150   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 151 };
 152
 153 /* Hook up ip4-local features */
 154 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 155 {
 156   .arc_name = "ip4-local",
 157   .node_name = "nat44-hairpinning",
 158   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 159 };
 160 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 161 {
 162   .arc_name = "ip4-local",
 163   .node_name = "nat44-ed-hairpinning",
 164   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 165 };
 166
 167
 168 /* *INDENT-OFF* */
 169 VLIB_PLUGIN_REGISTER () = {
 170     .version = VPP_BUILD_VER,
 171     .description = "Network Address Translation",
 172 };
 173 /* *INDENT-ON* */
 174
 175 vlib_node_registration_t nat44_classify_node;
 176 vlib_node_registration_t nat44_ed_classify_node;
 177 vlib_node_registration_t nat44_det_classify_node;
 178 vlib_node_registration_t nat44_handoff_classify_node;
 179
 180 typedef enum {
 181   NAT44_CLASSIFY_NEXT_IN2OUT,
 182   NAT44_CLASSIFY_NEXT_OUT2IN,
 183   NAT44_CLASSIFY_N_NEXT,
 184 } nat44_classify_next_t;
 185
 186 void
 187 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 188 {
 189   snat_session_key_t key;
 190   clib_bihash_kv_8_8_t kv;
 191   nat_ed_ses_key_t ed_key;
 192   clib_bihash_kv_16_8_t ed_kv;
 193   int i;
 194   snat_address_t *a;
 195   snat_main_per_thread_data_t *tsm =
 196     vec_elt_at_index (sm->per_thread_data, thread_index);
 197
 198   if (is_fwd_bypass_session (s))
 199     {
 200       ed_key.l_addr = s->in2out.addr;
 201       ed_key.r_addr = s->ext_host_addr;
 202       ed_key.l_port = s->in2out.port;
 203       ed_key.r_port = s->ext_host_port;
 204       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 205       ed_key.fib_index = 0;
 206       ed_kv.key[0] = ed_key.as_u64[0];
 207       ed_kv.key[1] = ed_key.as_u64[1];
 208       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 209         nat_log_warn ("in2out_ed key del failed");
 210       return;
 211     }
 212
 213   /* session lookup tables */
 214   if (is_ed_session (s))
 215     {
 216       ed_key.l_addr = s->out2in.addr;
 217       ed_key.r_addr = s->ext_host_addr;
 218       ed_key.fib_index = s->out2in.fib_index;
 219       if (snat_is_unk_proto_session (s))
 220         {
 221           ed_key.proto = s->in2out.port;
 222           ed_key.r_port = 0;
 223           ed_key.l_port = 0;
 224         }
 225       else
 226         {
 227           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 228           ed_key.l_port = s->out2in.port;
 229           ed_key.r_port = s->ext_host_port;
 230         }
 231       ed_kv.key[0] = ed_key.as_u64[0];
 232       ed_kv.key[1] = ed_key.as_u64[1];
 233       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 234         nat_log_warn ("out2in_ed key del failed");
 235
 236       ed_key.l_addr = s->in2out.addr;
 237       ed_key.fib_index = s->in2out.fib_index;
 238       if (!snat_is_unk_proto_session (s))
 239         ed_key.l_port = s->in2out.port;
 240       if (is_twice_nat_session (s))
 241         {
 242           ed_key.r_addr = s->ext_host_nat_addr;
 243           ed_key.r_port = s->ext_host_nat_port;
 244         }
 245       ed_kv.key[0] = ed_key.as_u64[0];
 246       ed_kv.key[1] = ed_key.as_u64[1];
 247       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 248         nat_log_warn ("in2out_ed key del failed");
 249     }
 250   else
 251     {
 252       kv.key = s->in2out.as_u64;
 253       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 254         nat_log_warn ("in2out key del failed");
 255       kv.key = s->out2in.as_u64;
 256       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 257         nat_log_warn ("out2in key del failed");
 258     }
 259
 260   if (snat_is_unk_proto_session (s))
 261     return;
 262
 263   /* log NAT event */
 264   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 265                                       s->out2in.addr.as_u32,
 266                                       s->in2out.protocol,
 267                                       s->in2out.port,
 268                                       s->out2in.port,
 269                                       s->in2out.fib_index);
 270
 271   /* Twice NAT address and port for external host */
 272   if (is_twice_nat_session (s))
 273     {
 274       for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 275         {
 276           key.protocol = s->in2out.protocol;
 277           key.port = s->ext_host_nat_port;
 278           a = sm->twice_nat_addresses + i;
 279           if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 280             {
 281               snat_free_outside_address_and_port (sm->twice_nat_addresses,
 282                                                   thread_index, &key, i);
 283               break;
 284             }
 285         }
 286     }
 287
 288   if (snat_is_session_static (s))
 289     return;
 290
 291   if (s->outside_address_index != ~0)
 292     snat_free_outside_address_and_port (sm->addresses, thread_index,
 293                                         &s->out2in, s->outside_address_index);
 294 }
 295
 296 snat_user_t *
 297 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 298                         u32 thread_index)
 299 {
 300   snat_user_t *u = 0;
 301   snat_user_key_t user_key;
 302   clib_bihash_kv_8_8_t kv, value;
 303   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 304   dlist_elt_t * per_user_list_head_elt;
 305
 306   user_key.addr.as_u32 = addr->as_u32;
 307   user_key.fib_index = fib_index;
 308   kv.key = user_key.as_u64;
 309
 310   /* Ever heard of the "user" = src ip4 address before? */
 311   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 312     {
 313       /* no, make a new one */
 314       pool_get (tsm->users, u);
 315       memset (u, 0, sizeof (*u));
 316       u->addr.as_u32 = addr->as_u32;
 317       u->fib_index = fib_index;
 318
 319       pool_get (tsm->list_pool, per_user_list_head_elt);
 320
 321       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 322         tsm->list_pool;
 323
 324       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 325
 326       kv.value = u - tsm->users;
 327
 328       /* add user */
 329       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 330         nat_log_warn ("user_hash keay add failed");
 331     }
 332   else
 333     {
 334       u = pool_elt_at_index (tsm->users, value.value);
 335     }
 336
 337   return u;
 338 }
 339
 340 snat_session_t *
 341 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 342 {
 343   snat_session_t *s;
 344   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 345   u32 oldest_per_user_translation_list_index, session_index;
 346   dlist_elt_t * oldest_per_user_translation_list_elt;
 347   dlist_elt_t * per_user_translation_list_elt;
 348
 349   /* Over quota? Recycle the least recently used translation */
 350   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 351     {
 352       oldest_per_user_translation_list_index =
 353         clib_dlist_remove_head (tsm->list_pool,
 354                                 u->sessions_per_user_list_head_index);
 355
 356       ASSERT (oldest_per_user_translation_list_index != ~0);
 357
 358       /* Add it back to the end of the LRU list */
 359       clib_dlist_addtail (tsm->list_pool,
 360                           u->sessions_per_user_list_head_index,
 361                           oldest_per_user_translation_list_index);
 362       /* Get the list element */
 363       oldest_per_user_translation_list_elt =
 364         pool_elt_at_index (tsm->list_pool,
 365                            oldest_per_user_translation_list_index);
 366
 367       /* Get the session index from the list element */
 368       session_index = oldest_per_user_translation_list_elt->value;
 369
 370       /* Get the session */
 371       s = pool_elt_at_index (tsm->sessions, session_index);
 372       nat_free_session_data (sm, s, thread_index);
 373       if (snat_is_session_static(s))
 374         u->nstaticsessions--;
 375       else
 376         u->nsessions--;
 377       s->outside_address_index = ~0;
 378       s->flags = 0;
 379       s->total_bytes = 0;
 380       s->total_pkts = 0;
 381       s->state = 0;
 382       s->ext_host_addr.as_u32 = 0;
 383       s->ext_host_port = 0;
 384       s->ext_host_nat_addr.as_u32 = 0;
 385       s->ext_host_nat_port = 0;
 386     }
 387   else
 388     {
 389       pool_get (tsm->sessions, s);
 390       memset (s, 0, sizeof (*s));
 391       s->outside_address_index = ~0;
 392
 393       /* Create list elts */
 394       pool_get (tsm->list_pool, per_user_translation_list_elt);
 395       clib_dlist_init (tsm->list_pool,
 396                        per_user_translation_list_elt - tsm->list_pool);
 397
 398       per_user_translation_list_elt->value = s - tsm->sessions;
 399       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 400       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 401
 402       clib_dlist_addtail (tsm->list_pool,
 403                           s->per_user_list_head_index,
 404                           per_user_translation_list_elt - tsm->list_pool);
 405     }
 406
 407   return s;
 408 }
 409
 410 typedef struct {
 411   u8 next_in2out;
 412 } nat44_classify_trace_t;
 413
 414 static u8 * format_nat44_classify_trace (u8 * s, va_list * args)
 415 {
 416   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 417   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 418   nat44_classify_trace_t *t = va_arg (*args, nat44_classify_trace_t *);
 419   char *next;
 420
 421   next = t->next_in2out ? "nat44-in2out" : "nat44-out2in";
 422
 423   s = format (s, "nat44-classify: next %s", next);
 424
 425   return s;
 426 }
 427
 428 static inline uword
 429 nat44_classify_node_fn_inline (vlib_main_t * vm,
 430                                vlib_node_runtime_t * node,
 431                                vlib_frame_t * frame,
 432                                int is_ed)
 433 {
 434   u32 n_left_from, * from, * to_next;
 435   nat44_classify_next_t next_index;
 436   snat_main_t *sm = &snat_main;
 437   snat_static_mapping_t *m;
 438   u32 thread_index = vm->thread_index;
 439   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 440
 441   from = vlib_frame_vector_args (frame);
 442   n_left_from = frame->n_vectors;
 443   next_index = node->cached_next_index;
 444
 445   while (n_left_from > 0)
 446     {
 447       u32 n_left_to_next;
 448
 449       vlib_get_next_frame (vm, node, next_index,
 450                            to_next, n_left_to_next);
 451
 452       while (n_left_from > 0 && n_left_to_next > 0)
 453         {
 454           u32 bi0;
 455           vlib_buffer_t *b0;
 456           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0;
 457           ip4_header_t *ip0;
 458           snat_address_t *ap;
 459           snat_session_key_t m_key0;
 460           clib_bihash_kv_8_8_t kv0, value0;
 461           clib_bihash_kv_16_8_t ed_kv0, ed_value0;
 462           udp_header_t *udp0;
 463
 464           /* speculatively enqueue b0 to the current next frame */
 465           bi0 = from[0];
 466           to_next[0] = bi0;
 467           from += 1;
 468           to_next += 1;
 469           n_left_from -= 1;
 470           n_left_to_next -= 1;
 471
 472           b0 = vlib_get_buffer (vm, bi0);
 473           ip0 = vlib_buffer_get_current (b0);
 474           udp0 = ip4_next_header (ip0);
 475
 476           if (is_ed)
 477             {
 478               sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 479               rx_fib_index0 =
 480                 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 481                                                      sw_if_index0);
 482               make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address,
 483                           ip0->protocol, rx_fib_index0, udp0->src_port,
 484                           udp0->dst_port);
 485               if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, &ed_value0))
 486                 goto enqueue0;
 487             }
 488
 489           vec_foreach (ap, sm->addresses)
 490             {
 491               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 492                 {
 493                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 494                   goto enqueue0;
 495                 }
 496             }
 497
 498           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 499             {
 500               m_key0.addr = ip0->dst_address;
 501               m_key0.port = 0;
 502               m_key0.protocol = 0;
 503               m_key0.fib_index = 0;
 504               kv0.key = m_key0.as_u64;
 505               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 506                 {
 507                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 508                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 509                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 510                   goto enqueue0;
 511                 }
 512               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 513               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 514               kv0.key = m_key0.as_u64;
 515               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 516                 {
 517                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 518                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 519                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 520                 }
 521             }
 522
 523         enqueue0:
 524           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 525                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 526             {
 527               nat44_classify_trace_t *t =
 528                   vlib_add_trace (vm, node, b0, sizeof (*t));
 529               t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
 530             }
 531
 532           /* verify speculative enqueue, maybe switch current next frame */
 533           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 534                                            to_next, n_left_to_next,
 535                                            bi0, next0);
 536         }
 537
 538       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 539     }
 540
 541   return frame->n_vectors;
 542 }
 543
 544 static uword
 545 nat44_classify_node_fn (vlib_main_t * vm,
 546                         vlib_node_runtime_t * node,
 547                         vlib_frame_t * frame)
 548 {
 549   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 550 };
 551
 552 VLIB_REGISTER_NODE (nat44_classify_node) = {
 553   .function = nat44_classify_node_fn,
 554   .name = "nat44-classify",
 555   .vector_size = sizeof (u32),
 556   .format_trace = format_nat44_classify_trace,
 557   .type = VLIB_NODE_TYPE_INTERNAL,
 558   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 559   .next_nodes = {
 560     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 561     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 562   },
 563 };
 564
 565 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 566                               nat44_classify_node_fn);
 567 static uword
 568 nat44_ed_classify_node_fn (vlib_main_t * vm,
 569                            vlib_node_runtime_t * node,
 570                            vlib_frame_t * frame)
 571 {
 572   return nat44_classify_node_fn_inline (vm, node, frame, 1);
 573 };
 574
 575 VLIB_REGISTER_NODE (nat44_ed_classify_node) = {
 576   .function = nat44_ed_classify_node_fn,
 577   .name = "nat44-ed-classify",
 578   .vector_size = sizeof (u32),
 579   .format_trace = format_nat44_classify_trace,
 580   .type = VLIB_NODE_TYPE_INTERNAL,
 581   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 582   .next_nodes = {
 583     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out",
 584     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in",
 585   },
 586 };
 587
 588 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_classify_node,
 589                               nat44_ed_classify_node_fn);
 590
 591 static uword
 592 nat44_det_classify_node_fn (vlib_main_t * vm,
 593                             vlib_node_runtime_t * node,
 594                             vlib_frame_t * frame)
 595 {
 596   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 597 };
 598
 599 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 600   .function = nat44_det_classify_node_fn,
 601   .name = "nat44-det-classify",
 602   .vector_size = sizeof (u32),
 603   .format_trace = format_nat44_classify_trace,
 604   .type = VLIB_NODE_TYPE_INTERNAL,
 605   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 606   .next_nodes = {
 607     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 608     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 609   },
 610 };
 611
 612 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 613                               nat44_det_classify_node_fn);
 614
 615 static uword
 616 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 617                                 vlib_node_runtime_t * node,
 618                                 vlib_frame_t * frame)
 619 {
 620   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 621 };
 622
 623 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 624   .function = nat44_handoff_classify_node_fn,
 625   .name = "nat44-handoff-classify",
 626   .vector_size = sizeof (u32),
 627   .format_trace = format_nat44_classify_trace,
 628   .type = VLIB_NODE_TYPE_INTERNAL,
 629   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 630   .next_nodes = {
 631     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 632     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 633   },
 634 };
 635
 636 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 637                               nat44_handoff_classify_node_fn);
 638
 639 /**
 640  * @brief Add/del NAT address to FIB.
 641  *
 642  * Add the external NAT address to the FIB as receive entries. This ensures
 643  * that VPP will reply to ARP for this address and we don't need to enable
 644  * proxy ARP on the outside interface.
 645  *
 646  * @param addr IPv4 address.
 647  * @param plen address prefix length
 648  * @param sw_if_index Interface.
 649  * @param is_add If 0 delete, otherwise add.
 650  */
 651 void
 652 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 653                           int is_add)
 654 {
 655   fib_prefix_t prefix = {
 656     .fp_len = p_len,
 657     .fp_proto = FIB_PROTOCOL_IP4,
 658     .fp_addr = {
 659         .ip4.as_u32 = addr->as_u32,
 660     },
 661   };
 662   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 663
 664   if (is_add)
 665     fib_table_entry_update_one_path(fib_index,
 666                                     &prefix,
 667                                     FIB_SOURCE_PLUGIN_LOW,
 668                                     (FIB_ENTRY_FLAG_CONNECTED |
 669                                      FIB_ENTRY_FLAG_LOCAL |
 670                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 671                                     DPO_PROTO_IP4,
 672                                     NULL,
 673                                     sw_if_index,
 674                                     ~0,
 675                                     1,
 676                                     NULL,
 677                                     FIB_ROUTE_PATH_FLAG_NONE);
 678   else
 679     fib_table_entry_delete(fib_index,
 680                            &prefix,
 681                            FIB_SOURCE_PLUGIN_LOW);
 682 }
 683
 684 int snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 685                        u8 twice_nat)
 686 {
 687   snat_address_t * ap;
 688   snat_interface_t *i;
 689   vlib_thread_main_t *tm = vlib_get_thread_main ();
 690
 691   if (twice_nat && !sm->endpoint_dependent)
 692     return VNET_API_ERROR_FEATURE_DISABLED;
 693
 694   /* Check if address already exists */
 695   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 696     {
 697       if (ap->addr.as_u32 == addr->as_u32)
 698         return VNET_API_ERROR_VALUE_EXIST;
 699     }
 700
 701   if (twice_nat)
 702     vec_add2 (sm->twice_nat_addresses, ap, 1);
 703   else
 704     vec_add2 (sm->addresses, ap, 1);
 705
 706   ap->addr = *addr;
 707   if (vrf_id != ~0)
 708     ap->fib_index =
 709       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 710                                          FIB_SOURCE_PLUGIN_LOW);
 711   else
 712     ap->fib_index = ~0;
 713 #define _(N, i, n, s) \
 714   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 715   ap->busy_##n##_ports = 0; \
 716   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 717   foreach_snat_protocol
 718 #undef _
 719
 720   if (twice_nat)
 721     return 0;
 722
 723   /* Add external address to FIB */
 724   pool_foreach (i, sm->interfaces,
 725   ({
 726     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 727       continue;
 728
 729     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 730     break;
 731   }));
 732   pool_foreach (i, sm->output_feature_interfaces,
 733   ({
 734     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 735       continue;
 736
 737     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 738     break;
 739   }));
 740
 741   return 0;
 742 }
 743
 744 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 745                                                    ip4_address_t addr)
 746 {
 747   snat_static_mapping_t *m;
 748   pool_foreach (m, sm->static_mappings,
 749   ({
 750       if (m->external_addr.as_u32 == addr.as_u32)
 751         return 1;
 752   }));
 753
 754   return 0;
 755 }
 756
 757 void increment_v4_address (ip4_address_t * a)
 758 {
 759   u32 v;
 760
 761   v = clib_net_to_host_u32(a->as_u32) + 1;
 762   a->as_u32 = clib_host_to_net_u32(v);
 763 }
 764
 765 static void
 766 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 767                                        ip4_address_t l_addr,
 768                                        u16 l_port,
 769                                        u32 sw_if_index,
 770                                        u16 e_port,
 771                                        u32 vrf_id,
 772                                        snat_protocol_t proto,
 773                                        int addr_only,
 774                                        int is_add,
 775                                        u8 * tag)
 776 {
 777   snat_static_map_resolve_t *rp;
 778
 779   vec_add2 (sm->to_resolve, rp, 1);
 780   rp->l_addr.as_u32 = l_addr.as_u32;
 781   rp->l_port = l_port;
 782   rp->sw_if_index = sw_if_index;
 783   rp->e_port = e_port;
 784   rp->vrf_id = vrf_id;
 785   rp->proto = proto;
 786   rp->addr_only = addr_only;
 787   rp->is_add = is_add;
 788   rp->tag = vec_dup (tag);
 789 }
 790
 791 /**
 792  * @brief Add static mapping.
 793  *
 794  * Create static mapping between local addr+port and external addr+port.
 795  *
 796  * @param l_addr Local IPv4 address.
 797  * @param e_addr External IPv4 address.
 798  * @param l_port Local port number.
 799  * @param e_port External port number.
 800  * @param vrf_id VRF ID.
 801  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 802  * @param sw_if_index External port instead of specific IP address.
 803  * @param is_add If 0 delete static mapping, otherwise add.
 804  * @param twice_nat If value is TWICE_NAT then translate external host address
 805  *                  and port.
 806  *                  If value is TWICE_NAT_SELF then translate external host
 807  *                  address and port whenever external host address equals
 808  *                  local address of internal host.
 809  * @param out2in_only If 1 rule match only out2in direction
 810  * @param tag - opaque string tag
 811  *
 812  * @returns
 813  */
 814 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 815                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 816                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 817                             twice_nat_type_t twice_nat, u8 out2in_only,
 818                             u8 * tag)
 819 {
 820   snat_main_t * sm = &snat_main;
 821   snat_static_mapping_t *m;
 822   snat_session_key_t m_key;
 823   clib_bihash_kv_8_8_t kv, value;
 824   snat_address_t *a = 0;
 825   u32 fib_index = ~0;
 826   uword * p;
 827   snat_interface_t *interface;
 828   int i;
 829   snat_main_per_thread_data_t *tsm;
 830   snat_user_key_t u_key;
 831   snat_user_t *u;
 832   dlist_elt_t * head, * elt;
 833   u32 elt_index, head_index;
 834   u32 ses_index;
 835   u64 user_index;
 836   snat_session_t * s;
 837   snat_static_map_resolve_t *rp, *rp_match = 0;
 838
 839   if (!sm->endpoint_dependent)
 840     {
 841       if (twice_nat || out2in_only)
 842         return VNET_API_ERROR_FEATURE_DISABLED;
 843     }
 844
 845   /* If the external address is a specific interface address */
 846   if (sw_if_index != ~0)
 847     {
 848       ip4_address_t * first_int_addr;
 849
 850       for (i = 0; i < vec_len (sm->to_resolve); i++)
 851         {
 852           rp = sm->to_resolve + i;
 853           if (rp->sw_if_index != sw_if_index ||
 854               rp->l_addr.as_u32 != l_addr.as_u32 ||
 855               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 856             continue;
 857
 858           if (!addr_only)
 859             {
 860               if (rp->l_port != l_port || rp->e_port != e_port || rp->proto != proto)
 861                 continue;
 862             }
 863
 864           rp_match = rp;
 865           break;
 866         }
 867
 868       /* Might be already set... */
 869       first_int_addr = ip4_interface_first_address
 870         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 871
 872       if (is_add)
 873         {
 874           if (rp_match)
 875             return VNET_API_ERROR_VALUE_EXIST;
 876
 877           snat_add_static_mapping_when_resolved
 878             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 879              addr_only,  is_add, tag);
 880
 881           /* DHCP resolution required? */
 882           if (first_int_addr == 0)
 883             {
 884               return 0;
 885             }
 886           else
 887             {
 888               e_addr.as_u32 = first_int_addr->as_u32;
 889               /* Identity mapping? */
 890               if (l_addr.as_u32 == 0)
 891                 l_addr.as_u32 = e_addr.as_u32;
 892             }
 893         }
 894       else
 895         {
 896           if (!rp_match)
 897             return VNET_API_ERROR_NO_SUCH_ENTRY;
 898
 899           vec_del1 (sm->to_resolve, i);
 900
 901           if (first_int_addr)
 902             {
 903               e_addr.as_u32 = first_int_addr->as_u32;
 904               /* Identity mapping? */
 905               if (l_addr.as_u32 == 0)
 906                 l_addr.as_u32 = e_addr.as_u32;
 907             }
 908           else
 909             return 0;
 910         }
 911     }
 912
 913   m_key.addr = e_addr;
 914   m_key.port = addr_only ? 0 : e_port;
 915   m_key.protocol = addr_only ? 0 : proto;
 916   m_key.fib_index = 0;
 917   kv.key = m_key.as_u64;
 918   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 919     m = 0;
 920   else
 921     m = pool_elt_at_index (sm->static_mappings, value.value);
 922
 923   if (is_add)
 924     {
 925       if (m)
 926         return VNET_API_ERROR_VALUE_EXIST;
 927
 928       if (twice_nat && addr_only)
 929         return VNET_API_ERROR_UNSUPPORTED;
 930
 931       /* Convert VRF id to FIB index */
 932       if (vrf_id != ~0)
 933         {
 934           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 935           if (!p)
 936             return VNET_API_ERROR_NO_SUCH_FIB;
 937           fib_index = p[0];
 938         }
 939       /* If not specified use inside VRF id from SNAT plugin startup config */
 940       else
 941         {
 942           fib_index = sm->inside_fib_index;
 943           vrf_id = sm->inside_vrf_id;
 944         }
 945
 946       if (!out2in_only)
 947         {
 948           m_key.addr = l_addr;
 949           m_key.port = addr_only ? 0 : l_port;
 950           m_key.protocol = addr_only ? 0 : proto;
 951           m_key.fib_index = fib_index;
 952           kv.key = m_key.as_u64;
 953           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 954             return VNET_API_ERROR_VALUE_EXIST;
 955         }
 956
 957       /* Find external address in allocated addresses and reserve port for
 958          address and port pair mapping when dynamic translations enabled */
 959       if (!(addr_only || sm->static_mapping_only || out2in_only))
 960         {
 961           for (i = 0; i < vec_len (sm->addresses); i++)
 962             {
 963               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 964                 {
 965                   a = sm->addresses + i;
 966                   /* External port must be unused */
 967                   switch (proto)
 968                     {
 969 #define _(N, j, n, s) \
 970                     case SNAT_PROTOCOL_##N: \
 971                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 972                         return VNET_API_ERROR_INVALID_VALUE; \
 973                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 974                       if (e_port > 1024) \
 975                         { \
 976                           a->busy_##n##_ports++; \
 977                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
 978                         } \
 979                       break;
 980                       foreach_snat_protocol
 981 #undef _
 982                     default:
 983                       nat_log_info ("unknown protocol");
 984                       return VNET_API_ERROR_INVALID_VALUE_2;
 985                     }
 986                   break;
 987                 }
 988             }
 989           /* External address must be allocated */
 990           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 991             {
 992               if (sw_if_index != ~0)
 993                 {
 994                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 995                     {
 996                       rp = sm->to_resolve + i;
 997                       if (rp->addr_only)
 998                          continue;
 999                       if (rp->sw_if_index != sw_if_index &&
1000                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1001                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1002                           rp->e_port != e_port && rp->proto != proto)
1003                         continue;
1004
1005                       vec_del1 (sm->to_resolve, i);
1006                       break;
1007                     }
1008                 }
1009               return VNET_API_ERROR_NO_SUCH_ENTRY;
1010             }
1011         }
1012
1013       pool_get (sm->static_mappings, m);
1014       memset (m, 0, sizeof (*m));
1015       m->tag = vec_dup (tag);
1016       m->local_addr = l_addr;
1017       m->external_addr = e_addr;
1018       m->addr_only = addr_only;
1019       m->vrf_id = vrf_id;
1020       m->fib_index = fib_index;
1021       m->twice_nat = twice_nat;
1022       m->out2in_only = out2in_only;
1023       if (!addr_only)
1024         {
1025           m->local_port = l_port;
1026           m->external_port = e_port;
1027           m->proto = proto;
1028         }
1029
1030       if (sm->num_workers > 1)
1031         {
1032           ip4_header_t ip = {
1033             .src_address = m->local_addr,
1034           };
1035           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
1036           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1037         }
1038       else
1039         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1040
1041       m_key.addr = m->local_addr;
1042       m_key.port = m->local_port;
1043       m_key.protocol = m->proto;
1044       m_key.fib_index = m->fib_index;
1045       kv.key = m_key.as_u64;
1046       kv.value = m - sm->static_mappings;
1047       if (!out2in_only)
1048         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1049
1050       m_key.addr = m->external_addr;
1051       m_key.port = m->external_port;
1052       m_key.fib_index = 0;
1053       kv.key = m_key.as_u64;
1054       kv.value = m - sm->static_mappings;
1055       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
1056
1057       /* Delete dynamic sessions matching local address (+ local port) */
1058       if (!(sm->static_mapping_only))
1059         {
1060           u_key.addr = m->local_addr;
1061           u_key.fib_index = m->fib_index;
1062           kv.key = u_key.as_u64;
1063           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1064             {
1065               user_index = value.value;
1066               u = pool_elt_at_index (tsm->users, user_index);
1067               if (u->nsessions)
1068                 {
1069                   head_index = u->sessions_per_user_list_head_index;
1070                   head = pool_elt_at_index (tsm->list_pool, head_index);
1071                   elt_index = head->next;
1072                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1073                   ses_index = elt->value;
1074                   while (ses_index != ~0)
1075                     {
1076                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1077                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1078                       ses_index = elt->value;
1079
1080                       if (snat_is_session_static (s))
1081                         continue;
1082
1083                       if (!addr_only && (clib_net_to_host_u16 (s->in2out.port) != m->local_port))
1084                         continue;
1085
1086                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1087                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1088
1089                       if (!addr_only)
1090                         break;
1091                     }
1092                 }
1093             }
1094         }
1095     }
1096   else
1097     {
1098       if (!m)
1099         {
1100           if (sw_if_index != ~0)
1101             return 0;
1102           else
1103             return VNET_API_ERROR_NO_SUCH_ENTRY;
1104         }
1105
1106       /* Free external address port */
1107       if (!(addr_only || sm->static_mapping_only || out2in_only))
1108         {
1109           for (i = 0; i < vec_len (sm->addresses); i++)
1110             {
1111               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1112                 {
1113                   a = sm->addresses + i;
1114                   switch (proto)
1115                     {
1116 #define _(N, j, n, s) \
1117                     case SNAT_PROTOCOL_##N: \
1118                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1119                       if (e_port > 1024) \
1120                         { \
1121                           a->busy_##n##_ports--; \
1122                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1123                         } \
1124                       break;
1125                       foreach_snat_protocol
1126 #undef _
1127                     default:
1128                       nat_log_info ("unknown protocol");
1129                       return VNET_API_ERROR_INVALID_VALUE_2;
1130                     }
1131                   break;
1132                 }
1133             }
1134         }
1135
1136       if (sm->num_workers > 1)
1137         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1138       else
1139         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1140
1141       m_key.addr = m->local_addr;
1142       m_key.port = m->local_port;
1143       m_key.protocol = m->proto;
1144       m_key.fib_index = m->fib_index;
1145       kv.key = m_key.as_u64;
1146       if (!out2in_only)
1147         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1148
1149       m_key.addr = m->external_addr;
1150       m_key.port = m->external_port;
1151       m_key.fib_index = 0;
1152       kv.key = m_key.as_u64;
1153       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1154
1155       /* Delete session(s) for static mapping if exist */
1156       if (!(sm->static_mapping_only) ||
1157           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1158         {
1159           u_key.addr = m->local_addr;
1160           u_key.fib_index = m->fib_index;
1161           kv.key = u_key.as_u64;
1162           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1163             {
1164               user_index = value.value;
1165               u = pool_elt_at_index (tsm->users, user_index);
1166               if (u->nstaticsessions)
1167                 {
1168                   head_index = u->sessions_per_user_list_head_index;
1169                   head = pool_elt_at_index (tsm->list_pool, head_index);
1170                   elt_index = head->next;
1171                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1172                   ses_index = elt->value;
1173                   while (ses_index != ~0)
1174                     {
1175                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1176                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1177                       ses_index = elt->value;
1178
1179                       if (!addr_only)
1180                         {
1181                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
1182                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1183                             continue;
1184                         }
1185
1186                       if (is_lb_session (s))
1187                         continue;
1188
1189                       if (!snat_is_session_static (s))
1190                         continue;
1191
1192                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1193                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1194
1195                       if (!addr_only)
1196                         break;
1197                     }
1198                   if (addr_only && (u->nstaticsessions == 0) && (u->nsessions == 0))
1199                     {
1200                       pool_put (tsm->users, u);
1201                       clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 0);
1202                     }
1203                 }
1204             }
1205         }
1206
1207       vec_free (m->tag);
1208       vec_free (m->workers);
1209       /* Delete static mapping from pool */
1210       pool_put (sm->static_mappings, m);
1211     }
1212
1213   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1214     return 0;
1215
1216   /* Add/delete external address to FIB */
1217   pool_foreach (interface, sm->interfaces,
1218   ({
1219     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1220       continue;
1221
1222     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1223     break;
1224   }));
1225   pool_foreach (interface, sm->output_feature_interfaces,
1226   ({
1227     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1228       continue;
1229
1230     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1231     break;
1232   }));
1233
1234   return 0;
1235 }
1236
1237 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1238                                      snat_protocol_t proto,
1239                                      nat44_lb_addr_port_t *locals, u8 is_add,
1240                                      twice_nat_type_t twice_nat, u8 out2in_only,
1241                                      u8 *tag)
1242 {
1243   snat_main_t * sm = &snat_main;
1244   snat_static_mapping_t *m;
1245   snat_session_key_t m_key;
1246   clib_bihash_kv_8_8_t kv, value;
1247   snat_address_t *a = 0;
1248   int i;
1249   nat44_lb_addr_port_t *local;
1250   u32 elt_index, head_index, ses_index;
1251   snat_main_per_thread_data_t *tsm;
1252   snat_user_key_t u_key;
1253   snat_user_t *u;
1254   snat_session_t * s;
1255   dlist_elt_t * head, * elt;
1256   uword *bitmap = 0;
1257
1258   if (!sm->endpoint_dependent)
1259     return VNET_API_ERROR_FEATURE_DISABLED;
1260
1261   m_key.addr = e_addr;
1262   m_key.port = e_port;
1263   m_key.protocol = proto;
1264   m_key.fib_index = 0;
1265   kv.key = m_key.as_u64;
1266   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1267     m = 0;
1268   else
1269     m = pool_elt_at_index (sm->static_mappings, value.value);
1270
1271   if (is_add)
1272     {
1273       if (m)
1274         return VNET_API_ERROR_VALUE_EXIST;
1275
1276       if (vec_len (locals) < 2)
1277         return VNET_API_ERROR_INVALID_VALUE;
1278
1279       /* Find external address in allocated addresses and reserve port for
1280          address and port pair mapping when dynamic translations enabled */
1281       if (!(sm->static_mapping_only || out2in_only))
1282         {
1283           for (i = 0; i < vec_len (sm->addresses); i++)
1284             {
1285               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1286                 {
1287                   a = sm->addresses + i;
1288                   /* External port must be unused */
1289                   switch (proto)
1290                     {
1291 #define _(N, j, n, s) \
1292                     case SNAT_PROTOCOL_##N: \
1293                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1294                         return VNET_API_ERROR_INVALID_VALUE; \
1295                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1296                       if (e_port > 1024) \
1297                         { \
1298                           a->busy_##n##_ports++; \
1299                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1300                         } \
1301                       break;
1302                       foreach_snat_protocol
1303 #undef _
1304                     default:
1305                       nat_log_info ("unknown protocol");
1306                       return VNET_API_ERROR_INVALID_VALUE_2;
1307                     }
1308                   break;
1309                 }
1310             }
1311           /* External address must be allocated */
1312           if (!a)
1313             return VNET_API_ERROR_NO_SUCH_ENTRY;
1314         }
1315
1316       pool_get (sm->static_mappings, m);
1317       memset (m, 0, sizeof (*m));
1318       m->tag = vec_dup (tag);
1319       m->external_addr = e_addr;
1320       m->addr_only = 0;
1321       m->external_port = e_port;
1322       m->proto = proto;
1323       m->twice_nat = twice_nat;
1324       m->out2in_only = out2in_only;
1325
1326       m_key.addr = m->external_addr;
1327       m_key.port = m->external_port;
1328       m_key.protocol = m->proto;
1329       m_key.fib_index = 0;
1330       kv.key = m_key.as_u64;
1331       kv.value = m - sm->static_mappings;
1332       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1333         {
1334           nat_log_err ("static_mapping_by_external key add failed");
1335           return VNET_API_ERROR_UNSPECIFIED;
1336         }
1337
1338       m_key.fib_index = m->fib_index;
1339       for (i = 0; i < vec_len (locals); i++)
1340         {
1341           locals[i].fib_index = fib_table_find_or_create_and_lock (
1342             FIB_PROTOCOL_IP4, locals[i].vrf_id, FIB_SOURCE_PLUGIN_LOW);
1343           m_key.addr = locals[i].addr;
1344           m_key.fib_index = locals[i].fib_index;
1345           if (!out2in_only)
1346             {
1347               m_key.port = locals[i].port;
1348               kv.key = m_key.as_u64;
1349               kv.value = m - sm->static_mappings;
1350               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1351             }
1352           locals[i].prefix = (i == 0) ? locals[i].probability :\
1353             (locals[i - 1].prefix + locals[i].probability);
1354           vec_add1 (m->locals, locals[i]);
1355           if (sm->num_workers > 1)
1356             {
1357               ip4_header_t ip = {
1358                 .src_address = locals[i].addr,
1359               };
1360               bitmap = clib_bitmap_set (
1361                 bitmap, sm->worker_in2out_cb (&ip, m->fib_index), 1);
1362             }
1363         }
1364
1365       /* Assign workers */
1366       if (sm->num_workers > 1)
1367         {
1368           clib_bitmap_foreach (i, bitmap,
1369             ({
1370                vec_add1(m->workers, i);
1371             }));
1372         }
1373     }
1374   else
1375     {
1376       if (!m)
1377         return VNET_API_ERROR_NO_SUCH_ENTRY;
1378
1379       /* Free external address port */
1380       if (!(sm->static_mapping_only || out2in_only))
1381         {
1382           for (i = 0; i < vec_len (sm->addresses); i++)
1383             {
1384               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1385                 {
1386                   a = sm->addresses + i;
1387                   switch (proto)
1388                     {
1389 #define _(N, j, n, s) \
1390                     case SNAT_PROTOCOL_##N: \
1391                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1392                       if (e_port > 1024) \
1393                         { \
1394                           a->busy_##n##_ports--; \
1395                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1396                         } \
1397                       break;
1398                       foreach_snat_protocol
1399 #undef _
1400                     default:
1401                       nat_log_info ("unknown protocol");
1402                       return VNET_API_ERROR_INVALID_VALUE_2;
1403                     }
1404                   break;
1405                 }
1406             }
1407         }
1408
1409       m_key.addr = m->external_addr;
1410       m_key.port = m->external_port;
1411       m_key.protocol = m->proto;
1412       m_key.fib_index = 0;
1413       kv.key = m_key.as_u64;
1414       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1415         {
1416           nat_log_err ("static_mapping_by_external key del failed");
1417           return VNET_API_ERROR_UNSPECIFIED;
1418         }
1419
1420       vec_foreach (local, m->locals)
1421         {
1422           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1423                             FIB_SOURCE_PLUGIN_LOW);
1424           m_key.addr = local->addr;
1425           if (!out2in_only)
1426             {
1427               m_key.port = local->port;
1428               m_key.fib_index = local->fib_index;
1429               kv.key = m_key.as_u64;
1430               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1431                 {
1432                   nat_log_err ("static_mapping_by_local key del failed");
1433                   return VNET_API_ERROR_UNSPECIFIED;
1434                 }
1435             }
1436
1437           if (sm->num_workers > 1)
1438             {
1439               ip4_header_t ip = {
1440                 .src_address = local->addr,
1441               };
1442               tsm = vec_elt_at_index (sm->per_thread_data,
1443                                       sm->worker_in2out_cb (&ip, m->fib_index));
1444             }
1445           else
1446             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1447
1448           /* Delete sessions */
1449           u_key.addr = local->addr;
1450           u_key.fib_index = m->fib_index;
1451           kv.key = u_key.as_u64;
1452           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1453             {
1454               u = pool_elt_at_index (tsm->users, value.value);
1455               if (u->nstaticsessions)
1456                 {
1457                   head_index = u->sessions_per_user_list_head_index;
1458                   head = pool_elt_at_index (tsm->list_pool, head_index);
1459                   elt_index = head->next;
1460                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1461                   ses_index = elt->value;
1462                   while (ses_index != ~0)
1463                     {
1464                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1465                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1466                       ses_index = elt->value;
1467
1468                       if (!(is_lb_session (s)))
1469                         continue;
1470
1471                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) &&
1472                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1473                         continue;
1474
1475                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1476                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1477                     }
1478                 }
1479             }
1480         }
1481       vec_free(m->locals);
1482       vec_free(m->tag);
1483       vec_free(m->workers);
1484
1485       pool_put (sm->static_mappings, m);
1486     }
1487
1488   return 0;
1489 }
1490
1491 int
1492 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1493                   u8 twice_nat)
1494 {
1495   snat_address_t *a = 0;
1496   snat_session_t *ses;
1497   u32 *ses_to_be_removed = 0, *ses_index;
1498   snat_main_per_thread_data_t *tsm;
1499   snat_static_mapping_t *m;
1500   snat_interface_t *interface;
1501   int i;
1502   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1503
1504   /* Find SNAT address */
1505   for (i=0; i < vec_len (addresses); i++)
1506     {
1507       if (addresses[i].addr.as_u32 == addr.as_u32)
1508         {
1509           a = addresses + i;
1510           break;
1511         }
1512     }
1513   if (!a)
1514     return VNET_API_ERROR_NO_SUCH_ENTRY;
1515
1516   if (delete_sm)
1517     {
1518       pool_foreach (m, sm->static_mappings,
1519       ({
1520           if (m->external_addr.as_u32 == addr.as_u32)
1521             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1522                                             m->local_port, m->external_port,
1523                                             m->vrf_id, m->addr_only, ~0,
1524                                             m->proto, 0, m->twice_nat,
1525                                             m->out2in_only, m->tag);
1526       }));
1527     }
1528   else
1529     {
1530       /* Check if address is used in some static mapping */
1531       if (is_snat_address_used_in_static_mapping(sm, addr))
1532         {
1533           nat_log_notice ("address used in static mapping");
1534           return VNET_API_ERROR_UNSPECIFIED;
1535         }
1536     }
1537
1538   if (a->fib_index != ~0)
1539     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1540                      FIB_SOURCE_PLUGIN_LOW);
1541
1542   /* Delete sessions using address */
1543   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1544     {
1545       vec_foreach (tsm, sm->per_thread_data)
1546         {
1547           pool_foreach (ses, tsm->sessions, ({
1548             if (ses->out2in.addr.as_u32 == addr.as_u32)
1549               {
1550                 ses->outside_address_index = ~0;
1551                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1552                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1553               }
1554           }));
1555
1556           vec_foreach (ses_index, ses_to_be_removed)
1557             {
1558               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1559               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1560             }
1561
1562           vec_free (ses_to_be_removed);
1563        }
1564     }
1565
1566   if (twice_nat)
1567     {
1568       vec_del1 (sm->twice_nat_addresses, i);
1569       return 0;
1570     }
1571   else
1572     vec_del1 (sm->addresses, i);
1573
1574   /* Delete external address from FIB */
1575   pool_foreach (interface, sm->interfaces,
1576   ({
1577     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1578       continue;
1579
1580     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1581     break;
1582   }));
1583   pool_foreach (interface, sm->output_feature_interfaces,
1584   ({
1585     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1586       continue;
1587
1588     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1589     break;
1590   }));
1591
1592   return 0;
1593 }
1594
1595 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1596 {
1597   snat_main_t *sm = &snat_main;
1598   snat_interface_t *i;
1599   const char * feature_name, *del_feature_name;
1600   snat_address_t * ap;
1601   snat_static_mapping_t * m;
1602   snat_det_map_t * dm;
1603   nat_outside_fib_t *outside_fib;
1604   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1605                                                        sw_if_index);
1606
1607   if (sm->out2in_dpo && !is_inside)
1608     return VNET_API_ERROR_UNSUPPORTED;
1609
1610   pool_foreach (i, sm->output_feature_interfaces,
1611   ({
1612     if (i->sw_if_index == sw_if_index)
1613       return VNET_API_ERROR_VALUE_EXIST;
1614   }));
1615
1616   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1617     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1618   else
1619     {
1620       if (sm->num_workers > 1 && !sm->deterministic)
1621         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1622       else if (sm->deterministic)
1623         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1624       else if (sm->endpoint_dependent)
1625         feature_name = is_inside ?  "nat44-ed-in2out" : "nat44-ed-out2in";
1626       else
1627         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1628     }
1629
1630   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1631     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1632                                                       NAT_FQ_NELTS);
1633
1634   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1635     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1636                                                       NAT_FQ_NELTS);
1637
1638   if (!is_inside)
1639     {
1640       vec_foreach (outside_fib, sm->outside_fibs)
1641         {
1642           if (outside_fib->fib_index == fib_index)
1643             {
1644               if (is_del)
1645                 {
1646                   outside_fib->refcount--;
1647                   if (!outside_fib->refcount)
1648                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1649                 }
1650               else
1651                 outside_fib->refcount++;
1652               goto feature_set;
1653             }
1654         }
1655       if (!is_del)
1656         {
1657           vec_add2 (sm->outside_fibs, outside_fib, 1);
1658           outside_fib->refcount = 1;
1659           outside_fib->fib_index = fib_index;
1660         }
1661     }
1662 feature_set:
1663   pool_foreach (i, sm->interfaces,
1664   ({
1665     if (i->sw_if_index == sw_if_index)
1666       {
1667         if (is_del)
1668           {
1669             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1670               {
1671                 if (is_inside)
1672                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1673                 else
1674                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1675
1676                 if (sm->num_workers > 1 && !sm->deterministic)
1677                   {
1678                     del_feature_name = "nat44-handoff-classify";
1679                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1680                                                  "nat44-out2in-worker-handoff";
1681                   }
1682                 else if (sm->deterministic)
1683                   {
1684                     del_feature_name = "nat44-det-classify";
1685                     feature_name = !is_inside ?  "nat44-det-in2out" :
1686                                                  "nat44-det-out2in";
1687                   }
1688                 else if (sm->endpoint_dependent)
1689                   {
1690                     del_feature_name = "nat44-ed-classify";
1691                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1692                                                  "nat44-ed-out2in";
1693                   }
1694                 else
1695                   {
1696                     del_feature_name = "nat44-classify";
1697                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1698                   }
1699
1700                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1701                                              sw_if_index, 0, 0, 0);
1702                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1703                                              sw_if_index, 1, 0, 0);
1704                 if (!is_inside)
1705                   {
1706                     if (sm->endpoint_dependent)
1707                       vnet_feature_enable_disable ("ip4-local",
1708                                                    "nat44-ed-hairpinning",
1709                                                    sw_if_index, 1, 0, 0);
1710                     else
1711                       vnet_feature_enable_disable ("ip4-local",
1712                                                    "nat44-hairpinning",
1713                                                    sw_if_index, 1, 0, 0);
1714                   }
1715               }
1716             else
1717               {
1718                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1719                                              sw_if_index, 0, 0, 0);
1720                 pool_put (sm->interfaces, i);
1721                 if (is_inside)
1722                   {
1723                     if (sm->endpoint_dependent)
1724                       vnet_feature_enable_disable ("ip4-local",
1725                                                    "nat44-ed-hairpinning",
1726                                                    sw_if_index, 0, 0, 0);
1727                     else
1728                       vnet_feature_enable_disable ("ip4-local",
1729                                                    "nat44-hairpinning",
1730                                                    sw_if_index, 0, 0, 0);
1731                   }
1732               }
1733           }
1734         else
1735           {
1736             if ((nat_interface_is_inside(i) && is_inside) ||
1737                 (nat_interface_is_outside(i) && !is_inside))
1738               return 0;
1739
1740             if (sm->num_workers > 1 && !sm->deterministic)
1741               {
1742                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1743                                                  "nat44-out2in-worker-handoff";
1744                 feature_name = "nat44-handoff-classify";
1745               }
1746             else if (sm->deterministic)
1747               {
1748                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1749                                                  "nat44-det-out2in";
1750                 feature_name = "nat44-det-classify";
1751               }
1752             else if (sm->endpoint_dependent)
1753               {
1754                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1755                                                  "nat44-ed-out2in";
1756                 feature_name = "nat44-ed-classify";
1757               }
1758             else
1759               {
1760                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1761                 feature_name = "nat44-classify";
1762               }
1763
1764             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1765                                          sw_if_index, 0, 0, 0);
1766             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1767                                          sw_if_index, 1, 0, 0);
1768             if (!is_inside)
1769               {
1770                 if (sm->endpoint_dependent)
1771                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1772                                                sw_if_index, 0, 0, 0);
1773                 else
1774                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1775                                                sw_if_index, 0, 0, 0);
1776               }
1777             goto set_flags;
1778           }
1779
1780         goto fib;
1781       }
1782   }));
1783
1784   if (is_del)
1785     return VNET_API_ERROR_NO_SUCH_ENTRY;
1786
1787   pool_get (sm->interfaces, i);
1788   i->sw_if_index = sw_if_index;
1789   i->flags = 0;
1790   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1791
1792   if (is_inside && !sm->out2in_dpo)
1793     {
1794       if (sm->endpoint_dependent)
1795         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1796                                      sw_if_index, 1, 0, 0);
1797       else
1798         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1799                                      sw_if_index, 1, 0, 0);
1800     }
1801
1802 set_flags:
1803   if (is_inside)
1804     {
1805       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1806       return 0;
1807     }
1808   else
1809     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1810
1811   /* Add/delete external addresses to FIB */
1812 fib:
1813   vec_foreach (ap, sm->addresses)
1814     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1815
1816   pool_foreach (m, sm->static_mappings,
1817   ({
1818     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1819       continue;
1820
1821     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1822   }));
1823
1824   pool_foreach (dm, sm->det_maps,
1825   ({
1826     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1827   }));
1828
1829   return 0;
1830 }
1831
1832 int snat_interface_add_del_output_feature (u32 sw_if_index,
1833                                            u8 is_inside,
1834                                            int is_del)
1835 {
1836   snat_main_t *sm = &snat_main;
1837   snat_interface_t *i;
1838   snat_address_t * ap;
1839   snat_static_mapping_t * m;
1840
1841   if (sm->deterministic ||
1842       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1843     return VNET_API_ERROR_UNSUPPORTED;
1844
1845   pool_foreach (i, sm->interfaces,
1846   ({
1847     if (i->sw_if_index == sw_if_index)
1848       return VNET_API_ERROR_VALUE_EXIST;
1849   }));
1850
1851   if (is_inside)
1852     {
1853       if (sm->endpoint_dependent)
1854         {
1855           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1856                                        sw_if_index, !is_del, 0, 0);
1857           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1858                                        sw_if_index, !is_del, 0, 0);
1859         }
1860       else
1861         {
1862           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1863                                        sw_if_index, !is_del, 0, 0);
1864           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1865                                        sw_if_index, !is_del, 0, 0);
1866         }
1867       goto fq;
1868     }
1869
1870   if (sm->num_workers > 1)
1871     {
1872       vnet_feature_enable_disable ("ip4-unicast",
1873                                    "nat44-out2in-worker-handoff",
1874                                    sw_if_index, !is_del, 0, 0);
1875       vnet_feature_enable_disable ("ip4-output",
1876                                    "nat44-in2out-output-worker-handoff",
1877                                    sw_if_index, !is_del, 0, 0);
1878     }
1879   else
1880     {
1881       if (sm->endpoint_dependent)
1882         {
1883           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
1884                                        sw_if_index, !is_del, 0, 0);
1885           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
1886                                        sw_if_index, !is_del, 0, 0);
1887         }
1888       else
1889         {
1890           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
1891                                        sw_if_index, !is_del, 0, 0);
1892           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1893                                        sw_if_index, !is_del, 0, 0);
1894         }
1895     }
1896
1897 fq:
1898   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1899     sm->fq_in2out_output_index =
1900       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1901
1902   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1903     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1904
1905   pool_foreach (i, sm->output_feature_interfaces,
1906   ({
1907     if (i->sw_if_index == sw_if_index)
1908       {
1909         if (is_del)
1910           pool_put (sm->output_feature_interfaces, i);
1911         else
1912           return VNET_API_ERROR_VALUE_EXIST;
1913
1914         goto fib;
1915       }
1916   }));
1917
1918   if (is_del)
1919     return VNET_API_ERROR_NO_SUCH_ENTRY;
1920
1921   pool_get (sm->output_feature_interfaces, i);
1922   i->sw_if_index = sw_if_index;
1923   i->flags = 0;
1924   if (is_inside)
1925     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1926   else
1927     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1928
1929   /* Add/delete external addresses to FIB */
1930 fib:
1931   if (is_inside)
1932     return 0;
1933
1934   vec_foreach (ap, sm->addresses)
1935     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1936
1937   pool_foreach (m, sm->static_mappings,
1938   ({
1939     if (!(m->addr_only)  || (m->local_addr.as_u32 == m->external_addr.as_u32))
1940       continue;
1941
1942     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1943   }));
1944
1945   return 0;
1946 }
1947
1948 int snat_set_workers (uword * bitmap)
1949 {
1950   snat_main_t *sm = &snat_main;
1951   int i, j = 0;
1952
1953   if (sm->num_workers < 2)
1954     return VNET_API_ERROR_FEATURE_DISABLED;
1955
1956   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1957     return VNET_API_ERROR_INVALID_WORKER;
1958
1959   vec_free (sm->workers);
1960   clib_bitmap_foreach (i, bitmap,
1961     ({
1962       vec_add1(sm->workers, i);
1963       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1964       j++;
1965     }));
1966
1967   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
1968   sm->num_snat_thread = _vec_len (sm->workers);
1969
1970   return 0;
1971 }
1972
1973
1974 static void
1975 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
1976                                        uword opaque,
1977                                        u32 sw_if_index,
1978                                        ip4_address_t * address,
1979                                        u32 address_length,
1980                                        u32 if_address_index,
1981                                        u32 is_delete);
1982
1983 static void
1984 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
1985                                  uword opaque,
1986                                  u32 sw_if_index,
1987                                  ip4_address_t * address,
1988                                  u32 address_length,
1989                                  u32 if_address_index,
1990                                  u32 is_delete);
1991
1992 static int
1993 nat_alloc_addr_and_port_default (snat_address_t * addresses,
1994                                  u32 fib_index,
1995                                  u32 thread_index,
1996                                  snat_session_key_t * k,
1997                                  u32 * address_indexp,
1998                                  u16 port_per_thread,
1999                                  u32 snat_thread_index);
2000
2001 static clib_error_t * snat_init (vlib_main_t * vm)
2002 {
2003   snat_main_t * sm = &snat_main;
2004   clib_error_t * error = 0;
2005   ip4_main_t * im = &ip4_main;
2006   ip_lookup_main_t * lm = &im->lookup_main;
2007   uword *p;
2008   vlib_thread_registration_t *tr;
2009   vlib_thread_main_t *tm = vlib_get_thread_main ();
2010   uword *bitmap = 0;
2011   u32 i;
2012   ip4_add_del_interface_address_callback_t cb4;
2013   vlib_node_t * error_drop_node;
2014
2015   sm->vlib_main = vm;
2016   sm->vnet_main = vnet_get_main();
2017   sm->ip4_main = im;
2018   sm->ip4_lookup_main = lm;
2019   sm->api_main = &api_main;
2020   sm->first_worker_index = 0;
2021   sm->num_workers = 0;
2022   sm->num_snat_thread = 1;
2023   sm->workers = 0;
2024   sm->port_per_thread = 0xffff - 1024;
2025   sm->fq_in2out_index = ~0;
2026   sm->fq_out2in_index = ~0;
2027   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2028   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2029   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2030   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2031   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2032   sm->forwarding_enabled = 0;
2033   sm->log_class = vlib_log_register_class ("nat", 0);
2034   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2035   sm->error_node_index = error_drop_node->index;
2036
2037   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2038   if (p)
2039     {
2040       tr = (vlib_thread_registration_t *) p[0];
2041       if (tr)
2042         {
2043           sm->num_workers = tr->count;
2044           sm->first_worker_index = tr->first_index;
2045         }
2046     }
2047
2048   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2049
2050   /* Use all available workers by default */
2051   if (sm->num_workers > 1)
2052     {
2053       for (i=0; i < sm->num_workers; i++)
2054         bitmap = clib_bitmap_set (bitmap, i, 1);
2055       snat_set_workers(bitmap);
2056       clib_bitmap_free (bitmap);
2057     }
2058   else
2059     {
2060       sm->per_thread_data[0].snat_thread_index = 0;
2061     }
2062
2063   error = snat_api_init(vm, sm);
2064   if (error)
2065     return error;
2066
2067   /* Set up the interface address add/del callback */
2068   cb4.function = snat_ip4_add_del_interface_address_cb;
2069   cb4.function_opaque = 0;
2070
2071   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2072
2073   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2074   cb4.function_opaque = 0;
2075
2076   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2077
2078   nat_dpo_module_init ();
2079
2080   /* Init IPFIX logging */
2081   snat_ipfix_logging_init(vm);
2082
2083   /* Init NAT64 */
2084   error = nat64_init(vm);
2085   if (error)
2086     return error;
2087
2088   dslite_init(vm);
2089
2090   nat66_init();
2091
2092   /* Init virtual fragmenentation reassembly */
2093   return nat_reass_init(vm);
2094 }
2095
2096 VLIB_INIT_FUNCTION (snat_init);
2097
2098 void snat_free_outside_address_and_port (snat_address_t * addresses,
2099                                          u32 thread_index,
2100                                          snat_session_key_t * k,
2101                                          u32 address_index)
2102 {
2103   snat_address_t *a;
2104   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2105
2106   ASSERT (address_index < vec_len (addresses));
2107
2108   a = addresses + address_index;
2109
2110   switch (k->protocol)
2111     {
2112 #define _(N, i, n, s) \
2113     case SNAT_PROTOCOL_##N: \
2114       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2115         port_host_byte_order) == 1); \
2116       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2117         port_host_byte_order, 0); \
2118       a->busy_##n##_ports--; \
2119       a->busy_##n##_ports_per_thread[thread_index]--; \
2120       break;
2121       foreach_snat_protocol
2122 #undef _
2123     default:
2124       nat_log_info ("unknown protocol");
2125       return;
2126     }
2127 }
2128
2129 /**
2130  * @brief Match NAT44 static mapping.
2131  *
2132  * @param sm          NAT main.
2133  * @param match       Address and port to match.
2134  * @param mapping     External or local address and port of the matched mapping.
2135  * @param by_external If 0 match by local address otherwise match by external
2136  *                    address.
2137  * @param is_addr_only If matched mapping is address only
2138  * @param twice_nat If matched mapping is twice NAT.
2139  * @param lb If matched mapping is load-balanced.
2140  *
2141  * @returns 0 if match found otherwise 1.
2142  */
2143 int snat_static_mapping_match (snat_main_t * sm,
2144                                snat_session_key_t match,
2145                                snat_session_key_t * mapping,
2146                                u8 by_external,
2147                                u8 *is_addr_only,
2148                                twice_nat_type_t *twice_nat,
2149                                u8 *lb)
2150 {
2151   clib_bihash_kv_8_8_t kv, value;
2152   snat_static_mapping_t *m;
2153   snat_session_key_t m_key;
2154   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2155   u32 rand, lo = 0, hi, mid;
2156
2157   m_key.fib_index = match.fib_index;
2158   if (by_external)
2159     {
2160       mapping_hash = &sm->static_mapping_by_external;
2161       m_key.fib_index = 0;
2162     }
2163
2164   m_key.addr = match.addr;
2165   m_key.port = clib_net_to_host_u16 (match.port);
2166   m_key.protocol = match.protocol;
2167
2168   kv.key = m_key.as_u64;
2169
2170   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2171     {
2172       /* Try address only mapping */
2173       m_key.port = 0;
2174       m_key.protocol = 0;
2175       kv.key = m_key.as_u64;
2176       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2177         return 1;
2178     }
2179
2180   m = pool_elt_at_index (sm->static_mappings, value.value);
2181
2182   if (by_external)
2183     {
2184       if (vec_len (m->locals))
2185         {
2186 get_local:
2187           hi = vec_len (m->locals) - 1;
2188           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2189           while (lo < hi)
2190             {
2191               mid = ((hi - lo) >> 1) + lo;
2192               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2193             }
2194           if (!(m->locals[lo].prefix >= rand))
2195             return 1;
2196           if (PREDICT_FALSE (sm->num_workers > 1))
2197             {
2198               ip4_header_t ip = {
2199                 .src_address = m->locals[lo].addr,
2200               };
2201               if (sm->worker_in2out_cb (&ip, m->fib_index) != vlib_get_thread_index ())
2202                 goto get_local;
2203             }
2204           mapping->addr = m->locals[lo].addr;
2205           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2206           mapping->fib_index = m->locals[lo].fib_index;
2207         }
2208       else
2209         {
2210           mapping->fib_index = m->fib_index;
2211           mapping->addr = m->local_addr;
2212           /* Address only mapping doesn't change port */
2213           mapping->port = m->addr_only ? match.port
2214             : clib_host_to_net_u16 (m->local_port);
2215         }
2216       mapping->protocol = m->proto;
2217     }
2218   else
2219     {
2220       mapping->addr = m->external_addr;
2221       /* Address only mapping doesn't change port */
2222       mapping->port = m->addr_only ? match.port
2223         : clib_host_to_net_u16 (m->external_port);
2224       mapping->fib_index = sm->outside_fib_index;
2225     }
2226
2227   if (PREDICT_FALSE(is_addr_only != 0))
2228     *is_addr_only = m->addr_only;
2229
2230   if (PREDICT_FALSE(twice_nat != 0))
2231     *twice_nat = m->twice_nat;
2232
2233   if (PREDICT_FALSE(lb != 0))
2234     *lb = vec_len (m->locals) > 0;
2235
2236   return 0;
2237 }
2238
2239 static_always_inline u16
2240 snat_random_port (u16 min, u16 max)
2241 {
2242   snat_main_t *sm = &snat_main;
2243   return min + random_u32 (&sm->random_seed) /
2244     (random_u32_max() / (max - min + 1) + 1);
2245 }
2246
2247 int
2248 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2249                                      u32 fib_index,
2250                                      u32 thread_index,
2251                                      snat_session_key_t * k,
2252                                      u32 * address_indexp,
2253                                      u16 port_per_thread,
2254                                      u32 snat_thread_index)
2255 {
2256   snat_main_t *sm = &snat_main;
2257
2258   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2259                                  address_indexp, port_per_thread,
2260                                  snat_thread_index);
2261 }
2262
2263 static int
2264 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2265                                  u32 fib_index,
2266                                  u32 thread_index,
2267                                  snat_session_key_t * k,
2268                                  u32 * address_indexp,
2269                                  u16 port_per_thread,
2270                                  u32 snat_thread_index)
2271 {
2272   int i, gi = 0;
2273   snat_address_t *a, *ga = 0;
2274   u32 portnum;
2275
2276   for (i = 0; i < vec_len (addresses); i++)
2277     {
2278       a = addresses + i;
2279       switch (k->protocol)
2280         {
2281 #define _(N, j, n, s) \
2282         case SNAT_PROTOCOL_##N: \
2283           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2284             { \
2285               if (a->fib_index == fib_index) \
2286                 { \
2287                   while (1) \
2288                     { \
2289                       portnum = (port_per_thread * \
2290                         snat_thread_index) + \
2291                         snat_random_port(1, port_per_thread) + 1024; \
2292                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2293                         continue; \
2294                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2295                       a->busy_##n##_ports_per_thread[thread_index]++; \
2296                       a->busy_##n##_ports++; \
2297                       k->addr = a->addr; \
2298                       k->port = clib_host_to_net_u16(portnum); \
2299                       *address_indexp = i; \
2300                       return 0; \
2301                     } \
2302                 } \
2303               else if (a->fib_index == ~0) \
2304                 { \
2305                   ga = a; \
2306                   gi = i; \
2307                 } \
2308             } \
2309           break;
2310           foreach_snat_protocol
2311 #undef _
2312         default:
2313           nat_log_info ("unknown protocol");
2314           return 1;
2315         }
2316
2317     }
2318
2319   if (ga)
2320     {
2321       a = ga;
2322       switch (k->protocol)
2323         {
2324 #define _(N, j, n, s) \
2325         case SNAT_PROTOCOL_##N: \
2326           while (1) \
2327             { \
2328               portnum = (port_per_thread * \
2329                 snat_thread_index) + \
2330                 snat_random_port(1, port_per_thread) + 1024; \
2331               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2332                 continue; \
2333               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2334               a->busy_##n##_ports_per_thread[thread_index]++; \
2335               a->busy_##n##_ports++; \
2336               k->addr = a->addr; \
2337               k->port = clib_host_to_net_u16(portnum); \
2338               *address_indexp = gi; \
2339               return 0; \
2340             }
2341           break;
2342           foreach_snat_protocol
2343 #undef _
2344         default:
2345           nat_log_info ("unknown protocol");
2346           return 1;
2347         }
2348     }
2349
2350   /* Totally out of translations to use... */
2351   snat_ipfix_logging_addresses_exhausted(0);
2352   return 1;
2353 }
2354
2355 static int
2356 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2357                               u32 fib_index,
2358                               u32 thread_index,
2359                               snat_session_key_t * k,
2360                               u32 * address_indexp,
2361                               u16 port_per_thread,
2362                               u32 snat_thread_index)
2363 {
2364   snat_main_t *sm = &snat_main;
2365   snat_address_t *a = addresses;
2366   u16 m, ports, portnum, A, j;
2367   m = 16 - (sm->psid_offset + sm->psid_length);
2368   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2369
2370   if (!vec_len (addresses))
2371     goto exhausted;
2372
2373   switch (k->protocol)
2374     {
2375 #define _(N, i, n, s) \
2376     case SNAT_PROTOCOL_##N: \
2377       if (a->busy_##n##_ports < ports) \
2378         { \
2379           while (1) \
2380             { \
2381               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2382               j = snat_random_port(0, pow2_mask(m)); \
2383               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2384               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2385                 continue; \
2386               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2387               a->busy_##n##_ports++; \
2388               k->addr = a->addr; \
2389               k->port = clib_host_to_net_u16 (portnum); \
2390               *address_indexp = i; \
2391               return 0; \
2392             } \
2393         } \
2394       break;
2395       foreach_snat_protocol
2396 #undef _
2397     default:
2398       nat_log_info ("unknown protocol");
2399       return 1;
2400     }
2401
2402 exhausted:
2403   /* Totally out of translations to use... */
2404   snat_ipfix_logging_addresses_exhausted(0);
2405   return 1;
2406 }
2407
2408 void
2409 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2410 {
2411   dpo_id_t dpo_v4 = DPO_INVALID;
2412   fib_prefix_t pfx = {
2413     .fp_proto = FIB_PROTOCOL_IP4,
2414     .fp_len = 32,
2415     .fp_addr.ip4.as_u32 = addr.as_u32,
2416   };
2417
2418   if (is_add)
2419     {
2420       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2421       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2422                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2423       dpo_reset (&dpo_v4);
2424     }
2425   else
2426     {
2427       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2428     }
2429 }
2430
2431 uword
2432 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2433 {
2434   u32 *r = va_arg (*args, u32 *);
2435
2436   if (0);
2437 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2438   foreach_snat_protocol
2439 #undef _
2440   else
2441     return 0;
2442   return 1;
2443 }
2444
2445 u8 *
2446 format_snat_protocol (u8 * s, va_list * args)
2447 {
2448   u32 i = va_arg (*args, u32);
2449   u8 *t = 0;
2450
2451   switch (i)
2452     {
2453 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2454       foreach_snat_protocol
2455 #undef _
2456     default:
2457       s = format (s, "unknown");
2458       return s;
2459     }
2460   s = format (s, "%s", t);
2461   return s;
2462 }
2463
2464 u8 * format_snat_key (u8 * s, va_list * args);
2465
2466 u8 *
2467 format_session_kvp (u8 * s, va_list * args)
2468 {
2469   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2470   snat_session_key_t k;
2471
2472   k.as_u64 = v->key;
2473
2474   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2475
2476   return s;
2477 }
2478
2479 u8 *
2480 format_static_mapping_kvp (u8 * s, va_list * args)
2481 {
2482   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2483   snat_session_key_t k;
2484
2485   k.as_u64 = v->key;
2486
2487   s = format (s, "%U static-mapping-index %llu", format_snat_key, &k, v->value);
2488
2489   return s;
2490 }
2491
2492 u8 *
2493 format_user_kvp (u8 * s, va_list * args)
2494 {
2495   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2496   snat_user_key_t k;
2497
2498   k.as_u64 = v->key;
2499
2500   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2501               k.fib_index, v->value);
2502
2503   return s;
2504 }
2505
2506 u8 *
2507 format_ed_session_kvp (u8 * s, va_list * args)
2508 {
2509   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2510   nat_ed_ses_key_t k;
2511
2512   k.as_u64[0] = v->key[0];
2513   k.as_u64[1] = v->key[1];
2514
2515   s = format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2516               format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2517               format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2518               format_ip_protocol, k.proto, k.fib_index, v->value);
2519
2520   return s;
2521 }
2522
2523 static u32
2524 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2525 {
2526   snat_main_t *sm = &snat_main;
2527   u32 next_worker_index = 0;
2528   u32 hash;
2529
2530   next_worker_index = sm->first_worker_index;
2531   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2532          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2533
2534   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2535     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2536   else
2537     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2538
2539   return next_worker_index;
2540 }
2541
2542 static u32
2543 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2544 {
2545   snat_main_t *sm = &snat_main;
2546   udp_header_t *udp;
2547   u16 port;
2548   snat_session_key_t m_key;
2549   clib_bihash_kv_8_8_t kv, value;
2550   snat_static_mapping_t *m;
2551   u32 proto;
2552   u32 next_worker_index = 0;
2553
2554   /* first try static mappings without port */
2555   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2556     {
2557       m_key.addr = ip0->dst_address;
2558       m_key.port = 0;
2559       m_key.protocol = 0;
2560       m_key.fib_index = rx_fib_index0;
2561       kv.key = m_key.as_u64;
2562       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2563         {
2564           m = pool_elt_at_index (sm->static_mappings, value.value);
2565           return m->workers[0];
2566         }
2567     }
2568
2569   proto = ip_proto_to_snat_proto (ip0->protocol);
2570   udp = ip4_next_header (ip0);
2571   port = udp->dst_port;
2572
2573   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2574     {
2575       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2576         return vlib_get_thread_index ();
2577
2578       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2579         {
2580           nat_reass_ip4_t *reass;
2581
2582           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2583                                       ip0->fragment_id, ip0->protocol);
2584
2585           if (reass && (reass->thread_index != (u32) ~ 0))
2586             return reass->thread_index;
2587           else
2588             return vlib_get_thread_index ();
2589         }
2590     }
2591
2592   /* unknown protocol */
2593   if (PREDICT_FALSE (proto == ~0))
2594     {
2595       /* use current thread */
2596       return vlib_get_thread_index ();
2597     }
2598
2599   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2600     {
2601       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2602       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2603       if (!icmp_is_error_message (icmp))
2604         port = echo->identifier;
2605       else
2606         {
2607           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2608           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2609           void *l4_header = ip4_next_header (inner_ip);
2610           switch (proto)
2611             {
2612             case SNAT_PROTOCOL_ICMP:
2613               icmp = (icmp46_header_t*)l4_header;
2614               echo = (icmp_echo_header_t *)(icmp + 1);
2615               port = echo->identifier;
2616               break;
2617             case SNAT_PROTOCOL_UDP:
2618             case SNAT_PROTOCOL_TCP:
2619               port = ((tcp_udp_header_t*)l4_header)->src_port;
2620               break;
2621             default:
2622               return vlib_get_thread_index ();
2623             }
2624         }
2625     }
2626
2627   /* try static mappings with port */
2628   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2629     {
2630       m_key.addr = ip0->dst_address;
2631       m_key.port = clib_net_to_host_u16 (port);
2632       m_key.protocol = proto;
2633       m_key.fib_index = rx_fib_index0;
2634       kv.key = m_key.as_u64;
2635       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2636         {
2637           m = pool_elt_at_index (sm->static_mappings, value.value);
2638           return m->workers[0];
2639         }
2640     }
2641
2642   /* worker by outside port */
2643   next_worker_index = sm->first_worker_index;
2644   next_worker_index +=
2645     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2646   return next_worker_index;
2647 }
2648
2649 static u32
2650 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2651 {
2652   snat_main_t *sm = &snat_main;
2653   clib_bihash_kv_8_8_t kv, value;
2654   u32 proto, next_worker_index = 0;
2655   udp_header_t *udp;
2656   u16 port;
2657   snat_static_mapping_t *m;
2658   u32 hash;
2659
2660   /* first try static mappings without port */
2661   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2662     {
2663       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2664       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2665         {
2666           m = pool_elt_at_index (sm->static_mappings, value.value);
2667           return m->workers[0];
2668         }
2669     }
2670
2671   proto = ip_proto_to_snat_proto (ip->protocol);
2672
2673   /* unknown protocol */
2674   if (PREDICT_FALSE (proto == ~0))
2675     {
2676       /* use current thread */
2677       return vlib_get_thread_index ();
2678     }
2679
2680   udp = ip4_next_header (ip);
2681   port = udp->dst_port;
2682
2683   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2684     {
2685       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2686       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2687       if (!icmp_is_error_message (icmp))
2688         port = echo->identifier;
2689       else
2690         {
2691           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2692           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2693           void *l4_header = ip4_next_header (inner_ip);
2694           switch (proto)
2695             {
2696             case SNAT_PROTOCOL_ICMP:
2697               icmp = (icmp46_header_t*)l4_header;
2698               echo = (icmp_echo_header_t *)(icmp + 1);
2699               port = echo->identifier;
2700               break;
2701             case SNAT_PROTOCOL_UDP:
2702             case SNAT_PROTOCOL_TCP:
2703               port = ((tcp_udp_header_t*)l4_header)->src_port;
2704               break;
2705             default:
2706               return vlib_get_thread_index ();
2707             }
2708         }
2709     }
2710
2711   /* try static mappings with port */
2712   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2713     {
2714       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2715                   clib_net_to_host_u16 (port));
2716       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2717         {
2718           m = pool_elt_at_index (sm->static_mappings, value.value);
2719           if (!vec_len(m->locals))
2720             return m->workers[0];
2721
2722           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2723                  (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >>24);
2724
2725           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2726             return m->workers[hash & (_vec_len (m->workers) - 1)];
2727           else
2728             return m->workers[hash % _vec_len (m->workers)];
2729         }
2730     }
2731
2732   /* worker by outside port */
2733   next_worker_index = sm->first_worker_index;
2734   next_worker_index +=
2735     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2736
2737   return next_worker_index;
2738 }
2739
2740 static clib_error_t *
2741 snat_config (vlib_main_t * vm, unformat_input_t * input)
2742 {
2743   snat_main_t * sm = &snat_main;
2744   nat66_main_t * nm = &nat66_main;
2745   u32 translation_buckets = 1024;
2746   u32 translation_memory_size = 128<<20;
2747   u32 user_buckets = 128;
2748   u32 user_memory_size = 64<<20;
2749   u32 max_translations_per_user = 100;
2750   u32 outside_vrf_id = 0;
2751   u32 outside_ip6_vrf_id = 0;
2752   u32 inside_vrf_id = 0;
2753   u32 static_mapping_buckets = 1024;
2754   u32 static_mapping_memory_size = 64<<20;
2755   u32 nat64_bib_buckets = 1024;
2756   u32 nat64_bib_memory_size = 128 << 20;
2757   u32 nat64_st_buckets = 2048;
2758   u32 nat64_st_memory_size = 256 << 20;
2759   u8 static_mapping_only = 0;
2760   u8 static_mapping_connection_tracking = 0;
2761   snat_main_per_thread_data_t *tsm;
2762   dslite_main_t * dm = &dslite_main;
2763
2764   sm->deterministic = 0;
2765   sm->out2in_dpo = 0;
2766   sm->endpoint_dependent = 0;
2767
2768   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2769     {
2770       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2771         ;
2772       else if (unformat (input, "translation hash memory %d",
2773                          &translation_memory_size));
2774       else if (unformat (input, "user hash buckets %d", &user_buckets))
2775         ;
2776       else if (unformat (input, "user hash memory %d",
2777                          &user_memory_size))
2778         ;
2779       else if (unformat (input, "max translations per user %d",
2780                          &max_translations_per_user))
2781         ;
2782       else if (unformat (input, "outside VRF id %d",
2783                          &outside_vrf_id))
2784         ;
2785       else if (unformat (input, "outside ip6 VRF id %d",
2786                          &outside_ip6_vrf_id))
2787         ;
2788       else if (unformat (input, "inside VRF id %d",
2789                          &inside_vrf_id))
2790         ;
2791       else if (unformat (input, "static mapping only"))
2792         {
2793           static_mapping_only = 1;
2794           if (unformat (input, "connection tracking"))
2795             static_mapping_connection_tracking = 1;
2796         }
2797       else if (unformat (input, "deterministic"))
2798         sm->deterministic = 1;
2799       else if (unformat (input, "nat64 bib hash buckets %d",
2800                          &nat64_bib_buckets))
2801         ;
2802       else if (unformat (input, "nat64 bib hash memory %d",
2803                          &nat64_bib_memory_size))
2804         ;
2805       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2806         ;
2807       else if (unformat (input, "nat64 st hash memory %d",
2808                          &nat64_st_memory_size))
2809         ;
2810       else if (unformat (input, "out2in dpo"))
2811         sm->out2in_dpo = 1;
2812       else if (unformat (input, "dslite ce"))
2813         dslite_set_ce(dm, 1);
2814       else if (unformat (input, "endpoint-dependent"))
2815         sm->endpoint_dependent = 1;
2816       else
2817         return clib_error_return (0, "unknown input '%U'",
2818                                   format_unformat_error, input);
2819     }
2820
2821   /* for show commands, etc. */
2822   sm->translation_buckets = translation_buckets;
2823   sm->translation_memory_size = translation_memory_size;
2824   /* do not exceed load factor 10 */
2825   sm->max_translations = 10 * translation_buckets;
2826   sm->user_buckets = user_buckets;
2827   sm->user_memory_size = user_memory_size;
2828   sm->max_translations_per_user = max_translations_per_user;
2829   sm->outside_vrf_id = outside_vrf_id;
2830   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2831                                                              outside_vrf_id,
2832                                                              FIB_SOURCE_PLUGIN_HI);
2833   nm->outside_vrf_id = outside_ip6_vrf_id;
2834   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2835                                                              outside_ip6_vrf_id,
2836                                                              FIB_SOURCE_PLUGIN_HI);
2837   sm->inside_vrf_id = inside_vrf_id;
2838   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2839                                                             inside_vrf_id,
2840                                                             FIB_SOURCE_PLUGIN_HI);
2841   sm->static_mapping_only = static_mapping_only;
2842   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
2843
2844   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
2845                  nat64_st_memory_size);
2846
2847   if (sm->deterministic)
2848     {
2849       sm->in2out_node_index = snat_det_in2out_node.index;
2850       sm->in2out_output_node_index = ~0;
2851       sm->out2in_node_index = snat_det_out2in_node.index;
2852       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
2853       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
2854     }
2855   else
2856     {
2857       if (sm->endpoint_dependent)
2858         {
2859           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2860           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2861           sm->in2out_node_index = nat44_ed_in2out_node.index;
2862           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
2863           sm->out2in_node_index = nat44_ed_out2in_node.index;
2864           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
2865           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
2866         }
2867       else
2868         {
2869           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2870           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2871           sm->in2out_node_index = snat_in2out_node.index;
2872           sm->in2out_output_node_index = snat_in2out_output_node.index;
2873           sm->out2in_node_index = snat_out2in_node.index;
2874           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2875           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2876         }
2877       if (!static_mapping_only ||
2878           (static_mapping_only && static_mapping_connection_tracking))
2879         {
2880           vec_foreach (tsm, sm->per_thread_data)
2881             {
2882               if (sm->endpoint_dependent)
2883                 {
2884                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
2885                                          translation_buckets,
2886                                          translation_memory_size);
2887                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
2888                                                       format_ed_session_kvp);
2889
2890                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
2891                                          translation_buckets,
2892                                          translation_memory_size);
2893                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
2894                                                       format_ed_session_kvp);
2895                 }
2896               else
2897                 {
2898                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
2899                                         translation_buckets,
2900                                         translation_memory_size);
2901                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
2902                                                      format_session_kvp);
2903
2904                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
2905                                         translation_buckets,
2906                                         translation_memory_size);
2907                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
2908                                                      format_session_kvp);
2909                 }
2910
2911               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
2912                                     user_memory_size);
2913               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
2914                                                  format_user_kvp);
2915             }
2916
2917         }
2918       else
2919         {
2920           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2921           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2922         }
2923       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2924                             "static_mapping_by_local", static_mapping_buckets,
2925                             static_mapping_memory_size);
2926       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
2927                                          format_static_mapping_kvp);
2928
2929       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2930                             "static_mapping_by_external", static_mapping_buckets,
2931                             static_mapping_memory_size);
2932       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
2933                                          format_static_mapping_kvp);
2934     }
2935
2936   return 0;
2937 }
2938
2939 VLIB_CONFIG_FUNCTION (snat_config, "nat");
2940
2941 u8 * format_snat_session_state (u8 * s, va_list * args)
2942 {
2943   u32 i = va_arg (*args, u32);
2944   u8 *t = 0;
2945
2946   switch (i)
2947     {
2948 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
2949     foreach_snat_session_state
2950 #undef _
2951     default:
2952       t = format (t, "unknown");
2953     }
2954   s = format (s, "%s", t);
2955   return s;
2956 }
2957
2958 u8 * format_snat_key (u8 * s, va_list * args)
2959 {
2960   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
2961
2962   s = format (s, "%U proto %U port %d fib %d",
2963               format_ip4_address, &key->addr,
2964               format_snat_protocol, key->protocol,
2965               clib_net_to_host_u16 (key->port), key->fib_index);
2966   return s;
2967 }
2968
2969 u8 * format_snat_session (u8 * s, va_list * args)
2970 {
2971   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
2972   snat_session_t * sess = va_arg (*args, snat_session_t *);
2973
2974   if (snat_is_unk_proto_session (sess))
2975     {
2976       s = format (s, "  i2o %U proto %u fib %u\n",
2977                   format_ip4_address, &sess->in2out.addr,
2978                   clib_net_to_host_u16 (sess->in2out.port),
2979                   sess->in2out.fib_index);
2980       s = format (s, "    o2i %U proto %u fib %u\n",
2981                   format_ip4_address, &sess->out2in.addr,
2982                   clib_net_to_host_u16 (sess->out2in.port),
2983                   sess->out2in.fib_index);
2984     }
2985   else
2986     {
2987       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
2988       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
2989     }
2990   if (is_ed_session (sess) || is_fwd_bypass_session (sess))
2991     {
2992       if (is_twice_nat_session (sess))
2993         {
2994           s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
2995                       format_ip4_address, &sess->ext_host_addr,
2996                       clib_net_to_host_u16 (sess->ext_host_port),
2997                       format_ip4_address, &sess->ext_host_nat_addr,
2998                       clib_net_to_host_u16 (sess->ext_host_nat_port));
2999         }
3000       else
3001         {
3002           if (sess->ext_host_addr.as_u32)
3003               s = format (s, "       external host %U:%u\n",
3004                           format_ip4_address, &sess->ext_host_addr,
3005                           clib_net_to_host_u16 (sess->ext_host_port));
3006         }
3007     }
3008   s = format (s, "       index %llu\n", sess - sm->sessions);
3009   s = format (s, "       last heard %.2f\n", sess->last_heard);
3010   s = format (s, "       total pkts %d, total bytes %lld\n",
3011               sess->total_pkts, sess->total_bytes);
3012   if (snat_is_session_static (sess))
3013     s = format (s, "       static translation\n");
3014   else
3015     s = format (s, "       dynamic translation\n");
3016   if (is_fwd_bypass_session (sess))
3017     s = format (s, "       forwarding-bypass\n");
3018   if (is_lb_session (sess))
3019     s = format (s, "       load-balancing\n");
3020   if (is_twice_nat_session (sess))
3021     s = format (s, "       twice-nat\n");
3022
3023   return s;
3024 }
3025
3026 u8 * format_snat_user (u8 * s, va_list * args)
3027 {
3028   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3029   snat_user_t * u = va_arg (*args, snat_user_t *);
3030   int verbose = va_arg (*args, int);
3031   dlist_elt_t * head, * elt;
3032   u32 elt_index, head_index;
3033   u32 session_index;
3034   snat_session_t * sess;
3035
3036   s = format (s, "%U: %d dynamic translations, %d static translations\n",
3037               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
3038
3039   if (verbose == 0)
3040     return s;
3041
3042   if (u->nsessions || u->nstaticsessions)
3043     {
3044       head_index = u->sessions_per_user_list_head_index;
3045       head = pool_elt_at_index (sm->list_pool, head_index);
3046
3047       elt_index = head->next;
3048       elt = pool_elt_at_index (sm->list_pool, elt_index);
3049       session_index = elt->value;
3050
3051       while (session_index != ~0)
3052         {
3053           sess = pool_elt_at_index (sm->sessions, session_index);
3054
3055           s = format (s, "  %U\n", format_snat_session, sm, sess);
3056
3057           elt_index = elt->next;
3058           elt = pool_elt_at_index (sm->list_pool, elt_index);
3059           session_index = elt->value;
3060         }
3061     }
3062
3063   return s;
3064 }
3065
3066 u8 * format_snat_static_mapping (u8 * s, va_list * args)
3067 {
3068   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
3069   nat44_lb_addr_port_t *local;
3070
3071   if (m->addr_only)
3072       s = format (s, "local %U external %U vrf %d %s %s",
3073                   format_ip4_address, &m->local_addr,
3074                   format_ip4_address, &m->external_addr,
3075                   m->vrf_id,
3076                   m->twice_nat == TWICE_NAT ? "twice-nat" :
3077                   m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3078                   m->out2in_only ? "out2in-only" : "");
3079   else
3080    {
3081       if (vec_len (m->locals))
3082         {
3083           s = format (s, "%U external %U:%d %s %s",
3084                       format_snat_protocol, m->proto,
3085                       format_ip4_address, &m->external_addr, m->external_port,
3086                       m->twice_nat == TWICE_NAT ? "twice-nat" :
3087                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3088                       m->out2in_only ? "out2in-only" : "");
3089           vec_foreach (local, m->locals)
3090             s = format (s, "\n  local %U:%d vrf %d probability %d\%",
3091                         format_ip4_address, &local->addr, local->port,
3092                         local->vrf_id, local->probability);
3093         }
3094       else
3095         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
3096                     format_snat_protocol, m->proto,
3097                     format_ip4_address, &m->local_addr, m->local_port,
3098                     format_ip4_address, &m->external_addr, m->external_port,
3099                     m->vrf_id,
3100                     m->twice_nat == TWICE_NAT ? "twice-nat" :
3101                     m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3102                     m->out2in_only ? "out2in-only" : "");
3103    }
3104   return s;
3105 }
3106
3107 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
3108 {
3109   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
3110   vnet_main_t *vnm = vnet_get_main();
3111
3112   if (m->addr_only)
3113       s = format (s, "local %U external %U vrf %d",
3114                   format_ip4_address, &m->l_addr,
3115                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3116                   m->vrf_id);
3117   else
3118       s = format (s, "%U local %U:%d external %U:%d vrf %d",
3119                   format_snat_protocol, m->proto,
3120                   format_ip4_address, &m->l_addr, m->l_port,
3121                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3122                   m->e_port, m->vrf_id);
3123
3124   return s;
3125 }
3126
3127 u8 * format_det_map_ses (u8 * s, va_list * args)
3128 {
3129   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
3130   ip4_address_t in_addr, out_addr;
3131   u32 in_offset, out_offset;
3132   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
3133   u32 * i = va_arg (*args, u32 *);
3134
3135   u32 user_index = *i / SNAT_DET_SES_PER_USER;
3136   in_addr.as_u32 = clib_host_to_net_u32 (
3137     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
3138   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
3139     clib_net_to_host_u32(det_map->in_addr.as_u32);
3140   out_offset = in_offset / det_map->sharing_ratio;
3141   out_addr.as_u32 = clib_host_to_net_u32(
3142     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
3143   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
3144               format_ip4_address, &in_addr,
3145               clib_net_to_host_u16 (ses->in_port),
3146               format_ip4_address, &out_addr,
3147               clib_net_to_host_u16 (ses->out.out_port),
3148               format_ip4_address, &ses->out.ext_host_addr,
3149               clib_net_to_host_u16 (ses->out.ext_host_port),
3150               format_snat_session_state, ses->state,
3151               ses->expire);
3152
3153   return s;
3154 }
3155
3156 static void
3157 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3158                                  uword opaque,
3159                                  u32 sw_if_index,
3160                                  ip4_address_t * address,
3161                                  u32 address_length,
3162                                  u32 if_address_index,
3163                                  u32 is_delete)
3164 {
3165   snat_main_t *sm = &snat_main;
3166   snat_static_map_resolve_t *rp;
3167   snat_static_mapping_t *m;
3168   snat_session_key_t m_key;
3169   clib_bihash_kv_8_8_t kv, value;
3170   int i, rv;
3171   ip4_address_t l_addr;
3172
3173   for (i = 0; i < vec_len (sm->to_resolve); i++)
3174     {
3175       rp = sm->to_resolve + i;
3176       if (rp->addr_only == 0)
3177         continue;
3178       if (rp->sw_if_index == sw_if_index)
3179         goto match;
3180     }
3181
3182   return;
3183
3184 match:
3185   m_key.addr.as_u32 = address->as_u32;
3186   m_key.port = rp->addr_only ? 0 : rp->e_port;
3187   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3188   m_key.fib_index = sm->outside_fib_index;
3189   kv.key = m_key.as_u64;
3190   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3191     m = 0;
3192   else
3193     m = pool_elt_at_index (sm->static_mappings, value.value);
3194
3195   if (!is_delete)
3196     {
3197       /* Don't trip over lease renewal, static config */
3198       if (m)
3199         return;
3200     }
3201   else
3202     {
3203       if (!m)
3204         return;
3205     }
3206
3207   /* Indetity mapping? */
3208   if (rp->l_addr.as_u32 == 0)
3209     l_addr.as_u32 = address[0].as_u32;
3210   else
3211     l_addr.as_u32 = rp->l_addr.as_u32;
3212   /* Add the static mapping */
3213   rv = snat_add_static_mapping (l_addr,
3214                                 address[0],
3215                                 rp->l_port,
3216                                 rp->e_port,
3217                                 rp->vrf_id,
3218                                 rp->addr_only,
3219                                 ~0 /* sw_if_index */,
3220                                 rp->proto,
3221                                 !is_delete,
3222                                 0, 0, rp->tag);
3223   if (rv)
3224     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3225 }
3226
3227 static void
3228 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3229                                        uword opaque,
3230                                        u32 sw_if_index,
3231                                        ip4_address_t * address,
3232                                        u32 address_length,
3233                                        u32 if_address_index,
3234                                        u32 is_delete)
3235 {
3236   snat_main_t *sm = &snat_main;
3237   snat_static_map_resolve_t *rp;
3238   ip4_address_t l_addr;
3239   int i, j;
3240   int rv;
3241   u8 twice_nat = 0;
3242   snat_address_t *addresses = sm->addresses;
3243
3244   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
3245     {
3246       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3247           goto match;
3248     }
3249
3250   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
3251     {
3252       twice_nat = 1;
3253       addresses = sm->twice_nat_addresses;
3254       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3255           goto match;
3256     }
3257
3258   return;
3259
3260 match:
3261   if (!is_delete)
3262     {
3263       /* Don't trip over lease renewal, static config */
3264       for (j = 0; j < vec_len(addresses); j++)
3265         if (addresses[j].addr.as_u32 == address->as_u32)
3266           return;
3267
3268       (void) snat_add_address (sm, address, ~0, twice_nat);
3269       /* Scan static map resolution vector */
3270       for (j = 0; j < vec_len (sm->to_resolve); j++)
3271         {
3272           rp = sm->to_resolve + j;
3273           if (rp->addr_only)
3274             continue;
3275           /* On this interface? */
3276           if (rp->sw_if_index == sw_if_index)
3277             {
3278               /* Indetity mapping? */
3279               if (rp->l_addr.as_u32 == 0)
3280                 l_addr.as_u32 = address[0].as_u32;
3281               else
3282                 l_addr.as_u32 = rp->l_addr.as_u32;
3283               /* Add the static mapping */
3284               rv = snat_add_static_mapping (l_addr,
3285                                             address[0],
3286                                             rp->l_port,
3287                                             rp->e_port,
3288                                             rp->vrf_id,
3289                                             rp->addr_only,
3290                                             ~0 /* sw_if_index */,
3291                                             rp->proto,
3292                                             rp->is_add,
3293                                             0, 0, rp->tag);
3294               if (rv)
3295                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3296             }
3297         }
3298       return;
3299     }
3300   else
3301     {
3302       (void) snat_del_address(sm, address[0], 1, twice_nat);
3303       return;
3304     }
3305 }
3306
3307
3308 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
3309                                 u8 twice_nat)
3310 {
3311   ip4_main_t * ip4_main = sm->ip4_main;
3312   ip4_address_t * first_int_addr;
3313   snat_static_map_resolve_t *rp;
3314   u32 *indices_to_delete = 0;
3315   int i, j;
3316   u32 *auto_add_sw_if_indices =
3317     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3318
3319   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
3320                                                 0 /* just want the address*/);
3321
3322   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
3323     {
3324       if (auto_add_sw_if_indices[i] == sw_if_index)
3325         {
3326           if (is_del)
3327             {
3328               /* if have address remove it */
3329               if (first_int_addr)
3330                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3331               else
3332                 {
3333                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3334                     {
3335                       rp = sm->to_resolve + j;
3336                       if (rp->sw_if_index == sw_if_index)
3337                         vec_add1 (indices_to_delete, j);
3338                     }
3339                   if (vec_len(indices_to_delete))
3340                     {
3341                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3342                         vec_del1(sm->to_resolve, j);
3343                       vec_free(indices_to_delete);
3344                     }
3345                 }
3346               if (twice_nat)
3347                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3348               else
3349                 vec_del1(sm->auto_add_sw_if_indices, i);
3350             }
3351           else
3352             return VNET_API_ERROR_VALUE_EXIST;
3353
3354           return 0;
3355         }
3356     }
3357
3358   if (is_del)
3359     return VNET_API_ERROR_NO_SUCH_ENTRY;
3360
3361   /* add to the auto-address list */
3362   if (twice_nat)
3363     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3364   else
3365     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3366
3367   /* If the address is already bound - or static - add it now */
3368   if (first_int_addr)
3369       (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3370
3371   return 0;
3372 }
3373
3374 int
3375 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3376                    snat_protocol_t proto, u32 vrf_id, int is_in)
3377 {
3378   snat_main_per_thread_data_t *tsm;
3379   clib_bihash_kv_8_8_t kv, value;
3380   ip4_header_t ip;
3381   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3382   snat_session_key_t key;
3383   snat_session_t *s;
3384   clib_bihash_8_8_t *t;
3385
3386   if (sm->endpoint_dependent)
3387     return VNET_API_ERROR_UNSUPPORTED;
3388
3389   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3390   if (sm->num_workers > 1)
3391     tsm =
3392       vec_elt_at_index (sm->per_thread_data,
3393                         sm->worker_in2out_cb (&ip, fib_index));
3394   else
3395     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3396
3397   key.addr.as_u32 = addr->as_u32;
3398   key.port = clib_host_to_net_u16 (port);
3399   key.protocol = proto;
3400   key.fib_index = fib_index;
3401   kv.key = key.as_u64;
3402   t = is_in ? &tsm->in2out : &tsm->out2in;
3403   if (!clib_bihash_search_8_8 (t, &kv, &value))
3404     {
3405       if (pool_is_free_index (tsm->sessions, value.value))
3406         return VNET_API_ERROR_UNSPECIFIED;
3407
3408       s = pool_elt_at_index (tsm->sessions, value.value);
3409       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3410       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3411       return 0;
3412     }
3413
3414   return VNET_API_ERROR_NO_SUCH_ENTRY;
3415 }
3416
3417 int
3418 nat44_del_ed_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3419                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3420                       u32 vrf_id, int is_in)
3421 {
3422   ip4_header_t ip;
3423   clib_bihash_16_8_t *t;
3424   nat_ed_ses_key_t key;
3425   clib_bihash_kv_16_8_t kv, value;
3426   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3427   snat_session_t *s;
3428   snat_main_per_thread_data_t *tsm;
3429
3430   if (!sm->endpoint_dependent)
3431     return VNET_API_ERROR_FEATURE_DISABLED;
3432
3433   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3434   if (sm->num_workers > 1)
3435     tsm =
3436       vec_elt_at_index (sm->per_thread_data,
3437                         sm->worker_in2out_cb (&ip, fib_index));
3438   else
3439     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3440
3441   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3442   key.l_addr.as_u32 = addr->as_u32;
3443   key.r_addr.as_u32 = eh_addr->as_u32;
3444   key.l_port = clib_host_to_net_u16 (port);
3445   key.r_port = clib_host_to_net_u16 (eh_port);
3446   key.proto = proto;
3447   key.fib_index = clib_host_to_net_u32 (fib_index);
3448   kv.key[0] = key.as_u64[0];
3449   kv.key[1] = key.as_u64[1];
3450   if (clib_bihash_search_16_8 (t, &kv, &value))
3451     return VNET_API_ERROR_NO_SUCH_ENTRY;
3452
3453   if (pool_is_free_index (tsm->sessions, value.value))
3454     return VNET_API_ERROR_UNSPECIFIED;
3455   s = pool_elt_at_index (tsm->sessions, value.value);
3456   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3457   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3458   return 0;
3459 }
3460
3461 void
3462 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3463 {
3464   snat_main_t *sm = &snat_main;
3465
3466   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3467   sm->psid = psid;
3468   sm->psid_offset = psid_offset;
3469   sm->psid_length = psid_length;
3470 }
3471
3472 void
3473 nat_set_alloc_addr_and_port_default (void)
3474 {
3475   snat_main_t *sm = &snat_main;
3476
3477   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3478 }
3479