src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <nat/nat_inlines.h>
  31 #include <vnet/fib/fib_table.h>
  32 #include <vnet/fib/ip4_fib.h>
  33
  34 #include <vpp/app/version.h>
  35
  36 snat_main_t snat_main;
  37
  38
  39 /* Hook up input features */
  40 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  41   .arc_name = "ip4-unicast",
  42   .node_name = "nat44-in2out",
  43   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  44 };
  45 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  46   .arc_name = "ip4-unicast",
  47   .node_name = "nat44-out2in",
  48   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  49                                "ip4-dhcp-client-detect"),
  50 };
  51 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  52   .arc_name = "ip4-unicast",
  53   .node_name = "nat44-classify",
  54   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  55 };
  56 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  57   .arc_name = "ip4-unicast",
  58   .node_name = "nat44-det-in2out",
  59   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  60 };
  61 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  62   .arc_name = "ip4-unicast",
  63   .node_name = "nat44-det-out2in",
  64   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  65                                "ip4-dhcp-client-detect"),
  66 };
  67 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  68   .arc_name = "ip4-unicast",
  69   .node_name = "nat44-det-classify",
  70   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  71 };
  72 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
  73   .arc_name = "ip4-unicast",
  74   .node_name = "nat44-ed-in2out",
  75   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  76 };
  77 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
  78   .arc_name = "ip4-unicast",
  79   .node_name = "nat44-ed-out2in",
  80   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  81                                "ip4-dhcp-client-detect"),
  82 };
  83 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  84   .arc_name = "ip4-unicast",
  85   .node_name = "nat44-ed-classify",
  86   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  87 };
  88 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat44-in2out-worker-handoff",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  92 };
  93 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-out2in-worker-handoff",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  97                                "ip4-dhcp-client-detect"),
  98 };
  99 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 100   .arc_name = "ip4-unicast",
 101   .node_name = "nat44-handoff-classify",
 102   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 103 };
 104 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-fast",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 108 };
 109 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-fast",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-hairpin-dst",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-hairpin-dst",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 124 };
 125
 126 /* Hook up output features */
 127 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 128   .arc_name = "ip4-output",
 129   .node_name = "nat44-in2out-output",
 130   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 136 };
 137 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 138   .arc_name = "ip4-output",
 139   .node_name = "nat44-hairpin-src",
 140   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 141 };
 142 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 143   .arc_name = "ip4-output",
 144   .node_name = "nat44-ed-in2out-output",
 145   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 146 };
 147 VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
 148   .arc_name = "ip4-output",
 149   .node_name = "nat44-ed-hairpin-src",
 150   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 151 };
 152
 153 /* Hook up ip4-local features */
 154 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 155 {
 156   .arc_name = "ip4-local",
 157   .node_name = "nat44-hairpinning",
 158   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 159 };
 160 VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
 161 {
 162   .arc_name = "ip4-local",
 163   .node_name = "nat44-ed-hairpinning",
 164   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 165 };
 166
 167
 168 /* *INDENT-OFF* */
 169 VLIB_PLUGIN_REGISTER () = {
 170     .version = VPP_BUILD_VER,
 171     .description = "Network Address Translation",
 172 };
 173 /* *INDENT-ON* */
 174
 175 vlib_node_registration_t nat44_classify_node;
 176 vlib_node_registration_t nat44_ed_classify_node;
 177 vlib_node_registration_t nat44_det_classify_node;
 178 vlib_node_registration_t nat44_handoff_classify_node;
 179
 180 typedef enum {
 181   NAT44_CLASSIFY_NEXT_IN2OUT,
 182   NAT44_CLASSIFY_NEXT_OUT2IN,
 183   NAT44_CLASSIFY_N_NEXT,
 184 } nat44_classify_next_t;
 185
 186 void
 187 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 188 {
 189   snat_session_key_t key;
 190   clib_bihash_kv_8_8_t kv;
 191   nat_ed_ses_key_t ed_key;
 192   clib_bihash_kv_16_8_t ed_kv;
 193   snat_main_per_thread_data_t *tsm =
 194     vec_elt_at_index (sm->per_thread_data, thread_index);
 195
 196   if (is_fwd_bypass_session (s))
 197     {
 198       ed_key.l_addr = s->in2out.addr;
 199       ed_key.r_addr = s->ext_host_addr;
 200       ed_key.l_port = s->in2out.port;
 201       ed_key.r_port = s->ext_host_port;
 202       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 203       ed_key.fib_index = 0;
 204       ed_kv.key[0] = ed_key.as_u64[0];
 205       ed_kv.key[1] = ed_key.as_u64[1];
 206       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 207         nat_log_warn ("in2out_ed key del failed");
 208       return;
 209     }
 210
 211   /* session lookup tables */
 212   if (is_ed_session (s))
 213     {
 214       ed_key.l_addr = s->out2in.addr;
 215       ed_key.r_addr = s->ext_host_addr;
 216       ed_key.fib_index = s->out2in.fib_index;
 217       if (snat_is_unk_proto_session (s))
 218         {
 219           ed_key.proto = s->in2out.port;
 220           ed_key.r_port = 0;
 221           ed_key.l_port = 0;
 222         }
 223       else
 224         {
 225           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 226           ed_key.l_port = s->out2in.port;
 227           ed_key.r_port = s->ext_host_port;
 228         }
 229       ed_kv.key[0] = ed_key.as_u64[0];
 230       ed_kv.key[1] = ed_key.as_u64[1];
 231       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 232         nat_log_warn ("out2in_ed key del failed");
 233
 234       ed_key.l_addr = s->in2out.addr;
 235       ed_key.fib_index = s->in2out.fib_index;
 236       if (!snat_is_unk_proto_session (s))
 237         ed_key.l_port = s->in2out.port;
 238       if (is_twice_nat_session (s))
 239         {
 240           ed_key.r_addr = s->ext_host_nat_addr;
 241           ed_key.r_port = s->ext_host_nat_port;
 242         }
 243       ed_kv.key[0] = ed_key.as_u64[0];
 244       ed_kv.key[1] = ed_key.as_u64[1];
 245       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
 246         nat_log_warn ("in2out_ed key del failed");
 247     }
 248   else
 249     {
 250       kv.key = s->in2out.as_u64;
 251       if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 252         nat_log_warn ("in2out key del failed");
 253       kv.key = s->out2in.as_u64;
 254       if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 255         nat_log_warn ("out2in key del failed");
 256     }
 257
 258   if (snat_is_unk_proto_session (s))
 259     return;
 260
 261   /* log NAT event */
 262   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 263                                       s->out2in.addr.as_u32,
 264                                       s->in2out.protocol,
 265                                       s->in2out.port,
 266                                       s->out2in.port,
 267                                       s->in2out.fib_index);
 268
 269   /* Twice NAT address and port for external host */
 270   if (is_twice_nat_session (s))
 271     {
 272       key.protocol = s->in2out.protocol;
 273       key.port = s->ext_host_nat_port;
 274       key.addr.as_u32 = s->ext_host_nat_addr.as_u32;
 275       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 276                                           thread_index, &key);
 277     }
 278
 279   if (snat_is_session_static (s))
 280     return;
 281
 282   if (s->outside_address_index != ~0)
 283     snat_free_outside_address_and_port (sm->addresses, thread_index,
 284                                         &s->out2in);
 285 }
 286
 287 snat_user_t *
 288 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 289                         u32 thread_index)
 290 {
 291   snat_user_t *u = 0;
 292   snat_user_key_t user_key;
 293   clib_bihash_kv_8_8_t kv, value;
 294   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 295   dlist_elt_t * per_user_list_head_elt;
 296
 297   user_key.addr.as_u32 = addr->as_u32;
 298   user_key.fib_index = fib_index;
 299   kv.key = user_key.as_u64;
 300
 301   /* Ever heard of the "user" = src ip4 address before? */
 302   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 303     {
 304       /* no, make a new one */
 305       pool_get (tsm->users, u);
 306       memset (u, 0, sizeof (*u));
 307       u->addr.as_u32 = addr->as_u32;
 308       u->fib_index = fib_index;
 309
 310       pool_get (tsm->list_pool, per_user_list_head_elt);
 311
 312       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 313         tsm->list_pool;
 314
 315       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 316
 317       kv.value = u - tsm->users;
 318
 319       /* add user */
 320       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 321         nat_log_warn ("user_hash keay add failed");
 322     }
 323   else
 324     {
 325       u = pool_elt_at_index (tsm->users, value.value);
 326     }
 327
 328   return u;
 329 }
 330
 331 snat_session_t *
 332 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 333 {
 334   snat_session_t *s;
 335   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 336   u32 oldest_per_user_translation_list_index, session_index;
 337   dlist_elt_t * oldest_per_user_translation_list_elt;
 338   dlist_elt_t * per_user_translation_list_elt;
 339
 340   /* Over quota? Recycle the least recently used translation */
 341   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 342     {
 343       oldest_per_user_translation_list_index =
 344         clib_dlist_remove_head (tsm->list_pool,
 345                                 u->sessions_per_user_list_head_index);
 346
 347       ASSERT (oldest_per_user_translation_list_index != ~0);
 348
 349       /* Add it back to the end of the LRU list */
 350       clib_dlist_addtail (tsm->list_pool,
 351                           u->sessions_per_user_list_head_index,
 352                           oldest_per_user_translation_list_index);
 353       /* Get the list element */
 354       oldest_per_user_translation_list_elt =
 355         pool_elt_at_index (tsm->list_pool,
 356                            oldest_per_user_translation_list_index);
 357
 358       /* Get the session index from the list element */
 359       session_index = oldest_per_user_translation_list_elt->value;
 360
 361       /* Get the session */
 362       s = pool_elt_at_index (tsm->sessions, session_index);
 363       nat_free_session_data (sm, s, thread_index);
 364       if (snat_is_session_static(s))
 365         u->nstaticsessions--;
 366       else
 367         u->nsessions--;
 368       s->outside_address_index = ~0;
 369       s->flags = 0;
 370       s->total_bytes = 0;
 371       s->total_pkts = 0;
 372       s->state = 0;
 373       s->ext_host_addr.as_u32 = 0;
 374       s->ext_host_port = 0;
 375       s->ext_host_nat_addr.as_u32 = 0;
 376       s->ext_host_nat_port = 0;
 377     }
 378   else
 379     {
 380       pool_get (tsm->sessions, s);
 381       memset (s, 0, sizeof (*s));
 382       s->outside_address_index = ~0;
 383
 384       /* Create list elts */
 385       pool_get (tsm->list_pool, per_user_translation_list_elt);
 386       clib_dlist_init (tsm->list_pool,
 387                        per_user_translation_list_elt - tsm->list_pool);
 388
 389       per_user_translation_list_elt->value = s - tsm->sessions;
 390       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 391       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 392
 393       clib_dlist_addtail (tsm->list_pool,
 394                           s->per_user_list_head_index,
 395                           per_user_translation_list_elt - tsm->list_pool);
 396     }
 397
 398   return s;
 399 }
 400
 401 snat_session_t *
 402 nat_ed_session_alloc (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 403 {
 404   snat_session_t *s;
 405   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 406   dlist_elt_t * per_user_translation_list_elt;
 407
 408   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 409     {
 410       nat_log_warn ("max translations per user %U", format_ip4_address, &u->addr);
 411       snat_ipfix_logging_max_entries_per_user (sm->max_translations_per_user,
 412                                                u->addr.as_u32);
 413       return 0;
 414     }
 415
 416   pool_get (tsm->sessions, s);
 417   memset (s, 0, sizeof (*s));
 418   s->outside_address_index = ~0;
 419
 420   /* Create list elts */
 421   pool_get (tsm->list_pool, per_user_translation_list_elt);
 422   clib_dlist_init (tsm->list_pool,
 423                    per_user_translation_list_elt - tsm->list_pool);
 424
 425   per_user_translation_list_elt->value = s - tsm->sessions;
 426   s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 427   s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 428
 429   clib_dlist_addtail (tsm->list_pool,
 430                       s->per_user_list_head_index,
 431                       per_user_translation_list_elt - tsm->list_pool);
 432
 433   return s;
 434 }
 435
 436 typedef struct {
 437   u8 next_in2out;
 438 } nat44_classify_trace_t;
 439
 440 static u8 * format_nat44_classify_trace (u8 * s, va_list * args)
 441 {
 442   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 443   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 444   nat44_classify_trace_t *t = va_arg (*args, nat44_classify_trace_t *);
 445   char *next;
 446
 447   next = t->next_in2out ? "nat44-in2out" : "nat44-out2in";
 448
 449   s = format (s, "nat44-classify: next %s", next);
 450
 451   return s;
 452 }
 453
 454 static inline uword
 455 nat44_classify_node_fn_inline (vlib_main_t * vm,
 456                                vlib_node_runtime_t * node,
 457                                vlib_frame_t * frame,
 458                                int is_ed)
 459 {
 460   u32 n_left_from, * from, * to_next;
 461   nat44_classify_next_t next_index;
 462   snat_main_t *sm = &snat_main;
 463   snat_static_mapping_t *m;
 464   u32 thread_index = vm->thread_index;
 465   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 466
 467   from = vlib_frame_vector_args (frame);
 468   n_left_from = frame->n_vectors;
 469   next_index = node->cached_next_index;
 470
 471   while (n_left_from > 0)
 472     {
 473       u32 n_left_to_next;
 474
 475       vlib_get_next_frame (vm, node, next_index,
 476                            to_next, n_left_to_next);
 477
 478       while (n_left_from > 0 && n_left_to_next > 0)
 479         {
 480           u32 bi0;
 481           vlib_buffer_t *b0;
 482           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0;
 483           ip4_header_t *ip0;
 484           snat_address_t *ap;
 485           snat_session_key_t m_key0;
 486           clib_bihash_kv_8_8_t kv0, value0;
 487           clib_bihash_kv_16_8_t ed_kv0, ed_value0;
 488           udp_header_t *udp0;
 489
 490           /* speculatively enqueue b0 to the current next frame */
 491           bi0 = from[0];
 492           to_next[0] = bi0;
 493           from += 1;
 494           to_next += 1;
 495           n_left_from -= 1;
 496           n_left_to_next -= 1;
 497
 498           b0 = vlib_get_buffer (vm, bi0);
 499           ip0 = vlib_buffer_get_current (b0);
 500           udp0 = ip4_next_header (ip0);
 501
 502           if (is_ed)
 503             {
 504               sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 505               rx_fib_index0 =
 506                 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 507                                                      sw_if_index0);
 508               make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address,
 509                           ip0->protocol, rx_fib_index0, udp0->src_port,
 510                           udp0->dst_port);
 511               if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0, &ed_value0))
 512                 goto enqueue0;
 513             }
 514
 515           vec_foreach (ap, sm->addresses)
 516             {
 517               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 518                 {
 519                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 520                   goto enqueue0;
 521                 }
 522             }
 523
 524           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 525             {
 526               m_key0.addr = ip0->dst_address;
 527               m_key0.port = 0;
 528               m_key0.protocol = 0;
 529               m_key0.fib_index = 0;
 530               kv0.key = m_key0.as_u64;
 531               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 532                 {
 533                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 534                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 535                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 536                   goto enqueue0;
 537                 }
 538               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 539               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 540               kv0.key = m_key0.as_u64;
 541               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 542                 {
 543                   m = pool_elt_at_index (sm->static_mappings, value0.value);
 544                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
 545                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 546                 }
 547             }
 548
 549         enqueue0:
 550           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 551                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 552             {
 553               nat44_classify_trace_t *t =
 554                   vlib_add_trace (vm, node, b0, sizeof (*t));
 555               t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
 556             }
 557
 558           /* verify speculative enqueue, maybe switch current next frame */
 559           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 560                                            to_next, n_left_to_next,
 561                                            bi0, next0);
 562         }
 563
 564       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 565     }
 566
 567   return frame->n_vectors;
 568 }
 569
 570 static uword
 571 nat44_classify_node_fn (vlib_main_t * vm,
 572                         vlib_node_runtime_t * node,
 573                         vlib_frame_t * frame)
 574 {
 575   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 576 };
 577
 578 VLIB_REGISTER_NODE (nat44_classify_node) = {
 579   .function = nat44_classify_node_fn,
 580   .name = "nat44-classify",
 581   .vector_size = sizeof (u32),
 582   .format_trace = format_nat44_classify_trace,
 583   .type = VLIB_NODE_TYPE_INTERNAL,
 584   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 585   .next_nodes = {
 586     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 587     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 588   },
 589 };
 590
 591 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 592                               nat44_classify_node_fn);
 593 static uword
 594 nat44_ed_classify_node_fn (vlib_main_t * vm,
 595                            vlib_node_runtime_t * node,
 596                            vlib_frame_t * frame)
 597 {
 598   return nat44_classify_node_fn_inline (vm, node, frame, 1);
 599 };
 600
 601 VLIB_REGISTER_NODE (nat44_ed_classify_node) = {
 602   .function = nat44_ed_classify_node_fn,
 603   .name = "nat44-ed-classify",
 604   .vector_size = sizeof (u32),
 605   .format_trace = format_nat44_classify_trace,
 606   .type = VLIB_NODE_TYPE_INTERNAL,
 607   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 608   .next_nodes = {
 609     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out",
 610     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in",
 611   },
 612 };
 613
 614 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_classify_node,
 615                               nat44_ed_classify_node_fn);
 616
 617 static uword
 618 nat44_det_classify_node_fn (vlib_main_t * vm,
 619                             vlib_node_runtime_t * node,
 620                             vlib_frame_t * frame)
 621 {
 622   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 623 };
 624
 625 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 626   .function = nat44_det_classify_node_fn,
 627   .name = "nat44-det-classify",
 628   .vector_size = sizeof (u32),
 629   .format_trace = format_nat44_classify_trace,
 630   .type = VLIB_NODE_TYPE_INTERNAL,
 631   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 632   .next_nodes = {
 633     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 634     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 635   },
 636 };
 637
 638 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 639                               nat44_det_classify_node_fn);
 640
 641 static uword
 642 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 643                                 vlib_node_runtime_t * node,
 644                                 vlib_frame_t * frame)
 645 {
 646   return nat44_classify_node_fn_inline (vm, node, frame, 0);
 647 };
 648
 649 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 650   .function = nat44_handoff_classify_node_fn,
 651   .name = "nat44-handoff-classify",
 652   .vector_size = sizeof (u32),
 653   .format_trace = format_nat44_classify_trace,
 654   .type = VLIB_NODE_TYPE_INTERNAL,
 655   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 656   .next_nodes = {
 657     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 658     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 659   },
 660 };
 661
 662 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 663                               nat44_handoff_classify_node_fn);
 664
 665 /**
 666  * @brief Add/del NAT address to FIB.
 667  *
 668  * Add the external NAT address to the FIB as receive entries. This ensures
 669  * that VPP will reply to ARP for this address and we don't need to enable
 670  * proxy ARP on the outside interface.
 671  *
 672  * @param addr IPv4 address.
 673  * @param plen address prefix length
 674  * @param sw_if_index Interface.
 675  * @param is_add If 0 delete, otherwise add.
 676  */
 677 void
 678 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 679                           int is_add)
 680 {
 681   fib_prefix_t prefix = {
 682     .fp_len = p_len,
 683     .fp_proto = FIB_PROTOCOL_IP4,
 684     .fp_addr = {
 685         .ip4.as_u32 = addr->as_u32,
 686     },
 687   };
 688   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 689
 690   if (is_add)
 691     fib_table_entry_update_one_path(fib_index,
 692                                     &prefix,
 693                                     FIB_SOURCE_PLUGIN_LOW,
 694                                     (FIB_ENTRY_FLAG_CONNECTED |
 695                                      FIB_ENTRY_FLAG_LOCAL |
 696                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 697                                     DPO_PROTO_IP4,
 698                                     NULL,
 699                                     sw_if_index,
 700                                     ~0,
 701                                     1,
 702                                     NULL,
 703                                     FIB_ROUTE_PATH_FLAG_NONE);
 704   else
 705     fib_table_entry_delete(fib_index,
 706                            &prefix,
 707                            FIB_SOURCE_PLUGIN_LOW);
 708 }
 709
 710 int snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 711                        u8 twice_nat)
 712 {
 713   snat_address_t * ap;
 714   snat_interface_t *i;
 715   vlib_thread_main_t *tm = vlib_get_thread_main ();
 716
 717   if (twice_nat && !sm->endpoint_dependent)
 718     return VNET_API_ERROR_FEATURE_DISABLED;
 719
 720   /* Check if address already exists */
 721   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 722     {
 723       if (ap->addr.as_u32 == addr->as_u32)
 724         return VNET_API_ERROR_VALUE_EXIST;
 725     }
 726
 727   if (twice_nat)
 728     vec_add2 (sm->twice_nat_addresses, ap, 1);
 729   else
 730     vec_add2 (sm->addresses, ap, 1);
 731
 732   ap->addr = *addr;
 733   if (vrf_id != ~0)
 734     ap->fib_index =
 735       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 736                                          FIB_SOURCE_PLUGIN_LOW);
 737   else
 738     ap->fib_index = ~0;
 739 #define _(N, i, n, s) \
 740   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 741   ap->busy_##n##_ports = 0; \
 742   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 743   foreach_snat_protocol
 744 #undef _
 745
 746   if (twice_nat)
 747     return 0;
 748
 749   /* Add external address to FIB */
 750   pool_foreach (i, sm->interfaces,
 751   ({
 752     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 753       continue;
 754
 755     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 756     break;
 757   }));
 758   pool_foreach (i, sm->output_feature_interfaces,
 759   ({
 760     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 761       continue;
 762
 763     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 764     break;
 765   }));
 766
 767   return 0;
 768 }
 769
 770 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 771                                                    ip4_address_t addr)
 772 {
 773   snat_static_mapping_t *m;
 774   pool_foreach (m, sm->static_mappings,
 775   ({
 776       if (m->external_addr.as_u32 == addr.as_u32)
 777         return 1;
 778   }));
 779
 780   return 0;
 781 }
 782
 783 void increment_v4_address (ip4_address_t * a)
 784 {
 785   u32 v;
 786
 787   v = clib_net_to_host_u32(a->as_u32) + 1;
 788   a->as_u32 = clib_host_to_net_u32(v);
 789 }
 790
 791 static void
 792 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 793                                        ip4_address_t l_addr,
 794                                        u16 l_port,
 795                                        u32 sw_if_index,
 796                                        u16 e_port,
 797                                        u32 vrf_id,
 798                                        snat_protocol_t proto,
 799                                        int addr_only,
 800                                        int is_add,
 801                                        u8 * tag)
 802 {
 803   snat_static_map_resolve_t *rp;
 804
 805   vec_add2 (sm->to_resolve, rp, 1);
 806   rp->l_addr.as_u32 = l_addr.as_u32;
 807   rp->l_port = l_port;
 808   rp->sw_if_index = sw_if_index;
 809   rp->e_port = e_port;
 810   rp->vrf_id = vrf_id;
 811   rp->proto = proto;
 812   rp->addr_only = addr_only;
 813   rp->is_add = is_add;
 814   rp->tag = vec_dup (tag);
 815 }
 816
 817 /**
 818  * @brief Add static mapping.
 819  *
 820  * Create static mapping between local addr+port and external addr+port.
 821  *
 822  * @param l_addr Local IPv4 address.
 823  * @param e_addr External IPv4 address.
 824  * @param l_port Local port number.
 825  * @param e_port External port number.
 826  * @param vrf_id VRF ID.
 827  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 828  * @param sw_if_index External port instead of specific IP address.
 829  * @param is_add If 0 delete static mapping, otherwise add.
 830  * @param twice_nat If value is TWICE_NAT then translate external host address
 831  *                  and port.
 832  *                  If value is TWICE_NAT_SELF then translate external host
 833  *                  address and port whenever external host address equals
 834  *                  local address of internal host.
 835  * @param out2in_only If 1 rule match only out2in direction
 836  * @param tag - opaque string tag
 837  *
 838  * @returns
 839  */
 840 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 841                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 842                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 843                             twice_nat_type_t twice_nat, u8 out2in_only,
 844                             u8 * tag)
 845 {
 846   snat_main_t * sm = &snat_main;
 847   snat_static_mapping_t *m;
 848   snat_session_key_t m_key;
 849   clib_bihash_kv_8_8_t kv, value;
 850   snat_address_t *a = 0;
 851   u32 fib_index = ~0;
 852   uword * p;
 853   snat_interface_t *interface;
 854   int i;
 855   snat_main_per_thread_data_t *tsm;
 856   snat_user_key_t u_key;
 857   snat_user_t *u;
 858   dlist_elt_t * head, * elt;
 859   u32 elt_index, head_index;
 860   u32 ses_index;
 861   u64 user_index;
 862   snat_session_t * s;
 863   snat_static_map_resolve_t *rp, *rp_match = 0;
 864
 865   if (!sm->endpoint_dependent)
 866     {
 867       if (twice_nat || out2in_only)
 868         return VNET_API_ERROR_FEATURE_DISABLED;
 869     }
 870
 871   /* If the external address is a specific interface address */
 872   if (sw_if_index != ~0)
 873     {
 874       ip4_address_t * first_int_addr;
 875
 876       for (i = 0; i < vec_len (sm->to_resolve); i++)
 877         {
 878           rp = sm->to_resolve + i;
 879           if (rp->sw_if_index != sw_if_index ||
 880               rp->l_addr.as_u32 != l_addr.as_u32 ||
 881               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 882             continue;
 883
 884           if (!addr_only)
 885             {
 886               if (rp->l_port != l_port || rp->e_port != e_port || rp->proto != proto)
 887                 continue;
 888             }
 889
 890           rp_match = rp;
 891           break;
 892         }
 893
 894       /* Might be already set... */
 895       first_int_addr = ip4_interface_first_address
 896         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 897
 898       if (is_add)
 899         {
 900           if (rp_match)
 901             return VNET_API_ERROR_VALUE_EXIST;
 902
 903           snat_add_static_mapping_when_resolved
 904             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 905              addr_only,  is_add, tag);
 906
 907           /* DHCP resolution required? */
 908           if (first_int_addr == 0)
 909             {
 910               return 0;
 911             }
 912           else
 913             {
 914               e_addr.as_u32 = first_int_addr->as_u32;
 915               /* Identity mapping? */
 916               if (l_addr.as_u32 == 0)
 917                 l_addr.as_u32 = e_addr.as_u32;
 918             }
 919         }
 920       else
 921         {
 922           if (!rp_match)
 923             return VNET_API_ERROR_NO_SUCH_ENTRY;
 924
 925           vec_del1 (sm->to_resolve, i);
 926
 927           if (first_int_addr)
 928             {
 929               e_addr.as_u32 = first_int_addr->as_u32;
 930               /* Identity mapping? */
 931               if (l_addr.as_u32 == 0)
 932                 l_addr.as_u32 = e_addr.as_u32;
 933             }
 934           else
 935             return 0;
 936         }
 937     }
 938
 939   m_key.addr = e_addr;
 940   m_key.port = addr_only ? 0 : e_port;
 941   m_key.protocol = addr_only ? 0 : proto;
 942   m_key.fib_index = 0;
 943   kv.key = m_key.as_u64;
 944   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 945     m = 0;
 946   else
 947     m = pool_elt_at_index (sm->static_mappings, value.value);
 948
 949   if (is_add)
 950     {
 951       if (m)
 952         return VNET_API_ERROR_VALUE_EXIST;
 953
 954       if (twice_nat && addr_only)
 955         return VNET_API_ERROR_UNSUPPORTED;
 956
 957       /* Convert VRF id to FIB index */
 958       if (vrf_id != ~0)
 959         {
 960           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 961           if (!p)
 962             return VNET_API_ERROR_NO_SUCH_FIB;
 963           fib_index = p[0];
 964         }
 965       /* If not specified use inside VRF id from SNAT plugin startup config */
 966       else
 967         {
 968           fib_index = sm->inside_fib_index;
 969           vrf_id = sm->inside_vrf_id;
 970         }
 971
 972       if (!out2in_only)
 973         {
 974           m_key.addr = l_addr;
 975           m_key.port = addr_only ? 0 : l_port;
 976           m_key.protocol = addr_only ? 0 : proto;
 977           m_key.fib_index = fib_index;
 978           kv.key = m_key.as_u64;
 979           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 980             return VNET_API_ERROR_VALUE_EXIST;
 981         }
 982
 983       /* Find external address in allocated addresses and reserve port for
 984          address and port pair mapping when dynamic translations enabled */
 985       if (!(addr_only || sm->static_mapping_only || out2in_only))
 986         {
 987           for (i = 0; i < vec_len (sm->addresses); i++)
 988             {
 989               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 990                 {
 991                   a = sm->addresses + i;
 992                   /* External port must be unused */
 993                   switch (proto)
 994                     {
 995 #define _(N, j, n, s) \
 996                     case SNAT_PROTOCOL_##N: \
 997                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 998                         return VNET_API_ERROR_INVALID_VALUE; \
 999                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1000                       if (e_port > 1024) \
1001                         { \
1002                           a->busy_##n##_ports++; \
1003                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1004                         } \
1005                       break;
1006                       foreach_snat_protocol
1007 #undef _
1008                     default:
1009                       nat_log_info ("unknown protocol");
1010                       return VNET_API_ERROR_INVALID_VALUE_2;
1011                     }
1012                   break;
1013                 }
1014             }
1015           /* External address must be allocated */
1016           if (!a && (l_addr.as_u32 != e_addr.as_u32))
1017             {
1018               if (sw_if_index != ~0)
1019                 {
1020                   for (i = 0; i < vec_len (sm->to_resolve); i++)
1021                     {
1022                       rp = sm->to_resolve + i;
1023                       if (rp->addr_only)
1024                          continue;
1025                       if (rp->sw_if_index != sw_if_index &&
1026                           rp->l_addr.as_u32 != l_addr.as_u32 &&
1027                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
1028                           rp->e_port != e_port && rp->proto != proto)
1029                         continue;
1030
1031                       vec_del1 (sm->to_resolve, i);
1032                       break;
1033                     }
1034                 }
1035               return VNET_API_ERROR_NO_SUCH_ENTRY;
1036             }
1037         }
1038
1039       pool_get (sm->static_mappings, m);
1040       memset (m, 0, sizeof (*m));
1041       m->tag = vec_dup (tag);
1042       m->local_addr = l_addr;
1043       m->external_addr = e_addr;
1044       m->addr_only = addr_only;
1045       m->vrf_id = vrf_id;
1046       m->fib_index = fib_index;
1047       m->twice_nat = twice_nat;
1048       m->out2in_only = out2in_only;
1049       if (!addr_only)
1050         {
1051           m->local_port = l_port;
1052           m->external_port = e_port;
1053           m->proto = proto;
1054         }
1055
1056       if (sm->num_workers > 1)
1057         {
1058           ip4_header_t ip = {
1059             .src_address = m->local_addr,
1060           };
1061           vec_add1 (m->workers, sm->worker_in2out_cb (&ip, m->fib_index));
1062           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1063         }
1064       else
1065         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1066
1067       m_key.addr = m->local_addr;
1068       m_key.port = m->local_port;
1069       m_key.protocol = m->proto;
1070       m_key.fib_index = m->fib_index;
1071       kv.key = m_key.as_u64;
1072       kv.value = m - sm->static_mappings;
1073       if (!out2in_only)
1074         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1075
1076       m_key.addr = m->external_addr;
1077       m_key.port = m->external_port;
1078       m_key.fib_index = 0;
1079       kv.key = m_key.as_u64;
1080       kv.value = m - sm->static_mappings;
1081       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
1082
1083       /* Delete dynamic sessions matching local address (+ local port) */
1084       if (!(sm->static_mapping_only))
1085         {
1086           u_key.addr = m->local_addr;
1087           u_key.fib_index = m->fib_index;
1088           kv.key = u_key.as_u64;
1089           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1090             {
1091               user_index = value.value;
1092               u = pool_elt_at_index (tsm->users, user_index);
1093               if (u->nsessions)
1094                 {
1095                   head_index = u->sessions_per_user_list_head_index;
1096                   head = pool_elt_at_index (tsm->list_pool, head_index);
1097                   elt_index = head->next;
1098                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1099                   ses_index = elt->value;
1100                   while (ses_index != ~0)
1101                     {
1102                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1103                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1104                       ses_index = elt->value;
1105
1106                       if (snat_is_session_static (s))
1107                         continue;
1108
1109                       if (!addr_only && (clib_net_to_host_u16 (s->in2out.port) != m->local_port))
1110                         continue;
1111
1112                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1113                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1114
1115                       if (!addr_only && !sm->endpoint_dependent)
1116                         break;
1117                     }
1118                 }
1119             }
1120         }
1121     }
1122   else
1123     {
1124       if (!m)
1125         {
1126           if (sw_if_index != ~0)
1127             return 0;
1128           else
1129             return VNET_API_ERROR_NO_SUCH_ENTRY;
1130         }
1131
1132       /* Free external address port */
1133       if (!(addr_only || sm->static_mapping_only || out2in_only))
1134         {
1135           for (i = 0; i < vec_len (sm->addresses); i++)
1136             {
1137               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1138                 {
1139                   a = sm->addresses + i;
1140                   switch (proto)
1141                     {
1142 #define _(N, j, n, s) \
1143                     case SNAT_PROTOCOL_##N: \
1144                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1145                       if (e_port > 1024) \
1146                         { \
1147                           a->busy_##n##_ports--; \
1148                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1149                         } \
1150                       break;
1151                       foreach_snat_protocol
1152 #undef _
1153                     default:
1154                       nat_log_info ("unknown protocol");
1155                       return VNET_API_ERROR_INVALID_VALUE_2;
1156                     }
1157                   break;
1158                 }
1159             }
1160         }
1161
1162       if (sm->num_workers > 1)
1163         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1164       else
1165         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1166
1167       m_key.addr = m->local_addr;
1168       m_key.port = m->local_port;
1169       m_key.protocol = m->proto;
1170       m_key.fib_index = m->fib_index;
1171       kv.key = m_key.as_u64;
1172       if (!out2in_only)
1173         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1174
1175       m_key.addr = m->external_addr;
1176       m_key.port = m->external_port;
1177       m_key.fib_index = 0;
1178       kv.key = m_key.as_u64;
1179       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1180
1181       /* Delete session(s) for static mapping if exist */
1182       if (!(sm->static_mapping_only) ||
1183           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1184         {
1185           u_key.addr = m->local_addr;
1186           u_key.fib_index = m->fib_index;
1187           kv.key = u_key.as_u64;
1188           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1189             {
1190               user_index = value.value;
1191               u = pool_elt_at_index (tsm->users, user_index);
1192               if (u->nstaticsessions)
1193                 {
1194                   head_index = u->sessions_per_user_list_head_index;
1195                   head = pool_elt_at_index (tsm->list_pool, head_index);
1196                   elt_index = head->next;
1197                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1198                   ses_index = elt->value;
1199                   while (ses_index != ~0)
1200                     {
1201                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1202                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1203                       ses_index = elt->value;
1204
1205                       if (!addr_only)
1206                         {
1207                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
1208                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1209                             continue;
1210                         }
1211
1212                       if (is_lb_session (s))
1213                         continue;
1214
1215                       if (!snat_is_session_static (s))
1216                         continue;
1217
1218                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1219                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1220
1221                       if (!addr_only && !sm->endpoint_dependent)
1222                         break;
1223                     }
1224                 }
1225             }
1226         }
1227
1228       vec_free (m->tag);
1229       vec_free (m->workers);
1230       /* Delete static mapping from pool */
1231       pool_put (sm->static_mappings, m);
1232     }
1233
1234   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1235     return 0;
1236
1237   /* Add/delete external address to FIB */
1238   pool_foreach (interface, sm->interfaces,
1239   ({
1240     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1241       continue;
1242
1243     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1244     break;
1245   }));
1246   pool_foreach (interface, sm->output_feature_interfaces,
1247   ({
1248     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1249       continue;
1250
1251     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1252     break;
1253   }));
1254
1255   return 0;
1256 }
1257
1258 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1259                                      snat_protocol_t proto,
1260                                      nat44_lb_addr_port_t *locals, u8 is_add,
1261                                      twice_nat_type_t twice_nat, u8 out2in_only,
1262                                      u8 *tag)
1263 {
1264   snat_main_t * sm = &snat_main;
1265   snat_static_mapping_t *m;
1266   snat_session_key_t m_key;
1267   clib_bihash_kv_8_8_t kv, value;
1268   snat_address_t *a = 0;
1269   int i;
1270   nat44_lb_addr_port_t *local;
1271   u32 elt_index, head_index, ses_index;
1272   snat_main_per_thread_data_t *tsm;
1273   snat_user_key_t u_key;
1274   snat_user_t *u;
1275   snat_session_t * s;
1276   dlist_elt_t * head, * elt;
1277   uword *bitmap = 0;
1278
1279   if (!sm->endpoint_dependent)
1280     return VNET_API_ERROR_FEATURE_DISABLED;
1281
1282   m_key.addr = e_addr;
1283   m_key.port = e_port;
1284   m_key.protocol = proto;
1285   m_key.fib_index = 0;
1286   kv.key = m_key.as_u64;
1287   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1288     m = 0;
1289   else
1290     m = pool_elt_at_index (sm->static_mappings, value.value);
1291
1292   if (is_add)
1293     {
1294       if (m)
1295         return VNET_API_ERROR_VALUE_EXIST;
1296
1297       if (vec_len (locals) < 2)
1298         return VNET_API_ERROR_INVALID_VALUE;
1299
1300       /* Find external address in allocated addresses and reserve port for
1301          address and port pair mapping when dynamic translations enabled */
1302       if (!(sm->static_mapping_only || out2in_only))
1303         {
1304           for (i = 0; i < vec_len (sm->addresses); i++)
1305             {
1306               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1307                 {
1308                   a = sm->addresses + i;
1309                   /* External port must be unused */
1310                   switch (proto)
1311                     {
1312 #define _(N, j, n, s) \
1313                     case SNAT_PROTOCOL_##N: \
1314                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1315                         return VNET_API_ERROR_INVALID_VALUE; \
1316                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1317                       if (e_port > 1024) \
1318                         { \
1319                           a->busy_##n##_ports++; \
1320                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1321                         } \
1322                       break;
1323                       foreach_snat_protocol
1324 #undef _
1325                     default:
1326                       nat_log_info ("unknown protocol");
1327                       return VNET_API_ERROR_INVALID_VALUE_2;
1328                     }
1329                   break;
1330                 }
1331             }
1332           /* External address must be allocated */
1333           if (!a)
1334             return VNET_API_ERROR_NO_SUCH_ENTRY;
1335         }
1336
1337       pool_get (sm->static_mappings, m);
1338       memset (m, 0, sizeof (*m));
1339       m->tag = vec_dup (tag);
1340       m->external_addr = e_addr;
1341       m->addr_only = 0;
1342       m->external_port = e_port;
1343       m->proto = proto;
1344       m->twice_nat = twice_nat;
1345       m->out2in_only = out2in_only;
1346
1347       m_key.addr = m->external_addr;
1348       m_key.port = m->external_port;
1349       m_key.protocol = m->proto;
1350       m_key.fib_index = 0;
1351       kv.key = m_key.as_u64;
1352       kv.value = m - sm->static_mappings;
1353       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1354         {
1355           nat_log_err ("static_mapping_by_external key add failed");
1356           return VNET_API_ERROR_UNSPECIFIED;
1357         }
1358
1359       m_key.fib_index = m->fib_index;
1360       for (i = 0; i < vec_len (locals); i++)
1361         {
1362           locals[i].fib_index = fib_table_find_or_create_and_lock (
1363             FIB_PROTOCOL_IP4, locals[i].vrf_id, FIB_SOURCE_PLUGIN_LOW);
1364           m_key.addr = locals[i].addr;
1365           m_key.fib_index = locals[i].fib_index;
1366           if (!out2in_only)
1367             {
1368               m_key.port = locals[i].port;
1369               kv.key = m_key.as_u64;
1370               kv.value = m - sm->static_mappings;
1371               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1372             }
1373           locals[i].prefix = (i == 0) ? locals[i].probability :\
1374             (locals[i - 1].prefix + locals[i].probability);
1375           vec_add1 (m->locals, locals[i]);
1376           if (sm->num_workers > 1)
1377             {
1378               ip4_header_t ip = {
1379                 .src_address = locals[i].addr,
1380               };
1381               bitmap = clib_bitmap_set (
1382                 bitmap, sm->worker_in2out_cb (&ip, m->fib_index), 1);
1383             }
1384         }
1385
1386       /* Assign workers */
1387       if (sm->num_workers > 1)
1388         {
1389           clib_bitmap_foreach (i, bitmap,
1390             ({
1391                vec_add1(m->workers, i);
1392             }));
1393         }
1394     }
1395   else
1396     {
1397       if (!m)
1398         return VNET_API_ERROR_NO_SUCH_ENTRY;
1399
1400       /* Free external address port */
1401       if (!(sm->static_mapping_only || out2in_only))
1402         {
1403           for (i = 0; i < vec_len (sm->addresses); i++)
1404             {
1405               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1406                 {
1407                   a = sm->addresses + i;
1408                   switch (proto)
1409                     {
1410 #define _(N, j, n, s) \
1411                     case SNAT_PROTOCOL_##N: \
1412                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1413                       if (e_port > 1024) \
1414                         { \
1415                           a->busy_##n##_ports--; \
1416                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1417                         } \
1418                       break;
1419                       foreach_snat_protocol
1420 #undef _
1421                     default:
1422                       nat_log_info ("unknown protocol");
1423                       return VNET_API_ERROR_INVALID_VALUE_2;
1424                     }
1425                   break;
1426                 }
1427             }
1428         }
1429
1430       m_key.addr = m->external_addr;
1431       m_key.port = m->external_port;
1432       m_key.protocol = m->proto;
1433       m_key.fib_index = 0;
1434       kv.key = m_key.as_u64;
1435       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1436         {
1437           nat_log_err ("static_mapping_by_external key del failed");
1438           return VNET_API_ERROR_UNSPECIFIED;
1439         }
1440
1441       vec_foreach (local, m->locals)
1442         {
1443           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1444                             FIB_SOURCE_PLUGIN_LOW);
1445           m_key.addr = local->addr;
1446           if (!out2in_only)
1447             {
1448               m_key.port = local->port;
1449               m_key.fib_index = local->fib_index;
1450               kv.key = m_key.as_u64;
1451               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1452                 {
1453                   nat_log_err ("static_mapping_by_local key del failed");
1454                   return VNET_API_ERROR_UNSPECIFIED;
1455                 }
1456             }
1457
1458           if (sm->num_workers > 1)
1459             {
1460               ip4_header_t ip = {
1461                 .src_address = local->addr,
1462               };
1463               tsm = vec_elt_at_index (sm->per_thread_data,
1464                                       sm->worker_in2out_cb (&ip, m->fib_index));
1465             }
1466           else
1467             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1468
1469           /* Delete sessions */
1470           u_key.addr = local->addr;
1471           u_key.fib_index = m->fib_index;
1472           kv.key = u_key.as_u64;
1473           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1474             {
1475               u = pool_elt_at_index (tsm->users, value.value);
1476               if (u->nstaticsessions)
1477                 {
1478                   head_index = u->sessions_per_user_list_head_index;
1479                   head = pool_elt_at_index (tsm->list_pool, head_index);
1480                   elt_index = head->next;
1481                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1482                   ses_index = elt->value;
1483                   while (ses_index != ~0)
1484                     {
1485                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1486                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1487                       ses_index = elt->value;
1488
1489                       if (!(is_lb_session (s)))
1490                         continue;
1491
1492                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1493                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1494                         continue;
1495
1496                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1497                       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
1498                     }
1499                 }
1500             }
1501         }
1502       vec_free(m->locals);
1503       vec_free(m->tag);
1504       vec_free(m->workers);
1505
1506       pool_put (sm->static_mappings, m);
1507     }
1508
1509   return 0;
1510 }
1511
1512 int
1513 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1514                   u8 twice_nat)
1515 {
1516   snat_address_t *a = 0;
1517   snat_session_t *ses;
1518   u32 *ses_to_be_removed = 0, *ses_index;
1519   snat_main_per_thread_data_t *tsm;
1520   snat_static_mapping_t *m;
1521   snat_interface_t *interface;
1522   int i;
1523   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1524
1525   /* Find SNAT address */
1526   for (i=0; i < vec_len (addresses); i++)
1527     {
1528       if (addresses[i].addr.as_u32 == addr.as_u32)
1529         {
1530           a = addresses + i;
1531           break;
1532         }
1533     }
1534   if (!a)
1535     return VNET_API_ERROR_NO_SUCH_ENTRY;
1536
1537   if (delete_sm)
1538     {
1539       pool_foreach (m, sm->static_mappings,
1540       ({
1541           if (m->external_addr.as_u32 == addr.as_u32)
1542             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1543                                             m->local_port, m->external_port,
1544                                             m->vrf_id, m->addr_only, ~0,
1545                                             m->proto, 0, m->twice_nat,
1546                                             m->out2in_only, m->tag);
1547       }));
1548     }
1549   else
1550     {
1551       /* Check if address is used in some static mapping */
1552       if (is_snat_address_used_in_static_mapping(sm, addr))
1553         {
1554           nat_log_notice ("address used in static mapping");
1555           return VNET_API_ERROR_UNSPECIFIED;
1556         }
1557     }
1558
1559   if (a->fib_index != ~0)
1560     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1561                      FIB_SOURCE_PLUGIN_LOW);
1562
1563   /* Delete sessions using address */
1564   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1565     {
1566       vec_foreach (tsm, sm->per_thread_data)
1567         {
1568           pool_foreach (ses, tsm->sessions, ({
1569             if (ses->out2in.addr.as_u32 == addr.as_u32)
1570               {
1571                 ses->outside_address_index = ~0;
1572                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1573                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1574               }
1575           }));
1576
1577           vec_foreach (ses_index, ses_to_be_removed)
1578             {
1579               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1580               nat44_delete_session (sm, ses, tsm - sm->per_thread_data);
1581             }
1582
1583           vec_free (ses_to_be_removed);
1584        }
1585     }
1586
1587   if (twice_nat)
1588     {
1589       vec_del1 (sm->twice_nat_addresses, i);
1590       return 0;
1591     }
1592   else
1593     vec_del1 (sm->addresses, i);
1594
1595   /* Delete external address from FIB */
1596   pool_foreach (interface, sm->interfaces,
1597   ({
1598     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1599       continue;
1600
1601     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1602     break;
1603   }));
1604   pool_foreach (interface, sm->output_feature_interfaces,
1605   ({
1606     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1607       continue;
1608
1609     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1610     break;
1611   }));
1612
1613   return 0;
1614 }
1615
1616 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1617 {
1618   snat_main_t *sm = &snat_main;
1619   snat_interface_t *i;
1620   const char * feature_name, *del_feature_name;
1621   snat_address_t * ap;
1622   snat_static_mapping_t * m;
1623   snat_det_map_t * dm;
1624   nat_outside_fib_t *outside_fib;
1625   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1626                                                        sw_if_index);
1627
1628   if (sm->out2in_dpo && !is_inside)
1629     return VNET_API_ERROR_UNSUPPORTED;
1630
1631   pool_foreach (i, sm->output_feature_interfaces,
1632   ({
1633     if (i->sw_if_index == sw_if_index)
1634       return VNET_API_ERROR_VALUE_EXIST;
1635   }));
1636
1637   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1638     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1639   else
1640     {
1641       if (sm->num_workers > 1 && !sm->deterministic)
1642         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1643       else if (sm->deterministic)
1644         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1645       else if (sm->endpoint_dependent)
1646         feature_name = is_inside ?  "nat44-ed-in2out" : "nat44-ed-out2in";
1647       else
1648         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1649     }
1650
1651   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1652     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1653                                                       NAT_FQ_NELTS);
1654
1655   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1656     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1657                                                       NAT_FQ_NELTS);
1658
1659   if (!is_inside)
1660     {
1661       vec_foreach (outside_fib, sm->outside_fibs)
1662         {
1663           if (outside_fib->fib_index == fib_index)
1664             {
1665               if (is_del)
1666                 {
1667                   outside_fib->refcount--;
1668                   if (!outside_fib->refcount)
1669                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1670                 }
1671               else
1672                 outside_fib->refcount++;
1673               goto feature_set;
1674             }
1675         }
1676       if (!is_del)
1677         {
1678           vec_add2 (sm->outside_fibs, outside_fib, 1);
1679           outside_fib->refcount = 1;
1680           outside_fib->fib_index = fib_index;
1681         }
1682     }
1683 feature_set:
1684   pool_foreach (i, sm->interfaces,
1685   ({
1686     if (i->sw_if_index == sw_if_index)
1687       {
1688         if (is_del)
1689           {
1690             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1691               {
1692                 if (is_inside)
1693                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1694                 else
1695                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1696
1697                 if (sm->num_workers > 1 && !sm->deterministic)
1698                   {
1699                     del_feature_name = "nat44-handoff-classify";
1700                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1701                                                  "nat44-out2in-worker-handoff";
1702                   }
1703                 else if (sm->deterministic)
1704                   {
1705                     del_feature_name = "nat44-det-classify";
1706                     feature_name = !is_inside ?  "nat44-det-in2out" :
1707                                                  "nat44-det-out2in";
1708                   }
1709                 else if (sm->endpoint_dependent)
1710                   {
1711                     del_feature_name = "nat44-ed-classify";
1712                     feature_name = !is_inside ?  "nat44-ed-in2out" :
1713                                                  "nat44-ed-out2in";
1714                   }
1715                 else
1716                   {
1717                     del_feature_name = "nat44-classify";
1718                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1719                   }
1720
1721                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1722                                              sw_if_index, 0, 0, 0);
1723                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1724                                              sw_if_index, 1, 0, 0);
1725                 if (!is_inside)
1726                   {
1727                     if (sm->endpoint_dependent)
1728                       vnet_feature_enable_disable ("ip4-local",
1729                                                    "nat44-ed-hairpinning",
1730                                                    sw_if_index, 1, 0, 0);
1731                     else if (!sm->deterministic)
1732                       vnet_feature_enable_disable ("ip4-local",
1733                                                    "nat44-hairpinning",
1734                                                    sw_if_index, 1, 0, 0);
1735                   }
1736               }
1737             else
1738               {
1739                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1740                                              sw_if_index, 0, 0, 0);
1741                 pool_put (sm->interfaces, i);
1742                 if (is_inside)
1743                   {
1744                     if (sm->endpoint_dependent)
1745                       vnet_feature_enable_disable ("ip4-local",
1746                                                    "nat44-ed-hairpinning",
1747                                                    sw_if_index, 0, 0, 0);
1748                     else if (!sm->deterministic)
1749                       vnet_feature_enable_disable ("ip4-local",
1750                                                    "nat44-hairpinning",
1751                                                    sw_if_index, 0, 0, 0);
1752                   }
1753               }
1754           }
1755         else
1756           {
1757             if ((nat_interface_is_inside(i) && is_inside) ||
1758                 (nat_interface_is_outside(i) && !is_inside))
1759               return 0;
1760
1761             if (sm->num_workers > 1 && !sm->deterministic)
1762               {
1763                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1764                                                  "nat44-out2in-worker-handoff";
1765                 feature_name = "nat44-handoff-classify";
1766               }
1767             else if (sm->deterministic)
1768               {
1769                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1770                                                  "nat44-det-out2in";
1771                 feature_name = "nat44-det-classify";
1772               }
1773             else if (sm->endpoint_dependent)
1774               {
1775                 del_feature_name = !is_inside ?  "nat44-ed-in2out" :
1776                                                  "nat44-ed-out2in";
1777                 feature_name = "nat44-ed-classify";
1778               }
1779             else
1780               {
1781                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1782                 feature_name = "nat44-classify";
1783               }
1784
1785             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1786                                          sw_if_index, 0, 0, 0);
1787             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1788                                          sw_if_index, 1, 0, 0);
1789             if (!is_inside)
1790               {
1791                 if (sm->endpoint_dependent)
1792                   vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1793                                                sw_if_index, 0, 0, 0);
1794                 else if (!sm->deterministic)
1795                   vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1796                                                sw_if_index, 0, 0, 0);
1797               }
1798             goto set_flags;
1799           }
1800
1801         goto fib;
1802       }
1803   }));
1804
1805   if (is_del)
1806     return VNET_API_ERROR_NO_SUCH_ENTRY;
1807
1808   pool_get (sm->interfaces, i);
1809   i->sw_if_index = sw_if_index;
1810   i->flags = 0;
1811   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1812
1813   if (is_inside && !sm->out2in_dpo)
1814     {
1815       if (sm->endpoint_dependent)
1816         vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
1817                                      sw_if_index, 1, 0, 0);
1818       else if (!sm->deterministic)
1819         vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1820                                      sw_if_index, 1, 0, 0);
1821     }
1822
1823 set_flags:
1824   if (is_inside)
1825     {
1826       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1827       return 0;
1828     }
1829   else
1830     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1831
1832   /* Add/delete external addresses to FIB */
1833 fib:
1834   vec_foreach (ap, sm->addresses)
1835     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1836
1837   pool_foreach (m, sm->static_mappings,
1838   ({
1839     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1840       continue;
1841
1842     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1843   }));
1844
1845   pool_foreach (dm, sm->det_maps,
1846   ({
1847     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1848   }));
1849
1850   return 0;
1851 }
1852
1853 int snat_interface_add_del_output_feature (u32 sw_if_index,
1854                                            u8 is_inside,
1855                                            int is_del)
1856 {
1857   snat_main_t *sm = &snat_main;
1858   snat_interface_t *i;
1859   snat_address_t * ap;
1860   snat_static_mapping_t * m;
1861
1862   if (sm->deterministic ||
1863       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1864     return VNET_API_ERROR_UNSUPPORTED;
1865
1866   pool_foreach (i, sm->interfaces,
1867   ({
1868     if (i->sw_if_index == sw_if_index)
1869       return VNET_API_ERROR_VALUE_EXIST;
1870   }));
1871
1872   if (is_inside)
1873     {
1874       if (sm->endpoint_dependent)
1875         {
1876           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
1877                                        sw_if_index, !is_del, 0, 0);
1878           vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
1879                                        sw_if_index, !is_del, 0, 0);
1880         }
1881       else
1882         {
1883           vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1884                                        sw_if_index, !is_del, 0, 0);
1885           vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1886                                        sw_if_index, !is_del, 0, 0);
1887         }
1888       goto fq;
1889     }
1890
1891   if (sm->num_workers > 1)
1892     {
1893       vnet_feature_enable_disable ("ip4-unicast",
1894                                    "nat44-out2in-worker-handoff",
1895                                    sw_if_index, !is_del, 0, 0);
1896       vnet_feature_enable_disable ("ip4-output",
1897                                    "nat44-in2out-output-worker-handoff",
1898                                    sw_if_index, !is_del, 0, 0);
1899     }
1900   else
1901     {
1902       if (sm->endpoint_dependent)
1903         {
1904           vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
1905                                        sw_if_index, !is_del, 0, 0);
1906           vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
1907                                        sw_if_index, !is_del, 0, 0);
1908         }
1909       else
1910         {
1911           vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in",
1912                                        sw_if_index, !is_del, 0, 0);
1913           vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1914                                        sw_if_index, !is_del, 0, 0);
1915         }
1916     }
1917
1918 fq:
1919   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1920     sm->fq_in2out_output_index =
1921       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1922
1923   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1924     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1925
1926   pool_foreach (i, sm->output_feature_interfaces,
1927   ({
1928     if (i->sw_if_index == sw_if_index)
1929       {
1930         if (is_del)
1931           pool_put (sm->output_feature_interfaces, i);
1932         else
1933           return VNET_API_ERROR_VALUE_EXIST;
1934
1935         goto fib;
1936       }
1937   }));
1938
1939   if (is_del)
1940     return VNET_API_ERROR_NO_SUCH_ENTRY;
1941
1942   pool_get (sm->output_feature_interfaces, i);
1943   i->sw_if_index = sw_if_index;
1944   i->flags = 0;
1945   if (is_inside)
1946     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1947   else
1948     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1949
1950   /* Add/delete external addresses to FIB */
1951 fib:
1952   if (is_inside)
1953     return 0;
1954
1955   vec_foreach (ap, sm->addresses)
1956     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1957
1958   pool_foreach (m, sm->static_mappings,
1959   ({
1960     if (!(m->addr_only)  || (m->local_addr.as_u32 == m->external_addr.as_u32))
1961       continue;
1962
1963     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1964   }));
1965
1966   return 0;
1967 }
1968
1969 int snat_set_workers (uword * bitmap)
1970 {
1971   snat_main_t *sm = &snat_main;
1972   int i, j = 0;
1973
1974   if (sm->num_workers < 2)
1975     return VNET_API_ERROR_FEATURE_DISABLED;
1976
1977   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1978     return VNET_API_ERROR_INVALID_WORKER;
1979
1980   vec_free (sm->workers);
1981   clib_bitmap_foreach (i, bitmap,
1982     ({
1983       vec_add1(sm->workers, i);
1984       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1985       j++;
1986     }));
1987
1988   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
1989   sm->num_snat_thread = _vec_len (sm->workers);
1990
1991   return 0;
1992 }
1993
1994
1995 static void
1996 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
1997                                        uword opaque,
1998                                        u32 sw_if_index,
1999                                        ip4_address_t * address,
2000                                        u32 address_length,
2001                                        u32 if_address_index,
2002                                        u32 is_delete);
2003
2004 static void
2005 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2006                                  uword opaque,
2007                                  u32 sw_if_index,
2008                                  ip4_address_t * address,
2009                                  u32 address_length,
2010                                  u32 if_address_index,
2011                                  u32 is_delete);
2012
2013 static int
2014 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2015                                  u32 fib_index,
2016                                  u32 thread_index,
2017                                  snat_session_key_t * k,
2018                                  u32 * address_indexp,
2019                                  u16 port_per_thread,
2020                                  u32 snat_thread_index);
2021
2022 static clib_error_t * snat_init (vlib_main_t * vm)
2023 {
2024   snat_main_t * sm = &snat_main;
2025   clib_error_t * error = 0;
2026   ip4_main_t * im = &ip4_main;
2027   ip_lookup_main_t * lm = &im->lookup_main;
2028   uword *p;
2029   vlib_thread_registration_t *tr;
2030   vlib_thread_main_t *tm = vlib_get_thread_main ();
2031   uword *bitmap = 0;
2032   u32 i;
2033   ip4_add_del_interface_address_callback_t cb4;
2034   vlib_node_t * error_drop_node;
2035
2036   sm->vlib_main = vm;
2037   sm->vnet_main = vnet_get_main();
2038   sm->ip4_main = im;
2039   sm->ip4_lookup_main = lm;
2040   sm->api_main = &api_main;
2041   sm->first_worker_index = 0;
2042   sm->num_workers = 0;
2043   sm->num_snat_thread = 1;
2044   sm->workers = 0;
2045   sm->port_per_thread = 0xffff - 1024;
2046   sm->fq_in2out_index = ~0;
2047   sm->fq_out2in_index = ~0;
2048   sm->udp_timeout = SNAT_UDP_TIMEOUT;
2049   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
2050   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
2051   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
2052   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
2053   sm->forwarding_enabled = 0;
2054   sm->log_class = vlib_log_register_class ("nat", 0);
2055   error_drop_node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
2056   sm->error_node_index = error_drop_node->index;
2057
2058   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2059   if (p)
2060     {
2061       tr = (vlib_thread_registration_t *) p[0];
2062       if (tr)
2063         {
2064           sm->num_workers = tr->count;
2065           sm->first_worker_index = tr->first_index;
2066         }
2067     }
2068
2069   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
2070
2071   /* Use all available workers by default */
2072   if (sm->num_workers > 1)
2073     {
2074       for (i=0; i < sm->num_workers; i++)
2075         bitmap = clib_bitmap_set (bitmap, i, 1);
2076       snat_set_workers(bitmap);
2077       clib_bitmap_free (bitmap);
2078     }
2079   else
2080     {
2081       sm->per_thread_data[0].snat_thread_index = 0;
2082     }
2083
2084   error = snat_api_init(vm, sm);
2085   if (error)
2086     return error;
2087
2088   /* Set up the interface address add/del callback */
2089   cb4.function = snat_ip4_add_del_interface_address_cb;
2090   cb4.function_opaque = 0;
2091
2092   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2093
2094   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
2095   cb4.function_opaque = 0;
2096
2097   vec_add1 (im->add_del_interface_address_callbacks, cb4);
2098
2099   nat_dpo_module_init ();
2100
2101   /* Init IPFIX logging */
2102   snat_ipfix_logging_init(vm);
2103
2104   /* Init NAT64 */
2105   error = nat64_init(vm);
2106   if (error)
2107     return error;
2108
2109   dslite_init(vm);
2110
2111   nat66_init();
2112
2113   /* Init virtual fragmenentation reassembly */
2114   return nat_reass_init(vm);
2115 }
2116
2117 VLIB_INIT_FUNCTION (snat_init);
2118
2119 void snat_free_outside_address_and_port (snat_address_t * addresses,
2120                                          u32 thread_index,
2121                                          snat_session_key_t * k)
2122 {
2123   snat_address_t *a;
2124   u32 address_index;
2125   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
2126
2127   for (address_index = 0; address_index < vec_len (addresses); address_index++)
2128     {
2129       if (addresses[address_index].addr.as_u32 == k->addr.as_u32)
2130         break;
2131     }
2132
2133   ASSERT (address_index < vec_len (addresses));
2134
2135   a = addresses + address_index;
2136
2137   switch (k->protocol)
2138     {
2139 #define _(N, i, n, s) \
2140     case SNAT_PROTOCOL_##N: \
2141       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
2142         port_host_byte_order) == 1); \
2143       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
2144         port_host_byte_order, 0); \
2145       a->busy_##n##_ports--; \
2146       a->busy_##n##_ports_per_thread[thread_index]--; \
2147       break;
2148       foreach_snat_protocol
2149 #undef _
2150     default:
2151       nat_log_info ("unknown protocol");
2152       return;
2153     }
2154 }
2155
2156 /**
2157  * @brief Match NAT44 static mapping.
2158  *
2159  * @param sm          NAT main.
2160  * @param match       Address and port to match.
2161  * @param mapping     External or local address and port of the matched mapping.
2162  * @param by_external If 0 match by local address otherwise match by external
2163  *                    address.
2164  * @param is_addr_only If matched mapping is address only
2165  * @param twice_nat If matched mapping is twice NAT.
2166  * @param lb If matched mapping is load-balanced.
2167  *
2168  * @returns 0 if match found otherwise 1.
2169  */
2170 int snat_static_mapping_match (snat_main_t * sm,
2171                                snat_session_key_t match,
2172                                snat_session_key_t * mapping,
2173                                u8 by_external,
2174                                u8 *is_addr_only,
2175                                twice_nat_type_t *twice_nat,
2176                                u8 *lb)
2177 {
2178   clib_bihash_kv_8_8_t kv, value;
2179   snat_static_mapping_t *m;
2180   snat_session_key_t m_key;
2181   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2182   u32 rand, lo = 0, hi, mid;
2183
2184   m_key.fib_index = match.fib_index;
2185   if (by_external)
2186     {
2187       mapping_hash = &sm->static_mapping_by_external;
2188       m_key.fib_index = 0;
2189     }
2190
2191   m_key.addr = match.addr;
2192   m_key.port = clib_net_to_host_u16 (match.port);
2193   m_key.protocol = match.protocol;
2194
2195   kv.key = m_key.as_u64;
2196
2197   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2198     {
2199       /* Try address only mapping */
2200       m_key.port = 0;
2201       m_key.protocol = 0;
2202       kv.key = m_key.as_u64;
2203       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2204         return 1;
2205     }
2206
2207   m = pool_elt_at_index (sm->static_mappings, value.value);
2208
2209   if (by_external)
2210     {
2211       if (vec_len (m->locals))
2212         {
2213 get_local:
2214           hi = vec_len (m->locals) - 1;
2215           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2216           while (lo < hi)
2217             {
2218               mid = ((hi - lo) >> 1) + lo;
2219               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2220             }
2221           if (!(m->locals[lo].prefix >= rand))
2222             return 1;
2223           if (PREDICT_FALSE (sm->num_workers > 1))
2224             {
2225               ip4_header_t ip = {
2226                 .src_address = m->locals[lo].addr,
2227               };
2228               if (sm->worker_in2out_cb (&ip, m->fib_index) != vlib_get_thread_index ())
2229                 goto get_local;
2230             }
2231           mapping->addr = m->locals[lo].addr;
2232           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2233           mapping->fib_index = m->locals[lo].fib_index;
2234         }
2235       else
2236         {
2237           mapping->fib_index = m->fib_index;
2238           mapping->addr = m->local_addr;
2239           /* Address only mapping doesn't change port */
2240           mapping->port = m->addr_only ? match.port
2241             : clib_host_to_net_u16 (m->local_port);
2242         }
2243       mapping->protocol = m->proto;
2244     }
2245   else
2246     {
2247       mapping->addr = m->external_addr;
2248       /* Address only mapping doesn't change port */
2249       mapping->port = m->addr_only ? match.port
2250         : clib_host_to_net_u16 (m->external_port);
2251       mapping->fib_index = sm->outside_fib_index;
2252     }
2253
2254   if (PREDICT_FALSE(is_addr_only != 0))
2255     *is_addr_only = m->addr_only;
2256
2257   if (PREDICT_FALSE(twice_nat != 0))
2258     *twice_nat = m->twice_nat;
2259
2260   if (PREDICT_FALSE(lb != 0))
2261     *lb = vec_len (m->locals) > 0;
2262
2263   return 0;
2264 }
2265
2266 static_always_inline u16
2267 snat_random_port (u16 min, u16 max)
2268 {
2269   snat_main_t *sm = &snat_main;
2270   return min + random_u32 (&sm->random_seed) /
2271     (random_u32_max() / (max - min + 1) + 1);
2272 }
2273
2274 int
2275 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2276                                      u32 fib_index,
2277                                      u32 thread_index,
2278                                      snat_session_key_t * k,
2279                                      u32 * address_indexp,
2280                                      u16 port_per_thread,
2281                                      u32 snat_thread_index)
2282 {
2283   snat_main_t *sm = &snat_main;
2284
2285   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2286                                  address_indexp, port_per_thread,
2287                                  snat_thread_index);
2288 }
2289
2290 static int
2291 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2292                                  u32 fib_index,
2293                                  u32 thread_index,
2294                                  snat_session_key_t * k,
2295                                  u32 * address_indexp,
2296                                  u16 port_per_thread,
2297                                  u32 snat_thread_index)
2298 {
2299   int i, gi = 0;
2300   snat_address_t *a, *ga = 0;
2301   u32 portnum;
2302
2303   for (i = 0; i < vec_len (addresses); i++)
2304     {
2305       a = addresses + i;
2306       switch (k->protocol)
2307         {
2308 #define _(N, j, n, s) \
2309         case SNAT_PROTOCOL_##N: \
2310           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2311             { \
2312               if (a->fib_index == fib_index) \
2313                 { \
2314                   while (1) \
2315                     { \
2316                       portnum = (port_per_thread * \
2317                         snat_thread_index) + \
2318                         snat_random_port(1, port_per_thread) + 1024; \
2319                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2320                         continue; \
2321                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2322                       a->busy_##n##_ports_per_thread[thread_index]++; \
2323                       a->busy_##n##_ports++; \
2324                       k->addr = a->addr; \
2325                       k->port = clib_host_to_net_u16(portnum); \
2326                       *address_indexp = i; \
2327                       return 0; \
2328                     } \
2329                 } \
2330               else if (a->fib_index == ~0) \
2331                 { \
2332                   ga = a; \
2333                   gi = i; \
2334                 } \
2335             } \
2336           break;
2337           foreach_snat_protocol
2338 #undef _
2339         default:
2340           nat_log_info ("unknown protocol");
2341           return 1;
2342         }
2343
2344     }
2345
2346   if (ga)
2347     {
2348       a = ga;
2349       switch (k->protocol)
2350         {
2351 #define _(N, j, n, s) \
2352         case SNAT_PROTOCOL_##N: \
2353           while (1) \
2354             { \
2355               portnum = (port_per_thread * \
2356                 snat_thread_index) + \
2357                 snat_random_port(1, port_per_thread) + 1024; \
2358               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2359                 continue; \
2360               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2361               a->busy_##n##_ports_per_thread[thread_index]++; \
2362               a->busy_##n##_ports++; \
2363               k->addr = a->addr; \
2364               k->port = clib_host_to_net_u16(portnum); \
2365               *address_indexp = gi; \
2366               return 0; \
2367             }
2368           break;
2369           foreach_snat_protocol
2370 #undef _
2371         default:
2372           nat_log_info ("unknown protocol");
2373           return 1;
2374         }
2375     }
2376
2377   /* Totally out of translations to use... */
2378   snat_ipfix_logging_addresses_exhausted(0);
2379   return 1;
2380 }
2381
2382 static int
2383 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2384                               u32 fib_index,
2385                               u32 thread_index,
2386                               snat_session_key_t * k,
2387                               u32 * address_indexp,
2388                               u16 port_per_thread,
2389                               u32 snat_thread_index)
2390 {
2391   snat_main_t *sm = &snat_main;
2392   snat_address_t *a = addresses;
2393   u16 m, ports, portnum, A, j;
2394   m = 16 - (sm->psid_offset + sm->psid_length);
2395   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2396
2397   if (!vec_len (addresses))
2398     goto exhausted;
2399
2400   switch (k->protocol)
2401     {
2402 #define _(N, i, n, s) \
2403     case SNAT_PROTOCOL_##N: \
2404       if (a->busy_##n##_ports < ports) \
2405         { \
2406           while (1) \
2407             { \
2408               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2409               j = snat_random_port(0, pow2_mask(m)); \
2410               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2411               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2412                 continue; \
2413               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2414               a->busy_##n##_ports++; \
2415               k->addr = a->addr; \
2416               k->port = clib_host_to_net_u16 (portnum); \
2417               *address_indexp = i; \
2418               return 0; \
2419             } \
2420         } \
2421       break;
2422       foreach_snat_protocol
2423 #undef _
2424     default:
2425       nat_log_info ("unknown protocol");
2426       return 1;
2427     }
2428
2429 exhausted:
2430   /* Totally out of translations to use... */
2431   snat_ipfix_logging_addresses_exhausted(0);
2432   return 1;
2433 }
2434
2435 void
2436 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2437 {
2438   dpo_id_t dpo_v4 = DPO_INVALID;
2439   fib_prefix_t pfx = {
2440     .fp_proto = FIB_PROTOCOL_IP4,
2441     .fp_len = 32,
2442     .fp_addr.ip4.as_u32 = addr.as_u32,
2443   };
2444
2445   if (is_add)
2446     {
2447       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2448       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2449                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2450       dpo_reset (&dpo_v4);
2451     }
2452   else
2453     {
2454       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2455     }
2456 }
2457
2458 uword
2459 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2460 {
2461   u32 *r = va_arg (*args, u32 *);
2462
2463   if (0);
2464 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2465   foreach_snat_protocol
2466 #undef _
2467   else
2468     return 0;
2469   return 1;
2470 }
2471
2472 u8 *
2473 format_snat_protocol (u8 * s, va_list * args)
2474 {
2475   u32 i = va_arg (*args, u32);
2476   u8 *t = 0;
2477
2478   switch (i)
2479     {
2480 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2481       foreach_snat_protocol
2482 #undef _
2483     default:
2484       s = format (s, "unknown");
2485       return s;
2486     }
2487   s = format (s, "%s", t);
2488   return s;
2489 }
2490
2491 u8 * format_snat_key (u8 * s, va_list * args);
2492 u8 * format_static_mapping_key (u8 * s, va_list * args);
2493
2494 u8 *
2495 format_session_kvp (u8 * s, va_list * args)
2496 {
2497   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2498   snat_session_key_t k;
2499
2500   k.as_u64 = v->key;
2501
2502   s = format (s, "%U session-index %llu", format_snat_key, &k, v->value);
2503
2504   return s;
2505 }
2506
2507 u8 *
2508 format_static_mapping_kvp (u8 * s, va_list * args)
2509 {
2510   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2511   snat_session_key_t k;
2512
2513   k.as_u64 = v->key;
2514
2515   s = format (s, "%U static-mapping-index %llu",
2516               format_static_mapping_key, &k, v->value);
2517
2518   return s;
2519 }
2520
2521 u8 *
2522 format_user_kvp (u8 * s, va_list * args)
2523 {
2524   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
2525   snat_user_key_t k;
2526
2527   k.as_u64 = v->key;
2528
2529   s = format (s, "%U fib %d user-index %llu", format_ip4_address, &k.addr,
2530               k.fib_index, v->value);
2531
2532   return s;
2533 }
2534
2535 u8 *
2536 format_ed_session_kvp (u8 * s, va_list * args)
2537 {
2538   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
2539   nat_ed_ses_key_t k;
2540
2541   k.as_u64[0] = v->key[0];
2542   k.as_u64[1] = v->key[1];
2543
2544   s = format (s, "local %U:%d remote %U:%d proto %U fib %d session-index %llu",
2545               format_ip4_address, &k.l_addr, clib_net_to_host_u16 (k.l_port),
2546               format_ip4_address, &k.r_addr, clib_net_to_host_u16 (k.r_port),
2547               format_ip_protocol, k.proto, k.fib_index, v->value);
2548
2549   return s;
2550 }
2551
2552 static u32
2553 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2554 {
2555   snat_main_t *sm = &snat_main;
2556   u32 next_worker_index = 0;
2557   u32 hash;
2558
2559   next_worker_index = sm->first_worker_index;
2560   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2561          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2562
2563   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2564     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2565   else
2566     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2567
2568   return next_worker_index;
2569 }
2570
2571 static u32
2572 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2573 {
2574   snat_main_t *sm = &snat_main;
2575   udp_header_t *udp;
2576   u16 port;
2577   snat_session_key_t m_key;
2578   clib_bihash_kv_8_8_t kv, value;
2579   snat_static_mapping_t *m;
2580   u32 proto;
2581   u32 next_worker_index = 0;
2582
2583   /* first try static mappings without port */
2584   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2585     {
2586       m_key.addr = ip0->dst_address;
2587       m_key.port = 0;
2588       m_key.protocol = 0;
2589       m_key.fib_index = rx_fib_index0;
2590       kv.key = m_key.as_u64;
2591       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2592         {
2593           m = pool_elt_at_index (sm->static_mappings, value.value);
2594           return m->workers[0];
2595         }
2596     }
2597
2598   proto = ip_proto_to_snat_proto (ip0->protocol);
2599   udp = ip4_next_header (ip0);
2600   port = udp->dst_port;
2601
2602   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2603     {
2604       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2605         return vlib_get_thread_index ();
2606
2607       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2608         {
2609           nat_reass_ip4_t *reass;
2610
2611           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2612                                       ip0->fragment_id, ip0->protocol);
2613
2614           if (reass && (reass->thread_index != (u32) ~ 0))
2615             return reass->thread_index;
2616           else
2617             return vlib_get_thread_index ();
2618         }
2619     }
2620
2621   /* unknown protocol */
2622   if (PREDICT_FALSE (proto == ~0))
2623     {
2624       /* use current thread */
2625       return vlib_get_thread_index ();
2626     }
2627
2628   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2629     {
2630       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2631       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2632       if (!icmp_is_error_message (icmp))
2633         port = echo->identifier;
2634       else
2635         {
2636           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2637           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2638           void *l4_header = ip4_next_header (inner_ip);
2639           switch (proto)
2640             {
2641             case SNAT_PROTOCOL_ICMP:
2642               icmp = (icmp46_header_t*)l4_header;
2643               echo = (icmp_echo_header_t *)(icmp + 1);
2644               port = echo->identifier;
2645               break;
2646             case SNAT_PROTOCOL_UDP:
2647             case SNAT_PROTOCOL_TCP:
2648               port = ((tcp_udp_header_t*)l4_header)->src_port;
2649               break;
2650             default:
2651               return vlib_get_thread_index ();
2652             }
2653         }
2654     }
2655
2656   /* try static mappings with port */
2657   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2658     {
2659       m_key.addr = ip0->dst_address;
2660       m_key.port = clib_net_to_host_u16 (port);
2661       m_key.protocol = proto;
2662       m_key.fib_index = rx_fib_index0;
2663       kv.key = m_key.as_u64;
2664       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2665         {
2666           m = pool_elt_at_index (sm->static_mappings, value.value);
2667           return m->workers[0];
2668         }
2669     }
2670
2671   /* worker by outside port */
2672   next_worker_index = sm->first_worker_index;
2673   next_worker_index +=
2674     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2675   return next_worker_index;
2676 }
2677
2678 static u32
2679 nat44_ed_get_worker_out2in_cb (ip4_header_t * ip, u32 rx_fib_index)
2680 {
2681   snat_main_t *sm = &snat_main;
2682   clib_bihash_kv_8_8_t kv, value;
2683   u32 proto, next_worker_index = 0;
2684   udp_header_t *udp;
2685   u16 port;
2686   snat_static_mapping_t *m;
2687   u32 hash;
2688
2689   /* first try static mappings without port */
2690   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2691     {
2692       make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
2693       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2694         {
2695           m = pool_elt_at_index (sm->static_mappings, value.value);
2696           return m->workers[0];
2697         }
2698     }
2699
2700   proto = ip_proto_to_snat_proto (ip->protocol);
2701
2702   /* unknown protocol */
2703   if (PREDICT_FALSE (proto == ~0))
2704     {
2705       /* use current thread */
2706       return vlib_get_thread_index ();
2707     }
2708
2709   udp = ip4_next_header (ip);
2710   port = udp->dst_port;
2711
2712   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2713     {
2714       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2715       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2716       if (!icmp_is_error_message (icmp))
2717         port = echo->identifier;
2718       else
2719         {
2720           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2721           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2722           void *l4_header = ip4_next_header (inner_ip);
2723           switch (proto)
2724             {
2725             case SNAT_PROTOCOL_ICMP:
2726               icmp = (icmp46_header_t*)l4_header;
2727               echo = (icmp_echo_header_t *)(icmp + 1);
2728               port = echo->identifier;
2729               break;
2730             case SNAT_PROTOCOL_UDP:
2731             case SNAT_PROTOCOL_TCP:
2732               port = ((tcp_udp_header_t*)l4_header)->src_port;
2733               break;
2734             default:
2735               return vlib_get_thread_index ();
2736             }
2737         }
2738     }
2739
2740   /* try static mappings with port */
2741   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2742     {
2743       make_sm_kv (&kv, &ip->dst_address, proto, rx_fib_index,
2744                   clib_net_to_host_u16 (port));
2745       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2746         {
2747           m = pool_elt_at_index (sm->static_mappings, value.value);
2748           if (!vec_len(m->locals))
2749             return m->workers[0];
2750
2751           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2752                  (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >>24);
2753
2754           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2755             return m->workers[hash & (_vec_len (m->workers) - 1)];
2756           else
2757             return m->workers[hash % _vec_len (m->workers)];
2758         }
2759     }
2760
2761   /* worker by outside port */
2762   next_worker_index = sm->first_worker_index;
2763   next_worker_index +=
2764     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2765
2766   return next_worker_index;
2767 }
2768
2769 static clib_error_t *
2770 snat_config (vlib_main_t * vm, unformat_input_t * input)
2771 {
2772   snat_main_t * sm = &snat_main;
2773   nat66_main_t * nm = &nat66_main;
2774   u32 translation_buckets = 1024;
2775   u32 translation_memory_size = 128<<20;
2776   u32 user_buckets = 128;
2777   u32 user_memory_size = 64<<20;
2778   u32 max_translations_per_user = 100;
2779   u32 outside_vrf_id = 0;
2780   u32 outside_ip6_vrf_id = 0;
2781   u32 inside_vrf_id = 0;
2782   u32 static_mapping_buckets = 1024;
2783   u32 static_mapping_memory_size = 64<<20;
2784   u32 nat64_bib_buckets = 1024;
2785   u32 nat64_bib_memory_size = 128 << 20;
2786   u32 nat64_st_buckets = 2048;
2787   u32 nat64_st_memory_size = 256 << 20;
2788   u8 static_mapping_only = 0;
2789   u8 static_mapping_connection_tracking = 0;
2790   snat_main_per_thread_data_t *tsm;
2791   dslite_main_t * dm = &dslite_main;
2792
2793   sm->deterministic = 0;
2794   sm->out2in_dpo = 0;
2795   sm->endpoint_dependent = 0;
2796
2797   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2798     {
2799       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2800         ;
2801       else if (unformat (input, "translation hash memory %d",
2802                          &translation_memory_size));
2803       else if (unformat (input, "user hash buckets %d", &user_buckets))
2804         ;
2805       else if (unformat (input, "user hash memory %d",
2806                          &user_memory_size))
2807         ;
2808       else if (unformat (input, "max translations per user %d",
2809                          &max_translations_per_user))
2810         ;
2811       else if (unformat (input, "outside VRF id %d",
2812                          &outside_vrf_id))
2813         ;
2814       else if (unformat (input, "outside ip6 VRF id %d",
2815                          &outside_ip6_vrf_id))
2816         ;
2817       else if (unformat (input, "inside VRF id %d",
2818                          &inside_vrf_id))
2819         ;
2820       else if (unformat (input, "static mapping only"))
2821         {
2822           static_mapping_only = 1;
2823           if (unformat (input, "connection tracking"))
2824             static_mapping_connection_tracking = 1;
2825         }
2826       else if (unformat (input, "deterministic"))
2827         sm->deterministic = 1;
2828       else if (unformat (input, "nat64 bib hash buckets %d",
2829                          &nat64_bib_buckets))
2830         ;
2831       else if (unformat (input, "nat64 bib hash memory %d",
2832                          &nat64_bib_memory_size))
2833         ;
2834       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2835         ;
2836       else if (unformat (input, "nat64 st hash memory %d",
2837                          &nat64_st_memory_size))
2838         ;
2839       else if (unformat (input, "out2in dpo"))
2840         sm->out2in_dpo = 1;
2841       else if (unformat (input, "dslite ce"))
2842         dslite_set_ce(dm, 1);
2843       else if (unformat (input, "endpoint-dependent"))
2844         sm->endpoint_dependent = 1;
2845       else
2846         return clib_error_return (0, "unknown input '%U'",
2847                                   format_unformat_error, input);
2848     }
2849
2850   if (sm->deterministic && sm->endpoint_dependent)
2851     return clib_error_return (
2852       0, "deterministic and endpoint-dependent modes are mutually exclusive");
2853
2854   if (static_mapping_only && (sm->deterministic || sm->endpoint_dependent))
2855     return clib_error_return (
2856       0, "static mapping only mode available only for simple nat");
2857
2858   if (sm->out2in_dpo && (sm->deterministic || sm->endpoint_dependent))
2859     return clib_error_return (
2860       0, "out2in dpo mode available only for simple nat");
2861
2862   /* for show commands, etc. */
2863   sm->translation_buckets = translation_buckets;
2864   sm->translation_memory_size = translation_memory_size;
2865   /* do not exceed load factor 10 */
2866   sm->max_translations = 10 * translation_buckets;
2867   sm->user_buckets = user_buckets;
2868   sm->user_memory_size = user_memory_size;
2869   sm->max_translations_per_user = max_translations_per_user;
2870   sm->outside_vrf_id = outside_vrf_id;
2871   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2872                                                              outside_vrf_id,
2873                                                              FIB_SOURCE_PLUGIN_HI);
2874   nm->outside_vrf_id = outside_ip6_vrf_id;
2875   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2876                                                              outside_ip6_vrf_id,
2877                                                              FIB_SOURCE_PLUGIN_HI);
2878   sm->inside_vrf_id = inside_vrf_id;
2879   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2880                                                             inside_vrf_id,
2881                                                             FIB_SOURCE_PLUGIN_HI);
2882   sm->static_mapping_only = static_mapping_only;
2883   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
2884
2885   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
2886                  nat64_st_memory_size);
2887
2888   if (sm->deterministic)
2889     {
2890       sm->in2out_node_index = snat_det_in2out_node.index;
2891       sm->in2out_output_node_index = ~0;
2892       sm->out2in_node_index = snat_det_out2in_node.index;
2893       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
2894       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
2895     }
2896   else
2897     {
2898       if (sm->endpoint_dependent)
2899         {
2900           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2901           sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
2902           sm->in2out_node_index = nat44_ed_in2out_node.index;
2903           sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
2904           sm->out2in_node_index = nat44_ed_out2in_node.index;
2905           sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
2906           sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
2907         }
2908       else
2909         {
2910           sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2911           sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2912           sm->in2out_node_index = snat_in2out_node.index;
2913           sm->in2out_output_node_index = snat_in2out_output_node.index;
2914           sm->out2in_node_index = snat_out2in_node.index;
2915           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2916           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2917         }
2918       if (!static_mapping_only ||
2919           (static_mapping_only && static_mapping_connection_tracking))
2920         {
2921           vec_foreach (tsm, sm->per_thread_data)
2922             {
2923               if (sm->endpoint_dependent)
2924                 {
2925                   clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed",
2926                                          translation_buckets,
2927                                          translation_memory_size);
2928                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed,
2929                                                       format_ed_session_kvp);
2930
2931                   clib_bihash_init_16_8 (&tsm->out2in_ed, "out2in-ed",
2932                                          translation_buckets,
2933                                          translation_memory_size);
2934                   clib_bihash_set_kvp_format_fn_16_8 (&tsm->out2in_ed,
2935                                                       format_ed_session_kvp);
2936                 }
2937               else
2938                 {
2939                   clib_bihash_init_8_8 (&tsm->in2out, "in2out",
2940                                         translation_buckets,
2941                                         translation_memory_size);
2942                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->in2out,
2943                                                      format_session_kvp);
2944
2945                   clib_bihash_init_8_8 (&tsm->out2in, "out2in",
2946                                         translation_buckets,
2947                                         translation_memory_size);
2948                   clib_bihash_set_kvp_format_fn_8_8 (&tsm->out2in,
2949                                                      format_session_kvp);
2950                 }
2951
2952               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
2953                                     user_memory_size);
2954               clib_bihash_set_kvp_format_fn_8_8 (&tsm->user_hash,
2955                                                  format_user_kvp);
2956             }
2957
2958         }
2959       else
2960         {
2961           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2962           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2963         }
2964       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2965                             "static_mapping_by_local", static_mapping_buckets,
2966                             static_mapping_memory_size);
2967       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
2968                                          format_static_mapping_kvp);
2969
2970       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2971                             "static_mapping_by_external", static_mapping_buckets,
2972                             static_mapping_memory_size);
2973       clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
2974                                          format_static_mapping_kvp);
2975     }
2976
2977   return 0;
2978 }
2979
2980 VLIB_CONFIG_FUNCTION (snat_config, "nat");
2981
2982 u8 * format_snat_session_state (u8 * s, va_list * args)
2983 {
2984   u32 i = va_arg (*args, u32);
2985   u8 *t = 0;
2986
2987   switch (i)
2988     {
2989 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
2990     foreach_snat_session_state
2991 #undef _
2992     default:
2993       t = format (t, "unknown");
2994     }
2995   s = format (s, "%s", t);
2996   return s;
2997 }
2998
2999 u8 * format_snat_key (u8 * s, va_list * args)
3000 {
3001   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3002
3003   s = format (s, "%U proto %U port %d fib %d",
3004               format_ip4_address, &key->addr,
3005               format_snat_protocol, key->protocol,
3006               clib_net_to_host_u16 (key->port), key->fib_index);
3007   return s;
3008 }
3009
3010 u8 * format_static_mapping_key (u8 * s, va_list * args)
3011 {
3012   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
3013
3014   s = format (s, "%U proto %U port %d fib %d",
3015               format_ip4_address, &key->addr,
3016               format_snat_protocol, key->protocol,
3017               key->port, key->fib_index);
3018   return s;
3019 }
3020
3021 u8 * format_snat_session (u8 * s, va_list * args)
3022 {
3023   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3024   snat_session_t * sess = va_arg (*args, snat_session_t *);
3025
3026   if (snat_is_unk_proto_session (sess))
3027     {
3028       s = format (s, "  i2o %U proto %u fib %u\n",
3029                   format_ip4_address, &sess->in2out.addr,
3030                   clib_net_to_host_u16 (sess->in2out.port),
3031                   sess->in2out.fib_index);
3032       s = format (s, "    o2i %U proto %u fib %u\n",
3033                   format_ip4_address, &sess->out2in.addr,
3034                   clib_net_to_host_u16 (sess->out2in.port),
3035                   sess->out2in.fib_index);
3036     }
3037   else
3038     {
3039       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
3040       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
3041     }
3042   if (is_ed_session (sess) || is_fwd_bypass_session (sess))
3043     {
3044       if (is_twice_nat_session (sess))
3045         {
3046           s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
3047                       format_ip4_address, &sess->ext_host_addr,
3048                       clib_net_to_host_u16 (sess->ext_host_port),
3049                       format_ip4_address, &sess->ext_host_nat_addr,
3050                       clib_net_to_host_u16 (sess->ext_host_nat_port));
3051         }
3052       else
3053         {
3054           if (sess->ext_host_addr.as_u32)
3055               s = format (s, "       external host %U:%u\n",
3056                           format_ip4_address, &sess->ext_host_addr,
3057                           clib_net_to_host_u16 (sess->ext_host_port));
3058         }
3059     }
3060   s = format (s, "       index %llu\n", sess - sm->sessions);
3061   s = format (s, "       last heard %.2f\n", sess->last_heard);
3062   s = format (s, "       total pkts %d, total bytes %lld\n",
3063               sess->total_pkts, sess->total_bytes);
3064   if (snat_is_session_static (sess))
3065     s = format (s, "       static translation\n");
3066   else
3067     s = format (s, "       dynamic translation\n");
3068   if (is_fwd_bypass_session (sess))
3069     s = format (s, "       forwarding-bypass\n");
3070   if (is_lb_session (sess))
3071     s = format (s, "       load-balancing\n");
3072   if (is_twice_nat_session (sess))
3073     s = format (s, "       twice-nat\n");
3074
3075   return s;
3076 }
3077
3078 u8 * format_snat_user (u8 * s, va_list * args)
3079 {
3080   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
3081   snat_user_t * u = va_arg (*args, snat_user_t *);
3082   int verbose = va_arg (*args, int);
3083   dlist_elt_t * head, * elt;
3084   u32 elt_index, head_index;
3085   u32 session_index;
3086   snat_session_t * sess;
3087
3088   s = format (s, "%U: %d dynamic translations, %d static translations\n",
3089               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
3090
3091   if (verbose == 0)
3092     return s;
3093
3094   if (u->nsessions || u->nstaticsessions)
3095     {
3096       head_index = u->sessions_per_user_list_head_index;
3097       head = pool_elt_at_index (sm->list_pool, head_index);
3098
3099       elt_index = head->next;
3100       elt = pool_elt_at_index (sm->list_pool, elt_index);
3101       session_index = elt->value;
3102
3103       while (session_index != ~0)
3104         {
3105           sess = pool_elt_at_index (sm->sessions, session_index);
3106
3107           s = format (s, "  %U\n", format_snat_session, sm, sess);
3108
3109           elt_index = elt->next;
3110           elt = pool_elt_at_index (sm->list_pool, elt_index);
3111           session_index = elt->value;
3112         }
3113     }
3114
3115   return s;
3116 }
3117
3118 u8 * format_snat_static_mapping (u8 * s, va_list * args)
3119 {
3120   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
3121   nat44_lb_addr_port_t *local;
3122
3123   if (m->addr_only)
3124       s = format (s, "local %U external %U vrf %d %s %s",
3125                   format_ip4_address, &m->local_addr,
3126                   format_ip4_address, &m->external_addr,
3127                   m->vrf_id,
3128                   m->twice_nat == TWICE_NAT ? "twice-nat" :
3129                   m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3130                   m->out2in_only ? "out2in-only" : "");
3131   else
3132    {
3133       if (vec_len (m->locals))
3134         {
3135           s = format (s, "%U external %U:%d %s %s",
3136                       format_snat_protocol, m->proto,
3137                       format_ip4_address, &m->external_addr, m->external_port,
3138                       m->twice_nat == TWICE_NAT ? "twice-nat" :
3139                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3140                       m->out2in_only ? "out2in-only" : "");
3141           vec_foreach (local, m->locals)
3142             s = format (s, "\n  local %U:%d vrf %d probability %d\%",
3143                         format_ip4_address, &local->addr, local->port,
3144                         local->vrf_id, local->probability);
3145         }
3146       else
3147         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
3148                     format_snat_protocol, m->proto,
3149                     format_ip4_address, &m->local_addr, m->local_port,
3150                     format_ip4_address, &m->external_addr, m->external_port,
3151                     m->vrf_id,
3152                     m->twice_nat == TWICE_NAT ? "twice-nat" :
3153                     m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
3154                     m->out2in_only ? "out2in-only" : "");
3155    }
3156   return s;
3157 }
3158
3159 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
3160 {
3161   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
3162   vnet_main_t *vnm = vnet_get_main();
3163
3164   if (m->addr_only)
3165       s = format (s, "local %U external %U vrf %d",
3166                   format_ip4_address, &m->l_addr,
3167                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3168                   m->vrf_id);
3169   else
3170       s = format (s, "%U local %U:%d external %U:%d vrf %d",
3171                   format_snat_protocol, m->proto,
3172                   format_ip4_address, &m->l_addr, m->l_port,
3173                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
3174                   m->e_port, m->vrf_id);
3175
3176   return s;
3177 }
3178
3179 u8 * format_det_map_ses (u8 * s, va_list * args)
3180 {
3181   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
3182   ip4_address_t in_addr, out_addr;
3183   u32 in_offset, out_offset;
3184   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
3185   u32 * i = va_arg (*args, u32 *);
3186
3187   u32 user_index = *i / SNAT_DET_SES_PER_USER;
3188   in_addr.as_u32 = clib_host_to_net_u32 (
3189     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
3190   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
3191     clib_net_to_host_u32(det_map->in_addr.as_u32);
3192   out_offset = in_offset / det_map->sharing_ratio;
3193   out_addr.as_u32 = clib_host_to_net_u32(
3194     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
3195   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
3196               format_ip4_address, &in_addr,
3197               clib_net_to_host_u16 (ses->in_port),
3198               format_ip4_address, &out_addr,
3199               clib_net_to_host_u16 (ses->out.out_port),
3200               format_ip4_address, &ses->out.ext_host_addr,
3201               clib_net_to_host_u16 (ses->out.ext_host_port),
3202               format_snat_session_state, ses->state,
3203               ses->expire);
3204
3205   return s;
3206 }
3207
3208 static void
3209 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3210                                  uword opaque,
3211                                  u32 sw_if_index,
3212                                  ip4_address_t * address,
3213                                  u32 address_length,
3214                                  u32 if_address_index,
3215                                  u32 is_delete)
3216 {
3217   snat_main_t *sm = &snat_main;
3218   snat_static_map_resolve_t *rp;
3219   snat_static_mapping_t *m;
3220   snat_session_key_t m_key;
3221   clib_bihash_kv_8_8_t kv, value;
3222   int i, rv;
3223   ip4_address_t l_addr;
3224
3225   for (i = 0; i < vec_len (sm->to_resolve); i++)
3226     {
3227       rp = sm->to_resolve + i;
3228       if (rp->addr_only == 0)
3229         continue;
3230       if (rp->sw_if_index == sw_if_index)
3231         goto match;
3232     }
3233
3234   return;
3235
3236 match:
3237   m_key.addr.as_u32 = address->as_u32;
3238   m_key.port = rp->addr_only ? 0 : rp->e_port;
3239   m_key.protocol = rp->addr_only ? 0 : rp->proto;
3240   m_key.fib_index = sm->outside_fib_index;
3241   kv.key = m_key.as_u64;
3242   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3243     m = 0;
3244   else
3245     m = pool_elt_at_index (sm->static_mappings, value.value);
3246
3247   if (!is_delete)
3248     {
3249       /* Don't trip over lease renewal, static config */
3250       if (m)
3251         return;
3252     }
3253   else
3254     {
3255       if (!m)
3256         return;
3257     }
3258
3259   /* Indetity mapping? */
3260   if (rp->l_addr.as_u32 == 0)
3261     l_addr.as_u32 = address[0].as_u32;
3262   else
3263     l_addr.as_u32 = rp->l_addr.as_u32;
3264   /* Add the static mapping */
3265   rv = snat_add_static_mapping (l_addr,
3266                                 address[0],
3267                                 rp->l_port,
3268                                 rp->e_port,
3269                                 rp->vrf_id,
3270                                 rp->addr_only,
3271                                 ~0 /* sw_if_index */,
3272                                 rp->proto,
3273                                 !is_delete,
3274                                 0, 0, rp->tag);
3275   if (rv)
3276     nat_log_notice ("snat_add_static_mapping returned %d", rv);
3277 }
3278
3279 static void
3280 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3281                                        uword opaque,
3282                                        u32 sw_if_index,
3283                                        ip4_address_t * address,
3284                                        u32 address_length,
3285                                        u32 if_address_index,
3286                                        u32 is_delete)
3287 {
3288   snat_main_t *sm = &snat_main;
3289   snat_static_map_resolve_t *rp;
3290   ip4_address_t l_addr;
3291   int i, j;
3292   int rv;
3293   u8 twice_nat = 0;
3294   snat_address_t *addresses = sm->addresses;
3295
3296   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
3297     {
3298       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3299           goto match;
3300     }
3301
3302   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
3303     {
3304       twice_nat = 1;
3305       addresses = sm->twice_nat_addresses;
3306       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3307           goto match;
3308     }
3309
3310   return;
3311
3312 match:
3313   if (!is_delete)
3314     {
3315       /* Don't trip over lease renewal, static config */
3316       for (j = 0; j < vec_len(addresses); j++)
3317         if (addresses[j].addr.as_u32 == address->as_u32)
3318           return;
3319
3320       (void) snat_add_address (sm, address, ~0, twice_nat);
3321       /* Scan static map resolution vector */
3322       for (j = 0; j < vec_len (sm->to_resolve); j++)
3323         {
3324           rp = sm->to_resolve + j;
3325           if (rp->addr_only)
3326             continue;
3327           /* On this interface? */
3328           if (rp->sw_if_index == sw_if_index)
3329             {
3330               /* Indetity mapping? */
3331               if (rp->l_addr.as_u32 == 0)
3332                 l_addr.as_u32 = address[0].as_u32;
3333               else
3334                 l_addr.as_u32 = rp->l_addr.as_u32;
3335               /* Add the static mapping */
3336               rv = snat_add_static_mapping (l_addr,
3337                                             address[0],
3338                                             rp->l_port,
3339                                             rp->e_port,
3340                                             rp->vrf_id,
3341                                             rp->addr_only,
3342                                             ~0 /* sw_if_index */,
3343                                             rp->proto,
3344                                             rp->is_add,
3345                                             0, 0, rp->tag);
3346               if (rv)
3347                 nat_log_notice ("snat_add_static_mapping returned %d", rv);
3348             }
3349         }
3350       return;
3351     }
3352   else
3353     {
3354       (void) snat_del_address(sm, address[0], 1, twice_nat);
3355       return;
3356     }
3357 }
3358
3359
3360 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
3361                                 u8 twice_nat)
3362 {
3363   ip4_main_t * ip4_main = sm->ip4_main;
3364   ip4_address_t * first_int_addr;
3365   snat_static_map_resolve_t *rp;
3366   u32 *indices_to_delete = 0;
3367   int i, j;
3368   u32 *auto_add_sw_if_indices =
3369     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3370
3371   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
3372                                                 0 /* just want the address*/);
3373
3374   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
3375     {
3376       if (auto_add_sw_if_indices[i] == sw_if_index)
3377         {
3378           if (is_del)
3379             {
3380               /* if have address remove it */
3381               if (first_int_addr)
3382                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3383               else
3384                 {
3385                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3386                     {
3387                       rp = sm->to_resolve + j;
3388                       if (rp->sw_if_index == sw_if_index)
3389                         vec_add1 (indices_to_delete, j);
3390                     }
3391                   if (vec_len(indices_to_delete))
3392                     {
3393                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3394                         vec_del1(sm->to_resolve, j);
3395                       vec_free(indices_to_delete);
3396                     }
3397                 }
3398               if (twice_nat)
3399                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3400               else
3401                 vec_del1(sm->auto_add_sw_if_indices, i);
3402             }
3403           else
3404             return VNET_API_ERROR_VALUE_EXIST;
3405
3406           return 0;
3407         }
3408     }
3409
3410   if (is_del)
3411     return VNET_API_ERROR_NO_SUCH_ENTRY;
3412
3413   /* add to the auto-address list */
3414   if (twice_nat)
3415     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3416   else
3417     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3418
3419   /* If the address is already bound - or static - add it now */
3420   if (first_int_addr)
3421       (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3422
3423   return 0;
3424 }
3425
3426 int
3427 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3428                    snat_protocol_t proto, u32 vrf_id, int is_in)
3429 {
3430   snat_main_per_thread_data_t *tsm;
3431   clib_bihash_kv_8_8_t kv, value;
3432   ip4_header_t ip;
3433   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3434   snat_session_key_t key;
3435   snat_session_t *s;
3436   clib_bihash_8_8_t *t;
3437
3438   if (sm->endpoint_dependent)
3439     return VNET_API_ERROR_UNSUPPORTED;
3440
3441   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3442   if (sm->num_workers > 1)
3443     tsm =
3444       vec_elt_at_index (sm->per_thread_data,
3445                         sm->worker_in2out_cb (&ip, fib_index));
3446   else
3447     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3448
3449   key.addr.as_u32 = addr->as_u32;
3450   key.port = clib_host_to_net_u16 (port);
3451   key.protocol = proto;
3452   key.fib_index = fib_index;
3453   kv.key = key.as_u64;
3454   t = is_in ? &tsm->in2out : &tsm->out2in;
3455   if (!clib_bihash_search_8_8 (t, &kv, &value))
3456     {
3457       if (pool_is_free_index (tsm->sessions, value.value))
3458         return VNET_API_ERROR_UNSPECIFIED;
3459
3460       s = pool_elt_at_index (tsm->sessions, value.value);
3461       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3462       nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3463       return 0;
3464     }
3465
3466   return VNET_API_ERROR_NO_SUCH_ENTRY;
3467 }
3468
3469 int
3470 nat44_del_ed_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3471                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3472                       u32 vrf_id, int is_in)
3473 {
3474   ip4_header_t ip;
3475   clib_bihash_16_8_t *t;
3476   nat_ed_ses_key_t key;
3477   clib_bihash_kv_16_8_t kv, value;
3478   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3479   snat_session_t *s;
3480   snat_main_per_thread_data_t *tsm;
3481
3482   if (!sm->endpoint_dependent)
3483     return VNET_API_ERROR_FEATURE_DISABLED;
3484
3485   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3486   if (sm->num_workers > 1)
3487     tsm =
3488       vec_elt_at_index (sm->per_thread_data,
3489                         sm->worker_in2out_cb (&ip, fib_index));
3490   else
3491     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3492
3493   t = is_in ? &tsm->in2out_ed : &tsm->out2in_ed;
3494   key.l_addr.as_u32 = addr->as_u32;
3495   key.r_addr.as_u32 = eh_addr->as_u32;
3496   key.l_port = clib_host_to_net_u16 (port);
3497   key.r_port = clib_host_to_net_u16 (eh_port);
3498   key.proto = proto;
3499   key.fib_index = clib_host_to_net_u32 (fib_index);
3500   kv.key[0] = key.as_u64[0];
3501   kv.key[1] = key.as_u64[1];
3502   if (clib_bihash_search_16_8 (t, &kv, &value))
3503     return VNET_API_ERROR_NO_SUCH_ENTRY;
3504
3505   if (pool_is_free_index (tsm->sessions, value.value))
3506     return VNET_API_ERROR_UNSPECIFIED;
3507   s = pool_elt_at_index (tsm->sessions, value.value);
3508   nat_free_session_data (sm, s, tsm - sm->per_thread_data);
3509   nat44_delete_session (sm, s, tsm - sm->per_thread_data);
3510   return 0;
3511 }
3512
3513 void
3514 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3515 {
3516   snat_main_t *sm = &snat_main;
3517
3518   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3519   sm->psid = psid;
3520   sm->psid_offset = psid_offset;
3521   sm->psid_length = psid_length;
3522 }
3523
3524 void
3525 nat_set_alloc_addr_and_port_default (void)
3526 {
3527   snat_main_t *sm = &snat_main;
3528
3529   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3530 }
3531