src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <vnet/fib/fib_table.h>
  31 #include <vnet/fib/ip4_fib.h>
  32
  33 #include <vpp/app/version.h>
  34
  35 snat_main_t snat_main;
  36
  37
  38 /* Hook up input features */
  39 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  40   .arc_name = "ip4-unicast",
  41   .node_name = "nat44-in2out",
  42   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  43 };
  44 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  45   .arc_name = "ip4-unicast",
  46   .node_name = "nat44-out2in",
  47   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  48 };
  49 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat44-classify",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  53 };
  54 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-det-in2out",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-out2in",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-classify",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  68 };
  69 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  70   .arc_name = "ip4-unicast",
  71   .node_name = "nat44-in2out-worker-handoff",
  72   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  73 };
  74 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  75   .arc_name = "ip4-unicast",
  76   .node_name = "nat44-out2in-worker-handoff",
  77   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  78 };
  79 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat44-handoff-classify",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  83 };
  84 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-in2out-fast",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-out2in-fast",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  93 };
  94 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-hairpin-dst",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  98 };
  99
 100 /* Hook up output features */
 101 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 102   .arc_name = "ip4-output",
 103   .node_name = "nat44-in2out-output",
 104   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 105 };
 106 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 107   .arc_name = "ip4-output",
 108   .node_name = "nat44-in2out-output-worker-handoff",
 109   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 110 };
 111 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 112   .arc_name = "ip4-output",
 113   .node_name = "nat44-hairpin-src",
 114   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 115 };
 116
 117 /* Hook up ip4-local features */
 118 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 119 {
 120   .arc_name = "ip4-local",
 121   .node_name = "nat44-hairpinning",
 122   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 123 };
 124
 125
 126 /* *INDENT-OFF* */
 127 VLIB_PLUGIN_REGISTER () = {
 128     .version = VPP_BUILD_VER,
 129     .description = "Network Address Translation",
 130 };
 131 /* *INDENT-ON* */
 132
 133 vlib_node_registration_t nat44_classify_node;
 134 vlib_node_registration_t nat44_det_classify_node;
 135 vlib_node_registration_t nat44_handoff_classify_node;
 136
 137 typedef enum {
 138   NAT44_CLASSIFY_NEXT_IN2OUT,
 139   NAT44_CLASSIFY_NEXT_OUT2IN,
 140   NAT44_CLASSIFY_N_NEXT,
 141 } nat44_classify_next_t;
 142
 143 void
 144 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 145 {
 146   snat_session_key_t key;
 147   clib_bihash_kv_8_8_t kv;
 148   nat_ed_ses_key_t ed_key;
 149   clib_bihash_kv_16_8_t ed_kv;
 150   int i;
 151   snat_address_t *a;
 152   snat_main_per_thread_data_t *tsm =
 153     vec_elt_at_index (sm->per_thread_data, thread_index);
 154
 155   if (is_fwd_bypass_session (s))
 156     {
 157       ed_key.l_addr = s->in2out.addr;
 158       ed_key.r_addr = s->ext_host_addr;
 159       ed_key.l_port = s->in2out.port;
 160       ed_key.r_port = s->ext_host_port;
 161       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 162       ed_key.fib_index = 0;
 163       ed_kv.key[0] = ed_key.as_u64[0];
 164       ed_kv.key[1] = ed_key.as_u64[1];
 165       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &ed_kv, 0))
 166         clib_warning ("in2out_ed key del failed");
 167       return;
 168     }
 169
 170   /* Endpoint dependent session lookup tables */
 171   if (is_ed_session (s))
 172     {
 173       ed_key.l_addr = s->out2in.addr;
 174       ed_key.r_addr = s->ext_host_addr;
 175       ed_key.fib_index = s->out2in.fib_index;
 176       if (snat_is_unk_proto_session (s))
 177         {
 178           ed_key.proto = s->in2out.port;
 179           ed_key.r_port = 0;
 180           ed_key.l_port = 0;
 181         }
 182       else
 183         {
 184           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 185           ed_key.l_port = s->out2in.port;
 186           ed_key.r_port = s->ext_host_port;
 187         }
 188       ed_kv.key[0] = ed_key.as_u64[0];
 189       ed_kv.key[1] = ed_key.as_u64[1];
 190       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 191         clib_warning ("out2in_ed key del failed");
 192
 193       ed_key.l_addr = s->in2out.addr;
 194       ed_key.fib_index = s->in2out.fib_index;
 195       if (!snat_is_unk_proto_session (s))
 196         ed_key.l_port = s->in2out.port;
 197       if (is_twice_nat_session (s))
 198         {
 199           ed_key.r_addr = s->ext_host_nat_addr;
 200           ed_key.r_port = s->ext_host_nat_port;
 201         }
 202       ed_kv.key[0] = ed_key.as_u64[0];
 203       ed_kv.key[1] = ed_key.as_u64[1];
 204       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &ed_kv, 0))
 205         clib_warning ("in2out_ed key del failed");
 206     }
 207
 208   if (snat_is_unk_proto_session (s))
 209     return;
 210
 211   /* log NAT event */
 212   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 213                                       s->out2in.addr.as_u32,
 214                                       s->in2out.protocol,
 215                                       s->in2out.port,
 216                                       s->out2in.port,
 217                                       s->in2out.fib_index);
 218
 219   /* Twice NAT address and port for external host */
 220   if (is_twice_nat_session (s))
 221     {
 222       for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 223         {
 224           key.protocol = s->in2out.protocol;
 225           key.port = s->ext_host_nat_port;
 226           a = sm->twice_nat_addresses + i;
 227           if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 228             {
 229               snat_free_outside_address_and_port (sm->twice_nat_addresses,
 230                                                   thread_index, &key, i);
 231               break;
 232             }
 233         }
 234     }
 235
 236   if (is_ed_session (s))
 237     return;
 238
 239   /* Session lookup tables */
 240   kv.key = s->in2out.as_u64;
 241   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 242     clib_warning ("in2out key del failed");
 243   kv.key = s->out2in.as_u64;
 244   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 245     clib_warning ("out2in key del failed");
 246
 247   if (snat_is_session_static (s))
 248     return;
 249
 250   if (s->outside_address_index != ~0)
 251     snat_free_outside_address_and_port (sm->addresses, thread_index,
 252                                         &s->out2in, s->outside_address_index);
 253 }
 254
 255 snat_user_t *
 256 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 257                         u32 thread_index)
 258 {
 259   snat_user_t *u = 0;
 260   snat_user_key_t user_key;
 261   clib_bihash_kv_8_8_t kv, value;
 262   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 263   dlist_elt_t * per_user_list_head_elt;
 264
 265   user_key.addr.as_u32 = addr->as_u32;
 266   user_key.fib_index = fib_index;
 267   kv.key = user_key.as_u64;
 268
 269   /* Ever heard of the "user" = src ip4 address before? */
 270   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 271     {
 272       /* no, make a new one */
 273       pool_get (tsm->users, u);
 274       memset (u, 0, sizeof (*u));
 275       u->addr.as_u32 = addr->as_u32;
 276       u->fib_index = fib_index;
 277
 278       pool_get (tsm->list_pool, per_user_list_head_elt);
 279
 280       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 281         tsm->list_pool;
 282
 283       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 284
 285       kv.value = u - tsm->users;
 286
 287       /* add user */
 288       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 289         clib_warning ("user_hash keay add failed");
 290     }
 291   else
 292     {
 293       u = pool_elt_at_index (tsm->users, value.value);
 294     }
 295
 296   return u;
 297 }
 298
 299 snat_session_t *
 300 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 301 {
 302   snat_session_t *s;
 303   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 304   u32 oldest_per_user_translation_list_index, session_index;
 305   dlist_elt_t * oldest_per_user_translation_list_elt;
 306   dlist_elt_t * per_user_translation_list_elt;
 307
 308   /* Over quota? Recycle the least recently used translation */
 309   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 310     {
 311       oldest_per_user_translation_list_index =
 312         clib_dlist_remove_head (tsm->list_pool,
 313                                 u->sessions_per_user_list_head_index);
 314
 315       ASSERT (oldest_per_user_translation_list_index != ~0);
 316
 317       /* Add it back to the end of the LRU list */
 318       clib_dlist_addtail (tsm->list_pool,
 319                           u->sessions_per_user_list_head_index,
 320                           oldest_per_user_translation_list_index);
 321       /* Get the list element */
 322       oldest_per_user_translation_list_elt =
 323         pool_elt_at_index (tsm->list_pool,
 324                            oldest_per_user_translation_list_index);
 325
 326       /* Get the session index from the list element */
 327       session_index = oldest_per_user_translation_list_elt->value;
 328
 329       /* Get the session */
 330       s = pool_elt_at_index (tsm->sessions, session_index);
 331       nat_free_session_data (sm, s, thread_index);
 332       s->outside_address_index = ~0;
 333       s->flags = 0;
 334       s->total_bytes = 0;
 335       s->total_pkts = 0;
 336     }
 337   else
 338     {
 339       pool_get (tsm->sessions, s);
 340       memset (s, 0, sizeof (*s));
 341       s->outside_address_index = ~0;
 342
 343       /* Create list elts */
 344       pool_get (tsm->list_pool, per_user_translation_list_elt);
 345       clib_dlist_init (tsm->list_pool,
 346                        per_user_translation_list_elt - tsm->list_pool);
 347
 348       per_user_translation_list_elt->value = s - tsm->sessions;
 349       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 350       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 351
 352       clib_dlist_addtail (tsm->list_pool,
 353                           s->per_user_list_head_index,
 354                           per_user_translation_list_elt - tsm->list_pool);
 355     }
 356
 357   return s;
 358 }
 359
 360 static inline uword
 361 nat44_classify_node_fn_inline (vlib_main_t * vm,
 362                                vlib_node_runtime_t * node,
 363                                vlib_frame_t * frame)
 364 {
 365   u32 n_left_from, * from, * to_next;
 366   nat44_classify_next_t next_index;
 367   snat_main_t *sm = &snat_main;
 368
 369   from = vlib_frame_vector_args (frame);
 370   n_left_from = frame->n_vectors;
 371   next_index = node->cached_next_index;
 372
 373   while (n_left_from > 0)
 374     {
 375       u32 n_left_to_next;
 376
 377       vlib_get_next_frame (vm, node, next_index,
 378                            to_next, n_left_to_next);
 379
 380       while (n_left_from > 0 && n_left_to_next > 0)
 381         {
 382           u32 bi0;
 383           vlib_buffer_t *b0;
 384           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT;
 385           ip4_header_t *ip0;
 386           snat_address_t *ap;
 387           snat_session_key_t m_key0;
 388           clib_bihash_kv_8_8_t kv0, value0;
 389
 390           /* speculatively enqueue b0 to the current next frame */
 391           bi0 = from[0];
 392           to_next[0] = bi0;
 393           from += 1;
 394           to_next += 1;
 395           n_left_from -= 1;
 396           n_left_to_next -= 1;
 397
 398           b0 = vlib_get_buffer (vm, bi0);
 399           ip0 = vlib_buffer_get_current (b0);
 400
 401           vec_foreach (ap, sm->addresses)
 402             {
 403               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 404                 {
 405                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 406                   goto enqueue0;
 407                 }
 408             }
 409
 410           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 411             {
 412               m_key0.addr = ip0->dst_address;
 413               m_key0.port = 0;
 414               m_key0.protocol = 0;
 415               m_key0.fib_index = sm->outside_fib_index;
 416               kv0.key = m_key0.as_u64;
 417               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 418                 {
 419                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 420                   goto enqueue0;
 421                 }
 422               udp_header_t * udp0 = ip4_next_header (ip0);
 423               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 424               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 425               kv0.key = m_key0.as_u64;
 426               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 427                 next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 428             }
 429
 430         enqueue0:
 431           /* verify speculative enqueue, maybe switch current next frame */
 432           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 433                                            to_next, n_left_to_next,
 434                                            bi0, next0);
 435         }
 436
 437       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 438     }
 439
 440   return frame->n_vectors;
 441 }
 442
 443 static uword
 444 nat44_classify_node_fn (vlib_main_t * vm,
 445                         vlib_node_runtime_t * node,
 446                         vlib_frame_t * frame)
 447 {
 448   return nat44_classify_node_fn_inline (vm, node, frame);
 449 };
 450
 451 VLIB_REGISTER_NODE (nat44_classify_node) = {
 452   .function = nat44_classify_node_fn,
 453   .name = "nat44-classify",
 454   .vector_size = sizeof (u32),
 455   .type = VLIB_NODE_TYPE_INTERNAL,
 456   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 457   .next_nodes = {
 458     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 459     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 460   },
 461 };
 462
 463 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 464                               nat44_classify_node_fn);
 465
 466 static uword
 467 nat44_det_classify_node_fn (vlib_main_t * vm,
 468                             vlib_node_runtime_t * node,
 469                             vlib_frame_t * frame)
 470 {
 471   return nat44_classify_node_fn_inline (vm, node, frame);
 472 };
 473
 474 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 475   .function = nat44_det_classify_node_fn,
 476   .name = "nat44-det-classify",
 477   .vector_size = sizeof (u32),
 478   .type = VLIB_NODE_TYPE_INTERNAL,
 479   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 480   .next_nodes = {
 481     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 482     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 483   },
 484 };
 485
 486 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 487                               nat44_det_classify_node_fn);
 488
 489 static uword
 490 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 491                                 vlib_node_runtime_t * node,
 492                                 vlib_frame_t * frame)
 493 {
 494   return nat44_classify_node_fn_inline (vm, node, frame);
 495 };
 496
 497 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 498   .function = nat44_handoff_classify_node_fn,
 499   .name = "nat44-handoff-classify",
 500   .vector_size = sizeof (u32),
 501   .type = VLIB_NODE_TYPE_INTERNAL,
 502   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 503   .next_nodes = {
 504     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 505     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 506   },
 507 };
 508
 509 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 510                               nat44_handoff_classify_node_fn);
 511
 512 /**
 513  * @brief Add/del NAT address to FIB.
 514  *
 515  * Add the external NAT address to the FIB as receive entries. This ensures
 516  * that VPP will reply to ARP for this address and we don't need to enable
 517  * proxy ARP on the outside interface.
 518  *
 519  * @param addr IPv4 address.
 520  * @param plen address prefix length
 521  * @param sw_if_index Interface.
 522  * @param is_add If 0 delete, otherwise add.
 523  */
 524 void
 525 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 526                           int is_add)
 527 {
 528   fib_prefix_t prefix = {
 529     .fp_len = p_len,
 530     .fp_proto = FIB_PROTOCOL_IP4,
 531     .fp_addr = {
 532         .ip4.as_u32 = addr->as_u32,
 533     },
 534   };
 535   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 536
 537   if (is_add)
 538     fib_table_entry_update_one_path(fib_index,
 539                                     &prefix,
 540                                     FIB_SOURCE_PLUGIN_LOW,
 541                                     (FIB_ENTRY_FLAG_CONNECTED |
 542                                      FIB_ENTRY_FLAG_LOCAL |
 543                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 544                                     DPO_PROTO_IP4,
 545                                     NULL,
 546                                     sw_if_index,
 547                                     ~0,
 548                                     1,
 549                                     NULL,
 550                                     FIB_ROUTE_PATH_FLAG_NONE);
 551   else
 552     fib_table_entry_delete(fib_index,
 553                            &prefix,
 554                            FIB_SOURCE_PLUGIN_LOW);
 555 }
 556
 557 void snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 558                        u8 twice_nat)
 559 {
 560   snat_address_t * ap;
 561   snat_interface_t *i;
 562   vlib_thread_main_t *tm = vlib_get_thread_main ();
 563
 564   /* Check if address already exists */
 565   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 566     {
 567       if (ap->addr.as_u32 == addr->as_u32)
 568         return;
 569     }
 570
 571   if (twice_nat)
 572     vec_add2 (sm->twice_nat_addresses, ap, 1);
 573   else
 574     vec_add2 (sm->addresses, ap, 1);
 575
 576   ap->addr = *addr;
 577   if (vrf_id != ~0)
 578     ap->fib_index =
 579       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 580                                          FIB_SOURCE_PLUGIN_LOW);
 581   else
 582     ap->fib_index = ~0;
 583 #define _(N, i, n, s) \
 584   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 585   ap->busy_##n##_ports = 0; \
 586   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 587   foreach_snat_protocol
 588 #undef _
 589
 590   if (twice_nat)
 591     return;
 592
 593   /* Add external address to FIB */
 594   pool_foreach (i, sm->interfaces,
 595   ({
 596     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 597       continue;
 598
 599     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 600     break;
 601   }));
 602   pool_foreach (i, sm->output_feature_interfaces,
 603   ({
 604     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 605       continue;
 606
 607     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 608     break;
 609   }));
 610 }
 611
 612 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 613                                                    ip4_address_t addr)
 614 {
 615   snat_static_mapping_t *m;
 616   pool_foreach (m, sm->static_mappings,
 617   ({
 618       if (m->external_addr.as_u32 == addr.as_u32)
 619         return 1;
 620   }));
 621
 622   return 0;
 623 }
 624
 625 void increment_v4_address (ip4_address_t * a)
 626 {
 627   u32 v;
 628
 629   v = clib_net_to_host_u32(a->as_u32) + 1;
 630   a->as_u32 = clib_host_to_net_u32(v);
 631 }
 632
 633 static void
 634 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 635                                        ip4_address_t l_addr,
 636                                        u16 l_port,
 637                                        u32 sw_if_index,
 638                                        u16 e_port,
 639                                        u32 vrf_id,
 640                                        snat_protocol_t proto,
 641                                        int addr_only,
 642                                        int is_add,
 643                                        u8 * tag)
 644 {
 645   snat_static_map_resolve_t *rp;
 646
 647   vec_add2 (sm->to_resolve, rp, 1);
 648   rp->l_addr.as_u32 = l_addr.as_u32;
 649   rp->l_port = l_port;
 650   rp->sw_if_index = sw_if_index;
 651   rp->e_port = e_port;
 652   rp->vrf_id = vrf_id;
 653   rp->proto = proto;
 654   rp->addr_only = addr_only;
 655   rp->is_add = is_add;
 656   rp->tag = vec_dup (tag);
 657 }
 658
 659 /**
 660  * @brief Add static mapping.
 661  *
 662  * Create static mapping between local addr+port and external addr+port.
 663  *
 664  * @param l_addr Local IPv4 address.
 665  * @param e_addr External IPv4 address.
 666  * @param l_port Local port number.
 667  * @param e_port External port number.
 668  * @param vrf_id VRF ID.
 669  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 670  * @param sw_if_index External port instead of specific IP address.
 671  * @param is_add If 0 delete static mapping, otherwise add.
 672  * @param twice_nat If value is TWICE_NAT then translate external host address
 673  *                  and port.
 674  *                  If value is TWICE_NAT_SELF then translate external host
 675  *                  address and port whenever external host address equals
 676  *                  local address of internal host.
 677  * @param out2in_only If 1 rule match only out2in direction
 678  * @param tag - opaque string tag
 679  *
 680  * @returns
 681  */
 682 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 683                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 684                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 685                             twice_nat_type_t twice_nat, u8 out2in_only,
 686                             u8 * tag)
 687 {
 688   snat_main_t * sm = &snat_main;
 689   snat_static_mapping_t *m;
 690   snat_session_key_t m_key;
 691   clib_bihash_kv_8_8_t kv, value;
 692   snat_address_t *a = 0;
 693   u32 fib_index = ~0;
 694   uword * p;
 695   snat_interface_t *interface;
 696   int i;
 697   snat_main_per_thread_data_t *tsm;
 698   snat_user_key_t u_key;
 699   snat_user_t *u;
 700   dlist_elt_t * head, * elt;
 701   u32 elt_index, head_index;
 702   u32 ses_index;
 703   u64 user_index;
 704   snat_session_t * s;
 705   snat_static_map_resolve_t *rp, *rp_match = 0;
 706
 707   /* If the external address is a specific interface address */
 708   if (sw_if_index != ~0)
 709     {
 710       ip4_address_t * first_int_addr;
 711
 712       for (i = 0; i < vec_len (sm->to_resolve); i++)
 713         {
 714           rp = sm->to_resolve + i;
 715           if (rp->sw_if_index != sw_if_index &&
 716               rp->l_addr.as_u32 != l_addr.as_u32 &&
 717               rp->vrf_id != vrf_id && rp->addr_only != addr_only)
 718             continue;
 719
 720           if (!addr_only)
 721             {
 722               if (rp->l_port != l_port && rp->e_port != e_port && rp->proto != proto)
 723                 continue;
 724             }
 725
 726           rp_match = rp;
 727           break;
 728         }
 729
 730       /* Might be already set... */
 731       first_int_addr = ip4_interface_first_address
 732         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 733
 734       if (is_add)
 735         {
 736           if (rp_match)
 737             return VNET_API_ERROR_VALUE_EXIST;
 738
 739           snat_add_static_mapping_when_resolved
 740             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 741              addr_only,  is_add, tag);
 742
 743           /* DHCP resolution required? */
 744           if (first_int_addr == 0)
 745             {
 746               return 0;
 747             }
 748           else
 749             {
 750               e_addr.as_u32 = first_int_addr->as_u32;
 751               /* Identity mapping? */
 752               if (l_addr.as_u32 == 0)
 753                 l_addr.as_u32 = e_addr.as_u32;
 754             }
 755         }
 756       else
 757         {
 758           if (!rp_match)
 759             return VNET_API_ERROR_NO_SUCH_ENTRY;
 760
 761           vec_del1 (sm->to_resolve, i);
 762
 763           if (first_int_addr)
 764             {
 765               e_addr.as_u32 = first_int_addr->as_u32;
 766               /* Identity mapping? */
 767               if (l_addr.as_u32 == 0)
 768                 l_addr.as_u32 = e_addr.as_u32;
 769             }
 770           else
 771             return 0;
 772         }
 773     }
 774
 775   m_key.addr = e_addr;
 776   m_key.port = addr_only ? 0 : e_port;
 777   m_key.protocol = addr_only ? 0 : proto;
 778   m_key.fib_index = sm->outside_fib_index;
 779   kv.key = m_key.as_u64;
 780   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 781     m = 0;
 782   else
 783     m = pool_elt_at_index (sm->static_mappings, value.value);
 784
 785   if (is_add)
 786     {
 787       if (m)
 788         return VNET_API_ERROR_VALUE_EXIST;
 789
 790       if (twice_nat && addr_only)
 791         return VNET_API_ERROR_UNSUPPORTED;
 792
 793       /* Convert VRF id to FIB index */
 794       if (vrf_id != ~0)
 795         {
 796           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 797           if (!p)
 798             return VNET_API_ERROR_NO_SUCH_FIB;
 799           fib_index = p[0];
 800         }
 801       /* If not specified use inside VRF id from SNAT plugin startup config */
 802       else
 803         {
 804           fib_index = sm->inside_fib_index;
 805           vrf_id = sm->inside_vrf_id;
 806         }
 807
 808       if (!out2in_only)
 809         {
 810           m_key.addr = l_addr;
 811           m_key.port = addr_only ? 0 : l_port;
 812           m_key.protocol = addr_only ? 0 : proto;
 813           m_key.fib_index = fib_index;
 814           kv.key = m_key.as_u64;
 815           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 816             return VNET_API_ERROR_VALUE_EXIST;
 817         }
 818
 819       /* Find external address in allocated addresses and reserve port for
 820          address and port pair mapping when dynamic translations enabled */
 821       if (!(addr_only || sm->static_mapping_only || out2in_only))
 822         {
 823           for (i = 0; i < vec_len (sm->addresses); i++)
 824             {
 825               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 826                 {
 827                   a = sm->addresses + i;
 828                   /* External port must be unused */
 829                   switch (proto)
 830                     {
 831 #define _(N, j, n, s) \
 832                     case SNAT_PROTOCOL_##N: \
 833                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 834                         return VNET_API_ERROR_INVALID_VALUE; \
 835                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 836                       if (e_port > 1024) \
 837                         { \
 838                           a->busy_##n##_ports++; \
 839                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
 840                         } \
 841                       break;
 842                       foreach_snat_protocol
 843 #undef _
 844                     default:
 845                       clib_warning("unknown_protocol");
 846                       return VNET_API_ERROR_INVALID_VALUE_2;
 847                     }
 848                   break;
 849                 }
 850             }
 851           /* External address must be allocated */
 852           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 853             {
 854               if (sw_if_index != ~0)
 855                 {
 856                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 857                     {
 858                       rp = sm->to_resolve + i;
 859                       if (rp->addr_only)
 860                          continue;
 861                       if (rp->sw_if_index != sw_if_index &&
 862                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 863                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 864                           rp->e_port != e_port && rp->proto != proto)
 865                         continue;
 866
 867                       vec_del1 (sm->to_resolve, i);
 868                       break;
 869                     }
 870                 }
 871               return VNET_API_ERROR_NO_SUCH_ENTRY;
 872             }
 873         }
 874
 875       pool_get (sm->static_mappings, m);
 876       memset (m, 0, sizeof (*m));
 877       m->tag = vec_dup (tag);
 878       m->local_addr = l_addr;
 879       m->external_addr = e_addr;
 880       m->addr_only = addr_only;
 881       m->vrf_id = vrf_id;
 882       m->fib_index = fib_index;
 883       m->twice_nat = twice_nat;
 884       m->out2in_only = out2in_only;
 885       if (!addr_only)
 886         {
 887           m->local_port = l_port;
 888           m->external_port = e_port;
 889           m->proto = proto;
 890         }
 891
 892       if (sm->workers)
 893         {
 894           ip4_header_t ip = {
 895             .src_address = m->local_addr,
 896           };
 897           m->worker_index = sm->worker_in2out_cb (&ip, m->fib_index);
 898           tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
 899         }
 900       else
 901         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 902
 903       m_key.addr = m->local_addr;
 904       m_key.port = m->local_port;
 905       m_key.protocol = m->proto;
 906       m_key.fib_index = m->fib_index;
 907       kv.key = m_key.as_u64;
 908       kv.value = m - sm->static_mappings;
 909       if (!out2in_only)
 910         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
 911       if (twice_nat || out2in_only)
 912         {
 913           m_key.port = clib_host_to_net_u16 (l_port);
 914           kv.key = m_key.as_u64;
 915           kv.value = ~0ULL;
 916           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 1))
 917             clib_warning ("in2out key add failed");
 918         }
 919
 920       m_key.addr = m->external_addr;
 921       m_key.port = m->external_port;
 922       m_key.fib_index = sm->outside_fib_index;
 923       kv.key = m_key.as_u64;
 924       kv.value = m - sm->static_mappings;
 925       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
 926       if (twice_nat || out2in_only)
 927         {
 928           m_key.port = clib_host_to_net_u16 (e_port);
 929           kv.key = m_key.as_u64;
 930           kv.value = ~0ULL;
 931           if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 1))
 932             clib_warning ("out2in key add failed");
 933         }
 934
 935       /* Delete dynamic sessions matching local address (+ local port) */
 936       if (!(sm->static_mapping_only))
 937         {
 938           u_key.addr = m->local_addr;
 939           u_key.fib_index = m->fib_index;
 940           kv.key = u_key.as_u64;
 941           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 942             {
 943               user_index = value.value;
 944               u = pool_elt_at_index (tsm->users, user_index);
 945               if (u->nsessions)
 946                 {
 947                   head_index = u->sessions_per_user_list_head_index;
 948                   head = pool_elt_at_index (tsm->list_pool, head_index);
 949                   elt_index = head->next;
 950                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 951                   ses_index = elt->value;
 952                   while (ses_index != ~0)
 953                     {
 954                       s =  pool_elt_at_index (tsm->sessions, ses_index);
 955                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 956                       ses_index = elt->value;
 957
 958                       if (snat_is_session_static (s))
 959                         continue;
 960
 961                       if (!addr_only)
 962                         {
 963                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
 964                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
 965                             continue;
 966                         }
 967
 968                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
 969                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
 970                       pool_put_index (tsm->list_pool, s->per_user_index);
 971                       pool_put (tsm->sessions, s);
 972                       u->nsessions--;
 973
 974                       if (!addr_only)
 975                         break;
 976                     }
 977                 }
 978             }
 979         }
 980     }
 981   else
 982     {
 983       if (!m)
 984         {
 985           if (sw_if_index != ~0)
 986             return 0;
 987           else
 988             return VNET_API_ERROR_NO_SUCH_ENTRY;
 989         }
 990
 991       /* Free external address port */
 992       if (!(addr_only || sm->static_mapping_only || out2in_only))
 993         {
 994           for (i = 0; i < vec_len (sm->addresses); i++)
 995             {
 996               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 997                 {
 998                   a = sm->addresses + i;
 999                   switch (proto)
1000                     {
1001 #define _(N, j, n, s) \
1002                     case SNAT_PROTOCOL_##N: \
1003                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1004                       if (e_port > 1024) \
1005                         { \
1006                           a->busy_##n##_ports--; \
1007                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1008                         } \
1009                       break;
1010                       foreach_snat_protocol
1011 #undef _
1012                     default:
1013                       clib_warning("unknown_protocol");
1014                       return VNET_API_ERROR_INVALID_VALUE_2;
1015                     }
1016                   break;
1017                 }
1018             }
1019         }
1020
1021       if (sm->num_workers > 1)
1022         tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
1023       else
1024         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1025
1026       m_key.addr = m->local_addr;
1027       m_key.port = m->local_port;
1028       m_key.protocol = m->proto;
1029       m_key.fib_index = m->fib_index;
1030       kv.key = m_key.as_u64;
1031       if (!out2in_only)
1032         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1033       if (twice_nat || out2in_only)
1034         {
1035           m_key.port = clib_host_to_net_u16 (m->local_port);
1036           kv.key = m_key.as_u64;
1037           kv.value = ~0ULL;
1038           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 0))
1039             clib_warning ("in2out key del failed");
1040         }
1041
1042       m_key.addr = m->external_addr;
1043       m_key.port = m->external_port;
1044       m_key.fib_index = sm->outside_fib_index;
1045       kv.key = m_key.as_u64;
1046       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1047       if (twice_nat || out2in_only)
1048         {
1049           m_key.port = clib_host_to_net_u16 (m->external_port);
1050           kv.key = m_key.as_u64;
1051           kv.value = ~0ULL;
1052           if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 0))
1053             clib_warning ("in2out key del failed");
1054         }
1055
1056       /* Delete session(s) for static mapping if exist */
1057       if (!(sm->static_mapping_only) ||
1058           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1059         {
1060           u_key.addr = m->local_addr;
1061           u_key.fib_index = m->fib_index;
1062           kv.key = u_key.as_u64;
1063           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1064             {
1065               user_index = value.value;
1066               u = pool_elt_at_index (tsm->users, user_index);
1067               if (u->nstaticsessions)
1068                 {
1069                   head_index = u->sessions_per_user_list_head_index;
1070                   head = pool_elt_at_index (tsm->list_pool, head_index);
1071                   elt_index = head->next;
1072                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1073                   ses_index = elt->value;
1074                   while (ses_index != ~0)
1075                     {
1076                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1077                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1078                       ses_index = elt->value;
1079
1080                       if (!addr_only)
1081                         {
1082                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
1083                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1084                             continue;
1085                         }
1086
1087                       if (is_lb_session (s))
1088                         continue;
1089
1090                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1091                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
1092                       pool_put_index (tsm->list_pool, s->per_user_index);
1093                       pool_put (tsm->sessions, s);
1094                       u->nstaticsessions--;
1095
1096                       if (!addr_only)
1097                         break;
1098                     }
1099                   if (addr_only && (u->nstaticsessions == 0))
1100                     {
1101                       pool_put (tsm->users, u);
1102                       clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 0);
1103                     }
1104                 }
1105             }
1106         }
1107
1108       vec_free (m->tag);
1109       /* Delete static mapping from pool */
1110       pool_put (sm->static_mappings, m);
1111     }
1112
1113   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1114     return 0;
1115
1116   /* Add/delete external address to FIB */
1117   pool_foreach (interface, sm->interfaces,
1118   ({
1119     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1120       continue;
1121
1122     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1123     break;
1124   }));
1125   pool_foreach (interface, sm->output_feature_interfaces,
1126   ({
1127     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1128       continue;
1129
1130     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1131     break;
1132   }));
1133
1134   return 0;
1135 }
1136
1137 static int lb_local_exists (nat44_lb_addr_port_t * local,
1138                             ip4_address_t * e_addr, u16 e_port)
1139 {
1140   snat_main_t *sm = &snat_main;
1141   snat_static_mapping_t *m;
1142   nat44_lb_addr_port_t *ap;
1143
1144   /* *INDENT-OFF* */
1145   pool_foreach (m, sm->static_mappings,
1146   ({
1147       if (vec_len(m->locals))
1148         {
1149           if (m->external_port == e_port && m->external_addr.as_u32 == e_addr->as_u32)
1150             continue;
1151
1152           vec_foreach (ap, m->locals)
1153           {
1154             if (ap->port == local->port && ap->addr.as_u32 == local->addr.as_u32)
1155               return 1;
1156           }
1157         }
1158   }));
1159   /* *INDENT-ON* */
1160
1161   return 0;
1162 }
1163
1164 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1165                                      snat_protocol_t proto, u32 vrf_id,
1166                                      nat44_lb_addr_port_t *locals, u8 is_add,
1167                                      twice_nat_type_t twice_nat, u8 out2in_only,
1168                                      u8 *tag)
1169 {
1170   snat_main_t * sm = &snat_main;
1171   snat_static_mapping_t *m;
1172   snat_session_key_t m_key;
1173   clib_bihash_kv_8_8_t kv, value;
1174   u32 fib_index;
1175   snat_address_t *a = 0;
1176   int i;
1177   nat44_lb_addr_port_t *local;
1178   u32 worker_index = 0, elt_index, head_index, ses_index;
1179   snat_main_per_thread_data_t *tsm;
1180   snat_user_key_t u_key;
1181   snat_user_t *u;
1182   snat_session_t * s;
1183   dlist_elt_t * head, * elt;
1184
1185   m_key.addr = e_addr;
1186   m_key.port = e_port;
1187   m_key.protocol = proto;
1188   m_key.fib_index = sm->outside_fib_index;
1189   kv.key = m_key.as_u64;
1190   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1191     m = 0;
1192   else
1193     m = pool_elt_at_index (sm->static_mappings, value.value);
1194
1195   if (is_add)
1196     {
1197       if (m)
1198         return VNET_API_ERROR_VALUE_EXIST;
1199
1200       if (vec_len (locals) < 2)
1201         return VNET_API_ERROR_INVALID_VALUE;
1202
1203       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1204                                                      vrf_id,
1205                                                      FIB_SOURCE_PLUGIN_LOW);
1206
1207       /* Find external address in allocated addresses and reserve port for
1208          address and port pair mapping when dynamic translations enabled */
1209       if (!(sm->static_mapping_only || out2in_only))
1210         {
1211           for (i = 0; i < vec_len (sm->addresses); i++)
1212             {
1213               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1214                 {
1215                   a = sm->addresses + i;
1216                   /* External port must be unused */
1217                   switch (proto)
1218                     {
1219 #define _(N, j, n, s) \
1220                     case SNAT_PROTOCOL_##N: \
1221                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1222                         return VNET_API_ERROR_INVALID_VALUE; \
1223                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1224                       if (e_port > 1024) \
1225                         { \
1226                           a->busy_##n##_ports++; \
1227                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1228                         } \
1229                       break;
1230                       foreach_snat_protocol
1231 #undef _
1232                     default:
1233                       clib_warning("unknown_protocol");
1234                       return VNET_API_ERROR_INVALID_VALUE_2;
1235                     }
1236                   break;
1237                 }
1238             }
1239           /* External address must be allocated */
1240           if (!a)
1241             return VNET_API_ERROR_NO_SUCH_ENTRY;
1242         }
1243
1244       pool_get (sm->static_mappings, m);
1245       memset (m, 0, sizeof (*m));
1246       m->tag = vec_dup (tag);
1247       m->external_addr = e_addr;
1248       m->addr_only = 0;
1249       m->vrf_id = vrf_id;
1250       m->fib_index = fib_index;
1251       m->external_port = e_port;
1252       m->proto = proto;
1253       m->twice_nat = twice_nat;
1254       m->out2in_only = out2in_only;
1255
1256       m_key.addr = m->external_addr;
1257       m_key.port = m->external_port;
1258       m_key.protocol = m->proto;
1259       m_key.fib_index = sm->outside_fib_index;
1260       kv.key = m_key.as_u64;
1261       kv.value = m - sm->static_mappings;
1262       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1263         {
1264           clib_warning ("static_mapping_by_external key add failed");
1265           return VNET_API_ERROR_UNSPECIFIED;
1266         }
1267
1268       /* Assign worker */
1269       if (sm->workers)
1270         {
1271           worker_index = sm->first_worker_index +
1272             sm->workers[sm->next_worker++ % vec_len (sm->workers)];
1273           tsm = vec_elt_at_index (sm->per_thread_data, worker_index);
1274           m->worker_index = worker_index;
1275         }
1276       else
1277         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1278
1279       m_key.port = clib_host_to_net_u16 (m->external_port);
1280       kv.key = m_key.as_u64;
1281       kv.value = ~0ULL;
1282       if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 1))
1283         {
1284           clib_warning ("out2in key add failed");
1285           return VNET_API_ERROR_UNSPECIFIED;
1286         }
1287
1288       m_key.fib_index = m->fib_index;
1289       for (i = 0; i < vec_len (locals); i++)
1290         {
1291           m_key.addr = locals[i].addr;
1292           if (!out2in_only)
1293             {
1294               m_key.port = locals[i].port;
1295               kv.key = m_key.as_u64;
1296               kv.value = m - sm->static_mappings;
1297               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1298             }
1299           locals[i].prefix = (i == 0) ? locals[i].probability :\
1300             (locals[i - 1].prefix + locals[i].probability);
1301           vec_add1 (m->locals, locals[i]);
1302
1303           m_key.port = clib_host_to_net_u16 (locals[i].port);
1304           kv.key = m_key.as_u64;
1305           kv.value = ~0ULL;
1306           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 1))
1307             {
1308               clib_warning ("in2out key add failed");
1309               return VNET_API_ERROR_UNSPECIFIED;
1310             }
1311         }
1312     }
1313   else
1314     {
1315       if (!m)
1316         return VNET_API_ERROR_NO_SUCH_ENTRY;
1317
1318       fib_table_unlock (m->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1319
1320       /* Free external address port */
1321       if (!(sm->static_mapping_only || out2in_only))
1322         {
1323           for (i = 0; i < vec_len (sm->addresses); i++)
1324             {
1325               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1326                 {
1327                   a = sm->addresses + i;
1328                   switch (proto)
1329                     {
1330 #define _(N, j, n, s) \
1331                     case SNAT_PROTOCOL_##N: \
1332                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1333                       if (e_port > 1024) \
1334                         { \
1335                           a->busy_##n##_ports--; \
1336                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1337                         } \
1338                       break;
1339                       foreach_snat_protocol
1340 #undef _
1341                     default:
1342                       clib_warning("unknown_protocol");
1343                       return VNET_API_ERROR_INVALID_VALUE_2;
1344                     }
1345                   break;
1346                 }
1347             }
1348         }
1349
1350       tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
1351       m_key.addr = m->external_addr;
1352       m_key.port = m->external_port;
1353       m_key.protocol = m->proto;
1354       m_key.fib_index = sm->outside_fib_index;
1355       kv.key = m_key.as_u64;
1356       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1357         {
1358           clib_warning ("static_mapping_by_external key del failed");
1359           return VNET_API_ERROR_UNSPECIFIED;
1360         }
1361
1362       m_key.port = clib_host_to_net_u16 (m->external_port);
1363       kv.key = m_key.as_u64;
1364       if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 0))
1365         {
1366           clib_warning ("outi2in key del failed");
1367           return VNET_API_ERROR_UNSPECIFIED;
1368         }
1369
1370       vec_foreach (local, m->locals)
1371         {
1372           m_key.addr = local->addr;
1373           if (!out2in_only)
1374             {
1375               m_key.port = local->port;
1376               m_key.fib_index = m->fib_index;
1377               kv.key = m_key.as_u64;
1378               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1379                 {
1380                   clib_warning ("static_mapping_by_local key del failed");
1381                   return VNET_API_ERROR_UNSPECIFIED;
1382                 }
1383             }
1384
1385           if (!lb_local_exists(local, &e_addr, e_port))
1386             {
1387               m_key.port = clib_host_to_net_u16 (local->port);
1388               kv.key = m_key.as_u64;
1389               if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 0))
1390                 {
1391                   clib_warning ("in2out key del failed");
1392                   return VNET_API_ERROR_UNSPECIFIED;
1393                 }
1394             }
1395           /* Delete sessions */
1396           u_key.addr = local->addr;
1397           u_key.fib_index = m->fib_index;
1398           kv.key = u_key.as_u64;
1399           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1400             {
1401               u = pool_elt_at_index (tsm->users, value.value);
1402               if (u->nstaticsessions)
1403                 {
1404                   head_index = u->sessions_per_user_list_head_index;
1405                   head = pool_elt_at_index (tsm->list_pool, head_index);
1406                   elt_index = head->next;
1407                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1408                   ses_index = elt->value;
1409                   while (ses_index != ~0)
1410                     {
1411                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1412                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1413                       ses_index = elt->value;
1414
1415                       if (!(is_lb_session (s)))
1416                         continue;
1417
1418                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) &&
1419                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1420                         continue;
1421
1422                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1423                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
1424                       pool_put_index (tsm->list_pool, s->per_user_index);
1425                       pool_put (tsm->sessions, s);
1426                       u->nstaticsessions--;
1427                     }
1428                 }
1429             }
1430         }
1431       vec_free(m->locals);
1432       vec_free(m->tag);
1433
1434       pool_put (sm->static_mappings, m);
1435     }
1436
1437   return 0;
1438 }
1439
1440 int
1441 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1442                   u8 twice_nat)
1443 {
1444   snat_address_t *a = 0;
1445   snat_session_t *ses;
1446   u32 *ses_to_be_removed = 0, *ses_index;
1447   clib_bihash_kv_8_8_t kv, value;
1448   snat_user_key_t user_key;
1449   snat_user_t *u;
1450   snat_main_per_thread_data_t *tsm;
1451   snat_static_mapping_t *m;
1452   snat_interface_t *interface;
1453   int i;
1454   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1455
1456   /* Find SNAT address */
1457   for (i=0; i < vec_len (addresses); i++)
1458     {
1459       if (addresses[i].addr.as_u32 == addr.as_u32)
1460         {
1461           a = addresses + i;
1462           break;
1463         }
1464     }
1465   if (!a)
1466     return VNET_API_ERROR_NO_SUCH_ENTRY;
1467
1468   if (delete_sm)
1469     {
1470       pool_foreach (m, sm->static_mappings,
1471       ({
1472           if (m->external_addr.as_u32 == addr.as_u32)
1473             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1474                                             m->local_port, m->external_port,
1475                                             m->vrf_id, m->addr_only, ~0,
1476                                             m->proto, 0, m->twice_nat,
1477                                             m->out2in_only, m->tag);
1478       }));
1479     }
1480   else
1481     {
1482       /* Check if address is used in some static mapping */
1483       if (is_snat_address_used_in_static_mapping(sm, addr))
1484         {
1485           clib_warning ("address used in static mapping");
1486           return VNET_API_ERROR_UNSPECIFIED;
1487         }
1488     }
1489
1490   if (a->fib_index != ~0)
1491     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1492                      FIB_SOURCE_PLUGIN_LOW);
1493
1494   /* Delete sessions using address */
1495   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1496     {
1497       vec_foreach (tsm, sm->per_thread_data)
1498         {
1499           pool_foreach (ses, tsm->sessions, ({
1500             if (ses->out2in.addr.as_u32 == addr.as_u32)
1501               {
1502                 ses->outside_address_index = ~0;
1503                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1504                 clib_dlist_remove (tsm->list_pool, ses->per_user_index);
1505                 pool_put_index (tsm->list_pool, ses->per_user_index);
1506                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1507                 user_key.addr = ses->in2out.addr;
1508                 user_key.fib_index = ses->in2out.fib_index;
1509                 kv.key = user_key.as_u64;
1510                 if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1511                   {
1512                     u = pool_elt_at_index (tsm->users, value.value);
1513                     u->nsessions--;
1514                   }
1515               }
1516           }));
1517
1518           vec_foreach (ses_index, ses_to_be_removed)
1519             pool_put_index (tsm->sessions, ses_index[0]);
1520
1521           vec_free (ses_to_be_removed);
1522        }
1523     }
1524
1525   if (twice_nat)
1526     {
1527       vec_del1 (sm->twice_nat_addresses, i);
1528       return 0;
1529     }
1530   else
1531     vec_del1 (sm->addresses, i);
1532
1533   /* Delete external address from FIB */
1534   pool_foreach (interface, sm->interfaces,
1535   ({
1536     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1537       continue;
1538
1539     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1540     break;
1541   }));
1542   pool_foreach (interface, sm->output_feature_interfaces,
1543   ({
1544     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1545       continue;
1546
1547     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1548     break;
1549   }));
1550
1551   return 0;
1552 }
1553
1554 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1555 {
1556   snat_main_t *sm = &snat_main;
1557   snat_interface_t *i;
1558   const char * feature_name, *del_feature_name;
1559   snat_address_t * ap;
1560   snat_static_mapping_t * m;
1561   snat_det_map_t * dm;
1562
1563   if (sm->out2in_dpo && !is_inside)
1564     return VNET_API_ERROR_UNSUPPORTED;
1565
1566   pool_foreach (i, sm->output_feature_interfaces,
1567   ({
1568     if (i->sw_if_index == sw_if_index)
1569       return VNET_API_ERROR_VALUE_EXIST;
1570   }));
1571
1572   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1573     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1574   else
1575     {
1576       if (sm->num_workers > 1 && !sm->deterministic)
1577         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1578       else if (sm->deterministic)
1579         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1580       else
1581         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1582     }
1583
1584   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1585     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index, 0);
1586
1587   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1588     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1589
1590   pool_foreach (i, sm->interfaces,
1591   ({
1592     if (i->sw_if_index == sw_if_index)
1593       {
1594         if (is_del)
1595           {
1596             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1597               {
1598                 if (is_inside)
1599                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1600                 else
1601                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1602
1603                 if (sm->num_workers > 1 && !sm->deterministic)
1604                   {
1605                     del_feature_name = "nat44-handoff-classify";
1606                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1607                                                  "nat44-out2in-worker-handoff";
1608                   }
1609                 else if (sm->deterministic)
1610                   {
1611                     del_feature_name = "nat44-det-classify";
1612                     feature_name = !is_inside ?  "nat44-det-in2out" :
1613                                                  "nat44-det-out2in";
1614                   }
1615                 else
1616                   {
1617                     del_feature_name = "nat44-classify";
1618                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1619                   }
1620
1621                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1622                                              sw_if_index, 0, 0, 0);
1623                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1624                                              sw_if_index, 1, 0, 0);
1625               }
1626             else
1627               {
1628                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1629                                              sw_if_index, 0, 0, 0);
1630                 pool_put (sm->interfaces, i);
1631               }
1632           }
1633         else
1634           {
1635             if ((nat_interface_is_inside(i) && is_inside) ||
1636                 (nat_interface_is_outside(i) && !is_inside))
1637               return 0;
1638
1639             if (sm->num_workers > 1 && !sm->deterministic)
1640               {
1641                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1642                                                  "nat44-out2in-worker-handoff";
1643                 feature_name = "nat44-handoff-classify";
1644               }
1645             else if (sm->deterministic)
1646               {
1647                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1648                                                  "nat44-det-out2in";
1649                 feature_name = "nat44-det-classify";
1650               }
1651             else
1652               {
1653                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1654                 feature_name = "nat44-classify";
1655               }
1656
1657             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1658                                          sw_if_index, 0, 0, 0);
1659             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1660                                          sw_if_index, 1, 0, 0);
1661             goto set_flags;
1662           }
1663
1664         goto fib;
1665       }
1666   }));
1667
1668   if (is_del)
1669     return VNET_API_ERROR_NO_SUCH_ENTRY;
1670
1671   pool_get (sm->interfaces, i);
1672   i->sw_if_index = sw_if_index;
1673   i->flags = 0;
1674   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1675
1676 set_flags:
1677   if (is_inside)
1678     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1679   else
1680     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1681
1682   /* Add/delete external addresses to FIB */
1683 fib:
1684   if (is_inside && !sm->out2in_dpo)
1685     {
1686       vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1687                                    sw_if_index, !is_del, 0, 0);
1688       return 0;
1689     }
1690
1691   vec_foreach (ap, sm->addresses)
1692     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1693
1694   pool_foreach (m, sm->static_mappings,
1695   ({
1696     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1697       continue;
1698
1699     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1700   }));
1701
1702   pool_foreach (dm, sm->det_maps,
1703   ({
1704     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1705   }));
1706
1707   return 0;
1708 }
1709
1710 int snat_interface_add_del_output_feature (u32 sw_if_index,
1711                                            u8 is_inside,
1712                                            int is_del)
1713 {
1714   snat_main_t *sm = &snat_main;
1715   snat_interface_t *i;
1716   snat_address_t * ap;
1717   snat_static_mapping_t * m;
1718
1719   if (sm->deterministic ||
1720       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1721     return VNET_API_ERROR_UNSUPPORTED;
1722
1723   pool_foreach (i, sm->interfaces,
1724   ({
1725     if (i->sw_if_index == sw_if_index)
1726       return VNET_API_ERROR_VALUE_EXIST;
1727   }));
1728
1729   if (is_inside)
1730     {
1731       vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1732                                    sw_if_index, !is_del, 0, 0);
1733       vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1734                                    sw_if_index, !is_del, 0, 0);
1735       goto fq;
1736     }
1737
1738   if (sm->num_workers > 1)
1739     {
1740       vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in-worker-handoff",
1741                                    sw_if_index, !is_del, 0, 0);
1742       vnet_feature_enable_disable ("ip4-output",
1743                                    "nat44-in2out-output-worker-handoff",
1744                                    sw_if_index, !is_del, 0, 0);
1745     }
1746   else
1747     {
1748       vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in", sw_if_index,
1749                                    !is_del, 0, 0);
1750       vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1751                                    sw_if_index, !is_del, 0, 0);
1752     }
1753
1754 fq:
1755   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1756     sm->fq_in2out_output_index =
1757       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1758
1759   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1760     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1761
1762   pool_foreach (i, sm->output_feature_interfaces,
1763   ({
1764     if (i->sw_if_index == sw_if_index)
1765       {
1766         if (is_del)
1767           pool_put (sm->output_feature_interfaces, i);
1768         else
1769           return VNET_API_ERROR_VALUE_EXIST;
1770
1771         goto fib;
1772       }
1773   }));
1774
1775   if (is_del)
1776     return VNET_API_ERROR_NO_SUCH_ENTRY;
1777
1778   pool_get (sm->output_feature_interfaces, i);
1779   i->sw_if_index = sw_if_index;
1780   i->flags = 0;
1781   if (is_inside)
1782     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1783   else
1784     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1785
1786   /* Add/delete external addresses to FIB */
1787 fib:
1788   if (is_inside)
1789     return 0;
1790
1791   vec_foreach (ap, sm->addresses)
1792     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1793
1794   pool_foreach (m, sm->static_mappings,
1795   ({
1796     if (!(m->addr_only))
1797       continue;
1798
1799     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1800   }));
1801
1802   return 0;
1803 }
1804
1805 int snat_set_workers (uword * bitmap)
1806 {
1807   snat_main_t *sm = &snat_main;
1808   int i, j = 0;
1809
1810   if (sm->num_workers < 2)
1811     return VNET_API_ERROR_FEATURE_DISABLED;
1812
1813   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1814     return VNET_API_ERROR_INVALID_WORKER;
1815
1816   vec_free (sm->workers);
1817   clib_bitmap_foreach (i, bitmap,
1818     ({
1819       vec_add1(sm->workers, i);
1820       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1821       j++;
1822     }));
1823
1824   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
1825   sm->num_snat_thread = _vec_len (sm->workers);
1826
1827   return 0;
1828 }
1829
1830
1831 static void
1832 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
1833                                        uword opaque,
1834                                        u32 sw_if_index,
1835                                        ip4_address_t * address,
1836                                        u32 address_length,
1837                                        u32 if_address_index,
1838                                        u32 is_delete);
1839
1840 static void
1841 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
1842                                  uword opaque,
1843                                  u32 sw_if_index,
1844                                  ip4_address_t * address,
1845                                  u32 address_length,
1846                                  u32 if_address_index,
1847                                  u32 is_delete);
1848
1849 static int
1850 nat_alloc_addr_and_port_default (snat_address_t * addresses,
1851                                  u32 fib_index,
1852                                  u32 thread_index,
1853                                  snat_session_key_t * k,
1854                                  u32 * address_indexp,
1855                                  u16 port_per_thread,
1856                                  u32 snat_thread_index);
1857
1858 static clib_error_t * snat_init (vlib_main_t * vm)
1859 {
1860   snat_main_t * sm = &snat_main;
1861   clib_error_t * error = 0;
1862   ip4_main_t * im = &ip4_main;
1863   ip_lookup_main_t * lm = &im->lookup_main;
1864   uword *p;
1865   vlib_thread_registration_t *tr;
1866   vlib_thread_main_t *tm = vlib_get_thread_main ();
1867   uword *bitmap = 0;
1868   u32 i;
1869   ip4_add_del_interface_address_callback_t cb4;
1870
1871   sm->vlib_main = vm;
1872   sm->vnet_main = vnet_get_main();
1873   sm->ip4_main = im;
1874   sm->ip4_lookup_main = lm;
1875   sm->api_main = &api_main;
1876   sm->first_worker_index = 0;
1877   sm->next_worker = 0;
1878   sm->num_workers = 0;
1879   sm->num_snat_thread = 1;
1880   sm->workers = 0;
1881   sm->port_per_thread = 0xffff - 1024;
1882   sm->fq_in2out_index = ~0;
1883   sm->fq_out2in_index = ~0;
1884   sm->udp_timeout = SNAT_UDP_TIMEOUT;
1885   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
1886   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
1887   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
1888   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
1889   sm->forwarding_enabled = 0;
1890
1891   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
1892   if (p)
1893     {
1894       tr = (vlib_thread_registration_t *) p[0];
1895       if (tr)
1896         {
1897           sm->num_workers = tr->count;
1898           sm->first_worker_index = tr->first_index;
1899         }
1900     }
1901
1902   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
1903
1904   /* Use all available workers by default */
1905   if (sm->num_workers > 1)
1906     {
1907       for (i=0; i < sm->num_workers; i++)
1908         bitmap = clib_bitmap_set (bitmap, i, 1);
1909       snat_set_workers(bitmap);
1910       clib_bitmap_free (bitmap);
1911     }
1912   else
1913     {
1914       sm->per_thread_data[0].snat_thread_index = 0;
1915     }
1916
1917   error = snat_api_init(vm, sm);
1918   if (error)
1919     return error;
1920
1921   /* Set up the interface address add/del callback */
1922   cb4.function = snat_ip4_add_del_interface_address_cb;
1923   cb4.function_opaque = 0;
1924
1925   vec_add1 (im->add_del_interface_address_callbacks, cb4);
1926
1927   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
1928   cb4.function_opaque = 0;
1929
1930   vec_add1 (im->add_del_interface_address_callbacks, cb4);
1931
1932   nat_dpo_module_init ();
1933
1934   /* Init IPFIX logging */
1935   snat_ipfix_logging_init(vm);
1936
1937   /* Init NAT64 */
1938   error = nat64_init(vm);
1939   if (error)
1940     return error;
1941
1942   dslite_init(vm);
1943
1944   nat66_init();
1945
1946   /* Init virtual fragmenentation reassembly */
1947   return nat_reass_init(vm);
1948 }
1949
1950 VLIB_INIT_FUNCTION (snat_init);
1951
1952 void snat_free_outside_address_and_port (snat_address_t * addresses,
1953                                          u32 thread_index,
1954                                          snat_session_key_t * k,
1955                                          u32 address_index)
1956 {
1957   snat_address_t *a;
1958   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
1959
1960   ASSERT (address_index < vec_len (addresses));
1961
1962   a = addresses + address_index;
1963
1964   switch (k->protocol)
1965     {
1966 #define _(N, i, n, s) \
1967     case SNAT_PROTOCOL_##N: \
1968       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
1969         port_host_byte_order) == 1); \
1970       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
1971         port_host_byte_order, 0); \
1972       a->busy_##n##_ports--; \
1973       a->busy_##n##_ports_per_thread[thread_index]--; \
1974       break;
1975       foreach_snat_protocol
1976 #undef _
1977     default:
1978       clib_warning("unknown_protocol");
1979       return;
1980     }
1981 }
1982
1983 /**
1984  * @brief Match NAT44 static mapping.
1985  *
1986  * @param sm          NAT main.
1987  * @param match       Address and port to match.
1988  * @param mapping     External or local address and port of the matched mapping.
1989  * @param by_external If 0 match by local address otherwise match by external
1990  *                    address.
1991  * @param is_addr_only If matched mapping is address only
1992  * @param twice_nat If matched mapping is twice NAT.
1993  * @param lb If matched mapping is load-balanced.
1994  *
1995  * @returns 0 if match found otherwise 1.
1996  */
1997 int snat_static_mapping_match (snat_main_t * sm,
1998                                snat_session_key_t match,
1999                                snat_session_key_t * mapping,
2000                                u8 by_external,
2001                                u8 *is_addr_only,
2002                                twice_nat_type_t *twice_nat,
2003                                u8 *lb)
2004 {
2005   clib_bihash_kv_8_8_t kv, value;
2006   snat_static_mapping_t *m;
2007   snat_session_key_t m_key;
2008   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2009   u32 rand, lo = 0, hi, mid;
2010
2011   if (by_external)
2012     mapping_hash = &sm->static_mapping_by_external;
2013
2014   m_key.addr = match.addr;
2015   m_key.port = clib_net_to_host_u16 (match.port);
2016   m_key.protocol = match.protocol;
2017   m_key.fib_index = match.fib_index;
2018
2019   kv.key = m_key.as_u64;
2020
2021   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2022     {
2023       /* Try address only mapping */
2024       m_key.port = 0;
2025       m_key.protocol = 0;
2026       kv.key = m_key.as_u64;
2027       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2028         return 1;
2029     }
2030
2031   m = pool_elt_at_index (sm->static_mappings, value.value);
2032
2033   if (by_external)
2034     {
2035       if (vec_len (m->locals))
2036         {
2037           hi = vec_len (m->locals) - 1;
2038           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2039           while (lo < hi)
2040             {
2041               mid = ((hi - lo) >> 1) + lo;
2042               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2043             }
2044           if (!(m->locals[lo].prefix >= rand))
2045             return 1;
2046           mapping->addr = m->locals[lo].addr;
2047           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2048         }
2049       else
2050         {
2051           mapping->addr = m->local_addr;
2052           /* Address only mapping doesn't change port */
2053           mapping->port = m->addr_only ? match.port
2054             : clib_host_to_net_u16 (m->local_port);
2055         }
2056       mapping->fib_index = m->fib_index;
2057       mapping->protocol = m->proto;
2058     }
2059   else
2060     {
2061       mapping->addr = m->external_addr;
2062       /* Address only mapping doesn't change port */
2063       mapping->port = m->addr_only ? match.port
2064         : clib_host_to_net_u16 (m->external_port);
2065       mapping->fib_index = sm->outside_fib_index;
2066     }
2067
2068   if (PREDICT_FALSE(is_addr_only != 0))
2069     *is_addr_only = m->addr_only;
2070
2071   if (PREDICT_FALSE(twice_nat != 0))
2072     *twice_nat = m->twice_nat;
2073
2074   if (PREDICT_FALSE(lb != 0))
2075     *lb = vec_len (m->locals) > 0;
2076
2077   return 0;
2078 }
2079
2080 static_always_inline u16
2081 snat_random_port (u16 min, u16 max)
2082 {
2083   snat_main_t *sm = &snat_main;
2084   return min + random_u32 (&sm->random_seed) /
2085     (random_u32_max() / (max - min + 1) + 1);
2086 }
2087
2088 int
2089 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2090                                      u32 fib_index,
2091                                      u32 thread_index,
2092                                      snat_session_key_t * k,
2093                                      u32 * address_indexp,
2094                                      u16 port_per_thread,
2095                                      u32 snat_thread_index)
2096 {
2097   snat_main_t *sm = &snat_main;
2098
2099   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2100                                  address_indexp, port_per_thread,
2101                                  snat_thread_index);
2102 }
2103
2104 static int
2105 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2106                                  u32 fib_index,
2107                                  u32 thread_index,
2108                                  snat_session_key_t * k,
2109                                  u32 * address_indexp,
2110                                  u16 port_per_thread,
2111                                  u32 snat_thread_index)
2112 {
2113   int i, gi = 0;
2114   snat_address_t *a, *ga = 0;
2115   u32 portnum;
2116
2117   for (i = 0; i < vec_len (addresses); i++)
2118     {
2119       a = addresses + i;
2120       switch (k->protocol)
2121         {
2122 #define _(N, j, n, s) \
2123         case SNAT_PROTOCOL_##N: \
2124           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2125             { \
2126               if (a->fib_index == fib_index) \
2127                 { \
2128                   while (1) \
2129                     { \
2130                       portnum = (port_per_thread * \
2131                         snat_thread_index) + \
2132                         snat_random_port(1, port_per_thread) + 1024; \
2133                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2134                         continue; \
2135                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2136                       a->busy_##n##_ports_per_thread[thread_index]++; \
2137                       a->busy_##n##_ports++; \
2138                       k->addr = a->addr; \
2139                       k->port = clib_host_to_net_u16(portnum); \
2140                       *address_indexp = i; \
2141                       return 0; \
2142                     } \
2143                 } \
2144               else if (a->fib_index == ~0) \
2145                 { \
2146                   ga = a; \
2147                   gi = i; \
2148                 } \
2149             } \
2150           break;
2151           foreach_snat_protocol
2152 #undef _
2153         default:
2154           clib_warning("unknown protocol");
2155           return 1;
2156         }
2157
2158     }
2159
2160   if (ga)
2161     {
2162       a = ga;
2163       switch (k->protocol)
2164         {
2165 #define _(N, j, n, s) \
2166         case SNAT_PROTOCOL_##N: \
2167           while (1) \
2168             { \
2169               portnum = (port_per_thread * \
2170                 snat_thread_index) + \
2171                 snat_random_port(1, port_per_thread) + 1024; \
2172               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2173                 continue; \
2174               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2175               a->busy_##n##_ports_per_thread[thread_index]++; \
2176               a->busy_##n##_ports++; \
2177               k->addr = a->addr; \
2178               k->port = clib_host_to_net_u16(portnum); \
2179               *address_indexp = gi; \
2180               return 0; \
2181             }
2182           break;
2183           foreach_snat_protocol
2184 #undef _
2185         default:
2186           clib_warning ("unknown protocol");
2187           return 1;
2188         }
2189     }
2190
2191   /* Totally out of translations to use... */
2192   snat_ipfix_logging_addresses_exhausted(0);
2193   return 1;
2194 }
2195
2196 static int
2197 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2198                               u32 fib_index,
2199                               u32 thread_index,
2200                               snat_session_key_t * k,
2201                               u32 * address_indexp,
2202                               u16 port_per_thread,
2203                               u32 snat_thread_index)
2204 {
2205   snat_main_t *sm = &snat_main;
2206   snat_address_t *a = addresses;
2207   u16 m, ports, portnum, A, j;
2208   m = 16 - (sm->psid_offset + sm->psid_length);
2209   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2210
2211   if (!vec_len (addresses))
2212     goto exhausted;
2213
2214   switch (k->protocol)
2215     {
2216 #define _(N, i, n, s) \
2217     case SNAT_PROTOCOL_##N: \
2218       if (a->busy_##n##_ports < ports) \
2219         { \
2220           while (1) \
2221             { \
2222               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2223               j = snat_random_port(0, pow2_mask(m)); \
2224               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2225               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2226                 continue; \
2227               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2228               a->busy_##n##_ports++; \
2229               k->addr = a->addr; \
2230               k->port = clib_host_to_net_u16 (portnum); \
2231               *address_indexp = i; \
2232               return 0; \
2233             } \
2234         } \
2235       break;
2236       foreach_snat_protocol
2237 #undef _
2238     default:
2239       clib_warning("unknown protocol");
2240       return 1;
2241     }
2242
2243 exhausted:
2244   /* Totally out of translations to use... */
2245   snat_ipfix_logging_addresses_exhausted(0);
2246   return 1;
2247 }
2248
2249 void
2250 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2251 {
2252   dpo_id_t dpo_v4 = DPO_INVALID;
2253   fib_prefix_t pfx = {
2254     .fp_proto = FIB_PROTOCOL_IP4,
2255     .fp_len = 32,
2256     .fp_addr.ip4.as_u32 = addr.as_u32,
2257   };
2258
2259   if (is_add)
2260     {
2261       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2262       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2263                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2264       dpo_reset (&dpo_v4);
2265     }
2266   else
2267     {
2268       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2269     }
2270 }
2271
2272 uword
2273 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2274 {
2275   u32 *r = va_arg (*args, u32 *);
2276
2277   if (0);
2278 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2279   foreach_snat_protocol
2280 #undef _
2281   else
2282     return 0;
2283   return 1;
2284 }
2285
2286 u8 *
2287 format_snat_protocol (u8 * s, va_list * args)
2288 {
2289   u32 i = va_arg (*args, u32);
2290   u8 *t = 0;
2291
2292   switch (i)
2293     {
2294 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2295       foreach_snat_protocol
2296 #undef _
2297     default:
2298       s = format (s, "unknown");
2299       return s;
2300     }
2301   s = format (s, "%s", t);
2302   return s;
2303 }
2304
2305 static u32
2306 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2307 {
2308   snat_main_t *sm = &snat_main;
2309   u32 next_worker_index = 0;
2310   u32 hash;
2311
2312   next_worker_index = sm->first_worker_index;
2313   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2314          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2315
2316   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2317     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2318   else
2319     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2320
2321   return next_worker_index;
2322 }
2323
2324 static u32
2325 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2326 {
2327   snat_main_t *sm = &snat_main;
2328   udp_header_t *udp;
2329   u16 port;
2330   snat_session_key_t m_key;
2331   clib_bihash_kv_8_8_t kv, value;
2332   snat_static_mapping_t *m;
2333   nat_ed_ses_key_t key;
2334   clib_bihash_kv_16_8_t s_kv, s_value;
2335   snat_main_per_thread_data_t *tsm;
2336   snat_session_t *s;
2337   int i;
2338   u32 proto;
2339   u32 next_worker_index = 0;
2340
2341   /* first try static mappings without port */
2342   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2343     {
2344       m_key.addr = ip0->dst_address;
2345       m_key.port = 0;
2346       m_key.protocol = 0;
2347       m_key.fib_index = rx_fib_index0;
2348       kv.key = m_key.as_u64;
2349       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2350         {
2351           m = pool_elt_at_index (sm->static_mappings, value.value);
2352           return m->worker_index;
2353         }
2354     }
2355
2356   proto = ip_proto_to_snat_proto (ip0->protocol);
2357   udp = ip4_next_header (ip0);
2358   port = udp->dst_port;
2359
2360   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2361     {
2362       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2363         return vlib_get_thread_index ();
2364
2365       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2366         {
2367           nat_reass_ip4_t *reass;
2368
2369           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2370                                       ip0->fragment_id, ip0->protocol);
2371
2372           if (reass && (reass->thread_index != (u32) ~ 0))
2373             return reass->thread_index;
2374           else
2375             return vlib_get_thread_index ();
2376         }
2377     }
2378
2379   /* unknown protocol */
2380   if (PREDICT_FALSE (proto == ~0))
2381     {
2382       key.l_addr = ip0->dst_address;
2383       key.r_addr = ip0->src_address;
2384       key.fib_index = rx_fib_index0;
2385       key.proto = ip0->protocol;
2386       key.r_port = 0;
2387       key.l_port = 0;
2388       s_kv.key[0] = key.as_u64[0];
2389       s_kv.key[1] = key.as_u64[1];
2390
2391       if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
2392         {
2393           for (i = 0; i < _vec_len (sm->per_thread_data); i++)
2394             {
2395               tsm = vec_elt_at_index (sm->per_thread_data, i);
2396               if (!pool_is_free_index(tsm->sessions, s_value.value))
2397                 {
2398                   s = pool_elt_at_index (tsm->sessions, s_value.value);
2399                   if (s->out2in.addr.as_u32 == ip0->dst_address.as_u32 &&
2400                       s->out2in.port == ip0->protocol &&
2401                       snat_is_unk_proto_session (s))
2402                     return i;
2403                 }
2404             }
2405          }
2406
2407       /* if no session use current thread */
2408       return vlib_get_thread_index ();
2409     }
2410
2411   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2412     {
2413       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2414       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2415       if (!icmp_is_error_message (icmp))
2416         port = echo->identifier;
2417       else
2418         {
2419           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2420           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2421           void *l4_header = ip4_next_header (inner_ip);
2422           switch (proto)
2423             {
2424             case SNAT_PROTOCOL_ICMP:
2425               icmp = (icmp46_header_t*)l4_header;
2426               echo = (icmp_echo_header_t *)(icmp + 1);
2427               port = echo->identifier;
2428               break;
2429             case SNAT_PROTOCOL_UDP:
2430             case SNAT_PROTOCOL_TCP:
2431               port = ((tcp_udp_header_t*)l4_header)->src_port;
2432               break;
2433             default:
2434               return vlib_get_thread_index ();
2435             }
2436         }
2437     }
2438
2439   /* try static mappings with port */
2440   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2441     {
2442       m_key.addr = ip0->dst_address;
2443       m_key.port = clib_net_to_host_u16 (port);
2444       m_key.protocol = proto;
2445       m_key.fib_index = rx_fib_index0;
2446       kv.key = m_key.as_u64;
2447       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2448         {
2449           m = pool_elt_at_index (sm->static_mappings, value.value);
2450           return m->worker_index;
2451         }
2452     }
2453
2454   /* worker by outside port */
2455   next_worker_index = sm->first_worker_index;
2456   next_worker_index +=
2457     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2458   return next_worker_index;
2459 }
2460
2461 static clib_error_t *
2462 snat_config (vlib_main_t * vm, unformat_input_t * input)
2463 {
2464   snat_main_t * sm = &snat_main;
2465   nat66_main_t * nm = &nat66_main;
2466   u32 translation_buckets = 1024;
2467   u32 translation_memory_size = 128<<20;
2468   u32 user_buckets = 128;
2469   u32 user_memory_size = 64<<20;
2470   u32 max_translations_per_user = 100;
2471   u32 outside_vrf_id = 0;
2472   u32 outside_ip6_vrf_id = 0;
2473   u32 inside_vrf_id = 0;
2474   u32 static_mapping_buckets = 1024;
2475   u32 static_mapping_memory_size = 64<<20;
2476   u32 nat64_bib_buckets = 1024;
2477   u32 nat64_bib_memory_size = 128 << 20;
2478   u32 nat64_st_buckets = 2048;
2479   u32 nat64_st_memory_size = 256 << 20;
2480   u8 static_mapping_only = 0;
2481   u8 static_mapping_connection_tracking = 0;
2482   snat_main_per_thread_data_t *tsm;
2483   dslite_main_t * dm = &dslite_main;
2484
2485   sm->deterministic = 0;
2486   sm->out2in_dpo = 0;
2487
2488   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2489     {
2490       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2491         ;
2492       else if (unformat (input, "translation hash memory %d",
2493                          &translation_memory_size));
2494       else if (unformat (input, "user hash buckets %d", &user_buckets))
2495         ;
2496       else if (unformat (input, "user hash memory %d",
2497                          &user_memory_size))
2498         ;
2499       else if (unformat (input, "max translations per user %d",
2500                          &max_translations_per_user))
2501         ;
2502       else if (unformat (input, "outside VRF id %d",
2503                          &outside_vrf_id))
2504         ;
2505       else if (unformat (input, "outside ip6 VRF id %d",
2506                          &outside_ip6_vrf_id))
2507         ;
2508       else if (unformat (input, "inside VRF id %d",
2509                          &inside_vrf_id))
2510         ;
2511       else if (unformat (input, "static mapping only"))
2512         {
2513           static_mapping_only = 1;
2514           if (unformat (input, "connection tracking"))
2515             static_mapping_connection_tracking = 1;
2516         }
2517       else if (unformat (input, "deterministic"))
2518         sm->deterministic = 1;
2519       else if (unformat (input, "nat64 bib hash buckets %d",
2520                          &nat64_bib_buckets))
2521         ;
2522       else if (unformat (input, "nat64 bib hash memory %d",
2523                          &nat64_bib_memory_size))
2524         ;
2525       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2526         ;
2527       else if (unformat (input, "nat64 st hash memory %d",
2528                          &nat64_st_memory_size))
2529         ;
2530       else if (unformat (input, "out2in dpo"))
2531         sm->out2in_dpo = 1;
2532       else if (unformat (input, "dslite ce"))
2533         dslite_set_ce(dm, 1);
2534       else
2535         return clib_error_return (0, "unknown input '%U'",
2536                                   format_unformat_error, input);
2537     }
2538
2539   /* for show commands, etc. */
2540   sm->translation_buckets = translation_buckets;
2541   sm->translation_memory_size = translation_memory_size;
2542   /* do not exceed load factor 10 */
2543   sm->max_translations = 10 * translation_buckets;
2544   sm->user_buckets = user_buckets;
2545   sm->user_memory_size = user_memory_size;
2546   sm->max_translations_per_user = max_translations_per_user;
2547   sm->outside_vrf_id = outside_vrf_id;
2548   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2549                                                              outside_vrf_id,
2550                                                              FIB_SOURCE_PLUGIN_HI);
2551   nm->outside_vrf_id = outside_ip6_vrf_id;
2552   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2553                                                              outside_ip6_vrf_id,
2554                                                              FIB_SOURCE_PLUGIN_HI);
2555   sm->inside_vrf_id = inside_vrf_id;
2556   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2557                                                             inside_vrf_id,
2558                                                             FIB_SOURCE_PLUGIN_HI);
2559   sm->static_mapping_only = static_mapping_only;
2560   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
2561
2562   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
2563                  nat64_st_memory_size);
2564
2565   if (sm->deterministic)
2566     {
2567       sm->in2out_node_index = snat_det_in2out_node.index;
2568       sm->in2out_output_node_index = ~0;
2569       sm->out2in_node_index = snat_det_out2in_node.index;
2570       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
2571       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
2572     }
2573   else
2574     {
2575       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2576       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2577       sm->in2out_node_index = snat_in2out_node.index;
2578       sm->in2out_output_node_index = snat_in2out_output_node.index;
2579       sm->out2in_node_index = snat_out2in_node.index;
2580       if (!static_mapping_only ||
2581           (static_mapping_only && static_mapping_connection_tracking))
2582         {
2583           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2584           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2585
2586           vec_foreach (tsm, sm->per_thread_data)
2587             {
2588               clib_bihash_init_8_8 (&tsm->in2out, "in2out", translation_buckets,
2589                                     translation_memory_size);
2590
2591               clib_bihash_init_8_8 (&tsm->out2in, "out2in", translation_buckets,
2592                                     translation_memory_size);
2593
2594               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
2595                                     user_memory_size);
2596             }
2597
2598           clib_bihash_init_16_8 (&sm->in2out_ed, "in2out-ed",
2599                                  translation_buckets, translation_memory_size);
2600
2601           clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
2602                                  translation_buckets, translation_memory_size);
2603         }
2604       else
2605         {
2606           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2607           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2608         }
2609       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2610                             "static_mapping_by_local", static_mapping_buckets,
2611                             static_mapping_memory_size);
2612
2613       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2614                             "static_mapping_by_external", static_mapping_buckets,
2615                             static_mapping_memory_size);
2616     }
2617
2618   return 0;
2619 }
2620
2621 VLIB_CONFIG_FUNCTION (snat_config, "nat");
2622
2623 u8 * format_snat_session_state (u8 * s, va_list * args)
2624 {
2625   u32 i = va_arg (*args, u32);
2626   u8 *t = 0;
2627
2628   switch (i)
2629     {
2630 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
2631     foreach_snat_session_state
2632 #undef _
2633     default:
2634       t = format (t, "unknown");
2635     }
2636   s = format (s, "%s", t);
2637   return s;
2638 }
2639
2640 u8 * format_snat_key (u8 * s, va_list * args)
2641 {
2642   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
2643
2644   s = format (s, "%U proto %U port %d fib %d",
2645               format_ip4_address, &key->addr,
2646               format_snat_protocol, key->protocol,
2647               clib_net_to_host_u16 (key->port), key->fib_index);
2648   return s;
2649 }
2650
2651 u8 * format_snat_session (u8 * s, va_list * args)
2652 {
2653   snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *);
2654   snat_session_t * sess = va_arg (*args, snat_session_t *);
2655
2656   if (snat_is_unk_proto_session (sess))
2657     {
2658       s = format (s, "  i2o %U proto %u fib %u\n",
2659                   format_ip4_address, &sess->in2out.addr,
2660                   clib_net_to_host_u16 (sess->in2out.port),
2661                   sess->in2out.fib_index);
2662       s = format (s, "    o2i %U proto %u fib %u\n",
2663                   format_ip4_address, &sess->out2in.addr,
2664                   clib_net_to_host_u16 (sess->out2in.port),
2665                   sess->out2in.fib_index);
2666     }
2667   else
2668     {
2669       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
2670       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
2671     }
2672   if (is_twice_nat_session (sess))
2673     {
2674       s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
2675                   format_ip4_address, &sess->ext_host_addr,
2676                   clib_net_to_host_u16 (sess->ext_host_port),
2677                   format_ip4_address, &sess->ext_host_nat_addr,
2678                   clib_net_to_host_u16 (sess->ext_host_nat_port));
2679     }
2680   else
2681     {
2682       if (sess->ext_host_addr.as_u32)
2683           s = format (s, "       external host %U\n",
2684                       format_ip4_address, &sess->ext_host_addr);
2685     }
2686   s = format (s, "       last heard %.2f\n", sess->last_heard);
2687   s = format (s, "       total pkts %d, total bytes %lld\n",
2688               sess->total_pkts, sess->total_bytes);
2689   if (snat_is_session_static (sess))
2690     s = format (s, "       static translation\n");
2691   else
2692     s = format (s, "       dynamic translation\n");
2693   if (sess->flags & SNAT_SESSION_FLAG_LOAD_BALANCING)
2694     s = format (s, "       load-balancing\n");
2695   if (is_twice_nat_session (sess))
2696     s = format (s, "       twice-nat\n");
2697
2698   return s;
2699 }
2700
2701 u8 * format_snat_user (u8 * s, va_list * args)
2702 {
2703   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
2704   snat_user_t * u = va_arg (*args, snat_user_t *);
2705   int verbose = va_arg (*args, int);
2706   dlist_elt_t * head, * elt;
2707   u32 elt_index, head_index;
2708   u32 session_index;
2709   snat_session_t * sess;
2710
2711   s = format (s, "%U: %d dynamic translations, %d static translations\n",
2712               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
2713
2714   if (verbose == 0)
2715     return s;
2716
2717   if (u->nsessions || u->nstaticsessions)
2718     {
2719       head_index = u->sessions_per_user_list_head_index;
2720       head = pool_elt_at_index (sm->list_pool, head_index);
2721
2722       elt_index = head->next;
2723       elt = pool_elt_at_index (sm->list_pool, elt_index);
2724       session_index = elt->value;
2725
2726       while (session_index != ~0)
2727         {
2728           sess = pool_elt_at_index (sm->sessions, session_index);
2729
2730           s = format (s, "  %U\n", format_snat_session, sm, sess);
2731
2732           elt_index = elt->next;
2733           elt = pool_elt_at_index (sm->list_pool, elt_index);
2734           session_index = elt->value;
2735         }
2736     }
2737
2738   return s;
2739 }
2740
2741 u8 * format_snat_static_mapping (u8 * s, va_list * args)
2742 {
2743   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
2744   nat44_lb_addr_port_t *local;
2745
2746   if (m->addr_only)
2747       s = format (s, "local %U external %U vrf %d %s",
2748                   format_ip4_address, &m->local_addr,
2749                   format_ip4_address, &m->external_addr,
2750                   m->vrf_id,
2751                   m->twice_nat == TWICE_NAT ? "twice-nat" :
2752                   m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "");
2753   else
2754    {
2755       if (vec_len (m->locals))
2756         {
2757           s = format (s, "%U vrf %d external %U:%d %s %s",
2758                       format_snat_protocol, m->proto,
2759                       m->vrf_id,
2760                       format_ip4_address, &m->external_addr, m->external_port,
2761                       m->twice_nat == TWICE_NAT ? "twice-nat" :
2762                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
2763                       m->out2in_only ? "out2in-only" : "");
2764           vec_foreach (local, m->locals)
2765             s = format (s, "\n  local %U:%d probability %d\%",
2766                         format_ip4_address, &local->addr, local->port,
2767                         local->probability);
2768         }
2769       else
2770         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
2771                     format_snat_protocol, m->proto,
2772                     format_ip4_address, &m->local_addr, m->local_port,
2773                     format_ip4_address, &m->external_addr, m->external_port,
2774                     m->vrf_id,
2775                     m->twice_nat == TWICE_NAT ? "twice-nat" :
2776                     m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
2777                     m->out2in_only ? "out2in-only" : "");
2778    }
2779   return s;
2780 }
2781
2782 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
2783 {
2784   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
2785   vnet_main_t *vnm = vnet_get_main();
2786
2787   if (m->addr_only)
2788       s = format (s, "local %U external %U vrf %d",
2789                   format_ip4_address, &m->l_addr,
2790                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
2791                   m->vrf_id);
2792   else
2793       s = format (s, "%U local %U:%d external %U:%d vrf %d",
2794                   format_snat_protocol, m->proto,
2795                   format_ip4_address, &m->l_addr, m->l_port,
2796                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
2797                   m->e_port, m->vrf_id);
2798
2799   return s;
2800 }
2801
2802 u8 * format_det_map_ses (u8 * s, va_list * args)
2803 {
2804   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
2805   ip4_address_t in_addr, out_addr;
2806   u32 in_offset, out_offset;
2807   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
2808   u32 * i = va_arg (*args, u32 *);
2809
2810   u32 user_index = *i / SNAT_DET_SES_PER_USER;
2811   in_addr.as_u32 = clib_host_to_net_u32 (
2812     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
2813   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
2814     clib_net_to_host_u32(det_map->in_addr.as_u32);
2815   out_offset = in_offset / det_map->sharing_ratio;
2816   out_addr.as_u32 = clib_host_to_net_u32(
2817     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
2818   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
2819               format_ip4_address, &in_addr,
2820               clib_net_to_host_u16 (ses->in_port),
2821               format_ip4_address, &out_addr,
2822               clib_net_to_host_u16 (ses->out.out_port),
2823               format_ip4_address, &ses->out.ext_host_addr,
2824               clib_net_to_host_u16 (ses->out.ext_host_port),
2825               format_snat_session_state, ses->state,
2826               ses->expire);
2827
2828   return s;
2829 }
2830
2831 static void
2832 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2833                                  uword opaque,
2834                                  u32 sw_if_index,
2835                                  ip4_address_t * address,
2836                                  u32 address_length,
2837                                  u32 if_address_index,
2838                                  u32 is_delete)
2839 {
2840   snat_main_t *sm = &snat_main;
2841   snat_static_map_resolve_t *rp;
2842   snat_static_mapping_t *m;
2843   snat_session_key_t m_key;
2844   clib_bihash_kv_8_8_t kv, value;
2845   int i, rv;
2846   ip4_address_t l_addr;
2847
2848   for (i = 0; i < vec_len (sm->to_resolve); i++)
2849     {
2850       rp = sm->to_resolve + i;
2851       if (rp->addr_only == 0)
2852         continue;
2853       if (rp->sw_if_index == sw_if_index)
2854         goto match;
2855     }
2856
2857   return;
2858
2859 match:
2860   m_key.addr.as_u32 = address->as_u32;
2861   m_key.port = rp->addr_only ? 0 : rp->e_port;
2862   m_key.protocol = rp->addr_only ? 0 : rp->proto;
2863   m_key.fib_index = sm->outside_fib_index;
2864   kv.key = m_key.as_u64;
2865   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2866     m = 0;
2867   else
2868     m = pool_elt_at_index (sm->static_mappings, value.value);
2869
2870   if (!is_delete)
2871     {
2872       /* Don't trip over lease renewal, static config */
2873       if (m)
2874         return;
2875     }
2876   else
2877     {
2878       if (!m)
2879         return;
2880     }
2881
2882   /* Indetity mapping? */
2883   if (rp->l_addr.as_u32 == 0)
2884     l_addr.as_u32 = address[0].as_u32;
2885   else
2886     l_addr.as_u32 = rp->l_addr.as_u32;
2887   /* Add the static mapping */
2888   rv = snat_add_static_mapping (l_addr,
2889                                 address[0],
2890                                 rp->l_port,
2891                                 rp->e_port,
2892                                 rp->vrf_id,
2893                                 rp->addr_only,
2894                                 ~0 /* sw_if_index */,
2895                                 rp->proto,
2896                                 !is_delete,
2897                                 0, 0, rp->tag);
2898   if (rv)
2899     clib_warning ("snat_add_static_mapping returned %d", rv);
2900 }
2901
2902 static void
2903 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2904                                        uword opaque,
2905                                        u32 sw_if_index,
2906                                        ip4_address_t * address,
2907                                        u32 address_length,
2908                                        u32 if_address_index,
2909                                        u32 is_delete)
2910 {
2911   snat_main_t *sm = &snat_main;
2912   snat_static_map_resolve_t *rp;
2913   ip4_address_t l_addr;
2914   int i, j;
2915   int rv;
2916   u8 twice_nat = 0;
2917   snat_address_t *addresses = sm->addresses;
2918
2919   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
2920     {
2921       if (sw_if_index == sm->auto_add_sw_if_indices[i])
2922           goto match;
2923     }
2924
2925   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
2926     {
2927       twice_nat = 1;
2928       addresses = sm->twice_nat_addresses;
2929       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
2930           goto match;
2931     }
2932
2933   return;
2934
2935 match:
2936   if (!is_delete)
2937     {
2938       /* Don't trip over lease renewal, static config */
2939       for (j = 0; j < vec_len(addresses); j++)
2940         if (addresses[j].addr.as_u32 == address->as_u32)
2941           return;
2942
2943       snat_add_address (sm, address, ~0, twice_nat);
2944       /* Scan static map resolution vector */
2945       for (j = 0; j < vec_len (sm->to_resolve); j++)
2946         {
2947           rp = sm->to_resolve + j;
2948           if (rp->addr_only)
2949             continue;
2950           /* On this interface? */
2951           if (rp->sw_if_index == sw_if_index)
2952             {
2953               /* Indetity mapping? */
2954               if (rp->l_addr.as_u32 == 0)
2955                 l_addr.as_u32 = address[0].as_u32;
2956               else
2957                 l_addr.as_u32 = rp->l_addr.as_u32;
2958               /* Add the static mapping */
2959               rv = snat_add_static_mapping (l_addr,
2960                                             address[0],
2961                                             rp->l_port,
2962                                             rp->e_port,
2963                                             rp->vrf_id,
2964                                             rp->addr_only,
2965                                             ~0 /* sw_if_index */,
2966                                             rp->proto,
2967                                             rp->is_add,
2968                                             0, 0, rp->tag);
2969               if (rv)
2970                 clib_warning ("snat_add_static_mapping returned %d",
2971                               rv);
2972             }
2973         }
2974       return;
2975     }
2976   else
2977     {
2978       (void) snat_del_address(sm, address[0], 1, twice_nat);
2979       return;
2980     }
2981 }
2982
2983
2984 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
2985                                 u8 twice_nat)
2986 {
2987   ip4_main_t * ip4_main = sm->ip4_main;
2988   ip4_address_t * first_int_addr;
2989   snat_static_map_resolve_t *rp;
2990   u32 *indices_to_delete = 0;
2991   int i, j;
2992   u32 *auto_add_sw_if_indices =
2993     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
2994
2995   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
2996                                                 0 /* just want the address*/);
2997
2998   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
2999     {
3000       if (auto_add_sw_if_indices[i] == sw_if_index)
3001         {
3002           if (is_del)
3003             {
3004               /* if have address remove it */
3005               if (first_int_addr)
3006                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3007               else
3008                 {
3009                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3010                     {
3011                       rp = sm->to_resolve + j;
3012                       if (rp->sw_if_index == sw_if_index)
3013                         vec_add1 (indices_to_delete, j);
3014                     }
3015                   if (vec_len(indices_to_delete))
3016                     {
3017                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3018                         vec_del1(sm->to_resolve, j);
3019                       vec_free(indices_to_delete);
3020                     }
3021                 }
3022               if (twice_nat)
3023                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3024               else
3025                 vec_del1(sm->auto_add_sw_if_indices, i);
3026             }
3027           else
3028             return VNET_API_ERROR_VALUE_EXIST;
3029
3030           return 0;
3031         }
3032     }
3033
3034   if (is_del)
3035     return VNET_API_ERROR_NO_SUCH_ENTRY;
3036
3037   /* add to the auto-address list */
3038   if (twice_nat)
3039     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3040   else
3041     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3042
3043   /* If the address is already bound - or static - add it now */
3044   if (first_int_addr)
3045       snat_add_address (sm, first_int_addr, ~0, twice_nat);
3046
3047   return 0;
3048 }
3049
3050 int
3051 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3052                    snat_protocol_t proto, u32 vrf_id, int is_in)
3053 {
3054   snat_main_per_thread_data_t *tsm;
3055   clib_bihash_kv_8_8_t kv, value;
3056   ip4_header_t ip;
3057   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3058   snat_session_key_t key;
3059   snat_session_t *s;
3060   clib_bihash_8_8_t *t;
3061   snat_user_key_t u_key;
3062   snat_user_t *u;
3063
3064   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3065   if (sm->num_workers > 1)
3066     tsm =
3067       vec_elt_at_index (sm->per_thread_data,
3068                         sm->worker_in2out_cb (&ip, fib_index));
3069   else
3070     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3071
3072   key.addr.as_u32 = addr->as_u32;
3073   key.port = clib_host_to_net_u16 (port);
3074   key.protocol = proto;
3075   key.fib_index = fib_index;
3076   kv.key = key.as_u64;
3077   t = is_in ? &tsm->in2out : &tsm->out2in;
3078   if (!clib_bihash_search_8_8 (t, &kv, &value))
3079     {
3080       s = pool_elt_at_index (tsm->sessions, value.value);
3081       kv.key = s->in2out.as_u64;
3082       clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0);
3083       kv.key = s->out2in.as_u64;
3084       clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0);
3085       u_key.addr = s->in2out.addr;
3086       u_key.fib_index = s->in2out.fib_index;
3087       kv.key = u_key.as_u64;
3088       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
3089         {
3090           u = pool_elt_at_index (tsm->users, value.value);
3091           u->nsessions--;
3092         }
3093       clib_dlist_remove (tsm->list_pool, s->per_user_index);
3094       pool_put (tsm->sessions, s);
3095       return 0;
3096     }
3097
3098   return VNET_API_ERROR_NO_SUCH_ENTRY;
3099 }
3100
3101 void
3102 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3103 {
3104   snat_main_t *sm = &snat_main;
3105
3106   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3107   sm->psid = psid;
3108   sm->psid_offset = psid_offset;
3109   sm->psid_length = psid_length;
3110 }
3111
3112 void
3113 nat_set_alloc_addr_and_port_default (void)
3114 {
3115   snat_main_t *sm = &snat_main;
3116
3117   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3118 }
3119