src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <vnet/fib/fib_table.h>
  31 #include <vnet/fib/ip4_fib.h>
  32
  33 #include <vpp/app/version.h>
  34
  35 snat_main_t snat_main;
  36
  37
  38 /* Hook up input features */
  39 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  40   .arc_name = "ip4-unicast",
  41   .node_name = "nat44-in2out",
  42   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  43 };
  44 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  45   .arc_name = "ip4-unicast",
  46   .node_name = "nat44-out2in",
  47   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  48 };
  49 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat44-classify",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  53 };
  54 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-det-in2out",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-out2in",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-classify",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  68 };
  69 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  70   .arc_name = "ip4-unicast",
  71   .node_name = "nat44-in2out-worker-handoff",
  72   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  73 };
  74 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  75   .arc_name = "ip4-unicast",
  76   .node_name = "nat44-out2in-worker-handoff",
  77   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  78 };
  79 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat44-handoff-classify",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  83 };
  84 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-in2out-fast",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-out2in-fast",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  93 };
  94 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-hairpin-dst",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  98 };
  99
 100 /* Hook up output features */
 101 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 102   .arc_name = "ip4-output",
 103   .node_name = "nat44-in2out-output",
 104   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 105 };
 106 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 107   .arc_name = "ip4-output",
 108   .node_name = "nat44-in2out-output-worker-handoff",
 109   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 110 };
 111 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 112   .arc_name = "ip4-output",
 113   .node_name = "nat44-hairpin-src",
 114   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 115 };
 116
 117 /* Hook up ip4-local features */
 118 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 119 {
 120   .arc_name = "ip4-local",
 121   .node_name = "nat44-hairpinning",
 122   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 123 };
 124
 125
 126 /* *INDENT-OFF* */
 127 VLIB_PLUGIN_REGISTER () = {
 128     .version = VPP_BUILD_VER,
 129     .description = "Network Address Translation",
 130 };
 131 /* *INDENT-ON* */
 132
 133 vlib_node_registration_t nat44_classify_node;
 134 vlib_node_registration_t nat44_det_classify_node;
 135 vlib_node_registration_t nat44_handoff_classify_node;
 136
 137 typedef enum {
 138   NAT44_CLASSIFY_NEXT_IN2OUT,
 139   NAT44_CLASSIFY_NEXT_OUT2IN,
 140   NAT44_CLASSIFY_N_NEXT,
 141 } nat44_classify_next_t;
 142
 143 void
 144 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 145 {
 146   snat_session_key_t key;
 147   clib_bihash_kv_8_8_t kv;
 148   nat_ed_ses_key_t ed_key;
 149   clib_bihash_kv_16_8_t ed_kv;
 150   int i;
 151   snat_address_t *a;
 152   snat_main_per_thread_data_t *tsm =
 153     vec_elt_at_index (sm->per_thread_data, thread_index);
 154
 155   if (is_fwd_bypass_session (s))
 156     {
 157       ed_key.l_addr = s->in2out.addr;
 158       ed_key.r_addr = s->ext_host_addr;
 159       ed_key.l_port = s->in2out.port;
 160       ed_key.r_port = s->ext_host_port;
 161       ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 162       ed_key.fib_index = 0;
 163       ed_kv.key[0] = ed_key.as_u64[0];
 164       ed_kv.key[1] = ed_key.as_u64[1];
 165       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &ed_kv, 0))
 166         clib_warning ("in2out_ed key del failed");
 167       return;
 168     }
 169
 170   /* Endpoint dependent session lookup tables */
 171   if (is_ed_session (s))
 172     {
 173       ed_key.l_addr = s->out2in.addr;
 174       ed_key.r_addr = s->ext_host_addr;
 175       ed_key.fib_index = s->out2in.fib_index;
 176       if (snat_is_unk_proto_session (s))
 177         {
 178           ed_key.proto = s->in2out.port;
 179           ed_key.r_port = 0;
 180           ed_key.l_port = 0;
 181         }
 182       else
 183         {
 184           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 185           ed_key.l_port = s->out2in.port;
 186           ed_key.r_port = s->ext_host_port;
 187         }
 188       ed_kv.key[0] = ed_key.as_u64[0];
 189       ed_kv.key[1] = ed_key.as_u64[1];
 190       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 191         clib_warning ("out2in_ed key del failed");
 192
 193       ed_key.l_addr = s->in2out.addr;
 194       ed_key.fib_index = s->in2out.fib_index;
 195       if (!snat_is_unk_proto_session (s))
 196         ed_key.l_port = s->in2out.port;
 197       if (is_twice_nat_session (s))
 198         {
 199           ed_key.r_addr = s->ext_host_nat_addr;
 200           ed_key.r_port = s->ext_host_nat_port;
 201         }
 202       ed_kv.key[0] = ed_key.as_u64[0];
 203       ed_kv.key[1] = ed_key.as_u64[1];
 204       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &ed_kv, 0))
 205         clib_warning ("in2out_ed key del failed");
 206     }
 207
 208   if (snat_is_unk_proto_session (s))
 209     return;
 210
 211   /* log NAT event */
 212   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 213                                       s->out2in.addr.as_u32,
 214                                       s->in2out.protocol,
 215                                       s->in2out.port,
 216                                       s->out2in.port,
 217                                       s->in2out.fib_index);
 218
 219   /* Twice NAT address and port for external host */
 220   if (is_twice_nat_session (s))
 221     {
 222       for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 223         {
 224           key.protocol = s->in2out.protocol;
 225           key.port = s->ext_host_nat_port;
 226           a = sm->twice_nat_addresses + i;
 227           if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 228             {
 229               snat_free_outside_address_and_port (sm->twice_nat_addresses,
 230                                                   thread_index, &key, i);
 231               break;
 232             }
 233         }
 234     }
 235
 236   if (is_ed_session (s))
 237     return;
 238
 239   /* Session lookup tables */
 240   kv.key = s->in2out.as_u64;
 241   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 242     clib_warning ("in2out key del failed");
 243   kv.key = s->out2in.as_u64;
 244   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 245     clib_warning ("out2in key del failed");
 246
 247   if (snat_is_session_static (s))
 248     return;
 249
 250   if (s->outside_address_index != ~0)
 251     snat_free_outside_address_and_port (sm->addresses, thread_index,
 252                                         &s->out2in, s->outside_address_index);
 253 }
 254
 255 snat_user_t *
 256 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 257                         u32 thread_index)
 258 {
 259   snat_user_t *u = 0;
 260   snat_user_key_t user_key;
 261   clib_bihash_kv_8_8_t kv, value;
 262   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 263   dlist_elt_t * per_user_list_head_elt;
 264
 265   user_key.addr.as_u32 = addr->as_u32;
 266   user_key.fib_index = fib_index;
 267   kv.key = user_key.as_u64;
 268
 269   /* Ever heard of the "user" = src ip4 address before? */
 270   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 271     {
 272       /* no, make a new one */
 273       pool_get (tsm->users, u);
 274       memset (u, 0, sizeof (*u));
 275       u->addr.as_u32 = addr->as_u32;
 276       u->fib_index = fib_index;
 277
 278       pool_get (tsm->list_pool, per_user_list_head_elt);
 279
 280       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 281         tsm->list_pool;
 282
 283       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 284
 285       kv.value = u - tsm->users;
 286
 287       /* add user */
 288       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 289         clib_warning ("user_hash keay add failed");
 290     }
 291   else
 292     {
 293       u = pool_elt_at_index (tsm->users, value.value);
 294     }
 295
 296   return u;
 297 }
 298
 299 snat_session_t *
 300 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 301 {
 302   snat_session_t *s;
 303   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 304   u32 oldest_per_user_translation_list_index, session_index;
 305   dlist_elt_t * oldest_per_user_translation_list_elt;
 306   dlist_elt_t * per_user_translation_list_elt;
 307
 308   /* Over quota? Recycle the least recently used translation */
 309   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 310     {
 311       oldest_per_user_translation_list_index =
 312         clib_dlist_remove_head (tsm->list_pool,
 313                                 u->sessions_per_user_list_head_index);
 314
 315       ASSERT (oldest_per_user_translation_list_index != ~0);
 316
 317       /* Add it back to the end of the LRU list */
 318       clib_dlist_addtail (tsm->list_pool,
 319                           u->sessions_per_user_list_head_index,
 320                           oldest_per_user_translation_list_index);
 321       /* Get the list element */
 322       oldest_per_user_translation_list_elt =
 323         pool_elt_at_index (tsm->list_pool,
 324                            oldest_per_user_translation_list_index);
 325
 326       /* Get the session index from the list element */
 327       session_index = oldest_per_user_translation_list_elt->value;
 328
 329       /* Get the session */
 330       s = pool_elt_at_index (tsm->sessions, session_index);
 331       nat_free_session_data (sm, s, thread_index);
 332       s->outside_address_index = ~0;
 333       s->flags = 0;
 334       s->total_bytes = 0;
 335       s->total_pkts = 0;
 336     }
 337   else
 338     {
 339       pool_get (tsm->sessions, s);
 340       memset (s, 0, sizeof (*s));
 341       s->outside_address_index = ~0;
 342
 343       /* Create list elts */
 344       pool_get (tsm->list_pool, per_user_translation_list_elt);
 345       clib_dlist_init (tsm->list_pool,
 346                        per_user_translation_list_elt - tsm->list_pool);
 347
 348       per_user_translation_list_elt->value = s - tsm->sessions;
 349       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 350       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 351
 352       clib_dlist_addtail (tsm->list_pool,
 353                           s->per_user_list_head_index,
 354                           per_user_translation_list_elt - tsm->list_pool);
 355     }
 356
 357   return s;
 358 }
 359
 360 static inline uword
 361 nat44_classify_node_fn_inline (vlib_main_t * vm,
 362                                vlib_node_runtime_t * node,
 363                                vlib_frame_t * frame)
 364 {
 365   u32 n_left_from, * from, * to_next;
 366   nat44_classify_next_t next_index;
 367   snat_main_t *sm = &snat_main;
 368
 369   from = vlib_frame_vector_args (frame);
 370   n_left_from = frame->n_vectors;
 371   next_index = node->cached_next_index;
 372
 373   while (n_left_from > 0)
 374     {
 375       u32 n_left_to_next;
 376
 377       vlib_get_next_frame (vm, node, next_index,
 378                            to_next, n_left_to_next);
 379
 380       while (n_left_from > 0 && n_left_to_next > 0)
 381         {
 382           u32 bi0;
 383           vlib_buffer_t *b0;
 384           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT;
 385           ip4_header_t *ip0;
 386           snat_address_t *ap;
 387           snat_session_key_t m_key0;
 388           clib_bihash_kv_8_8_t kv0, value0;
 389
 390           /* speculatively enqueue b0 to the current next frame */
 391           bi0 = from[0];
 392           to_next[0] = bi0;
 393           from += 1;
 394           to_next += 1;
 395           n_left_from -= 1;
 396           n_left_to_next -= 1;
 397
 398           b0 = vlib_get_buffer (vm, bi0);
 399           ip0 = vlib_buffer_get_current (b0);
 400
 401           vec_foreach (ap, sm->addresses)
 402             {
 403               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 404                 {
 405                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 406                   goto enqueue0;
 407                 }
 408             }
 409
 410           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 411             {
 412               m_key0.addr = ip0->dst_address;
 413               m_key0.port = 0;
 414               m_key0.protocol = 0;
 415               m_key0.fib_index = sm->outside_fib_index;
 416               kv0.key = m_key0.as_u64;
 417               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 418                 {
 419                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 420                   goto enqueue0;
 421                 }
 422               udp_header_t * udp0 = ip4_next_header (ip0);
 423               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 424               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 425               kv0.key = m_key0.as_u64;
 426               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 427                 next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 428             }
 429
 430         enqueue0:
 431           /* verify speculative enqueue, maybe switch current next frame */
 432           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 433                                            to_next, n_left_to_next,
 434                                            bi0, next0);
 435         }
 436
 437       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 438     }
 439
 440   return frame->n_vectors;
 441 }
 442
 443 static uword
 444 nat44_classify_node_fn (vlib_main_t * vm,
 445                         vlib_node_runtime_t * node,
 446                         vlib_frame_t * frame)
 447 {
 448   return nat44_classify_node_fn_inline (vm, node, frame);
 449 };
 450
 451 VLIB_REGISTER_NODE (nat44_classify_node) = {
 452   .function = nat44_classify_node_fn,
 453   .name = "nat44-classify",
 454   .vector_size = sizeof (u32),
 455   .type = VLIB_NODE_TYPE_INTERNAL,
 456   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 457   .next_nodes = {
 458     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 459     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 460   },
 461 };
 462
 463 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 464                               nat44_classify_node_fn);
 465
 466 static uword
 467 nat44_det_classify_node_fn (vlib_main_t * vm,
 468                             vlib_node_runtime_t * node,
 469                             vlib_frame_t * frame)
 470 {
 471   return nat44_classify_node_fn_inline (vm, node, frame);
 472 };
 473
 474 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 475   .function = nat44_det_classify_node_fn,
 476   .name = "nat44-det-classify",
 477   .vector_size = sizeof (u32),
 478   .type = VLIB_NODE_TYPE_INTERNAL,
 479   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 480   .next_nodes = {
 481     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 482     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 483   },
 484 };
 485
 486 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 487                               nat44_det_classify_node_fn);
 488
 489 static uword
 490 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 491                                 vlib_node_runtime_t * node,
 492                                 vlib_frame_t * frame)
 493 {
 494   return nat44_classify_node_fn_inline (vm, node, frame);
 495 };
 496
 497 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 498   .function = nat44_handoff_classify_node_fn,
 499   .name = "nat44-handoff-classify",
 500   .vector_size = sizeof (u32),
 501   .type = VLIB_NODE_TYPE_INTERNAL,
 502   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 503   .next_nodes = {
 504     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 505     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 506   },
 507 };
 508
 509 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 510                               nat44_handoff_classify_node_fn);
 511
 512 /**
 513  * @brief Add/del NAT address to FIB.
 514  *
 515  * Add the external NAT address to the FIB as receive entries. This ensures
 516  * that VPP will reply to ARP for this address and we don't need to enable
 517  * proxy ARP on the outside interface.
 518  *
 519  * @param addr IPv4 address.
 520  * @param plen address prefix length
 521  * @param sw_if_index Interface.
 522  * @param is_add If 0 delete, otherwise add.
 523  */
 524 void
 525 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 526                           int is_add)
 527 {
 528   fib_prefix_t prefix = {
 529     .fp_len = p_len,
 530     .fp_proto = FIB_PROTOCOL_IP4,
 531     .fp_addr = {
 532         .ip4.as_u32 = addr->as_u32,
 533     },
 534   };
 535   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 536
 537   if (is_add)
 538     fib_table_entry_update_one_path(fib_index,
 539                                     &prefix,
 540                                     FIB_SOURCE_PLUGIN_LOW,
 541                                     (FIB_ENTRY_FLAG_CONNECTED |
 542                                      FIB_ENTRY_FLAG_LOCAL |
 543                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 544                                     DPO_PROTO_IP4,
 545                                     NULL,
 546                                     sw_if_index,
 547                                     ~0,
 548                                     1,
 549                                     NULL,
 550                                     FIB_ROUTE_PATH_FLAG_NONE);
 551   else
 552     fib_table_entry_delete(fib_index,
 553                            &prefix,
 554                            FIB_SOURCE_PLUGIN_LOW);
 555 }
 556
 557 void snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 558                        u8 twice_nat)
 559 {
 560   snat_address_t * ap;
 561   snat_interface_t *i;
 562   vlib_thread_main_t *tm = vlib_get_thread_main ();
 563
 564   /* Check if address already exists */
 565   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 566     {
 567       if (ap->addr.as_u32 == addr->as_u32)
 568         return;
 569     }
 570
 571   if (twice_nat)
 572     vec_add2 (sm->twice_nat_addresses, ap, 1);
 573   else
 574     vec_add2 (sm->addresses, ap, 1);
 575
 576   ap->addr = *addr;
 577   if (vrf_id != ~0)
 578     ap->fib_index =
 579       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 580                                          FIB_SOURCE_PLUGIN_LOW);
 581   else
 582     ap->fib_index = ~0;
 583 #define _(N, i, n, s) \
 584   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 585   ap->busy_##n##_ports = 0; \
 586   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 587   foreach_snat_protocol
 588 #undef _
 589
 590   if (twice_nat)
 591     return;
 592
 593   /* Add external address to FIB */
 594   pool_foreach (i, sm->interfaces,
 595   ({
 596     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 597       continue;
 598
 599     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 600     break;
 601   }));
 602   pool_foreach (i, sm->output_feature_interfaces,
 603   ({
 604     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 605       continue;
 606
 607     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 608     break;
 609   }));
 610 }
 611
 612 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 613                                                    ip4_address_t addr)
 614 {
 615   snat_static_mapping_t *m;
 616   pool_foreach (m, sm->static_mappings,
 617   ({
 618       if (m->external_addr.as_u32 == addr.as_u32)
 619         return 1;
 620   }));
 621
 622   return 0;
 623 }
 624
 625 void increment_v4_address (ip4_address_t * a)
 626 {
 627   u32 v;
 628
 629   v = clib_net_to_host_u32(a->as_u32) + 1;
 630   a->as_u32 = clib_host_to_net_u32(v);
 631 }
 632
 633 static void
 634 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 635                                        ip4_address_t l_addr,
 636                                        u16 l_port,
 637                                        u32 sw_if_index,
 638                                        u16 e_port,
 639                                        u32 vrf_id,
 640                                        snat_protocol_t proto,
 641                                        int addr_only,
 642                                        int is_add,
 643                                        u8 * tag)
 644 {
 645   snat_static_map_resolve_t *rp;
 646
 647   vec_add2 (sm->to_resolve, rp, 1);
 648   rp->l_addr.as_u32 = l_addr.as_u32;
 649   rp->l_port = l_port;
 650   rp->sw_if_index = sw_if_index;
 651   rp->e_port = e_port;
 652   rp->vrf_id = vrf_id;
 653   rp->proto = proto;
 654   rp->addr_only = addr_only;
 655   rp->is_add = is_add;
 656   rp->tag = vec_dup (tag);
 657 }
 658
 659 /**
 660  * @brief Add static mapping.
 661  *
 662  * Create static mapping between local addr+port and external addr+port.
 663  *
 664  * @param l_addr Local IPv4 address.
 665  * @param e_addr External IPv4 address.
 666  * @param l_port Local port number.
 667  * @param e_port External port number.
 668  * @param vrf_id VRF ID.
 669  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 670  * @param sw_if_index External port instead of specific IP address.
 671  * @param is_add If 0 delete static mapping, otherwise add.
 672  * @param twice_nat If 1 translate external host address and port.
 673  * @param out2in_only If 1 rule match only out2in direction
 674  * @param tag - opaque string tag
 675  *
 676  * @returns
 677  */
 678 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 679                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 680                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 681                             u8 twice_nat, u8 out2in_only, u8 * tag)
 682 {
 683   snat_main_t * sm = &snat_main;
 684   snat_static_mapping_t *m;
 685   snat_session_key_t m_key;
 686   clib_bihash_kv_8_8_t kv, value;
 687   snat_address_t *a = 0;
 688   u32 fib_index = ~0;
 689   uword * p;
 690   snat_interface_t *interface;
 691   int i;
 692   snat_main_per_thread_data_t *tsm;
 693   snat_user_key_t u_key;
 694   snat_user_t *u;
 695   dlist_elt_t * head, * elt;
 696   u32 elt_index, head_index;
 697   u32 ses_index;
 698   u64 user_index;
 699   snat_session_t * s;
 700   snat_static_map_resolve_t *rp, *rp_match = 0;
 701
 702   /* If the external address is a specific interface address */
 703   if (sw_if_index != ~0)
 704     {
 705       ip4_address_t * first_int_addr;
 706
 707       for (i = 0; i < vec_len (sm->to_resolve); i++)
 708         {
 709           rp = sm->to_resolve + i;
 710           if (rp->sw_if_index != sw_if_index &&
 711               rp->l_addr.as_u32 != l_addr.as_u32 &&
 712               rp->vrf_id != vrf_id && rp->addr_only != addr_only)
 713             continue;
 714
 715           if (!addr_only)
 716             {
 717               if (rp->l_port != l_port && rp->e_port != e_port && rp->proto != proto)
 718                 continue;
 719             }
 720
 721           rp_match = rp;
 722           break;
 723         }
 724
 725       /* Might be already set... */
 726       first_int_addr = ip4_interface_first_address
 727         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 728
 729       if (is_add)
 730         {
 731           if (rp_match)
 732             return VNET_API_ERROR_VALUE_EXIST;
 733
 734           snat_add_static_mapping_when_resolved
 735             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 736              addr_only,  is_add, tag);
 737
 738           /* DHCP resolution required? */
 739           if (first_int_addr == 0)
 740             {
 741               return 0;
 742             }
 743           else
 744             {
 745               e_addr.as_u32 = first_int_addr->as_u32;
 746               /* Identity mapping? */
 747               if (l_addr.as_u32 == 0)
 748                 l_addr.as_u32 = e_addr.as_u32;
 749             }
 750         }
 751       else
 752         {
 753           if (!rp_match)
 754             return VNET_API_ERROR_NO_SUCH_ENTRY;
 755
 756           vec_del1 (sm->to_resolve, i);
 757
 758           if (first_int_addr)
 759             {
 760               e_addr.as_u32 = first_int_addr->as_u32;
 761               /* Identity mapping? */
 762               if (l_addr.as_u32 == 0)
 763                 l_addr.as_u32 = e_addr.as_u32;
 764             }
 765           else
 766             return 0;
 767         }
 768     }
 769
 770   m_key.addr = e_addr;
 771   m_key.port = addr_only ? 0 : e_port;
 772   m_key.protocol = addr_only ? 0 : proto;
 773   m_key.fib_index = sm->outside_fib_index;
 774   kv.key = m_key.as_u64;
 775   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 776     m = 0;
 777   else
 778     m = pool_elt_at_index (sm->static_mappings, value.value);
 779
 780   if (is_add)
 781     {
 782       if (m)
 783         return VNET_API_ERROR_VALUE_EXIST;
 784
 785       if (twice_nat && addr_only)
 786         return VNET_API_ERROR_UNSUPPORTED;
 787
 788       /* Convert VRF id to FIB index */
 789       if (vrf_id != ~0)
 790         {
 791           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 792           if (!p)
 793             return VNET_API_ERROR_NO_SUCH_FIB;
 794           fib_index = p[0];
 795         }
 796       /* If not specified use inside VRF id from SNAT plugin startup config */
 797       else
 798         {
 799           fib_index = sm->inside_fib_index;
 800           vrf_id = sm->inside_vrf_id;
 801         }
 802
 803       if (!out2in_only)
 804         {
 805           m_key.addr = l_addr;
 806           m_key.port = addr_only ? 0 : l_port;
 807           m_key.protocol = addr_only ? 0 : proto;
 808           m_key.fib_index = fib_index;
 809           kv.key = m_key.as_u64;
 810           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 811             return VNET_API_ERROR_VALUE_EXIST;
 812         }
 813
 814       /* Find external address in allocated addresses and reserve port for
 815          address and port pair mapping when dynamic translations enabled */
 816       if (!(addr_only || sm->static_mapping_only || out2in_only))
 817         {
 818           for (i = 0; i < vec_len (sm->addresses); i++)
 819             {
 820               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 821                 {
 822                   a = sm->addresses + i;
 823                   /* External port must be unused */
 824                   switch (proto)
 825                     {
 826 #define _(N, j, n, s) \
 827                     case SNAT_PROTOCOL_##N: \
 828                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 829                         return VNET_API_ERROR_INVALID_VALUE; \
 830                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 831                       if (e_port > 1024) \
 832                         { \
 833                           a->busy_##n##_ports++; \
 834                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
 835                         } \
 836                       break;
 837                       foreach_snat_protocol
 838 #undef _
 839                     default:
 840                       clib_warning("unknown_protocol");
 841                       return VNET_API_ERROR_INVALID_VALUE_2;
 842                     }
 843                   break;
 844                 }
 845             }
 846           /* External address must be allocated */
 847           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 848             {
 849               if (sw_if_index != ~0)
 850                 {
 851                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 852                     {
 853                       rp = sm->to_resolve + i;
 854                       if (rp->addr_only)
 855                          continue;
 856                       if (rp->sw_if_index != sw_if_index &&
 857                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 858                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 859                           rp->e_port != e_port && rp->proto != proto)
 860                         continue;
 861
 862                       vec_del1 (sm->to_resolve, i);
 863                       break;
 864                     }
 865                 }
 866               return VNET_API_ERROR_NO_SUCH_ENTRY;
 867             }
 868         }
 869
 870       pool_get (sm->static_mappings, m);
 871       memset (m, 0, sizeof (*m));
 872       m->tag = vec_dup (tag);
 873       m->local_addr = l_addr;
 874       m->external_addr = e_addr;
 875       m->addr_only = addr_only;
 876       m->vrf_id = vrf_id;
 877       m->fib_index = fib_index;
 878       m->twice_nat = twice_nat;
 879       m->out2in_only = out2in_only;
 880       if (!addr_only)
 881         {
 882           m->local_port = l_port;
 883           m->external_port = e_port;
 884           m->proto = proto;
 885         }
 886
 887       if (sm->workers)
 888         {
 889           ip4_header_t ip = {
 890             .src_address = m->local_addr,
 891           };
 892           m->worker_index = sm->worker_in2out_cb (&ip, m->fib_index);
 893           tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
 894         }
 895       else
 896         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 897
 898       m_key.addr = m->local_addr;
 899       m_key.port = m->local_port;
 900       m_key.protocol = m->proto;
 901       m_key.fib_index = m->fib_index;
 902       kv.key = m_key.as_u64;
 903       kv.value = m - sm->static_mappings;
 904       if (!out2in_only)
 905         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
 906       if (twice_nat || out2in_only)
 907         {
 908           m_key.port = clib_host_to_net_u16 (l_port);
 909           kv.key = m_key.as_u64;
 910           kv.value = ~0ULL;
 911           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 1))
 912             clib_warning ("in2out key add failed");
 913         }
 914
 915       m_key.addr = m->external_addr;
 916       m_key.port = m->external_port;
 917       m_key.fib_index = sm->outside_fib_index;
 918       kv.key = m_key.as_u64;
 919       kv.value = m - sm->static_mappings;
 920       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
 921       if (twice_nat || out2in_only)
 922         {
 923           m_key.port = clib_host_to_net_u16 (e_port);
 924           kv.key = m_key.as_u64;
 925           kv.value = ~0ULL;
 926           if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 1))
 927             clib_warning ("out2in key add failed");
 928         }
 929
 930       /* Delete dynamic sessions matching local address (+ local port) */
 931       if (!(sm->static_mapping_only))
 932         {
 933           u_key.addr = m->local_addr;
 934           u_key.fib_index = m->fib_index;
 935           kv.key = u_key.as_u64;
 936           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 937             {
 938               user_index = value.value;
 939               u = pool_elt_at_index (tsm->users, user_index);
 940               if (u->nsessions)
 941                 {
 942                   head_index = u->sessions_per_user_list_head_index;
 943                   head = pool_elt_at_index (tsm->list_pool, head_index);
 944                   elt_index = head->next;
 945                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 946                   ses_index = elt->value;
 947                   while (ses_index != ~0)
 948                     {
 949                       s =  pool_elt_at_index (tsm->sessions, ses_index);
 950                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 951                       ses_index = elt->value;
 952
 953                       if (snat_is_session_static (s))
 954                         continue;
 955
 956                       if (!addr_only)
 957                         {
 958                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
 959                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
 960                             continue;
 961                         }
 962
 963                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
 964                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
 965                       pool_put_index (tsm->list_pool, s->per_user_index);
 966                       pool_put (tsm->sessions, s);
 967                       u->nsessions--;
 968
 969                       if (!addr_only)
 970                         break;
 971                     }
 972                 }
 973             }
 974         }
 975     }
 976   else
 977     {
 978       if (!m)
 979         {
 980           if (sw_if_index != ~0)
 981             return 0;
 982           else
 983             return VNET_API_ERROR_NO_SUCH_ENTRY;
 984         }
 985
 986       /* Free external address port */
 987       if (!(addr_only || sm->static_mapping_only || out2in_only))
 988         {
 989           for (i = 0; i < vec_len (sm->addresses); i++)
 990             {
 991               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 992                 {
 993                   a = sm->addresses + i;
 994                   switch (proto)
 995                     {
 996 #define _(N, j, n, s) \
 997                     case SNAT_PROTOCOL_##N: \
 998                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
 999                       if (e_port > 1024) \
1000                         { \
1001                           a->busy_##n##_ports--; \
1002                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1003                         } \
1004                       break;
1005                       foreach_snat_protocol
1006 #undef _
1007                     default:
1008                       clib_warning("unknown_protocol");
1009                       return VNET_API_ERROR_INVALID_VALUE_2;
1010                     }
1011                   break;
1012                 }
1013             }
1014         }
1015
1016       if (sm->num_workers > 1)
1017         tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
1018       else
1019         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1020
1021       m_key.addr = m->local_addr;
1022       m_key.port = m->local_port;
1023       m_key.protocol = m->proto;
1024       m_key.fib_index = m->fib_index;
1025       kv.key = m_key.as_u64;
1026       if (!out2in_only)
1027         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1028       if (twice_nat || out2in_only)
1029         {
1030           m_key.port = clib_host_to_net_u16 (m->local_port);
1031           kv.key = m_key.as_u64;
1032           kv.value = ~0ULL;
1033           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 0))
1034             clib_warning ("in2out key del failed");
1035         }
1036
1037       m_key.addr = m->external_addr;
1038       m_key.port = m->external_port;
1039       m_key.fib_index = sm->outside_fib_index;
1040       kv.key = m_key.as_u64;
1041       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1042       if (twice_nat || out2in_only)
1043         {
1044           m_key.port = clib_host_to_net_u16 (m->external_port);
1045           kv.key = m_key.as_u64;
1046           kv.value = ~0ULL;
1047           if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 0))
1048             clib_warning ("in2out key del failed");
1049         }
1050
1051       /* Delete session(s) for static mapping if exist */
1052       if (!(sm->static_mapping_only) ||
1053           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1054         {
1055           u_key.addr = m->local_addr;
1056           u_key.fib_index = m->fib_index;
1057           kv.key = u_key.as_u64;
1058           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1059             {
1060               user_index = value.value;
1061               u = pool_elt_at_index (tsm->users, user_index);
1062               if (u->nstaticsessions)
1063                 {
1064                   head_index = u->sessions_per_user_list_head_index;
1065                   head = pool_elt_at_index (tsm->list_pool, head_index);
1066                   elt_index = head->next;
1067                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1068                   ses_index = elt->value;
1069                   while (ses_index != ~0)
1070                     {
1071                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1072                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1073                       ses_index = elt->value;
1074
1075                       if (!addr_only)
1076                         {
1077                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
1078                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1079                             continue;
1080                         }
1081
1082                       if (is_lb_session (s))
1083                         continue;
1084
1085                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1086                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
1087                       pool_put_index (tsm->list_pool, s->per_user_index);
1088                       pool_put (tsm->sessions, s);
1089                       u->nstaticsessions--;
1090
1091                       if (!addr_only)
1092                         break;
1093                     }
1094                   if (addr_only && (u->nstaticsessions == 0))
1095                     {
1096                       pool_put (tsm->users, u);
1097                       clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 0);
1098                     }
1099                 }
1100             }
1101         }
1102
1103       vec_free (m->tag);
1104       /* Delete static mapping from pool */
1105       pool_put (sm->static_mappings, m);
1106     }
1107
1108   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1109     return 0;
1110
1111   /* Add/delete external address to FIB */
1112   pool_foreach (interface, sm->interfaces,
1113   ({
1114     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1115       continue;
1116
1117     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1118     break;
1119   }));
1120   pool_foreach (interface, sm->output_feature_interfaces,
1121   ({
1122     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1123       continue;
1124
1125     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1126     break;
1127   }));
1128
1129   return 0;
1130 }
1131
1132 static int lb_local_exists (nat44_lb_addr_port_t * local,
1133                             ip4_address_t * e_addr, u16 e_port)
1134 {
1135   snat_main_t *sm = &snat_main;
1136   snat_static_mapping_t *m;
1137   nat44_lb_addr_port_t *ap;
1138
1139   /* *INDENT-OFF* */
1140   pool_foreach (m, sm->static_mappings,
1141   ({
1142       if (vec_len(m->locals))
1143         {
1144           if (m->external_port == e_port && m->external_addr.as_u32 == e_addr->as_u32)
1145             continue;
1146
1147           vec_foreach (ap, m->locals)
1148           {
1149             if (ap->port == local->port && ap->addr.as_u32 == local->addr.as_u32)
1150               return 1;
1151           }
1152         }
1153   }));
1154   /* *INDENT-ON* */
1155
1156   return 0;
1157 }
1158
1159 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1160                                      snat_protocol_t proto, u32 vrf_id,
1161                                      nat44_lb_addr_port_t *locals, u8 is_add,
1162                                      u8 twice_nat, u8 out2in_only, u8 *tag)
1163 {
1164   snat_main_t * sm = &snat_main;
1165   snat_static_mapping_t *m;
1166   snat_session_key_t m_key;
1167   clib_bihash_kv_8_8_t kv, value;
1168   u32 fib_index;
1169   snat_address_t *a = 0;
1170   int i;
1171   nat44_lb_addr_port_t *local;
1172   u32 worker_index = 0, elt_index, head_index, ses_index;
1173   snat_main_per_thread_data_t *tsm;
1174   snat_user_key_t u_key;
1175   snat_user_t *u;
1176   snat_session_t * s;
1177   dlist_elt_t * head, * elt;
1178
1179   m_key.addr = e_addr;
1180   m_key.port = e_port;
1181   m_key.protocol = proto;
1182   m_key.fib_index = sm->outside_fib_index;
1183   kv.key = m_key.as_u64;
1184   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1185     m = 0;
1186   else
1187     m = pool_elt_at_index (sm->static_mappings, value.value);
1188
1189   if (is_add)
1190     {
1191       if (m)
1192         return VNET_API_ERROR_VALUE_EXIST;
1193
1194       if (vec_len (locals) < 2)
1195         return VNET_API_ERROR_INVALID_VALUE;
1196
1197       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1198                                                      vrf_id,
1199                                                      FIB_SOURCE_PLUGIN_LOW);
1200
1201       /* Find external address in allocated addresses and reserve port for
1202          address and port pair mapping when dynamic translations enabled */
1203       if (!(sm->static_mapping_only || out2in_only))
1204         {
1205           for (i = 0; i < vec_len (sm->addresses); i++)
1206             {
1207               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1208                 {
1209                   a = sm->addresses + i;
1210                   /* External port must be unused */
1211                   switch (proto)
1212                     {
1213 #define _(N, j, n, s) \
1214                     case SNAT_PROTOCOL_##N: \
1215                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1216                         return VNET_API_ERROR_INVALID_VALUE; \
1217                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1218                       if (e_port > 1024) \
1219                         { \
1220                           a->busy_##n##_ports++; \
1221                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1222                         } \
1223                       break;
1224                       foreach_snat_protocol
1225 #undef _
1226                     default:
1227                       clib_warning("unknown_protocol");
1228                       return VNET_API_ERROR_INVALID_VALUE_2;
1229                     }
1230                   break;
1231                 }
1232             }
1233           /* External address must be allocated */
1234           if (!a)
1235             return VNET_API_ERROR_NO_SUCH_ENTRY;
1236         }
1237
1238       pool_get (sm->static_mappings, m);
1239       memset (m, 0, sizeof (*m));
1240       m->tag = vec_dup (tag);
1241       m->external_addr = e_addr;
1242       m->addr_only = 0;
1243       m->vrf_id = vrf_id;
1244       m->fib_index = fib_index;
1245       m->external_port = e_port;
1246       m->proto = proto;
1247       m->twice_nat = twice_nat;
1248       m->out2in_only = out2in_only;
1249
1250       m_key.addr = m->external_addr;
1251       m_key.port = m->external_port;
1252       m_key.protocol = m->proto;
1253       m_key.fib_index = sm->outside_fib_index;
1254       kv.key = m_key.as_u64;
1255       kv.value = m - sm->static_mappings;
1256       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1257         {
1258           clib_warning ("static_mapping_by_external key add failed");
1259           return VNET_API_ERROR_UNSPECIFIED;
1260         }
1261
1262       /* Assign worker */
1263       if (sm->workers)
1264         {
1265           worker_index = sm->first_worker_index +
1266             sm->workers[sm->next_worker++ % vec_len (sm->workers)];
1267           tsm = vec_elt_at_index (sm->per_thread_data, worker_index);
1268           m->worker_index = worker_index;
1269         }
1270       else
1271         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1272
1273       m_key.port = clib_host_to_net_u16 (m->external_port);
1274       kv.key = m_key.as_u64;
1275       kv.value = ~0ULL;
1276       if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 1))
1277         {
1278           clib_warning ("out2in key add failed");
1279           return VNET_API_ERROR_UNSPECIFIED;
1280         }
1281
1282       m_key.fib_index = m->fib_index;
1283       for (i = 0; i < vec_len (locals); i++)
1284         {
1285           m_key.addr = locals[i].addr;
1286           if (!out2in_only)
1287             {
1288               m_key.port = locals[i].port;
1289               kv.key = m_key.as_u64;
1290               kv.value = m - sm->static_mappings;
1291               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1292             }
1293           locals[i].prefix = (i == 0) ? locals[i].probability :\
1294             (locals[i - 1].prefix + locals[i].probability);
1295           vec_add1 (m->locals, locals[i]);
1296
1297           m_key.port = clib_host_to_net_u16 (locals[i].port);
1298           kv.key = m_key.as_u64;
1299           kv.value = ~0ULL;
1300           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 1))
1301             {
1302               clib_warning ("in2out key add failed");
1303               return VNET_API_ERROR_UNSPECIFIED;
1304             }
1305         }
1306     }
1307   else
1308     {
1309       if (!m)
1310         return VNET_API_ERROR_NO_SUCH_ENTRY;
1311
1312       fib_table_unlock (m->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1313
1314       /* Free external address port */
1315       if (!(sm->static_mapping_only || out2in_only))
1316         {
1317           for (i = 0; i < vec_len (sm->addresses); i++)
1318             {
1319               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1320                 {
1321                   a = sm->addresses + i;
1322                   switch (proto)
1323                     {
1324 #define _(N, j, n, s) \
1325                     case SNAT_PROTOCOL_##N: \
1326                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1327                       if (e_port > 1024) \
1328                         { \
1329                           a->busy_##n##_ports--; \
1330                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1331                         } \
1332                       break;
1333                       foreach_snat_protocol
1334 #undef _
1335                     default:
1336                       clib_warning("unknown_protocol");
1337                       return VNET_API_ERROR_INVALID_VALUE_2;
1338                     }
1339                   break;
1340                 }
1341             }
1342         }
1343
1344       tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
1345       m_key.addr = m->external_addr;
1346       m_key.port = m->external_port;
1347       m_key.protocol = m->proto;
1348       m_key.fib_index = sm->outside_fib_index;
1349       kv.key = m_key.as_u64;
1350       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1351         {
1352           clib_warning ("static_mapping_by_external key del failed");
1353           return VNET_API_ERROR_UNSPECIFIED;
1354         }
1355
1356       m_key.port = clib_host_to_net_u16 (m->external_port);
1357       kv.key = m_key.as_u64;
1358       if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 0))
1359         {
1360           clib_warning ("outi2in key del failed");
1361           return VNET_API_ERROR_UNSPECIFIED;
1362         }
1363
1364       vec_foreach (local, m->locals)
1365         {
1366           m_key.addr = local->addr;
1367           if (!out2in_only)
1368             {
1369               m_key.port = local->port;
1370               m_key.fib_index = m->fib_index;
1371               kv.key = m_key.as_u64;
1372               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1373                 {
1374                   clib_warning ("static_mapping_by_local key del failed");
1375                   return VNET_API_ERROR_UNSPECIFIED;
1376                 }
1377             }
1378
1379           if (!lb_local_exists(local, &e_addr, e_port))
1380             {
1381               m_key.port = clib_host_to_net_u16 (local->port);
1382               kv.key = m_key.as_u64;
1383               if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 0))
1384                 {
1385                   clib_warning ("in2out key del failed");
1386                   return VNET_API_ERROR_UNSPECIFIED;
1387                 }
1388             }
1389           /* Delete sessions */
1390           u_key.addr = local->addr;
1391           u_key.fib_index = m->fib_index;
1392           kv.key = u_key.as_u64;
1393           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1394             {
1395               u = pool_elt_at_index (tsm->users, value.value);
1396               if (u->nstaticsessions)
1397                 {
1398                   head_index = u->sessions_per_user_list_head_index;
1399                   head = pool_elt_at_index (tsm->list_pool, head_index);
1400                   elt_index = head->next;
1401                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1402                   ses_index = elt->value;
1403                   while (ses_index != ~0)
1404                     {
1405                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1406                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1407                       ses_index = elt->value;
1408
1409                       if (!(is_lb_session (s)))
1410                         continue;
1411
1412                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) &&
1413                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1414                         continue;
1415
1416                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1417                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
1418                       pool_put_index (tsm->list_pool, s->per_user_index);
1419                       pool_put (tsm->sessions, s);
1420                       u->nstaticsessions--;
1421                     }
1422                 }
1423             }
1424         }
1425       vec_free(m->locals);
1426       vec_free(m->tag);
1427
1428       pool_put (sm->static_mappings, m);
1429     }
1430
1431   return 0;
1432 }
1433
1434 int
1435 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1436                   u8 twice_nat)
1437 {
1438   snat_address_t *a = 0;
1439   snat_session_t *ses;
1440   u32 *ses_to_be_removed = 0, *ses_index;
1441   clib_bihash_kv_8_8_t kv, value;
1442   snat_user_key_t user_key;
1443   snat_user_t *u;
1444   snat_main_per_thread_data_t *tsm;
1445   snat_static_mapping_t *m;
1446   snat_interface_t *interface;
1447   int i;
1448   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1449
1450   /* Find SNAT address */
1451   for (i=0; i < vec_len (addresses); i++)
1452     {
1453       if (addresses[i].addr.as_u32 == addr.as_u32)
1454         {
1455           a = addresses + i;
1456           break;
1457         }
1458     }
1459   if (!a)
1460     return VNET_API_ERROR_NO_SUCH_ENTRY;
1461
1462   if (delete_sm)
1463     {
1464       pool_foreach (m, sm->static_mappings,
1465       ({
1466           if (m->external_addr.as_u32 == addr.as_u32)
1467             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1468                                             m->local_port, m->external_port,
1469                                             m->vrf_id, m->addr_only, ~0,
1470                                             m->proto, 0, m->twice_nat,
1471                                             m->out2in_only, m->tag);
1472       }));
1473     }
1474   else
1475     {
1476       /* Check if address is used in some static mapping */
1477       if (is_snat_address_used_in_static_mapping(sm, addr))
1478         {
1479           clib_warning ("address used in static mapping");
1480           return VNET_API_ERROR_UNSPECIFIED;
1481         }
1482     }
1483
1484   if (a->fib_index != ~0)
1485     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1486                      FIB_SOURCE_PLUGIN_LOW);
1487
1488   /* Delete sessions using address */
1489   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1490     {
1491       vec_foreach (tsm, sm->per_thread_data)
1492         {
1493           pool_foreach (ses, tsm->sessions, ({
1494             if (ses->out2in.addr.as_u32 == addr.as_u32)
1495               {
1496                 ses->outside_address_index = ~0;
1497                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1498                 clib_dlist_remove (tsm->list_pool, ses->per_user_index);
1499                 pool_put_index (tsm->list_pool, ses->per_user_index);
1500                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1501                 user_key.addr = ses->in2out.addr;
1502                 user_key.fib_index = ses->in2out.fib_index;
1503                 kv.key = user_key.as_u64;
1504                 if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1505                   {
1506                     u = pool_elt_at_index (tsm->users, value.value);
1507                     u->nsessions--;
1508                   }
1509               }
1510           }));
1511
1512           vec_foreach (ses_index, ses_to_be_removed)
1513             pool_put_index (tsm->sessions, ses_index[0]);
1514
1515           vec_free (ses_to_be_removed);
1516        }
1517     }
1518
1519   if (twice_nat)
1520     {
1521       vec_del1 (sm->twice_nat_addresses, i);
1522       return 0;
1523     }
1524   else
1525     vec_del1 (sm->addresses, i);
1526
1527   /* Delete external address from FIB */
1528   pool_foreach (interface, sm->interfaces,
1529   ({
1530     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1531       continue;
1532
1533     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1534     break;
1535   }));
1536   pool_foreach (interface, sm->output_feature_interfaces,
1537   ({
1538     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1539       continue;
1540
1541     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1542     break;
1543   }));
1544
1545   return 0;
1546 }
1547
1548 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1549 {
1550   snat_main_t *sm = &snat_main;
1551   snat_interface_t *i;
1552   const char * feature_name, *del_feature_name;
1553   snat_address_t * ap;
1554   snat_static_mapping_t * m;
1555   snat_det_map_t * dm;
1556
1557   if (sm->out2in_dpo && !is_inside)
1558     return VNET_API_ERROR_UNSUPPORTED;
1559
1560   pool_foreach (i, sm->output_feature_interfaces,
1561   ({
1562     if (i->sw_if_index == sw_if_index)
1563       return VNET_API_ERROR_VALUE_EXIST;
1564   }));
1565
1566   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1567     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1568   else
1569     {
1570       if (sm->num_workers > 1 && !sm->deterministic)
1571         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1572       else if (sm->deterministic)
1573         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1574       else
1575         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1576     }
1577
1578   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1579     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index, 0);
1580
1581   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1582     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1583
1584   pool_foreach (i, sm->interfaces,
1585   ({
1586     if (i->sw_if_index == sw_if_index)
1587       {
1588         if (is_del)
1589           {
1590             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1591               {
1592                 if (is_inside)
1593                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1594                 else
1595                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1596
1597                 if (sm->num_workers > 1 && !sm->deterministic)
1598                   {
1599                     del_feature_name = "nat44-handoff-classify";
1600                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1601                                                  "nat44-out2in-worker-handoff";
1602                   }
1603                 else if (sm->deterministic)
1604                   {
1605                     del_feature_name = "nat44-det-classify";
1606                     feature_name = !is_inside ?  "nat44-det-in2out" :
1607                                                  "nat44-det-out2in";
1608                   }
1609                 else
1610                   {
1611                     del_feature_name = "nat44-classify";
1612                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1613                   }
1614
1615                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1616                                              sw_if_index, 0, 0, 0);
1617                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1618                                              sw_if_index, 1, 0, 0);
1619               }
1620             else
1621               {
1622                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1623                                              sw_if_index, 0, 0, 0);
1624                 pool_put (sm->interfaces, i);
1625               }
1626           }
1627         else
1628           {
1629             if ((nat_interface_is_inside(i) && is_inside) ||
1630                 (nat_interface_is_outside(i) && !is_inside))
1631               return 0;
1632
1633             if (sm->num_workers > 1 && !sm->deterministic)
1634               {
1635                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1636                                                  "nat44-out2in-worker-handoff";
1637                 feature_name = "nat44-handoff-classify";
1638               }
1639             else if (sm->deterministic)
1640               {
1641                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1642                                                  "nat44-det-out2in";
1643                 feature_name = "nat44-det-classify";
1644               }
1645             else
1646               {
1647                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1648                 feature_name = "nat44-classify";
1649               }
1650
1651             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1652                                          sw_if_index, 0, 0, 0);
1653             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1654                                          sw_if_index, 1, 0, 0);
1655             goto set_flags;
1656           }
1657
1658         goto fib;
1659       }
1660   }));
1661
1662   if (is_del)
1663     return VNET_API_ERROR_NO_SUCH_ENTRY;
1664
1665   pool_get (sm->interfaces, i);
1666   i->sw_if_index = sw_if_index;
1667   i->flags = 0;
1668   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1669
1670 set_flags:
1671   if (is_inside)
1672     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1673   else
1674     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1675
1676   /* Add/delete external addresses to FIB */
1677 fib:
1678   if (is_inside && !sm->out2in_dpo)
1679     {
1680       vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1681                                    sw_if_index, !is_del, 0, 0);
1682       return 0;
1683     }
1684
1685   vec_foreach (ap, sm->addresses)
1686     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1687
1688   pool_foreach (m, sm->static_mappings,
1689   ({
1690     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1691       continue;
1692
1693     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1694   }));
1695
1696   pool_foreach (dm, sm->det_maps,
1697   ({
1698     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1699   }));
1700
1701   return 0;
1702 }
1703
1704 int snat_interface_add_del_output_feature (u32 sw_if_index,
1705                                            u8 is_inside,
1706                                            int is_del)
1707 {
1708   snat_main_t *sm = &snat_main;
1709   snat_interface_t *i;
1710   snat_address_t * ap;
1711   snat_static_mapping_t * m;
1712
1713   if (sm->deterministic ||
1714       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1715     return VNET_API_ERROR_UNSUPPORTED;
1716
1717   pool_foreach (i, sm->interfaces,
1718   ({
1719     if (i->sw_if_index == sw_if_index)
1720       return VNET_API_ERROR_VALUE_EXIST;
1721   }));
1722
1723   if (is_inside)
1724     {
1725       vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1726                                    sw_if_index, !is_del, 0, 0);
1727       vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1728                                    sw_if_index, !is_del, 0, 0);
1729       goto fq;
1730     }
1731
1732   if (sm->num_workers > 1)
1733     {
1734       vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in-worker-handoff",
1735                                    sw_if_index, !is_del, 0, 0);
1736       vnet_feature_enable_disable ("ip4-output",
1737                                    "nat44-in2out-output-worker-handoff",
1738                                    sw_if_index, !is_del, 0, 0);
1739     }
1740   else
1741     {
1742       vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in", sw_if_index,
1743                                    !is_del, 0, 0);
1744       vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1745                                    sw_if_index, !is_del, 0, 0);
1746     }
1747
1748 fq:
1749   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1750     sm->fq_in2out_output_index =
1751       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1752
1753   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1754     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1755
1756   pool_foreach (i, sm->output_feature_interfaces,
1757   ({
1758     if (i->sw_if_index == sw_if_index)
1759       {
1760         if (is_del)
1761           pool_put (sm->output_feature_interfaces, i);
1762         else
1763           return VNET_API_ERROR_VALUE_EXIST;
1764
1765         goto fib;
1766       }
1767   }));
1768
1769   if (is_del)
1770     return VNET_API_ERROR_NO_SUCH_ENTRY;
1771
1772   pool_get (sm->output_feature_interfaces, i);
1773   i->sw_if_index = sw_if_index;
1774   i->flags = 0;
1775   if (is_inside)
1776     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1777   else
1778     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1779
1780   /* Add/delete external addresses to FIB */
1781 fib:
1782   if (is_inside)
1783     return 0;
1784
1785   vec_foreach (ap, sm->addresses)
1786     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1787
1788   pool_foreach (m, sm->static_mappings,
1789   ({
1790     if (!(m->addr_only))
1791       continue;
1792
1793     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1794   }));
1795
1796   return 0;
1797 }
1798
1799 int snat_set_workers (uword * bitmap)
1800 {
1801   snat_main_t *sm = &snat_main;
1802   int i, j = 0;
1803
1804   if (sm->num_workers < 2)
1805     return VNET_API_ERROR_FEATURE_DISABLED;
1806
1807   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1808     return VNET_API_ERROR_INVALID_WORKER;
1809
1810   vec_free (sm->workers);
1811   clib_bitmap_foreach (i, bitmap,
1812     ({
1813       vec_add1(sm->workers, i);
1814       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1815       j++;
1816     }));
1817
1818   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
1819   sm->num_snat_thread = _vec_len (sm->workers);
1820
1821   return 0;
1822 }
1823
1824
1825 static void
1826 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
1827                                        uword opaque,
1828                                        u32 sw_if_index,
1829                                        ip4_address_t * address,
1830                                        u32 address_length,
1831                                        u32 if_address_index,
1832                                        u32 is_delete);
1833
1834 static void
1835 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
1836                                  uword opaque,
1837                                  u32 sw_if_index,
1838                                  ip4_address_t * address,
1839                                  u32 address_length,
1840                                  u32 if_address_index,
1841                                  u32 is_delete);
1842
1843 static int
1844 nat_alloc_addr_and_port_default (snat_address_t * addresses,
1845                                  u32 fib_index,
1846                                  u32 thread_index,
1847                                  snat_session_key_t * k,
1848                                  u32 * address_indexp,
1849                                  u16 port_per_thread,
1850                                  u32 snat_thread_index);
1851
1852 static clib_error_t * snat_init (vlib_main_t * vm)
1853 {
1854   snat_main_t * sm = &snat_main;
1855   clib_error_t * error = 0;
1856   ip4_main_t * im = &ip4_main;
1857   ip_lookup_main_t * lm = &im->lookup_main;
1858   uword *p;
1859   vlib_thread_registration_t *tr;
1860   vlib_thread_main_t *tm = vlib_get_thread_main ();
1861   uword *bitmap = 0;
1862   u32 i;
1863   ip4_add_del_interface_address_callback_t cb4;
1864
1865   sm->vlib_main = vm;
1866   sm->vnet_main = vnet_get_main();
1867   sm->ip4_main = im;
1868   sm->ip4_lookup_main = lm;
1869   sm->api_main = &api_main;
1870   sm->first_worker_index = 0;
1871   sm->next_worker = 0;
1872   sm->num_workers = 0;
1873   sm->num_snat_thread = 1;
1874   sm->workers = 0;
1875   sm->port_per_thread = 0xffff - 1024;
1876   sm->fq_in2out_index = ~0;
1877   sm->fq_out2in_index = ~0;
1878   sm->udp_timeout = SNAT_UDP_TIMEOUT;
1879   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
1880   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
1881   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
1882   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
1883   sm->forwarding_enabled = 0;
1884
1885   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
1886   if (p)
1887     {
1888       tr = (vlib_thread_registration_t *) p[0];
1889       if (tr)
1890         {
1891           sm->num_workers = tr->count;
1892           sm->first_worker_index = tr->first_index;
1893         }
1894     }
1895
1896   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
1897
1898   /* Use all available workers by default */
1899   if (sm->num_workers > 1)
1900     {
1901       for (i=0; i < sm->num_workers; i++)
1902         bitmap = clib_bitmap_set (bitmap, i, 1);
1903       snat_set_workers(bitmap);
1904       clib_bitmap_free (bitmap);
1905     }
1906   else
1907     {
1908       sm->per_thread_data[0].snat_thread_index = 0;
1909     }
1910
1911   error = snat_api_init(vm, sm);
1912   if (error)
1913     return error;
1914
1915   /* Set up the interface address add/del callback */
1916   cb4.function = snat_ip4_add_del_interface_address_cb;
1917   cb4.function_opaque = 0;
1918
1919   vec_add1 (im->add_del_interface_address_callbacks, cb4);
1920
1921   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
1922   cb4.function_opaque = 0;
1923
1924   vec_add1 (im->add_del_interface_address_callbacks, cb4);
1925
1926   nat_dpo_module_init ();
1927
1928   /* Init IPFIX logging */
1929   snat_ipfix_logging_init(vm);
1930
1931   /* Init NAT64 */
1932   error = nat64_init(vm);
1933   if (error)
1934     return error;
1935
1936   dslite_init(vm);
1937
1938   nat66_init();
1939
1940   /* Init virtual fragmenentation reassembly */
1941   return nat_reass_init(vm);
1942 }
1943
1944 VLIB_INIT_FUNCTION (snat_init);
1945
1946 void snat_free_outside_address_and_port (snat_address_t * addresses,
1947                                          u32 thread_index,
1948                                          snat_session_key_t * k,
1949                                          u32 address_index)
1950 {
1951   snat_address_t *a;
1952   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
1953
1954   ASSERT (address_index < vec_len (addresses));
1955
1956   a = addresses + address_index;
1957
1958   switch (k->protocol)
1959     {
1960 #define _(N, i, n, s) \
1961     case SNAT_PROTOCOL_##N: \
1962       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
1963         port_host_byte_order) == 1); \
1964       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
1965         port_host_byte_order, 0); \
1966       a->busy_##n##_ports--; \
1967       a->busy_##n##_ports_per_thread[thread_index]--; \
1968       break;
1969       foreach_snat_protocol
1970 #undef _
1971     default:
1972       clib_warning("unknown_protocol");
1973       return;
1974     }
1975 }
1976
1977 /**
1978  * @brief Match NAT44 static mapping.
1979  *
1980  * @param sm          NAT main.
1981  * @param match       Address and port to match.
1982  * @param mapping     External or local address and port of the matched mapping.
1983  * @param by_external If 0 match by local address otherwise match by external
1984  *                    address.
1985  * @param is_addr_only If matched mapping is address only
1986  * @param twice_nat If matched mapping is twice NAT.
1987  * @param lb If matched mapping is load-balanced.
1988  *
1989  * @returns 0 if match found otherwise 1.
1990  */
1991 int snat_static_mapping_match (snat_main_t * sm,
1992                                snat_session_key_t match,
1993                                snat_session_key_t * mapping,
1994                                u8 by_external,
1995                                u8 *is_addr_only,
1996                                u8 *twice_nat,
1997                                u8 *lb)
1998 {
1999   clib_bihash_kv_8_8_t kv, value;
2000   snat_static_mapping_t *m;
2001   snat_session_key_t m_key;
2002   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
2003   u32 rand, lo = 0, hi, mid;
2004
2005   if (by_external)
2006     mapping_hash = &sm->static_mapping_by_external;
2007
2008   m_key.addr = match.addr;
2009   m_key.port = clib_net_to_host_u16 (match.port);
2010   m_key.protocol = match.protocol;
2011   m_key.fib_index = match.fib_index;
2012
2013   kv.key = m_key.as_u64;
2014
2015   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2016     {
2017       /* Try address only mapping */
2018       m_key.port = 0;
2019       m_key.protocol = 0;
2020       kv.key = m_key.as_u64;
2021       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2022         return 1;
2023     }
2024
2025   m = pool_elt_at_index (sm->static_mappings, value.value);
2026
2027   if (by_external)
2028     {
2029       if (vec_len (m->locals))
2030         {
2031           hi = vec_len (m->locals) - 1;
2032           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2033           while (lo < hi)
2034             {
2035               mid = ((hi - lo) >> 1) + lo;
2036               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2037             }
2038           if (!(m->locals[lo].prefix >= rand))
2039             return 1;
2040           mapping->addr = m->locals[lo].addr;
2041           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2042         }
2043       else
2044         {
2045           mapping->addr = m->local_addr;
2046           /* Address only mapping doesn't change port */
2047           mapping->port = m->addr_only ? match.port
2048             : clib_host_to_net_u16 (m->local_port);
2049         }
2050       mapping->fib_index = m->fib_index;
2051       mapping->protocol = m->proto;
2052     }
2053   else
2054     {
2055       mapping->addr = m->external_addr;
2056       /* Address only mapping doesn't change port */
2057       mapping->port = m->addr_only ? match.port
2058         : clib_host_to_net_u16 (m->external_port);
2059       mapping->fib_index = sm->outside_fib_index;
2060     }
2061
2062   if (PREDICT_FALSE(is_addr_only != 0))
2063     *is_addr_only = m->addr_only;
2064
2065   if (PREDICT_FALSE(twice_nat != 0))
2066     *twice_nat = m->twice_nat;
2067
2068   if (PREDICT_FALSE(lb != 0))
2069     *lb = vec_len (m->locals) > 0;
2070
2071   return 0;
2072 }
2073
2074 static_always_inline u16
2075 snat_random_port (u16 min, u16 max)
2076 {
2077   snat_main_t *sm = &snat_main;
2078   return min + random_u32 (&sm->random_seed) /
2079     (random_u32_max() / (max - min + 1) + 1);
2080 }
2081
2082 int
2083 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2084                                      u32 fib_index,
2085                                      u32 thread_index,
2086                                      snat_session_key_t * k,
2087                                      u32 * address_indexp,
2088                                      u16 port_per_thread,
2089                                      u32 snat_thread_index)
2090 {
2091   snat_main_t *sm = &snat_main;
2092
2093   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2094                                  address_indexp, port_per_thread,
2095                                  snat_thread_index);
2096 }
2097
2098 static int
2099 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2100                                  u32 fib_index,
2101                                  u32 thread_index,
2102                                  snat_session_key_t * k,
2103                                  u32 * address_indexp,
2104                                  u16 port_per_thread,
2105                                  u32 snat_thread_index)
2106 {
2107   int i, gi = 0;
2108   snat_address_t *a, *ga = 0;
2109   u32 portnum;
2110
2111   for (i = 0; i < vec_len (addresses); i++)
2112     {
2113       a = addresses + i;
2114       switch (k->protocol)
2115         {
2116 #define _(N, j, n, s) \
2117         case SNAT_PROTOCOL_##N: \
2118           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2119             { \
2120               if (a->fib_index == fib_index) \
2121                 { \
2122                   while (1) \
2123                     { \
2124                       portnum = (port_per_thread * \
2125                         snat_thread_index) + \
2126                         snat_random_port(1, port_per_thread) + 1024; \
2127                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2128                         continue; \
2129                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2130                       a->busy_##n##_ports_per_thread[thread_index]++; \
2131                       a->busy_##n##_ports++; \
2132                       k->addr = a->addr; \
2133                       k->port = clib_host_to_net_u16(portnum); \
2134                       *address_indexp = i; \
2135                       return 0; \
2136                     } \
2137                 } \
2138               else if (a->fib_index == ~0) \
2139                 { \
2140                   ga = a; \
2141                   gi = i; \
2142                 } \
2143             } \
2144           break;
2145           foreach_snat_protocol
2146 #undef _
2147         default:
2148           clib_warning("unknown protocol");
2149           return 1;
2150         }
2151
2152     }
2153
2154   if (ga)
2155     {
2156       a = ga;
2157       switch (k->protocol)
2158         {
2159 #define _(N, j, n, s) \
2160         case SNAT_PROTOCOL_##N: \
2161           while (1) \
2162             { \
2163               portnum = (port_per_thread * \
2164                 snat_thread_index) + \
2165                 snat_random_port(1, port_per_thread) + 1024; \
2166               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2167                 continue; \
2168               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2169               a->busy_##n##_ports_per_thread[thread_index]++; \
2170               a->busy_##n##_ports++; \
2171               k->addr = a->addr; \
2172               k->port = clib_host_to_net_u16(portnum); \
2173               *address_indexp = gi; \
2174               return 0; \
2175             }
2176           break;
2177           foreach_snat_protocol
2178 #undef _
2179         default:
2180           clib_warning ("unknown protocol");
2181           return 1;
2182         }
2183     }
2184
2185   /* Totally out of translations to use... */
2186   snat_ipfix_logging_addresses_exhausted(0);
2187   return 1;
2188 }
2189
2190 static int
2191 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2192                               u32 fib_index,
2193                               u32 thread_index,
2194                               snat_session_key_t * k,
2195                               u32 * address_indexp,
2196                               u16 port_per_thread,
2197                               u32 snat_thread_index)
2198 {
2199   snat_main_t *sm = &snat_main;
2200   snat_address_t *a = addresses;
2201   u16 m, ports, portnum, A, j;
2202   m = 16 - (sm->psid_offset + sm->psid_length);
2203   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2204
2205   if (!vec_len (addresses))
2206     goto exhausted;
2207
2208   switch (k->protocol)
2209     {
2210 #define _(N, i, n, s) \
2211     case SNAT_PROTOCOL_##N: \
2212       if (a->busy_##n##_ports < ports) \
2213         { \
2214           while (1) \
2215             { \
2216               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2217               j = snat_random_port(0, pow2_mask(m)); \
2218               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2219               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2220                 continue; \
2221               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2222               a->busy_##n##_ports++; \
2223               k->addr = a->addr; \
2224               k->port = clib_host_to_net_u16 (portnum); \
2225               *address_indexp = i; \
2226               return 0; \
2227             } \
2228         } \
2229       break;
2230       foreach_snat_protocol
2231 #undef _
2232     default:
2233       clib_warning("unknown protocol");
2234       return 1;
2235     }
2236
2237 exhausted:
2238   /* Totally out of translations to use... */
2239   snat_ipfix_logging_addresses_exhausted(0);
2240   return 1;
2241 }
2242
2243 void
2244 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2245 {
2246   dpo_id_t dpo_v4 = DPO_INVALID;
2247   fib_prefix_t pfx = {
2248     .fp_proto = FIB_PROTOCOL_IP4,
2249     .fp_len = 32,
2250     .fp_addr.ip4.as_u32 = addr.as_u32,
2251   };
2252
2253   if (is_add)
2254     {
2255       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2256       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2257                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2258       dpo_reset (&dpo_v4);
2259     }
2260   else
2261     {
2262       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2263     }
2264 }
2265
2266 uword
2267 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2268 {
2269   u32 *r = va_arg (*args, u32 *);
2270
2271   if (0);
2272 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2273   foreach_snat_protocol
2274 #undef _
2275   else
2276     return 0;
2277   return 1;
2278 }
2279
2280 u8 *
2281 format_snat_protocol (u8 * s, va_list * args)
2282 {
2283   u32 i = va_arg (*args, u32);
2284   u8 *t = 0;
2285
2286   switch (i)
2287     {
2288 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2289       foreach_snat_protocol
2290 #undef _
2291     default:
2292       s = format (s, "unknown");
2293       return s;
2294     }
2295   s = format (s, "%s", t);
2296   return s;
2297 }
2298
2299 static u32
2300 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2301 {
2302   snat_main_t *sm = &snat_main;
2303   u32 next_worker_index = 0;
2304   u32 hash;
2305
2306   next_worker_index = sm->first_worker_index;
2307   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2308          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2309
2310   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2311     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2312   else
2313     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2314
2315   return next_worker_index;
2316 }
2317
2318 static u32
2319 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2320 {
2321   snat_main_t *sm = &snat_main;
2322   udp_header_t *udp;
2323   u16 port;
2324   snat_session_key_t m_key;
2325   clib_bihash_kv_8_8_t kv, value;
2326   snat_static_mapping_t *m;
2327   nat_ed_ses_key_t key;
2328   clib_bihash_kv_16_8_t s_kv, s_value;
2329   snat_main_per_thread_data_t *tsm;
2330   snat_session_t *s;
2331   int i;
2332   u32 proto;
2333   u32 next_worker_index = 0;
2334
2335   /* first try static mappings without port */
2336   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2337     {
2338       m_key.addr = ip0->dst_address;
2339       m_key.port = 0;
2340       m_key.protocol = 0;
2341       m_key.fib_index = rx_fib_index0;
2342       kv.key = m_key.as_u64;
2343       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2344         {
2345           m = pool_elt_at_index (sm->static_mappings, value.value);
2346           return m->worker_index;
2347         }
2348     }
2349
2350   proto = ip_proto_to_snat_proto (ip0->protocol);
2351   udp = ip4_next_header (ip0);
2352   port = udp->dst_port;
2353
2354   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2355     {
2356       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2357         return vlib_get_thread_index ();
2358
2359       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2360         {
2361           nat_reass_ip4_t *reass;
2362
2363           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2364                                       ip0->fragment_id, ip0->protocol);
2365
2366           if (reass && (reass->thread_index != (u32) ~ 0))
2367             return reass->thread_index;
2368           else
2369             return vlib_get_thread_index ();
2370         }
2371     }
2372
2373   /* unknown protocol */
2374   if (PREDICT_FALSE (proto == ~0))
2375     {
2376       key.l_addr = ip0->dst_address;
2377       key.r_addr = ip0->src_address;
2378       key.fib_index = rx_fib_index0;
2379       key.proto = ip0->protocol;
2380       key.r_port = 0;
2381       key.l_port = 0;
2382       s_kv.key[0] = key.as_u64[0];
2383       s_kv.key[1] = key.as_u64[1];
2384
2385       if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
2386         {
2387           for (i = 0; i < _vec_len (sm->per_thread_data); i++)
2388             {
2389               tsm = vec_elt_at_index (sm->per_thread_data, i);
2390               if (!pool_is_free_index(tsm->sessions, s_value.value))
2391                 {
2392                   s = pool_elt_at_index (tsm->sessions, s_value.value);
2393                   if (s->out2in.addr.as_u32 == ip0->dst_address.as_u32 &&
2394                       s->out2in.port == ip0->protocol &&
2395                       snat_is_unk_proto_session (s))
2396                     return i;
2397                 }
2398             }
2399          }
2400
2401       /* if no session use current thread */
2402       return vlib_get_thread_index ();
2403     }
2404
2405   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2406     {
2407       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2408       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2409       if (!icmp_is_error_message (icmp))
2410         port = echo->identifier;
2411       else
2412         {
2413           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2414           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2415           void *l4_header = ip4_next_header (inner_ip);
2416           switch (proto)
2417             {
2418             case SNAT_PROTOCOL_ICMP:
2419               icmp = (icmp46_header_t*)l4_header;
2420               echo = (icmp_echo_header_t *)(icmp + 1);
2421               port = echo->identifier;
2422               break;
2423             case SNAT_PROTOCOL_UDP:
2424             case SNAT_PROTOCOL_TCP:
2425               port = ((tcp_udp_header_t*)l4_header)->src_port;
2426               break;
2427             default:
2428               return vlib_get_thread_index ();
2429             }
2430         }
2431     }
2432
2433   /* try static mappings with port */
2434   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2435     {
2436       m_key.addr = ip0->dst_address;
2437       m_key.port = clib_net_to_host_u16 (port);
2438       m_key.protocol = proto;
2439       m_key.fib_index = rx_fib_index0;
2440       kv.key = m_key.as_u64;
2441       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2442         {
2443           m = pool_elt_at_index (sm->static_mappings, value.value);
2444           return m->worker_index;
2445         }
2446     }
2447
2448   /* worker by outside port */
2449   next_worker_index = sm->first_worker_index;
2450   next_worker_index +=
2451     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2452   return next_worker_index;
2453 }
2454
2455 static clib_error_t *
2456 snat_config (vlib_main_t * vm, unformat_input_t * input)
2457 {
2458   snat_main_t * sm = &snat_main;
2459   nat66_main_t * nm = &nat66_main;
2460   u32 translation_buckets = 1024;
2461   u32 translation_memory_size = 128<<20;
2462   u32 user_buckets = 128;
2463   u32 user_memory_size = 64<<20;
2464   u32 max_translations_per_user = 100;
2465   u32 outside_vrf_id = 0;
2466   u32 outside_ip6_vrf_id = 0;
2467   u32 inside_vrf_id = 0;
2468   u32 static_mapping_buckets = 1024;
2469   u32 static_mapping_memory_size = 64<<20;
2470   u32 nat64_bib_buckets = 1024;
2471   u32 nat64_bib_memory_size = 128 << 20;
2472   u32 nat64_st_buckets = 2048;
2473   u32 nat64_st_memory_size = 256 << 20;
2474   u8 static_mapping_only = 0;
2475   u8 static_mapping_connection_tracking = 0;
2476   snat_main_per_thread_data_t *tsm;
2477   dslite_main_t * dm = &dslite_main;
2478
2479   sm->deterministic = 0;
2480   sm->out2in_dpo = 0;
2481
2482   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2483     {
2484       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2485         ;
2486       else if (unformat (input, "translation hash memory %d",
2487                          &translation_memory_size));
2488       else if (unformat (input, "user hash buckets %d", &user_buckets))
2489         ;
2490       else if (unformat (input, "user hash memory %d",
2491                          &user_memory_size))
2492         ;
2493       else if (unformat (input, "max translations per user %d",
2494                          &max_translations_per_user))
2495         ;
2496       else if (unformat (input, "outside VRF id %d",
2497                          &outside_vrf_id))
2498         ;
2499       else if (unformat (input, "outside ip6 VRF id %d",
2500                          &outside_ip6_vrf_id))
2501         ;
2502       else if (unformat (input, "inside VRF id %d",
2503                          &inside_vrf_id))
2504         ;
2505       else if (unformat (input, "static mapping only"))
2506         {
2507           static_mapping_only = 1;
2508           if (unformat (input, "connection tracking"))
2509             static_mapping_connection_tracking = 1;
2510         }
2511       else if (unformat (input, "deterministic"))
2512         sm->deterministic = 1;
2513       else if (unformat (input, "nat64 bib hash buckets %d",
2514                          &nat64_bib_buckets))
2515         ;
2516       else if (unformat (input, "nat64 bib hash memory %d",
2517                          &nat64_bib_memory_size))
2518         ;
2519       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2520         ;
2521       else if (unformat (input, "nat64 st hash memory %d",
2522                          &nat64_st_memory_size))
2523         ;
2524       else if (unformat (input, "out2in dpo"))
2525         sm->out2in_dpo = 1;
2526       else if (unformat (input, "dslite ce"))
2527         dslite_set_ce(dm, 1);
2528       else
2529         return clib_error_return (0, "unknown input '%U'",
2530                                   format_unformat_error, input);
2531     }
2532
2533   /* for show commands, etc. */
2534   sm->translation_buckets = translation_buckets;
2535   sm->translation_memory_size = translation_memory_size;
2536   /* do not exceed load factor 10 */
2537   sm->max_translations = 10 * translation_buckets;
2538   sm->user_buckets = user_buckets;
2539   sm->user_memory_size = user_memory_size;
2540   sm->max_translations_per_user = max_translations_per_user;
2541   sm->outside_vrf_id = outside_vrf_id;
2542   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2543                                                              outside_vrf_id,
2544                                                              FIB_SOURCE_PLUGIN_HI);
2545   nm->outside_vrf_id = outside_ip6_vrf_id;
2546   nm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2547                                                              outside_ip6_vrf_id,
2548                                                              FIB_SOURCE_PLUGIN_HI);
2549   sm->inside_vrf_id = inside_vrf_id;
2550   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2551                                                             inside_vrf_id,
2552                                                             FIB_SOURCE_PLUGIN_HI);
2553   sm->static_mapping_only = static_mapping_only;
2554   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
2555
2556   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
2557                  nat64_st_memory_size);
2558
2559   if (sm->deterministic)
2560     {
2561       sm->in2out_node_index = snat_det_in2out_node.index;
2562       sm->in2out_output_node_index = ~0;
2563       sm->out2in_node_index = snat_det_out2in_node.index;
2564       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
2565       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
2566     }
2567   else
2568     {
2569       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2570       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2571       sm->in2out_node_index = snat_in2out_node.index;
2572       sm->in2out_output_node_index = snat_in2out_output_node.index;
2573       sm->out2in_node_index = snat_out2in_node.index;
2574       if (!static_mapping_only ||
2575           (static_mapping_only && static_mapping_connection_tracking))
2576         {
2577           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2578           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2579
2580           vec_foreach (tsm, sm->per_thread_data)
2581             {
2582               clib_bihash_init_8_8 (&tsm->in2out, "in2out", translation_buckets,
2583                                     translation_memory_size);
2584
2585               clib_bihash_init_8_8 (&tsm->out2in, "out2in", translation_buckets,
2586                                     translation_memory_size);
2587
2588               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
2589                                     user_memory_size);
2590             }
2591
2592           clib_bihash_init_16_8 (&sm->in2out_ed, "in2out-ed",
2593                                  translation_buckets, translation_memory_size);
2594
2595           clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
2596                                  translation_buckets, translation_memory_size);
2597         }
2598       else
2599         {
2600           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2601           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2602         }
2603       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2604                             "static_mapping_by_local", static_mapping_buckets,
2605                             static_mapping_memory_size);
2606
2607       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2608                             "static_mapping_by_external", static_mapping_buckets,
2609                             static_mapping_memory_size);
2610     }
2611
2612   return 0;
2613 }
2614
2615 VLIB_CONFIG_FUNCTION (snat_config, "nat");
2616
2617 u8 * format_snat_session_state (u8 * s, va_list * args)
2618 {
2619   u32 i = va_arg (*args, u32);
2620   u8 *t = 0;
2621
2622   switch (i)
2623     {
2624 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
2625     foreach_snat_session_state
2626 #undef _
2627     default:
2628       t = format (t, "unknown");
2629     }
2630   s = format (s, "%s", t);
2631   return s;
2632 }
2633
2634 u8 * format_snat_key (u8 * s, va_list * args)
2635 {
2636   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
2637
2638   s = format (s, "%U proto %U port %d fib %d",
2639               format_ip4_address, &key->addr,
2640               format_snat_protocol, key->protocol,
2641               clib_net_to_host_u16 (key->port), key->fib_index);
2642   return s;
2643 }
2644
2645 u8 * format_snat_session (u8 * s, va_list * args)
2646 {
2647   snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *);
2648   snat_session_t * sess = va_arg (*args, snat_session_t *);
2649
2650   if (snat_is_unk_proto_session (sess))
2651     {
2652       s = format (s, "  i2o %U proto %u fib %u\n",
2653                   format_ip4_address, &sess->in2out.addr,
2654                   clib_net_to_host_u16 (sess->in2out.port),
2655                   sess->in2out.fib_index);
2656       s = format (s, "    o2i %U proto %u fib %u\n",
2657                   format_ip4_address, &sess->out2in.addr,
2658                   clib_net_to_host_u16 (sess->out2in.port),
2659                   sess->out2in.fib_index);
2660     }
2661   else
2662     {
2663       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
2664       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
2665     }
2666   if (is_twice_nat_session (sess))
2667     {
2668       s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
2669                   format_ip4_address, &sess->ext_host_addr,
2670                   clib_net_to_host_u16 (sess->ext_host_port),
2671                   format_ip4_address, &sess->ext_host_nat_addr,
2672                   clib_net_to_host_u16 (sess->ext_host_nat_port));
2673     }
2674   else
2675     {
2676       if (sess->ext_host_addr.as_u32)
2677           s = format (s, "       external host %U\n",
2678                       format_ip4_address, &sess->ext_host_addr);
2679     }
2680   s = format (s, "       last heard %.2f\n", sess->last_heard);
2681   s = format (s, "       total pkts %d, total bytes %lld\n",
2682               sess->total_pkts, sess->total_bytes);
2683   if (snat_is_session_static (sess))
2684     s = format (s, "       static translation\n");
2685   else
2686     s = format (s, "       dynamic translation\n");
2687   if (sess->flags & SNAT_SESSION_FLAG_LOAD_BALANCING)
2688     s = format (s, "       load-balancing\n");
2689   if (is_twice_nat_session (sess))
2690     s = format (s, "       twice-nat\n");
2691
2692   return s;
2693 }
2694
2695 u8 * format_snat_user (u8 * s, va_list * args)
2696 {
2697   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
2698   snat_user_t * u = va_arg (*args, snat_user_t *);
2699   int verbose = va_arg (*args, int);
2700   dlist_elt_t * head, * elt;
2701   u32 elt_index, head_index;
2702   u32 session_index;
2703   snat_session_t * sess;
2704
2705   s = format (s, "%U: %d dynamic translations, %d static translations\n",
2706               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
2707
2708   if (verbose == 0)
2709     return s;
2710
2711   if (u->nsessions || u->nstaticsessions)
2712     {
2713       head_index = u->sessions_per_user_list_head_index;
2714       head = pool_elt_at_index (sm->list_pool, head_index);
2715
2716       elt_index = head->next;
2717       elt = pool_elt_at_index (sm->list_pool, elt_index);
2718       session_index = elt->value;
2719
2720       while (session_index != ~0)
2721         {
2722           sess = pool_elt_at_index (sm->sessions, session_index);
2723
2724           s = format (s, "  %U\n", format_snat_session, sm, sess);
2725
2726           elt_index = elt->next;
2727           elt = pool_elt_at_index (sm->list_pool, elt_index);
2728           session_index = elt->value;
2729         }
2730     }
2731
2732   return s;
2733 }
2734
2735 u8 * format_snat_static_mapping (u8 * s, va_list * args)
2736 {
2737   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
2738   nat44_lb_addr_port_t *local;
2739
2740   if (m->addr_only)
2741       s = format (s, "local %U external %U vrf %d %s",
2742                   format_ip4_address, &m->local_addr,
2743                   format_ip4_address, &m->external_addr,
2744                   m->vrf_id, m->twice_nat ? "twice-nat" : "");
2745   else
2746    {
2747       if (vec_len (m->locals))
2748         {
2749           s = format (s, "%U vrf %d external %U:%d %s %s",
2750                       format_snat_protocol, m->proto,
2751                       m->vrf_id,
2752                       format_ip4_address, &m->external_addr, m->external_port,
2753                       m->twice_nat ? "twice-nat" : "",
2754                       m->out2in_only ? "out2in-only" : "");
2755           vec_foreach (local, m->locals)
2756             s = format (s, "\n  local %U:%d probability %d\%",
2757                         format_ip4_address, &local->addr, local->port,
2758                         local->probability);
2759         }
2760       else
2761         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
2762                     format_snat_protocol, m->proto,
2763                     format_ip4_address, &m->local_addr, m->local_port,
2764                     format_ip4_address, &m->external_addr, m->external_port,
2765                     m->vrf_id, m->twice_nat ? "twice-nat" : "",
2766                     m->out2in_only ? "out2in-only" : "");
2767    }
2768   return s;
2769 }
2770
2771 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
2772 {
2773   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
2774   vnet_main_t *vnm = vnet_get_main();
2775
2776   if (m->addr_only)
2777       s = format (s, "local %U external %U vrf %d",
2778                   format_ip4_address, &m->l_addr,
2779                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
2780                   m->vrf_id);
2781   else
2782       s = format (s, "%U local %U:%d external %U:%d vrf %d",
2783                   format_snat_protocol, m->proto,
2784                   format_ip4_address, &m->l_addr, m->l_port,
2785                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
2786                   m->e_port, m->vrf_id);
2787
2788   return s;
2789 }
2790
2791 u8 * format_det_map_ses (u8 * s, va_list * args)
2792 {
2793   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
2794   ip4_address_t in_addr, out_addr;
2795   u32 in_offset, out_offset;
2796   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
2797   u32 * i = va_arg (*args, u32 *);
2798
2799   u32 user_index = *i / SNAT_DET_SES_PER_USER;
2800   in_addr.as_u32 = clib_host_to_net_u32 (
2801     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
2802   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
2803     clib_net_to_host_u32(det_map->in_addr.as_u32);
2804   out_offset = in_offset / det_map->sharing_ratio;
2805   out_addr.as_u32 = clib_host_to_net_u32(
2806     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
2807   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
2808               format_ip4_address, &in_addr,
2809               clib_net_to_host_u16 (ses->in_port),
2810               format_ip4_address, &out_addr,
2811               clib_net_to_host_u16 (ses->out.out_port),
2812               format_ip4_address, &ses->out.ext_host_addr,
2813               clib_net_to_host_u16 (ses->out.ext_host_port),
2814               format_snat_session_state, ses->state,
2815               ses->expire);
2816
2817   return s;
2818 }
2819
2820 static void
2821 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2822                                  uword opaque,
2823                                  u32 sw_if_index,
2824                                  ip4_address_t * address,
2825                                  u32 address_length,
2826                                  u32 if_address_index,
2827                                  u32 is_delete)
2828 {
2829   snat_main_t *sm = &snat_main;
2830   snat_static_map_resolve_t *rp;
2831   snat_static_mapping_t *m;
2832   snat_session_key_t m_key;
2833   clib_bihash_kv_8_8_t kv, value;
2834   int i, rv;
2835   ip4_address_t l_addr;
2836
2837   for (i = 0; i < vec_len (sm->to_resolve); i++)
2838     {
2839       rp = sm->to_resolve + i;
2840       if (rp->addr_only == 0)
2841         continue;
2842       if (rp->sw_if_index == sw_if_index)
2843         goto match;
2844     }
2845
2846   return;
2847
2848 match:
2849   m_key.addr.as_u32 = address->as_u32;
2850   m_key.port = rp->addr_only ? 0 : rp->e_port;
2851   m_key.protocol = rp->addr_only ? 0 : rp->proto;
2852   m_key.fib_index = sm->outside_fib_index;
2853   kv.key = m_key.as_u64;
2854   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2855     m = 0;
2856   else
2857     m = pool_elt_at_index (sm->static_mappings, value.value);
2858
2859   if (!is_delete)
2860     {
2861       /* Don't trip over lease renewal, static config */
2862       if (m)
2863         return;
2864     }
2865   else
2866     {
2867       if (!m)
2868         return;
2869     }
2870
2871   /* Indetity mapping? */
2872   if (rp->l_addr.as_u32 == 0)
2873     l_addr.as_u32 = address[0].as_u32;
2874   else
2875     l_addr.as_u32 = rp->l_addr.as_u32;
2876   /* Add the static mapping */
2877   rv = snat_add_static_mapping (l_addr,
2878                                 address[0],
2879                                 rp->l_port,
2880                                 rp->e_port,
2881                                 rp->vrf_id,
2882                                 rp->addr_only,
2883                                 ~0 /* sw_if_index */,
2884                                 rp->proto,
2885                                 !is_delete,
2886                                 0, 0, rp->tag);
2887   if (rv)
2888     clib_warning ("snat_add_static_mapping returned %d", rv);
2889 }
2890
2891 static void
2892 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2893                                        uword opaque,
2894                                        u32 sw_if_index,
2895                                        ip4_address_t * address,
2896                                        u32 address_length,
2897                                        u32 if_address_index,
2898                                        u32 is_delete)
2899 {
2900   snat_main_t *sm = &snat_main;
2901   snat_static_map_resolve_t *rp;
2902   ip4_address_t l_addr;
2903   int i, j;
2904   int rv;
2905   u8 twice_nat = 0;
2906   snat_address_t *addresses = sm->addresses;
2907
2908   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
2909     {
2910       if (sw_if_index == sm->auto_add_sw_if_indices[i])
2911           goto match;
2912     }
2913
2914   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
2915     {
2916       twice_nat = 1;
2917       addresses = sm->twice_nat_addresses;
2918       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
2919           goto match;
2920     }
2921
2922   return;
2923
2924 match:
2925   if (!is_delete)
2926     {
2927       /* Don't trip over lease renewal, static config */
2928       for (j = 0; j < vec_len(addresses); j++)
2929         if (addresses[j].addr.as_u32 == address->as_u32)
2930           return;
2931
2932       snat_add_address (sm, address, ~0, twice_nat);
2933       /* Scan static map resolution vector */
2934       for (j = 0; j < vec_len (sm->to_resolve); j++)
2935         {
2936           rp = sm->to_resolve + j;
2937           if (rp->addr_only)
2938             continue;
2939           /* On this interface? */
2940           if (rp->sw_if_index == sw_if_index)
2941             {
2942               /* Indetity mapping? */
2943               if (rp->l_addr.as_u32 == 0)
2944                 l_addr.as_u32 = address[0].as_u32;
2945               else
2946                 l_addr.as_u32 = rp->l_addr.as_u32;
2947               /* Add the static mapping */
2948               rv = snat_add_static_mapping (l_addr,
2949                                             address[0],
2950                                             rp->l_port,
2951                                             rp->e_port,
2952                                             rp->vrf_id,
2953                                             rp->addr_only,
2954                                             ~0 /* sw_if_index */,
2955                                             rp->proto,
2956                                             rp->is_add,
2957                                             0, 0, rp->tag);
2958               if (rv)
2959                 clib_warning ("snat_add_static_mapping returned %d",
2960                               rv);
2961             }
2962         }
2963       return;
2964     }
2965   else
2966     {
2967       (void) snat_del_address(sm, address[0], 1, twice_nat);
2968       return;
2969     }
2970 }
2971
2972
2973 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
2974                                 u8 twice_nat)
2975 {
2976   ip4_main_t * ip4_main = sm->ip4_main;
2977   ip4_address_t * first_int_addr;
2978   snat_static_map_resolve_t *rp;
2979   u32 *indices_to_delete = 0;
2980   int i, j;
2981   u32 *auto_add_sw_if_indices =
2982     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
2983
2984   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
2985                                                 0 /* just want the address*/);
2986
2987   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
2988     {
2989       if (auto_add_sw_if_indices[i] == sw_if_index)
2990         {
2991           if (is_del)
2992             {
2993               /* if have address remove it */
2994               if (first_int_addr)
2995                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
2996               else
2997                 {
2998                   for (j = 0; j < vec_len (sm->to_resolve); j++)
2999                     {
3000                       rp = sm->to_resolve + j;
3001                       if (rp->sw_if_index == sw_if_index)
3002                         vec_add1 (indices_to_delete, j);
3003                     }
3004                   if (vec_len(indices_to_delete))
3005                     {
3006                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
3007                         vec_del1(sm->to_resolve, j);
3008                       vec_free(indices_to_delete);
3009                     }
3010                 }
3011               if (twice_nat)
3012                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
3013               else
3014                 vec_del1(sm->auto_add_sw_if_indices, i);
3015             }
3016           else
3017             return VNET_API_ERROR_VALUE_EXIST;
3018
3019           return 0;
3020         }
3021     }
3022
3023   if (is_del)
3024     return VNET_API_ERROR_NO_SUCH_ENTRY;
3025
3026   /* add to the auto-address list */
3027   if (twice_nat)
3028     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3029   else
3030     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
3031
3032   /* If the address is already bound - or static - add it now */
3033   if (first_int_addr)
3034       snat_add_address (sm, first_int_addr, ~0, twice_nat);
3035
3036   return 0;
3037 }
3038
3039 int
3040 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3041                    snat_protocol_t proto, u32 vrf_id, int is_in)
3042 {
3043   snat_main_per_thread_data_t *tsm;
3044   clib_bihash_kv_8_8_t kv, value;
3045   ip4_header_t ip;
3046   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3047   snat_session_key_t key;
3048   snat_session_t *s;
3049   clib_bihash_8_8_t *t;
3050   snat_user_key_t u_key;
3051   snat_user_t *u;
3052
3053   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3054   if (sm->num_workers > 1)
3055     tsm =
3056       vec_elt_at_index (sm->per_thread_data,
3057                         sm->worker_in2out_cb (&ip, fib_index));
3058   else
3059     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3060
3061   key.addr.as_u32 = addr->as_u32;
3062   key.port = clib_host_to_net_u16 (port);
3063   key.protocol = proto;
3064   key.fib_index = fib_index;
3065   kv.key = key.as_u64;
3066   t = is_in ? &tsm->in2out : &tsm->out2in;
3067   if (!clib_bihash_search_8_8 (t, &kv, &value))
3068     {
3069       s = pool_elt_at_index (tsm->sessions, value.value);
3070       kv.key = s->in2out.as_u64;
3071       clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0);
3072       kv.key = s->out2in.as_u64;
3073       clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0);
3074       u_key.addr = s->in2out.addr;
3075       u_key.fib_index = s->in2out.fib_index;
3076       kv.key = u_key.as_u64;
3077       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
3078         {
3079           u = pool_elt_at_index (tsm->users, value.value);
3080           u->nsessions--;
3081         }
3082       clib_dlist_remove (tsm->list_pool, s->per_user_index);
3083       pool_put (tsm->sessions, s);
3084       return 0;
3085     }
3086
3087   return VNET_API_ERROR_NO_SUCH_ENTRY;
3088 }
3089
3090 void
3091 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3092 {
3093   snat_main_t *sm = &snat_main;
3094
3095   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3096   sm->psid = psid;
3097   sm->psid_offset = psid_offset;
3098   sm->psid_length = psid_length;
3099 }
3100
3101 void
3102 nat_set_alloc_addr_and_port_default (void)
3103 {
3104   snat_main_t *sm = &snat_main;
3105
3106   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3107 }
3108