src/plugins/nat/nat.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/plugin/plugin.h>
  22 #include <nat/nat.h>
  23 #include <nat/nat_dpo.h>
  24 #include <nat/nat_ipfix_logging.h>
  25 #include <nat/nat_det.h>
  26 #include <nat/nat64.h>
  27 #include <nat/nat66.h>
  28 #include <nat/dslite.h>
  29 #include <nat/nat_reass.h>
  30 #include <vnet/fib/fib_table.h>
  31 #include <vnet/fib/ip4_fib.h>
  32
  33 #include <vpp/app/version.h>
  34
  35 snat_main_t snat_main;
  36
  37
  38 /* Hook up input features */
  39 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
  40   .arc_name = "ip4-unicast",
  41   .node_name = "nat44-in2out",
  42   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  43 };
  44 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
  45   .arc_name = "ip4-unicast",
  46   .node_name = "nat44-out2in",
  47   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  48 };
  49 VNET_FEATURE_INIT (ip4_nat_classify, static) = {
  50   .arc_name = "ip4-unicast",
  51   .node_name = "nat44-classify",
  52   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  53 };
  54 VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
  55   .arc_name = "ip4-unicast",
  56   .node_name = "nat44-det-in2out",
  57   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  58 };
  59 VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
  60   .arc_name = "ip4-unicast",
  61   .node_name = "nat44-det-out2in",
  62   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  63 };
  64 VNET_FEATURE_INIT (ip4_nat_det_classify, static) = {
  65   .arc_name = "ip4-unicast",
  66   .node_name = "nat44-det-classify",
  67   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  68 };
  69 VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
  70   .arc_name = "ip4-unicast",
  71   .node_name = "nat44-in2out-worker-handoff",
  72   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  73 };
  74 VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
  75   .arc_name = "ip4-unicast",
  76   .node_name = "nat44-out2in-worker-handoff",
  77   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  78 };
  79 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat44-handoff-classify",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  83 };
  84 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat44-in2out-fast",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  88 };
  89 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
  90   .arc_name = "ip4-unicast",
  91   .node_name = "nat44-out2in-fast",
  92   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  93 };
  94 VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
  95   .arc_name = "ip4-unicast",
  96   .node_name = "nat44-hairpin-dst",
  97   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  98 };
  99
 100 /* Hook up output features */
 101 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 102   .arc_name = "ip4-output",
 103   .node_name = "nat44-in2out-output",
 104   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 105 };
 106 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 107   .arc_name = "ip4-output",
 108   .node_name = "nat44-in2out-output-worker-handoff",
 109   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 110 };
 111 VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
 112   .arc_name = "ip4-output",
 113   .node_name = "nat44-hairpin-src",
 114   .runs_after = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 115 };
 116
 117 /* Hook up ip4-local features */
 118 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
 119 {
 120   .arc_name = "ip4-local",
 121   .node_name = "nat44-hairpinning",
 122   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 123 };
 124
 125
 126 /* *INDENT-OFF* */
 127 VLIB_PLUGIN_REGISTER () = {
 128     .version = VPP_BUILD_VER,
 129     .description = "Network Address Translation",
 130 };
 131 /* *INDENT-ON* */
 132
 133 vlib_node_registration_t nat44_classify_node;
 134 vlib_node_registration_t nat44_det_classify_node;
 135 vlib_node_registration_t nat44_handoff_classify_node;
 136
 137 typedef enum {
 138   NAT44_CLASSIFY_NEXT_IN2OUT,
 139   NAT44_CLASSIFY_NEXT_OUT2IN,
 140   NAT44_CLASSIFY_N_NEXT,
 141 } nat44_classify_next_t;
 142
 143 void
 144 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index)
 145 {
 146   snat_session_key_t key;
 147   clib_bihash_kv_8_8_t kv;
 148   nat_ed_ses_key_t ed_key;
 149   clib_bihash_kv_16_8_t ed_kv;
 150   int i;
 151   snat_address_t *a;
 152   snat_main_per_thread_data_t *tsm =
 153     vec_elt_at_index (sm->per_thread_data, thread_index);
 154
 155   /* Endpoint dependent session lookup tables */
 156   if (is_ed_session (s))
 157     {
 158       ed_key.l_addr = s->out2in.addr;
 159       ed_key.r_addr = s->ext_host_addr;
 160       ed_key.fib_index = s->out2in.fib_index;
 161       if (snat_is_unk_proto_session (s))
 162         {
 163           ed_key.proto = s->in2out.port;
 164           ed_key.r_port = 0;
 165           ed_key.l_port = 0;
 166         }
 167       else
 168         {
 169           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 170           ed_key.l_port = s->out2in.port;
 171           ed_key.r_port = s->ext_host_port;
 172         }
 173       ed_kv.key[0] = ed_key.as_u64[0];
 174       ed_kv.key[1] = ed_key.as_u64[1];
 175       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
 176         clib_warning ("out2in_ed key del failed");
 177
 178       ed_key.l_addr = s->in2out.addr;
 179       ed_key.fib_index = s->in2out.fib_index;
 180       if (!snat_is_unk_proto_session (s))
 181         ed_key.l_port = s->in2out.port;
 182       if (is_twice_nat_session (s))
 183         {
 184           ed_key.r_addr = s->ext_host_nat_addr;
 185           ed_key.r_port = s->ext_host_nat_port;
 186         }
 187       ed_kv.key[0] = ed_key.as_u64[0];
 188       ed_kv.key[1] = ed_key.as_u64[1];
 189       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &ed_kv, 0))
 190         clib_warning ("in2out_ed key del failed");
 191     }
 192
 193   if (snat_is_unk_proto_session (s))
 194     return;
 195
 196   /* log NAT event */
 197   snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
 198                                       s->out2in.addr.as_u32,
 199                                       s->in2out.protocol,
 200                                       s->in2out.port,
 201                                       s->out2in.port,
 202                                       s->in2out.fib_index);
 203
 204   /* Twice NAT address and port for external host */
 205   if (is_twice_nat_session (s))
 206     {
 207       for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 208         {
 209           key.protocol = s->in2out.protocol;
 210           key.port = s->ext_host_nat_port;
 211           a = sm->twice_nat_addresses + i;
 212           if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 213             {
 214               snat_free_outside_address_and_port (sm->twice_nat_addresses,
 215                                                   thread_index, &key, i);
 216               break;
 217             }
 218         }
 219     }
 220
 221   if (is_ed_session (s))
 222     return;
 223
 224   /* Session lookup tables */
 225   kv.key = s->in2out.as_u64;
 226   if (clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0))
 227     clib_warning ("in2out key del failed");
 228   kv.key = s->out2in.as_u64;
 229   if (clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0))
 230     clib_warning ("out2in key del failed");
 231
 232   if (snat_is_session_static (s))
 233     return;
 234
 235   if (s->outside_address_index != ~0)
 236     snat_free_outside_address_and_port (sm->addresses, thread_index,
 237                                         &s->out2in, s->outside_address_index);
 238 }
 239
 240 snat_user_t *
 241 nat_user_get_or_create (snat_main_t *sm, ip4_address_t *addr, u32 fib_index,
 242                         u32 thread_index)
 243 {
 244   snat_user_t *u = 0;
 245   snat_user_key_t user_key;
 246   clib_bihash_kv_8_8_t kv, value;
 247   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 248   dlist_elt_t * per_user_list_head_elt;
 249
 250   user_key.addr.as_u32 = addr->as_u32;
 251   user_key.fib_index = fib_index;
 252   kv.key = user_key.as_u64;
 253
 254   /* Ever heard of the "user" = src ip4 address before? */
 255   if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 256     {
 257       /* no, make a new one */
 258       pool_get (tsm->users, u);
 259       memset (u, 0, sizeof (*u));
 260       u->addr.as_u32 = addr->as_u32;
 261       u->fib_index = fib_index;
 262
 263       pool_get (tsm->list_pool, per_user_list_head_elt);
 264
 265       u->sessions_per_user_list_head_index = per_user_list_head_elt -
 266         tsm->list_pool;
 267
 268       clib_dlist_init (tsm->list_pool, u->sessions_per_user_list_head_index);
 269
 270       kv.value = u - tsm->users;
 271
 272       /* add user */
 273       if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
 274         clib_warning ("user_hash keay add failed");
 275     }
 276   else
 277     {
 278       u = pool_elt_at_index (tsm->users, value.value);
 279     }
 280
 281   return u;
 282 }
 283
 284 snat_session_t *
 285 nat_session_alloc_or_recycle (snat_main_t *sm, snat_user_t *u, u32 thread_index)
 286 {
 287   snat_session_t *s;
 288   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 289   u32 oldest_per_user_translation_list_index, session_index;
 290   dlist_elt_t * oldest_per_user_translation_list_elt;
 291   dlist_elt_t * per_user_translation_list_elt;
 292
 293   /* Over quota? Recycle the least recently used translation */
 294   if ((u->nsessions + u->nstaticsessions) >= sm->max_translations_per_user)
 295     {
 296       oldest_per_user_translation_list_index =
 297         clib_dlist_remove_head (tsm->list_pool,
 298                                 u->sessions_per_user_list_head_index);
 299
 300       ASSERT (oldest_per_user_translation_list_index != ~0);
 301
 302       /* Add it back to the end of the LRU list */
 303       clib_dlist_addtail (tsm->list_pool,
 304                           u->sessions_per_user_list_head_index,
 305                           oldest_per_user_translation_list_index);
 306       /* Get the list element */
 307       oldest_per_user_translation_list_elt =
 308         pool_elt_at_index (tsm->list_pool,
 309                            oldest_per_user_translation_list_index);
 310
 311       /* Get the session index from the list element */
 312       session_index = oldest_per_user_translation_list_elt->value;
 313
 314       /* Get the session */
 315       s = pool_elt_at_index (tsm->sessions, session_index);
 316       nat_free_session_data (sm, s, thread_index);
 317       s->outside_address_index = ~0;
 318       s->flags = 0;
 319       s->total_bytes = 0;
 320       s->total_pkts = 0;
 321     }
 322   else
 323     {
 324       pool_get (tsm->sessions, s);
 325       memset (s, 0, sizeof (*s));
 326       s->outside_address_index = ~0;
 327
 328       /* Create list elts */
 329       pool_get (tsm->list_pool, per_user_translation_list_elt);
 330       clib_dlist_init (tsm->list_pool,
 331                        per_user_translation_list_elt - tsm->list_pool);
 332
 333       per_user_translation_list_elt->value = s - tsm->sessions;
 334       s->per_user_index = per_user_translation_list_elt - tsm->list_pool;
 335       s->per_user_list_head_index = u->sessions_per_user_list_head_index;
 336
 337       clib_dlist_addtail (tsm->list_pool,
 338                           s->per_user_list_head_index,
 339                           per_user_translation_list_elt - tsm->list_pool);
 340     }
 341
 342   return s;
 343 }
 344
 345 static inline uword
 346 nat44_classify_node_fn_inline (vlib_main_t * vm,
 347                                vlib_node_runtime_t * node,
 348                                vlib_frame_t * frame)
 349 {
 350   u32 n_left_from, * from, * to_next;
 351   nat44_classify_next_t next_index;
 352   snat_main_t *sm = &snat_main;
 353
 354   from = vlib_frame_vector_args (frame);
 355   n_left_from = frame->n_vectors;
 356   next_index = node->cached_next_index;
 357
 358   while (n_left_from > 0)
 359     {
 360       u32 n_left_to_next;
 361
 362       vlib_get_next_frame (vm, node, next_index,
 363                            to_next, n_left_to_next);
 364
 365       while (n_left_from > 0 && n_left_to_next > 0)
 366         {
 367           u32 bi0;
 368           vlib_buffer_t *b0;
 369           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT;
 370           ip4_header_t *ip0;
 371           snat_address_t *ap;
 372           snat_session_key_t m_key0;
 373           clib_bihash_kv_8_8_t kv0, value0;
 374
 375           /* speculatively enqueue b0 to the current next frame */
 376           bi0 = from[0];
 377           to_next[0] = bi0;
 378           from += 1;
 379           to_next += 1;
 380           n_left_from -= 1;
 381           n_left_to_next -= 1;
 382
 383           b0 = vlib_get_buffer (vm, bi0);
 384           ip0 = vlib_buffer_get_current (b0);
 385
 386           vec_foreach (ap, sm->addresses)
 387             {
 388               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
 389                 {
 390                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 391                   goto enqueue0;
 392                 }
 393             }
 394
 395           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
 396             {
 397               m_key0.addr = ip0->dst_address;
 398               m_key0.port = 0;
 399               m_key0.protocol = 0;
 400               m_key0.fib_index = sm->outside_fib_index;
 401               kv0.key = m_key0.as_u64;
 402               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 403                 {
 404                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 405                   goto enqueue0;
 406                 }
 407               udp_header_t * udp0 = ip4_next_header (ip0);
 408               m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
 409               m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
 410               kv0.key = m_key0.as_u64;
 411               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv0, &value0))
 412                 next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
 413             }
 414
 415         enqueue0:
 416           /* verify speculative enqueue, maybe switch current next frame */
 417           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 418                                            to_next, n_left_to_next,
 419                                            bi0, next0);
 420         }
 421
 422       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 423     }
 424
 425   return frame->n_vectors;
 426 }
 427
 428 static uword
 429 nat44_classify_node_fn (vlib_main_t * vm,
 430                         vlib_node_runtime_t * node,
 431                         vlib_frame_t * frame)
 432 {
 433   return nat44_classify_node_fn_inline (vm, node, frame);
 434 };
 435
 436 VLIB_REGISTER_NODE (nat44_classify_node) = {
 437   .function = nat44_classify_node_fn,
 438   .name = "nat44-classify",
 439   .vector_size = sizeof (u32),
 440   .type = VLIB_NODE_TYPE_INTERNAL,
 441   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 442   .next_nodes = {
 443     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
 444     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
 445   },
 446 };
 447
 448 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node,
 449                               nat44_classify_node_fn);
 450
 451 static uword
 452 nat44_det_classify_node_fn (vlib_main_t * vm,
 453                             vlib_node_runtime_t * node,
 454                             vlib_frame_t * frame)
 455 {
 456   return nat44_classify_node_fn_inline (vm, node, frame);
 457 };
 458
 459 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
 460   .function = nat44_det_classify_node_fn,
 461   .name = "nat44-det-classify",
 462   .vector_size = sizeof (u32),
 463   .type = VLIB_NODE_TYPE_INTERNAL,
 464   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 465   .next_nodes = {
 466     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
 467     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
 468   },
 469 };
 470
 471 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
 472                               nat44_det_classify_node_fn);
 473
 474 static uword
 475 nat44_handoff_classify_node_fn (vlib_main_t * vm,
 476                                 vlib_node_runtime_t * node,
 477                                 vlib_frame_t * frame)
 478 {
 479   return nat44_classify_node_fn_inline (vm, node, frame);
 480 };
 481
 482 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
 483   .function = nat44_handoff_classify_node_fn,
 484   .name = "nat44-handoff-classify",
 485   .vector_size = sizeof (u32),
 486   .type = VLIB_NODE_TYPE_INTERNAL,
 487   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
 488   .next_nodes = {
 489     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
 490     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
 491   },
 492 };
 493
 494 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
 495                               nat44_handoff_classify_node_fn);
 496
 497 /**
 498  * @brief Add/del NAT address to FIB.
 499  *
 500  * Add the external NAT address to the FIB as receive entries. This ensures
 501  * that VPP will reply to ARP for this address and we don't need to enable
 502  * proxy ARP on the outside interface.
 503  *
 504  * @param addr IPv4 address.
 505  * @param plen address prefix length
 506  * @param sw_if_index Interface.
 507  * @param is_add If 0 delete, otherwise add.
 508  */
 509 void
 510 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 511                           int is_add)
 512 {
 513   fib_prefix_t prefix = {
 514     .fp_len = p_len,
 515     .fp_proto = FIB_PROTOCOL_IP4,
 516     .fp_addr = {
 517         .ip4.as_u32 = addr->as_u32,
 518     },
 519   };
 520   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
 521
 522   if (is_add)
 523     fib_table_entry_update_one_path(fib_index,
 524                                     &prefix,
 525                                     FIB_SOURCE_PLUGIN_LOW,
 526                                     (FIB_ENTRY_FLAG_CONNECTED |
 527                                      FIB_ENTRY_FLAG_LOCAL |
 528                                      FIB_ENTRY_FLAG_EXCLUSIVE),
 529                                     DPO_PROTO_IP4,
 530                                     NULL,
 531                                     sw_if_index,
 532                                     ~0,
 533                                     1,
 534                                     NULL,
 535                                     FIB_ROUTE_PATH_FLAG_NONE);
 536   else
 537     fib_table_entry_delete(fib_index,
 538                            &prefix,
 539                            FIB_SOURCE_PLUGIN_LOW);
 540 }
 541
 542 void snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id,
 543                        u8 twice_nat)
 544 {
 545   snat_address_t * ap;
 546   snat_interface_t *i;
 547   vlib_thread_main_t *tm = vlib_get_thread_main ();
 548
 549   /* Check if address already exists */
 550   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 551     {
 552       if (ap->addr.as_u32 == addr->as_u32)
 553         return;
 554     }
 555
 556   if (twice_nat)
 557     vec_add2 (sm->twice_nat_addresses, ap, 1);
 558   else
 559     vec_add2 (sm->addresses, ap, 1);
 560
 561   ap->addr = *addr;
 562   if (vrf_id != ~0)
 563     ap->fib_index =
 564       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 565                                          FIB_SOURCE_PLUGIN_LOW);
 566   else
 567     ap->fib_index = ~0;
 568 #define _(N, i, n, s) \
 569   clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535); \
 570   ap->busy_##n##_ports = 0; \
 571   vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 572   foreach_snat_protocol
 573 #undef _
 574
 575   if (twice_nat)
 576     return;
 577
 578   /* Add external address to FIB */
 579   pool_foreach (i, sm->interfaces,
 580   ({
 581     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 582       continue;
 583
 584     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 585     break;
 586   }));
 587   pool_foreach (i, sm->output_feature_interfaces,
 588   ({
 589     if (nat_interface_is_inside(i) || sm->out2in_dpo)
 590       continue;
 591
 592     snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
 593     break;
 594   }));
 595 }
 596
 597 static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
 598                                                    ip4_address_t addr)
 599 {
 600   snat_static_mapping_t *m;
 601   pool_foreach (m, sm->static_mappings,
 602   ({
 603       if (m->external_addr.as_u32 == addr.as_u32)
 604         return 1;
 605   }));
 606
 607   return 0;
 608 }
 609
 610 void increment_v4_address (ip4_address_t * a)
 611 {
 612   u32 v;
 613
 614   v = clib_net_to_host_u32(a->as_u32) + 1;
 615   a->as_u32 = clib_host_to_net_u32(v);
 616 }
 617
 618 static void
 619 snat_add_static_mapping_when_resolved (snat_main_t * sm,
 620                                        ip4_address_t l_addr,
 621                                        u16 l_port,
 622                                        u32 sw_if_index,
 623                                        u16 e_port,
 624                                        u32 vrf_id,
 625                                        snat_protocol_t proto,
 626                                        int addr_only,
 627                                        int is_add,
 628                                        u8 * tag)
 629 {
 630   snat_static_map_resolve_t *rp;
 631
 632   vec_add2 (sm->to_resolve, rp, 1);
 633   rp->l_addr.as_u32 = l_addr.as_u32;
 634   rp->l_port = l_port;
 635   rp->sw_if_index = sw_if_index;
 636   rp->e_port = e_port;
 637   rp->vrf_id = vrf_id;
 638   rp->proto = proto;
 639   rp->addr_only = addr_only;
 640   rp->is_add = is_add;
 641   rp->tag = vec_dup (tag);
 642 }
 643
 644 /**
 645  * @brief Add static mapping.
 646  *
 647  * Create static mapping between local addr+port and external addr+port.
 648  *
 649  * @param l_addr Local IPv4 address.
 650  * @param e_addr External IPv4 address.
 651  * @param l_port Local port number.
 652  * @param e_port External port number.
 653  * @param vrf_id VRF ID.
 654  * @param addr_only If 0 address port and pair mapping, otherwise address only.
 655  * @param sw_if_index External port instead of specific IP address.
 656  * @param is_add If 0 delete static mapping, otherwise add.
 657  * @param twice_nat If 1 translate external host address and port.
 658  * @param out2in_only If 1 rule match only out2in direction
 659  * @param tag - opaque string tag
 660  *
 661  * @returns
 662  */
 663 int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 664                             u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 665                             u32 sw_if_index, snat_protocol_t proto, int is_add,
 666                             u8 twice_nat, u8 out2in_only, u8 * tag)
 667 {
 668   snat_main_t * sm = &snat_main;
 669   snat_static_mapping_t *m;
 670   snat_session_key_t m_key;
 671   clib_bihash_kv_8_8_t kv, value;
 672   snat_address_t *a = 0;
 673   u32 fib_index = ~0;
 674   uword * p;
 675   snat_interface_t *interface;
 676   int i;
 677   snat_main_per_thread_data_t *tsm;
 678   snat_user_key_t u_key;
 679   snat_user_t *u;
 680   dlist_elt_t * head, * elt;
 681   u32 elt_index, head_index;
 682   u32 ses_index;
 683   u64 user_index;
 684   snat_session_t * s;
 685   snat_static_map_resolve_t *rp, *rp_match = 0;
 686
 687   /* If the external address is a specific interface address */
 688   if (sw_if_index != ~0)
 689     {
 690       ip4_address_t * first_int_addr;
 691
 692       for (i = 0; i < vec_len (sm->to_resolve); i++)
 693         {
 694           rp = sm->to_resolve + i;
 695           if (rp->sw_if_index != sw_if_index &&
 696               rp->l_addr.as_u32 != l_addr.as_u32 &&
 697               rp->vrf_id != vrf_id && rp->addr_only != addr_only)
 698             continue;
 699
 700           if (!addr_only)
 701             {
 702               if (rp->l_port != l_port && rp->e_port != e_port && rp->proto != proto)
 703                 continue;
 704             }
 705
 706           rp_match = rp;
 707           break;
 708         }
 709
 710       /* Might be already set... */
 711       first_int_addr = ip4_interface_first_address
 712         (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
 713
 714       if (is_add)
 715         {
 716           if (rp_match)
 717             return VNET_API_ERROR_VALUE_EXIST;
 718
 719           snat_add_static_mapping_when_resolved
 720             (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
 721              addr_only,  is_add, tag);
 722
 723           /* DHCP resolution required? */
 724           if (first_int_addr == 0)
 725             {
 726               return 0;
 727             }
 728           else
 729             {
 730               e_addr.as_u32 = first_int_addr->as_u32;
 731               /* Identity mapping? */
 732               if (l_addr.as_u32 == 0)
 733                 l_addr.as_u32 = e_addr.as_u32;
 734             }
 735         }
 736       else
 737         {
 738           if (!rp_match)
 739             return VNET_API_ERROR_NO_SUCH_ENTRY;
 740
 741           vec_del1 (sm->to_resolve, i);
 742
 743           if (first_int_addr)
 744             {
 745               e_addr.as_u32 = first_int_addr->as_u32;
 746               /* Identity mapping? */
 747               if (l_addr.as_u32 == 0)
 748                 l_addr.as_u32 = e_addr.as_u32;
 749             }
 750           else
 751             return 0;
 752         }
 753     }
 754
 755   m_key.addr = e_addr;
 756   m_key.port = addr_only ? 0 : e_port;
 757   m_key.protocol = addr_only ? 0 : proto;
 758   m_key.fib_index = sm->outside_fib_index;
 759   kv.key = m_key.as_u64;
 760   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 761     m = 0;
 762   else
 763     m = pool_elt_at_index (sm->static_mappings, value.value);
 764
 765   if (is_add)
 766     {
 767       if (m)
 768         return VNET_API_ERROR_VALUE_EXIST;
 769
 770       if (twice_nat && addr_only)
 771         return VNET_API_ERROR_UNSUPPORTED;
 772
 773       /* Convert VRF id to FIB index */
 774       if (vrf_id != ~0)
 775         {
 776           p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
 777           if (!p)
 778             return VNET_API_ERROR_NO_SUCH_FIB;
 779           fib_index = p[0];
 780         }
 781       /* If not specified use inside VRF id from SNAT plugin startup config */
 782       else
 783         {
 784           fib_index = sm->inside_fib_index;
 785           vrf_id = sm->inside_vrf_id;
 786         }
 787
 788       if (!out2in_only)
 789         {
 790           m_key.addr = l_addr;
 791           m_key.port = addr_only ? 0 : l_port;
 792           m_key.protocol = addr_only ? 0 : proto;
 793           m_key.fib_index = fib_index;
 794           kv.key = m_key.as_u64;
 795           if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 796             return VNET_API_ERROR_VALUE_EXIST;
 797         }
 798
 799       /* Find external address in allocated addresses and reserve port for
 800          address and port pair mapping when dynamic translations enabled */
 801       if (!(addr_only || sm->static_mapping_only || out2in_only))
 802         {
 803           for (i = 0; i < vec_len (sm->addresses); i++)
 804             {
 805               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 806                 {
 807                   a = sm->addresses + i;
 808                   /* External port must be unused */
 809                   switch (proto)
 810                     {
 811 #define _(N, j, n, s) \
 812                     case SNAT_PROTOCOL_##N: \
 813                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
 814                         return VNET_API_ERROR_INVALID_VALUE; \
 815                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
 816                       if (e_port > 1024) \
 817                         { \
 818                           a->busy_##n##_ports++; \
 819                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
 820                         } \
 821                       break;
 822                       foreach_snat_protocol
 823 #undef _
 824                     default:
 825                       clib_warning("unknown_protocol");
 826                       return VNET_API_ERROR_INVALID_VALUE_2;
 827                     }
 828                   break;
 829                 }
 830             }
 831           /* External address must be allocated */
 832           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 833             {
 834               if (sw_if_index != ~0)
 835                 {
 836                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 837                     {
 838                       rp = sm->to_resolve + i;
 839                       if (rp->addr_only)
 840                          continue;
 841                       if (rp->sw_if_index != sw_if_index &&
 842                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 843                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 844                           rp->e_port != e_port && rp->proto != proto)
 845                         continue;
 846
 847                       vec_del1 (sm->to_resolve, i);
 848                       break;
 849                     }
 850                 }
 851               return VNET_API_ERROR_NO_SUCH_ENTRY;
 852             }
 853         }
 854
 855       pool_get (sm->static_mappings, m);
 856       memset (m, 0, sizeof (*m));
 857       m->tag = vec_dup (tag);
 858       m->local_addr = l_addr;
 859       m->external_addr = e_addr;
 860       m->addr_only = addr_only;
 861       m->vrf_id = vrf_id;
 862       m->fib_index = fib_index;
 863       m->twice_nat = twice_nat;
 864       m->out2in_only = out2in_only;
 865       if (!addr_only)
 866         {
 867           m->local_port = l_port;
 868           m->external_port = e_port;
 869           m->proto = proto;
 870         }
 871
 872       if (sm->workers)
 873         {
 874           ip4_header_t ip = {
 875             .src_address = m->local_addr,
 876           };
 877           m->worker_index = sm->worker_in2out_cb (&ip, m->fib_index);
 878           tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
 879         }
 880       else
 881         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 882
 883       m_key.addr = m->local_addr;
 884       m_key.port = m->local_port;
 885       m_key.protocol = m->proto;
 886       m_key.fib_index = m->fib_index;
 887       kv.key = m_key.as_u64;
 888       kv.value = m - sm->static_mappings;
 889       if (!out2in_only)
 890         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
 891       if (twice_nat || out2in_only)
 892         {
 893           m_key.port = clib_host_to_net_u16 (l_port);
 894           kv.key = m_key.as_u64;
 895           kv.value = ~0ULL;
 896           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 1))
 897             clib_warning ("in2out key add failed");
 898         }
 899
 900       m_key.addr = m->external_addr;
 901       m_key.port = m->external_port;
 902       m_key.fib_index = sm->outside_fib_index;
 903       kv.key = m_key.as_u64;
 904       kv.value = m - sm->static_mappings;
 905       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
 906       if (twice_nat || out2in_only)
 907         {
 908           m_key.port = clib_host_to_net_u16 (e_port);
 909           kv.key = m_key.as_u64;
 910           kv.value = ~0ULL;
 911           if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 1))
 912             clib_warning ("out2in key add failed");
 913         }
 914
 915       /* Delete dynamic sessions matching local address (+ local port) */
 916       if (!(sm->static_mapping_only))
 917         {
 918           u_key.addr = m->local_addr;
 919           u_key.fib_index = m->fib_index;
 920           kv.key = u_key.as_u64;
 921           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
 922             {
 923               user_index = value.value;
 924               u = pool_elt_at_index (tsm->users, user_index);
 925               if (u->nsessions)
 926                 {
 927                   head_index = u->sessions_per_user_list_head_index;
 928                   head = pool_elt_at_index (tsm->list_pool, head_index);
 929                   elt_index = head->next;
 930                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
 931                   ses_index = elt->value;
 932                   while (ses_index != ~0)
 933                     {
 934                       s =  pool_elt_at_index (tsm->sessions, ses_index);
 935                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
 936                       ses_index = elt->value;
 937
 938                       if (snat_is_session_static (s))
 939                         continue;
 940
 941                       if (!addr_only)
 942                         {
 943                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
 944                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
 945                             continue;
 946                         }
 947
 948                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
 949                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
 950                       pool_put_index (tsm->list_pool, s->per_user_index);
 951                       pool_put (tsm->sessions, s);
 952                       u->nsessions--;
 953
 954                       if (!addr_only)
 955                         break;
 956                     }
 957                 }
 958             }
 959         }
 960     }
 961   else
 962     {
 963       if (!m)
 964         {
 965           if (sw_if_index != ~0)
 966             return 0;
 967           else
 968             return VNET_API_ERROR_NO_SUCH_ENTRY;
 969         }
 970
 971       /* Free external address port */
 972       if (!(addr_only || sm->static_mapping_only || out2in_only))
 973         {
 974           for (i = 0; i < vec_len (sm->addresses); i++)
 975             {
 976               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 977                 {
 978                   a = sm->addresses + i;
 979                   switch (proto)
 980                     {
 981 #define _(N, j, n, s) \
 982                     case SNAT_PROTOCOL_##N: \
 983                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
 984                       if (e_port > 1024) \
 985                         { \
 986                           a->busy_##n##_ports--; \
 987                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
 988                         } \
 989                       break;
 990                       foreach_snat_protocol
 991 #undef _
 992                     default:
 993                       clib_warning("unknown_protocol");
 994                       return VNET_API_ERROR_INVALID_VALUE_2;
 995                     }
 996                   break;
 997                 }
 998             }
 999         }
1000
1001       if (sm->num_workers > 1)
1002         tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
1003       else
1004         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1005
1006       m_key.addr = m->local_addr;
1007       m_key.port = m->local_port;
1008       m_key.protocol = m->proto;
1009       m_key.fib_index = m->fib_index;
1010       kv.key = m_key.as_u64;
1011       if (!out2in_only)
1012         clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
1013       if (twice_nat || out2in_only)
1014         {
1015           m_key.port = clib_host_to_net_u16 (m->local_port);
1016           kv.key = m_key.as_u64;
1017           kv.value = ~0ULL;
1018           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 0))
1019             clib_warning ("in2out key del failed");
1020         }
1021
1022       m_key.addr = m->external_addr;
1023       m_key.port = m->external_port;
1024       m_key.fib_index = sm->outside_fib_index;
1025       kv.key = m_key.as_u64;
1026       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
1027       if (twice_nat || out2in_only)
1028         {
1029           m_key.port = clib_host_to_net_u16 (m->external_port);
1030           kv.key = m_key.as_u64;
1031           kv.value = ~0ULL;
1032           if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 0))
1033             clib_warning ("in2out key del failed");
1034         }
1035
1036       /* Delete session(s) for static mapping if exist */
1037       if (!(sm->static_mapping_only) ||
1038           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
1039         {
1040           u_key.addr = m->local_addr;
1041           u_key.fib_index = m->fib_index;
1042           kv.key = u_key.as_u64;
1043           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1044             {
1045               user_index = value.value;
1046               u = pool_elt_at_index (tsm->users, user_index);
1047               if (u->nstaticsessions)
1048                 {
1049                   head_index = u->sessions_per_user_list_head_index;
1050                   head = pool_elt_at_index (tsm->list_pool, head_index);
1051                   elt_index = head->next;
1052                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1053                   ses_index = elt->value;
1054                   while (ses_index != ~0)
1055                     {
1056                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1057                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1058                       ses_index = elt->value;
1059
1060                       if (!addr_only)
1061                         {
1062                           if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
1063                               (clib_net_to_host_u16 (s->out2in.port) != e_port))
1064                             continue;
1065                         }
1066
1067                       if (s->flags & SNAT_SESSION_FLAG_LOAD_BALANCING)
1068                           continue;
1069
1070                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1071                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
1072                       pool_put_index (tsm->list_pool, s->per_user_index);
1073                       pool_put (tsm->sessions, s);
1074                       u->nstaticsessions--;
1075
1076                       if (!addr_only)
1077                         break;
1078                     }
1079                   if (addr_only && (u->nstaticsessions == 0))
1080                     {
1081                       pool_put (tsm->users, u);
1082                       clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 0);
1083                     }
1084                 }
1085             }
1086         }
1087
1088       vec_free (m->tag);
1089       /* Delete static mapping from pool */
1090       pool_put (sm->static_mappings, m);
1091     }
1092
1093   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
1094     return 0;
1095
1096   /* Add/delete external address to FIB */
1097   pool_foreach (interface, sm->interfaces,
1098   ({
1099     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1100       continue;
1101
1102     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1103     break;
1104   }));
1105   pool_foreach (interface, sm->output_feature_interfaces,
1106   ({
1107     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1108       continue;
1109
1110     snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
1111     break;
1112   }));
1113
1114   return 0;
1115 }
1116
1117 static int lb_local_exists (nat44_lb_addr_port_t * local,
1118                             ip4_address_t * e_addr, u16 e_port)
1119 {
1120   snat_main_t *sm = &snat_main;
1121   snat_static_mapping_t *m;
1122   nat44_lb_addr_port_t *ap;
1123
1124   /* *INDENT-OFF* */
1125   pool_foreach (m, sm->static_mappings,
1126   ({
1127       if (vec_len(m->locals))
1128         {
1129           if (m->external_port == e_port && m->external_addr.as_u32 == e_addr->as_u32)
1130             continue;
1131
1132           vec_foreach (ap, m->locals)
1133           {
1134             if (ap->port == local->port && ap->addr.as_u32 == local->addr.as_u32)
1135               return 1;
1136           }
1137         }
1138   }));
1139   /* *INDENT-ON* */
1140
1141   return 0;
1142 }
1143
1144 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1145                                      snat_protocol_t proto, u32 vrf_id,
1146                                      nat44_lb_addr_port_t *locals, u8 is_add,
1147                                      u8 twice_nat, u8 out2in_only, u8 *tag)
1148 {
1149   snat_main_t * sm = &snat_main;
1150   snat_static_mapping_t *m;
1151   snat_session_key_t m_key;
1152   clib_bihash_kv_8_8_t kv, value;
1153   u32 fib_index;
1154   snat_address_t *a = 0;
1155   int i;
1156   nat44_lb_addr_port_t *local;
1157   u32 worker_index = 0, elt_index, head_index, ses_index;
1158   snat_main_per_thread_data_t *tsm;
1159   snat_user_key_t u_key;
1160   snat_user_t *u;
1161   snat_session_t * s;
1162   dlist_elt_t * head, * elt;
1163
1164   m_key.addr = e_addr;
1165   m_key.port = e_port;
1166   m_key.protocol = proto;
1167   m_key.fib_index = sm->outside_fib_index;
1168   kv.key = m_key.as_u64;
1169   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1170     m = 0;
1171   else
1172     m = pool_elt_at_index (sm->static_mappings, value.value);
1173
1174   if (is_add)
1175     {
1176       if (m)
1177         return VNET_API_ERROR_VALUE_EXIST;
1178
1179       if (vec_len (locals) < 2)
1180         return VNET_API_ERROR_INVALID_VALUE;
1181
1182       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
1183                                                      vrf_id,
1184                                                      FIB_SOURCE_PLUGIN_LOW);
1185
1186       /* Find external address in allocated addresses and reserve port for
1187          address and port pair mapping when dynamic translations enabled */
1188       if (!(sm->static_mapping_only || out2in_only))
1189         {
1190           for (i = 0; i < vec_len (sm->addresses); i++)
1191             {
1192               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1193                 {
1194                   a = sm->addresses + i;
1195                   /* External port must be unused */
1196                   switch (proto)
1197                     {
1198 #define _(N, j, n, s) \
1199                     case SNAT_PROTOCOL_##N: \
1200                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
1201                         return VNET_API_ERROR_INVALID_VALUE; \
1202                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
1203                       if (e_port > 1024) \
1204                         { \
1205                           a->busy_##n##_ports++; \
1206                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]++; \
1207                         } \
1208                       break;
1209                       foreach_snat_protocol
1210 #undef _
1211                     default:
1212                       clib_warning("unknown_protocol");
1213                       return VNET_API_ERROR_INVALID_VALUE_2;
1214                     }
1215                   break;
1216                 }
1217             }
1218           /* External address must be allocated */
1219           if (!a)
1220             return VNET_API_ERROR_NO_SUCH_ENTRY;
1221         }
1222
1223       pool_get (sm->static_mappings, m);
1224       memset (m, 0, sizeof (*m));
1225       m->tag = vec_dup (tag);
1226       m->external_addr = e_addr;
1227       m->addr_only = 0;
1228       m->vrf_id = vrf_id;
1229       m->fib_index = fib_index;
1230       m->external_port = e_port;
1231       m->proto = proto;
1232       m->twice_nat = twice_nat;
1233       m->out2in_only = out2in_only;
1234
1235       m_key.addr = m->external_addr;
1236       m_key.port = m->external_port;
1237       m_key.protocol = m->proto;
1238       m_key.fib_index = sm->outside_fib_index;
1239       kv.key = m_key.as_u64;
1240       kv.value = m - sm->static_mappings;
1241       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
1242         {
1243           clib_warning ("static_mapping_by_external key add failed");
1244           return VNET_API_ERROR_UNSPECIFIED;
1245         }
1246
1247       /* Assign worker */
1248       if (sm->workers)
1249         {
1250           worker_index = sm->first_worker_index +
1251             sm->workers[sm->next_worker++ % vec_len (sm->workers)];
1252           tsm = vec_elt_at_index (sm->per_thread_data, worker_index);
1253           m->worker_index = worker_index;
1254         }
1255       else
1256         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1257
1258       m_key.port = clib_host_to_net_u16 (m->external_port);
1259       kv.key = m_key.as_u64;
1260       kv.value = ~0ULL;
1261       if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 1))
1262         {
1263           clib_warning ("out2in key add failed");
1264           return VNET_API_ERROR_UNSPECIFIED;
1265         }
1266
1267       m_key.fib_index = m->fib_index;
1268       for (i = 0; i < vec_len (locals); i++)
1269         {
1270           m_key.addr = locals[i].addr;
1271           if (!out2in_only)
1272             {
1273               m_key.port = locals[i].port;
1274               kv.key = m_key.as_u64;
1275               kv.value = m - sm->static_mappings;
1276               clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
1277             }
1278           locals[i].prefix = (i == 0) ? locals[i].probability :\
1279             (locals[i - 1].prefix + locals[i].probability);
1280           vec_add1 (m->locals, locals[i]);
1281
1282           m_key.port = clib_host_to_net_u16 (locals[i].port);
1283           kv.key = m_key.as_u64;
1284           kv.value = ~0ULL;
1285           if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 1))
1286             {
1287               clib_warning ("in2out key add failed");
1288               return VNET_API_ERROR_UNSPECIFIED;
1289             }
1290         }
1291     }
1292   else
1293     {
1294       if (!m)
1295         return VNET_API_ERROR_NO_SUCH_ENTRY;
1296
1297       fib_table_unlock (m->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
1298
1299       /* Free external address port */
1300       if (!(sm->static_mapping_only || out2in_only))
1301         {
1302           for (i = 0; i < vec_len (sm->addresses); i++)
1303             {
1304               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1305                 {
1306                   a = sm->addresses + i;
1307                   switch (proto)
1308                     {
1309 #define _(N, j, n, s) \
1310                     case SNAT_PROTOCOL_##N: \
1311                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
1312                       if (e_port > 1024) \
1313                         { \
1314                           a->busy_##n##_ports--; \
1315                           a->busy_##n##_ports_per_thread[(e_port - 1024) / sm->port_per_thread]--; \
1316                         } \
1317                       break;
1318                       foreach_snat_protocol
1319 #undef _
1320                     default:
1321                       clib_warning("unknown_protocol");
1322                       return VNET_API_ERROR_INVALID_VALUE_2;
1323                     }
1324                   break;
1325                 }
1326             }
1327         }
1328
1329       tsm = vec_elt_at_index (sm->per_thread_data, m->worker_index);
1330       m_key.addr = m->external_addr;
1331       m_key.port = m->external_port;
1332       m_key.protocol = m->proto;
1333       m_key.fib_index = sm->outside_fib_index;
1334       kv.key = m_key.as_u64;
1335       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
1336         {
1337           clib_warning ("static_mapping_by_external key del failed");
1338           return VNET_API_ERROR_UNSPECIFIED;
1339         }
1340
1341       m_key.port = clib_host_to_net_u16 (m->external_port);
1342       kv.key = m_key.as_u64;
1343       if (clib_bihash_add_del_8_8(&tsm->out2in, &kv, 0))
1344         {
1345           clib_warning ("outi2in key del failed");
1346           return VNET_API_ERROR_UNSPECIFIED;
1347         }
1348
1349       vec_foreach (local, m->locals)
1350         {
1351           m_key.addr = local->addr;
1352           if (!out2in_only)
1353             {
1354               m_key.port = local->port;
1355               m_key.fib_index = m->fib_index;
1356               kv.key = m_key.as_u64;
1357               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
1358                 {
1359                   clib_warning ("static_mapping_by_local key del failed");
1360                   return VNET_API_ERROR_UNSPECIFIED;
1361                 }
1362             }
1363
1364           if (!lb_local_exists(local, &e_addr, e_port))
1365             {
1366               m_key.port = clib_host_to_net_u16 (local->port);
1367               kv.key = m_key.as_u64;
1368               if (clib_bihash_add_del_8_8(&tsm->in2out, &kv, 0))
1369                 {
1370                   clib_warning ("in2out key del failed");
1371                   return VNET_API_ERROR_UNSPECIFIED;
1372                 }
1373             }
1374           /* Delete sessions */
1375           u_key.addr = local->addr;
1376           u_key.fib_index = m->fib_index;
1377           kv.key = u_key.as_u64;
1378           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1379             {
1380               u = pool_elt_at_index (tsm->users, value.value);
1381               if (u->nstaticsessions)
1382                 {
1383                   head_index = u->sessions_per_user_list_head_index;
1384                   head = pool_elt_at_index (tsm->list_pool, head_index);
1385                   elt_index = head->next;
1386                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1387                   ses_index = elt->value;
1388                   while (ses_index != ~0)
1389                     {
1390                       s =  pool_elt_at_index (tsm->sessions, ses_index);
1391                       elt = pool_elt_at_index (tsm->list_pool, elt->next);
1392                       ses_index = elt->value;
1393
1394                       if ((s->in2out.addr.as_u32 != local->addr.as_u32) &&
1395                           (clib_net_to_host_u16 (s->in2out.port) != local->port))
1396                         continue;
1397
1398                       nat_free_session_data (sm, s, tsm - sm->per_thread_data);
1399                       clib_dlist_remove (tsm->list_pool, s->per_user_index);
1400                       pool_put_index (tsm->list_pool, s->per_user_index);
1401                       pool_put (tsm->sessions, s);
1402                       u->nstaticsessions--;
1403                     }
1404                 }
1405             }
1406         }
1407       vec_free(m->locals);
1408       vec_free(m->tag);
1409
1410       pool_put (sm->static_mappings, m);
1411     }
1412
1413   return 0;
1414 }
1415
1416 int
1417 snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm,
1418                   u8 twice_nat)
1419 {
1420   snat_address_t *a = 0;
1421   snat_session_t *ses;
1422   u32 *ses_to_be_removed = 0, *ses_index;
1423   clib_bihash_kv_8_8_t kv, value;
1424   snat_user_key_t user_key;
1425   snat_user_t *u;
1426   snat_main_per_thread_data_t *tsm;
1427   snat_static_mapping_t *m;
1428   snat_interface_t *interface;
1429   int i;
1430   snat_address_t *addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1431
1432   /* Find SNAT address */
1433   for (i=0; i < vec_len (addresses); i++)
1434     {
1435       if (addresses[i].addr.as_u32 == addr.as_u32)
1436         {
1437           a = addresses + i;
1438           break;
1439         }
1440     }
1441   if (!a)
1442     return VNET_API_ERROR_NO_SUCH_ENTRY;
1443
1444   if (delete_sm)
1445     {
1446       pool_foreach (m, sm->static_mappings,
1447       ({
1448           if (m->external_addr.as_u32 == addr.as_u32)
1449             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1450                                             m->local_port, m->external_port,
1451                                             m->vrf_id, m->addr_only, ~0,
1452                                             m->proto, 0, m->twice_nat,
1453                                             m->out2in_only, m->tag);
1454       }));
1455     }
1456   else
1457     {
1458       /* Check if address is used in some static mapping */
1459       if (is_snat_address_used_in_static_mapping(sm, addr))
1460         {
1461           clib_warning ("address used in static mapping");
1462           return VNET_API_ERROR_UNSPECIFIED;
1463         }
1464     }
1465
1466   if (a->fib_index != ~0)
1467     fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
1468                      FIB_SOURCE_PLUGIN_LOW);
1469
1470   /* Delete sessions using address */
1471   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1472     {
1473       vec_foreach (tsm, sm->per_thread_data)
1474         {
1475           pool_foreach (ses, tsm->sessions, ({
1476             if (ses->out2in.addr.as_u32 == addr.as_u32)
1477               {
1478                 ses->outside_address_index = ~0;
1479                 nat_free_session_data (sm, ses, tsm - sm->per_thread_data);
1480                 clib_dlist_remove (tsm->list_pool, ses->per_user_index);
1481                 pool_put_index (tsm->list_pool, ses->per_user_index);
1482                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1483                 user_key.addr = ses->in2out.addr;
1484                 user_key.fib_index = ses->in2out.fib_index;
1485                 kv.key = user_key.as_u64;
1486                 if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1487                   {
1488                     u = pool_elt_at_index (tsm->users, value.value);
1489                     u->nsessions--;
1490                   }
1491               }
1492           }));
1493
1494           vec_foreach (ses_index, ses_to_be_removed)
1495             pool_put_index (tsm->sessions, ses_index[0]);
1496
1497           vec_free (ses_to_be_removed);
1498        }
1499     }
1500
1501   if (twice_nat)
1502     {
1503       vec_del1 (sm->twice_nat_addresses, i);
1504       return 0;
1505     }
1506   else
1507     vec_del1 (sm->addresses, i);
1508
1509   /* Delete external address from FIB */
1510   pool_foreach (interface, sm->interfaces,
1511   ({
1512     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1513       continue;
1514
1515     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1516     break;
1517   }));
1518   pool_foreach (interface, sm->output_feature_interfaces,
1519   ({
1520     if (nat_interface_is_inside(interface) || sm->out2in_dpo)
1521       continue;
1522
1523     snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
1524     break;
1525   }));
1526
1527   return 0;
1528 }
1529
1530 int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1531 {
1532   snat_main_t *sm = &snat_main;
1533   snat_interface_t *i;
1534   const char * feature_name, *del_feature_name;
1535   snat_address_t * ap;
1536   snat_static_mapping_t * m;
1537   snat_det_map_t * dm;
1538
1539   if (sm->out2in_dpo && !is_inside)
1540     return VNET_API_ERROR_UNSUPPORTED;
1541
1542   pool_foreach (i, sm->output_feature_interfaces,
1543   ({
1544     if (i->sw_if_index == sw_if_index)
1545       return VNET_API_ERROR_VALUE_EXIST;
1546   }));
1547
1548   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1549     feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
1550   else
1551     {
1552       if (sm->num_workers > 1 && !sm->deterministic)
1553         feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
1554       else if (sm->deterministic)
1555         feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
1556       else
1557         feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
1558     }
1559
1560   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1561     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index, 0);
1562
1563   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
1564     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1565
1566   pool_foreach (i, sm->interfaces,
1567   ({
1568     if (i->sw_if_index == sw_if_index)
1569       {
1570         if (is_del)
1571           {
1572             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1573               {
1574                 if (is_inside)
1575                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1576                 else
1577                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1578
1579                 if (sm->num_workers > 1 && !sm->deterministic)
1580                   {
1581                     del_feature_name = "nat44-handoff-classify";
1582                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1583                                                  "nat44-out2in-worker-handoff";
1584                   }
1585                 else if (sm->deterministic)
1586                   {
1587                     del_feature_name = "nat44-det-classify";
1588                     feature_name = !is_inside ?  "nat44-det-in2out" :
1589                                                  "nat44-det-out2in";
1590                   }
1591                 else
1592                   {
1593                     del_feature_name = "nat44-classify";
1594                     feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1595                   }
1596
1597                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1598                                              sw_if_index, 0, 0, 0);
1599                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1600                                              sw_if_index, 1, 0, 0);
1601               }
1602             else
1603               {
1604                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1605                                              sw_if_index, 0, 0, 0);
1606                 pool_put (sm->interfaces, i);
1607               }
1608           }
1609         else
1610           {
1611             if ((nat_interface_is_inside(i) && is_inside) ||
1612                 (nat_interface_is_outside(i) && !is_inside))
1613               return 0;
1614
1615             if (sm->num_workers > 1 && !sm->deterministic)
1616               {
1617                 del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1618                                                  "nat44-out2in-worker-handoff";
1619                 feature_name = "nat44-handoff-classify";
1620               }
1621             else if (sm->deterministic)
1622               {
1623                 del_feature_name = !is_inside ?  "nat44-det-in2out" :
1624                                                  "nat44-det-out2in";
1625                 feature_name = "nat44-det-classify";
1626               }
1627             else
1628               {
1629                 del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
1630                 feature_name = "nat44-classify";
1631               }
1632
1633             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1634                                          sw_if_index, 0, 0, 0);
1635             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1636                                          sw_if_index, 1, 0, 0);
1637             goto set_flags;
1638           }
1639
1640         goto fib;
1641       }
1642   }));
1643
1644   if (is_del)
1645     return VNET_API_ERROR_NO_SUCH_ENTRY;
1646
1647   pool_get (sm->interfaces, i);
1648   i->sw_if_index = sw_if_index;
1649   i->flags = 0;
1650   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0, 0);
1651
1652 set_flags:
1653   if (is_inside)
1654     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1655   else
1656     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1657
1658   /* Add/delete external addresses to FIB */
1659 fib:
1660   if (is_inside && !sm->out2in_dpo)
1661     {
1662       vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
1663                                    sw_if_index, !is_del, 0, 0);
1664       return 0;
1665     }
1666
1667   vec_foreach (ap, sm->addresses)
1668     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1669
1670   pool_foreach (m, sm->static_mappings,
1671   ({
1672     if (!(m->addr_only) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1673       continue;
1674
1675     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1676   }));
1677
1678   pool_foreach (dm, sm->det_maps,
1679   ({
1680     snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
1681   }));
1682
1683   return 0;
1684 }
1685
1686 int snat_interface_add_del_output_feature (u32 sw_if_index,
1687                                            u8 is_inside,
1688                                            int is_del)
1689 {
1690   snat_main_t *sm = &snat_main;
1691   snat_interface_t *i;
1692   snat_address_t * ap;
1693   snat_static_mapping_t * m;
1694
1695   if (sm->deterministic ||
1696       (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
1697     return VNET_API_ERROR_UNSUPPORTED;
1698
1699   pool_foreach (i, sm->interfaces,
1700   ({
1701     if (i->sw_if_index == sw_if_index)
1702       return VNET_API_ERROR_VALUE_EXIST;
1703   }));
1704
1705   if (is_inside)
1706     {
1707       vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
1708                                    sw_if_index, !is_del, 0, 0);
1709       vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
1710                                    sw_if_index, !is_del, 0, 0);
1711       goto fq;
1712     }
1713
1714   if (sm->num_workers > 1)
1715     {
1716       vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in-worker-handoff",
1717                                    sw_if_index, !is_del, 0, 0);
1718       vnet_feature_enable_disable ("ip4-output",
1719                                    "nat44-in2out-output-worker-handoff",
1720                                    sw_if_index, !is_del, 0, 0);
1721     }
1722   else
1723     {
1724       vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in", sw_if_index,
1725                                    !is_del, 0, 0);
1726       vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
1727                                    sw_if_index, !is_del, 0, 0);
1728     }
1729
1730 fq:
1731   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1732     sm->fq_in2out_output_index =
1733       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1734
1735   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1736     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1737
1738   pool_foreach (i, sm->output_feature_interfaces,
1739   ({
1740     if (i->sw_if_index == sw_if_index)
1741       {
1742         if (is_del)
1743           pool_put (sm->output_feature_interfaces, i);
1744         else
1745           return VNET_API_ERROR_VALUE_EXIST;
1746
1747         goto fib;
1748       }
1749   }));
1750
1751   if (is_del)
1752     return VNET_API_ERROR_NO_SUCH_ENTRY;
1753
1754   pool_get (sm->output_feature_interfaces, i);
1755   i->sw_if_index = sw_if_index;
1756   i->flags = 0;
1757   if (is_inside)
1758     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1759   else
1760     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1761
1762   /* Add/delete external addresses to FIB */
1763 fib:
1764   if (is_inside)
1765     return 0;
1766
1767   vec_foreach (ap, sm->addresses)
1768     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1769
1770   pool_foreach (m, sm->static_mappings,
1771   ({
1772     if (!(m->addr_only))
1773       continue;
1774
1775     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1776   }));
1777
1778   return 0;
1779 }
1780
1781 int snat_set_workers (uword * bitmap)
1782 {
1783   snat_main_t *sm = &snat_main;
1784   int i, j = 0;
1785
1786   if (sm->num_workers < 2)
1787     return VNET_API_ERROR_FEATURE_DISABLED;
1788
1789   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1790     return VNET_API_ERROR_INVALID_WORKER;
1791
1792   vec_free (sm->workers);
1793   clib_bitmap_foreach (i, bitmap,
1794     ({
1795       vec_add1(sm->workers, i);
1796       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1797       j++;
1798     }));
1799
1800   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
1801   sm->num_snat_thread = _vec_len (sm->workers);
1802
1803   return 0;
1804 }
1805
1806
1807 static void
1808 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
1809                                        uword opaque,
1810                                        u32 sw_if_index,
1811                                        ip4_address_t * address,
1812                                        u32 address_length,
1813                                        u32 if_address_index,
1814                                        u32 is_delete);
1815
1816 static void
1817 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
1818                                  uword opaque,
1819                                  u32 sw_if_index,
1820                                  ip4_address_t * address,
1821                                  u32 address_length,
1822                                  u32 if_address_index,
1823                                  u32 is_delete);
1824
1825 static int
1826 nat_alloc_addr_and_port_default (snat_address_t * addresses,
1827                                  u32 fib_index,
1828                                  u32 thread_index,
1829                                  snat_session_key_t * k,
1830                                  u32 * address_indexp,
1831                                  u16 port_per_thread,
1832                                  u32 snat_thread_index);
1833
1834 static clib_error_t * snat_init (vlib_main_t * vm)
1835 {
1836   snat_main_t * sm = &snat_main;
1837   clib_error_t * error = 0;
1838   ip4_main_t * im = &ip4_main;
1839   ip_lookup_main_t * lm = &im->lookup_main;
1840   uword *p;
1841   vlib_thread_registration_t *tr;
1842   vlib_thread_main_t *tm = vlib_get_thread_main ();
1843   uword *bitmap = 0;
1844   u32 i;
1845   ip4_add_del_interface_address_callback_t cb4;
1846
1847   sm->vlib_main = vm;
1848   sm->vnet_main = vnet_get_main();
1849   sm->ip4_main = im;
1850   sm->ip4_lookup_main = lm;
1851   sm->api_main = &api_main;
1852   sm->first_worker_index = 0;
1853   sm->next_worker = 0;
1854   sm->num_workers = 0;
1855   sm->num_snat_thread = 1;
1856   sm->workers = 0;
1857   sm->port_per_thread = 0xffff - 1024;
1858   sm->fq_in2out_index = ~0;
1859   sm->fq_out2in_index = ~0;
1860   sm->udp_timeout = SNAT_UDP_TIMEOUT;
1861   sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
1862   sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
1863   sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
1864   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
1865   sm->forwarding_enabled = 0;
1866
1867   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
1868   if (p)
1869     {
1870       tr = (vlib_thread_registration_t *) p[0];
1871       if (tr)
1872         {
1873           sm->num_workers = tr->count;
1874           sm->first_worker_index = tr->first_index;
1875         }
1876     }
1877
1878   vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
1879
1880   /* Use all available workers by default */
1881   if (sm->num_workers > 1)
1882     {
1883       for (i=0; i < sm->num_workers; i++)
1884         bitmap = clib_bitmap_set (bitmap, i, 1);
1885       snat_set_workers(bitmap);
1886       clib_bitmap_free (bitmap);
1887     }
1888   else
1889     {
1890       sm->per_thread_data[0].snat_thread_index = 0;
1891     }
1892
1893   error = snat_api_init(vm, sm);
1894   if (error)
1895     return error;
1896
1897   /* Set up the interface address add/del callback */
1898   cb4.function = snat_ip4_add_del_interface_address_cb;
1899   cb4.function_opaque = 0;
1900
1901   vec_add1 (im->add_del_interface_address_callbacks, cb4);
1902
1903   cb4.function = nat_ip4_add_del_addr_only_sm_cb;
1904   cb4.function_opaque = 0;
1905
1906   vec_add1 (im->add_del_interface_address_callbacks, cb4);
1907
1908   nat_dpo_module_init ();
1909
1910   /* Init IPFIX logging */
1911   snat_ipfix_logging_init(vm);
1912
1913   /* Init NAT64 */
1914   error = nat64_init(vm);
1915   if (error)
1916     return error;
1917
1918   dslite_init(vm);
1919
1920   nat66_init();
1921
1922   /* Init virtual fragmenentation reassembly */
1923   return nat_reass_init(vm);
1924 }
1925
1926 VLIB_INIT_FUNCTION (snat_init);
1927
1928 void snat_free_outside_address_and_port (snat_address_t * addresses,
1929                                          u32 thread_index,
1930                                          snat_session_key_t * k,
1931                                          u32 address_index)
1932 {
1933   snat_address_t *a;
1934   u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
1935
1936   ASSERT (address_index < vec_len (addresses));
1937
1938   a = addresses + address_index;
1939
1940   switch (k->protocol)
1941     {
1942 #define _(N, i, n, s) \
1943     case SNAT_PROTOCOL_##N: \
1944       ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
1945         port_host_byte_order) == 1); \
1946       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
1947         port_host_byte_order, 0); \
1948       a->busy_##n##_ports--; \
1949       a->busy_##n##_ports_per_thread[thread_index]--; \
1950       break;
1951       foreach_snat_protocol
1952 #undef _
1953     default:
1954       clib_warning("unknown_protocol");
1955       return;
1956     }
1957 }
1958
1959 /**
1960  * @brief Match NAT44 static mapping.
1961  *
1962  * @param sm          NAT main.
1963  * @param match       Address and port to match.
1964  * @param mapping     External or local address and port of the matched mapping.
1965  * @param by_external If 0 match by local address otherwise match by external
1966  *                    address.
1967  * @param is_addr_only If matched mapping is address only
1968  * @param twice_nat If matched mapping is twice NAT.
1969  *
1970  * @returns 0 if match found otherwise 1.
1971  */
1972 int snat_static_mapping_match (snat_main_t * sm,
1973                                snat_session_key_t match,
1974                                snat_session_key_t * mapping,
1975                                u8 by_external,
1976                                u8 *is_addr_only,
1977                                u8 *twice_nat)
1978 {
1979   clib_bihash_kv_8_8_t kv, value;
1980   snat_static_mapping_t *m;
1981   snat_session_key_t m_key;
1982   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
1983   u32 rand, lo = 0, hi, mid;
1984
1985   if (by_external)
1986     mapping_hash = &sm->static_mapping_by_external;
1987
1988   m_key.addr = match.addr;
1989   m_key.port = clib_net_to_host_u16 (match.port);
1990   m_key.protocol = match.protocol;
1991   m_key.fib_index = match.fib_index;
1992
1993   kv.key = m_key.as_u64;
1994
1995   if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
1996     {
1997       /* Try address only mapping */
1998       m_key.port = 0;
1999       m_key.protocol = 0;
2000       kv.key = m_key.as_u64;
2001       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2002         return 1;
2003     }
2004
2005   m = pool_elt_at_index (sm->static_mappings, value.value);
2006
2007   if (by_external)
2008     {
2009       if (vec_len (m->locals))
2010         {
2011           hi = vec_len (m->locals) - 1;
2012           rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
2013           while (lo < hi)
2014             {
2015               mid = ((hi - lo) >> 1) + lo;
2016               (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
2017             }
2018           if (!(m->locals[lo].prefix >= rand))
2019             return 1;
2020           mapping->addr = m->locals[lo].addr;
2021           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
2022         }
2023       else
2024         {
2025           mapping->addr = m->local_addr;
2026           /* Address only mapping doesn't change port */
2027           mapping->port = m->addr_only ? match.port
2028             : clib_host_to_net_u16 (m->local_port);
2029         }
2030       mapping->fib_index = m->fib_index;
2031       mapping->protocol = m->proto;
2032     }
2033   else
2034     {
2035       mapping->addr = m->external_addr;
2036       /* Address only mapping doesn't change port */
2037       mapping->port = m->addr_only ? match.port
2038         : clib_host_to_net_u16 (m->external_port);
2039       mapping->fib_index = sm->outside_fib_index;
2040     }
2041
2042   if (PREDICT_FALSE(is_addr_only != 0))
2043     *is_addr_only = m->addr_only;
2044
2045   if (PREDICT_FALSE(twice_nat != 0))
2046     *twice_nat = m->twice_nat;
2047
2048   return 0;
2049 }
2050
2051 static_always_inline u16
2052 snat_random_port (u16 min, u16 max)
2053 {
2054   snat_main_t *sm = &snat_main;
2055   return min + random_u32 (&sm->random_seed) /
2056     (random_u32_max() / (max - min + 1) + 1);
2057 }
2058
2059 int
2060 snat_alloc_outside_address_and_port (snat_address_t * addresses,
2061                                      u32 fib_index,
2062                                      u32 thread_index,
2063                                      snat_session_key_t * k,
2064                                      u32 * address_indexp,
2065                                      u16 port_per_thread,
2066                                      u32 snat_thread_index)
2067 {
2068   snat_main_t *sm = &snat_main;
2069
2070   return sm->alloc_addr_and_port(addresses, fib_index, thread_index, k,
2071                                  address_indexp, port_per_thread,
2072                                  snat_thread_index);
2073 }
2074
2075 static int
2076 nat_alloc_addr_and_port_default (snat_address_t * addresses,
2077                                  u32 fib_index,
2078                                  u32 thread_index,
2079                                  snat_session_key_t * k,
2080                                  u32 * address_indexp,
2081                                  u16 port_per_thread,
2082                                  u32 snat_thread_index)
2083 {
2084   int i, gi = 0;
2085   snat_address_t *a, *ga = 0;
2086   u32 portnum;
2087
2088   for (i = 0; i < vec_len (addresses); i++)
2089     {
2090       a = addresses + i;
2091       switch (k->protocol)
2092         {
2093 #define _(N, j, n, s) \
2094         case SNAT_PROTOCOL_##N: \
2095           if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread) \
2096             { \
2097               if (a->fib_index == fib_index) \
2098                 { \
2099                   while (1) \
2100                     { \
2101                       portnum = (port_per_thread * \
2102                         snat_thread_index) + \
2103                         snat_random_port(1, port_per_thread) + 1024; \
2104                       if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2105                         continue; \
2106                       clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2107                       a->busy_##n##_ports_per_thread[thread_index]++; \
2108                       a->busy_##n##_ports++; \
2109                       k->addr = a->addr; \
2110                       k->port = clib_host_to_net_u16(portnum); \
2111                       *address_indexp = i; \
2112                       return 0; \
2113                     } \
2114                 } \
2115               else if (a->fib_index == ~0) \
2116                 { \
2117                   ga = a; \
2118                   gi = i; \
2119                 } \
2120             } \
2121           break;
2122           foreach_snat_protocol
2123 #undef _
2124         default:
2125           clib_warning("unknown protocol");
2126           return 1;
2127         }
2128
2129     }
2130
2131   if (ga)
2132     {
2133       a = ga;
2134       switch (k->protocol)
2135         {
2136 #define _(N, j, n, s) \
2137         case SNAT_PROTOCOL_##N: \
2138           while (1) \
2139             { \
2140               portnum = (port_per_thread * \
2141                 snat_thread_index) + \
2142                 snat_random_port(1, port_per_thread) + 1024; \
2143               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2144                 continue; \
2145               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2146               a->busy_##n##_ports_per_thread[thread_index]++; \
2147               a->busy_##n##_ports++; \
2148               k->addr = a->addr; \
2149               k->port = clib_host_to_net_u16(portnum); \
2150               *address_indexp = gi; \
2151               return 0; \
2152             }
2153           break;
2154           foreach_snat_protocol
2155 #undef _
2156         default:
2157           clib_warning ("unknown protocol");
2158           return 1;
2159         }
2160     }
2161
2162   /* Totally out of translations to use... */
2163   snat_ipfix_logging_addresses_exhausted(0);
2164   return 1;
2165 }
2166
2167 static int
2168 nat_alloc_addr_and_port_mape (snat_address_t * addresses,
2169                               u32 fib_index,
2170                               u32 thread_index,
2171                               snat_session_key_t * k,
2172                               u32 * address_indexp,
2173                               u16 port_per_thread,
2174                               u32 snat_thread_index)
2175 {
2176   snat_main_t *sm = &snat_main;
2177   snat_address_t *a = addresses;
2178   u16 m, ports, portnum, A, j;
2179   m = 16 - (sm->psid_offset + sm->psid_length);
2180   ports = (1 << (16 - sm->psid_length)) - (1 << m);
2181
2182   if (!vec_len (addresses))
2183     goto exhausted;
2184
2185   switch (k->protocol)
2186     {
2187 #define _(N, i, n, s) \
2188     case SNAT_PROTOCOL_##N: \
2189       if (a->busy_##n##_ports < ports) \
2190         { \
2191           while (1) \
2192             { \
2193               A = snat_random_port(1, pow2_mask(sm->psid_offset)); \
2194               j = snat_random_port(0, pow2_mask(m)); \
2195               portnum = A | (sm->psid << sm->psid_offset) | (j << (16 - m)); \
2196               if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
2197                 continue; \
2198               clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
2199               a->busy_##n##_ports++; \
2200               k->addr = a->addr; \
2201               k->port = clib_host_to_net_u16 (portnum); \
2202               *address_indexp = i; \
2203               return 0; \
2204             } \
2205         } \
2206       break;
2207       foreach_snat_protocol
2208 #undef _
2209     default:
2210       clib_warning("unknown protocol");
2211       return 1;
2212     }
2213
2214 exhausted:
2215   /* Totally out of translations to use... */
2216   snat_ipfix_logging_addresses_exhausted(0);
2217   return 1;
2218 }
2219
2220 void
2221 nat44_add_del_address_dpo (ip4_address_t addr, u8 is_add)
2222 {
2223   dpo_id_t dpo_v4 = DPO_INVALID;
2224   fib_prefix_t pfx = {
2225     .fp_proto = FIB_PROTOCOL_IP4,
2226     .fp_len = 32,
2227     .fp_addr.ip4.as_u32 = addr.as_u32,
2228   };
2229
2230   if (is_add)
2231     {
2232       nat_dpo_create (DPO_PROTO_IP4, 0, &dpo_v4);
2233       fib_table_entry_special_dpo_add (0, &pfx, FIB_SOURCE_PLUGIN_HI,
2234                                        FIB_ENTRY_FLAG_EXCLUSIVE, &dpo_v4);
2235       dpo_reset (&dpo_v4);
2236     }
2237   else
2238     {
2239       fib_table_entry_special_remove (0, &pfx, FIB_SOURCE_PLUGIN_HI);
2240     }
2241 }
2242
2243 uword
2244 unformat_snat_protocol (unformat_input_t * input, va_list * args)
2245 {
2246   u32 *r = va_arg (*args, u32 *);
2247
2248   if (0);
2249 #define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
2250   foreach_snat_protocol
2251 #undef _
2252   else
2253     return 0;
2254   return 1;
2255 }
2256
2257 u8 *
2258 format_snat_protocol (u8 * s, va_list * args)
2259 {
2260   u32 i = va_arg (*args, u32);
2261   u8 *t = 0;
2262
2263   switch (i)
2264     {
2265 #define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
2266       foreach_snat_protocol
2267 #undef _
2268     default:
2269       s = format (s, "unknown");
2270       return s;
2271     }
2272   s = format (s, "%s", t);
2273   return s;
2274 }
2275
2276 static u32
2277 snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2278 {
2279   snat_main_t *sm = &snat_main;
2280   u32 next_worker_index = 0;
2281   u32 hash;
2282
2283   next_worker_index = sm->first_worker_index;
2284   hash = ip0->src_address.as_u32 + (ip0->src_address.as_u32 >> 8) +
2285          (ip0->src_address.as_u32 >> 16) + (ip0->src_address.as_u32 >>24);
2286
2287   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2288     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2289   else
2290     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2291
2292   return next_worker_index;
2293 }
2294
2295 static u32
2296 snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
2297 {
2298   snat_main_t *sm = &snat_main;
2299   udp_header_t *udp;
2300   u16 port;
2301   snat_session_key_t m_key;
2302   clib_bihash_kv_8_8_t kv, value;
2303   snat_static_mapping_t *m;
2304   nat_ed_ses_key_t key;
2305   clib_bihash_kv_16_8_t s_kv, s_value;
2306   snat_main_per_thread_data_t *tsm;
2307   snat_session_t *s;
2308   int i;
2309   u32 proto;
2310   u32 next_worker_index = 0;
2311
2312   /* first try static mappings without port */
2313   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2314     {
2315       m_key.addr = ip0->dst_address;
2316       m_key.port = 0;
2317       m_key.protocol = 0;
2318       m_key.fib_index = rx_fib_index0;
2319       kv.key = m_key.as_u64;
2320       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2321         {
2322           m = pool_elt_at_index (sm->static_mappings, value.value);
2323           return m->worker_index;
2324         }
2325     }
2326
2327   proto = ip_proto_to_snat_proto (ip0->protocol);
2328   udp = ip4_next_header (ip0);
2329   port = udp->dst_port;
2330
2331   if (PREDICT_FALSE (ip4_is_fragment (ip0)))
2332     {
2333       if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
2334         return vlib_get_thread_index ();
2335
2336       if (PREDICT_TRUE (!ip4_is_first_fragment (ip0)))
2337         {
2338           nat_reass_ip4_t *reass;
2339
2340           reass = nat_ip4_reass_find (ip0->src_address, ip0->dst_address,
2341                                       ip0->fragment_id, ip0->protocol);
2342
2343           if (reass && (reass->thread_index != (u32) ~ 0))
2344             return reass->thread_index;
2345           else
2346             return vlib_get_thread_index ();
2347         }
2348     }
2349
2350   /* unknown protocol */
2351   if (PREDICT_FALSE (proto == ~0))
2352     {
2353       key.l_addr = ip0->dst_address;
2354       key.r_addr = ip0->src_address;
2355       key.fib_index = rx_fib_index0;
2356       key.proto = ip0->protocol;
2357       key.r_port = 0;
2358       key.l_port = 0;
2359       s_kv.key[0] = key.as_u64[0];
2360       s_kv.key[1] = key.as_u64[1];
2361
2362       if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
2363         {
2364           for (i = 0; i < _vec_len (sm->per_thread_data); i++)
2365             {
2366               tsm = vec_elt_at_index (sm->per_thread_data, i);
2367               if (!pool_is_free_index(tsm->sessions, s_value.value))
2368                 {
2369                   s = pool_elt_at_index (tsm->sessions, s_value.value);
2370                   if (s->out2in.addr.as_u32 == ip0->dst_address.as_u32 &&
2371                       s->out2in.port == ip0->protocol &&
2372                       snat_is_unk_proto_session (s))
2373                     return i;
2374                 }
2375             }
2376          }
2377
2378       /* if no session use current thread */
2379       return vlib_get_thread_index ();
2380     }
2381
2382   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_ICMP))
2383     {
2384       icmp46_header_t * icmp = (icmp46_header_t *) udp;
2385       icmp_echo_header_t *echo = (icmp_echo_header_t *)(icmp + 1);
2386       if (!icmp_is_error_message (icmp))
2387         port = echo->identifier;
2388       else
2389         {
2390           ip4_header_t *inner_ip = (ip4_header_t *)(echo + 1);
2391           proto = ip_proto_to_snat_proto (inner_ip->protocol);
2392           void *l4_header = ip4_next_header (inner_ip);
2393           switch (proto)
2394             {
2395             case SNAT_PROTOCOL_ICMP:
2396               icmp = (icmp46_header_t*)l4_header;
2397               echo = (icmp_echo_header_t *)(icmp + 1);
2398               port = echo->identifier;
2399               break;
2400             case SNAT_PROTOCOL_UDP:
2401             case SNAT_PROTOCOL_TCP:
2402               port = ((tcp_udp_header_t*)l4_header)->src_port;
2403               break;
2404             default:
2405               return vlib_get_thread_index ();
2406             }
2407         }
2408     }
2409
2410   /* try static mappings with port */
2411   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2412     {
2413       m_key.addr = ip0->dst_address;
2414       m_key.port = clib_net_to_host_u16 (port);
2415       m_key.protocol = proto;
2416       m_key.fib_index = rx_fib_index0;
2417       kv.key = m_key.as_u64;
2418       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2419         {
2420           m = pool_elt_at_index (sm->static_mappings, value.value);
2421           return m->worker_index;
2422         }
2423     }
2424
2425   /* worker by outside port */
2426   next_worker_index = sm->first_worker_index;
2427   next_worker_index +=
2428     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2429   return next_worker_index;
2430 }
2431
2432 static clib_error_t *
2433 snat_config (vlib_main_t * vm, unformat_input_t * input)
2434 {
2435   snat_main_t * sm = &snat_main;
2436   u32 translation_buckets = 1024;
2437   u32 translation_memory_size = 128<<20;
2438   u32 user_buckets = 128;
2439   u32 user_memory_size = 64<<20;
2440   u32 max_translations_per_user = 100;
2441   u32 outside_vrf_id = 0;
2442   u32 inside_vrf_id = 0;
2443   u32 static_mapping_buckets = 1024;
2444   u32 static_mapping_memory_size = 64<<20;
2445   u32 nat64_bib_buckets = 1024;
2446   u32 nat64_bib_memory_size = 128 << 20;
2447   u32 nat64_st_buckets = 2048;
2448   u32 nat64_st_memory_size = 256 << 20;
2449   u8 static_mapping_only = 0;
2450   u8 static_mapping_connection_tracking = 0;
2451   snat_main_per_thread_data_t *tsm;
2452   dslite_main_t * dm = &dslite_main;
2453
2454   sm->deterministic = 0;
2455   sm->out2in_dpo = 0;
2456
2457   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2458     {
2459       if (unformat (input, "translation hash buckets %d", &translation_buckets))
2460         ;
2461       else if (unformat (input, "translation hash memory %d",
2462                          &translation_memory_size));
2463       else if (unformat (input, "user hash buckets %d", &user_buckets))
2464         ;
2465       else if (unformat (input, "user hash memory %d",
2466                          &user_memory_size))
2467         ;
2468       else if (unformat (input, "max translations per user %d",
2469                          &max_translations_per_user))
2470         ;
2471       else if (unformat (input, "outside VRF id %d",
2472                          &outside_vrf_id))
2473         ;
2474       else if (unformat (input, "inside VRF id %d",
2475                          &inside_vrf_id))
2476         ;
2477       else if (unformat (input, "static mapping only"))
2478         {
2479           static_mapping_only = 1;
2480           if (unformat (input, "connection tracking"))
2481             static_mapping_connection_tracking = 1;
2482         }
2483       else if (unformat (input, "deterministic"))
2484         sm->deterministic = 1;
2485       else if (unformat (input, "nat64 bib hash buckets %d",
2486                          &nat64_bib_buckets))
2487         ;
2488       else if (unformat (input, "nat64 bib hash memory %d",
2489                          &nat64_bib_memory_size))
2490         ;
2491       else if (unformat (input, "nat64 st hash buckets %d", &nat64_st_buckets))
2492         ;
2493       else if (unformat (input, "nat64 st hash memory %d",
2494                          &nat64_st_memory_size))
2495         ;
2496       else if (unformat (input, "out2in dpo"))
2497         sm->out2in_dpo = 1;
2498       else if (unformat (input, "dslite ce"))
2499         dslite_set_ce(dm, 1);
2500       else
2501         return clib_error_return (0, "unknown input '%U'",
2502                                   format_unformat_error, input);
2503     }
2504
2505   /* for show commands, etc. */
2506   sm->translation_buckets = translation_buckets;
2507   sm->translation_memory_size = translation_memory_size;
2508   /* do not exceed load factor 10 */
2509   sm->max_translations = 10 * translation_buckets;
2510   sm->user_buckets = user_buckets;
2511   sm->user_memory_size = user_memory_size;
2512   sm->max_translations_per_user = max_translations_per_user;
2513   sm->outside_vrf_id = outside_vrf_id;
2514   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2515                                                              outside_vrf_id,
2516                                                              FIB_SOURCE_PLUGIN_HI);
2517   sm->inside_vrf_id = inside_vrf_id;
2518   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2519                                                             inside_vrf_id,
2520                                                             FIB_SOURCE_PLUGIN_HI);
2521   sm->static_mapping_only = static_mapping_only;
2522   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
2523
2524   nat64_set_hash(nat64_bib_buckets, nat64_bib_memory_size, nat64_st_buckets,
2525                  nat64_st_memory_size);
2526
2527   if (sm->deterministic)
2528     {
2529       sm->in2out_node_index = snat_det_in2out_node.index;
2530       sm->in2out_output_node_index = ~0;
2531       sm->out2in_node_index = snat_det_out2in_node.index;
2532       sm->icmp_match_in2out_cb = icmp_match_in2out_det;
2533       sm->icmp_match_out2in_cb = icmp_match_out2in_det;
2534     }
2535   else
2536     {
2537       sm->worker_in2out_cb = snat_get_worker_in2out_cb;
2538       sm->worker_out2in_cb = snat_get_worker_out2in_cb;
2539       sm->in2out_node_index = snat_in2out_node.index;
2540       sm->in2out_output_node_index = snat_in2out_output_node.index;
2541       sm->out2in_node_index = snat_out2in_node.index;
2542       if (!static_mapping_only ||
2543           (static_mapping_only && static_mapping_connection_tracking))
2544         {
2545           sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
2546           sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
2547
2548           vec_foreach (tsm, sm->per_thread_data)
2549             {
2550               clib_bihash_init_8_8 (&tsm->in2out, "in2out", translation_buckets,
2551                                     translation_memory_size);
2552
2553               clib_bihash_init_8_8 (&tsm->out2in, "out2in", translation_buckets,
2554                                     translation_memory_size);
2555
2556               clib_bihash_init_8_8 (&tsm->user_hash, "users", user_buckets,
2557                                     user_memory_size);
2558             }
2559
2560           clib_bihash_init_16_8 (&sm->in2out_ed, "in2out-ed",
2561                                  translation_buckets, translation_memory_size);
2562
2563           clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed",
2564                                  translation_buckets, translation_memory_size);
2565         }
2566       else
2567         {
2568           sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
2569           sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
2570         }
2571       clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2572                             "static_mapping_by_local", static_mapping_buckets,
2573                             static_mapping_memory_size);
2574
2575       clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2576                             "static_mapping_by_external", static_mapping_buckets,
2577                             static_mapping_memory_size);
2578     }
2579
2580   return 0;
2581 }
2582
2583 VLIB_CONFIG_FUNCTION (snat_config, "nat");
2584
2585 u8 * format_snat_session_state (u8 * s, va_list * args)
2586 {
2587   u32 i = va_arg (*args, u32);
2588   u8 *t = 0;
2589
2590   switch (i)
2591     {
2592 #define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
2593     foreach_snat_session_state
2594 #undef _
2595     default:
2596       t = format (t, "unknown");
2597     }
2598   s = format (s, "%s", t);
2599   return s;
2600 }
2601
2602 u8 * format_snat_key (u8 * s, va_list * args)
2603 {
2604   snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
2605
2606   s = format (s, "%U proto %U port %d fib %d",
2607               format_ip4_address, &key->addr,
2608               format_snat_protocol, key->protocol,
2609               clib_net_to_host_u16 (key->port), key->fib_index);
2610   return s;
2611 }
2612
2613 u8 * format_snat_session (u8 * s, va_list * args)
2614 {
2615   snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *);
2616   snat_session_t * sess = va_arg (*args, snat_session_t *);
2617
2618   if (snat_is_unk_proto_session (sess))
2619     {
2620       s = format (s, "  i2o %U proto %u fib %u\n",
2621                   format_ip4_address, &sess->in2out.addr,
2622                   clib_net_to_host_u16 (sess->in2out.port),
2623                   sess->in2out.fib_index);
2624       s = format (s, "    o2i %U proto %u fib %u\n",
2625                   format_ip4_address, &sess->out2in.addr,
2626                   clib_net_to_host_u16 (sess->out2in.port),
2627                   sess->out2in.fib_index);
2628     }
2629   else
2630     {
2631       s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
2632       s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
2633     }
2634   if (is_twice_nat_session (sess))
2635     {
2636       s = format (s, "       external host o2i %U:%d i2o %U:%d\n",
2637                   format_ip4_address, &sess->ext_host_addr,
2638                   clib_net_to_host_u16 (sess->ext_host_port),
2639                   format_ip4_address, &sess->ext_host_nat_addr,
2640                   clib_net_to_host_u16 (sess->ext_host_nat_port));
2641     }
2642   else
2643     {
2644       if (sess->ext_host_addr.as_u32)
2645           s = format (s, "       external host %U\n",
2646                       format_ip4_address, &sess->ext_host_addr);
2647     }
2648   s = format (s, "       last heard %.2f\n", sess->last_heard);
2649   s = format (s, "       total pkts %d, total bytes %lld\n",
2650               sess->total_pkts, sess->total_bytes);
2651   if (snat_is_session_static (sess))
2652     s = format (s, "       static translation\n");
2653   else
2654     s = format (s, "       dynamic translation\n");
2655   if (sess->flags & SNAT_SESSION_FLAG_LOAD_BALANCING)
2656     s = format (s, "       load-balancing\n");
2657   if (is_twice_nat_session (sess))
2658     s = format (s, "       twice-nat\n");
2659
2660   return s;
2661 }
2662
2663 u8 * format_snat_user (u8 * s, va_list * args)
2664 {
2665   snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
2666   snat_user_t * u = va_arg (*args, snat_user_t *);
2667   int verbose = va_arg (*args, int);
2668   dlist_elt_t * head, * elt;
2669   u32 elt_index, head_index;
2670   u32 session_index;
2671   snat_session_t * sess;
2672
2673   s = format (s, "%U: %d dynamic translations, %d static translations\n",
2674               format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
2675
2676   if (verbose == 0)
2677     return s;
2678
2679   if (u->nsessions || u->nstaticsessions)
2680     {
2681       head_index = u->sessions_per_user_list_head_index;
2682       head = pool_elt_at_index (sm->list_pool, head_index);
2683
2684       elt_index = head->next;
2685       elt = pool_elt_at_index (sm->list_pool, elt_index);
2686       session_index = elt->value;
2687
2688       while (session_index != ~0)
2689         {
2690           sess = pool_elt_at_index (sm->sessions, session_index);
2691
2692           s = format (s, "  %U\n", format_snat_session, sm, sess);
2693
2694           elt_index = elt->next;
2695           elt = pool_elt_at_index (sm->list_pool, elt_index);
2696           session_index = elt->value;
2697         }
2698     }
2699
2700   return s;
2701 }
2702
2703 u8 * format_snat_static_mapping (u8 * s, va_list * args)
2704 {
2705   snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
2706   nat44_lb_addr_port_t *local;
2707
2708   if (m->addr_only)
2709       s = format (s, "local %U external %U vrf %d %s",
2710                   format_ip4_address, &m->local_addr,
2711                   format_ip4_address, &m->external_addr,
2712                   m->vrf_id, m->twice_nat ? "twice-nat" : "");
2713   else
2714    {
2715       if (vec_len (m->locals))
2716         {
2717           s = format (s, "%U vrf %d external %U:%d %s %s",
2718                       format_snat_protocol, m->proto,
2719                       m->vrf_id,
2720                       format_ip4_address, &m->external_addr, m->external_port,
2721                       m->twice_nat ? "twice-nat" : "",
2722                       m->out2in_only ? "out2in-only" : "");
2723           vec_foreach (local, m->locals)
2724             s = format (s, "\n  local %U:%d probability %d\%",
2725                         format_ip4_address, &local->addr, local->port,
2726                         local->probability);
2727         }
2728       else
2729         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
2730                     format_snat_protocol, m->proto,
2731                     format_ip4_address, &m->local_addr, m->local_port,
2732                     format_ip4_address, &m->external_addr, m->external_port,
2733                     m->vrf_id, m->twice_nat ? "twice-nat" : "",
2734                     m->out2in_only ? "out2in-only" : "");
2735    }
2736   return s;
2737 }
2738
2739 u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
2740 {
2741   snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
2742   vnet_main_t *vnm = vnet_get_main();
2743
2744   if (m->addr_only)
2745       s = format (s, "local %U external %U vrf %d",
2746                   format_ip4_address, &m->l_addr,
2747                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
2748                   m->vrf_id);
2749   else
2750       s = format (s, "%U local %U:%d external %U:%d vrf %d",
2751                   format_snat_protocol, m->proto,
2752                   format_ip4_address, &m->l_addr, m->l_port,
2753                   format_vnet_sw_if_index_name, vnm, m->sw_if_index,
2754                   m->e_port, m->vrf_id);
2755
2756   return s;
2757 }
2758
2759 u8 * format_det_map_ses (u8 * s, va_list * args)
2760 {
2761   snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
2762   ip4_address_t in_addr, out_addr;
2763   u32 in_offset, out_offset;
2764   snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
2765   u32 * i = va_arg (*args, u32 *);
2766
2767   u32 user_index = *i / SNAT_DET_SES_PER_USER;
2768   in_addr.as_u32 = clib_host_to_net_u32 (
2769     clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
2770   in_offset = clib_net_to_host_u32(in_addr.as_u32) -
2771     clib_net_to_host_u32(det_map->in_addr.as_u32);
2772   out_offset = in_offset / det_map->sharing_ratio;
2773   out_addr.as_u32 = clib_host_to_net_u32(
2774     clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
2775   s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
2776               format_ip4_address, &in_addr,
2777               clib_net_to_host_u16 (ses->in_port),
2778               format_ip4_address, &out_addr,
2779               clib_net_to_host_u16 (ses->out.out_port),
2780               format_ip4_address, &ses->out.ext_host_addr,
2781               clib_net_to_host_u16 (ses->out.ext_host_port),
2782               format_snat_session_state, ses->state,
2783               ses->expire);
2784
2785   return s;
2786 }
2787
2788 static void
2789 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2790                                  uword opaque,
2791                                  u32 sw_if_index,
2792                                  ip4_address_t * address,
2793                                  u32 address_length,
2794                                  u32 if_address_index,
2795                                  u32 is_delete)
2796 {
2797   snat_main_t *sm = &snat_main;
2798   snat_static_map_resolve_t *rp;
2799   snat_static_mapping_t *m;
2800   snat_session_key_t m_key;
2801   clib_bihash_kv_8_8_t kv, value;
2802   int i, rv;
2803   ip4_address_t l_addr;
2804
2805   for (i = 0; i < vec_len (sm->to_resolve); i++)
2806     {
2807       rp = sm->to_resolve + i;
2808       if (rp->addr_only == 0)
2809         continue;
2810       if (rp->sw_if_index == sw_if_index)
2811         goto match;
2812     }
2813
2814   return;
2815
2816 match:
2817   m_key.addr.as_u32 = address->as_u32;
2818   m_key.port = rp->addr_only ? 0 : rp->e_port;
2819   m_key.protocol = rp->addr_only ? 0 : rp->proto;
2820   m_key.fib_index = sm->outside_fib_index;
2821   kv.key = m_key.as_u64;
2822   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2823     m = 0;
2824   else
2825     m = pool_elt_at_index (sm->static_mappings, value.value);
2826
2827   if (!is_delete)
2828     {
2829       /* Don't trip over lease renewal, static config */
2830       if (m)
2831         return;
2832     }
2833   else
2834     {
2835       if (!m)
2836         return;
2837     }
2838
2839   /* Indetity mapping? */
2840   if (rp->l_addr.as_u32 == 0)
2841     l_addr.as_u32 = address[0].as_u32;
2842   else
2843     l_addr.as_u32 = rp->l_addr.as_u32;
2844   /* Add the static mapping */
2845   rv = snat_add_static_mapping (l_addr,
2846                                 address[0],
2847                                 rp->l_port,
2848                                 rp->e_port,
2849                                 rp->vrf_id,
2850                                 rp->addr_only,
2851                                 ~0 /* sw_if_index */,
2852                                 rp->proto,
2853                                 !is_delete,
2854                                 0, 0, rp->tag);
2855   if (rv)
2856     clib_warning ("snat_add_static_mapping returned %d", rv);
2857 }
2858
2859 static void
2860 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2861                                        uword opaque,
2862                                        u32 sw_if_index,
2863                                        ip4_address_t * address,
2864                                        u32 address_length,
2865                                        u32 if_address_index,
2866                                        u32 is_delete)
2867 {
2868   snat_main_t *sm = &snat_main;
2869   snat_static_map_resolve_t *rp;
2870   ip4_address_t l_addr;
2871   int i, j;
2872   int rv;
2873   u8 twice_nat = 0;
2874   snat_address_t *addresses = sm->addresses;
2875
2876   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
2877     {
2878       if (sw_if_index == sm->auto_add_sw_if_indices[i])
2879           goto match;
2880     }
2881
2882   for (i = 0; i < vec_len(sm->auto_add_sw_if_indices_twice_nat); i++)
2883     {
2884       twice_nat = 1;
2885       addresses = sm->twice_nat_addresses;
2886       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
2887           goto match;
2888     }
2889
2890   return;
2891
2892 match:
2893   if (!is_delete)
2894     {
2895       /* Don't trip over lease renewal, static config */
2896       for (j = 0; j < vec_len(addresses); j++)
2897         if (addresses[j].addr.as_u32 == address->as_u32)
2898           return;
2899
2900       snat_add_address (sm, address, ~0, twice_nat);
2901       /* Scan static map resolution vector */
2902       for (j = 0; j < vec_len (sm->to_resolve); j++)
2903         {
2904           rp = sm->to_resolve + j;
2905           if (rp->addr_only)
2906             continue;
2907           /* On this interface? */
2908           if (rp->sw_if_index == sw_if_index)
2909             {
2910               /* Indetity mapping? */
2911               if (rp->l_addr.as_u32 == 0)
2912                 l_addr.as_u32 = address[0].as_u32;
2913               else
2914                 l_addr.as_u32 = rp->l_addr.as_u32;
2915               /* Add the static mapping */
2916               rv = snat_add_static_mapping (l_addr,
2917                                             address[0],
2918                                             rp->l_port,
2919                                             rp->e_port,
2920                                             rp->vrf_id,
2921                                             rp->addr_only,
2922                                             ~0 /* sw_if_index */,
2923                                             rp->proto,
2924                                             rp->is_add,
2925                                             0, 0, rp->tag);
2926               if (rv)
2927                 clib_warning ("snat_add_static_mapping returned %d",
2928                               rv);
2929             }
2930         }
2931       return;
2932     }
2933   else
2934     {
2935       (void) snat_del_address(sm, address[0], 1, twice_nat);
2936       return;
2937     }
2938 }
2939
2940
2941 int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del,
2942                                 u8 twice_nat)
2943 {
2944   ip4_main_t * ip4_main = sm->ip4_main;
2945   ip4_address_t * first_int_addr;
2946   snat_static_map_resolve_t *rp;
2947   u32 *indices_to_delete = 0;
2948   int i, j;
2949   u32 *auto_add_sw_if_indices =
2950     twice_nat ? sm->auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
2951
2952   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
2953                                                 0 /* just want the address*/);
2954
2955   for (i = 0; i < vec_len(auto_add_sw_if_indices); i++)
2956     {
2957       if (auto_add_sw_if_indices[i] == sw_if_index)
2958         {
2959           if (is_del)
2960             {
2961               /* if have address remove it */
2962               if (first_int_addr)
2963                   (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
2964               else
2965                 {
2966                   for (j = 0; j < vec_len (sm->to_resolve); j++)
2967                     {
2968                       rp = sm->to_resolve + j;
2969                       if (rp->sw_if_index == sw_if_index)
2970                         vec_add1 (indices_to_delete, j);
2971                     }
2972                   if (vec_len(indices_to_delete))
2973                     {
2974                       for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
2975                         vec_del1(sm->to_resolve, j);
2976                       vec_free(indices_to_delete);
2977                     }
2978                 }
2979               if (twice_nat)
2980                 vec_del1(sm->auto_add_sw_if_indices_twice_nat, i);
2981               else
2982                 vec_del1(sm->auto_add_sw_if_indices, i);
2983             }
2984           else
2985             return VNET_API_ERROR_VALUE_EXIST;
2986
2987           return 0;
2988         }
2989     }
2990
2991   if (is_del)
2992     return VNET_API_ERROR_NO_SUCH_ENTRY;
2993
2994   /* add to the auto-address list */
2995   if (twice_nat)
2996     vec_add1(sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
2997   else
2998     vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
2999
3000   /* If the address is already bound - or static - add it now */
3001   if (first_int_addr)
3002       snat_add_address (sm, first_int_addr, ~0, twice_nat);
3003
3004   return 0;
3005 }
3006
3007 int
3008 nat44_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3009                    snat_protocol_t proto, u32 vrf_id, int is_in)
3010 {
3011   snat_main_per_thread_data_t *tsm;
3012   clib_bihash_kv_8_8_t kv, value;
3013   ip4_header_t ip;
3014   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3015   snat_session_key_t key;
3016   snat_session_t *s;
3017   clib_bihash_8_8_t *t;
3018   snat_user_key_t u_key;
3019   snat_user_t *u;
3020
3021   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3022   if (sm->num_workers > 1)
3023     tsm =
3024       vec_elt_at_index (sm->per_thread_data,
3025                         sm->worker_in2out_cb (&ip, fib_index));
3026   else
3027     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3028
3029   key.addr.as_u32 = addr->as_u32;
3030   key.port = clib_host_to_net_u16 (port);
3031   key.protocol = proto;
3032   key.fib_index = fib_index;
3033   kv.key = key.as_u64;
3034   t = is_in ? &tsm->in2out : &tsm->out2in;
3035   if (!clib_bihash_search_8_8 (t, &kv, &value))
3036     {
3037       s = pool_elt_at_index (tsm->sessions, value.value);
3038       kv.key = s->in2out.as_u64;
3039       clib_bihash_add_del_8_8 (&tsm->in2out, &kv, 0);
3040       kv.key = s->out2in.as_u64;
3041       clib_bihash_add_del_8_8 (&tsm->out2in, &kv, 0);
3042       u_key.addr = s->in2out.addr;
3043       u_key.fib_index = s->in2out.fib_index;
3044       kv.key = u_key.as_u64;
3045       if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
3046         {
3047           u = pool_elt_at_index (tsm->users, value.value);
3048           u->nsessions--;
3049         }
3050       clib_dlist_remove (tsm->list_pool, s->per_user_index);
3051       pool_put (tsm->sessions, s);
3052       return 0;
3053     }
3054
3055   return VNET_API_ERROR_NO_SUCH_ENTRY;
3056 }
3057
3058 void
3059 nat_set_alloc_addr_and_port_mape (u16 psid, u16 psid_offset, u16 psid_length)
3060 {
3061   snat_main_t *sm = &snat_main;
3062
3063   sm->alloc_addr_and_port = nat_alloc_addr_and_port_mape;
3064   sm->psid = psid;
3065   sm->psid_offset = psid_offset;
3066   sm->psid_length = psid_length;
3067 }
3068
3069 void
3070 nat_set_alloc_addr_and_port_default (void)
3071 {
3072   snat_main_t *sm = &snat_main;
3073
3074   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
3075 }
3076