src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vpp/app/version.h>
  19
  20 #include <vnet/vnet.h>
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ip/ip4.h>
  23 #include <vnet/ip/ip_table.h>
  24 #include <vnet/ip/reass/ip4_sv_reass.h>
  25 #include <vnet/fib/fib_table.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <vnet/plugin/plugin.h>
  28 #include <vppinfra/bihash_16_8.h>
  29
  30 #include <nat/lib/log.h>
  31 #include <nat/lib/nat_syslog.h>
  32 #include <nat/lib/nat_inlines.h>
  33 #include <nat/lib/ipfix_logging.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 snat_main_t snat_main;
  40
  41 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  42                                                            u32 sw_if_index);
  43
  44 #define skip_if_disabled()                                                    \
  45   do                                                                          \
  46     {                                                                         \
  47       snat_main_t *sm = &snat_main;                                           \
  48       if (PREDICT_FALSE (!sm->enabled))                                       \
  49         return;                                                               \
  50     }                                                                         \
  51   while (0)
  52
  53 #define fail_if_enabled()                                                     \
  54   do                                                                          \
  55     {                                                                         \
  56       snat_main_t *sm = &snat_main;                                           \
  57       if (PREDICT_FALSE (sm->enabled))                                        \
  58         {                                                                     \
  59           nat_log_err ("plugin enabled");                                     \
  60           return 1;                                                           \
  61         }                                                                     \
  62     }                                                                         \
  63   while (0)
  64
  65 #define fail_if_disabled()                                                    \
  66   do                                                                          \
  67     {                                                                         \
  68       snat_main_t *sm = &snat_main;                                           \
  69       if (PREDICT_FALSE (!sm->enabled))                                       \
  70         {                                                                     \
  71           nat_log_err ("plugin disabled");                                    \
  72           return 1;                                                           \
  73         }                                                                     \
  74     }                                                                         \
  75   while (0)
  76
  77 /* Hook up input features */
  78 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  79   .arc_name = "ip4-unicast",
  80   .node_name = "nat-pre-in2out",
  81   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  82                                "ip4-sv-reassembly-feature"),
  83 };
  84 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  85   .arc_name = "ip4-unicast",
  86   .node_name = "nat-pre-out2in",
  87   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  88                                "ip4-dhcp-client-detect",
  89                                "ip4-sv-reassembly-feature"),
  90 };
  91 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  92   .arc_name = "ip4-unicast",
  93   .node_name = "nat44-in2out-worker-handoff",
  94   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  95 };
  96 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  97   .arc_name = "ip4-unicast",
  98   .node_name = "nat44-out2in-worker-handoff",
  99   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 100                                "ip4-dhcp-client-detect"),
 101 };
 102 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 103   .arc_name = "ip4-unicast",
 104   .node_name = "nat44-in2out",
 105   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 106 };
 107 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 108   .arc_name = "ip4-unicast",
 109   .node_name = "nat44-out2in",
 110   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 111                                "ip4-dhcp-client-detect"),
 112 };
 113 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 114   .arc_name = "ip4-unicast",
 115   .node_name = "nat44-ed-in2out",
 116   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 117 };
 118 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 119   .arc_name = "ip4-unicast",
 120   .node_name = "nat44-ed-out2in",
 121   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 122                                "ip4-dhcp-client-detect"),
 123 };
 124 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 125   .arc_name = "ip4-unicast",
 126   .node_name = "nat44-ed-classify",
 127   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 128 };
 129 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 130   .arc_name = "ip4-unicast",
 131   .node_name = "nat44-handoff-classify",
 132   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 133 };
 134 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 135   .arc_name = "ip4-unicast",
 136   .node_name = "nat44-in2out-fast",
 137   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 138 };
 139 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 140   .arc_name = "ip4-unicast",
 141   .node_name = "nat44-out2in-fast",
 142   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 143                                "ip4-dhcp-client-detect"),
 144 };
 145
 146 /* Hook up output features */
 147 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 148   .arc_name = "ip4-output",
 149   .node_name = "nat44-in2out-output",
 150   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 151   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 152 };
 153 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 154   .arc_name = "ip4-output",
 155   .node_name = "nat44-in2out-output-worker-handoff",
 156   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 157   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 158 };
 159 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 160   .arc_name = "ip4-output",
 161   .node_name = "nat-pre-in2out-output",
 162   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 163   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 164 };
 165 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 166   .arc_name = "ip4-output",
 167   .node_name = "nat44-ed-in2out-output",
 168   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 169   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 170 };
 171
 172 VLIB_PLUGIN_REGISTER () = {
 173     .version = VPP_BUILD_VER,
 174     .description = "Network Address Translation (NAT)",
 175 };
 176
 177 static void nat44_ed_db_init (u32 translations, u32 translation_buckets);
 178
 179 static void nat44_ed_db_free ();
 180
 181 u32 nat_calc_bihash_buckets (u32 n_elts);
 182
 183 u8 *
 184 format_session_kvp (u8 * s, va_list * args)
 185 {
 186   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 187
 188   s = format (s, "%U thread-index %llu session-index %llu", format_snat_key,
 189               v->key, nat_value_get_thread_index (v),
 190               nat_value_get_session_index (v));
 191
 192   return s;
 193 }
 194
 195 u8 *
 196 format_static_mapping_kvp (u8 * s, va_list * args)
 197 {
 198   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 199
 200   s = format (s, "%U static-mapping-index %llu",
 201               format_snat_key, v->key, v->value);
 202
 203   return s;
 204 }
 205
 206 u8 *
 207 format_ed_session_kvp (u8 * s, va_list * args)
 208 {
 209   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 210
 211   u8 proto;
 212   u16 r_port, l_port;
 213   ip4_address_t l_addr, r_addr;
 214   u32 fib_index;
 215
 216   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 217   s = format (s,
 218               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 219               "session-index %u",
 220               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 221               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 222               format_ip_protocol, proto, fib_index,
 223               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 224
 225   return s;
 226 }
 227
 228 void
 229 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 230                        u8 is_ha)
 231 {
 232       per_vrf_sessions_unregister_session (s, thread_index);
 233
 234       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 235         nat_elog_warn (sm, "flow hash del failed");
 236
 237       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 238         nat_elog_warn (sm, "flow hash del failed");
 239
 240   if (is_fwd_bypass_session (s))
 241     {
 242       return;
 243     }
 244
 245       if (is_affinity_sessions (s))
 246         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 247                              s->nat_proto, s->out2in.port);
 248
 249       if (!is_ha)
 250         nat_syslog_nat44_sdel (
 251           0, s->in2out.fib_index, &s->in2out.addr, s->in2out.port,
 252           &s->ext_host_nat_addr, s->ext_host_nat_port, &s->out2in.addr,
 253           s->out2in.port, &s->ext_host_addr, s->ext_host_port, s->nat_proto,
 254           is_twice_nat_session (s));
 255
 256   if (snat_is_unk_proto_session (s))
 257     return;
 258
 259   if (!is_ha)
 260     {
 261       /* log NAT event */
 262       nat_ipfix_logging_nat44_ses_delete (thread_index,
 263                                           s->in2out.addr.as_u32,
 264                                           s->out2in.addr.as_u32,
 265                                           s->nat_proto,
 266                                           s->in2out.port,
 267                                           s->out2in.port,
 268                                           s->in2out.fib_index);
 269     }
 270
 271   /* Twice NAT address and port for external host */
 272   if (is_twice_nat_session (s))
 273     {
 274       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 275                                           thread_index,
 276                                           &s->ext_host_nat_addr,
 277                                           s->ext_host_nat_port, s->nat_proto);
 278     }
 279
 280   if (snat_is_session_static (s))
 281     return;
 282
 283   snat_free_outside_address_and_port (sm->addresses, thread_index,
 284                                       &s->out2in.addr, s->out2in.port,
 285                                       s->nat_proto);
 286 }
 287
 288 void
 289 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 290                           int is_add)
 291 {
 292   snat_main_t *sm = &snat_main;
 293   fib_prefix_t prefix = {
 294     .fp_len = p_len,
 295     .fp_proto = FIB_PROTOCOL_IP4,
 296     .fp_addr = {
 297                 .ip4.as_u32 = addr->as_u32,
 298                 },
 299   };
 300   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 301
 302   if (is_add)
 303     fib_table_entry_update_one_path (fib_index,
 304                                      &prefix,
 305                                      sm->fib_src_low,
 306                                      (FIB_ENTRY_FLAG_CONNECTED |
 307                                       FIB_ENTRY_FLAG_LOCAL |
 308                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 309                                      DPO_PROTO_IP4,
 310                                      NULL,
 311                                      sw_if_index,
 312                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 313   else
 314     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 315 }
 316
 317 int
 318 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 319                   u8 twice_nat)
 320 {
 321   snat_address_t *ap;
 322   snat_interface_t *i;
 323   vlib_thread_main_t *tm = vlib_get_thread_main ();
 324
 325   /* Check if address already exists */
 326   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 327     {
 328       if (ap->addr.as_u32 == addr->as_u32)
 329         {
 330           nat_log_err ("address exist");
 331           return VNET_API_ERROR_VALUE_EXIST;
 332         }
 333     }
 334
 335   if (twice_nat)
 336     vec_add2 (sm->twice_nat_addresses, ap, 1);
 337   else
 338     vec_add2 (sm->addresses, ap, 1);
 339
 340   ap->addr = *addr;
 341   if (vrf_id != ~0)
 342     ap->fib_index =
 343       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 344                                          sm->fib_src_low);
 345   else
 346     ap->fib_index = ~0;
 347
 348   #define _(N, i, n, s) \
 349     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 350     ap->busy_##n##_ports = 0; \
 351     ap->busy_##n##_ports_per_thread = 0;\
 352     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 353     foreach_nat_protocol
 354   #undef _
 355
 356   if (twice_nat)
 357     return 0;
 358
 359   /* Add external address to FIB */
 360   pool_foreach (i, sm->interfaces)
 361    {
 362      if (nat_interface_is_inside (i))
 363        continue;
 364
 365      snat_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
 366      break;
 367   }
 368   pool_foreach (i, sm->output_feature_interfaces)
 369    {
 370      if (nat_interface_is_inside (i))
 371        continue;
 372
 373      snat_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
 374      break;
 375   }
 376
 377   return 0;
 378 }
 379
 380 static int
 381 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 382 {
 383   snat_static_mapping_t *m;
 384   pool_foreach (m, sm->static_mappings)
 385    {
 386       if (is_addr_only_static_mapping (m) ||
 387           is_out2in_only_static_mapping (m) ||
 388           is_identity_static_mapping (m))
 389         continue;
 390       if (m->external_addr.as_u32 == addr.as_u32)
 391         return 1;
 392   }
 393
 394   return 0;
 395 }
 396
 397 static void
 398 snat_add_static_mapping_when_resolved (snat_main_t *sm, ip4_address_t l_addr,
 399                                        u16 l_port, u32 sw_if_index, u16 e_port,
 400                                        u32 vrf_id, nat_protocol_t proto,
 401                                        int addr_only, u8 *tag, int twice_nat,
 402                                        int out2in_only, int identity_nat,
 403                                        ip4_address_t pool_addr, int exact)
 404 {
 405   snat_static_map_resolve_t *rp;
 406
 407   vec_add2 (sm->to_resolve, rp, 1);
 408   rp->l_addr.as_u32 = l_addr.as_u32;
 409   rp->l_port = l_port;
 410   rp->sw_if_index = sw_if_index;
 411   rp->e_port = e_port;
 412   rp->vrf_id = vrf_id;
 413   rp->proto = proto;
 414   rp->addr_only = addr_only;
 415   rp->twice_nat = twice_nat;
 416   rp->out2in_only = out2in_only;
 417   rp->identity_nat = identity_nat;
 418   rp->tag = vec_dup (tag);
 419   rp->pool_addr = pool_addr;
 420   rp->exact = exact;
 421 }
 422
 423 u32
 424 get_thread_idx_by_port (u16 e_port)
 425 {
 426   snat_main_t *sm = &snat_main;
 427   u32 thread_idx = sm->num_workers;
 428   if (sm->num_workers > 1)
 429     {
 430       thread_idx =
 431         sm->first_worker_index +
 432         sm->workers[(e_port - 1024) / sm->port_per_thread];
 433     }
 434   return thread_idx;
 435 }
 436
 437 void
 438 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 439                                     snat_main_per_thread_data_t * tsm,
 440                                     ip4_address_t l_addr,
 441                                     u16 l_port,
 442                                     u8 protocol,
 443                                     u32 fib_index, int addr_only,
 444                                     ip4_address_t e_addr, u16 e_port)
 445 {
 446   snat_session_t *s;
 447   u32 *indexes_to_free = NULL;
 448   pool_foreach (s, tsm->sessions) {
 449     if (s->in2out.fib_index != fib_index ||
 450         s->in2out.addr.as_u32 != l_addr.as_u32)
 451       {
 452         continue;
 453       }
 454     if (!addr_only)
 455       {
 456         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 457             s->out2in.port != e_port ||
 458             s->in2out.port != l_port ||
 459             s->nat_proto != protocol)
 460           continue;
 461       }
 462
 463     if (is_lb_session (s))
 464       continue;
 465     if (!snat_is_session_static (s))
 466       continue;
 467     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 468     vec_add1 (indexes_to_free, s - tsm->sessions);
 469     if (!addr_only)
 470       break;
 471   }
 472   u32 *ses_index;
 473   vec_foreach (ses_index, indexes_to_free)
 474   {
 475     s = pool_elt_at_index (tsm->sessions, *ses_index);
 476     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 477   }
 478   vec_free (indexes_to_free);
 479 }
 480
 481 int
 482 snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 483                          u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
 484                          u32 sw_if_index, nat_protocol_t proto, int is_add,
 485                          twice_nat_type_t twice_nat, u8 out2in_only, u8 *tag,
 486                          u8 identity_nat, ip4_address_t pool_addr, int exact)
 487 {
 488   snat_main_t *sm = &snat_main;
 489   snat_static_mapping_t *m;
 490   clib_bihash_kv_8_8_t kv, value;
 491   snat_address_t *a = 0;
 492   u32 fib_index = ~0;
 493   snat_interface_t *interface;
 494   snat_main_per_thread_data_t *tsm;
 495   snat_static_map_resolve_t *rp, *rp_match = 0;
 496   nat44_lb_addr_port_t *local;
 497   u32 find = ~0;
 498   int i;
 499
 500   /* If the external address is a specific interface address */
 501   if (sw_if_index != ~0)
 502     {
 503       ip4_address_t *first_int_addr;
 504
 505       for (i = 0; i < vec_len (sm->to_resolve); i++)
 506         {
 507           rp = sm->to_resolve + i;
 508           if (rp->sw_if_index != sw_if_index ||
 509               rp->l_addr.as_u32 != l_addr.as_u32 ||
 510               rp->vrf_id != vrf_id || rp->addr_only != addr_only)
 511             continue;
 512
 513           if (!addr_only)
 514             {
 515               if ((rp->l_port != l_port && rp->e_port != e_port)
 516                   || rp->proto != proto)
 517                 continue;
 518             }
 519
 520           rp_match = rp;
 521           break;
 522         }
 523
 524       /* Might be already set... */
 525       first_int_addr = ip4_interface_first_address
 526         (sm->ip4_main, sw_if_index, 0 /* just want the address */ );
 527
 528       if (is_add)
 529         {
 530           if (rp_match)
 531             return VNET_API_ERROR_VALUE_EXIST;
 532
 533           snat_add_static_mapping_when_resolved (
 534             sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto, addr_only,
 535             tag, twice_nat, out2in_only, identity_nat, pool_addr, exact);
 536
 537           /* DHCP resolution required? */
 538           if (first_int_addr == 0)
 539             {
 540               return 0;
 541             }
 542           else
 543             {
 544               e_addr.as_u32 = first_int_addr->as_u32;
 545               /* Identity mapping? */
 546               if (l_addr.as_u32 == 0)
 547                 l_addr.as_u32 = e_addr.as_u32;
 548             }
 549         }
 550       else
 551         {
 552           if (!rp_match)
 553             return VNET_API_ERROR_NO_SUCH_ENTRY;
 554
 555           vec_del1 (sm->to_resolve, i);
 556
 557           if (first_int_addr)
 558             {
 559               e_addr.as_u32 = first_int_addr->as_u32;
 560               /* Identity mapping? */
 561               if (l_addr.as_u32 == 0)
 562                 l_addr.as_u32 = e_addr.as_u32;
 563             }
 564           else
 565             return 0;
 566         }
 567     }
 568
 569   init_nat_k (&kv, e_addr, addr_only ? 0 : e_port, 0, addr_only ? 0 : proto);
 570   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 571     m = 0;
 572   else
 573     m = pool_elt_at_index (sm->static_mappings, value.value);
 574
 575   if (is_add)
 576     {
 577       if (m)
 578         {
 579           if (is_identity_static_mapping (m))
 580             {
 581               pool_foreach (local, m->locals)
 582                {
 583                 if (local->vrf_id == vrf_id)
 584                   return VNET_API_ERROR_VALUE_EXIST;
 585               }
 586               pool_get (m->locals, local);
 587               local->vrf_id = vrf_id;
 588               local->fib_index =
 589                 fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 590                                                    sm->fib_src_low);
 591               init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
 592                            m->proto, 0, m - sm->static_mappings);
 593               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 594               return 0;
 595             }
 596           else
 597             return VNET_API_ERROR_VALUE_EXIST;
 598         }
 599
 600       if (twice_nat && addr_only)
 601         return VNET_API_ERROR_UNSUPPORTED;
 602
 603       /* Convert VRF id to FIB index */
 604       if (vrf_id != ~0)
 605         fib_index =
 606           fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 607                                              sm->fib_src_low);
 608       /* If not specified use inside VRF id from SNAT plugin startup config */
 609       else
 610         {
 611           fib_index = sm->inside_fib_index;
 612           vrf_id = sm->inside_vrf_id;
 613           fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 614         }
 615
 616       if (!(out2in_only || identity_nat))
 617         {
 618           init_nat_k (&kv, l_addr, addr_only ? 0 : l_port, fib_index,
 619                       addr_only ? 0 : proto);
 620           if (!clib_bihash_search_8_8
 621               (&sm->static_mapping_by_local, &kv, &value))
 622             return VNET_API_ERROR_VALUE_EXIST;
 623         }
 624
 625       /* Find external address in allocated addresses and reserve port for
 626          address and port pair mapping when dynamic translations enabled */
 627       if (!(addr_only || sm->static_mapping_only || out2in_only))
 628         {
 629           for (i = 0; i < vec_len (sm->addresses); i++)
 630             {
 631               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 632                 {
 633                   a = sm->addresses + i;
 634                   /* External port must be unused */
 635                   switch (proto)
 636                     {
 637 #define _(N, j, n, s) \
 638                     case NAT_PROTOCOL_##N: \
 639                       if (a->busy_##n##_port_refcounts[e_port]) \
 640                         return VNET_API_ERROR_INVALID_VALUE; \
 641                       ++a->busy_##n##_port_refcounts[e_port]; \
 642                       if (e_port > 1024) \
 643                         { \
 644                           a->busy_##n##_ports++; \
 645                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 646                         } \
 647                       break;
 648                       foreach_nat_protocol
 649 #undef _
 650                         default : nat_elog_info (sm, "unknown protocol");
 651                       return VNET_API_ERROR_INVALID_VALUE_2;
 652                     }
 653                   break;
 654                 }
 655             }
 656           /* External address must be allocated */
 657           if (!a && (l_addr.as_u32 != e_addr.as_u32))
 658             {
 659               if (sw_if_index != ~0)
 660                 {
 661                   for (i = 0; i < vec_len (sm->to_resolve); i++)
 662                     {
 663                       rp = sm->to_resolve + i;
 664                       if (rp->addr_only)
 665                         continue;
 666                       if (rp->sw_if_index != sw_if_index &&
 667                           rp->l_addr.as_u32 != l_addr.as_u32 &&
 668                           rp->vrf_id != vrf_id && rp->l_port != l_port &&
 669                           rp->e_port != e_port && rp->proto != proto)
 670                         continue;
 671
 672                       vec_del1 (sm->to_resolve, i);
 673                       break;
 674                     }
 675                 }
 676               return VNET_API_ERROR_NO_SUCH_ENTRY;
 677             }
 678         }
 679
 680       pool_get (sm->static_mappings, m);
 681       clib_memset (m, 0, sizeof (*m));
 682       m->tag = vec_dup (tag);
 683       m->local_addr = l_addr;
 684       m->external_addr = e_addr;
 685       m->twice_nat = twice_nat;
 686
 687       if (twice_nat == TWICE_NAT && exact)
 688         {
 689           m->flags |= NAT_STATIC_MAPPING_FLAG_EXACT_ADDRESS;
 690           m->pool_addr = pool_addr;
 691         }
 692
 693       if (out2in_only)
 694         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 695       if (addr_only)
 696         m->flags |= NAT_STATIC_MAPPING_FLAG_ADDR_ONLY;
 697       if (identity_nat)
 698         {
 699           m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
 700           pool_get (m->locals, local);
 701           local->vrf_id = vrf_id;
 702           local->fib_index = fib_index;
 703         }
 704       else
 705         {
 706           m->vrf_id = vrf_id;
 707           m->fib_index = fib_index;
 708         }
 709       if (!addr_only)
 710         {
 711           m->local_port = l_port;
 712           m->external_port = e_port;
 713           m->proto = proto;
 714         }
 715
 716       if (sm->num_workers > 1)
 717         {
 718           ip4_header_t ip = {
 719             .src_address = m->local_addr,
 720           };
 721           vec_add1 (m->workers, nat44_ed_get_in2out_worker_index (
 722                                   0, &ip, m->fib_index, 0));
 723           tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 724         }
 725       else
 726         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 727
 728       if (!out2in_only)
 729         {
 730           init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto,
 731                        0, m - sm->static_mappings);
 732           clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 733         }
 734
 735       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
 736                    m - sm->static_mappings);
 737       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 738     }
 739   else
 740     {
 741       if (!m)
 742         {
 743           if (sw_if_index != ~0)
 744             return 0;
 745           else
 746             return VNET_API_ERROR_NO_SUCH_ENTRY;
 747         }
 748
 749       if (identity_nat)
 750         {
 751           if (vrf_id == ~0)
 752             vrf_id = sm->inside_vrf_id;
 753
 754           pool_foreach (local, m->locals)
 755            {
 756             if (local->vrf_id == vrf_id)
 757               find = local - m->locals;
 758           }
 759           if (find == ~0)
 760             return VNET_API_ERROR_NO_SUCH_ENTRY;
 761
 762           local = pool_elt_at_index (m->locals, find);
 763           fib_index = local->fib_index;
 764           pool_put (m->locals, local);
 765         }
 766       else
 767         fib_index = m->fib_index;
 768
 769       /* Free external address port */
 770       if (!(addr_only || sm->static_mapping_only || out2in_only))
 771         {
 772           for (i = 0; i < vec_len (sm->addresses); i++)
 773             {
 774               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 775                 {
 776                   a = sm->addresses + i;
 777                   switch (proto)
 778                     {
 779 #define _(N, j, n, s) \
 780                     case NAT_PROTOCOL_##N: \
 781                       --a->busy_##n##_port_refcounts[e_port]; \
 782                       if (e_port > 1024) \
 783                         { \
 784                           a->busy_##n##_ports--; \
 785                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
 786                         } \
 787                       break;
 788                       foreach_nat_protocol
 789 #undef _
 790                         default : nat_elog_info (sm, "unknown protocol");
 791                       return VNET_API_ERROR_INVALID_VALUE_2;
 792                     }
 793                   break;
 794                 }
 795             }
 796         }
 797
 798       if (sm->num_workers > 1)
 799         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 800       else
 801         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 802
 803       init_nat_k (&kv, m->local_addr, m->local_port, fib_index, m->proto);
 804       if (!out2in_only)
 805         clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
 806
 807       /* Delete session(s) for static mapping if exist */
 808       if (!(sm->static_mapping_only) ||
 809           (sm->static_mapping_only && sm->static_mapping_connection_tracking))
 810         {
 811           nat_ed_static_mapping_del_sessions (
 812             sm, tsm, m->local_addr, m->local_port, m->proto, fib_index,
 813             addr_only, e_addr, e_port);
 814         }
 815
 816       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 817       if (pool_elts (m->locals))
 818         return 0;
 819
 820       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
 821       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
 822
 823       vec_free (m->tag);
 824       vec_free (m->workers);
 825       /* Delete static mapping from pool */
 826       pool_put (sm->static_mappings, m);
 827     }
 828
 829   if (!addr_only || (l_addr.as_u32 == e_addr.as_u32))
 830     return 0;
 831
 832   /* Add/delete external address to FIB */
 833   pool_foreach (interface, sm->interfaces)
 834    {
 835      if (nat_interface_is_inside (interface))
 836        continue;
 837
 838      snat_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index, is_add);
 839      break;
 840   }
 841   pool_foreach (interface, sm->output_feature_interfaces)
 842    {
 843      if (nat_interface_is_inside (interface))
 844        continue;
 845
 846      snat_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index, is_add);
 847      break;
 848   }
 849
 850   return 0;
 851 }
 852
 853 int
 854 nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
 855                                  nat_protocol_t proto,
 856                                  nat44_lb_addr_port_t * locals, u8 is_add,
 857                                  twice_nat_type_t twice_nat, u8 out2in_only,
 858                                  u8 * tag, u32 affinity)
 859 {
 860   snat_main_t *sm = &snat_main;
 861   snat_static_mapping_t *m;
 862   clib_bihash_kv_8_8_t kv, value;
 863   snat_address_t *a = 0;
 864   int i;
 865   nat44_lb_addr_port_t *local;
 866   snat_main_per_thread_data_t *tsm;
 867   snat_session_t *s;
 868   uword *bitmap = 0;
 869
 870   init_nat_k (&kv, e_addr, e_port, 0, proto);
 871   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 872     m = 0;
 873   else
 874     m = pool_elt_at_index (sm->static_mappings, value.value);
 875
 876   if (is_add)
 877     {
 878       if (m)
 879         return VNET_API_ERROR_VALUE_EXIST;
 880
 881       if (vec_len (locals) < 2)
 882         return VNET_API_ERROR_INVALID_VALUE;
 883
 884       /* Find external address in allocated addresses and reserve port for
 885          address and port pair mapping when dynamic translations enabled */
 886       if (!(sm->static_mapping_only || out2in_only))
 887         {
 888           for (i = 0; i < vec_len (sm->addresses); i++)
 889             {
 890               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 891                 {
 892                   a = sm->addresses + i;
 893                   /* External port must be unused */
 894                   switch (proto)
 895                     {
 896 #define _(N, j, n, s) \
 897                     case NAT_PROTOCOL_##N: \
 898                       if (a->busy_##n##_port_refcounts[e_port]) \
 899                         return VNET_API_ERROR_INVALID_VALUE; \
 900                       ++a->busy_##n##_port_refcounts[e_port]; \
 901                       if (e_port > 1024) \
 902                         { \
 903                           a->busy_##n##_ports++; \
 904                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]++; \
 905                         } \
 906                       break;
 907                       foreach_nat_protocol
 908 #undef _
 909                         default : nat_elog_info (sm, "unknown protocol");
 910                       return VNET_API_ERROR_INVALID_VALUE_2;
 911                     }
 912                   break;
 913                 }
 914             }
 915           /* External address must be allocated */
 916           if (!a)
 917             return VNET_API_ERROR_NO_SUCH_ENTRY;
 918         }
 919
 920       pool_get (sm->static_mappings, m);
 921       clib_memset (m, 0, sizeof (*m));
 922       m->tag = vec_dup (tag);
 923       m->external_addr = e_addr;
 924       m->external_port = e_port;
 925       m->proto = proto;
 926       m->twice_nat = twice_nat;
 927       m->flags |= NAT_STATIC_MAPPING_FLAG_LB;
 928       if (out2in_only)
 929         m->flags |= NAT_STATIC_MAPPING_FLAG_OUT2IN_ONLY;
 930       m->affinity = affinity;
 931
 932       if (affinity)
 933         m->affinity_per_service_list_head_index =
 934           nat_affinity_get_per_service_list_head_index ();
 935       else
 936         m->affinity_per_service_list_head_index = ~0;
 937
 938       init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
 939                    m - sm->static_mappings);
 940       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
 941         {
 942           nat_elog_err (sm, "static_mapping_by_external key add failed");
 943           return VNET_API_ERROR_UNSPECIFIED;
 944         }
 945
 946       for (i = 0; i < vec_len (locals); i++)
 947         {
 948           locals[i].fib_index =
 949             fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
 950                                                locals[i].vrf_id,
 951                                                sm->fib_src_low);
 952           if (!out2in_only)
 953             {
 954               init_nat_kv (&kv, locals[i].addr, locals[i].port,
 955                            locals[i].fib_index, m->proto, 0,
 956                            m - sm->static_mappings);
 957               clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 958             }
 959           locals[i].prefix = (i == 0) ? locals[i].probability :
 960             (locals[i - 1].prefix + locals[i].probability);
 961           pool_get (m->locals, local);
 962           *local = locals[i];
 963           if (sm->num_workers > 1)
 964             {
 965               ip4_header_t ip = {
 966                 .src_address = locals[i].addr,
 967               };
 968               bitmap = clib_bitmap_set (
 969                 bitmap,
 970                 nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0), 1);
 971             }
 972         }
 973
 974       /* Assign workers */
 975       if (sm->num_workers > 1)
 976         {
 977           clib_bitmap_foreach (i, bitmap)
 978              {
 979                vec_add1(m->workers, i);
 980             }
 981         }
 982     }
 983   else
 984     {
 985       if (!m)
 986         return VNET_API_ERROR_NO_SUCH_ENTRY;
 987
 988       if (!is_lb_static_mapping (m))
 989         return VNET_API_ERROR_INVALID_VALUE;
 990
 991       /* Free external address port */
 992       if (!(sm->static_mapping_only || out2in_only))
 993         {
 994           for (i = 0; i < vec_len (sm->addresses); i++)
 995             {
 996               if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
 997                 {
 998                   a = sm->addresses + i;
 999                   switch (proto)
1000                     {
1001 #define _(N, j, n, s) \
1002                     case NAT_PROTOCOL_##N: \
1003                       --a->busy_##n##_port_refcounts[e_port]; \
1004                       if (e_port > 1024) \
1005                         { \
1006                           a->busy_##n##_ports--; \
1007                           a->busy_##n##_ports_per_thread[get_thread_idx_by_port(e_port)]--; \
1008                         } \
1009                       break;
1010                       foreach_nat_protocol
1011 #undef _
1012                         default : nat_elog_info (sm, "unknown protocol");
1013                       return VNET_API_ERROR_INVALID_VALUE_2;
1014                     }
1015                   break;
1016                 }
1017             }
1018         }
1019
1020       init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1021       if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1022         {
1023           nat_elog_err (sm, "static_mapping_by_external key del failed");
1024           return VNET_API_ERROR_UNSPECIFIED;
1025         }
1026
1027       pool_foreach (local, m->locals)
1028       {
1029           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
1030                             sm->fib_src_low);
1031           if (!out2in_only)
1032             {
1033               init_nat_k (&kv, local->addr, local->port, local->fib_index,
1034                           m->proto);
1035               if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv,
1036                                            0))
1037                 {
1038                   nat_elog_err (sm, "static_mapping_by_local key del failed");
1039                   return VNET_API_ERROR_UNSPECIFIED;
1040                 }
1041             }
1042
1043           if (sm->num_workers > 1)
1044             {
1045               ip4_header_t ip = {
1046                 .src_address = local->addr,
1047               };
1048               tsm = vec_elt_at_index (
1049                 sm->per_thread_data,
1050                 nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1051             }
1052           else
1053             tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1054
1055           /* Delete sessions */
1056           pool_foreach (s, tsm->sessions)
1057             {
1058               if (!(is_lb_session (s)))
1059                 continue;
1060
1061               if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1062                   s->in2out.port != local->port)
1063                 continue;
1064
1065               nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1066               nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1067             }
1068       }
1069       if (m->affinity)
1070         nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1071       pool_free (m->locals);
1072       vec_free (m->tag);
1073       vec_free (m->workers);
1074
1075       pool_put (sm->static_mappings, m);
1076     }
1077
1078   return 0;
1079 }
1080
1081 int
1082 nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
1083                                        ip4_address_t l_addr, u16 l_port,
1084                                        nat_protocol_t proto, u32 vrf_id,
1085                                        u8 probability, u8 is_add)
1086 {
1087   snat_main_t *sm = &snat_main;
1088   snat_static_mapping_t *m = 0;
1089   clib_bihash_kv_8_8_t kv, value;
1090   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1091   snat_main_per_thread_data_t *tsm;
1092   snat_session_t *s;
1093   u32 *locals = 0;
1094   uword *bitmap = 0;
1095   int i;
1096
1097   init_nat_k (&kv, e_addr, e_port, 0, proto);
1098   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1099     m = pool_elt_at_index (sm->static_mappings, value.value);
1100
1101   if (!m)
1102     return VNET_API_ERROR_NO_SUCH_ENTRY;
1103
1104   if (!is_lb_static_mapping (m))
1105     return VNET_API_ERROR_INVALID_VALUE;
1106
1107   pool_foreach (local, m->locals)
1108    {
1109     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1110         (local->vrf_id == vrf_id))
1111       {
1112         match_local = local;
1113         break;
1114       }
1115   }
1116
1117   if (is_add)
1118     {
1119       if (match_local)
1120         return VNET_API_ERROR_VALUE_EXIST;
1121
1122       pool_get (m->locals, local);
1123       clib_memset (local, 0, sizeof (*local));
1124       local->addr.as_u32 = l_addr.as_u32;
1125       local->port = l_port;
1126       local->probability = probability;
1127       local->vrf_id = vrf_id;
1128       local->fib_index =
1129         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1130                                            sm->fib_src_low);
1131
1132       if (!is_out2in_only_static_mapping (m))
1133         {
1134           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto, 0,
1135                        m - sm->static_mappings);
1136           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1137             nat_elog_err (sm, "static_mapping_by_local key add failed");
1138         }
1139     }
1140   else
1141     {
1142       if (!match_local)
1143         return VNET_API_ERROR_NO_SUCH_ENTRY;
1144
1145       if (pool_elts (m->locals) < 3)
1146         return VNET_API_ERROR_UNSPECIFIED;
1147
1148       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1149                         sm->fib_src_low);
1150
1151       if (!is_out2in_only_static_mapping (m))
1152         {
1153           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1154           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1155             nat_elog_err (sm, "static_mapping_by_local key del failed");
1156         }
1157
1158       if (sm->num_workers > 1)
1159         {
1160           ip4_header_t ip = {
1161             .src_address = local->addr,
1162           };
1163           tsm = vec_elt_at_index (
1164             sm->per_thread_data,
1165             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1166         }
1167       else
1168         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1169
1170       /* Delete sessions */
1171       pool_foreach (s, tsm->sessions) {
1172         if (!(is_lb_session (s)))
1173           continue;
1174
1175         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1176             s->in2out.port != match_local->port)
1177           continue;
1178
1179         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1180         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1181       }
1182
1183       pool_put (m->locals, match_local);
1184     }
1185
1186   vec_free (m->workers);
1187
1188   pool_foreach (local, m->locals)
1189    {
1190     vec_add1 (locals, local - m->locals);
1191     if (sm->num_workers > 1)
1192       {
1193         ip4_header_t ip;
1194         ip.src_address.as_u32 = local->addr.as_u32,
1195         bitmap = clib_bitmap_set (
1196           bitmap,
1197           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1198       }
1199   }
1200
1201   ASSERT (vec_len (locals) > 1);
1202
1203   local = pool_elt_at_index (m->locals, locals[0]);
1204   local->prefix = local->probability;
1205   for (i = 1; i < vec_len (locals); i++)
1206     {
1207       local = pool_elt_at_index (m->locals, locals[i]);
1208       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1209       local->prefix = local->probability + prev_local->prefix;
1210     }
1211
1212   /* Assign workers */
1213   if (sm->num_workers > 1)
1214     {
1215       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1216     }
1217
1218   return 0;
1219 }
1220
1221 int
1222 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1223                   u8 twice_nat)
1224 {
1225   snat_address_t *a = 0;
1226   snat_session_t *ses;
1227   u32 *ses_to_be_removed = 0, *ses_index;
1228   snat_main_per_thread_data_t *tsm;
1229   snat_static_mapping_t *m;
1230   snat_interface_t *interface;
1231   int i;
1232   snat_address_t *addresses =
1233     twice_nat ? sm->twice_nat_addresses : sm->addresses;
1234
1235   /* Find SNAT address */
1236   for (i = 0; i < vec_len (addresses); i++)
1237     {
1238       if (addresses[i].addr.as_u32 == addr.as_u32)
1239         {
1240           a = addresses + i;
1241           break;
1242         }
1243     }
1244   if (!a)
1245     {
1246       nat_log_err ("no such address");
1247       return VNET_API_ERROR_NO_SUCH_ENTRY;
1248     }
1249
1250   if (delete_sm)
1251     {
1252       ip4_address_t pool_addr = { 0 };
1253       pool_foreach (m, sm->static_mappings)
1254        {
1255           if (m->external_addr.as_u32 == addr.as_u32)
1256             (void) snat_add_static_mapping (m->local_addr, m->external_addr,
1257                                             m->local_port, m->external_port,
1258                                             m->vrf_id,
1259                                             is_addr_only_static_mapping(m), ~0,
1260                                             m->proto, 0 /* is_add */,
1261                                             m->twice_nat,
1262                                             is_out2in_only_static_mapping(m),
1263                                             m->tag,
1264                                             is_identity_static_mapping(m),
1265                                             pool_addr, 0);
1266       }
1267     }
1268   else
1269     {
1270       /* Check if address is used in some static mapping */
1271       if (is_snat_address_used_in_static_mapping (sm, addr))
1272         {
1273           nat_log_err ("address used in static mapping");
1274           return VNET_API_ERROR_UNSPECIFIED;
1275         }
1276     }
1277
1278   if (a->fib_index != ~0)
1279     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1280
1281   /* Delete sessions using address */
1282   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1283     {
1284       vec_foreach (tsm, sm->per_thread_data)
1285       {
1286         pool_foreach (ses, tsm->sessions)  {
1287           if (ses->out2in.addr.as_u32 == addr.as_u32)
1288             {
1289               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1290               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1291             }
1292         }
1293
1294             vec_foreach (ses_index, ses_to_be_removed)
1295             {
1296               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1297               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1298             }
1299
1300         vec_free (ses_to_be_removed);
1301       }
1302     }
1303
1304 #define _(N, i, n, s) \
1305   vec_free (a->busy_##n##_ports_per_thread);
1306   foreach_nat_protocol
1307 #undef _
1308
1309     if (twice_nat)
1310   {
1311     vec_del1 (sm->twice_nat_addresses, i);
1312     return 0;
1313   }
1314   else vec_del1 (sm->addresses, i);
1315
1316   /* Delete external address from FIB */
1317   pool_foreach (interface, sm->interfaces)
1318     {
1319       if (nat_interface_is_inside (interface))
1320         continue;
1321
1322       snat_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
1323       break;
1324     }
1325   pool_foreach (interface, sm->output_feature_interfaces)
1326    {
1327      if (nat_interface_is_inside (interface))
1328        continue;
1329
1330      snat_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
1331      break;
1332   }
1333
1334   return 0;
1335 }
1336
1337 void
1338 expire_per_vrf_sessions (u32 fib_index)
1339 {
1340   per_vrf_sessions_t *per_vrf_sessions;
1341   snat_main_per_thread_data_t *tsm;
1342   snat_main_t *sm = &snat_main;
1343
1344   vec_foreach (tsm, sm->per_thread_data)
1345     {
1346       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1347         {
1348           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1349               (per_vrf_sessions->tx_fib_index == fib_index))
1350             {
1351               per_vrf_sessions->expired = 1;
1352             }
1353         }
1354     }
1355 }
1356
1357 void
1358 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1359 {
1360   snat_main_t *sm = &snat_main;
1361   nat_fib_t *fib;
1362
1363   // we don't care if it is outside/inside fib
1364   // we just care about their ref_count
1365   // if it reaches 0 sessions should expire
1366   // because the fib isn't valid for NAT anymore
1367
1368   vec_foreach (fib, sm->fibs)
1369   {
1370     if (fib->fib_index == fib_index)
1371       {
1372         if (is_del)
1373           {
1374             fib->ref_count--;
1375             if (!fib->ref_count)
1376               {
1377                 vec_del1 (sm->fibs, fib - sm->fibs);
1378                 expire_per_vrf_sessions (fib_index);
1379               }
1380             return;
1381           }
1382         else
1383           fib->ref_count++;
1384       }
1385   }
1386   if (!is_del)
1387     {
1388       vec_add2 (sm->fibs, fib, 1);
1389       fib->ref_count = 1;
1390       fib->fib_index = fib_index;
1391     }
1392 }
1393
1394 int
1395 snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
1396 {
1397   snat_main_t *sm = &snat_main;
1398   snat_interface_t *i;
1399   const char *feature_name, *del_feature_name;
1400   snat_address_t *ap;
1401   snat_static_mapping_t *m;
1402   nat_outside_fib_t *outside_fib;
1403   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1404                                                        sw_if_index);
1405
1406   if (!sm->enabled)
1407     {
1408       nat_log_err ("nat44 is disabled");
1409       return VNET_API_ERROR_UNSUPPORTED;
1410     }
1411
1412   pool_foreach (i, sm->output_feature_interfaces)
1413    {
1414     if (i->sw_if_index == sw_if_index)
1415       {
1416         nat_log_err ("error interface already configured");
1417         return VNET_API_ERROR_VALUE_EXIST;
1418       }
1419   }
1420
1421   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1422     feature_name = is_inside ? "nat44-in2out-fast" : "nat44-out2in-fast";
1423   else
1424     {
1425       if (sm->num_workers > 1)
1426         feature_name =
1427           is_inside ? "nat44-in2out-worker-handoff" :
1428           "nat44-out2in-worker-handoff";
1429       else
1430         feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1431     }
1432
1433   ASSERT (sm->frame_queue_nelts > 0);
1434
1435   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
1436     sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
1437                                                       sm->frame_queue_nelts);
1438
1439   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1440     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
1441                                                       sm->frame_queue_nelts);
1442
1443   update_per_vrf_sessions_vec (fib_index, is_del);
1444
1445   if (!is_inside)
1446     {
1447       vec_foreach (outside_fib, sm->outside_fibs)
1448         {
1449           if (outside_fib->fib_index == fib_index)
1450             {
1451               if (is_del)
1452                 {
1453                   outside_fib->refcount--;
1454                   if (!outside_fib->refcount)
1455                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1456                 }
1457               else
1458                 outside_fib->refcount++;
1459               goto feature_set;
1460             }
1461         }
1462       if (!is_del)
1463         {
1464           vec_add2 (sm->outside_fibs, outside_fib, 1);
1465           outside_fib->refcount = 1;
1466           outside_fib->fib_index = fib_index;
1467         }
1468     }
1469
1470 feature_set:
1471   pool_foreach (i, sm->interfaces)
1472    {
1473     if (i->sw_if_index == sw_if_index)
1474       {
1475         if (is_del)
1476           {
1477             if (nat_interface_is_inside(i) && nat_interface_is_outside(i))
1478               {
1479                 if (is_inside)
1480                   i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1481                 else
1482                   i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1483
1484                 if (sm->num_workers > 1)
1485                   {
1486                     del_feature_name = "nat44-handoff-classify";
1487                     feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
1488                                                  "nat44-out2in-worker-handoff";
1489                   }
1490                 else
1491                   {
1492                     del_feature_name = "nat44-ed-classify";
1493                     feature_name =
1494                       !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1495                   }
1496
1497                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1498                 if (rv)
1499                   return rv;
1500                 vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1501                                              sw_if_index, 0, 0, 0);
1502                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1503                                              sw_if_index, 1, 0, 0);
1504               }
1505             else
1506               {
1507                 int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1508                 if (rv)
1509                   return rv;
1510                 vnet_feature_enable_disable ("ip4-unicast", feature_name,
1511                                              sw_if_index, 0, 0, 0);
1512                 pool_put (sm->interfaces, i);
1513               }
1514           }
1515         else
1516           {
1517             if ((nat_interface_is_inside (i) && is_inside) ||
1518                 (nat_interface_is_outside (i) && !is_inside))
1519               return 0;
1520
1521             if (sm->num_workers > 1)
1522               {
1523                 del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1524                                                 "nat44-out2in-worker-handoff";
1525                 feature_name = "nat44-handoff-classify";
1526               }
1527             else
1528               {
1529                 del_feature_name =
1530                   !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1531
1532                 feature_name = "nat44-ed-classify";
1533               }
1534
1535             int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1536             if (rv)
1537               return rv;
1538             vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1539                                          sw_if_index, 0, 0, 0);
1540             vnet_feature_enable_disable ("ip4-unicast", feature_name,
1541                                          sw_if_index, 1, 0, 0);
1542             goto set_flags;
1543           }
1544
1545         goto fib;
1546       }
1547   }
1548
1549   if (is_del)
1550     {
1551       nat_log_err ("error interface couldn't be found");
1552       return VNET_API_ERROR_NO_SUCH_ENTRY;
1553     }
1554
1555   pool_get (sm->interfaces, i);
1556   i->sw_if_index = sw_if_index;
1557   i->flags = 0;
1558   nat_validate_interface_counters (sm, sw_if_index);
1559
1560   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1, 0,
1561                                0);
1562
1563   int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1564   if (rv)
1565     return rv;
1566
1567 set_flags:
1568   if (is_inside)
1569     {
1570       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1571       return 0;
1572     }
1573   else
1574     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1575
1576   /* Add/delete external addresses to FIB */
1577 fib:
1578   vec_foreach (ap, sm->addresses)
1579     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1580
1581   pool_foreach (m, sm->static_mappings)
1582    {
1583     if (!(is_addr_only_static_mapping(m)) || (m->local_addr.as_u32 == m->external_addr.as_u32))
1584       continue;
1585
1586     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1587   }
1588
1589   return 0;
1590 }
1591
1592 int
1593 snat_interface_add_del_output_feature (u32 sw_if_index,
1594                                        u8 is_inside, int is_del)
1595 {
1596   snat_main_t *sm = &snat_main;
1597   snat_interface_t *i;
1598   snat_address_t *ap;
1599   snat_static_mapping_t *m;
1600   nat_outside_fib_t *outside_fib;
1601   u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1602                                                        sw_if_index);
1603
1604   if (!sm->enabled)
1605     {
1606       nat_log_err ("nat44 is disabled");
1607       return VNET_API_ERROR_UNSUPPORTED;
1608     }
1609
1610   if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
1611     {
1612       nat_log_err ("error unsupported");
1613       return VNET_API_ERROR_UNSUPPORTED;
1614     }
1615
1616   pool_foreach (i, sm->interfaces)
1617    {
1618     if (i->sw_if_index == sw_if_index)
1619       {
1620         nat_log_err ("error interface already configured");
1621         return VNET_API_ERROR_VALUE_EXIST;
1622       }
1623   }
1624
1625   update_per_vrf_sessions_vec (fib_index, is_del);
1626
1627   if (!is_inside)
1628     {
1629       vec_foreach (outside_fib, sm->outside_fibs)
1630         {
1631           if (outside_fib->fib_index == fib_index)
1632             {
1633               if (is_del)
1634                 {
1635                   outside_fib->refcount--;
1636                   if (!outside_fib->refcount)
1637                     vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1638                 }
1639               else
1640                 outside_fib->refcount++;
1641               goto feature_set;
1642             }
1643         }
1644       if (!is_del)
1645         {
1646           vec_add2 (sm->outside_fibs, outside_fib, 1);
1647           outside_fib->refcount = 1;
1648           outside_fib->fib_index = fib_index;
1649         }
1650     }
1651
1652 feature_set:
1653   if (is_inside)
1654     {
1655           int rv =
1656             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
1657           if (rv)
1658             return rv;
1659           rv =
1660             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
1661                                                             !is_del);
1662           if (rv)
1663             return rv;
1664       goto fq;
1665     }
1666
1667   if (sm->num_workers > 1)
1668     {
1669       int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
1670       if (rv)
1671         return rv;
1672       rv =
1673         ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, !is_del);
1674       if (rv)
1675         return rv;
1676       vnet_feature_enable_disable ("ip4-unicast",
1677                                    "nat44-out2in-worker-handoff",
1678                                    sw_if_index, !is_del, 0, 0);
1679       vnet_feature_enable_disable ("ip4-output",
1680                                    "nat44-in2out-output-worker-handoff",
1681                                    sw_if_index, !is_del, 0, 0);
1682     }
1683   else
1684     {
1685           int rv =
1686             ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, !is_del);
1687           if (rv)
1688             return rv;
1689           rv =
1690             ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index,
1691                                                             !is_del);
1692           if (rv)
1693             return rv;
1694           vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1695                                        sw_if_index, !is_del, 0, 0);
1696           vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1697                                        sw_if_index, !is_del, 0, 0);
1698     }
1699
1700 fq:
1701   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
1702     sm->fq_in2out_output_index =
1703       vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
1704
1705   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
1706     sm->fq_out2in_index =
1707       vlib_frame_queue_main_init (sm->out2in_node_index, 0);
1708
1709   pool_foreach (i, sm->output_feature_interfaces)
1710    {
1711     if (i->sw_if_index == sw_if_index)
1712       {
1713         if (is_del)
1714           pool_put (sm->output_feature_interfaces, i);
1715         else
1716           return VNET_API_ERROR_VALUE_EXIST;
1717
1718         goto fib;
1719       }
1720   }
1721
1722   if (is_del)
1723     {
1724       nat_log_err ("error interface couldn't be found");
1725       return VNET_API_ERROR_NO_SUCH_ENTRY;
1726     }
1727
1728   pool_get (sm->output_feature_interfaces, i);
1729   i->sw_if_index = sw_if_index;
1730   i->flags = 0;
1731   nat_validate_interface_counters (sm, sw_if_index);
1732   if (is_inside)
1733     i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1734   else
1735     i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1736
1737   /* Add/delete external addresses to FIB */
1738 fib:
1739   if (is_inside)
1740     return 0;
1741
1742   vec_foreach (ap, sm->addresses)
1743     snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
1744
1745   pool_foreach (m, sm->static_mappings)
1746    {
1747     if (!((is_addr_only_static_mapping(m)))  || (m->local_addr.as_u32 == m->external_addr.as_u32))
1748       continue;
1749
1750     snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
1751   }
1752
1753   return 0;
1754 }
1755
1756 int
1757 snat_set_workers (uword * bitmap)
1758 {
1759   snat_main_t *sm = &snat_main;
1760   int i, j = 0;
1761
1762   if (sm->num_workers < 2)
1763     return VNET_API_ERROR_FEATURE_DISABLED;
1764
1765   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
1766     return VNET_API_ERROR_INVALID_WORKER;
1767
1768   vec_free (sm->workers);
1769   clib_bitmap_foreach (i, bitmap)
1770     {
1771       vec_add1(sm->workers, i);
1772       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
1773       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
1774       j++;
1775     }
1776
1777   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
1778
1779   return 0;
1780 }
1781
1782 int
1783 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
1784 {
1785   fail_if_enabled ();
1786   snat_main_t *sm = &snat_main;
1787   sm->frame_queue_nelts = frame_queue_nelts;
1788   return 0;
1789 }
1790
1791 static void
1792 snat_update_outside_fib (ip4_main_t * im, uword opaque,
1793                          u32 sw_if_index, u32 new_fib_index,
1794                          u32 old_fib_index)
1795 {
1796   snat_main_t *sm = &snat_main;
1797   nat_outside_fib_t *outside_fib;
1798   snat_interface_t *i;
1799   u8 is_add = 1;
1800   u8 match = 0;
1801
1802   if (!sm->enabled || (new_fib_index == old_fib_index)
1803       || (!vec_len (sm->outside_fibs)))
1804     {
1805       return;
1806     }
1807
1808   pool_foreach (i, sm->interfaces)
1809      {
1810       if (i->sw_if_index == sw_if_index)
1811         {
1812           if (!(nat_interface_is_outside (i)))
1813             return;
1814           match = 1;
1815         }
1816     }
1817
1818   pool_foreach (i, sm->output_feature_interfaces)
1819      {
1820       if (i->sw_if_index == sw_if_index)
1821         {
1822           if (!(nat_interface_is_outside (i)))
1823             return;
1824           match = 1;
1825         }
1826     }
1827
1828   if (!match)
1829     return;
1830
1831   vec_foreach (outside_fib, sm->outside_fibs)
1832   {
1833     if (outside_fib->fib_index == old_fib_index)
1834       {
1835         outside_fib->refcount--;
1836         if (!outside_fib->refcount)
1837           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1838         break;
1839       }
1840   }
1841
1842   vec_foreach (outside_fib, sm->outside_fibs)
1843   {
1844     if (outside_fib->fib_index == new_fib_index)
1845       {
1846         outside_fib->refcount++;
1847         is_add = 0;
1848         break;
1849       }
1850   }
1851
1852   if (is_add)
1853     {
1854       vec_add2 (sm->outside_fibs, outside_fib, 1);
1855       outside_fib->refcount = 1;
1856       outside_fib->fib_index = new_fib_index;
1857     }
1858 }
1859
1860 static void
1861 snat_update_outside_fib (ip4_main_t * im, uword opaque,
1862                          u32 sw_if_index, u32 new_fib_index,
1863                          u32 old_fib_index);
1864
1865 static void
1866 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
1867                                        uword opaque,
1868                                        u32 sw_if_index,
1869                                        ip4_address_t * address,
1870                                        u32 address_length,
1871                                        u32 if_address_index, u32 is_delete);
1872
1873 static void
1874 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
1875                                  uword opaque,
1876                                  u32 sw_if_index,
1877                                  ip4_address_t * address,
1878                                  u32 address_length,
1879                                  u32 if_address_index, u32 is_delete);
1880
1881 void
1882 test_key_calc_split ()
1883 {
1884   ip4_address_t l_addr;
1885   l_addr.as_u8[0] = 1;
1886   l_addr.as_u8[1] = 1;
1887   l_addr.as_u8[2] = 1;
1888   l_addr.as_u8[3] = 1;
1889   ip4_address_t r_addr;
1890   r_addr.as_u8[0] = 2;
1891   r_addr.as_u8[1] = 2;
1892   r_addr.as_u8[2] = 2;
1893   r_addr.as_u8[3] = 2;
1894   u16 l_port = 40001;
1895   u16 r_port = 40301;
1896   u8 proto = 9;
1897   u32 fib_index = 9000001;
1898   u32 thread_index = 3000000001;
1899   u32 session_index = 3000000221;
1900   clib_bihash_kv_16_8_t kv;
1901   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
1902               thread_index, session_index);
1903   ip4_address_t l_addr2;
1904   ip4_address_t r_addr2;
1905   clib_memset (&l_addr2, 0, sizeof (l_addr2));
1906   clib_memset (&r_addr2, 0, sizeof (r_addr2));
1907   u16 l_port2 = 0;
1908   u16 r_port2 = 0;
1909   u8 proto2 = 0;
1910   u32 fib_index2 = 0;
1911   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
1912                &r_port2);
1913   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
1914   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
1915   ASSERT (l_port == l_port2);
1916   ASSERT (r_port == r_port2);
1917   ASSERT (proto == proto2);
1918   ASSERT (fib_index == fib_index2);
1919   ASSERT (thread_index == ed_value_get_thread_index (&kv));
1920   ASSERT (session_index == ed_value_get_session_index (&kv));
1921
1922   fib_index = 7001;
1923   proto = 5;
1924   nat_protocol_t proto3 = ~0;
1925   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
1926   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
1927   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
1928   ASSERT (l_port == l_port2);
1929   ASSERT (proto == proto3);
1930   ASSERT (fib_index == fib_index2);
1931 }
1932
1933 static clib_error_t *
1934 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
1935 {
1936   u32 fib_index;
1937
1938       // TODO: consider removing all NAT interfaces
1939       if (!is_add)
1940         {
1941           fib_index = ip4_fib_index_from_table_id (table_id);
1942           if (fib_index != ~0)
1943             expire_per_vrf_sessions (fib_index);
1944         }
1945   return 0;
1946 }
1947
1948 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
1949
1950 void
1951 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
1952 {
1953   vlib_node_t *node;
1954
1955   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
1956   sm->out2in_node_index = node->index;
1957
1958   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
1959   sm->in2out_node_index = node->index;
1960
1961   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
1962   sm->in2out_output_node_index = node->index;
1963 }
1964
1965 #define nat_validate_simple_counter(c, i)                                     \
1966   do                                                                          \
1967     {                                                                         \
1968       vlib_validate_simple_counter (&c, i);                                   \
1969       vlib_zero_simple_counter (&c, i);                                       \
1970     }                                                                         \
1971   while (0);
1972
1973 #define nat_init_simple_counter(c, n, sn)                                     \
1974   do                                                                          \
1975     {                                                                         \
1976       c.name = n;                                                             \
1977       c.stat_segment_name = sn;                                               \
1978       nat_validate_simple_counter (c, 0);                                     \
1979     }                                                                         \
1980   while (0);
1981
1982 static_always_inline void
1983 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
1984 {
1985 #define _(x)                                                                  \
1986   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
1987   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
1988   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
1989   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
1990   foreach_nat_counter;
1991 #undef _
1992   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
1993 }
1994
1995 static clib_error_t *
1996 nat_init (vlib_main_t * vm)
1997 {
1998   snat_main_t *sm = &snat_main;
1999   vlib_thread_main_t *tm = vlib_get_thread_main ();
2000   vlib_thread_registration_t *tr;
2001   ip4_add_del_interface_address_callback_t cbi = { 0 };
2002   ip4_table_bind_callback_t cbt = { 0 };
2003   u32 i, num_threads = 0;
2004   uword *p, *bitmap = 0;
2005
2006   clib_memset (sm, 0, sizeof (*sm));
2007
2008   // required
2009   sm->vnet_main = vnet_get_main ();
2010   // convenience
2011   sm->ip4_main = &ip4_main;
2012   sm->api_main = vlibapi_get_main ();
2013   sm->ip4_lookup_main = &ip4_main.lookup_main;
2014
2015   // frame queue indices used for handoff
2016   sm->fq_out2in_index = ~0;
2017   sm->fq_in2out_index = ~0;
2018   sm->fq_in2out_output_index = ~0;
2019
2020   sm->log_level = NAT_LOG_ERROR;
2021
2022   nat44_set_node_indexes (sm, vm);
2023   sm->log_class = vlib_log_register_class ("nat", 0);
2024   nat_ipfix_logging_init (vm);
2025
2026   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2027                            "/nat44-ed/total-sessions");
2028
2029 #define _(x)                                                                  \
2030   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2031                            "/nat44-ed/in2out/fastpath/" #x);                  \
2032   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2033                            "/nat44-ed/out2in/fastpath/" #x);                  \
2034   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2035                            "/nat44-ed/in2out/slowpath/" #x);                  \
2036   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2037                            "/nat44-ed/out2in/slowpath/" #x);
2038   foreach_nat_counter;
2039 #undef _
2040   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2041                            "/nat44-ed/hairpinning");
2042
2043   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2044   if (p)
2045     {
2046       tr = (vlib_thread_registration_t *) p[0];
2047       if (tr)
2048         {
2049           sm->num_workers = tr->count;
2050           sm->first_worker_index = tr->first_index;
2051         }
2052     }
2053   num_threads = tm->n_vlib_mains - 1;
2054   sm->port_per_thread = 0xffff - 1024;
2055   vec_validate (sm->per_thread_data, num_threads);
2056
2057   /* Use all available workers by default */
2058   if (sm->num_workers > 1)
2059     {
2060
2061       for (i = 0; i < sm->num_workers; i++)
2062         bitmap = clib_bitmap_set (bitmap, i, 1);
2063       snat_set_workers (bitmap);
2064       clib_bitmap_free (bitmap);
2065     }
2066   else
2067     sm->per_thread_data[0].snat_thread_index = 0;
2068
2069   /* callbacks to call when interface address changes. */
2070   cbi.function = snat_ip4_add_del_interface_address_cb;
2071   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2072   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2073   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2074
2075   /* callbacks to call when interface to table biding changes */
2076   cbt.function = snat_update_outside_fib;
2077   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2078
2079   sm->fib_src_low =
2080     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2081                          FIB_SOURCE_BH_SIMPLE);
2082   sm->fib_src_hi =
2083     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2084                          FIB_SOURCE_BH_SIMPLE);
2085
2086   nat_affinity_init (vm);
2087   test_key_calc_split ();
2088
2089   return nat44_api_hookup (vm);
2090 }
2091
2092 VLIB_INIT_FUNCTION (nat_init);
2093
2094 int
2095 nat44_plugin_enable (nat44_config_t c)
2096 {
2097   snat_main_t *sm = &snat_main;
2098
2099   fail_if_enabled ();
2100
2101   // UPDATE based on these appropriate API/CLI
2102   // c.static_mapping_only + c.connection_tracking
2103   //  - supported in NAT EI & NAT ED
2104   // c.out2in_dpo, c.static_mapping_only
2105   //  - supported in NAT EI
2106
2107   if (c.static_mapping_only && !c.connection_tracking)
2108     {
2109       nat_log_err ("unsupported combination of configuration");
2110       return 1;
2111     }
2112
2113   // nat44 feature configuration
2114   sm->static_mapping_only = c.static_mapping_only;
2115   sm->static_mapping_connection_tracking = c.connection_tracking;
2116
2117   sm->forwarding_enabled = 0;
2118   sm->mss_clamping = 0;
2119   sm->pat = (!c.static_mapping_only ||
2120              (c.static_mapping_only && c.connection_tracking));
2121
2122   if (!c.sessions)
2123     c.sessions = 63 * 1024;
2124
2125   sm->max_translations_per_thread = c.sessions;
2126   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2127
2128   // ED only feature
2129   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2130
2131   sm->inside_vrf_id = c.inside_vrf;
2132   sm->inside_fib_index =
2133     fib_table_find_or_create_and_lock
2134     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2135
2136   sm->outside_vrf_id = c.outside_vrf;
2137   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2138     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2139
2140   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets);
2141
2142   nat_affinity_enable ();
2143
2144   nat_reset_timeouts (&sm->timeouts);
2145
2146   vlib_zero_simple_counter (&sm->total_sessions, 0);
2147
2148   if (!sm->frame_queue_nelts)
2149     sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2150
2151   sm->enabled = 1;
2152   sm->rconfig = c;
2153
2154   return 0;
2155 }
2156
2157 void
2158 nat44_addresses_free (snat_address_t ** addresses)
2159 {
2160   snat_address_t *ap;
2161   vec_foreach (ap, *addresses)
2162     {
2163     #define _(N, i, n, s) \
2164       vec_free (ap->busy_##n##_ports_per_thread);
2165       foreach_nat_protocol
2166     #undef _
2167     }
2168   vec_free (*addresses);
2169   *addresses = 0;
2170 }
2171
2172 int
2173 nat44_plugin_disable ()
2174 {
2175   snat_main_t *sm = &snat_main;
2176   snat_interface_t *i, *vec;
2177   int error = 0;
2178
2179   fail_if_disabled ();
2180
2181   // first unregister all nodes from interfaces
2182   vec = vec_dup (sm->interfaces);
2183   vec_foreach (i, vec)
2184     {
2185       if (nat_interface_is_inside(i))
2186         error = snat_interface_add_del (i->sw_if_index, 1, 1);
2187       if (nat_interface_is_outside(i))
2188         error = snat_interface_add_del (i->sw_if_index, 0, 1);
2189
2190       if (error)
2191         {
2192           nat_log_err ("error occurred while removing interface %u",
2193                        i->sw_if_index);
2194         }
2195     }
2196   vec_free (vec);
2197   sm->interfaces = 0;
2198
2199   vec = vec_dup (sm->output_feature_interfaces);
2200   vec_foreach (i, vec)
2201     {
2202       if (nat_interface_is_inside(i))
2203         error = snat_interface_add_del_output_feature (i->sw_if_index, 1, 1);
2204       if (nat_interface_is_outside(i))
2205         error = snat_interface_add_del_output_feature (i->sw_if_index, 0, 1);
2206
2207       if (error)
2208         {
2209           nat_log_err ("error occurred while removing interface %u",
2210                        i->sw_if_index);
2211         }
2212     }
2213   vec_free (vec);
2214   sm->output_feature_interfaces = 0;
2215
2216   vec_free (sm->max_translations_per_fib);
2217
2218   nat44_ed_db_free ();
2219
2220   nat44_addresses_free (&sm->addresses);
2221   nat44_addresses_free (&sm->twice_nat_addresses);
2222
2223   vec_free (sm->to_resolve);
2224   vec_free (sm->auto_add_sw_if_indices);
2225   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2226
2227   sm->to_resolve = 0;
2228   sm->auto_add_sw_if_indices = 0;
2229   sm->auto_add_sw_if_indices_twice_nat = 0;
2230
2231   sm->forwarding_enabled = 0;
2232
2233   sm->enabled = 0;
2234   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2235
2236   return 0;
2237 }
2238
2239 void
2240 nat44_ed_forwarding_enable_disable (u8 is_enable)
2241 {
2242   snat_main_per_thread_data_t *tsm;
2243   snat_main_t *sm = &snat_main;
2244   snat_session_t *s;
2245
2246   u32 *ses_to_be_removed = 0, *ses_index;
2247
2248   sm->forwarding_enabled = is_enable != 0;
2249
2250   if (is_enable)
2251     return;
2252
2253   vec_foreach (tsm, sm->per_thread_data)
2254     {
2255       pool_foreach (s, tsm->sessions)
2256         {
2257           if (is_fwd_bypass_session (s))
2258             {
2259               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2260             }
2261         }
2262       vec_foreach (ses_index, ses_to_be_removed)
2263         {
2264           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2265           nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2266           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2267         }
2268
2269       vec_free (ses_to_be_removed);
2270     }
2271 }
2272
2273 void
2274 snat_free_outside_address_and_port (snat_address_t *addresses,
2275                                     u32 thread_index, ip4_address_t *addr,
2276                                     u16 port, nat_protocol_t protocol)
2277 {
2278   snat_main_t *sm = &snat_main;
2279   snat_address_t *a;
2280   u32 address_index;
2281   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2282
2283   for (address_index = 0; address_index < vec_len (addresses);
2284        address_index++)
2285     {
2286       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2287         break;
2288     }
2289
2290   ASSERT (address_index < vec_len (addresses));
2291
2292   a = addresses + address_index;
2293
2294   switch (protocol)
2295     {
2296 #define _(N, i, n, s) \
2297     case NAT_PROTOCOL_##N: \
2298       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2299       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2300       a->busy_##n##_ports--; \
2301       a->busy_##n##_ports_per_thread[thread_index]--; \
2302       break;
2303       foreach_nat_protocol
2304 #undef _
2305         default : nat_elog_info (sm, "unknown protocol");
2306       return;
2307     }
2308 }
2309
2310 int
2311 nat_set_outside_address_and_port (snat_address_t *addresses, u32 thread_index,
2312                                   ip4_address_t addr, u16 port,
2313                                   nat_protocol_t protocol)
2314 {
2315   snat_main_t *sm = &snat_main;
2316   snat_address_t *a = 0;
2317   u32 address_index;
2318   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2319
2320   for (address_index = 0; address_index < vec_len (addresses);
2321        address_index++)
2322     {
2323       if (addresses[address_index].addr.as_u32 != addr.as_u32)
2324         continue;
2325
2326       a = addresses + address_index;
2327       switch (protocol)
2328         {
2329 #define _(N, j, n, s) \
2330         case NAT_PROTOCOL_##N: \
2331           if (a->busy_##n##_port_refcounts[port_host_byte_order]) \
2332             return VNET_API_ERROR_INSTANCE_IN_USE; \
2333           ++a->busy_##n##_port_refcounts[port_host_byte_order]; \
2334           a->busy_##n##_ports_per_thread[thread_index]++; \
2335           a->busy_##n##_ports++; \
2336           return 0;
2337           foreach_nat_protocol
2338 #undef _
2339             default : nat_elog_info (sm, "unknown protocol");
2340           return 1;
2341         }
2342     }
2343
2344   return VNET_API_ERROR_NO_SUCH_ENTRY;
2345 }
2346
2347 int
2348 snat_static_mapping_match (vlib_main_t *vm, snat_main_t *sm,
2349                            ip4_address_t match_addr, u16 match_port,
2350                            u32 match_fib_index, nat_protocol_t match_protocol,
2351                            ip4_address_t *mapping_addr, u16 *mapping_port,
2352                            u32 *mapping_fib_index, u8 by_external,
2353                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2354                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2355                            u8 *is_identity_nat, snat_static_mapping_t **out)
2356 {
2357   clib_bihash_kv_8_8_t kv, value;
2358   clib_bihash_8_8_t *mapping_hash;
2359   snat_static_mapping_t *m;
2360   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2361   nat44_lb_addr_port_t *local;
2362   u8 backend_index;
2363
2364   if (!by_external)
2365     {
2366       mapping_hash = &sm->static_mapping_by_local;
2367       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2368                   match_protocol);
2369       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2370         {
2371           /* Try address only mapping */
2372           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2373           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2374             return 1;
2375         }
2376     }
2377   else
2378     {
2379       mapping_hash = &sm->static_mapping_by_external;
2380       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2381       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2382         {
2383           /* Try address only mapping */
2384           init_nat_k (&kv, match_addr, 0, 0, 0);
2385           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2386             return 1;
2387         }
2388     }
2389
2390   m = pool_elt_at_index (sm->static_mappings, value.value);
2391
2392   if (by_external)
2393     {
2394       if (is_lb_static_mapping (m))
2395         {
2396           if (PREDICT_FALSE (lb != 0))
2397             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2398           if (m->affinity && !nat_affinity_find_and_lock (
2399                                vm, ext_host_addr[0], match_addr,
2400                                match_protocol, match_port, &backend_index))
2401             {
2402               local = pool_elt_at_index (m->locals, backend_index);
2403               *mapping_addr = local->addr;
2404               *mapping_port = local->port;
2405               *mapping_fib_index = local->fib_index;
2406               goto end;
2407             }
2408           // pick locals matching this worker
2409           if (PREDICT_FALSE (sm->num_workers > 1))
2410             {
2411               u32 thread_index = vlib_get_thread_index ();
2412               pool_foreach_index (i, m->locals)
2413                {
2414                 local = pool_elt_at_index (m->locals, i);
2415
2416                 ip4_header_t ip = {
2417                   .src_address = local->addr,
2418                 };
2419
2420                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2421                                                       0) == thread_index)
2422                   {
2423                     vec_add1 (tmp, i);
2424                   }
2425                }
2426               ASSERT (vec_len (tmp) != 0);
2427             }
2428           else
2429             {
2430               pool_foreach_index (i, m->locals)
2431                {
2432                 vec_add1 (tmp, i);
2433               }
2434             }
2435           hi = vec_len (tmp) - 1;
2436           local = pool_elt_at_index (m->locals, tmp[hi]);
2437           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2438           while (lo < hi)
2439             {
2440               mid = ((hi - lo) >> 1) + lo;
2441               local = pool_elt_at_index (m->locals, tmp[mid]);
2442               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2443             }
2444           local = pool_elt_at_index (m->locals, tmp[lo]);
2445           if (!(local->prefix >= rand))
2446             return 1;
2447           *mapping_addr = local->addr;
2448           *mapping_port = local->port;
2449           *mapping_fib_index = local->fib_index;
2450           if (m->affinity)
2451             {
2452               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2453                                                 match_protocol, match_port,
2454                                                 tmp[lo], m->affinity,
2455                                                 m->affinity_per_service_list_head_index))
2456                 nat_elog_info (sm, "create affinity record failed");
2457             }
2458           vec_free (tmp);
2459         }
2460       else
2461         {
2462           if (PREDICT_FALSE (lb != 0))
2463             *lb = NO_LB_NAT;
2464           *mapping_fib_index = m->fib_index;
2465           *mapping_addr = m->local_addr;
2466           /* Address only mapping doesn't change port */
2467           *mapping_port = is_addr_only_static_mapping (m) ? match_port
2468             : m->local_port;
2469         }
2470     }
2471   else
2472     {
2473       *mapping_addr = m->external_addr;
2474       /* Address only mapping doesn't change port */
2475       *mapping_port = is_addr_only_static_mapping (m) ? match_port
2476         : m->external_port;
2477       *mapping_fib_index = sm->outside_fib_index;
2478     }
2479
2480 end:
2481   if (PREDICT_FALSE (is_addr_only != 0))
2482     *is_addr_only = is_addr_only_static_mapping (m);
2483
2484   if (PREDICT_FALSE (twice_nat != 0))
2485     *twice_nat = m->twice_nat;
2486
2487   if (PREDICT_FALSE (is_identity_nat != 0))
2488     *is_identity_nat = is_identity_static_mapping (m);
2489
2490   if (out != 0)
2491     *out = m;
2492
2493   return 0;
2494 }
2495
2496 u32
2497 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2498                                   u32 rx_fib_index, u8 is_output)
2499 {
2500   snat_main_t *sm = &snat_main;
2501   u32 next_worker_index = sm->first_worker_index;
2502   u32 hash;
2503
2504   clib_bihash_kv_16_8_t kv16, value16;
2505
2506   u32 fib_index = rx_fib_index;
2507   if (b)
2508     {
2509       if (PREDICT_FALSE (is_output))
2510         {
2511           fib_index = sm->outside_fib_index;
2512           nat_outside_fib_t *outside_fib;
2513           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2514           fib_prefix_t pfx = {
2515                   .fp_proto = FIB_PROTOCOL_IP4,
2516                   .fp_len = 32,
2517                   .fp_addr = {
2518                           .ip4.as_u32 = ip->dst_address.as_u32,
2519                   } ,
2520           };
2521
2522           switch (vec_len (sm->outside_fibs))
2523             {
2524             case 0:
2525               fib_index = sm->outside_fib_index;
2526               break;
2527             case 1:
2528               fib_index = sm->outside_fibs[0].fib_index;
2529               break;
2530             default:
2531               vec_foreach (outside_fib, sm->outside_fibs)
2532                 {
2533                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2534                   if (FIB_NODE_INDEX_INVALID != fei)
2535                     {
2536                       if (fib_entry_get_resolving_interface (fei) != ~0)
2537                         {
2538                           fib_index = outside_fib->fib_index;
2539                           break;
2540                         }
2541                     }
2542                 }
2543               break;
2544             }
2545         }
2546
2547       init_ed_k (&kv16, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2548                  ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2549                  fib_index, ip->protocol);
2550
2551       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2552         {
2553           next_worker_index = ed_value_get_thread_index (&value16);
2554           vnet_buffer2 (b)->nat.cached_session_index =
2555             ed_value_get_session_index (&value16);
2556           goto out;
2557         }
2558
2559       // dst NAT
2560       init_ed_k (&kv16, ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2561                  ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2562                  rx_fib_index, ip->protocol);
2563       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2564         {
2565           next_worker_index = ed_value_get_thread_index (&value16);
2566           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
2567             ed_value_get_session_index (&value16);
2568           goto out;
2569         }
2570     }
2571
2572   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2573     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2574
2575   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2576     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2577   else
2578     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2579
2580 out:
2581   if (PREDICT_TRUE (!is_output))
2582     {
2583       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
2584                               rx_fib_index,
2585                               clib_net_to_host_u32 (ip->src_address.as_u32),
2586                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2587     }
2588   else
2589     {
2590       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
2591                               next_worker_index, rx_fib_index,
2592                               clib_net_to_host_u32 (ip->src_address.as_u32),
2593                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2594     }
2595
2596   return next_worker_index;
2597 }
2598
2599 u32
2600 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2601                                   u32 rx_fib_index, u8 is_output)
2602 {
2603   snat_main_t *sm = &snat_main;
2604   clib_bihash_kv_8_8_t kv, value;
2605   clib_bihash_kv_16_8_t kv16, value16;
2606
2607   u32 proto, next_worker_index = 0;
2608   u16 port;
2609   snat_static_mapping_t *m;
2610   u32 hash;
2611
2612   proto = ip_proto_to_nat_proto (ip->protocol);
2613
2614   if (PREDICT_FALSE (proto == NAT_PROTOCOL_ICMP))
2615     {
2616       ip4_address_t lookup_saddr, lookup_daddr;
2617       u16 lookup_sport, lookup_dport;
2618       u8 lookup_protocol;
2619       if (!nat_get_icmp_session_lookup_values (
2620             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
2621             &lookup_protocol))
2622         {
2623           init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
2624                      lookup_dport, rx_fib_index, lookup_protocol);
2625           if (PREDICT_TRUE (
2626                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
2627             {
2628               next_worker_index = ed_value_get_thread_index (&value16);
2629               nat_elog_debug_handoff (
2630                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
2631                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
2632                 clib_net_to_host_u32 (ip->dst_address.as_u32));
2633               return next_worker_index;
2634             }
2635         }
2636     }
2637
2638   init_ed_k (&kv16, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2639              ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2640              rx_fib_index, ip->protocol);
2641
2642   if (PREDICT_TRUE (
2643         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
2644     {
2645       vnet_buffer2 (b)->nat.cached_session_index =
2646         ed_value_get_session_index (&value16);
2647       next_worker_index = ed_value_get_thread_index (&value16);
2648       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
2649                               next_worker_index, rx_fib_index,
2650                               clib_net_to_host_u32 (ip->src_address.as_u32),
2651                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2652       return next_worker_index;
2653     }
2654
2655   /* first try static mappings without port */
2656   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2657     {
2658       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
2659       if (!clib_bihash_search_8_8
2660           (&sm->static_mapping_by_external, &kv, &value))
2661         {
2662           m = pool_elt_at_index (sm->static_mappings, value.value);
2663           next_worker_index = m->workers[0];
2664           goto done;
2665         }
2666     }
2667
2668   /* unknown protocol */
2669   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
2670     {
2671       /* use current thread */
2672       next_worker_index = vlib_get_thread_index ();
2673       goto done;
2674     }
2675
2676   port = vnet_buffer (b)->ip.reass.l4_dst_port;
2677
2678   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2679     {
2680       udp_header_t *udp = ip4_next_header (ip);
2681       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2682       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2683       if (!icmp_type_is_error_message
2684           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
2685         port = vnet_buffer (b)->ip.reass.l4_src_port;
2686       else
2687         {
2688           /* if error message, then it's not fragmented and we can access it */
2689           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2690           proto = ip_proto_to_nat_proto (inner_ip->protocol);
2691           void *l4_header = ip4_next_header (inner_ip);
2692           switch (proto)
2693             {
2694             case NAT_PROTOCOL_ICMP:
2695               icmp = (icmp46_header_t *) l4_header;
2696               echo = (icmp_echo_header_t *) (icmp + 1);
2697               port = echo->identifier;
2698               break;
2699             case NAT_PROTOCOL_UDP:
2700             case NAT_PROTOCOL_TCP:
2701               port = ((tcp_udp_header_t *) l4_header)->src_port;
2702               break;
2703             default:
2704               next_worker_index = vlib_get_thread_index ();
2705               goto done;
2706             }
2707         }
2708     }
2709
2710   /* try static mappings with port */
2711   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2712     {
2713       init_nat_k (&kv, ip->dst_address, port, 0, proto);
2714       if (!clib_bihash_search_8_8
2715           (&sm->static_mapping_by_external, &kv, &value))
2716         {
2717           m = pool_elt_at_index (sm->static_mappings, value.value);
2718           if (!is_lb_static_mapping (m))
2719             {
2720               next_worker_index = m->workers[0];
2721               goto done;
2722             }
2723
2724           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2725             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2726
2727           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
2728             next_worker_index =
2729               m->workers[hash & (_vec_len (m->workers) - 1)];
2730           else
2731             next_worker_index = m->workers[hash % _vec_len (m->workers)];
2732           goto done;
2733         }
2734     }
2735
2736   /* worker by outside port */
2737   next_worker_index = sm->first_worker_index;
2738   next_worker_index +=
2739     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
2740
2741 done:
2742   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
2743                           rx_fib_index,
2744                           clib_net_to_host_u32 (ip->src_address.as_u32),
2745                           clib_net_to_host_u32 (ip->dst_address.as_u32));
2746   return next_worker_index;
2747 }
2748
2749 u32
2750 nat44_get_max_session_limit ()
2751 {
2752   snat_main_t *sm = &snat_main;
2753   u32 max_limit = 0, len = 0;
2754
2755   for (; len < vec_len (sm->max_translations_per_fib); len++)
2756     {
2757       if (max_limit < sm->max_translations_per_fib[len])
2758         max_limit = sm->max_translations_per_fib[len];
2759     }
2760   return max_limit;
2761 }
2762
2763 int
2764 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
2765 {
2766   snat_main_t *sm = &snat_main;
2767   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
2768   u32 len = vec_len (sm->max_translations_per_fib);
2769
2770   if (len <= fib_index)
2771     {
2772       vec_validate (sm->max_translations_per_fib, fib_index + 1);
2773
2774       for (; len < vec_len (sm->max_translations_per_fib); len++)
2775         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
2776     }
2777
2778   sm->max_translations_per_fib[fib_index] = session_limit;
2779   return 0;
2780 }
2781
2782 int
2783 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
2784 {
2785   snat_main_t *sm = &snat_main;
2786
2787   if (nat44_set_session_limit (session_limit, vrf_id))
2788     return 1;
2789   sm->max_translations_per_thread = nat44_get_max_session_limit ();
2790
2791   sm->translation_buckets =
2792     nat_calc_bihash_buckets (sm->max_translations_per_thread);
2793
2794   nat44_ed_sessions_clear ();
2795   return 0;
2796 }
2797
2798 static void
2799 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
2800                          u32 translation_buckets)
2801 {
2802   dlist_elt_t *head;
2803
2804   pool_alloc (tsm->sessions, translations);
2805   pool_alloc (tsm->lru_pool, translations);
2806
2807   pool_get (tsm->lru_pool, head);
2808   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
2809   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
2810
2811   pool_get (tsm->lru_pool, head);
2812   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
2813   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
2814
2815   pool_get (tsm->lru_pool, head);
2816   tsm->udp_lru_head_index = head - tsm->lru_pool;
2817   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
2818
2819   pool_get (tsm->lru_pool, head);
2820   tsm->icmp_lru_head_index = head - tsm->lru_pool;
2821   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
2822
2823   pool_get (tsm->lru_pool, head);
2824   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
2825   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
2826 }
2827
2828 static void
2829 reinit_ed_flow_hash ()
2830 {
2831   snat_main_t *sm = &snat_main;
2832   // we expect 2 flows per session, so multiply translation_buckets by 2
2833   clib_bihash_init_16_8 (
2834     &sm->flow_hash, "ed-flow-hash",
2835     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
2836   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
2837 }
2838
2839 static void
2840 nat44_ed_db_init (u32 translations, u32 translation_buckets)
2841 {
2842   snat_main_t *sm = &snat_main;
2843   snat_main_per_thread_data_t *tsm;
2844   u32 static_mapping_buckets = 1024;
2845   u32 static_mapping_memory_size = 64 << 20;
2846
2847   reinit_ed_flow_hash ();
2848
2849   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
2850                         "static_mapping_by_local", static_mapping_buckets,
2851                         static_mapping_memory_size);
2852   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
2853                                      format_static_mapping_kvp);
2854
2855   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
2856                         "static_mapping_by_external", static_mapping_buckets,
2857                         static_mapping_memory_size);
2858   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
2859                                      format_static_mapping_kvp);
2860
2861   if (sm->pat)
2862     {
2863       vec_foreach (tsm, sm->per_thread_data)
2864         {
2865           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
2866                                    sm->translation_buckets);
2867         }
2868     }
2869 }
2870
2871 static void
2872 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
2873 {
2874   pool_free (tsm->lru_pool);
2875   pool_free (tsm->sessions);
2876   vec_free (tsm->per_vrf_sessions_vec);
2877 }
2878
2879 static void
2880 nat44_ed_db_free ()
2881 {
2882   snat_main_t *sm = &snat_main;
2883   snat_main_per_thread_data_t *tsm;
2884
2885   pool_free (sm->static_mappings);
2886   clib_bihash_free_16_8 (&sm->flow_hash);
2887   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
2888   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
2889
2890   if (sm->pat)
2891     {
2892       vec_foreach (tsm, sm->per_thread_data)
2893         {
2894           nat44_ed_worker_db_free (tsm);
2895         }
2896     }
2897 }
2898
2899 void
2900 nat44_ed_sessions_clear ()
2901 {
2902   snat_main_t *sm = &snat_main;
2903   snat_main_per_thread_data_t *tsm;
2904
2905   reinit_ed_flow_hash ();
2906
2907   if (sm->pat)
2908     {
2909       vec_foreach (tsm, sm->per_thread_data)
2910         {
2911
2912           nat44_ed_worker_db_free (tsm);
2913           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
2914                                    sm->translation_buckets);
2915         }
2916     }
2917   vlib_zero_simple_counter (&sm->total_sessions, 0);
2918 }
2919
2920 static void
2921 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2922                                  uword opaque,
2923                                  u32 sw_if_index,
2924                                  ip4_address_t * address,
2925                                  u32 address_length,
2926                                  u32 if_address_index, u32 is_delete)
2927 {
2928   snat_main_t *sm = &snat_main;
2929   snat_static_map_resolve_t *rp;
2930   snat_static_mapping_t *m;
2931   clib_bihash_kv_8_8_t kv, value;
2932   int i, rv;
2933   ip4_address_t l_addr;
2934
2935   if (!sm->enabled)
2936     return;
2937
2938   for (i = 0; i < vec_len (sm->to_resolve); i++)
2939     {
2940       rp = sm->to_resolve + i;
2941       if (rp->addr_only == 0)
2942         continue;
2943       if (rp->sw_if_index == sw_if_index)
2944         goto match;
2945     }
2946
2947   return;
2948
2949 match:
2950   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
2951               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
2952   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2953     m = 0;
2954   else
2955     m = pool_elt_at_index (sm->static_mappings, value.value);
2956
2957   if (!is_delete)
2958     {
2959       /* Don't trip over lease renewal, static config */
2960       if (m)
2961         return;
2962     }
2963   else
2964     {
2965       if (!m)
2966         return;
2967     }
2968
2969   /* Indetity mapping? */
2970   if (rp->l_addr.as_u32 == 0)
2971     l_addr.as_u32 = address[0].as_u32;
2972   else
2973     l_addr.as_u32 = rp->l_addr.as_u32;
2974   /* Add the static mapping */
2975   rv = snat_add_static_mapping (l_addr,
2976                                 address[0],
2977                                 rp->l_port,
2978                                 rp->e_port,
2979                                 rp->vrf_id,
2980                                 rp->addr_only, ~0 /* sw_if_index */ ,
2981                                 rp->proto, !is_delete, rp->twice_nat,
2982                                 rp->out2in_only, rp->tag, rp->identity_nat,
2983                                 rp->pool_addr, rp->exact);
2984   if (rv)
2985     nat_elog_notice_X1 (sm, "snat_add_static_mapping returned %d", "i4", rv);
2986 }
2987
2988 static void
2989 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2990                                        uword opaque,
2991                                        u32 sw_if_index,
2992                                        ip4_address_t * address,
2993                                        u32 address_length,
2994                                        u32 if_address_index, u32 is_delete)
2995 {
2996   snat_main_t *sm = &snat_main;
2997   snat_static_map_resolve_t *rp;
2998   ip4_address_t l_addr;
2999   int i, j;
3000   int rv;
3001   u8 twice_nat = 0;
3002   snat_address_t *addresses = sm->addresses;
3003
3004   if (!sm->enabled)
3005     return;
3006
3007   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3008     {
3009       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3010         goto match;
3011     }
3012
3013   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3014     {
3015       twice_nat = 1;
3016       addresses = sm->twice_nat_addresses;
3017       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3018         goto match;
3019     }
3020
3021   return;
3022
3023 match:
3024   if (!is_delete)
3025     {
3026       /* Don't trip over lease renewal, static config */
3027       for (j = 0; j < vec_len (addresses); j++)
3028         if (addresses[j].addr.as_u32 == address->as_u32)
3029           return;
3030
3031       (void) snat_add_address (sm, address, ~0, twice_nat);
3032       /* Scan static map resolution vector */
3033       for (j = 0; j < vec_len (sm->to_resolve); j++)
3034         {
3035           rp = sm->to_resolve + j;
3036           if (rp->addr_only)
3037             continue;
3038           /* On this interface? */
3039           if (rp->sw_if_index == sw_if_index)
3040             {
3041               /* Indetity mapping? */
3042               if (rp->l_addr.as_u32 == 0)
3043                 l_addr.as_u32 = address[0].as_u32;
3044               else
3045                 l_addr.as_u32 = rp->l_addr.as_u32;
3046               /* Add the static mapping */
3047               rv = snat_add_static_mapping (
3048                 l_addr, address[0], rp->l_port, rp->e_port, rp->vrf_id,
3049                 rp->addr_only, ~0 /* sw_if_index */, rp->proto, 1,
3050                 rp->twice_nat, rp->out2in_only, rp->tag, rp->identity_nat,
3051                 rp->pool_addr, rp->exact);
3052               if (rv)
3053                 nat_elog_notice_X1 (sm, "snat_add_static_mapping returned %d",
3054                                     "i4", rv);
3055             }
3056         }
3057       return;
3058     }
3059   else
3060     {
3061       (void) snat_del_address (sm, address[0], 1, twice_nat);
3062       return;
3063     }
3064 }
3065
3066 int
3067 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3068                             u8 twice_nat)
3069 {
3070   ip4_main_t *ip4_main = sm->ip4_main;
3071   ip4_address_t *first_int_addr;
3072   snat_static_map_resolve_t *rp;
3073   u32 *indices_to_delete = 0;
3074   int i, j;
3075   u32 *auto_add_sw_if_indices =
3076     twice_nat ? sm->
3077     auto_add_sw_if_indices_twice_nat : sm->auto_add_sw_if_indices;
3078
3079   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0        /* just want the address */
3080     );
3081
3082   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3083     {
3084       if (auto_add_sw_if_indices[i] == sw_if_index)
3085         {
3086           if (is_del)
3087             {
3088               /* if have address remove it */
3089               if (first_int_addr)
3090                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3091               else
3092                 {
3093                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3094                     {
3095                       rp = sm->to_resolve + j;
3096                       if (rp->sw_if_index == sw_if_index)
3097                         vec_add1 (indices_to_delete, j);
3098                     }
3099                   if (vec_len (indices_to_delete))
3100                     {
3101                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3102                         vec_del1 (sm->to_resolve, j);
3103                       vec_free (indices_to_delete);
3104                     }
3105                 }
3106               if (twice_nat)
3107                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3108               else
3109                 vec_del1 (sm->auto_add_sw_if_indices, i);
3110             }
3111           else
3112             return VNET_API_ERROR_VALUE_EXIST;
3113
3114           return 0;
3115         }
3116     }
3117
3118   if (is_del)
3119     return VNET_API_ERROR_NO_SUCH_ENTRY;
3120
3121   /* add to the auto-address list */
3122   if (twice_nat)
3123     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3124   else
3125     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3126
3127   /* If the address is already bound - or static - add it now */
3128   if (first_int_addr)
3129     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3130
3131   return 0;
3132 }
3133
3134 int
3135 nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
3136                       ip4_address_t * eh_addr, u16 eh_port, u8 proto,
3137                       u32 vrf_id, int is_in)
3138 {
3139   ip4_header_t ip;
3140   clib_bihash_kv_16_8_t kv, value;
3141   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3142   snat_session_t *s;
3143   snat_main_per_thread_data_t *tsm;
3144
3145   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3146   if (sm->num_workers > 1)
3147     tsm = vec_elt_at_index (
3148       sm->per_thread_data,
3149       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3150   else
3151     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3152
3153   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3154   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3155     {
3156       return VNET_API_ERROR_NO_SUCH_ENTRY;
3157     }
3158
3159   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3160     return VNET_API_ERROR_UNSPECIFIED;
3161   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3162   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3163   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3164   return 0;
3165 }
3166
3167 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3168                                  vlib_node_runtime_t * node,
3169                                  vlib_frame_t * frame)
3170 {
3171   return 0;
3172 }
3173
3174 VLIB_REGISTER_NODE (nat_default_node) = {
3175   .name = "nat-default",
3176   .vector_size = sizeof (u32),
3177   .format_trace = 0,
3178   .type = VLIB_NODE_TYPE_INTERNAL,
3179   .n_errors = 0,
3180   .n_next_nodes = NAT_N_NEXT,
3181   .next_nodes = {
3182     [NAT_NEXT_DROP] = "error-drop",
3183     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3184     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3185     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3186     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3187     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3188     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3189     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3190     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3191     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3192   },
3193 };
3194
3195 void
3196 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3197 {
3198   f->l3_csum_delta = 0;
3199   f->l4_csum_delta = 0;
3200   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3201       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3202     {
3203       f->l3_csum_delta =
3204         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3205       f->l3_csum_delta =
3206         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3207     }
3208   else
3209     {
3210       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3211     }
3212   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3213       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3214     {
3215       f->l3_csum_delta =
3216         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3217       f->l3_csum_delta =
3218         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3219     }
3220   else
3221     {
3222       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3223     }
3224   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3225     {
3226       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3227       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3228     }
3229   else
3230     {
3231       f->rewrite.sport = f->match.sport;
3232     }
3233   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3234     {
3235       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3236       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3237     }
3238   else
3239     {
3240       f->rewrite.dport = f->match.dport;
3241     }
3242   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3243       f->rewrite.icmp_id != f->match.sport)
3244     {
3245       f->l4_csum_delta =
3246         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3247       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3248     }
3249   else
3250     {
3251       f->rewrite.icmp_id = f->match.sport;
3252     }
3253   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3254     {
3255     }
3256   else
3257     {
3258       f->rewrite.fib_index = f->match.fib_index;
3259     }
3260 }
3261
3262 static_always_inline int nat_6t_flow_icmp_translate (snat_main_t *sm,
3263                                                      vlib_buffer_t *b,
3264                                                      ip4_header_t *ip,
3265                                                      nat_6t_flow_t *f);
3266
3267 static_always_inline void
3268 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3269                            nat_6t_flow_t *f, nat_protocol_t proto,
3270                            int is_icmp_inner_ip4)
3271 {
3272   udp_header_t *udp = ip4_next_header (ip);
3273   tcp_header_t *tcp = (tcp_header_t *) udp;
3274
3275   if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
3276       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3277     {
3278       if (!is_icmp_inner_ip4)
3279         { // regular case
3280           ip->src_address = f->rewrite.saddr;
3281           ip->dst_address = f->rewrite.daddr;
3282           udp->src_port = f->rewrite.sport;
3283           udp->dst_port = f->rewrite.dport;
3284         }
3285       else
3286         { // icmp inner ip4 - reversed saddr/daddr
3287           ip->src_address = f->rewrite.daddr;
3288           ip->dst_address = f->rewrite.saddr;
3289           udp->src_port = f->rewrite.dport;
3290           udp->dst_port = f->rewrite.sport;
3291         }
3292
3293       if (NAT_PROTOCOL_TCP == proto)
3294         {
3295           ip_csum_t tcp_sum = tcp->checksum;
3296           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3297           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3298           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3299           tcp->checksum = ip_csum_fold (tcp_sum);
3300         }
3301       else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
3302         {
3303           ip_csum_t udp_sum = udp->checksum;
3304           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3305           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3306           udp->checksum = ip_csum_fold (udp_sum);
3307         }
3308     }
3309   else
3310     {
3311       if (!is_icmp_inner_ip4)
3312         { // regular case
3313           ip->src_address = f->rewrite.saddr;
3314           ip->dst_address = f->rewrite.daddr;
3315         }
3316       else
3317         { // icmp inner ip4 - reversed saddr/daddr
3318           ip->src_address = f->rewrite.daddr;
3319           ip->dst_address = f->rewrite.saddr;
3320         }
3321     }
3322
3323   ip_csum_t ip_sum = ip->checksum;
3324   ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3325   ip->checksum = ip_csum_fold (ip_sum);
3326   if (0xffff == ip->checksum)
3327     ip->checksum = 0;
3328   ASSERT (ip4_header_checksum_is_valid (ip));
3329 }
3330
3331 static_always_inline int
3332 nat_6t_flow_icmp_translate (snat_main_t *sm, vlib_buffer_t *b,
3333                             ip4_header_t *ip, nat_6t_flow_t *f)
3334 {
3335   if (IP_PROTOCOL_ICMP != ip->protocol)
3336     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3337
3338   icmp46_header_t *icmp = ip4_next_header (ip);
3339   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3340
3341   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3342     {
3343       if (icmp->checksum == 0)
3344         icmp->checksum = 0xffff;
3345
3346       if (!icmp_type_is_error_message (icmp->type))
3347         {
3348           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3349               (f->rewrite.icmp_id != echo->identifier))
3350             {
3351               ip_csum_t sum = icmp->checksum;
3352               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3353                                     icmp_echo_header_t,
3354                                     identifier /* changed member */);
3355               echo->identifier = f->rewrite.icmp_id;
3356               icmp->checksum = ip_csum_fold (sum);
3357             }
3358         }
3359       else
3360         {
3361           // errors are not fragmented
3362           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3363
3364           if (!ip4_header_checksum_is_valid (inner_ip))
3365             {
3366               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3367             }
3368
3369           nat_protocol_t inner_proto =
3370             ip_proto_to_nat_proto (inner_ip->protocol);
3371
3372           ip_csum_t icmp_sum = icmp->checksum;
3373
3374           switch (inner_proto)
3375             {
3376             case NAT_PROTOCOL_UDP:
3377             case NAT_PROTOCOL_TCP:
3378               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3379                                          1 /* is_icmp_inner_ip4 */);
3380               icmp_sum = ip_csum_sub_even (icmp_sum, f->l3_csum_delta);
3381               icmp->checksum = ip_csum_fold (icmp_sum);
3382               break;
3383             case NAT_PROTOCOL_ICMP:
3384               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3385                 {
3386                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3387                   icmp_echo_header_t *inner_echo =
3388                     (icmp_echo_header_t *) (inner_icmp + 1);
3389                   if (f->rewrite.icmp_id != inner_echo->identifier)
3390                     {
3391                       ip_csum_t sum = icmp->checksum;
3392                       sum = ip_csum_update (
3393                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3394                         icmp_echo_header_t, identifier /* changed member */);
3395                       icmp->checksum = ip_csum_fold (sum);
3396                       ip_csum_t inner_sum = inner_icmp->checksum;
3397                       inner_sum = ip_csum_update (
3398                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3399                         icmp_echo_header_t, identifier /* changed member */);
3400                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3401                       inner_echo->identifier = f->rewrite.icmp_id;
3402                     }
3403                 }
3404               break;
3405             default:
3406               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3407               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3408             }
3409         }
3410     }
3411   return NAT_ED_TRNSL_ERR_SUCCESS;
3412 }
3413
3414 nat_translation_error_e
3415 nat_6t_flow_buf_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3416                            nat_6t_flow_t *f, nat_protocol_t proto,
3417                            int is_output_feature)
3418 {
3419   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3420     {
3421       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3422     }
3423
3424   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */);
3425
3426   if (NAT_PROTOCOL_ICMP == proto)
3427     {
3428       return nat_6t_flow_icmp_translate (sm, b, ip, f);
3429     }
3430
3431   return NAT_ED_TRNSL_ERR_SUCCESS;
3432 }
3433
3434 u8 *
3435 format_nat_6t (u8 *s, va_list *args)
3436 {
3437   nat_6t_t *t = va_arg (*args, nat_6t_t *);
3438
3439   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
3440               format_ip4_address, t->saddr.as_u8,
3441               clib_net_to_host_u16 (t->sport), format_ip4_address,
3442               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
3443               format_ip_protocol, t->proto, t->fib_index);
3444   return s;
3445 }
3446
3447 u8 *
3448 format_nat_ed_translation_error (u8 *s, va_list *args)
3449 {
3450   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
3451
3452   switch (e)
3453     {
3454     case NAT_ED_TRNSL_ERR_SUCCESS:
3455       s = format (s, "success");
3456       break;
3457     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
3458       s = format (s, "translation-failed");
3459       break;
3460     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
3461       s = format (s, "flow-mismatch");
3462       break;
3463     }
3464   return s;
3465 }
3466
3467 u8 *
3468 format_nat_6t_flow (u8 *s, va_list *args)
3469 {
3470   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
3471
3472   s = format (s, "match: %U ", format_nat_6t, &f->match);
3473   int r = 0;
3474   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
3475     {
3476       s = format (s, "rewrite: saddr %U ", format_ip4_address,
3477                   f->rewrite.saddr.as_u8);
3478       r = 1;
3479     }
3480   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
3481     {
3482       if (!r)
3483         {
3484           s = format (s, "rewrite: ");
3485           r = 1;
3486         }
3487       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
3488     }
3489   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
3490     {
3491       if (!r)
3492         {
3493           s = format (s, "rewrite: ");
3494           r = 1;
3495         }
3496       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
3497     }
3498   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
3499     {
3500       if (!r)
3501         {
3502           s = format (s, "rewrite: ");
3503           r = 1;
3504         }
3505       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
3506     }
3507   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3508     {
3509       if (!r)
3510         {
3511           s = format (s, "rewrite: ");
3512           r = 1;
3513         }
3514       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
3515     }
3516   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3517     {
3518       if (!r)
3519         {
3520           s = format (s, "rewrite: ");
3521           r = 1;
3522         }
3523       s = format (s, "txfib %u ", f->rewrite.fib_index);
3524     }
3525   return s;
3526 }
3527
3528 /*
3529  * fd.io coding-style-patch-verification: ON
3530  *
3531  * Local Variables:
3532  * eval: (c-set-style "gnu")
3533  * End:
3534  */