src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vpp/app/version.h>
  19
  20 #include <vnet/vnet.h>
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ip/ip4.h>
  23 #include <vnet/ip/ip_table.h>
  24 #include <vnet/ip/reass/ip4_sv_reass.h>
  25 #include <vnet/fib/fib_table.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <vnet/plugin/plugin.h>
  28 #include <vppinfra/bihash_16_8.h>
  29
  30 #include <nat/lib/log.h>
  31 #include <nat/lib/nat_syslog.h>
  32 #include <nat/lib/nat_inlines.h>
  33 #include <nat/lib/ipfix_logging.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 #include <vpp/stats/stat_segment.h>
  40
  41 snat_main_t snat_main;
  42
  43 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  44                                                            u32 sw_if_index);
  45
  46 #define skip_if_disabled()                                                    \
  47   do                                                                          \
  48     {                                                                         \
  49       snat_main_t *sm = &snat_main;                                           \
  50       if (PREDICT_FALSE (!sm->enabled))                                       \
  51         return;                                                               \
  52     }                                                                         \
  53   while (0)
  54
  55 #define fail_if_enabled()                                                     \
  56   do                                                                          \
  57     {                                                                         \
  58       snat_main_t *sm = &snat_main;                                           \
  59       if (PREDICT_FALSE (sm->enabled))                                        \
  60         {                                                                     \
  61           nat_log_err ("plugin enabled");                                     \
  62           return 1;                                                           \
  63         }                                                                     \
  64     }                                                                         \
  65   while (0)
  66
  67 #define fail_if_disabled()                                                    \
  68   do                                                                          \
  69     {                                                                         \
  70       snat_main_t *sm = &snat_main;                                           \
  71       if (PREDICT_FALSE (!sm->enabled))                                       \
  72         {                                                                     \
  73           nat_log_err ("plugin disabled");                                    \
  74           return 1;                                                           \
  75         }                                                                     \
  76     }                                                                         \
  77   while (0)
  78
  79 /* Hook up input features */
  80 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat-pre-in2out",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat-pre-out2in",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  90                                "ip4-dhcp-client-detect",
  91                                "ip4-sv-reassembly-feature"),
  92 };
  93 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-in2out-worker-handoff",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  97 };
  98 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-out2in-worker-handoff",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 102                                "ip4-dhcp-client-detect"),
 103 };
 104 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-in2out",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-out2in",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 124                                "ip4-dhcp-client-detect"),
 125 };
 126 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 127   .arc_name = "ip4-unicast",
 128   .node_name = "nat44-ed-classify",
 129   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 130 };
 131 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 132   .arc_name = "ip4-unicast",
 133   .node_name = "nat44-handoff-classify",
 134   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 137   .arc_name = "ip4-unicast",
 138   .node_name = "nat44-in2out-fast",
 139   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 140 };
 141 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 142   .arc_name = "ip4-unicast",
 143   .node_name = "nat44-out2in-fast",
 144   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 145                                "ip4-dhcp-client-detect"),
 146 };
 147
 148 /* Hook up output features */
 149 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 150   .arc_name = "ip4-output",
 151   .node_name = "nat44-in2out-output",
 152   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 153   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 156   .arc_name = "ip4-output",
 157   .node_name = "nat44-in2out-output-worker-handoff",
 158   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 159   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 160 };
 161 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat-pre-in2out-output",
 164   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 165   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 166 };
 167 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 168   .arc_name = "ip4-output",
 169   .node_name = "nat44-ed-in2out-output",
 170   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 171   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 172 };
 173
 174 VLIB_PLUGIN_REGISTER () = {
 175     .version = VPP_BUILD_VER,
 176     .description = "Network Address Translation (NAT)",
 177 };
 178
 179 static void nat44_ed_db_init (u32 translations, u32 translation_buckets);
 180
 181 static void nat44_ed_db_free ();
 182
 183 u32 nat_calc_bihash_buckets (u32 n_elts);
 184
 185 u8 *
 186 format_session_kvp (u8 * s, va_list * args)
 187 {
 188   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 189
 190   s = format (s, "%U thread-index %llu session-index %llu", format_snat_key,
 191               v->key, nat_value_get_thread_index (v),
 192               nat_value_get_session_index (v));
 193
 194   return s;
 195 }
 196
 197 u8 *
 198 format_static_mapping_kvp (u8 * s, va_list * args)
 199 {
 200   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 201
 202   s = format (s, "%U static-mapping-index %llu",
 203               format_snat_key, v->key, v->value);
 204
 205   return s;
 206 }
 207
 208 u8 *
 209 format_ed_session_kvp (u8 * s, va_list * args)
 210 {
 211   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 212
 213   u8 proto;
 214   u16 r_port, l_port;
 215   ip4_address_t l_addr, r_addr;
 216   u32 fib_index;
 217
 218   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 219   s = format (s,
 220               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 221               "session-index %u",
 222               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 223               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 224               format_ip_protocol, proto, fib_index,
 225               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 226
 227   return s;
 228 }
 229
 230 void
 231 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 232                        u8 is_ha)
 233 {
 234       per_vrf_sessions_unregister_session (s, thread_index);
 235
 236       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 237         nat_elog_warn (sm, "flow hash del failed");
 238
 239       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 240         nat_elog_warn (sm, "flow hash del failed");
 241
 242       if (na44_ed_is_fwd_bypass_session (s))
 243         {
 244           return;
 245         }
 246
 247       if (nat44_ed_is_affinity_session (s))
 248         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, s->nat_proto,
 249                              s->out2in.port);
 250
 251       if (!is_ha)
 252         nat_syslog_nat44_sdel (
 253           0, s->in2out.fib_index, &s->in2out.addr, s->in2out.port,
 254           &s->ext_host_nat_addr, s->ext_host_nat_port, &s->out2in.addr,
 255           s->out2in.port, &s->ext_host_addr, s->ext_host_port, s->nat_proto,
 256           nat44_ed_is_twice_nat_session (s));
 257
 258       if (snat_is_unk_proto_session (s))
 259         return;
 260
 261       if (!is_ha)
 262         {
 263           /* log NAT event */
 264           nat_ipfix_logging_nat44_ses_delete (
 265             thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
 266             s->nat_proto, s->in2out.port, s->out2in.port, s->in2out.fib_index);
 267         }
 268
 269   /* Twice NAT address and port for external host */
 270       if (nat44_ed_is_twice_nat_session (s))
 271         {
 272           snat_free_outside_address_and_port (
 273             sm->twice_nat_addresses, thread_index, &s->ext_host_nat_addr,
 274             s->ext_host_nat_port, s->nat_proto);
 275         }
 276
 277       if (nat44_ed_is_session_static (s))
 278         return;
 279
 280       snat_free_outside_address_and_port (sm->addresses, thread_index,
 281                                           &s->out2in.addr, s->out2in.port,
 282                                           s->nat_proto);
 283 }
 284
 285 static int
 286 is_snat_address_used_in_static_mapping (snat_main_t *sm, ip4_address_t addr)
 287 {
 288   snat_static_mapping_t *m;
 289   pool_foreach (m, sm->static_mappings)
 290     {
 291       if (is_sm_addr_only (m->flags) || is_sm_out2in_only (m->flags) ||
 292           is_sm_identity_nat (m->flags))
 293         {
 294           continue;
 295         }
 296       if (m->external_addr.as_u32 == addr.as_u32)
 297         {
 298           return 1;
 299         }
 300     }
 301   return 0;
 302 }
 303
 304 static void
 305 nat44_ed_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
 306                               int is_add)
 307 {
 308   // Add the external NAT address to the FIB as receive entries. This ensures
 309   // that VPP will reply to ARP for this address and we don't need to enable
 310   // proxy ARP on the outside interface.
 311
 312   snat_main_t *sm = &snat_main;
 313   fib_prefix_t prefix = {
 314     .fp_len = p_len,
 315     .fp_proto = FIB_PROTOCOL_IP4,
 316     .fp_addr = {
 317                 .ip4.as_u32 = addr->as_u32,
 318                 },
 319   };
 320   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 321
 322   if (is_add)
 323     {
 324       fib_table_entry_update_one_path (fib_index, &prefix, sm->fib_src_low,
 325                                        (FIB_ENTRY_FLAG_CONNECTED |
 326                                         FIB_ENTRY_FLAG_LOCAL |
 327                                         FIB_ENTRY_FLAG_EXCLUSIVE),
 328                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
 329                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 330     }
 331   else
 332     {
 333       fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 334     }
 335 }
 336
 337 static void
 338 nat44_ed_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 339 {
 340   snat_main_t *sm = &snat_main;
 341   snat_interface_t *i;
 342
 343   pool_foreach (i, sm->interfaces)
 344     {
 345       if (nat44_ed_is_interface_outside (i))
 346         {
 347           nat44_ed_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 348         }
 349     }
 350   pool_foreach (i, sm->output_feature_interfaces)
 351     {
 352       if (nat44_ed_is_interface_outside (i))
 353         {
 354           nat44_ed_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 355         }
 356     }
 357 }
 358
 359 static_always_inline void
 360 nat44_ed_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 361 {
 362   snat_main_t *sm = &snat_main;
 363   snat_address_t *ap;
 364
 365   vec_foreach (ap, sm->addresses)
 366     {
 367       nat44_ed_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 368     }
 369 }
 370
 371 static_always_inline void
 372 nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 373 {
 374   snat_main_t *sm = &snat_main;
 375   snat_static_mapping_t *m;
 376
 377   pool_foreach (m, sm->static_mappings)
 378     {
 379       if (is_sm_addr_only (m->flags) && !is_sm_identity_nat (m->flags))
 380         {
 381           nat44_ed_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 382                                         is_add);
 383         }
 384     }
 385 }
 386
 387 int
 388 nat44_ed_add_address (ip4_address_t *addr, u32 vrf_id, u8 twice_nat)
 389 {
 390   snat_main_t *sm = &snat_main;
 391   vlib_thread_main_t *tm = vlib_get_thread_main ();
 392   snat_address_t *ap, *addresses;
 393
 394   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 395
 396   if (!sm->enabled)
 397     {
 398       return VNET_API_ERROR_UNSUPPORTED;
 399     }
 400
 401   // check if address already exists
 402   vec_foreach (ap, addresses)
 403     {
 404       if (ap->addr.as_u32 == addr->as_u32)
 405         {
 406           nat_log_err ("address exist");
 407           return VNET_API_ERROR_VALUE_EXIST;
 408         }
 409     }
 410
 411   if (twice_nat)
 412     {
 413       vec_add2 (sm->twice_nat_addresses, ap, 1);
 414     }
 415   else
 416     {
 417       vec_add2 (sm->addresses, ap, 1);
 418     }
 419
 420   ap->fib_index = ~0;
 421   ap->addr = *addr;
 422
 423   if (vrf_id != ~0)
 424     {
 425       ap->fib_index = fib_table_find_or_create_and_lock (
 426         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 427     }
 428
 429 #define _(N, i, n, s)                                                         \
 430   clib_memset (ap->busy_##n##_port_refcounts, 0,                              \
 431                sizeof (ap->busy_##n##_port_refcounts));                       \
 432   ap->busy_##n##_ports = 0;                                                   \
 433   ap->busy_##n##_ports_per_thread = 0;                                        \
 434   vec_validate_init_empty (ap->busy_##n##_ports_per_thread,                   \
 435                            tm->n_vlib_mains - 1, 0);
 436   foreach_nat_protocol
 437 #undef _
 438
 439     if (!twice_nat)
 440   {
 441     // if we don't have enabled interface we don't add address
 442     // to fib
 443     nat44_ed_add_del_addr_to_fib_foreach_out_if (addr, 1);
 444   }
 445   return 0;
 446 }
 447
 448 int
 449 nat44_ed_del_address (ip4_address_t addr, u8 delete_sm, u8 twice_nat)
 450 {
 451   snat_main_t *sm = &snat_main;
 452   snat_address_t *a = 0, *addresses;
 453   snat_session_t *ses;
 454   u32 *ses_to_be_removed = 0, *ses_index;
 455   snat_main_per_thread_data_t *tsm;
 456   snat_static_mapping_t *m;
 457   int j;
 458
 459   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 460
 461   for (j = 0; j < vec_len (addresses); j++)
 462     {
 463       if (addresses[j].addr.as_u32 == addr.as_u32)
 464         {
 465           a = addresses + j;
 466           break;
 467         }
 468     }
 469   if (!a)
 470     {
 471       nat_log_err ("no such address");
 472       return VNET_API_ERROR_NO_SUCH_ENTRY;
 473     }
 474
 475   if (delete_sm)
 476     {
 477       pool_foreach (m, sm->static_mappings)
 478         {
 479           if (m->external_addr.as_u32 == addr.as_u32)
 480             {
 481               nat44_ed_del_static_mapping (m->local_addr, m->external_addr,
 482                                            m->local_port, m->external_port,
 483                                            m->proto, m->vrf_id, ~0, m->flags);
 484             }
 485         }
 486     }
 487   else
 488     {
 489       // TODO: why ?
 490       // check if address is used in some static mapping
 491       if (is_snat_address_used_in_static_mapping (sm, addr))
 492         {
 493           nat_log_err ("address used in static mapping");
 494           return VNET_API_ERROR_UNSPECIFIED;
 495         }
 496     }
 497
 498   if (a->fib_index != ~0)
 499     {
 500       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 501     }
 502
 503   // delete sessions using address
 504   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
 505     {
 506       vec_foreach (tsm, sm->per_thread_data)
 507         {
 508           pool_foreach (ses, tsm->sessions)
 509             {
 510               if (ses->out2in.addr.as_u32 == addr.as_u32)
 511                 {
 512                   nat_free_session_data (sm, ses, tsm - sm->per_thread_data,
 513                                          0);
 514                   vec_add1 (ses_to_be_removed, ses - tsm->sessions);
 515                 }
 516             }
 517           vec_foreach (ses_index, ses_to_be_removed)
 518             {
 519               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
 520               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
 521             }
 522           vec_free (ses_to_be_removed);
 523         }
 524     }
 525
 526 #define _(N, i, n, s) vec_free (a->busy_##n##_ports_per_thread);
 527   foreach_nat_protocol
 528 #undef _
 529
 530     if (!twice_nat)
 531   {
 532     vec_del1 (sm->addresses, j);
 533     nat44_ed_add_del_addr_to_fib_foreach_out_if (&addr, 0);
 534   }
 535   else { vec_del1 (sm->twice_nat_addresses, j); }
 536   return 0;
 537 }
 538
 539 u32
 540 get_thread_idx_by_port (u16 e_port)
 541 {
 542   snat_main_t *sm = &snat_main;
 543   u32 thread_idx = sm->num_workers;
 544   if (sm->num_workers > 1)
 545     {
 546       thread_idx =
 547         sm->first_worker_index +
 548         sm->workers[(e_port - 1024) / sm->port_per_thread];
 549     }
 550   return thread_idx;
 551 }
 552
 553 void
 554 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 555                                     snat_main_per_thread_data_t * tsm,
 556                                     ip4_address_t l_addr,
 557                                     u16 l_port,
 558                                     u8 protocol,
 559                                     u32 fib_index, int addr_only,
 560                                     ip4_address_t e_addr, u16 e_port)
 561 {
 562   snat_session_t *s;
 563   u32 *indexes_to_free = NULL;
 564   pool_foreach (s, tsm->sessions) {
 565     if (s->in2out.fib_index != fib_index ||
 566         s->in2out.addr.as_u32 != l_addr.as_u32)
 567       {
 568         continue;
 569       }
 570     if (!addr_only)
 571       {
 572         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 573             s->out2in.port != e_port ||
 574             s->in2out.port != l_port ||
 575             s->nat_proto != protocol)
 576           continue;
 577       }
 578
 579     if (nat44_ed_is_lb_session (s))
 580       continue;
 581     if (!nat44_ed_is_session_static (s))
 582       continue;
 583     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 584     vec_add1 (indexes_to_free, s - tsm->sessions);
 585     if (!addr_only)
 586       break;
 587   }
 588   u32 *ses_index;
 589   vec_foreach (ses_index, indexes_to_free)
 590   {
 591     s = pool_elt_at_index (tsm->sessions, *ses_index);
 592     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 593   }
 594   vec_free (indexes_to_free);
 595 }
 596
 597 int
 598 nat44_ed_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
 599 {
 600   u32 ti = get_thread_idx_by_port (port);
 601   snat_main_t *sm = &snat_main;
 602   snat_address_t *a = 0;
 603   int i;
 604
 605   for (i = 0; i < vec_len (sm->addresses); i++)
 606     {
 607       a = sm->addresses + i;
 608
 609       if (a->addr.as_u32 != addr.as_u32)
 610         continue;
 611
 612       switch (proto)
 613         {
 614 #define _(N, j, n, s)                                                         \
 615   case NAT_PROTOCOL_##N:                                                      \
 616     if (a->busy_##n##_port_refcounts[port])                                   \
 617       goto done;                                                              \
 618     ++a->busy_##n##_port_refcounts[port];                                     \
 619     if (port > 1024)                                                          \
 620       {                                                                       \
 621         a->busy_##n##_ports++;                                                \
 622         a->busy_##n##_ports_per_thread[ti]++;                                 \
 623       }                                                                       \
 624     break;
 625           foreach_nat_protocol
 626 #undef _
 627             default : nat_elog_info (sm, "unknown protocol");
 628           goto done;
 629         }
 630
 631       return 0;
 632     }
 633
 634 done:
 635   return 1;
 636 }
 637
 638 int
 639 nat44_ed_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
 640 {
 641   u32 ti = get_thread_idx_by_port (port);
 642   snat_main_t *sm = &snat_main;
 643   snat_address_t *a = 0;
 644   int i;
 645
 646   for (i = 0; i < vec_len (sm->addresses); i++)
 647     {
 648       a = sm->addresses + i;
 649
 650       if (a->addr.as_u32 != addr.as_u32)
 651         continue;
 652
 653       switch (proto)
 654         {
 655 #define _(N, j, n, s)                                                         \
 656   case NAT_PROTOCOL_##N:                                                      \
 657     --a->busy_##n##_port_refcounts[port];                                     \
 658     if (port > 1024)                                                          \
 659       {                                                                       \
 660         a->busy_##n##_ports--;                                                \
 661         a->busy_##n##_ports_per_thread[ti]--;                                 \
 662       }                                                                       \
 663     break;
 664           foreach_nat_protocol
 665 #undef _
 666             default : nat_elog_info (sm, "unknown protocol");
 667           goto done;
 668         }
 669
 670       return 0;
 671     }
 672
 673 done:
 674   return 1;
 675 }
 676
 677 void
 678 nat44_ed_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 679                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 680                              u32 flags, ip4_address_t pool_addr, u8 *tag)
 681 {
 682   snat_static_map_resolve_t *rp;
 683   snat_main_t *sm = &snat_main;
 684
 685   vec_add2 (sm->to_resolve, rp, 1);
 686   rp->l_addr.as_u32 = l_addr.as_u32;
 687   rp->l_port = l_port;
 688   rp->e_port = e_port;
 689   rp->sw_if_index = sw_if_index;
 690   rp->vrf_id = vrf_id;
 691   rp->proto = proto;
 692   rp->flags = flags;
 693   rp->pool_addr = pool_addr;
 694   rp->tag = vec_dup (tag);
 695 }
 696
 697 int
 698 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 699                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 700                              u32 flags, int *out)
 701 {
 702   snat_static_map_resolve_t *rp;
 703   snat_main_t *sm = &snat_main;
 704   int i;
 705
 706   for (i = 0; i < vec_len (sm->to_resolve); i++)
 707     {
 708       rp = sm->to_resolve + i;
 709
 710       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 711         {
 712           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 713             {
 714               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 715                 {
 716                   if (rp->e_port != e_port || rp->proto != proto)
 717                     {
 718                       continue;
 719                     }
 720                 }
 721             }
 722           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 723             {
 724               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 725                 {
 726                   if (rp->l_port != l_port || rp->e_port != e_port ||
 727                       rp->proto != proto)
 728                     {
 729                       continue;
 730                     }
 731                 }
 732             }
 733           else
 734             {
 735               continue;
 736             }
 737           if (out)
 738             {
 739               *out = i;
 740             }
 741           return 0;
 742         }
 743     }
 744   return 1;
 745 }
 746
 747 int
 748 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 749                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 750                              u32 flags)
 751 {
 752   snat_main_t *sm = &snat_main;
 753   int i;
 754   if (!nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 755                                     sw_if_index, flags, &i))
 756     {
 757       vec_del1 (sm->to_resolve, i);
 758       return 0;
 759     }
 760   return 1;
 761 }
 762
 763 static_always_inline int
 764 nat44_ed_validate_sm_input (u32 flags)
 765 {
 766   // identity nat can be initiated only from inside interface
 767   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 768     {
 769       return VNET_API_ERROR_UNSUPPORTED;
 770     }
 771
 772   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 773     {
 774       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 775         {
 776           return VNET_API_ERROR_UNSUPPORTED;
 777         }
 778     }
 779   return 0;
 780 }
 781
 782 int
 783 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 784                              u16 l_port, u16 e_port, nat_protocol_t proto,
 785                              u32 vrf_id, u32 sw_if_index, u32 flags,
 786                              ip4_address_t pool_addr, u8 *tag)
 787 {
 788   snat_main_t *sm = &snat_main;
 789   clib_bihash_kv_8_8_t kv, value;
 790   nat44_lb_addr_port_t *local;
 791   snat_static_mapping_t *m;
 792   u32 fib_index = ~0;
 793   int rv;
 794
 795   if (!sm->enabled)
 796     {
 797       return VNET_API_ERROR_UNSUPPORTED;
 798     }
 799
 800   rv = nat44_ed_validate_sm_input (flags);
 801   if (rv != 0)
 802     {
 803       return rv;
 804     }
 805
 806   if (is_sm_addr_only (flags))
 807     {
 808       e_port = l_port = proto = 0;
 809     }
 810
 811   if (is_sm_switch_address (flags))
 812     {
 813       // this mapping is interface bound
 814       ip4_address_t *first_int_addr;
 815
 816       // check if this record isn't registered for resolve
 817       if (!nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 818                                         sw_if_index, flags, 0))
 819         {
 820           return VNET_API_ERROR_VALUE_EXIST;
 821         }
 822       // register record for resolve
 823       nat44_ed_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 824                                    sw_if_index, flags, pool_addr, tag);
 825
 826       first_int_addr =
 827         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 828       if (!first_int_addr)
 829         {
 830           // dhcp resolution required
 831           return 0;
 832         }
 833
 834       e_addr.as_u32 = first_int_addr->as_u32;
 835     }
 836
 837   if (is_sm_identity_nat (flags))
 838     {
 839       l_port = e_port;
 840       l_addr.as_u32 = e_addr.as_u32;
 841     }
 842
 843   // fib index 0
 844   init_nat_k (&kv, e_addr, e_port, 0, proto);
 845
 846   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 847     {
 848       m = pool_elt_at_index (sm->static_mappings, value.value);
 849       if (!is_sm_identity_nat (m->flags))
 850         {
 851           return VNET_API_ERROR_VALUE_EXIST;
 852         }
 853
 854       // case:
 855       // adding local identity nat record for different vrf table
 856       pool_foreach (local, m->locals)
 857         {
 858           if (local->vrf_id == vrf_id)
 859             {
 860               return VNET_API_ERROR_VALUE_EXIST;
 861             }
 862         }
 863
 864       pool_get (m->locals, local);
 865
 866       local->vrf_id = vrf_id;
 867       local->fib_index = fib_table_find_or_create_and_lock (
 868         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 869
 870       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
 871                    m->proto, 0, m - sm->static_mappings);
 872       clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 873
 874       return 0;
 875     }
 876
 877   if (vrf_id != ~0)
 878     {
 879       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 880                                                      sm->fib_src_low);
 881     }
 882   else
 883     {
 884       // fallback to default vrf
 885       vrf_id = sm->inside_vrf_id;
 886       fib_index = sm->inside_fib_index;
 887       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 888     }
 889
 890   // test if local mapping record doesn't exist
 891   // identity nat supports multiple records in local mapping
 892   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
 893     {
 894       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
 895       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 896         {
 897           return VNET_API_ERROR_VALUE_EXIST;
 898         }
 899     }
 900
 901   if (!(is_sm_out2in_only (flags) || is_sm_addr_only (flags) ||
 902         sm->static_mapping_only))
 903     {
 904       if (nat44_ed_reserve_port (e_addr, e_port, proto))
 905         {
 906           // remove resolve record
 907           if (is_sm_switch_address (flags) && !is_sm_identity_nat (flags))
 908             {
 909               nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto,
 910                                            vrf_id, sw_if_index, flags);
 911             }
 912           return VNET_API_ERROR_NO_SUCH_ENTRY;
 913         }
 914     }
 915
 916   pool_get (sm->static_mappings, m);
 917   clib_memset (m, 0, sizeof (*m));
 918
 919   m->flags = flags;
 920   m->local_addr = l_addr;
 921   m->external_addr = e_addr;
 922
 923   m->tag = vec_dup (tag);
 924
 925   if (is_sm_exact_address (flags) && is_sm_twice_nat (flags))
 926     {
 927       m->pool_addr = pool_addr;
 928     }
 929
 930   if (!is_sm_addr_only (flags))
 931     {
 932       m->local_port = l_port;
 933       m->external_port = e_port;
 934       m->proto = proto;
 935     }
 936
 937   if (is_sm_identity_nat (flags))
 938     {
 939       pool_get (m->locals, local);
 940
 941       local->vrf_id = vrf_id;
 942       local->fib_index = fib_index;
 943     }
 944   else
 945     {
 946       m->vrf_id = vrf_id;
 947       m->fib_index = fib_index;
 948
 949       // not identity && addr only
 950       if (is_sm_addr_only (flags))
 951         {
 952           nat44_ed_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
 953         }
 954     }
 955
 956   if (!is_sm_out2in_only (flags))
 957     {
 958       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
 959                    m - sm->static_mappings);
 960       clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 961     }
 962
 963   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
 964                m - sm->static_mappings);
 965   clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 966
 967   if (sm->num_workers > 1)
 968     {
 969       // store worker index for this record
 970       ip4_header_t ip = {
 971         .src_address = m->local_addr,
 972       };
 973       u32 worker_index;
 974       worker_index =
 975         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
 976       vec_add1 (m->workers, worker_index);
 977     }
 978
 979   return 0;
 980 }
 981
 982 int
 983 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 984                              u16 l_port, u16 e_port, nat_protocol_t proto,
 985                              u32 vrf_id, u32 sw_if_index, u32 flags)
 986 {
 987   snat_main_per_thread_data_t *tsm;
 988   snat_main_t *sm = &snat_main;
 989
 990   clib_bihash_kv_8_8_t kv, value;
 991   nat44_lb_addr_port_t *local;
 992   snat_static_mapping_t *m;
 993   u32 fib_index = ~0;
 994   int rv;
 995
 996   if (!sm->enabled)
 997     {
 998       return VNET_API_ERROR_UNSUPPORTED;
 999     }
1000
1001   rv = nat44_ed_validate_sm_input (flags);
1002   if (rv != 0)
1003     {
1004       return rv;
1005     }
1006
1007   if (is_sm_addr_only (flags))
1008     {
1009       e_port = l_port = proto = 0;
1010     }
1011
1012   if (is_sm_switch_address (flags))
1013     {
1014       // this mapping is interface bound
1015       ip4_address_t *first_int_addr;
1016
1017       // delete record registered for resolve
1018       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
1019                                        sw_if_index, flags))
1020         {
1021           return VNET_API_ERROR_NO_SUCH_ENTRY;
1022         }
1023
1024       first_int_addr =
1025         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
1026       if (!first_int_addr)
1027         {
1028           // dhcp resolution required
1029           return 0;
1030         }
1031
1032       e_addr.as_u32 = first_int_addr->as_u32;
1033     }
1034
1035   if (is_sm_identity_nat (flags))
1036     {
1037       l_port = e_port;
1038       l_addr.as_u32 = e_addr.as_u32;
1039     }
1040
1041   // fib index 0
1042   init_nat_k (&kv, e_addr, e_port, 0, proto);
1043
1044   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1045     {
1046       if (is_sm_switch_address (flags))
1047         {
1048           return 0;
1049         }
1050       return VNET_API_ERROR_NO_SUCH_ENTRY;
1051     }
1052
1053   m = pool_elt_at_index (sm->static_mappings, value.value);
1054
1055   if (is_sm_identity_nat (flags))
1056     {
1057       u8 failure = 1;
1058
1059       if (!is_sm_switch_address (flags))
1060         {
1061           vrf_id = sm->inside_vrf_id;
1062         }
1063
1064       pool_foreach (local, m->locals)
1065         {
1066           if (local->vrf_id == vrf_id)
1067             {
1068               local = pool_elt_at_index (m->locals, local - m->locals);
1069               fib_index = local->fib_index;
1070               pool_put (m->locals, local);
1071               failure = 0;
1072             }
1073         }
1074
1075       if (failure)
1076         {
1077           return VNET_API_ERROR_NO_SUCH_ENTRY;
1078         }
1079     }
1080   else
1081     {
1082       fib_index = m->fib_index;
1083     }
1084
1085   if (!(is_sm_out2in_only (flags) || is_sm_addr_only (flags) ||
1086         sm->static_mapping_only))
1087     {
1088       if (nat44_ed_free_port (e_addr, e_port, proto))
1089         {
1090           return VNET_API_ERROR_INVALID_VALUE;
1091         }
1092     }
1093
1094   if (!is_sm_out2in_only (flags))
1095     {
1096       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
1097       clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
1098     }
1099
1100   if (!sm->static_mapping_only || sm->static_mapping_connection_tracking)
1101     {
1102       // delete sessions for static mapping
1103       if (sm->num_workers > 1)
1104         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1105       else
1106         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1107
1108       nat_ed_static_mapping_del_sessions (
1109         sm, tsm, m->local_addr, m->local_port, m->proto, fib_index,
1110         is_sm_addr_only (flags), e_addr, e_port);
1111     }
1112
1113   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1114
1115   if (!pool_elts (m->locals))
1116     {
1117       // this is last record remove all required stuff
1118       // fib_index 0
1119       init_nat_k (&kv, e_addr, e_port, 0, proto);
1120       clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1121
1122       vec_free (m->tag);
1123       vec_free (m->workers);
1124       pool_put (sm->static_mappings, m);
1125
1126       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
1127         {
1128           nat44_ed_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
1129         }
1130     }
1131
1132   return 0;
1133 }
1134
1135 int
1136 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1137                                 nat_protocol_t proto,
1138                                 nat44_lb_addr_port_t *locals, u32 flags,
1139                                 u8 *tag, u32 affinity)
1140 {
1141   snat_main_t *sm = &snat_main;
1142   snat_static_mapping_t *m;
1143   clib_bihash_kv_8_8_t kv, value;
1144   snat_address_t *a = 0;
1145
1146   nat44_lb_addr_port_t *local;
1147   uword *bitmap = 0;
1148
1149   int i;
1150
1151   if (!sm->enabled)
1152     {
1153       return VNET_API_ERROR_UNSUPPORTED;
1154     }
1155
1156   init_nat_k (&kv, e_addr, e_port, 0, proto);
1157   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1158     {
1159       m = 0;
1160     }
1161   else
1162     {
1163       m = pool_elt_at_index (sm->static_mappings, value.value);
1164     }
1165
1166   if (m)
1167     {
1168       return VNET_API_ERROR_VALUE_EXIST;
1169     }
1170
1171   if (vec_len (locals) < 2)
1172     {
1173       return VNET_API_ERROR_INVALID_VALUE;
1174     }
1175
1176   if (!(sm->static_mapping_only || is_sm_out2in_only (flags)))
1177     {
1178       /* Find external address in allocated addresses and reserve port for
1179          address and port pair mapping when dynamic translations enabled */
1180       for (i = 0; i < vec_len (sm->addresses); i++)
1181         {
1182           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1183             {
1184               a = sm->addresses + i;
1185               /* External port must be unused */
1186               switch (proto)
1187                 {
1188 #define _(N, j, n, s)                                                         \
1189   case NAT_PROTOCOL_##N:                                                      \
1190     if (a->busy_##n##_port_refcounts[e_port])                                 \
1191       return VNET_API_ERROR_INVALID_VALUE;                                    \
1192     ++a->busy_##n##_port_refcounts[e_port];                                   \
1193     if (e_port > 1024)                                                        \
1194       {                                                                       \
1195         a->busy_##n##_ports++;                                                \
1196         a->busy_##n##_ports_per_thread[get_thread_idx_by_port (e_port)]++;    \
1197       }                                                                       \
1198     break;
1199                   foreach_nat_protocol
1200 #undef _
1201                     default : nat_elog_info (sm, "unknown protocol");
1202                   return VNET_API_ERROR_INVALID_VALUE_2;
1203                 }
1204               break;
1205             }
1206         }
1207       // external address must be allocated
1208       if (!a)
1209         {
1210           return VNET_API_ERROR_NO_SUCH_ENTRY;
1211         }
1212     }
1213
1214   pool_get (sm->static_mappings, m);
1215   clib_memset (m, 0, sizeof (*m));
1216   m->tag = vec_dup (tag);
1217   m->external_addr = e_addr;
1218   m->external_port = e_port;
1219   m->affinity = affinity;
1220   m->proto = proto;
1221
1222   m->flags = flags;
1223   m->flags |= NAT_SM_FLAG_LB;
1224
1225   if (affinity)
1226     m->affinity_per_service_list_head_index =
1227       nat_affinity_get_per_service_list_head_index ();
1228   else
1229     m->affinity_per_service_list_head_index = ~0;
1230
1231   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
1232                m - sm->static_mappings);
1233   if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1234     {
1235       nat_elog_err (sm, "static_mapping_by_external key add failed");
1236       return VNET_API_ERROR_UNSPECIFIED;
1237     }
1238
1239   for (i = 0; i < vec_len (locals); i++)
1240     {
1241       locals[i].fib_index = fib_table_find_or_create_and_lock (
1242         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1243       if (!is_sm_out2in_only (flags))
1244         {
1245           init_nat_kv (&kv, locals[i].addr, locals[i].port,
1246                        locals[i].fib_index, m->proto, 0,
1247                        m - sm->static_mappings);
1248           clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1249         }
1250       locals[i].prefix = (i == 0) ?
1251                            locals[i].probability :
1252                            (locals[i - 1].prefix + locals[i].probability);
1253       pool_get (m->locals, local);
1254       *local = locals[i];
1255       if (sm->num_workers > 1)
1256         {
1257           ip4_header_t ip = {
1258             .src_address = locals[i].addr,
1259           };
1260           bitmap = clib_bitmap_set (
1261             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1262             1);
1263         }
1264     }
1265
1266   /* Assign workers */
1267   if (sm->num_workers > 1)
1268     {
1269       clib_bitmap_foreach (i, bitmap)
1270         {
1271           vec_add1 (m->workers, i);
1272         }
1273     }
1274
1275   return 0;
1276 }
1277
1278 int
1279 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1280                                 nat_protocol_t proto, u32 flags)
1281 {
1282   snat_main_t *sm = &snat_main;
1283   snat_static_mapping_t *m;
1284   clib_bihash_kv_8_8_t kv, value;
1285   snat_address_t *a = 0;
1286
1287   nat44_lb_addr_port_t *local;
1288   snat_main_per_thread_data_t *tsm;
1289   snat_session_t *s;
1290   int i;
1291
1292   if (!sm->enabled)
1293     {
1294       return VNET_API_ERROR_UNSUPPORTED;
1295     }
1296
1297   init_nat_k (&kv, e_addr, e_port, 0, proto);
1298   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1299     m = 0;
1300   else
1301     m = pool_elt_at_index (sm->static_mappings, value.value);
1302
1303   if (!m)
1304     return VNET_API_ERROR_NO_SUCH_ENTRY;
1305
1306   if (!is_sm_lb (m->flags))
1307     return VNET_API_ERROR_INVALID_VALUE;
1308
1309   if (!(sm->static_mapping_only || is_sm_out2in_only (flags)))
1310     {
1311       for (i = 0; i < vec_len (sm->addresses); i++)
1312         {
1313           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1314             {
1315               a = sm->addresses + i;
1316               switch (proto)
1317                 {
1318 #define _(N, j, n, s)                                                         \
1319   case NAT_PROTOCOL_##N:                                                      \
1320     --a->busy_##n##_port_refcounts[e_port];                                   \
1321     if (e_port > 1024)                                                        \
1322       {                                                                       \
1323         a->busy_##n##_ports--;                                                \
1324         a->busy_##n##_ports_per_thread[get_thread_idx_by_port (e_port)]--;    \
1325       }                                                                       \
1326     break;
1327                   foreach_nat_protocol
1328 #undef _
1329                     default : nat_elog_info (sm, "unknown protocol");
1330                   return VNET_API_ERROR_INVALID_VALUE_2;
1331                 }
1332               break;
1333             }
1334         }
1335     }
1336
1337   init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1338   if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1339     {
1340       nat_elog_err (sm, "static_mapping_by_external key del failed");
1341       return VNET_API_ERROR_UNSPECIFIED;
1342     }
1343
1344   pool_foreach (local, m->locals)
1345     {
1346       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1347       if (!is_sm_out2in_only (flags))
1348         {
1349           init_nat_k (&kv, local->addr, local->port, local->fib_index,
1350                       m->proto);
1351           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1352             {
1353               nat_elog_err (sm, "static_mapping_by_local key del failed");
1354               return VNET_API_ERROR_UNSPECIFIED;
1355             }
1356         }
1357
1358       if (sm->num_workers > 1)
1359         {
1360           ip4_header_t ip = {
1361             .src_address = local->addr,
1362           };
1363           tsm = vec_elt_at_index (
1364             sm->per_thread_data,
1365             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1366         }
1367       else
1368         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1369
1370       /* Delete sessions */
1371       pool_foreach (s, tsm->sessions)
1372         {
1373           if (!(nat44_ed_is_lb_session (s)))
1374             continue;
1375
1376           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1377               s->in2out.port != local->port)
1378             continue;
1379
1380           nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1381           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1382         }
1383     }
1384
1385   if (m->affinity)
1386     {
1387       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1388     }
1389
1390   pool_free (m->locals);
1391   vec_free (m->tag);
1392   vec_free (m->workers);
1393   pool_put (sm->static_mappings, m);
1394
1395   return 0;
1396 }
1397
1398 int
1399 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1400                                           ip4_address_t l_addr, u16 l_port,
1401                                           nat_protocol_t proto, u32 vrf_id,
1402                                           u8 probability, u8 is_add)
1403 {
1404   snat_main_t *sm = &snat_main;
1405   snat_static_mapping_t *m = 0;
1406   clib_bihash_kv_8_8_t kv, value;
1407   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1408   snat_main_per_thread_data_t *tsm;
1409   snat_session_t *s;
1410   u32 *locals = 0;
1411   uword *bitmap = 0;
1412   int i;
1413
1414   if (!sm->enabled)
1415     {
1416       return VNET_API_ERROR_UNSUPPORTED;
1417     }
1418
1419   init_nat_k (&kv, e_addr, e_port, 0, proto);
1420   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1421     m = pool_elt_at_index (sm->static_mappings, value.value);
1422
1423   if (!m)
1424     {
1425       return VNET_API_ERROR_NO_SUCH_ENTRY;
1426     }
1427
1428   if (!is_sm_lb (m->flags))
1429     {
1430       return VNET_API_ERROR_INVALID_VALUE;
1431     }
1432
1433   pool_foreach (local, m->locals)
1434    {
1435     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1436         (local->vrf_id == vrf_id))
1437       {
1438         match_local = local;
1439         break;
1440       }
1441   }
1442
1443   if (is_add)
1444     {
1445       if (match_local)
1446         {
1447           return VNET_API_ERROR_VALUE_EXIST;
1448         }
1449
1450       pool_get (m->locals, local);
1451       clib_memset (local, 0, sizeof (*local));
1452       local->addr.as_u32 = l_addr.as_u32;
1453       local->port = l_port;
1454       local->probability = probability;
1455       local->vrf_id = vrf_id;
1456       local->fib_index =
1457         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1458                                            sm->fib_src_low);
1459
1460       if (!is_sm_out2in_only (m->flags))
1461         {
1462           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto, 0,
1463                        m - sm->static_mappings);
1464           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1465             nat_elog_err (sm, "static_mapping_by_local key add failed");
1466         }
1467     }
1468   else
1469     {
1470       if (!match_local)
1471         return VNET_API_ERROR_NO_SUCH_ENTRY;
1472
1473       if (pool_elts (m->locals) < 3)
1474         return VNET_API_ERROR_UNSPECIFIED;
1475
1476       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1477                         sm->fib_src_low);
1478
1479       if (!is_sm_out2in_only (m->flags))
1480         {
1481           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1482           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1483             nat_elog_err (sm, "static_mapping_by_local key del failed");
1484         }
1485
1486       if (sm->num_workers > 1)
1487         {
1488           ip4_header_t ip = {
1489             .src_address = local->addr,
1490           };
1491           tsm = vec_elt_at_index (
1492             sm->per_thread_data,
1493             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1494         }
1495       else
1496         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1497
1498       /* Delete sessions */
1499       pool_foreach (s, tsm->sessions) {
1500           if (!(nat44_ed_is_lb_session (s)))
1501             continue;
1502
1503           if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1504               s->in2out.port != match_local->port)
1505             continue;
1506
1507           nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1508           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1509       }
1510
1511       pool_put (m->locals, match_local);
1512     }
1513
1514   vec_free (m->workers);
1515
1516   pool_foreach (local, m->locals)
1517    {
1518     vec_add1 (locals, local - m->locals);
1519     if (sm->num_workers > 1)
1520       {
1521         ip4_header_t ip;
1522         ip.src_address.as_u32 = local->addr.as_u32,
1523         bitmap = clib_bitmap_set (
1524           bitmap,
1525           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1526       }
1527   }
1528
1529   ASSERT (vec_len (locals) > 1);
1530
1531   local = pool_elt_at_index (m->locals, locals[0]);
1532   local->prefix = local->probability;
1533   for (i = 1; i < vec_len (locals); i++)
1534     {
1535       local = pool_elt_at_index (m->locals, locals[i]);
1536       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1537       local->prefix = local->probability + prev_local->prefix;
1538     }
1539
1540   /* Assign workers */
1541   if (sm->num_workers > 1)
1542     {
1543       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1544     }
1545
1546   return 0;
1547 }
1548
1549 void
1550 expire_per_vrf_sessions (u32 fib_index)
1551 {
1552   per_vrf_sessions_t *per_vrf_sessions;
1553   snat_main_per_thread_data_t *tsm;
1554   snat_main_t *sm = &snat_main;
1555
1556   vec_foreach (tsm, sm->per_thread_data)
1557     {
1558       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1559         {
1560           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1561               (per_vrf_sessions->tx_fib_index == fib_index))
1562             {
1563               per_vrf_sessions->expired = 1;
1564             }
1565         }
1566     }
1567 }
1568
1569 void
1570 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1571 {
1572   snat_main_t *sm = &snat_main;
1573   nat_fib_t *fib;
1574
1575   // we don't care if it is outside/inside fib
1576   // we just care about their ref_count
1577   // if it reaches 0 sessions should expire
1578   // because the fib isn't valid for NAT anymore
1579
1580   vec_foreach (fib, sm->fibs)
1581   {
1582     if (fib->fib_index == fib_index)
1583       {
1584         if (is_del)
1585           {
1586             fib->ref_count--;
1587             if (!fib->ref_count)
1588               {
1589                 vec_del1 (sm->fibs, fib - sm->fibs);
1590                 expire_per_vrf_sessions (fib_index);
1591               }
1592             return;
1593           }
1594         else
1595           fib->ref_count++;
1596       }
1597   }
1598   if (!is_del)
1599     {
1600       vec_add2 (sm->fibs, fib, 1);
1601       fib->ref_count = 1;
1602       fib->fib_index = fib_index;
1603     }
1604 }
1605
1606 static_always_inline nat_outside_fib_t *
1607 nat44_ed_get_outside_fib (nat_outside_fib_t *outside_fibs, u32 fib_index)
1608 {
1609   nat_outside_fib_t *f;
1610   vec_foreach (f, outside_fibs)
1611     {
1612       if (f->fib_index == fib_index)
1613         {
1614           return f;
1615         }
1616     }
1617   return 0;
1618 }
1619
1620 static_always_inline snat_interface_t *
1621 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1622 {
1623   snat_interface_t *i;
1624   pool_foreach (i, interfaces)
1625     {
1626       if (i->sw_if_index == sw_if_index)
1627         {
1628           return i;
1629         }
1630     }
1631   return 0;
1632 }
1633
1634 int
1635 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1636 {
1637   const char *del_feature_name, *feature_name;
1638   snat_main_t *sm = &snat_main;
1639
1640   nat_outside_fib_t *outside_fib;
1641   snat_interface_t *i;
1642   u32 fib_index;
1643   int rv;
1644
1645   if (!sm->enabled)
1646     {
1647       nat_log_err ("nat44 is disabled");
1648       return VNET_API_ERROR_UNSUPPORTED;
1649     }
1650
1651   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1652     {
1653       nat_log_err ("error interface already configured");
1654       return VNET_API_ERROR_VALUE_EXIST;
1655     }
1656
1657   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1658   if (i)
1659     {
1660       if ((nat44_ed_is_interface_inside (i) && is_inside) ||
1661           (nat44_ed_is_interface_outside (i) && !is_inside))
1662         {
1663           return 0;
1664         }
1665       if (sm->num_workers > 1)
1666         {
1667           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1668                                           "nat44-out2in-worker-handoff";
1669           feature_name = "nat44-handoff-classify";
1670         }
1671       else
1672         {
1673           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1674
1675           feature_name = "nat44-ed-classify";
1676         }
1677
1678       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1679       if (rv)
1680         return rv;
1681       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1682                                    sw_if_index, 0, 0, 0);
1683       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1684                                    0, 0);
1685     }
1686   else
1687     {
1688       if (sm->num_workers > 1)
1689         {
1690           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1691                                      "nat44-out2in-worker-handoff";
1692         }
1693       else
1694         {
1695           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1696         }
1697
1698       nat_validate_interface_counters (sm, sw_if_index);
1699       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1700       if (rv)
1701         return rv;
1702       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1703                                    0, 0);
1704
1705       pool_get (sm->interfaces, i);
1706       i->sw_if_index = sw_if_index;
1707       i->flags = 0;
1708     }
1709
1710   fib_index =
1711     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1712
1713   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1714
1715   if (!is_inside)
1716     {
1717       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1718
1719       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1720       if (outside_fib)
1721         {
1722           outside_fib->refcount++;
1723         }
1724       else
1725         {
1726           vec_add2 (sm->outside_fibs, outside_fib, 1);
1727           outside_fib->fib_index = fib_index;
1728           outside_fib->refcount = 1;
1729         }
1730
1731       nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1732       nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1733     }
1734   else
1735     {
1736       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1737     }
1738
1739   return 0;
1740 }
1741
1742 int
1743 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1744 {
1745   const char *del_feature_name, *feature_name;
1746   snat_main_t *sm = &snat_main;
1747
1748   nat_outside_fib_t *outside_fib;
1749   snat_interface_t *i;
1750   u32 fib_index;
1751   int rv;
1752
1753   if (!sm->enabled)
1754     {
1755       nat_log_err ("nat44 is disabled");
1756       return VNET_API_ERROR_UNSUPPORTED;
1757     }
1758
1759   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1760   if (i == 0)
1761     {
1762       nat_log_err ("error interface couldn't be found");
1763       return VNET_API_ERROR_NO_SUCH_ENTRY;
1764     }
1765
1766   if (nat44_ed_is_interface_inside (i) && nat44_ed_is_interface_outside (i))
1767     {
1768       if (sm->num_workers > 1)
1769         {
1770           del_feature_name = "nat44-handoff-classify";
1771           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1772                                       "nat44-out2in-worker-handoff";
1773         }
1774       else
1775         {
1776           del_feature_name = "nat44-ed-classify";
1777           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1778         }
1779
1780       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1781       if (rv)
1782         {
1783           return rv;
1784         }
1785       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1786                                    sw_if_index, 0, 0, 0);
1787       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1788                                    0, 0);
1789
1790       if (is_inside)
1791         {
1792           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1793         }
1794       else
1795         {
1796           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1797         }
1798     }
1799   else
1800     {
1801       if (sm->num_workers > 1)
1802         {
1803           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1804                                      "nat44-out2in-worker-handoff";
1805         }
1806       else
1807         {
1808           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1809         }
1810
1811       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1812       if (rv)
1813         {
1814           return rv;
1815         }
1816       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1817                                    0, 0);
1818
1819       // remove interface
1820       pool_put (sm->interfaces, i);
1821     }
1822
1823   fib_index =
1824     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1825
1826   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
1827
1828   if (!is_inside)
1829     {
1830       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1831       if (outside_fib)
1832         {
1833           outside_fib->refcount--;
1834           if (!outside_fib->refcount)
1835             {
1836               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1837             }
1838         }
1839
1840       nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
1841       nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
1842     }
1843
1844   return 0;
1845 }
1846
1847 int
1848 nat44_ed_add_output_interface (u32 sw_if_index)
1849 {
1850   snat_main_t *sm = &snat_main;
1851
1852   nat_outside_fib_t *outside_fib;
1853   snat_interface_t *i;
1854   u32 fib_index;
1855   int rv;
1856
1857   if (!sm->enabled)
1858     {
1859       nat_log_err ("nat44 is disabled");
1860       return VNET_API_ERROR_UNSUPPORTED;
1861     }
1862
1863   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1864     {
1865       nat_log_err ("error interface already configured");
1866       return VNET_API_ERROR_VALUE_EXIST;
1867     }
1868
1869   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1870     {
1871       nat_log_err ("error interface already configured");
1872       return VNET_API_ERROR_VALUE_EXIST;
1873     }
1874
1875   if (sm->num_workers > 1)
1876     {
1877       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1878       if (rv)
1879         {
1880           return rv;
1881         }
1882
1883       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1884       if (rv)
1885         {
1886           return rv;
1887         }
1888
1889       vnet_feature_enable_disable (
1890         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1891       vnet_feature_enable_disable ("ip4-output",
1892                                    "nat44-in2out-output-worker-handoff",
1893                                    sw_if_index, 1, 0, 0);
1894     }
1895   else
1896     {
1897       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1898       if (rv)
1899         {
1900           return rv;
1901         }
1902
1903       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1904       if (rv)
1905         {
1906           return rv;
1907         }
1908
1909       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1910                                    sw_if_index, 1, 0, 0);
1911       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1912                                    sw_if_index, 1, 0, 0);
1913     }
1914
1915   nat_validate_interface_counters (sm, sw_if_index);
1916
1917   pool_get (sm->output_feature_interfaces, i);
1918   i->sw_if_index = sw_if_index;
1919   i->flags = 0;
1920   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1921   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1922
1923   fib_index =
1924     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1925   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1926
1927   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1928   if (outside_fib)
1929     {
1930       outside_fib->refcount++;
1931     }
1932   else
1933     {
1934       vec_add2 (sm->outside_fibs, outside_fib, 1);
1935       outside_fib->fib_index = fib_index;
1936       outside_fib->refcount = 1;
1937     }
1938
1939   nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1940   nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1941
1942   return 0;
1943 }
1944
1945 int
1946 nat44_ed_del_output_interface (u32 sw_if_index)
1947 {
1948   snat_main_t *sm = &snat_main;
1949
1950   nat_outside_fib_t *outside_fib;
1951   snat_interface_t *i;
1952   u32 fib_index;
1953   int rv;
1954
1955   if (!sm->enabled)
1956     {
1957       nat_log_err ("nat44 is disabled");
1958       return VNET_API_ERROR_UNSUPPORTED;
1959     }
1960
1961   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
1962   if (!i)
1963     {
1964       nat_log_err ("error interface couldn't be found");
1965       return VNET_API_ERROR_NO_SUCH_ENTRY;
1966     }
1967
1968   if (sm->num_workers > 1)
1969     {
1970       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1971       if (rv)
1972         {
1973           return rv;
1974         }
1975
1976       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1977       if (rv)
1978         {
1979           return rv;
1980         }
1981
1982       vnet_feature_enable_disable (
1983         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
1984       vnet_feature_enable_disable ("ip4-output",
1985                                    "nat44-in2out-output-worker-handoff",
1986                                    sw_if_index, 0, 0, 0);
1987     }
1988   else
1989     {
1990       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1991       if (rv)
1992         {
1993           return rv;
1994         }
1995
1996       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
1997       if (rv)
1998         {
1999           return rv;
2000         }
2001
2002       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2003                                    sw_if_index, 0, 0, 0);
2004       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2005                                    sw_if_index, 0, 0, 0);
2006     }
2007
2008   // remove interface
2009   pool_put (sm->output_feature_interfaces, i);
2010
2011   fib_index =
2012     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2013   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
2014
2015   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2016   if (outside_fib)
2017     {
2018       outside_fib->refcount--;
2019       if (!outside_fib->refcount)
2020         {
2021           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2022         }
2023     }
2024
2025   nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
2026   nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
2027
2028   return 0;
2029 }
2030
2031 int
2032 snat_set_workers (uword * bitmap)
2033 {
2034   snat_main_t *sm = &snat_main;
2035   int i, j = 0;
2036
2037   if (sm->num_workers < 2)
2038     return VNET_API_ERROR_FEATURE_DISABLED;
2039
2040   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2041     return VNET_API_ERROR_INVALID_WORKER;
2042
2043   vec_free (sm->workers);
2044   clib_bitmap_foreach (i, bitmap)
2045     {
2046       vec_add1(sm->workers, i);
2047       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2048       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2049       j++;
2050     }
2051
2052   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2053
2054   return 0;
2055 }
2056
2057 int
2058 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2059 {
2060   fail_if_enabled ();
2061   snat_main_t *sm = &snat_main;
2062   sm->frame_queue_nelts = frame_queue_nelts;
2063   return 0;
2064 }
2065
2066 static void
2067 nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque, u32 sw_if_index,
2068                                 u32 new_fib_index, u32 old_fib_index)
2069 {
2070   snat_main_t *sm = &snat_main;
2071   nat_outside_fib_t *outside_fib;
2072   snat_interface_t *i;
2073   u8 is_add = 1;
2074   u8 match = 0;
2075
2076   if (!sm->enabled || (new_fib_index == old_fib_index)
2077       || (!vec_len (sm->outside_fibs)))
2078     {
2079       return;
2080     }
2081
2082   pool_foreach (i, sm->interfaces)
2083      {
2084       if (i->sw_if_index == sw_if_index)
2085         {
2086           if (!(nat44_ed_is_interface_outside (i)))
2087             return;
2088           match = 1;
2089         }
2090     }
2091
2092   pool_foreach (i, sm->output_feature_interfaces)
2093      {
2094       if (i->sw_if_index == sw_if_index)
2095         {
2096           if (!(nat44_ed_is_interface_outside (i)))
2097             return;
2098           match = 1;
2099         }
2100     }
2101
2102   if (!match)
2103     return;
2104
2105   vec_foreach (outside_fib, sm->outside_fibs)
2106   {
2107     if (outside_fib->fib_index == old_fib_index)
2108       {
2109         outside_fib->refcount--;
2110         if (!outside_fib->refcount)
2111           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2112         break;
2113       }
2114   }
2115
2116   vec_foreach (outside_fib, sm->outside_fibs)
2117   {
2118     if (outside_fib->fib_index == new_fib_index)
2119       {
2120         outside_fib->refcount++;
2121         is_add = 0;
2122         break;
2123       }
2124   }
2125
2126   if (is_add)
2127     {
2128       vec_add2 (sm->outside_fibs, outside_fib, 1);
2129       outside_fib->refcount = 1;
2130       outside_fib->fib_index = new_fib_index;
2131     }
2132 }
2133
2134 static void nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque,
2135                                             u32 sw_if_index, u32 new_fib_index,
2136                                             u32 old_fib_index);
2137
2138 static void nat44_ed_add_del_interface_address_cb (
2139   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2140   u32 address_length, u32 if_address_index, u32 is_delete);
2141
2142 static void nat44_ed_add_del_static_mapping_addr_only_cb (
2143   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2144   u32 address_length, u32 if_address_index, u32 is_delete);
2145
2146 void
2147 test_key_calc_split ()
2148 {
2149   ip4_address_t l_addr;
2150   l_addr.as_u8[0] = 1;
2151   l_addr.as_u8[1] = 1;
2152   l_addr.as_u8[2] = 1;
2153   l_addr.as_u8[3] = 1;
2154   ip4_address_t r_addr;
2155   r_addr.as_u8[0] = 2;
2156   r_addr.as_u8[1] = 2;
2157   r_addr.as_u8[2] = 2;
2158   r_addr.as_u8[3] = 2;
2159   u16 l_port = 40001;
2160   u16 r_port = 40301;
2161   u8 proto = 9;
2162   u32 fib_index = 9000001;
2163   u32 thread_index = 3000000001;
2164   u32 session_index = 3000000221;
2165   clib_bihash_kv_16_8_t kv;
2166   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2167               thread_index, session_index);
2168   ip4_address_t l_addr2;
2169   ip4_address_t r_addr2;
2170   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2171   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2172   u16 l_port2 = 0;
2173   u16 r_port2 = 0;
2174   u8 proto2 = 0;
2175   u32 fib_index2 = 0;
2176   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2177                &r_port2);
2178   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2179   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2180   ASSERT (l_port == l_port2);
2181   ASSERT (r_port == r_port2);
2182   ASSERT (proto == proto2);
2183   ASSERT (fib_index == fib_index2);
2184   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2185   ASSERT (session_index == ed_value_get_session_index (&kv));
2186
2187   fib_index = 7001;
2188   proto = 5;
2189   nat_protocol_t proto3 = ~0;
2190   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2191   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2192   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2193   ASSERT (l_port == l_port2);
2194   ASSERT (proto == proto3);
2195   ASSERT (fib_index == fib_index2);
2196 }
2197
2198 static clib_error_t *
2199 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2200 {
2201   u32 fib_index;
2202   if (!is_add)
2203     {
2204       fib_index = ip4_fib_index_from_table_id (table_id);
2205       if (fib_index != ~0)
2206         {
2207           expire_per_vrf_sessions (fib_index);
2208         }
2209     }
2210   return 0;
2211 }
2212
2213 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2214
2215 void
2216 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2217 {
2218   vlib_node_t *node;
2219
2220   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2221   sm->out2in_node_index = node->index;
2222
2223   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2224   sm->in2out_node_index = node->index;
2225
2226   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2227   sm->in2out_output_node_index = node->index;
2228 }
2229
2230 #define nat_validate_simple_counter(c, i)                                     \
2231   do                                                                          \
2232     {                                                                         \
2233       vlib_validate_simple_counter (&c, i);                                   \
2234       vlib_zero_simple_counter (&c, i);                                       \
2235     }                                                                         \
2236   while (0);
2237
2238 #define nat_init_simple_counter(c, n, sn)                                     \
2239   do                                                                          \
2240     {                                                                         \
2241       c.name = n;                                                             \
2242       c.stat_segment_name = sn;                                               \
2243       nat_validate_simple_counter (c, 0);                                     \
2244     }                                                                         \
2245   while (0);
2246
2247 static_always_inline void
2248 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2249 {
2250 #define _(x)                                                                  \
2251   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2252   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2253   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2254   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2255   foreach_nat_counter;
2256 #undef _
2257   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2258 }
2259
2260 static clib_error_t *
2261 nat_init (vlib_main_t * vm)
2262 {
2263   snat_main_t *sm = &snat_main;
2264   vlib_thread_main_t *tm = vlib_get_thread_main ();
2265   vlib_thread_registration_t *tr;
2266   ip4_add_del_interface_address_callback_t cbi = { 0 };
2267   ip4_table_bind_callback_t cbt = { 0 };
2268   u32 i, num_threads = 0;
2269   uword *p, *bitmap = 0;
2270
2271   clib_memset (sm, 0, sizeof (*sm));
2272
2273   // required
2274   sm->vnet_main = vnet_get_main ();
2275   // convenience
2276   sm->ip4_main = &ip4_main;
2277   sm->api_main = vlibapi_get_main ();
2278   sm->ip4_lookup_main = &ip4_main.lookup_main;
2279
2280   // frame queue indices used for handoff
2281   sm->fq_out2in_index = ~0;
2282   sm->fq_in2out_index = ~0;
2283   sm->fq_in2out_output_index = ~0;
2284
2285   sm->log_level = NAT_LOG_ERROR;
2286
2287   nat44_set_node_indexes (sm, vm);
2288
2289   sm->log_class = vlib_log_register_class ("nat", 0);
2290   nat_ipfix_logging_init (vm);
2291
2292   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2293                            "/nat44-ed/total-sessions");
2294   sm->max_cfg_sessions_gauge = stat_segment_new_entry (
2295     (u8 *) "/nat44-ed/max-cfg-sessions", STAT_DIR_TYPE_SCALAR_INDEX);
2296
2297 #define _(x)                                                                  \
2298   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2299                            "/nat44-ed/in2out/fastpath/" #x);                  \
2300   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2301                            "/nat44-ed/out2in/fastpath/" #x);                  \
2302   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2303                            "/nat44-ed/in2out/slowpath/" #x);                  \
2304   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2305                            "/nat44-ed/out2in/slowpath/" #x);
2306   foreach_nat_counter;
2307 #undef _
2308   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2309                            "/nat44-ed/hairpinning");
2310
2311   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2312   if (p)
2313     {
2314       tr = (vlib_thread_registration_t *) p[0];
2315       if (tr)
2316         {
2317           sm->num_workers = tr->count;
2318           sm->first_worker_index = tr->first_index;
2319         }
2320     }
2321   num_threads = tm->n_vlib_mains - 1;
2322   sm->port_per_thread = 0xffff - 1024;
2323   vec_validate (sm->per_thread_data, num_threads);
2324
2325   /* Use all available workers by default */
2326   if (sm->num_workers > 1)
2327     {
2328       for (i = 0; i < sm->num_workers; i++)
2329         bitmap = clib_bitmap_set (bitmap, i, 1);
2330       snat_set_workers (bitmap);
2331       clib_bitmap_free (bitmap);
2332     }
2333   else
2334     {
2335       sm->per_thread_data[0].snat_thread_index = 0;
2336     }
2337
2338   /* callbacks to call when interface address changes. */
2339   cbi.function = nat44_ed_add_del_interface_address_cb;
2340   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2341   cbi.function = nat44_ed_add_del_static_mapping_addr_only_cb;
2342   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2343
2344   /* callbacks to call when interface to table biding changes */
2345   cbt.function = nat44_ed_update_outside_fib_cb;
2346   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2347
2348   sm->fib_src_low =
2349     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2350                          FIB_SOURCE_BH_SIMPLE);
2351   sm->fib_src_hi =
2352     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2353                          FIB_SOURCE_BH_SIMPLE);
2354
2355   nat_affinity_init (vm);
2356   test_key_calc_split ();
2357
2358   return nat44_api_hookup (vm);
2359 }
2360
2361 VLIB_INIT_FUNCTION (nat_init);
2362
2363 int
2364 nat44_plugin_enable (nat44_config_t c)
2365 {
2366   snat_main_t *sm = &snat_main;
2367
2368   fail_if_enabled ();
2369
2370   if (c.static_mapping_only && !c.connection_tracking)
2371     {
2372       nat_log_err ("unsupported combination of configuration");
2373       return 1;
2374     }
2375
2376   sm->static_mapping_only = c.static_mapping_only;
2377   sm->static_mapping_connection_tracking = c.connection_tracking;
2378
2379   sm->forwarding_enabled = 0;
2380   sm->mss_clamping = 0;
2381   sm->pat = (!c.static_mapping_only ||
2382              (c.static_mapping_only && c.connection_tracking));
2383
2384   if (!c.sessions)
2385     c.sessions = 63 * 1024;
2386
2387   sm->max_translations_per_thread = c.sessions;
2388   stat_segment_set_state_counter (sm->max_cfg_sessions_gauge,
2389                                   sm->max_translations_per_thread);
2390   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2391
2392   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2393
2394   sm->inside_vrf_id = c.inside_vrf;
2395   sm->inside_fib_index =
2396     fib_table_find_or_create_and_lock
2397     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2398
2399   sm->outside_vrf_id = c.outside_vrf;
2400   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2401     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2402
2403   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets);
2404
2405   nat_affinity_enable ();
2406
2407   nat_reset_timeouts (&sm->timeouts);
2408
2409   vlib_zero_simple_counter (&sm->total_sessions, 0);
2410
2411   if (!sm->frame_queue_nelts)
2412     {
2413       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2414     }
2415
2416   if (sm->num_workers > 1)
2417     {
2418       if (sm->fq_in2out_index == ~0)
2419         {
2420           sm->fq_in2out_index = vlib_frame_queue_main_init (
2421             sm->in2out_node_index, sm->frame_queue_nelts);
2422         }
2423       if (sm->fq_out2in_index == ~0)
2424         {
2425           sm->fq_out2in_index = vlib_frame_queue_main_init (
2426             sm->out2in_node_index, sm->frame_queue_nelts);
2427         }
2428       if (sm->fq_in2out_output_index == ~0)
2429         {
2430           sm->fq_in2out_output_index = vlib_frame_queue_main_init (
2431             sm->in2out_output_node_index, sm->frame_queue_nelts);
2432         }
2433     }
2434
2435   sm->enabled = 1;
2436   sm->rconfig = c;
2437
2438   return 0;
2439 }
2440
2441 void
2442 nat44_addresses_free (snat_address_t ** addresses)
2443 {
2444   snat_address_t *ap;
2445   vec_foreach (ap, *addresses)
2446     {
2447     #define _(N, i, n, s) \
2448       vec_free (ap->busy_##n##_ports_per_thread);
2449       foreach_nat_protocol
2450     #undef _
2451     }
2452   vec_free (*addresses);
2453   *addresses = 0;
2454 }
2455
2456 int
2457 nat44_plugin_disable ()
2458 {
2459   snat_main_t *sm = &snat_main;
2460   snat_interface_t *i, *pool;
2461   int error = 0;
2462
2463   fail_if_disabled ();
2464
2465   pool = pool_dup (sm->interfaces);
2466   pool_foreach (i, pool)
2467     {
2468       if (nat44_ed_is_interface_inside (i))
2469         {
2470           error = nat44_ed_del_interface (i->sw_if_index, 1);
2471         }
2472       if (nat44_ed_is_interface_outside (i))
2473         {
2474           error = nat44_ed_del_interface (i->sw_if_index, 0);
2475         }
2476       if (error)
2477         {
2478           nat_log_err ("error occurred while removing interface %u",
2479                        i->sw_if_index);
2480         }
2481     }
2482   pool_free (sm->interfaces);
2483   pool_free (pool);
2484   sm->interfaces = 0;
2485
2486   pool = pool_dup (sm->output_feature_interfaces);
2487   pool_foreach (i, pool)
2488     {
2489       error = nat44_ed_del_output_interface (i->sw_if_index);
2490       if (error)
2491         {
2492           nat_log_err ("error occurred while removing interface %u",
2493                        i->sw_if_index);
2494         }
2495     }
2496   pool_free (sm->output_feature_interfaces);
2497   pool_free (pool);
2498   sm->output_feature_interfaces = 0;
2499
2500   vec_free (sm->max_translations_per_fib);
2501
2502   nat44_ed_db_free ();
2503
2504   nat44_addresses_free (&sm->addresses);
2505   nat44_addresses_free (&sm->twice_nat_addresses);
2506
2507   vec_free (sm->to_resolve);
2508   vec_free (sm->auto_add_sw_if_indices);
2509   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2510
2511   sm->to_resolve = 0;
2512   sm->auto_add_sw_if_indices = 0;
2513   sm->auto_add_sw_if_indices_twice_nat = 0;
2514
2515   sm->forwarding_enabled = 0;
2516
2517   sm->enabled = 0;
2518   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2519
2520   return 0;
2521 }
2522
2523 void
2524 nat44_ed_forwarding_enable_disable (u8 is_enable)
2525 {
2526   snat_main_per_thread_data_t *tsm;
2527   snat_main_t *sm = &snat_main;
2528   snat_session_t *s;
2529
2530   u32 *ses_to_be_removed = 0, *ses_index;
2531
2532   sm->forwarding_enabled = is_enable != 0;
2533
2534   if (!sm->enabled || is_enable)
2535     {
2536       return;
2537     }
2538
2539   vec_foreach (tsm, sm->per_thread_data)
2540     {
2541       pool_foreach (s, tsm->sessions)
2542         {
2543           if (na44_ed_is_fwd_bypass_session (s))
2544             {
2545               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2546             }
2547         }
2548       vec_foreach (ses_index, ses_to_be_removed)
2549         {
2550           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2551           nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2552           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2553         }
2554
2555       vec_free (ses_to_be_removed);
2556     }
2557 }
2558
2559 void
2560 snat_free_outside_address_and_port (snat_address_t *addresses,
2561                                     u32 thread_index, ip4_address_t *addr,
2562                                     u16 port, nat_protocol_t protocol)
2563 {
2564   snat_main_t *sm = &snat_main;
2565   snat_address_t *a;
2566   u32 address_index;
2567   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2568
2569   for (address_index = 0; address_index < vec_len (addresses);
2570        address_index++)
2571     {
2572       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2573         break;
2574     }
2575
2576   ASSERT (address_index < vec_len (addresses));
2577
2578   a = addresses + address_index;
2579
2580   switch (protocol)
2581     {
2582 #define _(N, i, n, s) \
2583     case NAT_PROTOCOL_##N: \
2584       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2585       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2586       a->busy_##n##_ports--; \
2587       a->busy_##n##_ports_per_thread[thread_index]--; \
2588       break;
2589       foreach_nat_protocol
2590 #undef _
2591         default : nat_elog_info (sm, "unknown protocol");
2592       return;
2593     }
2594 }
2595
2596 int
2597 snat_static_mapping_match (vlib_main_t *vm, snat_main_t *sm,
2598                            ip4_address_t match_addr, u16 match_port,
2599                            u32 match_fib_index, nat_protocol_t match_protocol,
2600                            ip4_address_t *mapping_addr, u16 *mapping_port,
2601                            u32 *mapping_fib_index, u8 by_external,
2602                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2603                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2604                            u8 *is_identity_nat, snat_static_mapping_t **out)
2605 {
2606   clib_bihash_kv_8_8_t kv, value;
2607   clib_bihash_8_8_t *mapping_hash;
2608   snat_static_mapping_t *m;
2609   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2610   nat44_lb_addr_port_t *local;
2611   u8 backend_index;
2612
2613   if (!by_external)
2614     {
2615       mapping_hash = &sm->static_mapping_by_local;
2616       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2617                   match_protocol);
2618       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2619         {
2620           /* Try address only mapping */
2621           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2622           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2623             return 1;
2624         }
2625     }
2626   else
2627     {
2628       mapping_hash = &sm->static_mapping_by_external;
2629       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2630       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2631         {
2632           /* Try address only mapping */
2633           init_nat_k (&kv, match_addr, 0, 0, 0);
2634           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2635             return 1;
2636         }
2637     }
2638
2639   m = pool_elt_at_index (sm->static_mappings, value.value);
2640
2641   if (by_external)
2642     {
2643       if (is_sm_lb (m->flags))
2644         {
2645           if (PREDICT_FALSE (lb != 0))
2646             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2647           if (m->affinity && !nat_affinity_find_and_lock (
2648                                vm, ext_host_addr[0], match_addr,
2649                                match_protocol, match_port, &backend_index))
2650             {
2651               local = pool_elt_at_index (m->locals, backend_index);
2652               *mapping_addr = local->addr;
2653               *mapping_port = local->port;
2654               *mapping_fib_index = local->fib_index;
2655               goto end;
2656             }
2657           // pick locals matching this worker
2658           if (PREDICT_FALSE (sm->num_workers > 1))
2659             {
2660               u32 thread_index = vlib_get_thread_index ();
2661               pool_foreach_index (i, m->locals)
2662                {
2663                 local = pool_elt_at_index (m->locals, i);
2664
2665                 ip4_header_t ip = {
2666                   .src_address = local->addr,
2667                 };
2668
2669                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2670                                                       0) == thread_index)
2671                   {
2672                     vec_add1 (tmp, i);
2673                   }
2674                }
2675               ASSERT (vec_len (tmp) != 0);
2676             }
2677           else
2678             {
2679               pool_foreach_index (i, m->locals)
2680                {
2681                 vec_add1 (tmp, i);
2682               }
2683             }
2684           hi = vec_len (tmp) - 1;
2685           local = pool_elt_at_index (m->locals, tmp[hi]);
2686           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2687           while (lo < hi)
2688             {
2689               mid = ((hi - lo) >> 1) + lo;
2690               local = pool_elt_at_index (m->locals, tmp[mid]);
2691               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2692             }
2693           local = pool_elt_at_index (m->locals, tmp[lo]);
2694           if (!(local->prefix >= rand))
2695             return 1;
2696           *mapping_addr = local->addr;
2697           *mapping_port = local->port;
2698           *mapping_fib_index = local->fib_index;
2699           if (m->affinity)
2700             {
2701               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2702                                                 match_protocol, match_port,
2703                                                 tmp[lo], m->affinity,
2704                                                 m->affinity_per_service_list_head_index))
2705                 nat_elog_info (sm, "create affinity record failed");
2706             }
2707           vec_free (tmp);
2708         }
2709       else
2710         {
2711           if (PREDICT_FALSE (lb != 0))
2712             *lb = NO_LB_NAT;
2713           *mapping_fib_index = m->fib_index;
2714           *mapping_addr = m->local_addr;
2715           /* Address only mapping doesn't change port */
2716           *mapping_port =
2717             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2718         }
2719     }
2720   else
2721     {
2722       *mapping_addr = m->external_addr;
2723       /* Address only mapping doesn't change port */
2724       *mapping_port =
2725         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2726       *mapping_fib_index = sm->outside_fib_index;
2727     }
2728
2729 end:
2730   if (PREDICT_FALSE (is_addr_only != 0))
2731     *is_addr_only = is_sm_addr_only (m->flags);
2732
2733   if (PREDICT_FALSE (twice_nat != 0))
2734     {
2735       *twice_nat = TWICE_NAT_DISABLED;
2736
2737       if (is_sm_twice_nat (m->flags))
2738         {
2739           *twice_nat = TWICE_NAT;
2740         }
2741       else if (is_sm_self_twice_nat (m->flags))
2742         {
2743           *twice_nat = TWICE_NAT_SELF;
2744         }
2745     }
2746
2747   if (PREDICT_FALSE (is_identity_nat != 0))
2748     *is_identity_nat = is_sm_identity_nat (m->flags);
2749
2750   if (out != 0)
2751     *out = m;
2752
2753   return 0;
2754 }
2755
2756 u32
2757 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2758                                   u32 rx_fib_index, u8 is_output)
2759 {
2760   snat_main_t *sm = &snat_main;
2761   u32 next_worker_index = sm->first_worker_index;
2762   u32 hash;
2763
2764   clib_bihash_kv_16_8_t kv16, value16;
2765
2766   u32 fib_index = rx_fib_index;
2767   if (b)
2768     {
2769       if (PREDICT_FALSE (is_output))
2770         {
2771           fib_index = sm->outside_fib_index;
2772           nat_outside_fib_t *outside_fib;
2773           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2774           fib_prefix_t pfx = {
2775                   .fp_proto = FIB_PROTOCOL_IP4,
2776                   .fp_len = 32,
2777                   .fp_addr = {
2778                           .ip4.as_u32 = ip->dst_address.as_u32,
2779                   } ,
2780           };
2781
2782           switch (vec_len (sm->outside_fibs))
2783             {
2784             case 0:
2785               fib_index = sm->outside_fib_index;
2786               break;
2787             case 1:
2788               fib_index = sm->outside_fibs[0].fib_index;
2789               break;
2790             default:
2791               vec_foreach (outside_fib, sm->outside_fibs)
2792                 {
2793                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2794                   if (FIB_NODE_INDEX_INVALID != fei)
2795                     {
2796                       if (fib_entry_get_resolving_interface (fei) != ~0)
2797                         {
2798                           fib_index = outside_fib->fib_index;
2799                           break;
2800                         }
2801                     }
2802                 }
2803               break;
2804             }
2805         }
2806
2807       if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2808         {
2809           ip4_address_t lookup_saddr, lookup_daddr;
2810           u16 lookup_sport, lookup_dport;
2811           u8 lookup_protocol;
2812
2813           if (!nat_get_icmp_session_lookup_values (
2814                 b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr,
2815                 &lookup_dport, &lookup_protocol))
2816             {
2817               init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
2818                          lookup_dport, rx_fib_index, lookup_protocol);
2819               if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2820                 {
2821                   next_worker_index = ed_value_get_thread_index (&value16);
2822                   vnet_buffer2 (b)->nat.cached_session_index =
2823                     ed_value_get_session_index (&value16);
2824                   goto out;
2825                 }
2826             }
2827         }
2828
2829       init_ed_k (&kv16, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2830                  ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2831                  fib_index, ip->protocol);
2832
2833       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2834         {
2835           next_worker_index = ed_value_get_thread_index (&value16);
2836           vnet_buffer2 (b)->nat.cached_session_index =
2837             ed_value_get_session_index (&value16);
2838           goto out;
2839         }
2840
2841       // dst NAT
2842       init_ed_k (&kv16, ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2843                  ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2844                  rx_fib_index, ip->protocol);
2845       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2846         {
2847           next_worker_index = ed_value_get_thread_index (&value16);
2848           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
2849             ed_value_get_session_index (&value16);
2850           goto out;
2851         }
2852     }
2853
2854   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2855     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2856
2857   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2858     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2859   else
2860     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2861
2862 out:
2863   if (PREDICT_TRUE (!is_output))
2864     {
2865       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
2866                               rx_fib_index,
2867                               clib_net_to_host_u32 (ip->src_address.as_u32),
2868                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2869     }
2870   else
2871     {
2872       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
2873                               next_worker_index, rx_fib_index,
2874                               clib_net_to_host_u32 (ip->src_address.as_u32),
2875                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2876     }
2877
2878   return next_worker_index;
2879 }
2880
2881 u32
2882 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2883                                   u32 rx_fib_index, u8 is_output)
2884 {
2885   snat_main_t *sm = &snat_main;
2886   clib_bihash_kv_8_8_t kv, value;
2887   clib_bihash_kv_16_8_t kv16, value16;
2888
2889   u32 proto, next_worker_index = 0;
2890   u16 port;
2891   snat_static_mapping_t *m;
2892   u32 hash;
2893
2894   proto = ip_proto_to_nat_proto (ip->protocol);
2895
2896   if (PREDICT_FALSE (proto == NAT_PROTOCOL_ICMP))
2897     {
2898       ip4_address_t lookup_saddr, lookup_daddr;
2899       u16 lookup_sport, lookup_dport;
2900       u8 lookup_protocol;
2901       if (!nat_get_icmp_session_lookup_values (
2902             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
2903             &lookup_protocol))
2904         {
2905           init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
2906                      lookup_dport, rx_fib_index, lookup_protocol);
2907           if (PREDICT_TRUE (
2908                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
2909             {
2910               next_worker_index = ed_value_get_thread_index (&value16);
2911               nat_elog_debug_handoff (
2912                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
2913                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
2914                 clib_net_to_host_u32 (ip->dst_address.as_u32));
2915               return next_worker_index;
2916             }
2917         }
2918     }
2919
2920   init_ed_k (&kv16, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2921              ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2922              rx_fib_index, ip->protocol);
2923
2924   if (PREDICT_TRUE (
2925         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
2926     {
2927       vnet_buffer2 (b)->nat.cached_session_index =
2928         ed_value_get_session_index (&value16);
2929       next_worker_index = ed_value_get_thread_index (&value16);
2930       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
2931                               next_worker_index, rx_fib_index,
2932                               clib_net_to_host_u32 (ip->src_address.as_u32),
2933                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2934       return next_worker_index;
2935     }
2936
2937   /* first try static mappings without port */
2938   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2939     {
2940       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
2941       if (!clib_bihash_search_8_8
2942           (&sm->static_mapping_by_external, &kv, &value))
2943         {
2944           m = pool_elt_at_index (sm->static_mappings, value.value);
2945           next_worker_index = m->workers[0];
2946           goto done;
2947         }
2948     }
2949
2950   /* unknown protocol */
2951   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
2952     {
2953       /* use current thread */
2954       next_worker_index = vlib_get_thread_index ();
2955       goto done;
2956     }
2957
2958   port = vnet_buffer (b)->ip.reass.l4_dst_port;
2959
2960   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2961     {
2962       udp_header_t *udp = ip4_next_header (ip);
2963       icmp46_header_t *icmp = (icmp46_header_t *) udp;
2964       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
2965       if (!icmp_type_is_error_message
2966           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
2967         port = vnet_buffer (b)->ip.reass.l4_src_port;
2968       else
2969         {
2970           /* if error message, then it's not fragmented and we can access it */
2971           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
2972           proto = ip_proto_to_nat_proto (inner_ip->protocol);
2973           void *l4_header = ip4_next_header (inner_ip);
2974           switch (proto)
2975             {
2976             case NAT_PROTOCOL_ICMP:
2977               icmp = (icmp46_header_t *) l4_header;
2978               echo = (icmp_echo_header_t *) (icmp + 1);
2979               port = echo->identifier;
2980               break;
2981             case NAT_PROTOCOL_UDP:
2982             case NAT_PROTOCOL_TCP:
2983               port = ((tcp_udp_header_t *) l4_header)->src_port;
2984               break;
2985             default:
2986               next_worker_index = vlib_get_thread_index ();
2987               goto done;
2988             }
2989         }
2990     }
2991
2992   /* try static mappings with port */
2993   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2994     {
2995       init_nat_k (&kv, ip->dst_address, port, 0, proto);
2996       if (!clib_bihash_search_8_8
2997           (&sm->static_mapping_by_external, &kv, &value))
2998         {
2999           m = pool_elt_at_index (sm->static_mappings, value.value);
3000           if (!is_sm_lb (m->flags))
3001             {
3002               next_worker_index = m->workers[0];
3003               goto done;
3004             }
3005
3006           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3007             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3008
3009           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3010             next_worker_index =
3011               m->workers[hash & (_vec_len (m->workers) - 1)];
3012           else
3013             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3014           goto done;
3015         }
3016     }
3017
3018   /* worker by outside port */
3019   next_worker_index = sm->first_worker_index;
3020   next_worker_index +=
3021     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3022
3023 done:
3024   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3025                           rx_fib_index,
3026                           clib_net_to_host_u32 (ip->src_address.as_u32),
3027                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3028   return next_worker_index;
3029 }
3030
3031 u32
3032 nat44_get_max_session_limit ()
3033 {
3034   snat_main_t *sm = &snat_main;
3035   u32 max_limit = 0, len = 0;
3036
3037   for (; len < vec_len (sm->max_translations_per_fib); len++)
3038     {
3039       if (max_limit < sm->max_translations_per_fib[len])
3040         max_limit = sm->max_translations_per_fib[len];
3041     }
3042   return max_limit;
3043 }
3044
3045 int
3046 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3047 {
3048   snat_main_t *sm = &snat_main;
3049   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3050   u32 len = vec_len (sm->max_translations_per_fib);
3051
3052   if (len <= fib_index)
3053     {
3054       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3055
3056       for (; len < vec_len (sm->max_translations_per_fib); len++)
3057         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3058     }
3059
3060   sm->max_translations_per_fib[fib_index] = session_limit;
3061   return 0;
3062 }
3063
3064 int
3065 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3066 {
3067   snat_main_t *sm = &snat_main;
3068
3069   if (nat44_set_session_limit (session_limit, vrf_id))
3070     return 1;
3071   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3072
3073   stat_segment_set_state_counter (sm->max_cfg_sessions_gauge,
3074                                   sm->max_translations_per_thread);
3075
3076   sm->translation_buckets =
3077     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3078
3079   nat44_ed_sessions_clear ();
3080   return 0;
3081 }
3082
3083 static void
3084 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3085                          u32 translation_buckets)
3086 {
3087   dlist_elt_t *head;
3088
3089   pool_alloc (tsm->sessions, translations);
3090   pool_alloc (tsm->lru_pool, translations);
3091
3092   pool_get (tsm->lru_pool, head);
3093   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3094   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3095
3096   pool_get (tsm->lru_pool, head);
3097   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3098   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3099
3100   pool_get (tsm->lru_pool, head);
3101   tsm->udp_lru_head_index = head - tsm->lru_pool;
3102   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3103
3104   pool_get (tsm->lru_pool, head);
3105   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3106   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3107
3108   pool_get (tsm->lru_pool, head);
3109   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3110   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3111 }
3112
3113 static void
3114 reinit_ed_flow_hash ()
3115 {
3116   snat_main_t *sm = &snat_main;
3117   // we expect 2 flows per session, so multiply translation_buckets by 2
3118   clib_bihash_init_16_8 (
3119     &sm->flow_hash, "ed-flow-hash",
3120     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3121   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3122 }
3123
3124 static void
3125 nat44_ed_db_init (u32 translations, u32 translation_buckets)
3126 {
3127   snat_main_t *sm = &snat_main;
3128   snat_main_per_thread_data_t *tsm;
3129   u32 static_mapping_buckets = 1024;
3130   u32 static_mapping_memory_size = 64 << 20;
3131
3132   reinit_ed_flow_hash ();
3133
3134   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3135                         "static_mapping_by_local", static_mapping_buckets,
3136                         static_mapping_memory_size);
3137   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3138                                      format_static_mapping_kvp);
3139
3140   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3141                         "static_mapping_by_external", static_mapping_buckets,
3142                         static_mapping_memory_size);
3143   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3144                                      format_static_mapping_kvp);
3145
3146   if (sm->pat)
3147     {
3148       vec_foreach (tsm, sm->per_thread_data)
3149         {
3150           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3151                                    sm->translation_buckets);
3152         }
3153     }
3154 }
3155
3156 static void
3157 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3158 {
3159   pool_free (tsm->lru_pool);
3160   pool_free (tsm->sessions);
3161   vec_free (tsm->per_vrf_sessions_vec);
3162 }
3163
3164 static void
3165 nat44_ed_db_free ()
3166 {
3167   snat_main_t *sm = &snat_main;
3168   snat_main_per_thread_data_t *tsm;
3169
3170   pool_free (sm->static_mappings);
3171   clib_bihash_free_16_8 (&sm->flow_hash);
3172   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3173   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3174
3175   if (sm->pat)
3176     {
3177       vec_foreach (tsm, sm->per_thread_data)
3178         {
3179           nat44_ed_worker_db_free (tsm);
3180         }
3181     }
3182 }
3183
3184 void
3185 nat44_ed_sessions_clear ()
3186 {
3187   snat_main_t *sm = &snat_main;
3188   snat_main_per_thread_data_t *tsm;
3189
3190   reinit_ed_flow_hash ();
3191
3192   if (sm->pat)
3193     {
3194       vec_foreach (tsm, sm->per_thread_data)
3195         {
3196
3197           nat44_ed_worker_db_free (tsm);
3198           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3199                                    sm->translation_buckets);
3200         }
3201     }
3202   vlib_zero_simple_counter (&sm->total_sessions, 0);
3203 }
3204
3205 static void
3206 nat44_ed_add_del_static_mapping_addr_only_cb (
3207   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
3208   u32 address_length, u32 if_address_index, u32 is_delete)
3209 {
3210   clib_bihash_kv_8_8_t kv, value;
3211   snat_static_map_resolve_t *rp;
3212   snat_main_t *sm = &snat_main;
3213   snat_static_mapping_t *m;
3214   int i, rv = 0, match = 0;
3215
3216   if (!sm->enabled)
3217     {
3218       return;
3219     }
3220
3221   // find first addr_only resolve record by sw_if_index
3222   for (i = 0; i < vec_len (sm->to_resolve); i++)
3223     {
3224       rp = sm->to_resolve + i;
3225       if (rp->addr_only && rp->sw_if_index == sw_if_index)
3226         {
3227           match = 1;
3228           break;
3229         }
3230     }
3231   if (!match)
3232     {
3233       return;
3234     }
3235
3236   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
3237               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
3238
3239   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3240     {
3241       m = 0;
3242     }
3243   else
3244     {
3245       m = pool_elt_at_index (sm->static_mappings, value.value);
3246     }
3247
3248   if (is_delete)
3249     {
3250       if (m)
3251         {
3252           rv = nat44_ed_del_static_mapping (rp->l_addr, address[0], rp->l_port,
3253                                             rp->e_port, rp->proto, rp->vrf_id,
3254                                             ~0, rp->flags);
3255         }
3256     }
3257   else
3258     {
3259       if (!m)
3260         {
3261           rv = nat44_ed_add_static_mapping (
3262             rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3263             rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3264         }
3265       // else: don't trip over lease renewal, static config
3266     }
3267   if (rv)
3268     {
3269       nat_elog_notice_X1 (sm, "nat44_ed_del_static_mapping returned %d", "i4",
3270                           rv);
3271     }
3272 }
3273
3274 static_always_inline int
3275 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3276 {
3277   u32 *i;
3278   vec_foreach (i, sw_if_indices)
3279     {
3280       if (*i == sw_if_index)
3281         {
3282           return 1;
3283         }
3284     }
3285   return 0;
3286 }
3287
3288 static void
3289 nat44_ed_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3290                                        u32 sw_if_index, ip4_address_t *address,
3291                                        u32 address_length,
3292                                        u32 if_address_index, u32 is_delete)
3293 {
3294   snat_main_t *sm = &snat_main;
3295   snat_static_map_resolve_t *rp;
3296   snat_address_t *addresses = sm->addresses;
3297   u8 twice_nat = 0;
3298   int rv, i;
3299
3300   if (!sm->enabled)
3301     {
3302       return;
3303     }
3304
3305   if (!is_sw_if_index_reg_for_auto_resolve (sm->auto_add_sw_if_indices,
3306                                             sw_if_index))
3307     {
3308       if (!is_sw_if_index_reg_for_auto_resolve (
3309             sm->auto_add_sw_if_indices_twice_nat, sw_if_index))
3310         {
3311           return;
3312         }
3313       else
3314         {
3315           addresses = sm->twice_nat_addresses;
3316           twice_nat = 1;
3317         }
3318     }
3319
3320   if (!is_delete)
3321     {
3322       // don't trip over lease renewal, static config
3323       for (i = 0; i < vec_len (addresses); i++)
3324         {
3325           if (addresses[i].addr.as_u32 == address->as_u32)
3326             {
3327               return;
3328             }
3329         }
3330
3331       (void) nat44_ed_add_address (address, ~0, twice_nat);
3332
3333       // scan static mapping switch address resolution record vector
3334       for (i = 0; i < vec_len (sm->to_resolve); i++)
3335         {
3336           rp = sm->to_resolve + i;
3337           if (rp->addr_only)
3338             {
3339               continue;
3340             }
3341           if (rp->sw_if_index == sw_if_index)
3342             {
3343               rv = nat44_ed_add_static_mapping (
3344                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3345                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3346               if (rv)
3347                 {
3348                   nat_elog_notice_X1 (sm, "add_static_mapping returned %d",
3349                                       "i4", rv);
3350                 }
3351             }
3352         }
3353     }
3354   else
3355     {
3356       // remove all static mapping records
3357       (void) nat44_ed_del_address (address[0], 1, twice_nat);
3358     }
3359 }
3360
3361 int
3362 nat44_ed_add_interface_address (u32 sw_if_index, u8 twice_nat)
3363 {
3364   snat_main_t *sm = &snat_main;
3365   ip4_main_t *ip4_main = sm->ip4_main;
3366   ip4_address_t *first_int_addr;
3367   u32 *auto_add_sw_if_indices = twice_nat ?
3368                                   sm->auto_add_sw_if_indices_twice_nat :
3369                                   sm->auto_add_sw_if_indices;
3370   int i;
3371
3372   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3373     {
3374       if (auto_add_sw_if_indices[i] == sw_if_index)
3375         {
3376           return VNET_API_ERROR_VALUE_EXIST;
3377         }
3378     }
3379
3380   // add to the auto-address list
3381   if (twice_nat)
3382     {
3383       vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3384     }
3385   else
3386     {
3387       vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3388     }
3389
3390   // if the address is already bound - or static - add it now
3391   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3392   if (first_int_addr)
3393     {
3394       (void) nat44_ed_add_address (first_int_addr, ~0, twice_nat);
3395     }
3396
3397   return 0;
3398 }
3399
3400 int
3401 nat44_ed_del_interface_address (u32 sw_if_index, u8 twice_nat)
3402 {
3403   snat_main_t *sm = &snat_main;
3404   ip4_main_t *ip4_main = sm->ip4_main;
3405   ip4_address_t *first_int_addr;
3406   snat_static_map_resolve_t *rp;
3407   u32 *indices_to_delete = 0;
3408   int i, j;
3409   u32 *auto_add_sw_if_indices;
3410
3411   if (!sm->enabled)
3412     {
3413       return VNET_API_ERROR_UNSUPPORTED;
3414     }
3415
3416   auto_add_sw_if_indices = twice_nat ? sm->auto_add_sw_if_indices_twice_nat :
3417                                        sm->auto_add_sw_if_indices;
3418
3419   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3420     {
3421       if (auto_add_sw_if_indices[i] == sw_if_index)
3422         {
3423           first_int_addr =
3424             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3425           if (first_int_addr)
3426             {
3427               // remove all static mapping records
3428               (void) nat44_ed_del_address (first_int_addr[0], 1, twice_nat);
3429             }
3430           else
3431             {
3432               for (j = 0; j < vec_len (sm->to_resolve); j++)
3433                 {
3434                   rp = sm->to_resolve + j;
3435                   if (rp->sw_if_index == sw_if_index)
3436                     {
3437                       vec_add1 (indices_to_delete, j);
3438                     }
3439                 }
3440               if (vec_len (indices_to_delete))
3441                 {
3442                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3443                     {
3444                       vec_del1 (sm->to_resolve, j);
3445                     }
3446                   vec_free (indices_to_delete);
3447                 }
3448             }
3449
3450           if (twice_nat)
3451             {
3452               vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3453             }
3454           else
3455             {
3456               vec_del1 (sm->auto_add_sw_if_indices, i);
3457             }
3458           return 0;
3459         }
3460     }
3461   return VNET_API_ERROR_NO_SUCH_ENTRY;
3462 }
3463
3464 int
3465 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3466                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3467                       u32 vrf_id, int is_in)
3468 {
3469   ip4_header_t ip;
3470   clib_bihash_kv_16_8_t kv, value;
3471   u32 fib_index;
3472   snat_session_t *s;
3473   snat_main_per_thread_data_t *tsm;
3474
3475   if (!sm->enabled)
3476     {
3477       return VNET_API_ERROR_UNSUPPORTED;
3478     }
3479
3480   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3481   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3482   if (sm->num_workers > 1)
3483     tsm = vec_elt_at_index (
3484       sm->per_thread_data,
3485       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3486   else
3487     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3488
3489   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3490   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3491     {
3492       return VNET_API_ERROR_NO_SUCH_ENTRY;
3493     }
3494
3495   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3496     return VNET_API_ERROR_UNSPECIFIED;
3497   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3498   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3499   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3500   return 0;
3501 }
3502
3503 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3504                                  vlib_node_runtime_t * node,
3505                                  vlib_frame_t * frame)
3506 {
3507   return 0;
3508 }
3509
3510 VLIB_REGISTER_NODE (nat_default_node) = {
3511   .name = "nat-default",
3512   .vector_size = sizeof (u32),
3513   .format_trace = 0,
3514   .type = VLIB_NODE_TYPE_INTERNAL,
3515   .n_errors = 0,
3516   .n_next_nodes = NAT_N_NEXT,
3517   .next_nodes = {
3518     [NAT_NEXT_DROP] = "error-drop",
3519     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3520     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3521     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3522     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3523     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3524     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3525     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3526     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3527     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3528   },
3529 };
3530
3531 void
3532 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3533 {
3534   f->l3_csum_delta = 0;
3535   f->l4_csum_delta = 0;
3536   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3537       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3538     {
3539       f->l3_csum_delta =
3540         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3541       f->l3_csum_delta =
3542         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3543     }
3544   else
3545     {
3546       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3547     }
3548   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3549       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3550     {
3551       f->l3_csum_delta =
3552         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3553       f->l3_csum_delta =
3554         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3555     }
3556   else
3557     {
3558       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3559     }
3560   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3561     {
3562       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3563       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3564     }
3565   else
3566     {
3567       f->rewrite.sport = f->match.sport;
3568     }
3569   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3570     {
3571       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3572       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3573     }
3574   else
3575     {
3576       f->rewrite.dport = f->match.dport;
3577     }
3578   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3579       f->rewrite.icmp_id != f->match.sport)
3580     {
3581       f->l4_csum_delta =
3582         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3583       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3584     }
3585   else
3586     {
3587       f->rewrite.icmp_id = f->match.sport;
3588     }
3589   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3590     {
3591     }
3592   else
3593     {
3594       f->rewrite.fib_index = f->match.fib_index;
3595     }
3596 }
3597
3598 static_always_inline int
3599 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3600                             ip4_header_t *ip, nat_6t_flow_t *f);
3601
3602 static_always_inline void
3603 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3604                            nat_6t_flow_t *f, nat_protocol_t proto,
3605                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3606 {
3607   udp_header_t *udp = ip4_next_header (ip);
3608   tcp_header_t *tcp = (tcp_header_t *) udp;
3609
3610   if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
3611       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3612     {
3613       if (!is_icmp_inner_ip4)
3614         { // regular case
3615           ip->src_address = f->rewrite.saddr;
3616           ip->dst_address = f->rewrite.daddr;
3617           udp->src_port = f->rewrite.sport;
3618           udp->dst_port = f->rewrite.dport;
3619         }
3620       else
3621         { // icmp inner ip4 - reversed saddr/daddr
3622           ip->src_address = f->rewrite.daddr;
3623           ip->dst_address = f->rewrite.saddr;
3624           udp->src_port = f->rewrite.dport;
3625           udp->dst_port = f->rewrite.sport;
3626         }
3627
3628       if (NAT_PROTOCOL_TCP == proto)
3629         {
3630           ip_csum_t tcp_sum = tcp->checksum;
3631           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3632           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3633           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3634           tcp->checksum = ip_csum_fold (tcp_sum);
3635         }
3636       else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
3637         {
3638           ip_csum_t udp_sum = udp->checksum;
3639           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3640           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3641           udp->checksum = ip_csum_fold (udp_sum);
3642         }
3643     }
3644   else
3645     {
3646       if (!is_icmp_inner_ip4)
3647         { // regular case
3648           if (!skip_saddr_rewrite)
3649             {
3650               ip->src_address = f->rewrite.saddr;
3651             }
3652           ip->dst_address = f->rewrite.daddr;
3653         }
3654       else
3655         { // icmp inner ip4 - reversed saddr/daddr
3656           ip->src_address = f->rewrite.daddr;
3657           ip->dst_address = f->rewrite.saddr;
3658         }
3659     }
3660
3661   if (skip_saddr_rewrite)
3662     {
3663       ip->checksum = ip4_header_checksum (ip);
3664     }
3665   else
3666     {
3667       ip_csum_t ip_sum = ip->checksum;
3668       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3669       ip->checksum = ip_csum_fold (ip_sum);
3670     }
3671   if (0xffff == ip->checksum)
3672     ip->checksum = 0;
3673   ASSERT (ip4_header_checksum_is_valid (ip));
3674 }
3675
3676 static_always_inline int
3677 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3678 {
3679   int result = ((u8 *) object + size <=
3680                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3681                vlib_object_within_buffer_data (vm, b, object, size);
3682   return result;
3683 }
3684
3685 static_always_inline int
3686 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3687                             ip4_header_t *ip, nat_6t_flow_t *f)
3688 {
3689   if (IP_PROTOCOL_ICMP != ip->protocol)
3690     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3691
3692   icmp46_header_t *icmp = ip4_next_header (ip);
3693   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3694
3695   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3696     {
3697       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3698         {
3699           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3700         }
3701
3702       if (!icmp_type_is_error_message (icmp->type))
3703         {
3704           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3705               (f->rewrite.icmp_id != echo->identifier))
3706             {
3707               ip_csum_t sum = icmp->checksum;
3708               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3709                                     icmp_echo_header_t,
3710                                     identifier /* changed member */);
3711               echo->identifier = f->rewrite.icmp_id;
3712               icmp->checksum = ip_csum_fold (sum);
3713             }
3714         }
3715       else
3716         {
3717           ip_csum_t sum = ip_incremental_checksum (
3718             0, icmp,
3719             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3720           sum = (u16) ~ip_csum_fold (sum);
3721           if (sum != 0)
3722             {
3723               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3724             }
3725
3726           // errors are not fragmented
3727           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3728
3729           if (!ip4_header_checksum_is_valid (inner_ip))
3730             {
3731               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3732             }
3733
3734           nat_protocol_t inner_proto =
3735             ip_proto_to_nat_proto (inner_ip->protocol);
3736
3737           ip_csum_t old_icmp_sum = icmp->checksum;
3738           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3739           ip_csum_t old_udp_sum;
3740           ip_csum_t old_tcp_sum;
3741           ip_csum_t new_icmp_sum;
3742           udp_header_t *udp;
3743           tcp_header_t *tcp;
3744
3745           switch (inner_proto)
3746             {
3747             case NAT_PROTOCOL_UDP:
3748               udp = (udp_header_t *) (inner_ip + 1);
3749               if (!it_fits (vm, b, udp, sizeof (*udp)))
3750                 {
3751                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3752                 }
3753               old_udp_sum = udp->checksum;
3754               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3755                                          1 /* is_icmp_inner_ip4 */,
3756                                          0 /* skip_saddr_rewrite */);
3757               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3758               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3759               new_icmp_sum =
3760                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3761                                 inner_ip->checksum, ip4_header_t, checksum);
3762               new_icmp_sum =
3763                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3764                                 udp_header_t, checksum);
3765               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3766               icmp->checksum = new_icmp_sum;
3767               break;
3768             case NAT_PROTOCOL_TCP:
3769               tcp = (tcp_header_t *) (inner_ip + 1);
3770               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3771                 {
3772                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3773                 }
3774               old_tcp_sum = tcp->checksum;
3775               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3776                                          1 /* is_icmp_inner_ip4 */,
3777                                          0 /* skip_saddr_rewrite */);
3778               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3779               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3780               new_icmp_sum =
3781                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3782                                 inner_ip->checksum, ip4_header_t, checksum);
3783               new_icmp_sum =
3784                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3785                                 tcp_header_t, checksum);
3786               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3787               icmp->checksum = new_icmp_sum;
3788               break;
3789             case NAT_PROTOCOL_ICMP:
3790               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3791                 {
3792                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3793                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3794                     {
3795                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3796                     }
3797                   icmp_echo_header_t *inner_echo =
3798                     (icmp_echo_header_t *) (inner_icmp + 1);
3799                   if (f->rewrite.icmp_id != inner_echo->identifier)
3800                     {
3801                       ip_csum_t sum = icmp->checksum;
3802                       sum = ip_csum_update (
3803                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3804                         icmp_echo_header_t, identifier /* changed member */);
3805                       icmp->checksum = ip_csum_fold (sum);
3806                       ip_csum_t inner_sum = inner_icmp->checksum;
3807                       inner_sum = ip_csum_update (
3808                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3809                         icmp_echo_header_t, identifier /* changed member */);
3810                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3811                       inner_echo->identifier = f->rewrite.icmp_id;
3812                     }
3813                 }
3814               break;
3815             default:
3816               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3817               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3818             }
3819         }
3820     }
3821
3822   return NAT_ED_TRNSL_ERR_SUCCESS;
3823 }
3824
3825 static_always_inline nat_translation_error_e
3826 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3827                            ip4_header_t *ip, nat_6t_flow_t *f,
3828                            nat_protocol_t proto, int is_output_feature,
3829                            int is_i2o)
3830 {
3831   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3832     {
3833       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3834     }
3835
3836   if (NAT_PROTOCOL_ICMP == proto)
3837     {
3838       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3839         {
3840           // packet is returned from a router, not from destination
3841           // skip source address rewrite if in o2i path
3842           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3843                                      0 /* is_icmp_inner_ip4 */,
3844                                      !is_i2o /* skip_saddr_rewrite */);
3845         }
3846       else
3847         {
3848           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3849                                      0 /* is_icmp_inner_ip4 */,
3850                                      0 /* skip_saddr_rewrite */);
3851         }
3852       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
3853     }
3854
3855   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
3856                              0 /* skip_saddr_rewrite */);
3857
3858   return NAT_ED_TRNSL_ERR_SUCCESS;
3859 }
3860
3861 nat_translation_error_e
3862 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
3863                                vlib_buffer_t *b, ip4_header_t *ip,
3864                                nat_6t_flow_t *f, nat_protocol_t proto,
3865                                int is_output_feature)
3866 {
3867   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3868                                     1 /* is_i2o */);
3869 }
3870
3871 nat_translation_error_e
3872 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
3873                                vlib_buffer_t *b, ip4_header_t *ip,
3874                                nat_6t_flow_t *f, nat_protocol_t proto,
3875                                int is_output_feature)
3876 {
3877   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3878                                     0 /* is_i2o */);
3879 }
3880
3881 u8 *
3882 format_nat_6t (u8 *s, va_list *args)
3883 {
3884   nat_6t_t *t = va_arg (*args, nat_6t_t *);
3885
3886   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
3887               format_ip4_address, t->saddr.as_u8,
3888               clib_net_to_host_u16 (t->sport), format_ip4_address,
3889               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
3890               format_ip_protocol, t->proto, t->fib_index);
3891   return s;
3892 }
3893
3894 u8 *
3895 format_nat_ed_translation_error (u8 *s, va_list *args)
3896 {
3897   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
3898
3899   switch (e)
3900     {
3901     case NAT_ED_TRNSL_ERR_SUCCESS:
3902       s = format (s, "success");
3903       break;
3904     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
3905       s = format (s, "translation-failed");
3906       break;
3907     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
3908       s = format (s, "flow-mismatch");
3909       break;
3910     case NAT_ED_TRNSL_ERR_PACKET_TRUNCATED:
3911       s = format (s, "packet-truncated");
3912       break;
3913     case NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT:
3914       s = format (s, "inner-ip-corrupted");
3915       break;
3916     case NAT_ED_TRNSL_ERR_INVALID_CSUM:
3917       s = format (s, "invalid-checksum");
3918       break;
3919     }
3920   return s;
3921 }
3922
3923 u8 *
3924 format_nat_6t_flow (u8 *s, va_list *args)
3925 {
3926   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
3927
3928   s = format (s, "match: %U ", format_nat_6t, &f->match);
3929   int r = 0;
3930   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
3931     {
3932       s = format (s, "rewrite: saddr %U ", format_ip4_address,
3933                   f->rewrite.saddr.as_u8);
3934       r = 1;
3935     }
3936   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
3937     {
3938       if (!r)
3939         {
3940           s = format (s, "rewrite: ");
3941           r = 1;
3942         }
3943       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
3944     }
3945   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
3946     {
3947       if (!r)
3948         {
3949           s = format (s, "rewrite: ");
3950           r = 1;
3951         }
3952       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
3953     }
3954   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
3955     {
3956       if (!r)
3957         {
3958           s = format (s, "rewrite: ");
3959           r = 1;
3960         }
3961       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
3962     }
3963   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3964     {
3965       if (!r)
3966         {
3967           s = format (s, "rewrite: ");
3968           r = 1;
3969         }
3970       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
3971     }
3972   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3973     {
3974       if (!r)
3975         {
3976           s = format (s, "rewrite: ");
3977           r = 1;
3978         }
3979       s = format (s, "txfib %u ", f->rewrite.fib_index);
3980     }
3981   return s;
3982 }
3983
3984 /*
3985  * fd.io coding-style-patch-verification: ON
3986  *
3987  * Local Variables:
3988  * eval: (c-set-style "gnu")
3989  * End:
3990  */