src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vpp/app/version.h>
  19
  20 #include <vnet/vnet.h>
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ip/ip4.h>
  23 #include <vnet/ip/ip_table.h>
  24 #include <vnet/ip/reass/ip4_sv_reass.h>
  25 #include <vnet/fib/fib_table.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <vnet/plugin/plugin.h>
  28 #include <vppinfra/bihash_16_8.h>
  29
  30 #include <nat/lib/log.h>
  31 #include <nat/lib/nat_inlines.h>
  32 #include <nat/lib/ipfix_logging.h>
  33 #include <vnet/syslog/syslog.h>
  34 #include <nat/lib/nat_syslog_constants.h>
  35 #include <nat/lib/nat_syslog.h>
  36
  37 #include <nat/nat44-ed/nat44_ed.h>
  38 #include <nat/nat44-ed/nat44_ed_affinity.h>
  39 #include <nat/nat44-ed/nat44_ed_inlines.h>
  40
  41 #include <vpp/stats/stat_segment.h>
  42
  43 snat_main_t snat_main;
  44
  45 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  46                                                            u32 sw_if_index);
  47
  48 #define skip_if_disabled()                                                    \
  49   do                                                                          \
  50     {                                                                         \
  51       snat_main_t *sm = &snat_main;                                           \
  52       if (PREDICT_FALSE (!sm->enabled))                                       \
  53         return;                                                               \
  54     }                                                                         \
  55   while (0)
  56
  57 #define fail_if_enabled()                                                     \
  58   do                                                                          \
  59     {                                                                         \
  60       snat_main_t *sm = &snat_main;                                           \
  61       if (PREDICT_FALSE (sm->enabled))                                        \
  62         {                                                                     \
  63           nat_log_err ("plugin enabled");                                     \
  64           return 1;                                                           \
  65         }                                                                     \
  66     }                                                                         \
  67   while (0)
  68
  69 #define fail_if_disabled()                                                    \
  70   do                                                                          \
  71     {                                                                         \
  72       snat_main_t *sm = &snat_main;                                           \
  73       if (PREDICT_FALSE (!sm->enabled))                                       \
  74         {                                                                     \
  75           nat_log_err ("plugin disabled");                                    \
  76           return 1;                                                           \
  77         }                                                                     \
  78     }                                                                         \
  79   while (0)
  80
  81 /* Hook up input features */
  82 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  83   .arc_name = "ip4-unicast",
  84   .node_name = "nat-pre-in2out",
  85   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  86                                "ip4-sv-reassembly-feature"),
  87 };
  88 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  89   .arc_name = "ip4-unicast",
  90   .node_name = "nat-pre-out2in",
  91   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  92                                "ip4-dhcp-client-detect",
  93                                "ip4-sv-reassembly-feature"),
  94 };
  95 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  96   .arc_name = "ip4-unicast",
  97   .node_name = "nat44-in2out-worker-handoff",
  98   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  99 };
 100 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
 101   .arc_name = "ip4-unicast",
 102   .node_name = "nat44-out2in-worker-handoff",
 103   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 104                                "ip4-dhcp-client-detect"),
 105 };
 106 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 107   .arc_name = "ip4-unicast",
 108   .node_name = "nat44-in2out",
 109   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 110 };
 111 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 112   .arc_name = "ip4-unicast",
 113   .node_name = "nat44-out2in",
 114   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 115                                "ip4-dhcp-client-detect"),
 116 };
 117 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 118   .arc_name = "ip4-unicast",
 119   .node_name = "nat44-ed-in2out",
 120   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 121 };
 122 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 123   .arc_name = "ip4-unicast",
 124   .node_name = "nat44-ed-out2in",
 125   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 126                                "ip4-dhcp-client-detect"),
 127 };
 128 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 129   .arc_name = "ip4-unicast",
 130   .node_name = "nat44-ed-classify",
 131   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 132 };
 133 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 134   .arc_name = "ip4-unicast",
 135   .node_name = "nat44-handoff-classify",
 136   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 137 };
 138 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 139   .arc_name = "ip4-unicast",
 140   .node_name = "nat44-in2out-fast",
 141   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 142 };
 143 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 144   .arc_name = "ip4-unicast",
 145   .node_name = "nat44-out2in-fast",
 146   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 147                                "ip4-dhcp-client-detect"),
 148 };
 149
 150 /* Hook up output features */
 151 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 152   .arc_name = "ip4-output",
 153   .node_name = "nat44-in2out-output",
 154   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 155   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 156 };
 157 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 158   .arc_name = "ip4-output",
 159   .node_name = "nat44-in2out-output-worker-handoff",
 160   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 161   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 162 };
 163 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 164   .arc_name = "ip4-output",
 165   .node_name = "nat-pre-in2out-output",
 166   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 167   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 168 };
 169 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 170   .arc_name = "ip4-output",
 171   .node_name = "nat44-ed-in2out-output",
 172   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 173   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 174 };
 175
 176 VLIB_PLUGIN_REGISTER () = {
 177     .version = VPP_BUILD_VER,
 178     .description = "Network Address Translation (NAT)",
 179 };
 180
 181 static void nat44_ed_db_init (u32 translations, u32 translation_buckets);
 182 static void nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm);
 183
 184 static int nat44_ed_add_static_mapping_internal (
 185   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 186   ip_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 187   ip4_address_t pool_addr, u8 *tag);
 188 static int nat44_ed_del_static_mapping_internal (
 189   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 190   ip_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags);
 191
 192 u32 nat_calc_bihash_buckets (u32 n_elts);
 193
 194 u8 *
 195 format_ed_session_kvp (u8 * s, va_list * args)
 196 {
 197   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 198
 199   u8 proto;
 200   u16 r_port, l_port;
 201   ip4_address_t l_addr, r_addr;
 202   u32 fib_index;
 203
 204   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 205   s = format (s,
 206               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 207               "session-index %u",
 208               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 209               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 210               format_ip_protocol, proto, fib_index,
 211               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 212
 213   return s;
 214 }
 215
 216 static_always_inline int
 217 nat44_ed_sm_i2o_add (snat_main_t *sm, snat_static_mapping_t *m,
 218                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 219 {
 220   ASSERT (!pool_is_free (sm->static_mappings, m));
 221   clib_bihash_kv_16_8_t kv;
 222   nat44_ed_sm_init_i2o_kv (&kv, addr.as_u32, port, fib_index, proto,
 223                            m - sm->static_mappings);
 224   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 225 }
 226
 227 static_always_inline int
 228 nat44_ed_sm_i2o_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 229                      u32 fib_index, u8 proto)
 230 {
 231   clib_bihash_kv_16_8_t kv;
 232   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 233   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 234 }
 235
 236 static_always_inline int
 237 nat44_ed_sm_o2i_add (snat_main_t *sm, snat_static_mapping_t *m,
 238                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 239 {
 240   ASSERT (!pool_is_free (sm->static_mappings, m));
 241   clib_bihash_kv_16_8_t kv;
 242   nat44_ed_sm_init_o2i_kv (&kv, addr.as_u32, port, fib_index, proto,
 243                            m - sm->static_mappings);
 244   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 245 }
 246
 247 static_always_inline int
 248 nat44_ed_sm_o2i_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 249                      u32 fib_index, u8 proto)
 250 {
 251   clib_bihash_kv_16_8_t kv;
 252   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 253   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 254 }
 255
 256 void
 257 nat44_ed_free_session_data (snat_main_t *sm, snat_session_t *s,
 258                             u32 thread_index, u8 is_ha)
 259 {
 260   per_vrf_sessions_unregister_session (s, thread_index);
 261
 262   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 263     nat_elog_warn (sm, "flow hash del failed");
 264
 265   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 266     nat_elog_warn (sm, "flow hash del failed");
 267
 268   if (na44_ed_is_fwd_bypass_session (s))
 269     {
 270       return;
 271     }
 272
 273   if (nat44_ed_is_affinity_session (s))
 274     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, s->proto,
 275                          s->out2in.port);
 276
 277   if (!is_ha)
 278     nat_syslog_nat44_sdel (0, s->in2out.fib_index, &s->in2out.addr,
 279                            s->in2out.port, &s->ext_host_nat_addr,
 280                            s->ext_host_nat_port, &s->out2in.addr,
 281                            s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 282                            s->proto, nat44_ed_is_twice_nat_session (s));
 283
 284   if (!is_ha)
 285     {
 286       /* log NAT event */
 287       nat_ipfix_logging_nat44_ses_delete (
 288         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 289         s->in2out.port, s->out2in.port, s->in2out.fib_index);
 290     }
 291 }
 292
 293 static int
 294 is_snat_address_used_in_static_mapping (snat_main_t *sm, ip4_address_t addr)
 295 {
 296   snat_static_mapping_t *m;
 297   pool_foreach (m, sm->static_mappings)
 298     {
 299       if (is_sm_addr_only (m->flags) || is_sm_out2in_only (m->flags) ||
 300           is_sm_identity_nat (m->flags))
 301         {
 302           continue;
 303         }
 304       if (m->external_addr.as_u32 == addr.as_u32)
 305         {
 306           return 1;
 307         }
 308     }
 309   return 0;
 310 }
 311
 312 static ip_interface_address_t *
 313 nat44_ed_get_ip_interface_address (u32 sw_if_index, ip4_address_t addr)
 314 {
 315   snat_main_t *sm = &snat_main;
 316
 317   ip_lookup_main_t *lm = &sm->ip4_main->lookup_main;
 318   ip_interface_address_t *ia;
 319   ip4_address_t *ip4a;
 320
 321   foreach_ip_interface_address (
 322     lm, ia, sw_if_index, 1, ({
 323       ip4a = ip_interface_address_get_address (lm, ia);
 324       nat_log_debug ("sw_if_idx: %u addr: %U ? %U", sw_if_index,
 325                      format_ip4_address, ip4a, format_ip4_address, &addr);
 326       if (ip4a->as_u32 == addr.as_u32)
 327         {
 328           return ia;
 329         }
 330     }));
 331   return NULL;
 332 }
 333
 334 static int
 335 nat44_ed_resolve_nat_addr_len (snat_address_t *ap,
 336                                snat_interface_t *interfaces)
 337 {
 338   ip_interface_address_t *ia;
 339   snat_interface_t *i;
 340   u32 fib_index;
 341
 342   pool_foreach (i, interfaces)
 343     {
 344       if (!nat44_ed_is_interface_outside (i))
 345         {
 346           continue;
 347         }
 348
 349       fib_index = ip4_fib_table_get_index_for_sw_if_index (i->sw_if_index);
 350       if (fib_index != ap->fib_index)
 351         {
 352           continue;
 353         }
 354
 355       if ((ia = nat44_ed_get_ip_interface_address (i->sw_if_index, ap->addr)))
 356         {
 357           ap->addr_len = ia->address_length;
 358           ap->sw_if_index = i->sw_if_index;
 359           ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
 360                            << (32 - ap->addr_len);
 361
 362           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 363                          format_ip4_address, &ap->addr, ap->sw_if_index,
 364                          format_ip4_address, &ap->net, ap->addr_len);
 365           return 0;
 366         }
 367     }
 368   return 1;
 369 }
 370
 371 static void
 372 nat44_ed_update_outside_if_addresses (snat_address_t *ap)
 373 {
 374   snat_main_t *sm = &snat_main;
 375
 376   if (!nat44_ed_resolve_nat_addr_len (ap, sm->interfaces))
 377     {
 378       return;
 379     }
 380
 381   if (!nat44_ed_resolve_nat_addr_len (ap, sm->output_feature_interfaces))
 382     {
 383       return;
 384     }
 385 }
 386
 387 static void
 388 nat44_ed_bind_if_addr_to_nat_addr (u32 sw_if_index)
 389 {
 390   snat_main_t *sm = &snat_main;
 391   ip_interface_address_t *ia;
 392   snat_address_t *ap;
 393
 394   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 395
 396   vec_foreach (ap, sm->addresses)
 397     {
 398       if (fib_index != ap->fib_index)
 399         {
 400           continue;
 401         }
 402
 403       if ((ia = nat44_ed_get_ip_interface_address (sw_if_index, ap->addr)))
 404         {
 405           ap->addr_len = ia->address_length;
 406           ap->sw_if_index = sw_if_index;
 407           ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
 408                            << (32 - ap->addr_len);
 409
 410           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 411                          format_ip4_address, &ap->addr, ap->sw_if_index,
 412                          format_ip4_address, &ap->net, ap->addr_len);
 413           return;
 414         }
 415     }
 416 }
 417
 418 static void
 419 nat44_ed_add_del_addr_to_fib (ip4_address_t *addr, u8 p_len, u32 sw_if_index,
 420                               int is_add)
 421 {
 422   // Add the external NAT address to the FIB as receive entries. This ensures
 423   // that VPP will reply to ARP for this address and we don't need to enable
 424   // proxy ARP on the outside interface.
 425
 426   snat_main_t *sm = &snat_main;
 427   fib_prefix_t prefix = {
 428     .fp_len = p_len,
 429     .fp_proto = FIB_PROTOCOL_IP4,
 430     .fp_addr = {
 431                 .ip4.as_u32 = addr->as_u32,
 432                 },
 433   };
 434   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 435
 436   if (is_add)
 437     {
 438       fib_table_entry_update_one_path (fib_index, &prefix, sm->fib_src_low,
 439                                        (FIB_ENTRY_FLAG_CONNECTED |
 440                                         FIB_ENTRY_FLAG_LOCAL |
 441                                         FIB_ENTRY_FLAG_EXCLUSIVE),
 442                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
 443                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 444     }
 445   else
 446     {
 447       fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 448     }
 449 }
 450
 451 static void
 452 nat44_ed_add_del_addr_to_fib_foreach_out_if (ip4_address_t *addr, u8 is_add)
 453 {
 454   snat_main_t *sm = &snat_main;
 455   snat_interface_t *i;
 456
 457   pool_foreach (i, sm->interfaces)
 458     {
 459       if (nat44_ed_is_interface_outside (i))
 460         {
 461           nat44_ed_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 462         }
 463     }
 464   pool_foreach (i, sm->output_feature_interfaces)
 465     {
 466       if (nat44_ed_is_interface_outside (i))
 467         {
 468           nat44_ed_add_del_addr_to_fib (addr, 32, i->sw_if_index, is_add);
 469         }
 470     }
 471 }
 472
 473 static_always_inline void
 474 nat44_ed_add_del_addr_to_fib_foreach_addr (u32 sw_if_index, u8 is_add)
 475 {
 476   snat_main_t *sm = &snat_main;
 477   snat_address_t *ap;
 478
 479   vec_foreach (ap, sm->addresses)
 480     {
 481       nat44_ed_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, is_add);
 482     }
 483 }
 484
 485 static_always_inline void
 486 nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (u32 sw_if_index, u8 is_add)
 487 {
 488   snat_main_t *sm = &snat_main;
 489   snat_static_mapping_t *m;
 490
 491   pool_foreach (m, sm->static_mappings)
 492     {
 493       if (is_sm_addr_only (m->flags) && !is_sm_identity_nat (m->flags))
 494         {
 495           nat44_ed_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index,
 496                                         is_add);
 497         }
 498     }
 499 }
 500
 501 int
 502 nat44_ed_add_address (ip4_address_t *addr, u32 vrf_id, u8 twice_nat)
 503 {
 504   snat_main_t *sm = &snat_main;
 505   snat_address_t *ap, *addresses;
 506
 507   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 508
 509   if (!sm->enabled)
 510     {
 511       return VNET_API_ERROR_UNSUPPORTED;
 512     }
 513
 514   // check if address already exists
 515   vec_foreach (ap, addresses)
 516     {
 517       if (ap->addr.as_u32 == addr->as_u32)
 518         {
 519           nat_log_err ("address exist");
 520           return VNET_API_ERROR_VALUE_EXIST;
 521         }
 522     }
 523
 524   if (twice_nat)
 525     {
 526       vec_add2 (sm->twice_nat_addresses, ap, 1);
 527     }
 528   else
 529     {
 530       vec_add2 (sm->addresses, ap, 1);
 531     }
 532
 533   ap->addr_len = ~0;
 534   ap->fib_index = ~0;
 535   ap->addr = *addr;
 536
 537   if (vrf_id != ~0)
 538     {
 539       ap->fib_index = fib_table_find_or_create_and_lock (
 540         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 541     }
 542
 543   if (!twice_nat)
 544     {
 545       // if we don't have enabled interface we don't add address
 546       // to fib
 547       nat44_ed_add_del_addr_to_fib_foreach_out_if (addr, 1);
 548       nat44_ed_update_outside_if_addresses (ap);
 549     }
 550   return 0;
 551 }
 552
 553 int
 554 nat44_ed_del_address (ip4_address_t addr, u8 delete_sm, u8 twice_nat)
 555 {
 556   snat_main_t *sm = &snat_main;
 557   snat_address_t *a = 0, *addresses;
 558   snat_session_t *ses;
 559   u32 *ses_to_be_removed = 0, *ses_index;
 560   snat_main_per_thread_data_t *tsm;
 561   snat_static_mapping_t *m;
 562   int j;
 563
 564   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 565
 566   for (j = 0; j < vec_len (addresses); j++)
 567     {
 568       if (addresses[j].addr.as_u32 == addr.as_u32)
 569         {
 570           a = addresses + j;
 571           break;
 572         }
 573     }
 574   if (!a)
 575     {
 576       nat_log_err ("no such address");
 577       return VNET_API_ERROR_NO_SUCH_ENTRY;
 578     }
 579
 580   if (delete_sm)
 581     {
 582       pool_foreach (m, sm->static_mappings)
 583         {
 584           if (m->external_addr.as_u32 == addr.as_u32)
 585             {
 586               nat44_ed_del_static_mapping_internal (
 587                 m->local_addr, m->external_addr, m->local_port,
 588                 m->external_port, ip_proto_to_nat_proto (m->proto), m->vrf_id,
 589                 ~0, m->flags);
 590             }
 591         }
 592     }
 593   else
 594     {
 595       if (is_snat_address_used_in_static_mapping (sm, addr))
 596         {
 597           nat_log_err ("address used in static mapping");
 598           return VNET_API_ERROR_UNSPECIFIED;
 599         }
 600     }
 601
 602   // delete sessions using address
 603   vec_foreach (tsm, sm->per_thread_data)
 604     {
 605       pool_foreach (ses, tsm->sessions)
 606         {
 607           if (ses->out2in.addr.as_u32 == addr.as_u32)
 608             {
 609               nat44_ed_free_session_data (sm, ses, tsm - sm->per_thread_data,
 610                                           0);
 611               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
 612             }
 613         }
 614       vec_foreach (ses_index, ses_to_be_removed)
 615         {
 616           ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
 617           nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
 618         }
 619       vec_free (ses_to_be_removed);
 620     }
 621
 622   if (!twice_nat)
 623     {
 624       nat44_ed_add_del_addr_to_fib_foreach_out_if (&addr, 0);
 625     }
 626
 627   if (a->fib_index != ~0)
 628     {
 629       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 630     }
 631
 632   if (!twice_nat)
 633     {
 634       vec_del1 (sm->addresses, j);
 635     }
 636   else
 637     {
 638       vec_del1 (sm->twice_nat_addresses, j);
 639     }
 640
 641   return 0;
 642 }
 643
 644 u32
 645 get_thread_idx_by_port (u16 e_port)
 646 {
 647   snat_main_t *sm = &snat_main;
 648   u32 thread_idx = sm->num_workers;
 649   if (sm->num_workers > 1)
 650     {
 651       thread_idx =
 652         sm->first_worker_index +
 653         sm->workers[(e_port - 1024) / sm->port_per_thread];
 654     }
 655   return thread_idx;
 656 }
 657
 658 void
 659 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 660                                     snat_main_per_thread_data_t * tsm,
 661                                     ip4_address_t l_addr,
 662                                     u16 l_port,
 663                                     u8 protocol,
 664                                     u32 fib_index, int addr_only,
 665                                     ip4_address_t e_addr, u16 e_port)
 666 {
 667   snat_session_t *s;
 668   u32 *indexes_to_free = NULL;
 669   pool_foreach (s, tsm->sessions) {
 670     if (s->in2out.fib_index != fib_index ||
 671         s->in2out.addr.as_u32 != l_addr.as_u32)
 672       {
 673         continue;
 674       }
 675     if (!addr_only)
 676       {
 677         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 678             s->out2in.port != e_port || s->in2out.port != l_port ||
 679             s->proto != protocol)
 680           continue;
 681       }
 682
 683     if (nat44_ed_is_lb_session (s))
 684       continue;
 685     if (!nat44_ed_is_session_static (s))
 686       continue;
 687     nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 688     vec_add1 (indexes_to_free, s - tsm->sessions);
 689     if (!addr_only)
 690       break;
 691   }
 692   u32 *ses_index;
 693   vec_foreach (ses_index, indexes_to_free)
 694   {
 695     s = pool_elt_at_index (tsm->sessions, *ses_index);
 696     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 697   }
 698   vec_free (indexes_to_free);
 699 }
 700
 701 static_always_inline snat_static_mapping_t *
 702 nat44_ed_sm_lookup (snat_main_t *sm, clib_bihash_kv_16_8_t *kv)
 703 {
 704   clib_bihash_kv_16_8_t v;
 705   int rc = clib_bihash_search_16_8 (&sm->flow_hash, kv, &v);
 706   if (!rc)
 707     {
 708       ASSERT (0 == ed_value_get_thread_index (&v));
 709       return pool_elt_at_index (sm->static_mappings,
 710                                 ed_value_get_session_index (&v));
 711     }
 712   return NULL;
 713 }
 714
 715 snat_static_mapping_t *
 716 nat44_ed_sm_o2i_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 717                         u32 fib_index, u8 proto)
 718 {
 719   clib_bihash_kv_16_8_t kv;
 720   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 721   return nat44_ed_sm_lookup (sm, &kv);
 722 }
 723
 724 snat_static_mapping_t *
 725 nat44_ed_sm_i2o_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 726                         u32 fib_index, u8 proto)
 727 {
 728   clib_bihash_kv_16_8_t kv;
 729   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 730   return nat44_ed_sm_lookup (sm, &kv);
 731 }
 732
 733 void
 734 nat44_ed_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 735                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 736                              u32 flags, ip4_address_t pool_addr, u8 *tag)
 737 {
 738   snat_static_map_resolve_t *rp;
 739   snat_main_t *sm = &snat_main;
 740
 741   vec_add2 (sm->to_resolve, rp, 1);
 742   rp->l_addr.as_u32 = l_addr.as_u32;
 743   rp->l_port = l_port;
 744   rp->e_port = e_port;
 745   rp->sw_if_index = sw_if_index;
 746   rp->vrf_id = vrf_id;
 747   rp->proto = proto;
 748   rp->flags = flags;
 749   rp->pool_addr = pool_addr;
 750   rp->tag = vec_dup (tag);
 751 }
 752
 753 int
 754 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 755                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 756                              u32 flags, int *out)
 757 {
 758   snat_static_map_resolve_t *rp;
 759   snat_main_t *sm = &snat_main;
 760   int i;
 761
 762   for (i = 0; i < vec_len (sm->to_resolve); i++)
 763     {
 764       rp = sm->to_resolve + i;
 765
 766       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 767         {
 768           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 769             {
 770               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 771                 {
 772                   if (rp->e_port != e_port || rp->proto != proto)
 773                     {
 774                       continue;
 775                     }
 776                 }
 777             }
 778           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 779             {
 780               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 781                 {
 782                   if (rp->l_port != l_port || rp->e_port != e_port ||
 783                       rp->proto != proto)
 784                     {
 785                       continue;
 786                     }
 787                 }
 788             }
 789           else
 790             {
 791               continue;
 792             }
 793           if (out)
 794             {
 795               *out = i;
 796             }
 797           return 0;
 798         }
 799     }
 800   return 1;
 801 }
 802
 803 int
 804 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 805                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 806                              u32 flags)
 807 {
 808   snat_main_t *sm = &snat_main;
 809   int i;
 810   if (!nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 811                                     sw_if_index, flags, &i))
 812     {
 813       vec_del1 (sm->to_resolve, i);
 814       return 0;
 815     }
 816   return 1;
 817 }
 818
 819 static_always_inline int
 820 nat44_ed_validate_sm_input (u32 flags)
 821 {
 822   // identity nat can be initiated only from inside interface
 823   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 824     {
 825       return VNET_API_ERROR_UNSUPPORTED;
 826     }
 827
 828   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 829     {
 830       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 831         {
 832           return VNET_API_ERROR_UNSUPPORTED;
 833         }
 834     }
 835   return 0;
 836 }
 837
 838 snat_address_t *
 839 nat44_ed_addr_lookup (snat_main_t *sm, u32 addr)
 840 {
 841   for (int i = 0; i < vec_len (sm->addresses); ++i)
 842     {
 843       if (sm->addresses[i].addr.as_u32 == addr)
 844         return &sm->addresses[i];
 845     }
 846   return NULL;
 847 }
 848
 849 int
 850 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 851                              u16 l_port, u16 e_port, ip_protocol_t proto,
 852                              u32 vrf_id, u32 sw_if_index, u32 flags,
 853                              ip4_address_t pool_addr, u8 *tag)
 854 {
 855   snat_main_t *sm = &snat_main;
 856
 857   // interface bound mapping
 858   if (is_sm_switch_address (flags))
 859     {
 860       if (!nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 861                                         sw_if_index, flags, 0))
 862         {
 863           return VNET_API_ERROR_VALUE_EXIST;
 864         }
 865
 866       nat44_ed_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 867                                    sw_if_index, flags, pool_addr, tag);
 868       ip4_address_t *first_int_addr =
 869         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 870       if (!first_int_addr)
 871         {
 872           // dhcp resolution required
 873           return 0;
 874         }
 875
 876       e_addr.as_u32 = first_int_addr->as_u32;
 877     }
 878
 879   return nat44_ed_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
 880                                                proto, vrf_id, sw_if_index,
 881                                                flags, pool_addr, tag);
 882 }
 883
 884 int
 885 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 886                              u16 l_port, u16 e_port, ip_protocol_t proto,
 887                              u32 vrf_id, u32 sw_if_index, u32 flags)
 888 {
 889
 890   snat_main_t *sm = &snat_main;
 891
 892   // interface bound mapping
 893   if (is_sm_switch_address (flags))
 894     {
 895       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 896                                        sw_if_index, flags))
 897         {
 898           return VNET_API_ERROR_NO_SUCH_ENTRY;
 899         }
 900
 901       ip4_address_t *first_int_addr =
 902         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 903       if (!first_int_addr)
 904         {
 905           // dhcp resolution required
 906           return 0;
 907         }
 908
 909       e_addr.as_u32 = first_int_addr->as_u32;
 910     }
 911
 912   return nat44_ed_del_static_mapping_internal (
 913     l_addr, e_addr, l_port, e_port, proto, vrf_id, sw_if_index, flags);
 914 }
 915
 916 static int
 917 nat44_ed_add_static_mapping_internal (ip4_address_t l_addr,
 918                                       ip4_address_t e_addr, u16 l_port,
 919                                       u16 e_port, ip_protocol_t proto,
 920                                       u32 vrf_id, u32 sw_if_index, u32 flags,
 921                                       ip4_address_t pool_addr, u8 *tag)
 922 {
 923   snat_main_t *sm = &snat_main;
 924   nat44_lb_addr_port_t *local;
 925   snat_static_mapping_t *m;
 926   u32 fib_index = ~0;
 927   int rv;
 928
 929   if (!sm->enabled)
 930     {
 931       return VNET_API_ERROR_UNSUPPORTED;
 932     }
 933
 934   rv = nat44_ed_validate_sm_input (flags);
 935   if (rv != 0)
 936     {
 937       return rv;
 938     }
 939
 940   if (is_sm_addr_only (flags))
 941     {
 942       e_port = l_port = proto = 0;
 943     }
 944
 945   if (is_sm_identity_nat (flags))
 946     {
 947       l_port = e_port;
 948       l_addr.as_u32 = e_addr.as_u32;
 949     }
 950
 951   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
 952   if (m)
 953     {
 954       if (!is_sm_identity_nat (m->flags))
 955         {
 956           return VNET_API_ERROR_VALUE_EXIST;
 957         }
 958
 959       // case:
 960       // adding local identity nat record for different vrf table
 961       pool_foreach (local, m->locals)
 962         {
 963           if (local->vrf_id == vrf_id)
 964             {
 965               return VNET_API_ERROR_VALUE_EXIST;
 966             }
 967         }
 968
 969       pool_get (m->locals, local);
 970
 971       local->vrf_id = vrf_id;
 972       local->fib_index = fib_table_find_or_create_and_lock (
 973         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 974
 975       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port,
 976                            local->fib_index, m->proto);
 977
 978       return 0;
 979     }
 980
 981   if (vrf_id != ~0)
 982     {
 983       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 984                                                      sm->fib_src_low);
 985     }
 986   else
 987     {
 988       // fallback to default vrf
 989       vrf_id = sm->inside_vrf_id;
 990       fib_index = sm->inside_fib_index;
 991       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 992     }
 993
 994   // test if local mapping record doesn't exist
 995   // identity nat supports multiple records in local mapping
 996   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
 997     {
 998       if (nat44_ed_sm_i2o_lookup (sm, l_addr, l_port, fib_index, proto))
 999         {
1000           return VNET_API_ERROR_VALUE_EXIST;
1001         }
1002     }
1003
1004   if (!(is_sm_out2in_only (flags) || is_sm_addr_only (flags) ||
1005         sm->static_mapping_only))
1006     {
1007       if (!nat44_ed_addr_lookup (sm, e_addr.as_u32))
1008         {
1009           // remove resolve record
1010           if (is_sm_switch_address (flags) && !is_sm_identity_nat (flags))
1011             {
1012               nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto,
1013                                            vrf_id, sw_if_index, flags);
1014             }
1015           return VNET_API_ERROR_NO_SUCH_ENTRY;
1016         }
1017     }
1018
1019   pool_get (sm->static_mappings, m);
1020   clib_memset (m, 0, sizeof (*m));
1021
1022   m->flags = flags;
1023   m->local_addr = l_addr;
1024   m->external_addr = e_addr;
1025
1026   m->tag = vec_dup (tag);
1027
1028   if (is_sm_exact_address (flags) && is_sm_twice_nat (flags))
1029     {
1030       m->pool_addr = pool_addr;
1031     }
1032
1033   if (!is_sm_addr_only (flags))
1034     {
1035       m->local_port = l_port;
1036       m->external_port = e_port;
1037       m->proto = proto;
1038     }
1039
1040   if (is_sm_identity_nat (flags))
1041     {
1042       pool_get (m->locals, local);
1043
1044       local->vrf_id = vrf_id;
1045       local->fib_index = fib_index;
1046     }
1047   else
1048     {
1049       m->vrf_id = vrf_id;
1050       m->fib_index = fib_index;
1051
1052       // not identity && addr only
1053       if (is_sm_addr_only (flags))
1054         {
1055           nat44_ed_add_del_addr_to_fib_foreach_out_if (&e_addr, 1);
1056         }
1057     }
1058
1059   if (!is_sm_out2in_only (flags))
1060     {
1061       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port, fib_index,
1062                            m->proto);
1063     }
1064
1065   nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0, m->proto);
1066
1067   if (sm->num_workers > 1)
1068     {
1069       // store worker index for this record
1070       ip4_header_t ip = {
1071         .src_address = m->local_addr,
1072       };
1073       u32 worker_index;
1074       worker_index =
1075         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
1076       vec_add1 (m->workers, worker_index);
1077     }
1078
1079   return 0;
1080 }
1081
1082 static int
1083 nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
1084                                       ip4_address_t e_addr, u16 l_port,
1085                                       u16 e_port, ip_protocol_t proto,
1086                                       u32 vrf_id, u32 sw_if_index, u32 flags)
1087 {
1088   snat_main_per_thread_data_t *tsm;
1089   snat_main_t *sm = &snat_main;
1090
1091   nat44_lb_addr_port_t *local;
1092   snat_static_mapping_t *m;
1093   u32 fib_index = ~0;
1094   int rv;
1095
1096   if (!sm->enabled)
1097     {
1098       return VNET_API_ERROR_UNSUPPORTED;
1099     }
1100
1101   rv = nat44_ed_validate_sm_input (flags);
1102   if (rv != 0)
1103     {
1104       return rv;
1105     }
1106
1107   if (is_sm_addr_only (flags))
1108     {
1109       e_port = l_port = proto = 0;
1110     }
1111
1112   if (is_sm_identity_nat (flags))
1113     {
1114       l_port = e_port;
1115       l_addr.as_u32 = e_addr.as_u32;
1116     }
1117
1118   // fib index 0
1119   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1120
1121   if (!m)
1122     {
1123       if (is_sm_switch_address (flags))
1124         {
1125           return 0;
1126         }
1127       return VNET_API_ERROR_NO_SUCH_ENTRY;
1128     }
1129
1130   if (is_sm_identity_nat (flags))
1131     {
1132       u8 found = 0;
1133
1134       if (vrf_id == ~0)
1135         {
1136           vrf_id = sm->inside_vrf_id;
1137         }
1138
1139       pool_foreach (local, m->locals)
1140         {
1141           if (local->vrf_id == vrf_id)
1142             {
1143               local = pool_elt_at_index (m->locals, local - m->locals);
1144               fib_index = local->fib_index;
1145               pool_put (m->locals, local);
1146               found = 1;
1147             }
1148         }
1149
1150       if (!found)
1151         {
1152           return VNET_API_ERROR_NO_SUCH_ENTRY;
1153         }
1154     }
1155   else
1156     {
1157       fib_index = m->fib_index;
1158     }
1159
1160   if (!(is_sm_out2in_only (flags) || is_sm_addr_only (flags) ||
1161         sm->static_mapping_only))
1162     {
1163       if (!nat44_ed_addr_lookup (sm, e_addr.as_u32))
1164         {
1165           return VNET_API_ERROR_INVALID_VALUE;
1166         }
1167     }
1168
1169   if (!is_sm_out2in_only (flags))
1170     {
1171       nat44_ed_sm_i2o_del (sm, l_addr, l_port, fib_index, proto);
1172     }
1173
1174   if (!sm->static_mapping_only || sm->static_mapping_connection_tracking)
1175     {
1176       // delete sessions for static mapping
1177       if (sm->num_workers > 1)
1178         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1179       else
1180         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1181
1182       nat_ed_static_mapping_del_sessions (
1183         sm, tsm, m->local_addr, m->local_port, m->proto, fib_index,
1184         is_sm_addr_only (flags), e_addr, e_port);
1185     }
1186
1187   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1188
1189   if (!pool_elts (m->locals))
1190     {
1191       // this is last record remove all required stuff
1192       // fib_index 0
1193       nat44_ed_sm_o2i_del (sm, e_addr, e_port, 0, proto);
1194
1195       vec_free (m->tag);
1196       vec_free (m->workers);
1197       pool_put (sm->static_mappings, m);
1198
1199       if (is_sm_addr_only (flags) && !is_sm_identity_nat (flags))
1200         {
1201           nat44_ed_add_del_addr_to_fib_foreach_out_if (&e_addr, 0);
1202         }
1203     }
1204
1205   return 0;
1206 }
1207
1208 int
1209 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1210                                 ip_protocol_t proto,
1211                                 nat44_lb_addr_port_t *locals, u32 flags,
1212                                 u8 *tag, u32 affinity)
1213 {
1214   snat_main_t *sm = &snat_main;
1215   snat_static_mapping_t *m;
1216   snat_address_t *a = 0;
1217
1218   nat44_lb_addr_port_t *local;
1219   uword *bitmap = 0;
1220   int rc = 0;
1221
1222   int i;
1223
1224   if (!sm->enabled)
1225     {
1226       return VNET_API_ERROR_UNSUPPORTED;
1227     }
1228
1229   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1230
1231   if (m)
1232     {
1233       return VNET_API_ERROR_VALUE_EXIST;
1234     }
1235
1236   if (vec_len (locals) < 2)
1237     {
1238       return VNET_API_ERROR_INVALID_VALUE;
1239     }
1240
1241   if (!(sm->static_mapping_only || is_sm_out2in_only (flags)))
1242     {
1243       /* Find external address in allocated addresses and reserve port for
1244          address and port pair mapping when dynamic translations enabled */
1245       for (i = 0; i < vec_len (sm->addresses); i++)
1246         {
1247           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1248             {
1249               /* External port must be unused */
1250               a = sm->addresses + i;
1251               if (nat44_ed_sm_o2i_lookup (sm, a->addr, e_port, 0, proto))
1252                 {
1253                   return VNET_API_ERROR_VALUE_EXIST;
1254                 }
1255               break;
1256             }
1257         }
1258       // external address must be allocated
1259       if (!a)
1260         {
1261           return VNET_API_ERROR_NO_SUCH_ENTRY;
1262         }
1263     }
1264
1265   pool_get (sm->static_mappings, m);
1266   clib_memset (m, 0, sizeof (*m));
1267   m->tag = vec_dup (tag);
1268   m->external_addr = e_addr;
1269   m->external_port = e_port;
1270   m->affinity = affinity;
1271   m->proto = proto;
1272
1273   m->flags = flags;
1274   m->flags |= NAT_SM_FLAG_LB;
1275
1276   if (affinity)
1277     m->affinity_per_service_list_head_index =
1278       nat_affinity_get_per_service_list_head_index ();
1279   else
1280     m->affinity_per_service_list_head_index = ~0;
1281
1282   if (nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0,
1283                            m->proto))
1284     {
1285       nat_elog_err (sm, "sm o2i key add failed");
1286       return VNET_API_ERROR_UNSPECIFIED;
1287     }
1288
1289   for (i = 0; i < vec_len (locals); i++)
1290     {
1291       locals[i].fib_index = fib_table_find_or_create_and_lock (
1292         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1293       if (!is_sm_out2in_only (flags))
1294         {
1295           if (nat44_ed_sm_o2i_add (sm, m, e_addr, e_port, 0, proto))
1296             {
1297               nat_elog_err (sm, "sm o2i key add failed");
1298               rc = VNET_API_ERROR_UNSPECIFIED;
1299               // here we continue with add operation so that it can be safely
1300               // reversed in delete path - otherwise we'd have to track what
1301               // we've done and deal with partial cleanups and since bihash
1302               // adds are (extremely improbable) the only points of failure,
1303               // it's easier to just do it this way
1304             }
1305         }
1306       locals[i].prefix = (i == 0) ?
1307                            locals[i].probability :
1308                            (locals[i - 1].prefix + locals[i].probability);
1309       pool_get (m->locals, local);
1310       *local = locals[i];
1311       if (sm->num_workers > 1)
1312         {
1313           ip4_header_t ip = {
1314             .src_address = locals[i].addr,
1315           };
1316           bitmap = clib_bitmap_set (
1317             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1318             1);
1319         }
1320     }
1321
1322   /* Assign workers */
1323   if (sm->num_workers > 1)
1324     {
1325       clib_bitmap_foreach (i, bitmap)
1326         {
1327           vec_add1 (m->workers, i);
1328         }
1329     }
1330
1331   return rc;
1332 }
1333
1334 int
1335 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1336                                 ip_protocol_t proto, u32 flags)
1337 {
1338   snat_main_t *sm = &snat_main;
1339   snat_static_mapping_t *m;
1340
1341   nat44_lb_addr_port_t *local;
1342   snat_main_per_thread_data_t *tsm;
1343   snat_session_t *s;
1344
1345   if (!sm->enabled)
1346     {
1347       return VNET_API_ERROR_UNSUPPORTED;
1348     }
1349
1350   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1351   if (!m)
1352     return VNET_API_ERROR_NO_SUCH_ENTRY;
1353
1354   if (!is_sm_lb (m->flags))
1355     return VNET_API_ERROR_INVALID_VALUE;
1356
1357   if (nat44_ed_sm_o2i_del (sm, m->external_addr, m->external_port, 0,
1358                            m->proto))
1359     {
1360       nat_elog_err (sm, "sm o2i key del failed");
1361       return VNET_API_ERROR_UNSPECIFIED;
1362     }
1363
1364   pool_foreach (local, m->locals)
1365     {
1366       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1367       if (!is_sm_out2in_only (flags))
1368         {
1369           if (nat44_ed_sm_i2o_del (sm, local->addr, local->port,
1370                                    local->fib_index, m->proto))
1371             {
1372               nat_elog_err (sm, "sm i2o key del failed");
1373               return VNET_API_ERROR_UNSPECIFIED;
1374             }
1375         }
1376
1377       if (sm->num_workers > 1)
1378         {
1379           ip4_header_t ip = {
1380             .src_address = local->addr,
1381           };
1382           tsm = vec_elt_at_index (
1383             sm->per_thread_data,
1384             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1385         }
1386       else
1387         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1388
1389       /* Delete sessions */
1390       pool_foreach (s, tsm->sessions)
1391         {
1392           if (!(nat44_ed_is_lb_session (s)))
1393             continue;
1394
1395           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1396               s->in2out.port != local->port)
1397             continue;
1398
1399           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1400           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1401         }
1402     }
1403
1404   if (m->affinity)
1405     {
1406       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1407     }
1408
1409   pool_free (m->locals);
1410   vec_free (m->tag);
1411   vec_free (m->workers);
1412   pool_put (sm->static_mappings, m);
1413
1414   return 0;
1415 }
1416
1417 int
1418 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1419                                           ip4_address_t l_addr, u16 l_port,
1420                                           ip_protocol_t proto, u32 vrf_id,
1421                                           u8 probability, u8 is_add)
1422 {
1423   snat_main_t *sm = &snat_main;
1424   snat_static_mapping_t *m = 0;
1425   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1426   snat_main_per_thread_data_t *tsm;
1427   snat_session_t *s;
1428   u32 *locals = 0;
1429   uword *bitmap = 0;
1430   int i;
1431
1432   if (!sm->enabled)
1433     {
1434       return VNET_API_ERROR_UNSUPPORTED;
1435     }
1436
1437   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1438
1439   if (!m)
1440     {
1441       return VNET_API_ERROR_NO_SUCH_ENTRY;
1442     }
1443
1444   if (!is_sm_lb (m->flags))
1445     {
1446       return VNET_API_ERROR_INVALID_VALUE;
1447     }
1448
1449   pool_foreach (local, m->locals)
1450    {
1451     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1452         (local->vrf_id == vrf_id))
1453       {
1454         match_local = local;
1455         break;
1456       }
1457   }
1458
1459   if (is_add)
1460     {
1461       if (match_local)
1462         {
1463           return VNET_API_ERROR_VALUE_EXIST;
1464         }
1465
1466       pool_get (m->locals, local);
1467       clib_memset (local, 0, sizeof (*local));
1468       local->addr.as_u32 = l_addr.as_u32;
1469       local->port = l_port;
1470       local->probability = probability;
1471       local->vrf_id = vrf_id;
1472       local->fib_index =
1473         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1474                                            sm->fib_src_low);
1475
1476       if (!is_sm_out2in_only (m->flags))
1477         {
1478           if (nat44_ed_sm_i2o_add (sm, m, l_addr, l_port, local->fib_index,
1479                                    proto))
1480             {
1481               nat_elog_err (sm, "sm i2o key add failed");
1482               pool_put (m->locals, local);
1483               return VNET_API_ERROR_UNSPECIFIED;
1484             }
1485         }
1486     }
1487   else
1488     {
1489       if (!match_local)
1490         return VNET_API_ERROR_NO_SUCH_ENTRY;
1491
1492       if (pool_elts (m->locals) < 3)
1493         return VNET_API_ERROR_UNSPECIFIED;
1494
1495       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1496                         sm->fib_src_low);
1497
1498       if (!is_sm_out2in_only (m->flags))
1499         {
1500           if (nat44_ed_sm_i2o_del (sm, l_addr, l_port, match_local->fib_index,
1501                                    proto))
1502             nat_elog_err (sm, "sm i2o key del failed");
1503         }
1504
1505       if (sm->num_workers > 1)
1506         {
1507           ip4_header_t ip = {
1508             .src_address = local->addr,
1509           };
1510           tsm = vec_elt_at_index (
1511             sm->per_thread_data,
1512             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1513         }
1514       else
1515         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1516
1517       /* Delete sessions */
1518       pool_foreach (s, tsm->sessions) {
1519           if (!(nat44_ed_is_lb_session (s)))
1520             continue;
1521
1522           if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1523               s->in2out.port != match_local->port)
1524             continue;
1525
1526           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1527           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1528       }
1529
1530       pool_put (m->locals, match_local);
1531     }
1532
1533   vec_free (m->workers);
1534
1535   pool_foreach (local, m->locals)
1536    {
1537     vec_add1 (locals, local - m->locals);
1538     if (sm->num_workers > 1)
1539       {
1540         ip4_header_t ip;
1541         ip.src_address.as_u32 = local->addr.as_u32,
1542         bitmap = clib_bitmap_set (
1543           bitmap,
1544           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1545       }
1546   }
1547
1548   ASSERT (vec_len (locals) > 1);
1549
1550   local = pool_elt_at_index (m->locals, locals[0]);
1551   local->prefix = local->probability;
1552   for (i = 1; i < vec_len (locals); i++)
1553     {
1554       local = pool_elt_at_index (m->locals, locals[i]);
1555       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1556       local->prefix = local->probability + prev_local->prefix;
1557     }
1558
1559   /* Assign workers */
1560   if (sm->num_workers > 1)
1561     {
1562       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1563     }
1564
1565   return 0;
1566 }
1567
1568 void
1569 expire_per_vrf_sessions (u32 fib_index)
1570 {
1571   per_vrf_sessions_t *per_vrf_sessions;
1572   snat_main_per_thread_data_t *tsm;
1573   snat_main_t *sm = &snat_main;
1574
1575   vec_foreach (tsm, sm->per_thread_data)
1576     {
1577       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1578         {
1579           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1580               (per_vrf_sessions->tx_fib_index == fib_index))
1581             {
1582               per_vrf_sessions->expired = 1;
1583             }
1584         }
1585     }
1586 }
1587
1588 void
1589 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1590 {
1591   snat_main_t *sm = &snat_main;
1592   nat_fib_t *fib;
1593
1594   // we don't care if it is outside/inside fib
1595   // we just care about their ref_count
1596   // if it reaches 0 sessions should expire
1597   // because the fib isn't valid for NAT anymore
1598
1599   vec_foreach (fib, sm->fibs)
1600   {
1601     if (fib->fib_index == fib_index)
1602       {
1603         if (is_del)
1604           {
1605             fib->ref_count--;
1606             if (!fib->ref_count)
1607               {
1608                 vec_del1 (sm->fibs, fib - sm->fibs);
1609                 expire_per_vrf_sessions (fib_index);
1610               }
1611             return;
1612           }
1613         else
1614           fib->ref_count++;
1615       }
1616   }
1617   if (!is_del)
1618     {
1619       vec_add2 (sm->fibs, fib, 1);
1620       fib->ref_count = 1;
1621       fib->fib_index = fib_index;
1622     }
1623 }
1624
1625 static_always_inline nat_outside_fib_t *
1626 nat44_ed_get_outside_fib (nat_outside_fib_t *outside_fibs, u32 fib_index)
1627 {
1628   nat_outside_fib_t *f;
1629   vec_foreach (f, outside_fibs)
1630     {
1631       if (f->fib_index == fib_index)
1632         {
1633           return f;
1634         }
1635     }
1636   return 0;
1637 }
1638
1639 static_always_inline snat_interface_t *
1640 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1641 {
1642   snat_interface_t *i;
1643   pool_foreach (i, interfaces)
1644     {
1645       if (i->sw_if_index == sw_if_index)
1646         {
1647           return i;
1648         }
1649     }
1650   return 0;
1651 }
1652
1653 int
1654 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1655 {
1656   const char *del_feature_name, *feature_name;
1657   snat_main_t *sm = &snat_main;
1658
1659   nat_outside_fib_t *outside_fib;
1660   snat_interface_t *i;
1661   u32 fib_index;
1662   int rv;
1663
1664   if (!sm->enabled)
1665     {
1666       nat_log_err ("nat44 is disabled");
1667       return VNET_API_ERROR_UNSUPPORTED;
1668     }
1669
1670   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1671     {
1672       nat_log_err ("error interface already configured");
1673       return VNET_API_ERROR_VALUE_EXIST;
1674     }
1675
1676   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1677   if (i)
1678     {
1679       if ((nat44_ed_is_interface_inside (i) && is_inside) ||
1680           (nat44_ed_is_interface_outside (i) && !is_inside))
1681         {
1682           return 0;
1683         }
1684       if (sm->num_workers > 1)
1685         {
1686           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1687                                           "nat44-out2in-worker-handoff";
1688           feature_name = "nat44-handoff-classify";
1689         }
1690       else
1691         {
1692           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1693
1694           feature_name = "nat44-ed-classify";
1695         }
1696
1697       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1698       if (rv)
1699         return rv;
1700       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1701                                    sw_if_index, 0, 0, 0);
1702       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1703                                    0, 0);
1704     }
1705   else
1706     {
1707       if (sm->num_workers > 1)
1708         {
1709           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1710                                      "nat44-out2in-worker-handoff";
1711         }
1712       else
1713         {
1714           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1715         }
1716
1717       nat_validate_interface_counters (sm, sw_if_index);
1718       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1719       if (rv)
1720         return rv;
1721       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1722                                    0, 0);
1723
1724       pool_get (sm->interfaces, i);
1725       i->sw_if_index = sw_if_index;
1726       i->flags = 0;
1727     }
1728
1729   fib_index =
1730     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1731
1732   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1733
1734   if (!is_inside)
1735     {
1736       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1737
1738       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1739       if (outside_fib)
1740         {
1741           outside_fib->refcount++;
1742         }
1743       else
1744         {
1745           vec_add2 (sm->outside_fibs, outside_fib, 1);
1746           outside_fib->fib_index = fib_index;
1747           outside_fib->refcount = 1;
1748         }
1749
1750       nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1751       nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1752
1753       nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
1754     }
1755   else
1756     {
1757       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1758     }
1759
1760   return 0;
1761 }
1762
1763 int
1764 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1765 {
1766   const char *del_feature_name, *feature_name;
1767   snat_main_t *sm = &snat_main;
1768
1769   nat_outside_fib_t *outside_fib;
1770   snat_interface_t *i;
1771   u32 fib_index;
1772   int rv;
1773
1774   if (!sm->enabled)
1775     {
1776       nat_log_err ("nat44 is disabled");
1777       return VNET_API_ERROR_UNSUPPORTED;
1778     }
1779
1780   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1781   if (i == 0)
1782     {
1783       nat_log_err ("error interface couldn't be found");
1784       return VNET_API_ERROR_NO_SUCH_ENTRY;
1785     }
1786
1787   if (nat44_ed_is_interface_inside (i) && nat44_ed_is_interface_outside (i))
1788     {
1789       if (sm->num_workers > 1)
1790         {
1791           del_feature_name = "nat44-handoff-classify";
1792           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1793                                       "nat44-out2in-worker-handoff";
1794         }
1795       else
1796         {
1797           del_feature_name = "nat44-ed-classify";
1798           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1799         }
1800
1801       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1802       if (rv)
1803         {
1804           return rv;
1805         }
1806       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1807                                    sw_if_index, 0, 0, 0);
1808       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1809                                    0, 0);
1810
1811       if (is_inside)
1812         {
1813           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1814         }
1815       else
1816         {
1817           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1818         }
1819     }
1820   else
1821     {
1822       if (sm->num_workers > 1)
1823         {
1824           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1825                                      "nat44-out2in-worker-handoff";
1826         }
1827       else
1828         {
1829           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1830         }
1831
1832       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1833       if (rv)
1834         {
1835           return rv;
1836         }
1837       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1838                                    0, 0);
1839
1840       // remove interface
1841       pool_put (sm->interfaces, i);
1842     }
1843
1844   fib_index =
1845     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1846
1847   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
1848
1849   if (!is_inside)
1850     {
1851       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1852       if (outside_fib)
1853         {
1854           outside_fib->refcount--;
1855           if (!outside_fib->refcount)
1856             {
1857               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1858             }
1859         }
1860
1861       nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
1862       nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
1863     }
1864
1865   return 0;
1866 }
1867
1868 int
1869 nat44_ed_add_output_interface (u32 sw_if_index)
1870 {
1871   snat_main_t *sm = &snat_main;
1872
1873   nat_outside_fib_t *outside_fib;
1874   snat_interface_t *i;
1875   u32 fib_index;
1876   int rv;
1877
1878   if (!sm->enabled)
1879     {
1880       nat_log_err ("nat44 is disabled");
1881       return VNET_API_ERROR_UNSUPPORTED;
1882     }
1883
1884   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1885     {
1886       nat_log_err ("error interface already configured");
1887       return VNET_API_ERROR_VALUE_EXIST;
1888     }
1889
1890   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1891     {
1892       nat_log_err ("error interface already configured");
1893       return VNET_API_ERROR_VALUE_EXIST;
1894     }
1895
1896   if (sm->num_workers > 1)
1897     {
1898       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1899       if (rv)
1900         {
1901           return rv;
1902         }
1903
1904       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1905       if (rv)
1906         {
1907           return rv;
1908         }
1909
1910       vnet_feature_enable_disable (
1911         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1912       vnet_feature_enable_disable ("ip4-output",
1913                                    "nat44-in2out-output-worker-handoff",
1914                                    sw_if_index, 1, 0, 0);
1915     }
1916   else
1917     {
1918       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1919       if (rv)
1920         {
1921           return rv;
1922         }
1923
1924       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1925       if (rv)
1926         {
1927           return rv;
1928         }
1929
1930       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1931                                    sw_if_index, 1, 0, 0);
1932       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1933                                    sw_if_index, 1, 0, 0);
1934     }
1935
1936   nat_validate_interface_counters (sm, sw_if_index);
1937
1938   pool_get (sm->output_feature_interfaces, i);
1939   i->sw_if_index = sw_if_index;
1940   i->flags = 0;
1941   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1942   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1943
1944   fib_index =
1945     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1946   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1947
1948   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1949   if (outside_fib)
1950     {
1951       outside_fib->refcount++;
1952     }
1953   else
1954     {
1955       vec_add2 (sm->outside_fibs, outside_fib, 1);
1956       outside_fib->fib_index = fib_index;
1957       outside_fib->refcount = 1;
1958     }
1959
1960   nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 1);
1961   nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 1);
1962
1963   nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
1964
1965   return 0;
1966 }
1967
1968 int
1969 nat44_ed_del_output_interface (u32 sw_if_index)
1970 {
1971   snat_main_t *sm = &snat_main;
1972
1973   nat_outside_fib_t *outside_fib;
1974   snat_interface_t *i;
1975   u32 fib_index;
1976   int rv;
1977
1978   if (!sm->enabled)
1979     {
1980       nat_log_err ("nat44 is disabled");
1981       return VNET_API_ERROR_UNSUPPORTED;
1982     }
1983
1984   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
1985   if (!i)
1986     {
1987       nat_log_err ("error interface couldn't be found");
1988       return VNET_API_ERROR_NO_SUCH_ENTRY;
1989     }
1990
1991   if (sm->num_workers > 1)
1992     {
1993       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1994       if (rv)
1995         {
1996           return rv;
1997         }
1998
1999       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2000       if (rv)
2001         {
2002           return rv;
2003         }
2004
2005       vnet_feature_enable_disable (
2006         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
2007       vnet_feature_enable_disable ("ip4-output",
2008                                    "nat44-in2out-output-worker-handoff",
2009                                    sw_if_index, 0, 0, 0);
2010     }
2011   else
2012     {
2013       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2014       if (rv)
2015         {
2016           return rv;
2017         }
2018
2019       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2020       if (rv)
2021         {
2022           return rv;
2023         }
2024
2025       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2026                                    sw_if_index, 0, 0, 0);
2027       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2028                                    sw_if_index, 0, 0, 0);
2029     }
2030
2031   // remove interface
2032   pool_put (sm->output_feature_interfaces, i);
2033
2034   fib_index =
2035     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2036   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
2037
2038   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2039   if (outside_fib)
2040     {
2041       outside_fib->refcount--;
2042       if (!outside_fib->refcount)
2043         {
2044           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2045         }
2046     }
2047
2048   nat44_ed_add_del_addr_to_fib_foreach_addr (sw_if_index, 0);
2049   nat44_ed_add_del_addr_to_fib_foreach_addr_only_sm (sw_if_index, 0);
2050
2051   return 0;
2052 }
2053
2054 int
2055 snat_set_workers (uword * bitmap)
2056 {
2057   snat_main_t *sm = &snat_main;
2058   int i, j = 0;
2059
2060   if (sm->num_workers < 2)
2061     return VNET_API_ERROR_FEATURE_DISABLED;
2062
2063   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2064     return VNET_API_ERROR_INVALID_WORKER;
2065
2066   vec_free (sm->workers);
2067   clib_bitmap_foreach (i, bitmap)
2068     {
2069       vec_add1(sm->workers, i);
2070       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2071       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2072       j++;
2073     }
2074
2075   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2076
2077   return 0;
2078 }
2079
2080 int
2081 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2082 {
2083   fail_if_enabled ();
2084   snat_main_t *sm = &snat_main;
2085   sm->frame_queue_nelts = frame_queue_nelts;
2086   return 0;
2087 }
2088
2089 static void
2090 nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque, u32 sw_if_index,
2091                                 u32 new_fib_index, u32 old_fib_index)
2092 {
2093   snat_main_t *sm = &snat_main;
2094   nat_outside_fib_t *outside_fib;
2095   snat_interface_t *i;
2096   u8 is_add = 1;
2097   u8 match = 0;
2098
2099   if (!sm->enabled || (new_fib_index == old_fib_index)
2100       || (!vec_len (sm->outside_fibs)))
2101     {
2102       return;
2103     }
2104
2105   pool_foreach (i, sm->interfaces)
2106      {
2107       if (i->sw_if_index == sw_if_index)
2108         {
2109           if (!(nat44_ed_is_interface_outside (i)))
2110             return;
2111           match = 1;
2112         }
2113     }
2114
2115   pool_foreach (i, sm->output_feature_interfaces)
2116      {
2117       if (i->sw_if_index == sw_if_index)
2118         {
2119           if (!(nat44_ed_is_interface_outside (i)))
2120             return;
2121           match = 1;
2122         }
2123     }
2124
2125   if (!match)
2126     return;
2127
2128   vec_foreach (outside_fib, sm->outside_fibs)
2129   {
2130     if (outside_fib->fib_index == old_fib_index)
2131       {
2132         outside_fib->refcount--;
2133         if (!outside_fib->refcount)
2134           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2135         break;
2136       }
2137   }
2138
2139   vec_foreach (outside_fib, sm->outside_fibs)
2140   {
2141     if (outside_fib->fib_index == new_fib_index)
2142       {
2143         outside_fib->refcount++;
2144         is_add = 0;
2145         break;
2146       }
2147   }
2148
2149   if (is_add)
2150     {
2151       vec_add2 (sm->outside_fibs, outside_fib, 1);
2152       outside_fib->refcount = 1;
2153       outside_fib->fib_index = new_fib_index;
2154     }
2155 }
2156
2157 static void nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque,
2158                                             u32 sw_if_index, u32 new_fib_index,
2159                                             u32 old_fib_index);
2160
2161 static void nat44_ed_add_del_interface_address_cb (
2162   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2163   u32 address_length, u32 if_address_index, u32 is_delete);
2164
2165 static void nat44_ed_add_del_static_mapping_addr_only_cb (
2166   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2167   u32 address_length, u32 if_address_index, u32 is_delete);
2168
2169 void
2170 test_key_calc_split ()
2171 {
2172   ip4_address_t l_addr;
2173   l_addr.as_u8[0] = 1;
2174   l_addr.as_u8[1] = 1;
2175   l_addr.as_u8[2] = 1;
2176   l_addr.as_u8[3] = 1;
2177   ip4_address_t r_addr;
2178   r_addr.as_u8[0] = 2;
2179   r_addr.as_u8[1] = 2;
2180   r_addr.as_u8[2] = 2;
2181   r_addr.as_u8[3] = 2;
2182   u16 l_port = 40001;
2183   u16 r_port = 40301;
2184   u8 proto = 9;
2185   u32 fib_index = 9000001;
2186   u32 thread_index = 3000000001;
2187   u32 session_index = 3000000221;
2188   clib_bihash_kv_16_8_t kv;
2189   init_ed_kv (&kv, l_addr.as_u32, l_port, r_addr.as_u32, r_port, fib_index,
2190               proto, thread_index, session_index);
2191   ip4_address_t l_addr2;
2192   ip4_address_t r_addr2;
2193   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2194   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2195   u16 l_port2 = 0;
2196   u16 r_port2 = 0;
2197   u8 proto2 = 0;
2198   u32 fib_index2 = 0;
2199   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2200                &r_port2);
2201   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2202   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2203   ASSERT (l_port == l_port2);
2204   ASSERT (r_port == r_port2);
2205   ASSERT (proto == proto2);
2206   ASSERT (fib_index == fib_index2);
2207   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2208   ASSERT (session_index == ed_value_get_session_index (&kv));
2209 }
2210
2211 static clib_error_t *
2212 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2213 {
2214   u32 fib_index;
2215   if (!is_add)
2216     {
2217       fib_index = ip4_fib_index_from_table_id (table_id);
2218       if (fib_index != ~0)
2219         {
2220           expire_per_vrf_sessions (fib_index);
2221         }
2222     }
2223   return 0;
2224 }
2225
2226 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2227
2228 void
2229 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2230 {
2231   vlib_node_t *node;
2232
2233   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2234   sm->out2in_node_index = node->index;
2235
2236   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2237   sm->in2out_node_index = node->index;
2238
2239   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2240   sm->in2out_output_node_index = node->index;
2241 }
2242
2243 #define nat_validate_simple_counter(c, i)                                     \
2244   do                                                                          \
2245     {                                                                         \
2246       vlib_validate_simple_counter (&c, i);                                   \
2247       vlib_zero_simple_counter (&c, i);                                       \
2248     }                                                                         \
2249   while (0);
2250
2251 #define nat_init_simple_counter(c, n, sn)                                     \
2252   do                                                                          \
2253     {                                                                         \
2254       c.name = n;                                                             \
2255       c.stat_segment_name = sn;                                               \
2256       nat_validate_simple_counter (c, 0);                                     \
2257     }                                                                         \
2258   while (0);
2259
2260 static_always_inline void
2261 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2262 {
2263 #define _(x)                                                                  \
2264   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2265   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2266   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2267   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2268   foreach_nat_counter;
2269 #undef _
2270   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2271 }
2272
2273 static clib_error_t *
2274 nat_init (vlib_main_t * vm)
2275 {
2276   snat_main_t *sm = &snat_main;
2277   vlib_thread_main_t *tm = vlib_get_thread_main ();
2278   vlib_thread_registration_t *tr;
2279   ip4_add_del_interface_address_callback_t cbi = { 0 };
2280   ip4_table_bind_callback_t cbt = { 0 };
2281   u32 i, num_threads = 0;
2282   uword *p, *bitmap = 0;
2283
2284   clib_memset (sm, 0, sizeof (*sm));
2285
2286   // required
2287   sm->vnet_main = vnet_get_main ();
2288   // convenience
2289   sm->ip4_main = &ip4_main;
2290   sm->api_main = vlibapi_get_main ();
2291   sm->ip4_lookup_main = &ip4_main.lookup_main;
2292
2293   // frame queue indices used for handoff
2294   sm->fq_out2in_index = ~0;
2295   sm->fq_in2out_index = ~0;
2296   sm->fq_in2out_output_index = ~0;
2297
2298   sm->log_level = NAT_LOG_ERROR;
2299
2300   nat44_set_node_indexes (sm, vm);
2301
2302   sm->log_class = vlib_log_register_class ("nat", 0);
2303   nat_ipfix_logging_init (vm);
2304
2305   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2306                            "/nat44-ed/total-sessions");
2307   sm->max_cfg_sessions_gauge = stat_segment_new_entry (
2308     (u8 *) "/nat44-ed/max-cfg-sessions", STAT_DIR_TYPE_SCALAR_INDEX);
2309
2310 #define _(x)                                                                  \
2311   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2312                            "/nat44-ed/in2out/fastpath/" #x);                  \
2313   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2314                            "/nat44-ed/out2in/fastpath/" #x);                  \
2315   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2316                            "/nat44-ed/in2out/slowpath/" #x);                  \
2317   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2318                            "/nat44-ed/out2in/slowpath/" #x);
2319   foreach_nat_counter;
2320 #undef _
2321   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2322                            "/nat44-ed/hairpinning");
2323
2324   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2325   if (p)
2326     {
2327       tr = (vlib_thread_registration_t *) p[0];
2328       if (tr)
2329         {
2330           sm->num_workers = tr->count;
2331           sm->first_worker_index = tr->first_index;
2332         }
2333     }
2334   num_threads = tm->n_vlib_mains - 1;
2335   sm->port_per_thread = 0xffff - 1024;
2336   vec_validate (sm->per_thread_data, num_threads);
2337
2338   /* Use all available workers by default */
2339   if (sm->num_workers > 1)
2340     {
2341       for (i = 0; i < sm->num_workers; i++)
2342         bitmap = clib_bitmap_set (bitmap, i, 1);
2343       snat_set_workers (bitmap);
2344       clib_bitmap_free (bitmap);
2345     }
2346   else
2347     {
2348       sm->per_thread_data[0].snat_thread_index = 0;
2349     }
2350
2351   /* callbacks to call when interface address changes. */
2352   cbi.function = nat44_ed_add_del_interface_address_cb;
2353   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2354   cbi.function = nat44_ed_add_del_static_mapping_addr_only_cb;
2355   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2356
2357   /* callbacks to call when interface to table biding changes */
2358   cbt.function = nat44_ed_update_outside_fib_cb;
2359   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2360
2361   sm->fib_src_low =
2362     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2363                          FIB_SOURCE_BH_SIMPLE);
2364   sm->fib_src_hi =
2365     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2366                          FIB_SOURCE_BH_SIMPLE);
2367
2368   nat_affinity_init (vm);
2369   test_key_calc_split ();
2370
2371   return nat44_api_hookup (vm);
2372 }
2373
2374 VLIB_INIT_FUNCTION (nat_init);
2375
2376 int
2377 nat44_plugin_enable (nat44_config_t c)
2378 {
2379   snat_main_t *sm = &snat_main;
2380
2381   fail_if_enabled ();
2382
2383   if (c.static_mapping_only && !c.connection_tracking)
2384     {
2385       nat_log_err ("unsupported combination of configuration");
2386       return 1;
2387     }
2388
2389   sm->static_mapping_only = c.static_mapping_only;
2390   sm->static_mapping_connection_tracking = c.connection_tracking;
2391
2392   sm->forwarding_enabled = 0;
2393   sm->mss_clamping = 0;
2394   sm->pat = (!c.static_mapping_only ||
2395              (c.static_mapping_only && c.connection_tracking));
2396
2397   if (!c.sessions)
2398     c.sessions = 63 * 1024;
2399
2400   sm->max_translations_per_thread = c.sessions;
2401   stat_segment_set_state_counter (sm->max_cfg_sessions_gauge,
2402                                   sm->max_translations_per_thread);
2403   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2404
2405   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2406
2407   sm->inside_vrf_id = c.inside_vrf;
2408   sm->inside_fib_index =
2409     fib_table_find_or_create_and_lock
2410     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2411
2412   sm->outside_vrf_id = c.outside_vrf;
2413   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2414     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2415
2416   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets);
2417
2418   nat_affinity_enable ();
2419
2420   nat_reset_timeouts (&sm->timeouts);
2421
2422   vlib_zero_simple_counter (&sm->total_sessions, 0);
2423
2424   if (!sm->frame_queue_nelts)
2425     {
2426       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2427     }
2428
2429   if (sm->num_workers > 1)
2430     {
2431       if (sm->fq_in2out_index == ~0)
2432         {
2433           sm->fq_in2out_index = vlib_frame_queue_main_init (
2434             sm->in2out_node_index, sm->frame_queue_nelts);
2435         }
2436       if (sm->fq_out2in_index == ~0)
2437         {
2438           sm->fq_out2in_index = vlib_frame_queue_main_init (
2439             sm->out2in_node_index, sm->frame_queue_nelts);
2440         }
2441       if (sm->fq_in2out_output_index == ~0)
2442         {
2443           sm->fq_in2out_output_index = vlib_frame_queue_main_init (
2444             sm->in2out_output_node_index, sm->frame_queue_nelts);
2445         }
2446     }
2447
2448   sm->enabled = 1;
2449   sm->rconfig = c;
2450
2451   return 0;
2452 }
2453
2454 int
2455 nat44_ed_del_addresses ()
2456 {
2457   snat_main_t *sm = &snat_main;
2458   snat_address_t *a, *vec;
2459   int error = 0;
2460
2461   vec = vec_dup (sm->addresses);
2462   vec_foreach (a, vec)
2463     {
2464       error = nat44_ed_del_address (a->addr, 0, 0);
2465       if (error)
2466         {
2467           nat_log_err ("error occurred while removing adderess");
2468         }
2469     }
2470   vec_free (vec);
2471   vec_free (sm->addresses);
2472   sm->addresses = 0;
2473
2474   vec = vec_dup (sm->twice_nat_addresses);
2475   vec_foreach (a, vec)
2476     {
2477       error = nat44_ed_del_address (a->addr, 0, 1);
2478       if (error)
2479         {
2480           nat_log_err ("error occurred while removing adderess");
2481         }
2482     }
2483   vec_free (vec);
2484   vec_free (sm->twice_nat_addresses);
2485   sm->twice_nat_addresses = 0;
2486
2487   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2488   sm->auto_add_sw_if_indices_twice_nat = 0;
2489
2490   vec_free (sm->auto_add_sw_if_indices);
2491   sm->auto_add_sw_if_indices = 0;
2492
2493   return error;
2494 }
2495
2496 int
2497 nat44_ed_del_interfaces ()
2498 {
2499   snat_main_t *sm = &snat_main;
2500   snat_interface_t *i, *pool;
2501   int error = 0;
2502
2503   pool = pool_dup (sm->interfaces);
2504   pool_foreach (i, pool)
2505     {
2506       if (nat44_ed_is_interface_inside (i))
2507         {
2508           error = nat44_ed_del_interface (i->sw_if_index, 1);
2509         }
2510       if (nat44_ed_is_interface_outside (i))
2511         {
2512           error = nat44_ed_del_interface (i->sw_if_index, 0);
2513         }
2514
2515       if (error)
2516         {
2517           nat_log_err ("error occurred while removing interface");
2518         }
2519     }
2520   pool_free (pool);
2521   pool_free (sm->interfaces);
2522   sm->interfaces = 0;
2523   return error;
2524 }
2525
2526 int
2527 nat44_ed_del_output_interfaces ()
2528 {
2529   snat_main_t *sm = &snat_main;
2530   snat_interface_t *i, *pool;
2531   int error = 0;
2532
2533   pool = pool_dup (sm->output_feature_interfaces);
2534   pool_foreach (i, pool)
2535     {
2536       error = nat44_ed_del_output_interface (i->sw_if_index);
2537       if (error)
2538         {
2539           nat_log_err ("error occurred while removing output interface");
2540         }
2541     }
2542   pool_free (pool);
2543   pool_free (sm->output_feature_interfaces);
2544   sm->output_feature_interfaces = 0;
2545   return error;
2546 }
2547
2548 int
2549 nat44_ed_del_static_mappings ()
2550 {
2551   snat_main_t *sm = &snat_main;
2552   snat_static_mapping_t *m, *pool;
2553   int error = 0;
2554
2555   pool = pool_dup (sm->static_mappings);
2556   pool_foreach (m, pool)
2557     {
2558       error = nat44_ed_del_static_mapping_internal (
2559         m->local_addr, m->external_addr, m->local_port, m->external_port,
2560         m->proto, m->vrf_id, ~0, m->flags);
2561       if (error)
2562         {
2563           nat_log_err ("error occurred while removing mapping");
2564         }
2565     }
2566   pool_free (pool);
2567   pool_free (sm->static_mappings);
2568   sm->static_mappings = 0;
2569
2570   vec_free (sm->to_resolve);
2571   sm->to_resolve = 0;
2572
2573   return error;
2574 }
2575
2576 int
2577 nat44_plugin_disable ()
2578 {
2579   snat_main_per_thread_data_t *tsm;
2580   snat_main_t *sm = &snat_main;
2581   int rc, error = 0;
2582
2583   fail_if_disabled ();
2584
2585   rc = nat44_ed_del_static_mappings ();
2586   if (rc)
2587     error = 1;
2588
2589   rc = nat44_ed_del_addresses ();
2590   if (rc)
2591     error = 1;
2592
2593   rc = nat44_ed_del_interfaces ();
2594   if (rc)
2595     error = 1;
2596
2597   rc = nat44_ed_del_output_interfaces ();
2598   if (rc)
2599     error = 1;
2600
2601   vec_free (sm->max_translations_per_fib);
2602   sm->max_translations_per_fib = 0;
2603
2604   clib_bihash_free_16_8 (&sm->flow_hash);
2605
2606   if (sm->pat)
2607     {
2608       vec_foreach (tsm, sm->per_thread_data)
2609         {
2610           nat44_ed_worker_db_free (tsm);
2611         }
2612     }
2613
2614   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2615
2616   sm->forwarding_enabled = 0;
2617   sm->enabled = 0;
2618
2619   return error;
2620 }
2621
2622 void
2623 nat44_ed_forwarding_enable_disable (u8 is_enable)
2624 {
2625   snat_main_per_thread_data_t *tsm;
2626   snat_main_t *sm = &snat_main;
2627   snat_session_t *s;
2628
2629   u32 *ses_to_be_removed = 0, *ses_index;
2630
2631   sm->forwarding_enabled = is_enable != 0;
2632
2633   if (!sm->enabled || is_enable)
2634     {
2635       return;
2636     }
2637
2638   vec_foreach (tsm, sm->per_thread_data)
2639     {
2640       pool_foreach (s, tsm->sessions)
2641         {
2642           if (na44_ed_is_fwd_bypass_session (s))
2643             {
2644               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2645             }
2646         }
2647       vec_foreach (ses_index, ses_to_be_removed)
2648         {
2649           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2650           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2651           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2652         }
2653
2654       vec_free (ses_to_be_removed);
2655     }
2656 }
2657
2658 static_always_inline snat_static_mapping_t *
2659 nat44_ed_sm_match (snat_main_t *sm, ip4_address_t match_addr, u16 match_port,
2660                    u32 match_fib_index, ip_protocol_t match_protocol,
2661                    int by_external)
2662 {
2663   snat_static_mapping_t *m;
2664   if (!by_external)
2665     {
2666       m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port, match_fib_index,
2667                                   match_protocol);
2668       if (m)
2669         return m;
2670
2671       /* Try address only mapping */
2672       m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, match_fib_index, 0);
2673       if (m)
2674         return m;
2675
2676       if (sm->inside_fib_index != match_fib_index)
2677         {
2678           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2679                                       sm->inside_fib_index, match_protocol);
2680           if (m)
2681             return m;
2682
2683           /* Try address only mapping */
2684           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->inside_fib_index,
2685                                       0);
2686           if (m)
2687             return m;
2688         }
2689       if (sm->outside_fib_index != match_fib_index)
2690         {
2691           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2692                                       sm->outside_fib_index, match_protocol);
2693           if (m)
2694             return m;
2695
2696           /* Try address only mapping */
2697           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->outside_fib_index,
2698                                       0);
2699           if (m)
2700             return m;
2701         }
2702     }
2703   else
2704     {
2705       m =
2706         nat44_ed_sm_o2i_lookup (sm, match_addr, match_port, 0, match_protocol);
2707       if (m)
2708         return m;
2709
2710       /* Try address only mapping */
2711       m = nat44_ed_sm_o2i_lookup (sm, match_addr, 0, 0, 0);
2712       if (m)
2713         return m;
2714     }
2715   return 0;
2716 }
2717
2718 int
2719 snat_static_mapping_match (vlib_main_t *vm, snat_main_t *sm,
2720                            ip4_address_t match_addr, u16 match_port,
2721                            u32 match_fib_index, ip_protocol_t match_protocol,
2722                            ip4_address_t *mapping_addr, u16 *mapping_port,
2723                            u32 *mapping_fib_index, int by_external,
2724                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2725                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2726                            u8 *is_identity_nat, snat_static_mapping_t **out)
2727 {
2728   snat_static_mapping_t *m;
2729   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2730   nat44_lb_addr_port_t *local;
2731   u8 backend_index;
2732
2733   m = nat44_ed_sm_match (sm, match_addr, match_port, match_fib_index,
2734                          match_protocol, by_external);
2735   if (!m)
2736     {
2737       return 1;
2738     }
2739
2740   if (by_external)
2741     {
2742       if (is_sm_lb (m->flags))
2743         {
2744           if (PREDICT_FALSE (lb != 0))
2745             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2746           if (m->affinity && !nat_affinity_find_and_lock (
2747                                vm, ext_host_addr[0], match_addr,
2748                                match_protocol, match_port, &backend_index))
2749             {
2750               local = pool_elt_at_index (m->locals, backend_index);
2751               *mapping_addr = local->addr;
2752               *mapping_port = local->port;
2753               *mapping_fib_index = local->fib_index;
2754               goto end;
2755             }
2756           // pick locals matching this worker
2757           if (PREDICT_FALSE (sm->num_workers > 1))
2758             {
2759               u32 thread_index = vlib_get_thread_index ();
2760               pool_foreach_index (i, m->locals)
2761                {
2762                 local = pool_elt_at_index (m->locals, i);
2763
2764                 ip4_header_t ip = {
2765                   .src_address = local->addr,
2766                 };
2767
2768                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2769                                                       0) == thread_index)
2770                   {
2771                     vec_add1 (tmp, i);
2772                   }
2773                }
2774               ASSERT (vec_len (tmp) != 0);
2775             }
2776           else
2777             {
2778               pool_foreach_index (i, m->locals)
2779                {
2780                 vec_add1 (tmp, i);
2781               }
2782             }
2783           hi = vec_len (tmp) - 1;
2784           local = pool_elt_at_index (m->locals, tmp[hi]);
2785           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2786           while (lo < hi)
2787             {
2788               mid = ((hi - lo) >> 1) + lo;
2789               local = pool_elt_at_index (m->locals, tmp[mid]);
2790               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2791             }
2792           local = pool_elt_at_index (m->locals, tmp[lo]);
2793           if (!(local->prefix >= rand))
2794             return 1;
2795           *mapping_addr = local->addr;
2796           *mapping_port = local->port;
2797           *mapping_fib_index = local->fib_index;
2798           if (m->affinity)
2799             {
2800               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2801                                                 match_protocol, match_port,
2802                                                 tmp[lo], m->affinity,
2803                                                 m->affinity_per_service_list_head_index))
2804                 nat_elog_info (sm, "create affinity record failed");
2805             }
2806           vec_free (tmp);
2807         }
2808       else
2809         {
2810           if (PREDICT_FALSE (lb != 0))
2811             *lb = NO_LB_NAT;
2812           *mapping_fib_index = m->fib_index;
2813           *mapping_addr = m->local_addr;
2814           /* Address only mapping doesn't change port */
2815           *mapping_port =
2816             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2817         }
2818     }
2819   else
2820     {
2821       *mapping_addr = m->external_addr;
2822       /* Address only mapping doesn't change port */
2823       *mapping_port =
2824         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2825       *mapping_fib_index = sm->outside_fib_index;
2826     }
2827
2828 end:
2829   if (PREDICT_FALSE (is_addr_only != 0))
2830     *is_addr_only = is_sm_addr_only (m->flags);
2831
2832   if (PREDICT_FALSE (twice_nat != 0))
2833     {
2834       *twice_nat = TWICE_NAT_DISABLED;
2835
2836       if (is_sm_twice_nat (m->flags))
2837         {
2838           *twice_nat = TWICE_NAT;
2839         }
2840       else if (is_sm_self_twice_nat (m->flags))
2841         {
2842           *twice_nat = TWICE_NAT_SELF;
2843         }
2844     }
2845
2846   if (PREDICT_FALSE (is_identity_nat != 0))
2847     *is_identity_nat = is_sm_identity_nat (m->flags);
2848
2849   if (out != 0)
2850     *out = m;
2851
2852   return 0;
2853 }
2854
2855 u32
2856 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2857                                   u32 rx_fib_index, u8 is_output)
2858 {
2859   snat_main_t *sm = &snat_main;
2860   u32 next_worker_index = sm->first_worker_index;
2861   u32 hash;
2862
2863   clib_bihash_kv_16_8_t kv16, value16;
2864
2865   u32 fib_index = rx_fib_index;
2866   if (b)
2867     {
2868       if (PREDICT_FALSE (is_output))
2869         {
2870           fib_index = sm->outside_fib_index;
2871           nat_outside_fib_t *outside_fib;
2872           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2873           fib_prefix_t pfx = {
2874                   .fp_proto = FIB_PROTOCOL_IP4,
2875                   .fp_len = 32,
2876                   .fp_addr = {
2877                           .ip4.as_u32 = ip->dst_address.as_u32,
2878                   } ,
2879           };
2880
2881           switch (vec_len (sm->outside_fibs))
2882             {
2883             case 0:
2884               fib_index = sm->outside_fib_index;
2885               break;
2886             case 1:
2887               fib_index = sm->outside_fibs[0].fib_index;
2888               break;
2889             default:
2890               vec_foreach (outside_fib, sm->outside_fibs)
2891                 {
2892                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2893                   if (FIB_NODE_INDEX_INVALID != fei)
2894                     {
2895                       if (fib_entry_get_resolving_interface (fei) != ~0)
2896                         {
2897                           fib_index = outside_fib->fib_index;
2898                           break;
2899                         }
2900                     }
2901                 }
2902               break;
2903             }
2904         }
2905
2906       if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2907         {
2908           ip4_address_t lookup_saddr, lookup_daddr;
2909           u16 lookup_sport, lookup_dport;
2910           u8 lookup_protocol;
2911
2912           if (!nat_get_icmp_session_lookup_values (
2913                 b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr,
2914                 &lookup_dport, &lookup_protocol))
2915             {
2916               init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
2917                          lookup_daddr.as_u32, lookup_dport, rx_fib_index,
2918                          lookup_protocol);
2919               if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2920                 {
2921                   next_worker_index = ed_value_get_thread_index (&value16);
2922                   vnet_buffer2 (b)->nat.cached_session_index =
2923                     ed_value_get_session_index (&value16);
2924                   goto out;
2925                 }
2926             }
2927         }
2928
2929       init_ed_k (&kv16, ip->src_address.as_u32,
2930                  vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
2931                  vnet_buffer (b)->ip.reass.l4_dst_port, fib_index,
2932                  ip->protocol);
2933
2934       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2935         {
2936           next_worker_index = ed_value_get_thread_index (&value16);
2937           vnet_buffer2 (b)->nat.cached_session_index =
2938             ed_value_get_session_index (&value16);
2939           goto out;
2940         }
2941
2942       // dst NAT
2943       init_ed_k (&kv16, ip->dst_address.as_u32,
2944                  vnet_buffer (b)->ip.reass.l4_dst_port, ip->src_address.as_u32,
2945                  vnet_buffer (b)->ip.reass.l4_src_port, rx_fib_index,
2946                  ip->protocol);
2947       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2948         {
2949           next_worker_index = ed_value_get_thread_index (&value16);
2950           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
2951             ed_value_get_session_index (&value16);
2952           goto out;
2953         }
2954     }
2955
2956   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2957     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2958
2959   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2960     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2961   else
2962     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2963
2964 out:
2965   if (PREDICT_TRUE (!is_output))
2966     {
2967       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
2968                               rx_fib_index,
2969                               clib_net_to_host_u32 (ip->src_address.as_u32),
2970                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2971     }
2972   else
2973     {
2974       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
2975                               next_worker_index, rx_fib_index,
2976                               clib_net_to_host_u32 (ip->src_address.as_u32),
2977                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2978     }
2979
2980   return next_worker_index;
2981 }
2982
2983 u32
2984 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2985                                   u32 rx_fib_index, u8 is_output)
2986 {
2987   snat_main_t *sm = &snat_main;
2988   clib_bihash_kv_16_8_t kv16, value16;
2989
2990   u8 proto, next_worker_index = 0;
2991   u16 port;
2992   snat_static_mapping_t *m;
2993   u32 hash;
2994
2995   proto = ip->protocol;
2996
2997   if (PREDICT_FALSE (IP_PROTOCOL_ICMP == proto))
2998     {
2999       ip4_address_t lookup_saddr, lookup_daddr;
3000       u16 lookup_sport, lookup_dport;
3001       u8 lookup_protocol;
3002       if (!nat_get_icmp_session_lookup_values (
3003             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3004             &lookup_protocol))
3005         {
3006           init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3007                      lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3008                      lookup_protocol);
3009           if (PREDICT_TRUE (
3010                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3011             {
3012               next_worker_index = ed_value_get_thread_index (&value16);
3013               nat_elog_debug_handoff (
3014                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
3015                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
3016                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3017               return next_worker_index;
3018             }
3019         }
3020     }
3021
3022   init_ed_k (&kv16, ip->src_address.as_u32,
3023              vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3024              vnet_buffer (b)->ip.reass.l4_dst_port, rx_fib_index,
3025              ip->protocol);
3026
3027   if (PREDICT_TRUE (
3028         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3029     {
3030       vnet_buffer2 (b)->nat.cached_session_index =
3031         ed_value_get_session_index (&value16);
3032       next_worker_index = ed_value_get_thread_index (&value16);
3033       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
3034                               next_worker_index, rx_fib_index,
3035                               clib_net_to_host_u32 (ip->src_address.as_u32),
3036                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3037       return next_worker_index;
3038     }
3039
3040   /* first try static mappings without port */
3041   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3042     {
3043       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, proto);
3044       if (m)
3045         {
3046           {
3047             next_worker_index = m->workers[0];
3048             goto done;
3049           }
3050         }
3051     }
3052
3053   /* unknown protocol */
3054   if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto)))
3055     {
3056       /* use current thread */
3057       next_worker_index = vlib_get_thread_index ();
3058       goto done;
3059     }
3060
3061   port = vnet_buffer (b)->ip.reass.l4_dst_port;
3062
3063   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3064     {
3065       udp_header_t *udp = ip4_next_header (ip);
3066       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3067       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3068       if (!icmp_type_is_error_message
3069           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3070         port = vnet_buffer (b)->ip.reass.l4_src_port;
3071       else
3072         {
3073           /* if error message, then it's not fragmented and we can access it */
3074           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3075           proto = inner_ip->protocol;
3076           void *l4_header = ip4_next_header (inner_ip);
3077           switch (proto)
3078             {
3079             case IP_PROTOCOL_ICMP:
3080               icmp = (icmp46_header_t *) l4_header;
3081               echo = (icmp_echo_header_t *) (icmp + 1);
3082               port = echo->identifier;
3083               break;
3084             case IP_PROTOCOL_UDP:
3085               /* breakthrough */
3086             case IP_PROTOCOL_TCP:
3087               port = ((tcp_udp_header_t *) l4_header)->src_port;
3088               break;
3089             default:
3090               next_worker_index = vlib_get_thread_index ();
3091               goto done;
3092             }
3093         }
3094     }
3095
3096   /* try static mappings with port */
3097   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3098     {
3099       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, port, 0, proto);
3100       if (m)
3101         {
3102           if (!is_sm_lb (m->flags))
3103             {
3104               next_worker_index = m->workers[0];
3105               goto done;
3106             }
3107
3108           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3109             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3110
3111           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3112             next_worker_index =
3113               m->workers[hash & (_vec_len (m->workers) - 1)];
3114           else
3115             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3116           goto done;
3117         }
3118     }
3119
3120   /* worker by outside port */
3121   next_worker_index = sm->first_worker_index;
3122   next_worker_index +=
3123     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3124
3125 done:
3126   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3127                           rx_fib_index,
3128                           clib_net_to_host_u32 (ip->src_address.as_u32),
3129                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3130   return next_worker_index;
3131 }
3132
3133 u32
3134 nat44_get_max_session_limit ()
3135 {
3136   snat_main_t *sm = &snat_main;
3137   u32 max_limit = 0, len = 0;
3138
3139   for (; len < vec_len (sm->max_translations_per_fib); len++)
3140     {
3141       if (max_limit < sm->max_translations_per_fib[len])
3142         max_limit = sm->max_translations_per_fib[len];
3143     }
3144   return max_limit;
3145 }
3146
3147 int
3148 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3149 {
3150   snat_main_t *sm = &snat_main;
3151   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3152   u32 len = vec_len (sm->max_translations_per_fib);
3153
3154   if (len <= fib_index)
3155     {
3156       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3157
3158       for (; len < vec_len (sm->max_translations_per_fib); len++)
3159         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3160     }
3161
3162   sm->max_translations_per_fib[fib_index] = session_limit;
3163   return 0;
3164 }
3165
3166 int
3167 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3168 {
3169   snat_main_t *sm = &snat_main;
3170
3171   if (nat44_set_session_limit (session_limit, vrf_id))
3172     return 1;
3173   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3174
3175   stat_segment_set_state_counter (sm->max_cfg_sessions_gauge,
3176                                   sm->max_translations_per_thread);
3177
3178   sm->translation_buckets =
3179     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3180
3181   nat44_ed_sessions_clear ();
3182   return 0;
3183 }
3184
3185 static void
3186 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3187                          u32 translation_buckets)
3188 {
3189   dlist_elt_t *head;
3190
3191   pool_alloc (tsm->sessions, translations);
3192   pool_alloc (tsm->lru_pool, translations);
3193
3194   pool_get (tsm->lru_pool, head);
3195   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3196   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3197
3198   pool_get (tsm->lru_pool, head);
3199   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3200   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3201
3202   pool_get (tsm->lru_pool, head);
3203   tsm->udp_lru_head_index = head - tsm->lru_pool;
3204   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3205
3206   pool_get (tsm->lru_pool, head);
3207   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3208   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3209
3210   pool_get (tsm->lru_pool, head);
3211   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3212   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3213 }
3214
3215 static void
3216 reinit_ed_flow_hash ()
3217 {
3218   snat_main_t *sm = &snat_main;
3219   // we expect 2 flows per session, so multiply translation_buckets by 2
3220   clib_bihash_init_16_8 (
3221     &sm->flow_hash, "ed-flow-hash",
3222     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3223   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3224 }
3225
3226 static void
3227 nat44_ed_db_init (u32 translations, u32 translation_buckets)
3228 {
3229   snat_main_t *sm = &snat_main;
3230   snat_main_per_thread_data_t *tsm;
3231
3232   reinit_ed_flow_hash ();
3233
3234   if (sm->pat)
3235     {
3236       vec_foreach (tsm, sm->per_thread_data)
3237         {
3238           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3239                                    sm->translation_buckets);
3240         }
3241     }
3242 }
3243
3244 static void
3245 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3246 {
3247   pool_free (tsm->lru_pool);
3248   pool_free (tsm->sessions);
3249   vec_free (tsm->per_vrf_sessions_vec);
3250 }
3251
3252 void
3253 nat44_ed_sessions_clear ()
3254 {
3255   snat_main_t *sm = &snat_main;
3256   snat_main_per_thread_data_t *tsm;
3257
3258   reinit_ed_flow_hash ();
3259
3260   if (sm->pat)
3261     {
3262       vec_foreach (tsm, sm->per_thread_data)
3263         {
3264
3265           nat44_ed_worker_db_free (tsm);
3266           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3267                                    sm->translation_buckets);
3268         }
3269     }
3270   vlib_zero_simple_counter (&sm->total_sessions, 0);
3271 }
3272
3273 static void
3274 nat44_ed_add_del_static_mapping_addr_only_cb (
3275   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
3276   u32 address_length, u32 if_address_index, u32 is_delete)
3277 {
3278   snat_static_map_resolve_t *rp;
3279   snat_main_t *sm = &snat_main;
3280   snat_static_mapping_t *m;
3281   int i, rv = 0, match = 0;
3282
3283   if (!sm->enabled)
3284     {
3285       return;
3286     }
3287
3288   // find first addr_only resolve record by sw_if_index
3289   for (i = 0; i < vec_len (sm->to_resolve); i++)
3290     {
3291       rp = sm->to_resolve + i;
3292       if (is_sm_addr_only (rp->flags) && rp->sw_if_index == sw_if_index)
3293         {
3294           match = 1;
3295           break;
3296         }
3297     }
3298   if (!match)
3299     {
3300       return;
3301     }
3302
3303   m = nat44_ed_sm_o2i_lookup (
3304     sm, *address, is_sm_addr_only (rp->flags) ? 0 : rp->e_port, 0, rp->proto);
3305
3306   if (is_delete)
3307     {
3308       if (m)
3309         {
3310           rv = nat44_ed_del_static_mapping_internal (
3311             rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3312             rp->vrf_id, ~0, rp->flags);
3313         }
3314       if (rv)
3315         {
3316           nat_elog_notice_X1 (sm, "nat44_ed_del_static_mapping returned %d",
3317                               "i4", rv);
3318         }
3319     }
3320   else
3321     {
3322       if (!m)
3323         {
3324           rv = nat44_ed_add_static_mapping_internal (
3325             rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3326             rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3327         }
3328       // else: don't trip over lease renewal, static config
3329       if (rv)
3330         {
3331           nat_elog_notice_X1 (sm, "nat44_ed_add_static_mapping returned %d",
3332                               "i4", rv);
3333         }
3334     }
3335 }
3336
3337 static_always_inline int
3338 is_sw_if_index_reg_for_auto_resolve (u32 *sw_if_indices, u32 sw_if_index)
3339 {
3340   u32 *i;
3341   vec_foreach (i, sw_if_indices)
3342     {
3343       if (*i == sw_if_index)
3344         {
3345           return 1;
3346         }
3347     }
3348   return 0;
3349 }
3350
3351 static void
3352 nat44_ed_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3353                                        u32 sw_if_index, ip4_address_t *address,
3354                                        u32 address_length,
3355                                        u32 if_address_index, u32 is_delete)
3356 {
3357   snat_main_t *sm = &snat_main;
3358   snat_static_map_resolve_t *rp;
3359   snat_address_t *ap, *addresses = sm->addresses;
3360   u8 twice_nat = 0;
3361   int rv, i;
3362
3363   if (!sm->enabled)
3364     {
3365       return;
3366     }
3367
3368   if (!is_sw_if_index_reg_for_auto_resolve (sm->auto_add_sw_if_indices,
3369                                             sw_if_index))
3370     {
3371       if (!is_sw_if_index_reg_for_auto_resolve (
3372             sm->auto_add_sw_if_indices_twice_nat, sw_if_index))
3373         {
3374           // interface resolve
3375           u32 fib_index =
3376             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
3377           vec_foreach (ap, sm->addresses)
3378             {
3379               if ((fib_index == ap->fib_index) &&
3380                   (address->as_u32 == ap->addr.as_u32))
3381                 {
3382                   if (!is_delete)
3383                     {
3384                       ap->addr_len = address_length;
3385                       ap->sw_if_index = sw_if_index;
3386                       ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
3387                                        << (32 - ap->addr_len);
3388
3389                       nat_log_debug (
3390                         "pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
3391                         format_ip4_address, &ap->addr, ap->sw_if_index,
3392                         format_ip4_address, &ap->net, ap->addr_len);
3393                     }
3394                   else
3395                     {
3396                       ap->addr_len = ~0;
3397                     }
3398                   break;
3399                 }
3400             }
3401           return;
3402         }
3403       else
3404         {
3405           addresses = sm->twice_nat_addresses;
3406           twice_nat = 1;
3407         }
3408     }
3409
3410   if (!is_delete)
3411     {
3412       // don't trip over lease renewal, static config
3413       for (i = 0; i < vec_len (addresses); i++)
3414         {
3415           if (addresses[i].addr.as_u32 == address->as_u32)
3416             {
3417               return;
3418             }
3419         }
3420
3421       (void) nat44_ed_add_address (address, ~0, twice_nat);
3422
3423       // scan static mapping switch address resolution record vector
3424       for (i = 0; i < vec_len (sm->to_resolve); i++)
3425         {
3426           rp = sm->to_resolve + i;
3427           if (is_sm_addr_only (rp->flags))
3428             {
3429               continue;
3430             }
3431           if (rp->sw_if_index == sw_if_index)
3432             {
3433               rv = nat44_ed_add_static_mapping_internal (
3434                 rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3435                 rp->vrf_id, sw_if_index, rp->flags, rp->pool_addr, rp->tag);
3436               if (rv)
3437                 {
3438                   nat_elog_notice_X1 (
3439                     sm, "add_static_mapping_internal returned %d", "i4", rv);
3440                 }
3441             }
3442         }
3443     }
3444   else
3445     {
3446       // remove all static mapping records
3447       (void) nat44_ed_del_address (address[0], 1, twice_nat);
3448     }
3449 }
3450
3451 int
3452 nat44_ed_add_interface_address (u32 sw_if_index, u8 twice_nat)
3453 {
3454   snat_main_t *sm = &snat_main;
3455   ip4_main_t *ip4_main = sm->ip4_main;
3456   ip4_address_t *first_int_addr;
3457   u32 *auto_add_sw_if_indices = twice_nat ?
3458                                   sm->auto_add_sw_if_indices_twice_nat :
3459                                   sm->auto_add_sw_if_indices;
3460   int i;
3461
3462   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3463     {
3464       if (auto_add_sw_if_indices[i] == sw_if_index)
3465         {
3466           return VNET_API_ERROR_VALUE_EXIST;
3467         }
3468     }
3469
3470   // add to the auto-address list
3471   if (twice_nat)
3472     {
3473       vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3474     }
3475   else
3476     {
3477       vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3478     }
3479
3480   // if the address is already bound - or static - add it now
3481   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3482   if (first_int_addr)
3483     {
3484       (void) nat44_ed_add_address (first_int_addr, ~0, twice_nat);
3485     }
3486
3487   return 0;
3488 }
3489
3490 int
3491 nat44_ed_del_interface_address (u32 sw_if_index, u8 twice_nat)
3492 {
3493   snat_main_t *sm = &snat_main;
3494   ip4_main_t *ip4_main = sm->ip4_main;
3495   ip4_address_t *first_int_addr;
3496   snat_static_map_resolve_t *rp;
3497   u32 *indices_to_delete = 0;
3498   int i, j;
3499   u32 *auto_add_sw_if_indices;
3500
3501   if (!sm->enabled)
3502     {
3503       return VNET_API_ERROR_UNSUPPORTED;
3504     }
3505
3506   auto_add_sw_if_indices = twice_nat ? sm->auto_add_sw_if_indices_twice_nat :
3507                                        sm->auto_add_sw_if_indices;
3508
3509   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3510     {
3511       if (auto_add_sw_if_indices[i] == sw_if_index)
3512         {
3513           first_int_addr =
3514             ip4_interface_first_address (ip4_main, sw_if_index, 0);
3515           if (first_int_addr)
3516             {
3517               // remove all static mapping records
3518               (void) nat44_ed_del_address (first_int_addr[0], 1, twice_nat);
3519             }
3520           else
3521             {
3522               for (j = 0; j < vec_len (sm->to_resolve); j++)
3523                 {
3524                   rp = sm->to_resolve + j;
3525                   if (rp->sw_if_index == sw_if_index)
3526                     {
3527                       vec_add1 (indices_to_delete, j);
3528                     }
3529                 }
3530               if (vec_len (indices_to_delete))
3531                 {
3532                   for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3533                     {
3534                       vec_del1 (sm->to_resolve, j);
3535                     }
3536                   vec_free (indices_to_delete);
3537                 }
3538             }
3539
3540           if (twice_nat)
3541             {
3542               vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3543             }
3544           else
3545             {
3546               vec_del1 (sm->auto_add_sw_if_indices, i);
3547             }
3548           return 0;
3549         }
3550     }
3551   return VNET_API_ERROR_NO_SUCH_ENTRY;
3552 }
3553
3554 int
3555 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3556                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3557                       u32 vrf_id, int is_in)
3558 {
3559   ip4_header_t ip;
3560   clib_bihash_kv_16_8_t kv, value;
3561   u32 fib_index;
3562   snat_session_t *s;
3563   snat_main_per_thread_data_t *tsm;
3564
3565   if (!sm->enabled)
3566     {
3567       return VNET_API_ERROR_UNSUPPORTED;
3568     }
3569
3570   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3571   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3572   if (sm->num_workers > 1)
3573     tsm = vec_elt_at_index (
3574       sm->per_thread_data,
3575       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3576   else
3577     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3578
3579   init_ed_k (&kv, addr->as_u32, port, eh_addr->as_u32, eh_port, fib_index,
3580              proto);
3581   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3582     {
3583       return VNET_API_ERROR_NO_SUCH_ENTRY;
3584     }
3585
3586   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3587     return VNET_API_ERROR_UNSPECIFIED;
3588   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3589   nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3590   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3591   return 0;
3592 }
3593
3594 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3595                                  vlib_node_runtime_t * node,
3596                                  vlib_frame_t * frame)
3597 {
3598   return 0;
3599 }
3600
3601 VLIB_REGISTER_NODE (nat_default_node) = {
3602   .name = "nat-default",
3603   .vector_size = sizeof (u32),
3604   .format_trace = 0,
3605   .type = VLIB_NODE_TYPE_INTERNAL,
3606   .n_errors = 0,
3607   .n_next_nodes = NAT_N_NEXT,
3608   .next_nodes = {
3609     [NAT_NEXT_DROP] = "error-drop",
3610     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3611     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3612     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3613     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3614     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3615     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3616     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3617     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3618     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3619   },
3620 };
3621
3622 void
3623 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3624 {
3625   f->l3_csum_delta = 0;
3626   f->l4_csum_delta = 0;
3627   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3628       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3629     {
3630       f->l3_csum_delta =
3631         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3632       f->l3_csum_delta =
3633         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3634     }
3635   else
3636     {
3637       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3638     }
3639   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3640       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3641     {
3642       f->l3_csum_delta =
3643         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3644       f->l3_csum_delta =
3645         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3646     }
3647   else
3648     {
3649       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3650     }
3651   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3652     {
3653       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3654       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3655     }
3656   else
3657     {
3658       f->rewrite.sport = f->match.sport;
3659     }
3660   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3661     {
3662       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3663       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3664     }
3665   else
3666     {
3667       f->rewrite.dport = f->match.dport;
3668     }
3669   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3670       f->rewrite.icmp_id != f->match.sport)
3671     {
3672       f->l4_csum_delta =
3673         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3674       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3675     }
3676   else
3677     {
3678       f->rewrite.icmp_id = f->match.sport;
3679     }
3680   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3681     {
3682     }
3683   else
3684     {
3685       f->rewrite.fib_index = f->match.fib_index;
3686     }
3687 }
3688
3689 static_always_inline int
3690 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3691                             ip4_header_t *ip, nat_6t_flow_t *f);
3692
3693 static_always_inline void
3694 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3695                            nat_6t_flow_t *f, ip_protocol_t proto,
3696                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3697 {
3698   udp_header_t *udp = ip4_next_header (ip);
3699   tcp_header_t *tcp = (tcp_header_t *) udp;
3700
3701   if ((IP_PROTOCOL_TCP == proto || IP_PROTOCOL_UDP == proto) &&
3702       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3703     {
3704       if (!is_icmp_inner_ip4)
3705         { // regular case
3706           ip->src_address = f->rewrite.saddr;
3707           ip->dst_address = f->rewrite.daddr;
3708           udp->src_port = f->rewrite.sport;
3709           udp->dst_port = f->rewrite.dport;
3710         }
3711       else
3712         { // icmp inner ip4 - reversed saddr/daddr
3713           ip->src_address = f->rewrite.daddr;
3714           ip->dst_address = f->rewrite.saddr;
3715           udp->src_port = f->rewrite.dport;
3716           udp->dst_port = f->rewrite.sport;
3717         }
3718
3719       if (IP_PROTOCOL_TCP == proto)
3720         {
3721           ip_csum_t tcp_sum = tcp->checksum;
3722           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3723           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3724           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3725           tcp->checksum = ip_csum_fold (tcp_sum);
3726         }
3727       else if (IP_PROTOCOL_UDP == proto && udp->checksum)
3728         {
3729           ip_csum_t udp_sum = udp->checksum;
3730           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3731           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3732           udp->checksum = ip_csum_fold (udp_sum);
3733         }
3734     }
3735   else
3736     {
3737       if (!is_icmp_inner_ip4)
3738         { // regular case
3739           if (!skip_saddr_rewrite)
3740             {
3741               ip->src_address = f->rewrite.saddr;
3742             }
3743           ip->dst_address = f->rewrite.daddr;
3744         }
3745       else
3746         { // icmp inner ip4 - reversed saddr/daddr
3747           ip->src_address = f->rewrite.daddr;
3748           ip->dst_address = f->rewrite.saddr;
3749         }
3750     }
3751
3752   if (skip_saddr_rewrite)
3753     {
3754       ip->checksum = ip4_header_checksum (ip);
3755     }
3756   else
3757     {
3758       ip_csum_t ip_sum = ip->checksum;
3759       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3760       ip->checksum = ip_csum_fold (ip_sum);
3761     }
3762   if (0xffff == ip->checksum)
3763     ip->checksum = 0;
3764   ASSERT (ip4_header_checksum_is_valid (ip));
3765 }
3766
3767 static_always_inline int
3768 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3769 {
3770   int result = ((u8 *) object + size <=
3771                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3772                vlib_object_within_buffer_data (vm, b, object, size);
3773   return result;
3774 }
3775
3776 static_always_inline int
3777 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3778                             ip4_header_t *ip, nat_6t_flow_t *f)
3779 {
3780   if (IP_PROTOCOL_ICMP != ip->protocol)
3781     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3782
3783   icmp46_header_t *icmp = ip4_next_header (ip);
3784   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3785
3786   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3787     {
3788       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3789         {
3790           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3791         }
3792
3793       if (!icmp_type_is_error_message (icmp->type))
3794         {
3795           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3796               (f->rewrite.icmp_id != echo->identifier))
3797             {
3798               ip_csum_t sum = icmp->checksum;
3799               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3800                                     icmp_echo_header_t,
3801                                     identifier /* changed member */);
3802               echo->identifier = f->rewrite.icmp_id;
3803               icmp->checksum = ip_csum_fold (sum);
3804             }
3805         }
3806       else
3807         {
3808           ip_csum_t sum = ip_incremental_checksum (
3809             0, icmp,
3810             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3811           sum = (u16) ~ip_csum_fold (sum);
3812           if (sum != 0)
3813             {
3814               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3815             }
3816
3817           // errors are not fragmented
3818           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3819
3820           if (!ip4_header_checksum_is_valid (inner_ip))
3821             {
3822               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3823             }
3824
3825           ip_protocol_t inner_proto = inner_ip->protocol;
3826
3827           ip_csum_t old_icmp_sum = icmp->checksum;
3828           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3829           ip_csum_t old_udp_sum;
3830           ip_csum_t old_tcp_sum;
3831           ip_csum_t new_icmp_sum;
3832           udp_header_t *udp;
3833           tcp_header_t *tcp;
3834
3835           switch (inner_proto)
3836             {
3837             case IP_PROTOCOL_UDP:
3838               udp = (udp_header_t *) (inner_ip + 1);
3839               if (!it_fits (vm, b, udp, sizeof (*udp)))
3840                 {
3841                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3842                 }
3843               old_udp_sum = udp->checksum;
3844               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3845                                          1 /* is_icmp_inner_ip4 */,
3846                                          0 /* skip_saddr_rewrite */);
3847               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3848               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3849               new_icmp_sum =
3850                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3851                                 inner_ip->checksum, ip4_header_t, checksum);
3852               new_icmp_sum =
3853                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3854                                 udp_header_t, checksum);
3855               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3856               icmp->checksum = new_icmp_sum;
3857               break;
3858             case IP_PROTOCOL_TCP:
3859               tcp = (tcp_header_t *) (inner_ip + 1);
3860               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3861                 {
3862                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3863                 }
3864               old_tcp_sum = tcp->checksum;
3865               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3866                                          1 /* is_icmp_inner_ip4 */,
3867                                          0 /* skip_saddr_rewrite */);
3868               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3869               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3870               new_icmp_sum =
3871                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3872                                 inner_ip->checksum, ip4_header_t, checksum);
3873               new_icmp_sum =
3874                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3875                                 tcp_header_t, checksum);
3876               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3877               icmp->checksum = new_icmp_sum;
3878               break;
3879             case IP_PROTOCOL_ICMP:
3880               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3881                 {
3882                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3883                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3884                     {
3885                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3886                     }
3887                   icmp_echo_header_t *inner_echo =
3888                     (icmp_echo_header_t *) (inner_icmp + 1);
3889                   if (f->rewrite.icmp_id != inner_echo->identifier)
3890                     {
3891                       ip_csum_t sum = icmp->checksum;
3892                       sum = ip_csum_update (
3893                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3894                         icmp_echo_header_t, identifier /* changed member */);
3895                       icmp->checksum = ip_csum_fold (sum);
3896                       ip_csum_t inner_sum = inner_icmp->checksum;
3897                       inner_sum = ip_csum_update (
3898                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3899                         icmp_echo_header_t, identifier /* changed member */);
3900                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3901                       inner_echo->identifier = f->rewrite.icmp_id;
3902                     }
3903                 }
3904               break;
3905             default:
3906               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3907               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3908             }
3909         }
3910     }
3911
3912   return NAT_ED_TRNSL_ERR_SUCCESS;
3913 }
3914
3915 static_always_inline nat_translation_error_e
3916 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3917                            ip4_header_t *ip, nat_6t_flow_t *f,
3918                            ip_protocol_t proto, int is_output_feature,
3919                            int is_i2o)
3920 {
3921   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3922     {
3923       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3924     }
3925
3926   if (IP_PROTOCOL_ICMP == proto)
3927     {
3928       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3929         {
3930           // packet is returned from a router, not from destination
3931           // skip source address rewrite if in o2i path
3932           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3933                                      0 /* is_icmp_inner_ip4 */,
3934                                      !is_i2o /* skip_saddr_rewrite */);
3935         }
3936       else
3937         {
3938           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3939                                      0 /* is_icmp_inner_ip4 */,
3940                                      0 /* skip_saddr_rewrite */);
3941         }
3942       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
3943     }
3944
3945   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
3946                              0 /* skip_saddr_rewrite */);
3947
3948   return NAT_ED_TRNSL_ERR_SUCCESS;
3949 }
3950
3951 nat_translation_error_e
3952 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
3953                                vlib_buffer_t *b, ip4_header_t *ip,
3954                                nat_6t_flow_t *f, ip_protocol_t proto,
3955                                int is_output_feature)
3956 {
3957   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3958                                     1 /* is_i2o */);
3959 }
3960
3961 nat_translation_error_e
3962 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
3963                                vlib_buffer_t *b, ip4_header_t *ip,
3964                                nat_6t_flow_t *f, ip_protocol_t proto,
3965                                int is_output_feature)
3966 {
3967   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3968                                     0 /* is_i2o */);
3969 }
3970
3971 u8 *
3972 format_nat_6t (u8 *s, va_list *args)
3973 {
3974   nat_6t_t *t = va_arg (*args, nat_6t_t *);
3975
3976   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
3977               format_ip4_address, t->saddr.as_u8,
3978               clib_net_to_host_u16 (t->sport), format_ip4_address,
3979               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
3980               format_ip_protocol, t->proto, t->fib_index);
3981   return s;
3982 }
3983
3984 u8 *
3985 format_nat_ed_translation_error (u8 *s, va_list *args)
3986 {
3987   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
3988
3989   switch (e)
3990     {
3991     case NAT_ED_TRNSL_ERR_SUCCESS:
3992       s = format (s, "success");
3993       break;
3994     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
3995       s = format (s, "translation-failed");
3996       break;
3997     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
3998       s = format (s, "flow-mismatch");
3999       break;
4000     case NAT_ED_TRNSL_ERR_PACKET_TRUNCATED:
4001       s = format (s, "packet-truncated");
4002       break;
4003     case NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT:
4004       s = format (s, "inner-ip-corrupted");
4005       break;
4006     case NAT_ED_TRNSL_ERR_INVALID_CSUM:
4007       s = format (s, "invalid-checksum");
4008       break;
4009     }
4010   return s;
4011 }
4012
4013 u8 *
4014 format_nat_6t_flow (u8 *s, va_list *args)
4015 {
4016   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
4017
4018   s = format (s, "match: %U ", format_nat_6t, &f->match);
4019   int r = 0;
4020   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
4021     {
4022       s = format (s, "rewrite: saddr %U ", format_ip4_address,
4023                   f->rewrite.saddr.as_u8);
4024       r = 1;
4025     }
4026   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
4027     {
4028       if (!r)
4029         {
4030           s = format (s, "rewrite: ");
4031           r = 1;
4032         }
4033       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
4034     }
4035   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
4036     {
4037       if (!r)
4038         {
4039           s = format (s, "rewrite: ");
4040           r = 1;
4041         }
4042       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
4043     }
4044   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
4045     {
4046       if (!r)
4047         {
4048           s = format (s, "rewrite: ");
4049           r = 1;
4050         }
4051       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
4052     }
4053   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
4054     {
4055       if (!r)
4056         {
4057           s = format (s, "rewrite: ");
4058           r = 1;
4059         }
4060       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
4061     }
4062   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
4063     {
4064       if (!r)
4065         {
4066           s = format (s, "rewrite: ");
4067           r = 1;
4068         }
4069       s = format (s, "txfib %u ", f->rewrite.fib_index);
4070     }
4071   return s;
4072 }
4073
4074 static inline void
4075 nat_syslog_nat44_sess (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4076                        u16 isport, ip4_address_t *xsaddr, u16 xsport,
4077                        ip4_address_t *idaddr, u16 idport,
4078                        ip4_address_t *xdaddr, u16 xdport, u8 proto, u8 is_add,
4079                        u8 is_twicenat)
4080 {
4081   syslog_msg_t syslog_msg;
4082   fib_table_t *fib;
4083
4084   if (!syslog_is_enabled ())
4085     return;
4086
4087   if (syslog_severity_filter_block (SADD_SDEL_SEVERITY))
4088     return;
4089
4090   fib = fib_table_get (sfibix, FIB_PROTOCOL_IP4);
4091
4092   syslog_msg_init (&syslog_msg, NAT_FACILITY, SADD_SDEL_SEVERITY, NAT_APPNAME,
4093                    is_add ? SADD_MSGID : SDEL_MSGID);
4094
4095   syslog_msg_sd_init (&syslog_msg, NSESS_SDID);
4096   syslog_msg_add_sd_param (&syslog_msg, SSUBIX_SDPARAM_NAME, "%d", ssubix);
4097   syslog_msg_add_sd_param (&syslog_msg, SVLAN_SDPARAM_NAME, "%d",
4098                            fib->ft_table_id);
4099   syslog_msg_add_sd_param (&syslog_msg, IATYP_SDPARAM_NAME, IATYP_IPV4);
4100   syslog_msg_add_sd_param (&syslog_msg, ISADDR_SDPARAM_NAME, "%U",
4101                            format_ip4_address, isaddr);
4102   syslog_msg_add_sd_param (&syslog_msg, ISPORT_SDPARAM_NAME, "%d",
4103                            clib_net_to_host_u16 (isport));
4104   syslog_msg_add_sd_param (&syslog_msg, XATYP_SDPARAM_NAME, IATYP_IPV4);
4105   syslog_msg_add_sd_param (&syslog_msg, XSADDR_SDPARAM_NAME, "%U",
4106                            format_ip4_address, xsaddr);
4107   syslog_msg_add_sd_param (&syslog_msg, XSPORT_SDPARAM_NAME, "%d",
4108                            clib_net_to_host_u16 (xsport));
4109   syslog_msg_add_sd_param (&syslog_msg, PROTO_SDPARAM_NAME, "%d", proto);
4110   syslog_msg_add_sd_param (&syslog_msg, XDADDR_SDPARAM_NAME, "%U",
4111                            format_ip4_address, xdaddr);
4112   syslog_msg_add_sd_param (&syslog_msg, XDPORT_SDPARAM_NAME, "%d",
4113                            clib_net_to_host_u16 (xdport));
4114   if (is_twicenat)
4115     {
4116       syslog_msg_add_sd_param (&syslog_msg, IDADDR_SDPARAM_NAME, "%U",
4117                                format_ip4_address, idaddr);
4118       syslog_msg_add_sd_param (&syslog_msg, IDPORT_SDPARAM_NAME, "%d",
4119                                clib_net_to_host_u16 (idport));
4120     }
4121
4122   syslog_msg_send (&syslog_msg);
4123 }
4124
4125 void
4126 nat_syslog_nat44_sadd (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4127                        u16 isport, ip4_address_t *idaddr, u16 idport,
4128                        ip4_address_t *xsaddr, u16 xsport,
4129                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4130                        u8 is_twicenat)
4131 {
4132   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4133                          idaddr, idport, xdaddr, xdport, proto, 1,
4134                          is_twicenat);
4135 }
4136
4137 void
4138 nat_syslog_nat44_sdel (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4139                        u16 isport, ip4_address_t *idaddr, u16 idport,
4140                        ip4_address_t *xsaddr, u16 xsport,
4141                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4142                        u8 is_twicenat)
4143 {
4144   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4145                          idaddr, idport, xdaddr, xdport, proto, 0,
4146                          is_twicenat);
4147 }
4148
4149 /*
4150  * fd.io coding-style-patch-verification: ON
4151  *
4152  * Local Variables:
4153  * eval: (c-set-style "gnu")
4154  * End:
4155  */