src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vpp/app/version.h>
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/ip/ip_table.h>
  22 #include <vnet/ip/reass/ip4_sv_reass.h>
  23 #include <vnet/fib/fib_table.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/plugin/plugin.h>
  26 #include <vppinfra/bihash_16_8.h>
  27
  28 #include <nat/lib/log.h>
  29 #include <nat/lib/nat_inlines.h>
  30 #include <nat/lib/ipfix_logging.h>
  31 #include <vnet/syslog/syslog.h>
  32 #include <nat/lib/nat_syslog_constants.h>
  33 #include <nat/lib/nat_syslog.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 #include <vlib/stats/stats.h>
  40
  41 snat_main_t snat_main;
  42
  43 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  44                                                            u32 sw_if_index);
  45
  46 #define skip_if_disabled()                                                    \
  47   do                                                                          \
  48     {                                                                         \
  49       snat_main_t *sm = &snat_main;                                           \
  50       if (PREDICT_FALSE (!sm->enabled))                                       \
  51         return;                                                               \
  52     }                                                                         \
  53   while (0)
  54
  55 #define fail_if_enabled()                                                     \
  56   do                                                                          \
  57     {                                                                         \
  58       snat_main_t *sm = &snat_main;                                           \
  59       if (PREDICT_FALSE (sm->enabled))                                        \
  60         {                                                                     \
  61           nat_log_err ("plugin enabled");                                     \
  62           return 1;                                                           \
  63         }                                                                     \
  64     }                                                                         \
  65   while (0)
  66
  67 #define fail_if_disabled()                                                    \
  68   do                                                                          \
  69     {                                                                         \
  70       snat_main_t *sm = &snat_main;                                           \
  71       if (PREDICT_FALSE (!sm->enabled))                                       \
  72         {                                                                     \
  73           nat_log_err ("plugin disabled");                                    \
  74           return 1;                                                           \
  75         }                                                                     \
  76     }                                                                         \
  77   while (0)
  78
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-ed-classify",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  96                                "ip4-sv-reassembly-feature"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-handoff-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 102                                "ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-worker-handoff",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 108 };
 109 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-worker-handoff",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-in2out",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-out2in",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 124                                "ip4-dhcp-client-detect"),
 125 };
 126 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 127   .arc_name = "ip4-output",
 128   .node_name = "nat-pre-in2out-output",
 129   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 130   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 136   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-ed-in2out-output",
 141   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 142   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 143 };
 144
 145 VLIB_PLUGIN_REGISTER () = {
 146     .version = VPP_BUILD_VER,
 147     .description = "Network Address Translation (NAT)",
 148 };
 149
 150 static void nat44_ed_db_init (u32 translations, u32 translation_buckets);
 151 static void nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm);
 152
 153 static int nat44_ed_add_static_mapping_internal (
 154   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 155   ip_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 156   ip4_address_t pool_addr, u8 *tag);
 157 static int nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
 158                                                  ip4_address_t e_addr,
 159                                                  u16 l_port, u16 e_port,
 160                                                  ip_protocol_t proto,
 161                                                  u32 vrf_id, u32 flags);
 162
 163 u32 nat_calc_bihash_buckets (u32 n_elts);
 164
 165 static_always_inline int
 166 nat44_ed_sm_i2o_add (snat_main_t *sm, snat_static_mapping_t *m,
 167                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 168 {
 169   ASSERT (!pool_is_free (sm->static_mappings, m));
 170   clib_bihash_kv_16_8_t kv;
 171   nat44_ed_sm_init_i2o_kv (&kv, addr.as_u32, port, fib_index, proto,
 172                            m - sm->static_mappings);
 173   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 174 }
 175
 176 static_always_inline int
 177 nat44_ed_sm_i2o_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 178                      u32 fib_index, u8 proto)
 179 {
 180   clib_bihash_kv_16_8_t kv;
 181   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 182   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 183 }
 184
 185 static_always_inline int
 186 nat44_ed_sm_o2i_add (snat_main_t *sm, snat_static_mapping_t *m,
 187                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 188 {
 189   ASSERT (!pool_is_free (sm->static_mappings, m));
 190   clib_bihash_kv_16_8_t kv;
 191   nat44_ed_sm_init_o2i_kv (&kv, addr.as_u32, port, fib_index, proto,
 192                            m - sm->static_mappings);
 193   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 194 }
 195
 196 static_always_inline int
 197 nat44_ed_sm_o2i_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 198                      u32 fib_index, u8 proto)
 199 {
 200   clib_bihash_kv_16_8_t kv;
 201   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 202   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 203 }
 204
 205 void
 206 nat44_ed_free_session_data (snat_main_t *sm, snat_session_t *s,
 207                             u32 thread_index, u8 is_ha)
 208 {
 209   per_vrf_sessions_unregister_session (s, thread_index);
 210
 211   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 212     nat_elog_warn (sm, "flow hash del failed");
 213
 214   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 215     nat_elog_warn (sm, "flow hash del failed");
 216
 217   if (na44_ed_is_fwd_bypass_session (s))
 218     {
 219       return;
 220     }
 221
 222   if (nat44_ed_is_affinity_session (s))
 223     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, s->proto,
 224                          s->out2in.port);
 225
 226   if (!is_ha)
 227     nat_syslog_nat44_sdel (0, s->in2out.fib_index, &s->in2out.addr,
 228                            s->in2out.port, &s->ext_host_nat_addr,
 229                            s->ext_host_nat_port, &s->out2in.addr,
 230                            s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 231                            s->proto, nat44_ed_is_twice_nat_session (s));
 232
 233   if (!is_ha)
 234     {
 235       /* log NAT event */
 236       nat_ipfix_logging_nat44_ses_delete (
 237         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 238         s->in2out.port, s->out2in.port, s->in2out.fib_index);
 239     }
 240 }
 241
 242 static ip_interface_address_t *
 243 nat44_ed_get_ip_interface_address (u32 sw_if_index, ip4_address_t addr)
 244 {
 245   snat_main_t *sm = &snat_main;
 246
 247   ip_lookup_main_t *lm = &sm->ip4_main->lookup_main;
 248   ip_interface_address_t *ia;
 249   ip4_address_t *ip4a;
 250
 251   foreach_ip_interface_address (
 252     lm, ia, sw_if_index, 1, ({
 253       ip4a = ip_interface_address_get_address (lm, ia);
 254       nat_log_debug ("sw_if_idx: %u addr: %U ? %U", sw_if_index,
 255                      format_ip4_address, ip4a, format_ip4_address, &addr);
 256       if (ip4a->as_u32 == addr.as_u32)
 257         {
 258           return ia;
 259         }
 260     }));
 261   return NULL;
 262 }
 263
 264 static int
 265 nat44_ed_resolve_nat_addr_len (snat_address_t *ap,
 266                                snat_interface_t *interfaces)
 267 {
 268   ip_interface_address_t *ia;
 269   snat_interface_t *i;
 270   u32 fib_index;
 271
 272   pool_foreach (i, interfaces)
 273     {
 274       if (!nat44_ed_is_interface_outside (i))
 275         {
 276           continue;
 277         }
 278
 279       fib_index = ip4_fib_table_get_index_for_sw_if_index (i->sw_if_index);
 280       if (fib_index != ap->fib_index)
 281         {
 282           continue;
 283         }
 284
 285       if ((ia = nat44_ed_get_ip_interface_address (i->sw_if_index, ap->addr)))
 286         {
 287           ap->addr_len = ia->address_length;
 288           ap->sw_if_index = i->sw_if_index;
 289           ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
 290                            << (32 - ap->addr_len);
 291
 292           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 293                          format_ip4_address, &ap->addr, ap->sw_if_index,
 294                          format_ip4_address, &ap->net, ap->addr_len);
 295           return 0;
 296         }
 297     }
 298   return 1;
 299 }
 300
 301 static void
 302 nat44_ed_update_outside_if_addresses (snat_address_t *ap)
 303 {
 304   snat_main_t *sm = &snat_main;
 305
 306   if (!nat44_ed_resolve_nat_addr_len (ap, sm->interfaces))
 307     {
 308       return;
 309     }
 310
 311   if (!nat44_ed_resolve_nat_addr_len (ap, sm->output_feature_interfaces))
 312     {
 313       return;
 314     }
 315 }
 316
 317 static void
 318 nat44_ed_bind_if_addr_to_nat_addr (u32 sw_if_index)
 319 {
 320   snat_main_t *sm = &snat_main;
 321   ip_interface_address_t *ia;
 322   snat_address_t *ap;
 323
 324   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 325
 326   vec_foreach (ap, sm->addresses)
 327     {
 328       if (fib_index != ap->fib_index)
 329         {
 330           continue;
 331         }
 332
 333       if ((ia = nat44_ed_get_ip_interface_address (sw_if_index, ap->addr)))
 334         {
 335           ap->addr_len = ia->address_length;
 336           ap->sw_if_index = sw_if_index;
 337           ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
 338                            << (32 - ap->addr_len);
 339
 340           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 341                          format_ip4_address, &ap->addr, ap->sw_if_index,
 342                          format_ip4_address, &ap->net, ap->addr_len);
 343           return;
 344         }
 345     }
 346 }
 347
 348 static_always_inline snat_fib_entry_reg_t *
 349 nat44_ed_get_fib_entry_reg (ip4_address_t addr, u32 sw_if_index, int *out_idx)
 350 {
 351   snat_main_t *sm = &snat_main;
 352   snat_fib_entry_reg_t *fe;
 353   int i;
 354
 355   for (i = 0; i < vec_len (sm->fib_entry_reg); i++)
 356     {
 357       fe = sm->fib_entry_reg + i;
 358       if ((addr.as_u32 == fe->addr.as_u32) && (sw_if_index == fe->sw_if_index))
 359         {
 360           if (out_idx)
 361             {
 362               *out_idx = i;
 363             }
 364           return fe;
 365         }
 366     }
 367   return NULL;
 368 }
 369
 370 static void
 371 nat44_ed_add_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 372 {
 373   // Add the external NAT address to the FIB as receive entries. This ensures
 374   // that VPP will reply to ARP for this address and we don't need to enable
 375   // proxy ARP on the outside interface.
 376   snat_main_t *sm = &snat_main;
 377   snat_fib_entry_reg_t *fe;
 378
 379   if (!(fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, 0)))
 380     {
 381       fib_prefix_t prefix = {
 382         .fp_len = 32,
 383         .fp_proto = FIB_PROTOCOL_IP4,
 384         .fp_addr = {
 385                   .ip4.as_u32 = addr.as_u32,
 386                 },
 387       };
 388       u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 389       fib_table_entry_update_one_path (fib_index, &prefix, sm->fib_src_low,
 390                                        (FIB_ENTRY_FLAG_CONNECTED |
 391                                         FIB_ENTRY_FLAG_LOCAL |
 392                                         FIB_ENTRY_FLAG_EXCLUSIVE),
 393                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
 394                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 395
 396       vec_add2 (sm->fib_entry_reg, fe, 1);
 397       clib_memset (fe, 0, sizeof (*fe));
 398       fe->addr.as_u32 = addr.as_u32;
 399       fe->sw_if_index = sw_if_index;
 400     }
 401   fe->count++;
 402 }
 403
 404 static void
 405 nat44_ed_del_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 406 {
 407   snat_main_t *sm = &snat_main;
 408   snat_fib_entry_reg_t *fe;
 409   int i;
 410
 411   if ((fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, &i)))
 412     {
 413       fe->count--;
 414       if (0 == fe->count)
 415         {
 416           fib_prefix_t prefix = {
 417             .fp_len = 32,
 418             .fp_proto = FIB_PROTOCOL_IP4,
 419             .fp_addr = {
 420               .ip4.as_u32 = addr.as_u32,
 421                     },
 422           };
 423           u32 fib_index =
 424             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 425           fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 426           vec_del1 (sm->fib_entry_reg, i);
 427         }
 428     }
 429 }
 430
 431 static void
 432 nat44_ed_add_del_interface_fib_reg_entries (ip4_address_t addr, u8 is_add)
 433 {
 434   snat_main_t *sm = &snat_main;
 435   snat_interface_t *i;
 436
 437   pool_foreach (i, sm->interfaces)
 438     {
 439       if (nat44_ed_is_interface_outside (i))
 440         {
 441           if (is_add)
 442             {
 443               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 444             }
 445           else
 446             {
 447               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 448             }
 449         }
 450     }
 451   pool_foreach (i, sm->output_feature_interfaces)
 452     {
 453       if (nat44_ed_is_interface_outside (i))
 454         {
 455           if (is_add)
 456             {
 457               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 458             }
 459           else
 460             {
 461               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 462             }
 463         }
 464     }
 465 }
 466
 467 static_always_inline void
 468 nat44_ed_add_del_nat_addr_fib_reg_entries (u32 sw_if_index, u8 is_add)
 469 {
 470   snat_main_t *sm = &snat_main;
 471   snat_address_t *ap;
 472
 473   vec_foreach (ap, sm->addresses)
 474     {
 475       if (is_add)
 476         {
 477           nat44_ed_add_fib_entry_reg (ap->addr, sw_if_index);
 478         }
 479       else
 480         {
 481           nat44_ed_del_fib_entry_reg (ap->addr, sw_if_index);
 482         }
 483     }
 484 }
 485
 486 static_always_inline void
 487 nat44_ed_add_del_sm_fib_reg_entries (u32 sw_if_index, u8 is_add)
 488 {
 489   snat_main_t *sm = &snat_main;
 490   snat_static_mapping_t *m;
 491
 492   pool_foreach (m, sm->static_mappings)
 493     {
 494       if (is_add)
 495         {
 496           nat44_ed_add_fib_entry_reg (m->external_addr, sw_if_index);
 497         }
 498       else
 499         {
 500           nat44_ed_del_fib_entry_reg (m->external_addr, sw_if_index);
 501         }
 502     }
 503 }
 504
 505 int
 506 nat44_ed_add_address (ip4_address_t *addr, u32 vrf_id, u8 twice_nat)
 507 {
 508   snat_main_t *sm = &snat_main;
 509   snat_address_t *ap, *addresses;
 510
 511   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 512
 513   if (!sm->enabled)
 514     {
 515       return VNET_API_ERROR_UNSUPPORTED;
 516     }
 517
 518   // check if address already exists
 519   vec_foreach (ap, addresses)
 520     {
 521       if (ap->addr.as_u32 == addr->as_u32)
 522         {
 523           nat_log_err ("address exist");
 524           return VNET_API_ERROR_VALUE_EXIST;
 525         }
 526     }
 527
 528   if (twice_nat)
 529     {
 530       vec_add2 (sm->twice_nat_addresses, ap, 1);
 531     }
 532   else
 533     {
 534       vec_add2 (sm->addresses, ap, 1);
 535     }
 536
 537   ap->addr_len = ~0;
 538   ap->fib_index = ~0;
 539   ap->addr = *addr;
 540
 541   if (vrf_id != ~0)
 542     {
 543       ap->fib_index = fib_table_find_or_create_and_lock (
 544         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 545     }
 546
 547   if (!twice_nat)
 548     {
 549       // if we don't have enabled interface we don't add address
 550       // to fib
 551       nat44_ed_add_del_interface_fib_reg_entries (*addr, 1);
 552       nat44_ed_update_outside_if_addresses (ap);
 553     }
 554   return 0;
 555 }
 556
 557 int
 558 nat44_ed_del_address (ip4_address_t addr, u8 twice_nat)
 559 {
 560   snat_main_t *sm = &snat_main;
 561   snat_address_t *a = 0, *addresses;
 562   snat_session_t *ses;
 563   u32 *ses_to_be_removed = 0, *ses_index;
 564   snat_main_per_thread_data_t *tsm;
 565   int j;
 566
 567   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 568
 569   for (j = 0; j < vec_len (addresses); j++)
 570     {
 571       if (addresses[j].addr.as_u32 == addr.as_u32)
 572         {
 573           a = addresses + j;
 574           break;
 575         }
 576     }
 577   if (!a)
 578     {
 579       nat_log_err ("no such address");
 580       return VNET_API_ERROR_NO_SUCH_ENTRY;
 581     }
 582
 583   // delete dynamic sessions only
 584   vec_foreach (tsm, sm->per_thread_data)
 585     {
 586       pool_foreach (ses, tsm->sessions)
 587         {
 588           if (ses->flags & SNAT_SESSION_FLAG_STATIC_MAPPING)
 589             {
 590               continue;
 591             }
 592           if (ses->out2in.addr.as_u32 == addr.as_u32)
 593             {
 594               nat44_ed_free_session_data (sm, ses, tsm - sm->per_thread_data,
 595                                           0);
 596               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
 597             }
 598         }
 599       vec_foreach (ses_index, ses_to_be_removed)
 600         {
 601           ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
 602           nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
 603         }
 604       vec_free (ses_to_be_removed);
 605     }
 606
 607   if (!twice_nat)
 608     {
 609       nat44_ed_add_del_interface_fib_reg_entries (addr, 0);
 610     }
 611
 612   if (a->fib_index != ~0)
 613     {
 614       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 615     }
 616
 617   if (!twice_nat)
 618     {
 619       vec_del1 (sm->addresses, j);
 620     }
 621   else
 622     {
 623       vec_del1 (sm->twice_nat_addresses, j);
 624     }
 625
 626   return 0;
 627 }
 628
 629 vrf_table_t *
 630 nat44_ed_get_vrf_table (u32 table_vrf_id)
 631 {
 632   snat_main_t *sm = &snat_main;
 633   vrf_table_t *t;
 634
 635   pool_foreach (t, sm->vrf_tables)
 636     {
 637       if (table_vrf_id == t->table_vrf_id)
 638         {
 639           return t;
 640         }
 641     }
 642   return NULL;
 643 }
 644
 645 vrf_route_t *
 646 nat44_ed_get_vrf_route (vrf_table_t *t, u32 vrf_id)
 647 {
 648   vrf_route_t *r;
 649
 650   pool_foreach (r, t->routes)
 651     {
 652       if (vrf_id == r->vrf_id)
 653         {
 654           return r;
 655         }
 656     }
 657   return NULL;
 658 }
 659
 660 int
 661 nat44_ed_add_del_vrf_table (u32 table_vrf_id, bool is_add)
 662 {
 663   snat_main_t *sm = &snat_main;
 664   vrf_table_t *t;
 665   vrf_route_t *r;
 666
 667   t = nat44_ed_get_vrf_table (table_vrf_id);
 668   if (t)
 669     {
 670       if (is_add)
 671         {
 672           return VNET_API_ERROR_VALUE_EXIST;
 673         }
 674       pool_foreach (r, t->routes)
 675         {
 676           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 677         }
 678       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 679
 680       pool_free (t->routes);
 681       pool_put (sm->vrf_tables, t);
 682     }
 683   else
 684     {
 685       if (!is_add)
 686         {
 687           return VNET_API_ERROR_NO_SUCH_ENTRY;
 688         }
 689       pool_get (sm->vrf_tables, t);
 690       clib_memset (t, 0, sizeof (*t));
 691       t->table_vrf_id = table_vrf_id;
 692       t->table_fib_index = fib_table_find_or_create_and_lock (
 693         FIB_PROTOCOL_IP4, table_vrf_id, sm->fib_src_low);
 694     }
 695
 696   return 0;
 697 }
 698
 699 void
 700 nat44_ed_del_vrf_tables ()
 701 {
 702   snat_main_t *sm = &snat_main;
 703   vrf_table_t *t;
 704   vrf_route_t *r;
 705
 706   pool_foreach (t, sm->vrf_tables)
 707     {
 708       pool_foreach (r, t->routes)
 709         {
 710           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 711         }
 712       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 713       pool_free (t->routes);
 714     }
 715   pool_free (sm->vrf_tables);
 716 }
 717
 718 int
 719 nat44_ed_add_del_vrf_route (u32 table_vrf_id, u32 vrf_id, bool is_add)
 720 {
 721   snat_main_t *sm = &snat_main;
 722   vrf_table_t *t;
 723   vrf_route_t *r;
 724
 725   t = nat44_ed_get_vrf_table (table_vrf_id);
 726   if (!t)
 727     {
 728       return VNET_API_ERROR_NO_SUCH_ENTRY;
 729     }
 730
 731   r = nat44_ed_get_vrf_route (t, vrf_id);
 732   if (r)
 733     {
 734       if (is_add)
 735         {
 736           return VNET_API_ERROR_VALUE_EXIST;
 737         }
 738       fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 739       pool_put (t->routes, r);
 740     }
 741   else
 742     {
 743       if (!is_add)
 744         {
 745           return VNET_API_ERROR_NO_SUCH_ENTRY;
 746         }
 747       pool_get (t->routes, r);
 748       clib_memset (r, 0, sizeof (*r));
 749       r->vrf_id = vrf_id;
 750       r->fib_index = fib_table_find_or_create_and_lock (
 751         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 752     }
 753
 754   return 0;
 755 }
 756
 757 u32
 758 get_thread_idx_by_port (u16 e_port)
 759 {
 760   snat_main_t *sm = &snat_main;
 761   u32 thread_idx = sm->num_workers;
 762   if (sm->num_workers > 1)
 763     {
 764       thread_idx =
 765         sm->first_worker_index +
 766         sm->workers[(e_port - 1024) / sm->port_per_thread];
 767     }
 768   return thread_idx;
 769 }
 770
 771 void
 772 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 773                                     snat_main_per_thread_data_t * tsm,
 774                                     ip4_address_t l_addr,
 775                                     u16 l_port,
 776                                     u8 protocol,
 777                                     u32 fib_index, int addr_only,
 778                                     ip4_address_t e_addr, u16 e_port)
 779 {
 780   snat_session_t *s;
 781   u32 *indexes_to_free = NULL;
 782   pool_foreach (s, tsm->sessions) {
 783     if (s->in2out.fib_index != fib_index ||
 784         s->in2out.addr.as_u32 != l_addr.as_u32)
 785       {
 786         continue;
 787       }
 788     if (!addr_only)
 789       {
 790         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 791             s->out2in.port != e_port || s->in2out.port != l_port ||
 792             s->proto != protocol)
 793           continue;
 794       }
 795
 796     if (nat44_ed_is_lb_session (s))
 797       continue;
 798     if (!nat44_ed_is_session_static (s))
 799       continue;
 800     nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 801     vec_add1 (indexes_to_free, s - tsm->sessions);
 802     if (!addr_only)
 803       break;
 804   }
 805   u32 *ses_index;
 806   vec_foreach (ses_index, indexes_to_free)
 807   {
 808     s = pool_elt_at_index (tsm->sessions, *ses_index);
 809     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 810   }
 811   vec_free (indexes_to_free);
 812 }
 813
 814 static_always_inline snat_static_mapping_t *
 815 nat44_ed_sm_lookup (snat_main_t *sm, clib_bihash_kv_16_8_t *kv)
 816 {
 817   clib_bihash_kv_16_8_t v;
 818   int rc = clib_bihash_search_16_8 (&sm->flow_hash, kv, &v);
 819   if (!rc)
 820     {
 821       ASSERT (0 == ed_value_get_thread_index (&v));
 822       return pool_elt_at_index (sm->static_mappings,
 823                                 ed_value_get_session_index (&v));
 824     }
 825   return NULL;
 826 }
 827
 828 snat_static_mapping_t *
 829 nat44_ed_sm_o2i_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 830                         u32 fib_index, u8 proto)
 831 {
 832   clib_bihash_kv_16_8_t kv;
 833   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 834   return nat44_ed_sm_lookup (sm, &kv);
 835 }
 836
 837 snat_static_mapping_t *
 838 nat44_ed_sm_i2o_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 839                         u32 fib_index, u8 proto)
 840 {
 841   clib_bihash_kv_16_8_t kv;
 842   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 843   return nat44_ed_sm_lookup (sm, &kv);
 844 }
 845
 846 static snat_static_mapping_resolve_t *
 847 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 848                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 849                              u32 flags, int *out_idx)
 850 {
 851   snat_static_mapping_resolve_t *rp;
 852   snat_main_t *sm = &snat_main;
 853   int i;
 854
 855   for (i = 0; i < vec_len (sm->sm_to_resolve); i++)
 856     {
 857       rp = sm->sm_to_resolve + i;
 858
 859       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 860         {
 861           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 862             {
 863               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 864                 {
 865                   if (rp->e_port != e_port || rp->proto != proto)
 866                     {
 867                       continue;
 868                     }
 869                 }
 870             }
 871           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 872             {
 873               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 874                 {
 875                   if (rp->l_port != l_port || rp->e_port != e_port ||
 876                       rp->proto != proto)
 877                     {
 878                       continue;
 879                     }
 880                 }
 881             }
 882           else
 883             {
 884               continue;
 885             }
 886           if (out_idx)
 887             {
 888               *out_idx = i;
 889             }
 890           return rp;
 891         }
 892     }
 893   return NULL;
 894 }
 895
 896 static int
 897 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 898                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 899                              u32 flags)
 900 {
 901   snat_main_t *sm = &snat_main;
 902   int i;
 903   if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 904                                    sw_if_index, flags, &i))
 905     {
 906       vec_del1 (sm->sm_to_resolve, i);
 907       return 0;
 908     }
 909   return 1;
 910 }
 911
 912 static_always_inline int
 913 nat44_ed_validate_sm_input (u32 flags)
 914 {
 915   // identity nat can be initiated only from inside interface
 916   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 917     {
 918       return VNET_API_ERROR_UNSUPPORTED;
 919     }
 920
 921   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 922     {
 923       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 924         {
 925           return VNET_API_ERROR_UNSUPPORTED;
 926         }
 927     }
 928   return 0;
 929 }
 930
 931 int
 932 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 933                              u16 l_port, u16 e_port, ip_protocol_t proto,
 934                              u32 vrf_id, u32 sw_if_index, u32 flags,
 935                              ip4_address_t pool_addr, u8 *tag)
 936 {
 937   snat_static_mapping_resolve_t *rp;
 938   snat_main_t *sm = &snat_main;
 939   int rv;
 940
 941   if (!sm->enabled)
 942     {
 943       return VNET_API_ERROR_UNSUPPORTED;
 944     }
 945
 946   rv = nat44_ed_validate_sm_input (flags);
 947   if (rv != 0)
 948     {
 949       return rv;
 950     }
 951
 952   // interface bound mapping
 953   if (is_sm_switch_address (flags))
 954     {
 955       if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 956                                        sw_if_index, flags, 0))
 957         {
 958           return VNET_API_ERROR_VALUE_EXIST;
 959         }
 960
 961       vec_add2 (sm->sm_to_resolve, rp, 1);
 962       rp->l_addr.as_u32 = l_addr.as_u32;
 963       rp->l_port = l_port;
 964       rp->e_port = e_port;
 965       rp->sw_if_index = sw_if_index;
 966       rp->vrf_id = vrf_id;
 967       rp->proto = proto;
 968       rp->flags = flags;
 969       rp->pool_addr = pool_addr;
 970       rp->tag = vec_dup (tag);
 971       rp->is_resolved = 0;
 972
 973       ip4_address_t *first_int_addr =
 974         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 975       if (!first_int_addr)
 976         {
 977           return 0;
 978         }
 979
 980       e_addr.as_u32 = first_int_addr->as_u32;
 981       rp->is_resolved = 1;
 982     }
 983
 984   rv = nat44_ed_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
 985                                              proto, vrf_id, sw_if_index, flags,
 986                                              pool_addr, tag);
 987   if ((0 != rv) && is_sm_switch_address (flags))
 988     {
 989       nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 990                                    sw_if_index, flags);
 991     }
 992
 993   return rv;
 994 }
 995
 996 int
 997 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 998                              u16 l_port, u16 e_port, ip_protocol_t proto,
 999                              u32 vrf_id, u32 sw_if_index, u32 flags)
1000 {
1001   snat_main_t *sm = &snat_main;
1002   int rv;
1003
1004   if (!sm->enabled)
1005     {
1006       return VNET_API_ERROR_UNSUPPORTED;
1007     }
1008
1009   rv = nat44_ed_validate_sm_input (flags);
1010   if (rv != 0)
1011     {
1012       return rv;
1013     }
1014
1015   // interface bound mapping
1016   if (is_sm_switch_address (flags))
1017     {
1018       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
1019                                        sw_if_index, flags))
1020         {
1021           return VNET_API_ERROR_NO_SUCH_ENTRY;
1022         }
1023
1024       ip4_address_t *first_int_addr =
1025         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
1026       if (!first_int_addr)
1027         {
1028           // dhcp resolution required
1029           return 0;
1030         }
1031
1032       e_addr.as_u32 = first_int_addr->as_u32;
1033     }
1034
1035   return nat44_ed_del_static_mapping_internal (l_addr, e_addr, l_port, e_port,
1036                                                proto, vrf_id, flags);
1037 }
1038
1039 static int
1040 nat44_ed_add_static_mapping_internal (ip4_address_t l_addr,
1041                                       ip4_address_t e_addr, u16 l_port,
1042                                       u16 e_port, ip_protocol_t proto,
1043                                       u32 vrf_id, u32 sw_if_index, u32 flags,
1044                                       ip4_address_t pool_addr, u8 *tag)
1045 {
1046   snat_main_t *sm = &snat_main;
1047   nat44_lb_addr_port_t *local;
1048   snat_static_mapping_t *m;
1049   u32 fib_index = ~0;
1050
1051   if (is_sm_addr_only (flags))
1052     {
1053       e_port = l_port = proto = 0;
1054     }
1055
1056   if (is_sm_identity_nat (flags))
1057     {
1058       l_port = e_port;
1059       l_addr.as_u32 = e_addr.as_u32;
1060     }
1061
1062   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1063   if (m)
1064     {
1065       // case:
1066       // adding local identity nat record for different vrf table
1067
1068       if (!is_sm_identity_nat (m->flags))
1069         {
1070           return VNET_API_ERROR_VALUE_EXIST;
1071         }
1072
1073       pool_foreach (local, m->locals)
1074         {
1075           if (local->vrf_id == vrf_id)
1076             {
1077               return VNET_API_ERROR_VALUE_EXIST;
1078             }
1079         }
1080
1081       pool_get (m->locals, local);
1082
1083       local->vrf_id = vrf_id;
1084       local->fib_index = fib_table_find_or_create_and_lock (
1085         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
1086
1087       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port,
1088                            local->fib_index, m->proto);
1089
1090       return 0;
1091     }
1092
1093   if (vrf_id != ~0)
1094     {
1095       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1096                                                      sm->fib_src_low);
1097     }
1098   else
1099     {
1100       // fallback to default vrf
1101       vrf_id = sm->inside_vrf_id;
1102       fib_index = sm->inside_fib_index;
1103       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1104     }
1105
1106   // test if local mapping record doesn't exist
1107   // identity nat supports multiple records in local mapping
1108   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
1109     {
1110       if (nat44_ed_sm_i2o_lookup (sm, l_addr, l_port, fib_index, proto))
1111         {
1112           return VNET_API_ERROR_VALUE_EXIST;
1113         }
1114     }
1115
1116   pool_get (sm->static_mappings, m);
1117   clib_memset (m, 0, sizeof (*m));
1118
1119   m->flags = flags;
1120   m->local_addr = l_addr;
1121   m->external_addr = e_addr;
1122
1123   m->pool_addr = pool_addr;
1124   m->tag = vec_dup (tag);
1125
1126   if (!is_sm_addr_only (flags))
1127     {
1128       m->local_port = l_port;
1129       m->external_port = e_port;
1130       m->proto = proto;
1131     }
1132
1133   if (is_sm_identity_nat (flags))
1134     {
1135       pool_get (m->locals, local);
1136
1137       local->vrf_id = vrf_id;
1138       local->fib_index = fib_index;
1139     }
1140   else
1141     {
1142       m->vrf_id = vrf_id;
1143       m->fib_index = fib_index;
1144     }
1145
1146   if (!is_sm_out2in_only (flags))
1147     {
1148       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port, fib_index,
1149                            m->proto);
1150     }
1151
1152   nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0, m->proto);
1153
1154   if (sm->num_workers > 1)
1155     {
1156       // store worker index for this record
1157       ip4_header_t ip = {
1158         .src_address = m->local_addr,
1159       };
1160       u32 worker_index;
1161       worker_index =
1162         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
1163       vec_add1 (m->workers, worker_index);
1164     }
1165
1166   nat44_ed_add_del_interface_fib_reg_entries (e_addr, 1);
1167
1168   return 0;
1169 }
1170
1171 static int
1172 nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
1173                                       ip4_address_t e_addr, u16 l_port,
1174                                       u16 e_port, ip_protocol_t proto,
1175                                       u32 vrf_id, u32 flags)
1176 {
1177   snat_main_per_thread_data_t *tsm;
1178   snat_main_t *sm = &snat_main;
1179
1180   nat44_lb_addr_port_t *local;
1181   snat_static_mapping_t *m;
1182   u32 fib_index = ~0;
1183
1184   if (is_sm_addr_only (flags))
1185     {
1186       e_port = l_port = proto = 0;
1187     }
1188
1189   if (is_sm_identity_nat (flags))
1190     {
1191       l_port = e_port;
1192       l_addr.as_u32 = e_addr.as_u32;
1193     }
1194
1195   // fib index 0
1196   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1197   if (!m)
1198     {
1199       return VNET_API_ERROR_NO_SUCH_ENTRY;
1200     }
1201
1202   if (is_sm_identity_nat (flags))
1203     {
1204       u8 found = 0;
1205
1206       if (vrf_id == ~0)
1207         {
1208           vrf_id = sm->inside_vrf_id;
1209         }
1210
1211       pool_foreach (local, m->locals)
1212         {
1213           if (local->vrf_id == vrf_id)
1214             {
1215               local = pool_elt_at_index (m->locals, local - m->locals);
1216               fib_index = local->fib_index;
1217               pool_put (m->locals, local);
1218               found = 1;
1219             }
1220         }
1221
1222       if (!found)
1223         {
1224           return VNET_API_ERROR_NO_SUCH_ENTRY;
1225         }
1226     }
1227   else
1228     {
1229       fib_index = m->fib_index;
1230     }
1231
1232   if (!is_sm_out2in_only (flags))
1233     {
1234       nat44_ed_sm_i2o_del (sm, l_addr, l_port, fib_index, proto);
1235     }
1236
1237   // delete sessions for static mapping
1238   if (sm->num_workers > 1)
1239     {
1240       tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1241     }
1242   else
1243     {
1244       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1245     }
1246
1247   nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr, m->local_port,
1248                                       m->proto, fib_index,
1249                                       is_sm_addr_only (flags), e_addr, e_port);
1250
1251   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1252
1253   if (!pool_elts (m->locals))
1254     {
1255       // this is last record remove all required stuff
1256       // fib_index 0
1257       nat44_ed_sm_o2i_del (sm, e_addr, e_port, 0, proto);
1258
1259       vec_free (m->tag);
1260       vec_free (m->workers);
1261       pool_put (sm->static_mappings, m);
1262
1263       nat44_ed_add_del_interface_fib_reg_entries (e_addr, 0);
1264     }
1265
1266   return 0;
1267 }
1268
1269 int
1270 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1271                                 ip_protocol_t proto,
1272                                 nat44_lb_addr_port_t *locals, u32 flags,
1273                                 u8 *tag, u32 affinity)
1274 {
1275   snat_main_t *sm = &snat_main;
1276   snat_static_mapping_t *m;
1277   snat_address_t *a = 0;
1278
1279   nat44_lb_addr_port_t *local;
1280   uword *bitmap = 0;
1281   int rc = 0;
1282
1283   int i;
1284
1285   if (!sm->enabled)
1286     {
1287       return VNET_API_ERROR_UNSUPPORTED;
1288     }
1289
1290   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1291
1292   if (m)
1293     {
1294       return VNET_API_ERROR_VALUE_EXIST;
1295     }
1296
1297   if (vec_len (locals) < 2)
1298     {
1299       return VNET_API_ERROR_INVALID_VALUE;
1300     }
1301
1302   if (!is_sm_out2in_only (flags))
1303     {
1304       /* Find external address in allocated addresses and reserve port for
1305          address and port pair mapping when dynamic translations enabled */
1306       for (i = 0; i < vec_len (sm->addresses); i++)
1307         {
1308           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1309             {
1310               /* External port must be unused */
1311               a = sm->addresses + i;
1312               if (nat44_ed_sm_o2i_lookup (sm, a->addr, e_port, 0, proto))
1313                 {
1314                   return VNET_API_ERROR_VALUE_EXIST;
1315                 }
1316               break;
1317             }
1318         }
1319       // external address must be allocated
1320       if (!a)
1321         {
1322           return VNET_API_ERROR_NO_SUCH_ENTRY;
1323         }
1324     }
1325
1326   pool_get (sm->static_mappings, m);
1327   clib_memset (m, 0, sizeof (*m));
1328   m->tag = vec_dup (tag);
1329   m->external_addr = e_addr;
1330   m->external_port = e_port;
1331   m->affinity = affinity;
1332   m->proto = proto;
1333
1334   m->flags = flags;
1335   m->flags |= NAT_SM_FLAG_LB;
1336
1337   if (affinity)
1338     m->affinity_per_service_list_head_index =
1339       nat_affinity_get_per_service_list_head_index ();
1340   else
1341     m->affinity_per_service_list_head_index = ~0;
1342
1343   if (nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0,
1344                            m->proto))
1345     {
1346       nat_log_err ("sm o2i key add failed");
1347       return VNET_API_ERROR_UNSPECIFIED;
1348     }
1349
1350   for (i = 0; i < vec_len (locals); i++)
1351     {
1352       locals[i].fib_index = fib_table_find_or_create_and_lock (
1353         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1354       if (!is_sm_out2in_only (flags))
1355         {
1356           if (nat44_ed_sm_o2i_add (sm, m, e_addr, e_port, 0, proto))
1357             {
1358               nat_log_err ("sm o2i key add failed");
1359               rc = VNET_API_ERROR_UNSPECIFIED;
1360               // here we continue with add operation so that it can be safely
1361               // reversed in delete path - otherwise we'd have to track what
1362               // we've done and deal with partial cleanups and since bihash
1363               // adds are (extremely improbable) the only points of failure,
1364               // it's easier to just do it this way
1365             }
1366         }
1367       locals[i].prefix = (i == 0) ?
1368                            locals[i].probability :
1369                            (locals[i - 1].prefix + locals[i].probability);
1370       pool_get (m->locals, local);
1371       *local = locals[i];
1372       if (sm->num_workers > 1)
1373         {
1374           ip4_header_t ip = {
1375             .src_address = locals[i].addr,
1376           };
1377           bitmap = clib_bitmap_set (
1378             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1379             1);
1380         }
1381     }
1382
1383   /* Assign workers */
1384   if (sm->num_workers > 1)
1385     {
1386       clib_bitmap_foreach (i, bitmap)
1387         {
1388           vec_add1 (m->workers, i);
1389         }
1390     }
1391
1392   return rc;
1393 }
1394
1395 int
1396 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1397                                 ip_protocol_t proto, u32 flags)
1398 {
1399   snat_main_t *sm = &snat_main;
1400   snat_static_mapping_t *m;
1401
1402   nat44_lb_addr_port_t *local;
1403   snat_main_per_thread_data_t *tsm;
1404   snat_session_t *s;
1405
1406   if (!sm->enabled)
1407     {
1408       return VNET_API_ERROR_UNSUPPORTED;
1409     }
1410
1411   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1412   if (!m)
1413     return VNET_API_ERROR_NO_SUCH_ENTRY;
1414
1415   if (!is_sm_lb (m->flags))
1416     return VNET_API_ERROR_INVALID_VALUE;
1417
1418   if (nat44_ed_sm_o2i_del (sm, m->external_addr, m->external_port, 0,
1419                            m->proto))
1420     {
1421       nat_log_err ("sm o2i key del failed");
1422       return VNET_API_ERROR_UNSPECIFIED;
1423     }
1424
1425   pool_foreach (local, m->locals)
1426     {
1427       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1428       if (!is_sm_out2in_only (flags))
1429         {
1430           if (nat44_ed_sm_i2o_del (sm, local->addr, local->port,
1431                                    local->fib_index, m->proto))
1432             {
1433               nat_log_err ("sm i2o key del failed");
1434               return VNET_API_ERROR_UNSPECIFIED;
1435             }
1436         }
1437
1438       if (sm->num_workers > 1)
1439         {
1440           ip4_header_t ip = {
1441             .src_address = local->addr,
1442           };
1443           tsm = vec_elt_at_index (
1444             sm->per_thread_data,
1445             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1446         }
1447       else
1448         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1449
1450       /* Delete sessions */
1451       pool_foreach (s, tsm->sessions)
1452         {
1453           if (!(nat44_ed_is_lb_session (s)))
1454             continue;
1455
1456           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1457               s->in2out.port != local->port)
1458             continue;
1459
1460           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1461           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1462         }
1463     }
1464
1465   if (m->affinity)
1466     {
1467       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1468     }
1469
1470   pool_free (m->locals);
1471   vec_free (m->tag);
1472   vec_free (m->workers);
1473   pool_put (sm->static_mappings, m);
1474
1475   return 0;
1476 }
1477
1478 int
1479 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1480                                           ip4_address_t l_addr, u16 l_port,
1481                                           ip_protocol_t proto, u32 vrf_id,
1482                                           u8 probability, u8 is_add)
1483 {
1484   snat_main_t *sm = &snat_main;
1485   snat_static_mapping_t *m = 0;
1486   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1487   snat_main_per_thread_data_t *tsm;
1488   snat_session_t *s;
1489   u32 *locals = 0;
1490   uword *bitmap = 0;
1491   int i;
1492
1493   if (!sm->enabled)
1494     {
1495       return VNET_API_ERROR_UNSUPPORTED;
1496     }
1497
1498   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1499
1500   if (!m)
1501     {
1502       return VNET_API_ERROR_NO_SUCH_ENTRY;
1503     }
1504
1505   if (!is_sm_lb (m->flags))
1506     {
1507       return VNET_API_ERROR_INVALID_VALUE;
1508     }
1509
1510   pool_foreach (local, m->locals)
1511    {
1512     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1513         (local->vrf_id == vrf_id))
1514       {
1515         match_local = local;
1516         break;
1517       }
1518   }
1519
1520   if (is_add)
1521     {
1522       if (match_local)
1523         {
1524           return VNET_API_ERROR_VALUE_EXIST;
1525         }
1526
1527       pool_get (m->locals, local);
1528       clib_memset (local, 0, sizeof (*local));
1529       local->addr.as_u32 = l_addr.as_u32;
1530       local->port = l_port;
1531       local->probability = probability;
1532       local->vrf_id = vrf_id;
1533       local->fib_index =
1534         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1535                                            sm->fib_src_low);
1536
1537       if (!is_sm_out2in_only (m->flags))
1538         {
1539           if (nat44_ed_sm_i2o_add (sm, m, l_addr, l_port, local->fib_index,
1540                                    proto))
1541             {
1542               nat_log_err ("sm i2o key add failed");
1543               pool_put (m->locals, local);
1544               return VNET_API_ERROR_UNSPECIFIED;
1545             }
1546         }
1547     }
1548   else
1549     {
1550       if (!match_local)
1551         return VNET_API_ERROR_NO_SUCH_ENTRY;
1552
1553       if (pool_elts (m->locals) < 3)
1554         return VNET_API_ERROR_UNSPECIFIED;
1555
1556       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1557                         sm->fib_src_low);
1558
1559       if (!is_sm_out2in_only (m->flags))
1560         {
1561           if (nat44_ed_sm_i2o_del (sm, l_addr, l_port, match_local->fib_index,
1562                                    proto))
1563             nat_log_err ("sm i2o key del failed");
1564         }
1565
1566       if (sm->num_workers > 1)
1567         {
1568           ip4_header_t ip = {
1569             .src_address = local->addr,
1570           };
1571           tsm = vec_elt_at_index (
1572             sm->per_thread_data,
1573             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1574         }
1575       else
1576         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1577
1578       /* Delete sessions */
1579       pool_foreach (s, tsm->sessions) {
1580           if (!(nat44_ed_is_lb_session (s)))
1581             continue;
1582
1583           if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1584               s->in2out.port != match_local->port)
1585             continue;
1586
1587           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1588           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1589       }
1590
1591       pool_put (m->locals, match_local);
1592     }
1593
1594   vec_free (m->workers);
1595
1596   pool_foreach (local, m->locals)
1597    {
1598     vec_add1 (locals, local - m->locals);
1599     if (sm->num_workers > 1)
1600       {
1601         ip4_header_t ip;
1602         ip.src_address.as_u32 = local->addr.as_u32,
1603         bitmap = clib_bitmap_set (
1604           bitmap,
1605           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1606       }
1607   }
1608
1609   ASSERT (vec_len (locals) > 1);
1610
1611   local = pool_elt_at_index (m->locals, locals[0]);
1612   local->prefix = local->probability;
1613   for (i = 1; i < vec_len (locals); i++)
1614     {
1615       local = pool_elt_at_index (m->locals, locals[i]);
1616       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1617       local->prefix = local->probability + prev_local->prefix;
1618     }
1619
1620   /* Assign workers */
1621   if (sm->num_workers > 1)
1622     {
1623       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1624     }
1625
1626   return 0;
1627 }
1628
1629 void
1630 expire_per_vrf_sessions (u32 fib_index)
1631 {
1632   per_vrf_sessions_t *per_vrf_sessions;
1633   snat_main_per_thread_data_t *tsm;
1634   snat_main_t *sm = &snat_main;
1635
1636   vec_foreach (tsm, sm->per_thread_data)
1637     {
1638       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1639         {
1640           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1641               (per_vrf_sessions->tx_fib_index == fib_index))
1642             {
1643               per_vrf_sessions->expired = 1;
1644             }
1645         }
1646     }
1647 }
1648
1649 void
1650 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1651 {
1652   snat_main_t *sm = &snat_main;
1653   nat_fib_t *fib;
1654
1655   // we don't care if it is outside/inside fib
1656   // we just care about their ref_count
1657   // if it reaches 0 sessions should expire
1658   // because the fib isn't valid for NAT anymore
1659
1660   vec_foreach (fib, sm->fibs)
1661   {
1662     if (fib->fib_index == fib_index)
1663       {
1664         if (is_del)
1665           {
1666             fib->ref_count--;
1667             if (!fib->ref_count)
1668               {
1669                 vec_del1 (sm->fibs, fib - sm->fibs);
1670                 expire_per_vrf_sessions (fib_index);
1671               }
1672             return;
1673           }
1674         else
1675           fib->ref_count++;
1676       }
1677   }
1678   if (!is_del)
1679     {
1680       vec_add2 (sm->fibs, fib, 1);
1681       fib->ref_count = 1;
1682       fib->fib_index = fib_index;
1683     }
1684 }
1685
1686 static_always_inline nat_fib_t *
1687 nat44_ed_get_outside_fib (nat_fib_t *outside_fibs, u32 fib_index)
1688 {
1689   nat_fib_t *f;
1690   vec_foreach (f, outside_fibs)
1691     {
1692       if (f->fib_index == fib_index)
1693         {
1694           return f;
1695         }
1696     }
1697   return 0;
1698 }
1699
1700 static_always_inline snat_interface_t *
1701 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1702 {
1703   snat_interface_t *i;
1704   pool_foreach (i, interfaces)
1705     {
1706       if (i->sw_if_index == sw_if_index)
1707         {
1708           return i;
1709         }
1710     }
1711   return 0;
1712 }
1713
1714 int
1715 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1716 {
1717   const char *del_feature_name, *feature_name;
1718   snat_main_t *sm = &snat_main;
1719
1720   nat_fib_t *outside_fib;
1721   snat_interface_t *i;
1722   u32 fib_index;
1723   int rv;
1724
1725   if (!sm->enabled)
1726     {
1727       nat_log_err ("nat44 is disabled");
1728       return VNET_API_ERROR_UNSUPPORTED;
1729     }
1730
1731   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1732     {
1733       nat_log_err ("error interface already configured");
1734       return VNET_API_ERROR_VALUE_EXIST;
1735     }
1736
1737   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1738   if (i)
1739     {
1740       if ((nat44_ed_is_interface_inside (i) && is_inside) ||
1741           (nat44_ed_is_interface_outside (i) && !is_inside))
1742         {
1743           return 0;
1744         }
1745       if (sm->num_workers > 1)
1746         {
1747           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1748                                           "nat44-out2in-worker-handoff";
1749           feature_name = "nat44-handoff-classify";
1750         }
1751       else
1752         {
1753           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1754
1755           feature_name = "nat44-ed-classify";
1756         }
1757
1758       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1759       if (rv)
1760         return rv;
1761       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1762                                    sw_if_index, 0, 0, 0);
1763       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1764                                    0, 0);
1765     }
1766   else
1767     {
1768       if (sm->num_workers > 1)
1769         {
1770           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1771                                      "nat44-out2in-worker-handoff";
1772         }
1773       else
1774         {
1775           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1776         }
1777
1778       nat_validate_interface_counters (sm, sw_if_index);
1779       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1780       if (rv)
1781         return rv;
1782       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1783                                    0, 0);
1784
1785       pool_get (sm->interfaces, i);
1786       i->sw_if_index = sw_if_index;
1787       i->flags = 0;
1788     }
1789
1790   fib_index =
1791     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1792
1793   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1794
1795   if (!is_inside)
1796     {
1797       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1798
1799       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1800       if (outside_fib)
1801         {
1802           outside_fib->ref_count++;
1803         }
1804       else
1805         {
1806           vec_add2 (sm->outside_fibs, outside_fib, 1);
1807           outside_fib->fib_index = fib_index;
1808           outside_fib->ref_count = 1;
1809         }
1810
1811       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
1812       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
1813
1814       nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
1815     }
1816   else
1817     {
1818       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1819     }
1820
1821   return 0;
1822 }
1823
1824 int
1825 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1826 {
1827   const char *del_feature_name, *feature_name;
1828   snat_main_t *sm = &snat_main;
1829
1830   nat_fib_t *outside_fib;
1831   snat_interface_t *i;
1832   u32 fib_index;
1833   int rv;
1834
1835   if (!sm->enabled)
1836     {
1837       nat_log_err ("nat44 is disabled");
1838       return VNET_API_ERROR_UNSUPPORTED;
1839     }
1840
1841   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1842   if (i == 0)
1843     {
1844       nat_log_err ("error interface couldn't be found");
1845       return VNET_API_ERROR_NO_SUCH_ENTRY;
1846     }
1847
1848   if (nat44_ed_is_interface_inside (i) && nat44_ed_is_interface_outside (i))
1849     {
1850       if (sm->num_workers > 1)
1851         {
1852           del_feature_name = "nat44-handoff-classify";
1853           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1854                                       "nat44-out2in-worker-handoff";
1855         }
1856       else
1857         {
1858           del_feature_name = "nat44-ed-classify";
1859           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1860         }
1861
1862       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1863       if (rv)
1864         {
1865           return rv;
1866         }
1867       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1868                                    sw_if_index, 0, 0, 0);
1869       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1870                                    0, 0);
1871
1872       if (is_inside)
1873         {
1874           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1875         }
1876       else
1877         {
1878           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1879         }
1880     }
1881   else
1882     {
1883       if (sm->num_workers > 1)
1884         {
1885           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1886                                      "nat44-out2in-worker-handoff";
1887         }
1888       else
1889         {
1890           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1891         }
1892
1893       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1894       if (rv)
1895         {
1896           return rv;
1897         }
1898       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1899                                    0, 0);
1900
1901       // remove interface
1902       pool_put (sm->interfaces, i);
1903     }
1904
1905   fib_index =
1906     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1907
1908   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
1909
1910   if (!is_inside)
1911     {
1912       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1913       if (outside_fib)
1914         {
1915           outside_fib->ref_count--;
1916           if (!outside_fib->ref_count)
1917             {
1918               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1919             }
1920         }
1921
1922       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
1923       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
1924     }
1925
1926   return 0;
1927 }
1928
1929 int
1930 nat44_ed_add_output_interface (u32 sw_if_index)
1931 {
1932   snat_main_t *sm = &snat_main;
1933
1934   nat_fib_t *outside_fib;
1935   snat_interface_t *i;
1936   u32 fib_index;
1937   int rv;
1938
1939   if (!sm->enabled)
1940     {
1941       nat_log_err ("nat44 is disabled");
1942       return VNET_API_ERROR_UNSUPPORTED;
1943     }
1944
1945   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1946     {
1947       nat_log_err ("error interface already configured");
1948       return VNET_API_ERROR_VALUE_EXIST;
1949     }
1950
1951   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1952     {
1953       nat_log_err ("error interface already configured");
1954       return VNET_API_ERROR_VALUE_EXIST;
1955     }
1956
1957   if (sm->num_workers > 1)
1958     {
1959       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1960       if (rv)
1961         {
1962           return rv;
1963         }
1964
1965       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1966       if (rv)
1967         {
1968           return rv;
1969         }
1970
1971       vnet_feature_enable_disable (
1972         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1973       vnet_feature_enable_disable ("ip4-output",
1974                                    "nat44-in2out-output-worker-handoff",
1975                                    sw_if_index, 1, 0, 0);
1976     }
1977   else
1978     {
1979       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1980       if (rv)
1981         {
1982           return rv;
1983         }
1984
1985       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1986       if (rv)
1987         {
1988           return rv;
1989         }
1990
1991       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1992                                    sw_if_index, 1, 0, 0);
1993       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1994                                    sw_if_index, 1, 0, 0);
1995     }
1996
1997   nat_validate_interface_counters (sm, sw_if_index);
1998
1999   pool_get (sm->output_feature_interfaces, i);
2000   i->sw_if_index = sw_if_index;
2001   i->flags = 0;
2002   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2003   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2004
2005   fib_index =
2006     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2007   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
2008
2009   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2010   if (outside_fib)
2011     {
2012       outside_fib->ref_count++;
2013     }
2014   else
2015     {
2016       vec_add2 (sm->outside_fibs, outside_fib, 1);
2017       outside_fib->fib_index = fib_index;
2018       outside_fib->ref_count = 1;
2019     }
2020
2021   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
2022   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
2023
2024   nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
2025
2026   return 0;
2027 }
2028
2029 int
2030 nat44_ed_del_output_interface (u32 sw_if_index)
2031 {
2032   snat_main_t *sm = &snat_main;
2033
2034   nat_fib_t *outside_fib;
2035   snat_interface_t *i;
2036   u32 fib_index;
2037   int rv;
2038
2039   if (!sm->enabled)
2040     {
2041       nat_log_err ("nat44 is disabled");
2042       return VNET_API_ERROR_UNSUPPORTED;
2043     }
2044
2045   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2046   if (!i)
2047     {
2048       nat_log_err ("error interface couldn't be found");
2049       return VNET_API_ERROR_NO_SUCH_ENTRY;
2050     }
2051
2052   if (sm->num_workers > 1)
2053     {
2054       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2055       if (rv)
2056         {
2057           return rv;
2058         }
2059
2060       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2061       if (rv)
2062         {
2063           return rv;
2064         }
2065
2066       vnet_feature_enable_disable (
2067         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
2068       vnet_feature_enable_disable ("ip4-output",
2069                                    "nat44-in2out-output-worker-handoff",
2070                                    sw_if_index, 0, 0, 0);
2071     }
2072   else
2073     {
2074       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2075       if (rv)
2076         {
2077           return rv;
2078         }
2079
2080       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2081       if (rv)
2082         {
2083           return rv;
2084         }
2085
2086       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2087                                    sw_if_index, 0, 0, 0);
2088       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2089                                    sw_if_index, 0, 0, 0);
2090     }
2091
2092   // remove interface
2093   pool_put (sm->output_feature_interfaces, i);
2094
2095   fib_index =
2096     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2097   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
2098
2099   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2100   if (outside_fib)
2101     {
2102       outside_fib->ref_count--;
2103       if (!outside_fib->ref_count)
2104         {
2105           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2106         }
2107     }
2108
2109   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
2110   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
2111
2112   return 0;
2113 }
2114
2115 int
2116 snat_set_workers (uword * bitmap)
2117 {
2118   snat_main_t *sm = &snat_main;
2119   int i, j = 0;
2120
2121   if (sm->num_workers < 2)
2122     return VNET_API_ERROR_FEATURE_DISABLED;
2123
2124   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2125     return VNET_API_ERROR_INVALID_WORKER;
2126
2127   vec_free (sm->workers);
2128   clib_bitmap_foreach (i, bitmap)
2129     {
2130       vec_add1(sm->workers, i);
2131       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2132       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2133       j++;
2134     }
2135
2136   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2137
2138   return 0;
2139 }
2140
2141 int
2142 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2143 {
2144   fail_if_enabled ();
2145   snat_main_t *sm = &snat_main;
2146
2147   if ((sm->fq_in2out_index != ~0) || (sm->fq_out2in_index != ~0) ||
2148       (sm->fq_in2out_output_index != ~0))
2149     {
2150       // frame queu nelts can be set only before first
2151       // call to nat44_plugin_enable after that it
2152       // doesn't make sense
2153       nat_log_err ("Frame queue was already initialized. "
2154                    "Change is not possible");
2155       return 1;
2156     }
2157
2158   sm->frame_queue_nelts = frame_queue_nelts;
2159   return 0;
2160 }
2161
2162 static void
2163 nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque, u32 sw_if_index,
2164                                 u32 new_fib_index, u32 old_fib_index)
2165 {
2166   snat_main_t *sm = &snat_main;
2167   nat_fib_t *outside_fib;
2168   snat_interface_t *i;
2169   u8 is_add = 1;
2170   u8 match = 0;
2171
2172   if (!sm->enabled || (new_fib_index == old_fib_index)
2173       || (!vec_len (sm->outside_fibs)))
2174     {
2175       return;
2176     }
2177
2178   pool_foreach (i, sm->interfaces)
2179     {
2180       if (i->sw_if_index == sw_if_index)
2181         {
2182           if (!(nat44_ed_is_interface_outside (i)))
2183             return;
2184           match = 1;
2185         }
2186     }
2187
2188   pool_foreach (i, sm->output_feature_interfaces)
2189     {
2190       if (i->sw_if_index == sw_if_index)
2191         {
2192           if (!(nat44_ed_is_interface_outside (i)))
2193             return;
2194           match = 1;
2195         }
2196     }
2197
2198   if (!match)
2199     return;
2200
2201   vec_foreach (outside_fib, sm->outside_fibs)
2202     {
2203       if (outside_fib->fib_index == old_fib_index)
2204         {
2205           outside_fib->ref_count--;
2206           if (!outside_fib->ref_count)
2207             vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2208           break;
2209         }
2210     }
2211
2212   vec_foreach (outside_fib, sm->outside_fibs)
2213     {
2214       if (outside_fib->fib_index == new_fib_index)
2215         {
2216           outside_fib->ref_count++;
2217           is_add = 0;
2218           break;
2219         }
2220     }
2221
2222   if (is_add)
2223     {
2224       vec_add2 (sm->outside_fibs, outside_fib, 1);
2225       outside_fib->ref_count = 1;
2226       outside_fib->fib_index = new_fib_index;
2227     }
2228 }
2229
2230 static void nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque,
2231                                             u32 sw_if_index, u32 new_fib_index,
2232                                             u32 old_fib_index);
2233
2234 static void nat44_ed_add_del_interface_address_cb (
2235   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2236   u32 address_length, u32 if_address_index, u32 is_delete);
2237
2238 static void nat44_ed_add_del_static_mapping_cb (
2239   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2240   u32 address_length, u32 if_address_index, u32 is_delete);
2241
2242 void
2243 test_key_calc_split ()
2244 {
2245   ip4_address_t l_addr;
2246   l_addr.as_u8[0] = 1;
2247   l_addr.as_u8[1] = 1;
2248   l_addr.as_u8[2] = 1;
2249   l_addr.as_u8[3] = 1;
2250   ip4_address_t r_addr;
2251   r_addr.as_u8[0] = 2;
2252   r_addr.as_u8[1] = 2;
2253   r_addr.as_u8[2] = 2;
2254   r_addr.as_u8[3] = 2;
2255   u16 l_port = 40001;
2256   u16 r_port = 40301;
2257   u8 proto = 9;
2258   u32 fib_index = 9000001;
2259   u32 thread_index = 3000000001;
2260   u32 session_index = 3000000221;
2261   clib_bihash_kv_16_8_t kv;
2262   init_ed_kv (&kv, l_addr.as_u32, l_port, r_addr.as_u32, r_port, fib_index,
2263               proto, thread_index, session_index);
2264   ip4_address_t l_addr2;
2265   ip4_address_t r_addr2;
2266   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2267   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2268   u16 l_port2 = 0;
2269   u16 r_port2 = 0;
2270   u8 proto2 = 0;
2271   u32 fib_index2 = 0;
2272   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2273                &r_port2);
2274   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2275   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2276   ASSERT (l_port == l_port2);
2277   ASSERT (r_port == r_port2);
2278   ASSERT (proto == proto2);
2279   ASSERT (fib_index == fib_index2);
2280   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2281   ASSERT (session_index == ed_value_get_session_index (&kv));
2282 }
2283
2284 static clib_error_t *
2285 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2286 {
2287   u32 fib_index;
2288   if (!is_add)
2289     {
2290       fib_index = ip4_fib_index_from_table_id (table_id);
2291       if (fib_index != ~0)
2292         {
2293           expire_per_vrf_sessions (fib_index);
2294         }
2295     }
2296   return 0;
2297 }
2298
2299 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2300
2301 #define nat_validate_simple_counter(c, i)                                     \
2302   do                                                                          \
2303     {                                                                         \
2304       vlib_validate_simple_counter (&c, i);                                   \
2305       vlib_zero_simple_counter (&c, i);                                       \
2306     }                                                                         \
2307   while (0);
2308
2309 #define nat_init_simple_counter(c, n, sn)                                     \
2310   do                                                                          \
2311     {                                                                         \
2312       c.name = n;                                                             \
2313       c.stat_segment_name = sn;                                               \
2314       nat_validate_simple_counter (c, 0);                                     \
2315     }                                                                         \
2316   while (0);
2317
2318 static_always_inline void
2319 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2320 {
2321 #define _(x)                                                                  \
2322   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2323   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2324   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2325   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2326   foreach_nat_counter;
2327 #undef _
2328   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2329 }
2330
2331 static clib_error_t *
2332 nat_init (vlib_main_t * vm)
2333 {
2334   snat_main_t *sm = &snat_main;
2335   vlib_thread_main_t *tm = vlib_get_thread_main ();
2336   vlib_thread_registration_t *tr;
2337   ip4_add_del_interface_address_callback_t cbi = { 0 };
2338   ip4_table_bind_callback_t cbt = { 0 };
2339   u32 i, num_threads = 0;
2340   uword *p, *bitmap = 0;
2341
2342   clib_memset (sm, 0, sizeof (*sm));
2343
2344   // convenience
2345   sm->ip4_main = &ip4_main;
2346
2347   // frame queue indices used for handoff
2348   sm->fq_out2in_index = ~0;
2349   sm->fq_in2out_index = ~0;
2350   sm->fq_in2out_output_index = ~0;
2351
2352   sm->log_level = NAT_LOG_ERROR;
2353
2354   sm->log_class = vlib_log_register_class ("nat", 0);
2355   nat_ipfix_logging_init (vm);
2356
2357   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2358                            "/nat44-ed/total-sessions");
2359   sm->max_cfg_sessions_gauge =
2360     vlib_stats_add_gauge ("/nat44-ed/max-cfg-sessions");
2361
2362 #define _(x)                                                                  \
2363   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2364                            "/nat44-ed/in2out/fastpath/" #x);                  \
2365   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2366                            "/nat44-ed/out2in/fastpath/" #x);                  \
2367   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2368                            "/nat44-ed/in2out/slowpath/" #x);                  \
2369   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2370                            "/nat44-ed/out2in/slowpath/" #x);
2371   foreach_nat_counter;
2372 #undef _
2373   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2374                            "/nat44-ed/hairpinning");
2375
2376   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2377   if (p)
2378     {
2379       tr = (vlib_thread_registration_t *) p[0];
2380       if (tr)
2381         {
2382           sm->num_workers = tr->count;
2383           sm->first_worker_index = tr->first_index;
2384         }
2385     }
2386   num_threads = tm->n_vlib_mains - 1;
2387   sm->port_per_thread = 0xffff - 1024;
2388   vec_validate (sm->per_thread_data, num_threads);
2389
2390   /* Use all available workers by default */
2391   if (sm->num_workers > 1)
2392     {
2393       for (i = 0; i < sm->num_workers; i++)
2394         bitmap = clib_bitmap_set (bitmap, i, 1);
2395       snat_set_workers (bitmap);
2396       clib_bitmap_free (bitmap);
2397     }
2398   else
2399     {
2400       sm->per_thread_data[0].snat_thread_index = 0;
2401     }
2402
2403   /* callbacks to call when interface address changes. */
2404   cbi.function = nat44_ed_add_del_interface_address_cb;
2405   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2406   cbi.function = nat44_ed_add_del_static_mapping_cb;
2407   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2408
2409   /* callbacks to call when interface to table biding changes */
2410   cbt.function = nat44_ed_update_outside_fib_cb;
2411   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2412
2413   sm->fib_src_low =
2414     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2415                          FIB_SOURCE_BH_SIMPLE);
2416   sm->fib_src_hi =
2417     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2418                          FIB_SOURCE_BH_SIMPLE);
2419
2420   nat_affinity_init (vm);
2421   test_key_calc_split ();
2422
2423   return nat44_api_hookup (vm);
2424 }
2425
2426 VLIB_INIT_FUNCTION (nat_init);
2427
2428 int
2429 nat44_plugin_enable (nat44_config_t c)
2430 {
2431   snat_main_t *sm = &snat_main;
2432
2433   fail_if_enabled ();
2434
2435   sm->forwarding_enabled = 0;
2436   sm->mss_clamping = 0;
2437
2438   if (!c.sessions)
2439     c.sessions = 63 * 1024;
2440
2441   sm->max_translations_per_thread = c.sessions;
2442   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
2443                         sm->max_translations_per_thread);
2444   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2445
2446   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2447
2448   sm->inside_vrf_id = c.inside_vrf;
2449   sm->inside_fib_index =
2450     fib_table_find_or_create_and_lock
2451     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2452
2453   sm->outside_vrf_id = c.outside_vrf;
2454   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2455     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2456
2457   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets);
2458
2459   nat_affinity_enable ();
2460
2461   nat_reset_timeouts (&sm->timeouts);
2462
2463   vlib_zero_simple_counter (&sm->total_sessions, 0);
2464
2465   if (!sm->frame_queue_nelts)
2466     {
2467       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2468     }
2469
2470   if (sm->num_workers > 1)
2471     {
2472       vlib_main_t *vm = vlib_get_main ();
2473       vlib_node_t *node;
2474
2475       if (sm->fq_in2out_index == ~0)
2476         {
2477           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2478           sm->fq_in2out_index =
2479             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2480         }
2481       if (sm->fq_out2in_index == ~0)
2482         {
2483           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2484           sm->fq_out2in_index =
2485             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2486         }
2487       if (sm->fq_in2out_output_index == ~0)
2488         {
2489           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2490           sm->fq_in2out_output_index =
2491             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2492         }
2493     }
2494
2495   sm->enabled = 1;
2496   sm->rconfig = c;
2497
2498   return 0;
2499 }
2500
2501 int
2502 nat44_ed_del_addresses ()
2503 {
2504   snat_main_t *sm = &snat_main;
2505   snat_address_t *a, *vec;
2506   int error = 0;
2507
2508   vec = vec_dup (sm->addresses);
2509   vec_foreach (a, vec)
2510     {
2511       error = nat44_ed_del_address (a->addr, 0);
2512       if (error)
2513         {
2514           nat_log_err ("error occurred while removing adderess");
2515         }
2516     }
2517   vec_free (vec);
2518   vec_free (sm->addresses);
2519   sm->addresses = 0;
2520
2521   vec = vec_dup (sm->twice_nat_addresses);
2522   vec_foreach (a, vec)
2523     {
2524       error = nat44_ed_del_address (a->addr, 1);
2525       if (error)
2526         {
2527           nat_log_err ("error occurred while removing adderess");
2528         }
2529     }
2530   vec_free (vec);
2531   vec_free (sm->twice_nat_addresses);
2532   sm->twice_nat_addresses = 0;
2533
2534   vec_free (sm->addr_to_resolve);
2535   sm->addr_to_resolve = 0;
2536
2537   return error;
2538 }
2539
2540 int
2541 nat44_ed_del_interfaces ()
2542 {
2543   snat_main_t *sm = &snat_main;
2544   snat_interface_t *i, *pool;
2545   int error = 0;
2546
2547   pool = pool_dup (sm->interfaces);
2548   pool_foreach (i, pool)
2549     {
2550       if (nat44_ed_is_interface_inside (i))
2551         {
2552           error = nat44_ed_del_interface (i->sw_if_index, 1);
2553         }
2554       if (nat44_ed_is_interface_outside (i))
2555         {
2556           error = nat44_ed_del_interface (i->sw_if_index, 0);
2557         }
2558
2559       if (error)
2560         {
2561           nat_log_err ("error occurred while removing interface");
2562         }
2563     }
2564   pool_free (pool);
2565   pool_free (sm->interfaces);
2566   sm->interfaces = 0;
2567   return error;
2568 }
2569
2570 int
2571 nat44_ed_del_output_interfaces ()
2572 {
2573   snat_main_t *sm = &snat_main;
2574   snat_interface_t *i, *pool;
2575   int error = 0;
2576
2577   pool = pool_dup (sm->output_feature_interfaces);
2578   pool_foreach (i, pool)
2579     {
2580       error = nat44_ed_del_output_interface (i->sw_if_index);
2581       if (error)
2582         {
2583           nat_log_err ("error occurred while removing output interface");
2584         }
2585     }
2586   pool_free (pool);
2587   pool_free (sm->output_feature_interfaces);
2588   sm->output_feature_interfaces = 0;
2589   return error;
2590 }
2591
2592 int
2593 nat44_ed_del_static_mappings ()
2594 {
2595   snat_main_t *sm = &snat_main;
2596   snat_static_mapping_t *m, *pool;
2597   int error = 0;
2598
2599   pool = pool_dup (sm->static_mappings);
2600   pool_foreach (m, pool)
2601     {
2602       error = nat44_ed_del_static_mapping_internal (
2603         m->local_addr, m->external_addr, m->local_port, m->external_port,
2604         m->proto, m->vrf_id, m->flags);
2605       if (error)
2606         {
2607           nat_log_err ("error occurred while removing mapping");
2608         }
2609     }
2610   pool_free (pool);
2611   pool_free (sm->static_mappings);
2612   sm->static_mappings = 0;
2613
2614   vec_free (sm->sm_to_resolve);
2615   sm->sm_to_resolve = 0;
2616
2617   return error;
2618 }
2619
2620 int
2621 nat44_plugin_disable ()
2622 {
2623   snat_main_per_thread_data_t *tsm;
2624   snat_main_t *sm = &snat_main;
2625   int rc, error = 0;
2626
2627   fail_if_disabled ();
2628
2629   rc = nat44_ed_del_static_mappings ();
2630   if (rc)
2631     error = 1;
2632
2633   rc = nat44_ed_del_addresses ();
2634   if (rc)
2635     error = 1;
2636
2637   rc = nat44_ed_del_interfaces ();
2638   if (rc)
2639     error = 1;
2640
2641   rc = nat44_ed_del_output_interfaces ();
2642   if (rc)
2643     error = 1;
2644
2645   nat44_ed_del_vrf_tables ();
2646
2647   vec_free (sm->max_translations_per_fib);
2648   sm->max_translations_per_fib = 0;
2649
2650   clib_bihash_free_16_8 (&sm->flow_hash);
2651
2652   vec_foreach (tsm, sm->per_thread_data)
2653     {
2654       nat44_ed_worker_db_free (tsm);
2655     }
2656
2657   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2658
2659   nat_affinity_disable ();
2660
2661   sm->forwarding_enabled = 0;
2662   sm->enabled = 0;
2663
2664   return error;
2665 }
2666
2667 void
2668 nat44_ed_forwarding_enable_disable (u8 is_enable)
2669 {
2670   snat_main_per_thread_data_t *tsm;
2671   snat_main_t *sm = &snat_main;
2672   snat_session_t *s;
2673
2674   u32 *ses_to_be_removed = 0, *ses_index;
2675
2676   sm->forwarding_enabled = is_enable != 0;
2677
2678   if (!sm->enabled || is_enable)
2679     {
2680       return;
2681     }
2682
2683   vec_foreach (tsm, sm->per_thread_data)
2684     {
2685       pool_foreach (s, tsm->sessions)
2686         {
2687           if (na44_ed_is_fwd_bypass_session (s))
2688             {
2689               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2690             }
2691         }
2692       vec_foreach (ses_index, ses_to_be_removed)
2693         {
2694           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2695           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2696           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2697         }
2698
2699       vec_free (ses_to_be_removed);
2700     }
2701 }
2702
2703 static_always_inline snat_static_mapping_t *
2704 nat44_ed_sm_match (snat_main_t *sm, ip4_address_t match_addr, u16 match_port,
2705                    u32 match_fib_index, ip_protocol_t match_protocol,
2706                    int by_external)
2707 {
2708   snat_static_mapping_t *m;
2709   if (!by_external)
2710     {
2711       m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port, match_fib_index,
2712                                   match_protocol);
2713       if (m)
2714         return m;
2715
2716       // try address only mapping
2717       m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, match_fib_index, 0);
2718       if (m)
2719         return m;
2720
2721       // default static mapping fib index (based on configuration)
2722       if (sm->inside_fib_index != match_fib_index)
2723         {
2724           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2725                                       sm->inside_fib_index, match_protocol);
2726           if (m)
2727             return m;
2728
2729           // try address only mapping
2730           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->inside_fib_index,
2731                                       0);
2732           if (m)
2733             return m;
2734         }
2735       // TODO: this specific use case may be deprecated (needs testing)
2736       if (sm->outside_fib_index != match_fib_index)
2737         {
2738           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2739                                       sm->outside_fib_index, match_protocol);
2740           if (m)
2741             return m;
2742
2743           // try address only mapping
2744           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->outside_fib_index,
2745                                       0);
2746           if (m)
2747             return m;
2748         }
2749     }
2750   else
2751     {
2752       m =
2753         nat44_ed_sm_o2i_lookup (sm, match_addr, match_port, 0, match_protocol);
2754       if (m)
2755         return m;
2756
2757       // try address only mapping
2758       m = nat44_ed_sm_o2i_lookup (sm, match_addr, 0, 0, 0);
2759       if (m)
2760         return m;
2761     }
2762   return 0;
2763 }
2764
2765 int
2766 snat_static_mapping_match (vlib_main_t *vm, ip4_address_t match_addr,
2767                            u16 match_port, u32 match_fib_index,
2768                            ip_protocol_t match_protocol,
2769                            ip4_address_t *mapping_addr, u16 *mapping_port,
2770                            u32 *mapping_fib_index, int by_external,
2771                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2772                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2773                            u8 *is_identity_nat, snat_static_mapping_t **out)
2774 {
2775   snat_main_t *sm = &snat_main;
2776   snat_static_mapping_t *m;
2777   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2778   nat44_lb_addr_port_t *local;
2779   u8 backend_index;
2780
2781   m = nat44_ed_sm_match (sm, match_addr, match_port, match_fib_index,
2782                          match_protocol, by_external);
2783   if (!m)
2784     {
2785       return 1;
2786     }
2787
2788   if (by_external)
2789     {
2790       if (is_sm_lb (m->flags))
2791         {
2792           if (PREDICT_FALSE (lb != 0))
2793             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2794           if (m->affinity && !nat_affinity_find_and_lock (
2795                                vm, ext_host_addr[0], match_addr,
2796                                match_protocol, match_port, &backend_index))
2797             {
2798               local = pool_elt_at_index (m->locals, backend_index);
2799               *mapping_addr = local->addr;
2800               *mapping_port = local->port;
2801               *mapping_fib_index = local->fib_index;
2802               goto end;
2803             }
2804           // pick locals matching this worker
2805           if (PREDICT_FALSE (sm->num_workers > 1))
2806             {
2807               u32 thread_index = vlib_get_thread_index ();
2808               pool_foreach_index (i, m->locals)
2809                {
2810                 local = pool_elt_at_index (m->locals, i);
2811
2812                 ip4_header_t ip = {
2813                   .src_address = local->addr,
2814                 };
2815
2816                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2817                                                       0) == thread_index)
2818                   {
2819                     vec_add1 (tmp, i);
2820                   }
2821                }
2822               ASSERT (vec_len (tmp) != 0);
2823             }
2824           else
2825             {
2826               pool_foreach_index (i, m->locals)
2827                {
2828                 vec_add1 (tmp, i);
2829               }
2830             }
2831           hi = vec_len (tmp) - 1;
2832           local = pool_elt_at_index (m->locals, tmp[hi]);
2833           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2834           while (lo < hi)
2835             {
2836               mid = ((hi - lo) >> 1) + lo;
2837               local = pool_elt_at_index (m->locals, tmp[mid]);
2838               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2839             }
2840           local = pool_elt_at_index (m->locals, tmp[lo]);
2841           if (!(local->prefix >= rand))
2842             return 1;
2843           *mapping_addr = local->addr;
2844           *mapping_port = local->port;
2845           *mapping_fib_index = local->fib_index;
2846           if (m->affinity)
2847             {
2848               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2849                                                 match_protocol, match_port,
2850                                                 tmp[lo], m->affinity,
2851                                                 m->affinity_per_service_list_head_index))
2852                 nat_elog_info (sm, "create affinity record failed");
2853             }
2854           vec_free (tmp);
2855         }
2856       else
2857         {
2858           if (PREDICT_FALSE (lb != 0))
2859             *lb = NO_LB_NAT;
2860           *mapping_fib_index = m->fib_index;
2861           *mapping_addr = m->local_addr;
2862           /* Address only mapping doesn't change port */
2863           *mapping_port =
2864             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2865         }
2866     }
2867   else
2868     {
2869       *mapping_addr = m->external_addr;
2870       /* Address only mapping doesn't change port */
2871       *mapping_port =
2872         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2873       *mapping_fib_index = sm->outside_fib_index;
2874     }
2875
2876 end:
2877   if (PREDICT_FALSE (is_addr_only != 0))
2878     *is_addr_only = is_sm_addr_only (m->flags);
2879
2880   if (PREDICT_FALSE (twice_nat != 0))
2881     {
2882       *twice_nat = TWICE_NAT_DISABLED;
2883
2884       if (is_sm_twice_nat (m->flags))
2885         {
2886           *twice_nat = TWICE_NAT;
2887         }
2888       else if (is_sm_self_twice_nat (m->flags))
2889         {
2890           *twice_nat = TWICE_NAT_SELF;
2891         }
2892     }
2893
2894   if (PREDICT_FALSE (is_identity_nat != 0))
2895     *is_identity_nat = is_sm_identity_nat (m->flags);
2896
2897   if (out != 0)
2898     *out = m;
2899
2900   return 0;
2901 }
2902
2903 u32
2904 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2905                                   u32 rx_fib_index, u8 is_output)
2906 {
2907   snat_main_t *sm = &snat_main;
2908   u32 next_worker_index = sm->first_worker_index;
2909   u32 hash;
2910
2911   clib_bihash_kv_16_8_t kv16, value16;
2912
2913   u32 fib_index = rx_fib_index;
2914   if (b)
2915     {
2916       if (PREDICT_FALSE (is_output))
2917         {
2918           fib_index = sm->outside_fib_index;
2919           nat_fib_t *outside_fib;
2920           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2921           fib_prefix_t pfx = {
2922                   .fp_proto = FIB_PROTOCOL_IP4,
2923                   .fp_len = 32,
2924                   .fp_addr = {
2925                           .ip4.as_u32 = ip->dst_address.as_u32,
2926                   } ,
2927           };
2928
2929           switch (vec_len (sm->outside_fibs))
2930             {
2931             case 0:
2932               fib_index = sm->outside_fib_index;
2933               break;
2934             case 1:
2935               fib_index = sm->outside_fibs[0].fib_index;
2936               break;
2937             default:
2938               vec_foreach (outside_fib, sm->outside_fibs)
2939                 {
2940                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2941                   if (FIB_NODE_INDEX_INVALID != fei)
2942                     {
2943                       if (fib_entry_get_resolving_interface (fei) != ~0)
2944                         {
2945                           fib_index = outside_fib->fib_index;
2946                           break;
2947                         }
2948                     }
2949                 }
2950               break;
2951             }
2952         }
2953
2954       if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2955         {
2956           ip4_address_t lookup_saddr, lookup_daddr;
2957           u16 lookup_sport, lookup_dport;
2958           u8 lookup_protocol;
2959
2960           if (!nat_get_icmp_session_lookup_values (
2961                 b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr,
2962                 &lookup_dport, &lookup_protocol))
2963             {
2964               init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
2965                          lookup_daddr.as_u32, lookup_dport, rx_fib_index,
2966                          lookup_protocol);
2967               if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2968                 {
2969                   next_worker_index = ed_value_get_thread_index (&value16);
2970                   vnet_buffer2 (b)->nat.cached_session_index =
2971                     ed_value_get_session_index (&value16);
2972                   goto out;
2973                 }
2974             }
2975         }
2976
2977       init_ed_k (&kv16, ip->src_address.as_u32,
2978                  vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
2979                  vnet_buffer (b)->ip.reass.l4_dst_port, fib_index,
2980                  ip->protocol);
2981
2982       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2983         {
2984           next_worker_index = ed_value_get_thread_index (&value16);
2985           vnet_buffer2 (b)->nat.cached_session_index =
2986             ed_value_get_session_index (&value16);
2987           goto out;
2988         }
2989
2990       // dst NAT
2991       init_ed_k (&kv16, ip->dst_address.as_u32,
2992                  vnet_buffer (b)->ip.reass.l4_dst_port, ip->src_address.as_u32,
2993                  vnet_buffer (b)->ip.reass.l4_src_port, rx_fib_index,
2994                  ip->protocol);
2995       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2996         {
2997           next_worker_index = ed_value_get_thread_index (&value16);
2998           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
2999             ed_value_get_session_index (&value16);
3000           goto out;
3001         }
3002     }
3003
3004   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3005     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3006
3007   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3008     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3009   else
3010     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3011
3012 out:
3013   if (PREDICT_TRUE (!is_output))
3014     {
3015       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
3016                               rx_fib_index,
3017                               clib_net_to_host_u32 (ip->src_address.as_u32),
3018                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3019     }
3020   else
3021     {
3022       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
3023                               next_worker_index, rx_fib_index,
3024                               clib_net_to_host_u32 (ip->src_address.as_u32),
3025                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3026     }
3027
3028   return next_worker_index;
3029 }
3030
3031 u32
3032 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
3033                                   u32 rx_fib_index, u8 is_output)
3034 {
3035   snat_main_t *sm = &snat_main;
3036   clib_bihash_kv_16_8_t kv16, value16;
3037
3038   u8 proto, next_worker_index = 0;
3039   u16 port;
3040   snat_static_mapping_t *m;
3041   u32 hash;
3042
3043   proto = ip->protocol;
3044
3045   if (PREDICT_FALSE (IP_PROTOCOL_ICMP == proto))
3046     {
3047       ip4_address_t lookup_saddr, lookup_daddr;
3048       u16 lookup_sport, lookup_dport;
3049       u8 lookup_protocol;
3050       if (!nat_get_icmp_session_lookup_values (
3051             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3052             &lookup_protocol))
3053         {
3054           init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3055                      lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3056                      lookup_protocol);
3057           if (PREDICT_TRUE (
3058                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3059             {
3060               next_worker_index = ed_value_get_thread_index (&value16);
3061               nat_elog_debug_handoff (
3062                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
3063                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
3064                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3065               return next_worker_index;
3066             }
3067         }
3068     }
3069
3070   init_ed_k (&kv16, ip->src_address.as_u32,
3071              vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3072              vnet_buffer (b)->ip.reass.l4_dst_port, rx_fib_index,
3073              ip->protocol);
3074
3075   if (PREDICT_TRUE (
3076         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3077     {
3078       vnet_buffer2 (b)->nat.cached_session_index =
3079         ed_value_get_session_index (&value16);
3080       next_worker_index = ed_value_get_thread_index (&value16);
3081       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
3082                               next_worker_index, rx_fib_index,
3083                               clib_net_to_host_u32 (ip->src_address.as_u32),
3084                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3085       return next_worker_index;
3086     }
3087
3088   /* first try static mappings without port */
3089   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3090     {
3091       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, proto);
3092       if (m)
3093         {
3094           {
3095             next_worker_index = m->workers[0];
3096             goto done;
3097           }
3098         }
3099     }
3100
3101   /* unknown protocol */
3102   if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto)))
3103     {
3104       /* use current thread */
3105       next_worker_index = vlib_get_thread_index ();
3106       goto done;
3107     }
3108
3109   port = vnet_buffer (b)->ip.reass.l4_dst_port;
3110
3111   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3112     {
3113       udp_header_t *udp = ip4_next_header (ip);
3114       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3115       nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3116       if (!icmp_type_is_error_message
3117           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3118         port = vnet_buffer (b)->ip.reass.l4_src_port;
3119       else
3120         {
3121           /* if error message, then it's not fragmented and we can access it */
3122           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3123           proto = inner_ip->protocol;
3124           void *l4_header = ip4_next_header (inner_ip);
3125           switch (proto)
3126             {
3127             case IP_PROTOCOL_ICMP:
3128               icmp = (icmp46_header_t *) l4_header;
3129               echo = (nat_icmp_echo_header_t *) (icmp + 1);
3130               port = echo->identifier;
3131               break;
3132             case IP_PROTOCOL_UDP:
3133               /* breakthrough */
3134             case IP_PROTOCOL_TCP:
3135               port = ((nat_tcp_udp_header_t *) l4_header)->src_port;
3136               break;
3137             default:
3138               next_worker_index = vlib_get_thread_index ();
3139               goto done;
3140             }
3141         }
3142     }
3143
3144   /* try static mappings with port */
3145   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3146     {
3147       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, port, 0, proto);
3148       if (m)
3149         {
3150           if (!is_sm_lb (m->flags))
3151             {
3152               next_worker_index = m->workers[0];
3153               goto done;
3154             }
3155
3156           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3157             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3158
3159           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3160             next_worker_index =
3161               m->workers[hash & (_vec_len (m->workers) - 1)];
3162           else
3163             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3164           goto done;
3165         }
3166     }
3167
3168   /* worker by outside port */
3169   next_worker_index = sm->first_worker_index;
3170   next_worker_index +=
3171     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3172
3173 done:
3174   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3175                           rx_fib_index,
3176                           clib_net_to_host_u32 (ip->src_address.as_u32),
3177                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3178   return next_worker_index;
3179 }
3180
3181 u32
3182 nat44_get_max_session_limit ()
3183 {
3184   snat_main_t *sm = &snat_main;
3185   u32 max_limit = 0, len = 0;
3186
3187   for (; len < vec_len (sm->max_translations_per_fib); len++)
3188     {
3189       if (max_limit < sm->max_translations_per_fib[len])
3190         max_limit = sm->max_translations_per_fib[len];
3191     }
3192   return max_limit;
3193 }
3194
3195 int
3196 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3197 {
3198   snat_main_t *sm = &snat_main;
3199   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3200   u32 len = vec_len (sm->max_translations_per_fib);
3201
3202   if (len <= fib_index)
3203     {
3204       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3205
3206       for (; len < vec_len (sm->max_translations_per_fib); len++)
3207         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3208     }
3209
3210   sm->max_translations_per_fib[fib_index] = session_limit;
3211   return 0;
3212 }
3213
3214 int
3215 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3216 {
3217   snat_main_t *sm = &snat_main;
3218
3219   if (nat44_set_session_limit (session_limit, vrf_id))
3220     return 1;
3221   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3222
3223   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
3224                         sm->max_translations_per_thread);
3225
3226   sm->translation_buckets =
3227     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3228
3229   nat44_ed_sessions_clear ();
3230   return 0;
3231 }
3232
3233 static void
3234 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3235                          u32 translation_buckets)
3236 {
3237   dlist_elt_t *head;
3238
3239   pool_alloc (tsm->sessions, translations);
3240   pool_alloc (tsm->lru_pool, translations);
3241
3242   pool_get (tsm->lru_pool, head);
3243   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3244   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3245
3246   pool_get (tsm->lru_pool, head);
3247   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3248   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3249
3250   pool_get (tsm->lru_pool, head);
3251   tsm->udp_lru_head_index = head - tsm->lru_pool;
3252   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3253
3254   pool_get (tsm->lru_pool, head);
3255   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3256   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3257
3258   pool_get (tsm->lru_pool, head);
3259   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3260   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3261 }
3262
3263 static void
3264 reinit_ed_flow_hash ()
3265 {
3266   snat_main_t *sm = &snat_main;
3267   // we expect 2 flows per session, so multiply translation_buckets by 2
3268   clib_bihash_init_16_8 (
3269     &sm->flow_hash, "ed-flow-hash",
3270     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3271   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3272 }
3273
3274 static void
3275 nat44_ed_db_init (u32 translations, u32 translation_buckets)
3276 {
3277   snat_main_t *sm = &snat_main;
3278   snat_main_per_thread_data_t *tsm;
3279
3280   reinit_ed_flow_hash ();
3281
3282   vec_foreach (tsm, sm->per_thread_data)
3283     {
3284       nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3285                                sm->translation_buckets);
3286     }
3287 }
3288
3289 static void
3290 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3291 {
3292   pool_free (tsm->lru_pool);
3293   pool_free (tsm->sessions);
3294   vec_free (tsm->per_vrf_sessions_vec);
3295 }
3296
3297 void
3298 nat44_ed_sessions_clear ()
3299 {
3300   snat_main_t *sm = &snat_main;
3301   snat_main_per_thread_data_t *tsm;
3302
3303   reinit_ed_flow_hash ();
3304
3305   vec_foreach (tsm, sm->per_thread_data)
3306     {
3307       nat44_ed_worker_db_free (tsm);
3308       nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3309                                sm->translation_buckets);
3310     }
3311   vlib_zero_simple_counter (&sm->total_sessions, 0);
3312 }
3313
3314 static void
3315 nat44_ed_add_del_static_mapping_cb (ip4_main_t *im, uword opaque,
3316                                     u32 sw_if_index, ip4_address_t *address,
3317                                     u32 address_length, u32 if_address_index,
3318                                     u32 is_delete)
3319 {
3320   snat_static_mapping_resolve_t *rp;
3321   snat_main_t *sm = &snat_main;
3322   int rv = 0;
3323
3324   if (!sm->enabled)
3325     {
3326       return;
3327     }
3328
3329   vec_foreach (rp, sm->sm_to_resolve)
3330     {
3331       if (sw_if_index == rp->sw_if_index)
3332         {
3333           if (is_delete)
3334             {
3335               if (rp->is_resolved)
3336                 {
3337                   rv = nat44_ed_del_static_mapping_internal (
3338                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3339                     rp->vrf_id, rp->flags);
3340                   if (rv)
3341                     {
3342                       nat_log_err ("ed del static mapping failed");
3343                     }
3344                   else
3345                     {
3346                       rp->is_resolved = 0;
3347                     }
3348                 }
3349             }
3350           else
3351             {
3352               if (!rp->is_resolved)
3353                 {
3354                   rv = nat44_ed_add_static_mapping_internal (
3355                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3356                     rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3357                   if (rv)
3358                     {
3359                       nat_log_err ("ed add static mapping failed");
3360                     }
3361                   else
3362                     {
3363                       rp->is_resolved = 1;
3364                     }
3365                 }
3366             }
3367         }
3368     }
3369 }
3370
3371 static int
3372 nat44_ed_get_addr_resolve_record (u32 sw_if_index, u8 twice_nat, int *out)
3373 {
3374   snat_main_t *sm = &snat_main;
3375   snat_address_resolve_t *rp;
3376   int i;
3377
3378   for (i = 0; i < vec_len (sm->addr_to_resolve); i++)
3379     {
3380       rp = sm->addr_to_resolve + i;
3381
3382       if ((rp->sw_if_index == sw_if_index) && (rp->is_twice_nat == twice_nat))
3383         {
3384           if (out)
3385             {
3386               *out = i;
3387             }
3388           return 0;
3389         }
3390     }
3391   return 1;
3392 }
3393 static int
3394 nat44_ed_del_addr_resolve_record (u32 sw_if_index, u8 twice_nat)
3395 {
3396   snat_main_t *sm = &snat_main;
3397   int i;
3398   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3399     {
3400       vec_del1 (sm->addr_to_resolve, i);
3401       return 0;
3402     }
3403   return 1;
3404 }
3405
3406 static void
3407 nat44_ed_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3408                                        u32 sw_if_index, ip4_address_t *address,
3409                                        u32 address_length,
3410                                        u32 if_address_index, u32 is_delete)
3411 {
3412   snat_main_t *sm = &snat_main;
3413   snat_address_resolve_t *arp;
3414   snat_address_t *ap;
3415   u8 twice_nat = 0;
3416   int i, rv;
3417
3418   if (!sm->enabled)
3419     {
3420       return;
3421     }
3422
3423   if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3424     {
3425       twice_nat = 1;
3426       if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3427         {
3428           u32 fib_index =
3429             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
3430           vec_foreach (ap, sm->addresses)
3431             {
3432               if ((fib_index == ap->fib_index) &&
3433                   (address->as_u32 == ap->addr.as_u32))
3434                 {
3435                   if (!is_delete)
3436                     {
3437                       ap->addr_len = address_length;
3438                       ap->sw_if_index = sw_if_index;
3439                       ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
3440                                        << (32 - ap->addr_len);
3441
3442                       nat_log_debug (
3443                         "pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
3444                         format_ip4_address, &ap->addr, ap->sw_if_index,
3445                         format_ip4_address, &ap->net, ap->addr_len);
3446                     }
3447                   else
3448                     {
3449                       ap->addr_len = ~0;
3450                     }
3451                   break;
3452                 }
3453             }
3454           return;
3455         }
3456     }
3457
3458   arp = sm->addr_to_resolve + i;
3459
3460   if (!is_delete)
3461     {
3462       if (arp->is_resolved)
3463         {
3464           return;
3465         }
3466
3467       rv = nat44_ed_add_address (address, ~0, arp->is_twice_nat);
3468       if (0 == rv)
3469         {
3470           arp->is_resolved = 1;
3471         }
3472     }
3473   else
3474     {
3475       if (!arp->is_resolved)
3476         {
3477           return;
3478         }
3479
3480       rv = nat44_ed_del_address (address[0], arp->is_twice_nat);
3481       if (0 == rv)
3482         {
3483           arp->is_resolved = 0;
3484         }
3485     }
3486 }
3487
3488 int
3489 nat44_ed_add_interface_address (u32 sw_if_index, u8 twice_nat)
3490 {
3491   snat_main_t *sm = &snat_main;
3492   ip4_main_t *ip4_main = sm->ip4_main;
3493   ip4_address_t *first_int_addr;
3494   snat_address_resolve_t *ap;
3495   int rv;
3496
3497   if (!sm->enabled)
3498     {
3499       nat_log_err ("nat44 is disabled");
3500       return VNET_API_ERROR_UNSUPPORTED;
3501     }
3502
3503   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, 0))
3504     {
3505       return VNET_API_ERROR_VALUE_EXIST;
3506     }
3507
3508   vec_add2 (sm->addr_to_resolve, ap, 1);
3509   ap->sw_if_index = sw_if_index;
3510   ap->is_twice_nat = twice_nat;
3511   ap->is_resolved = 0;
3512
3513   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3514   if (first_int_addr)
3515     {
3516       rv = nat44_ed_add_address (first_int_addr, ~0, twice_nat);
3517       if (0 != rv)
3518         {
3519           nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat);
3520           return rv;
3521         }
3522       ap->is_resolved = 1;
3523     }
3524
3525   return 0;
3526 }
3527
3528 int
3529 nat44_ed_del_interface_address (u32 sw_if_index, u8 twice_nat)
3530 {
3531   snat_main_t *sm = &snat_main;
3532   ip4_main_t *ip4_main = sm->ip4_main;
3533   ip4_address_t *first_int_addr;
3534
3535   if (!sm->enabled)
3536     {
3537       nat_log_err ("nat44 is disabled");
3538       return VNET_API_ERROR_UNSUPPORTED;
3539     }
3540
3541   if (nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat))
3542     {
3543       return VNET_API_ERROR_NO_SUCH_ENTRY;
3544     }
3545
3546   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3547   if (first_int_addr)
3548     {
3549       return nat44_ed_del_address (first_int_addr[0], twice_nat);
3550     }
3551
3552   return 0;
3553 }
3554
3555 int
3556 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3557                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3558                       u32 vrf_id, int is_in)
3559 {
3560   ip4_header_t ip;
3561   clib_bihash_kv_16_8_t kv, value;
3562   u32 fib_index;
3563   snat_session_t *s;
3564   snat_main_per_thread_data_t *tsm;
3565
3566   if (!sm->enabled)
3567     {
3568       return VNET_API_ERROR_UNSUPPORTED;
3569     }
3570
3571   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3572   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3573   if (sm->num_workers > 1)
3574     tsm = vec_elt_at_index (
3575       sm->per_thread_data,
3576       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3577   else
3578     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3579
3580   init_ed_k (&kv, addr->as_u32, port, eh_addr->as_u32, eh_port, fib_index,
3581              proto);
3582   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3583     {
3584       return VNET_API_ERROR_NO_SUCH_ENTRY;
3585     }
3586
3587   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3588     return VNET_API_ERROR_UNSPECIFIED;
3589   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3590   nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3591   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3592   return 0;
3593 }
3594
3595 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3596                                  vlib_node_runtime_t * node,
3597                                  vlib_frame_t * frame)
3598 {
3599   return 0;
3600 }
3601
3602 VLIB_REGISTER_NODE (nat_default_node) = {
3603   .name = "nat-default",
3604   .vector_size = sizeof (u32),
3605   .format_trace = 0,
3606   .type = VLIB_NODE_TYPE_INTERNAL,
3607   .n_errors = 0,
3608   .n_next_nodes = NAT_N_NEXT,
3609   .next_nodes = {
3610     [NAT_NEXT_DROP] = "error-drop",
3611     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3612     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3613     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3614     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3615     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3616     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3617     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3618     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3619     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3620   },
3621 };
3622
3623 void
3624 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3625 {
3626   f->l3_csum_delta = 0;
3627   f->l4_csum_delta = 0;
3628   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3629       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3630     {
3631       f->l3_csum_delta =
3632         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3633       f->l3_csum_delta =
3634         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3635     }
3636   else
3637     {
3638       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3639     }
3640   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3641       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3642     {
3643       f->l3_csum_delta =
3644         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3645       f->l3_csum_delta =
3646         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3647     }
3648   else
3649     {
3650       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3651     }
3652   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3653     {
3654       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3655       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3656     }
3657   else
3658     {
3659       f->rewrite.sport = f->match.sport;
3660     }
3661   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3662     {
3663       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3664       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3665     }
3666   else
3667     {
3668       f->rewrite.dport = f->match.dport;
3669     }
3670   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3671       f->rewrite.icmp_id != f->match.sport)
3672     {
3673       f->l4_csum_delta =
3674         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3675       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3676     }
3677   else
3678     {
3679       f->rewrite.icmp_id = f->match.sport;
3680     }
3681   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3682     {
3683     }
3684   else
3685     {
3686       f->rewrite.fib_index = f->match.fib_index;
3687     }
3688 }
3689
3690 static_always_inline int
3691 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3692                             ip4_header_t *ip, nat_6t_flow_t *f);
3693
3694 static_always_inline void
3695 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3696                            nat_6t_flow_t *f, ip_protocol_t proto,
3697                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3698 {
3699   udp_header_t *udp = ip4_next_header (ip);
3700   tcp_header_t *tcp = (tcp_header_t *) udp;
3701
3702   if ((IP_PROTOCOL_TCP == proto || IP_PROTOCOL_UDP == proto) &&
3703       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3704     {
3705       if (!is_icmp_inner_ip4)
3706         { // regular case
3707           ip->src_address = f->rewrite.saddr;
3708           ip->dst_address = f->rewrite.daddr;
3709           udp->src_port = f->rewrite.sport;
3710           udp->dst_port = f->rewrite.dport;
3711         }
3712       else
3713         { // icmp inner ip4 - reversed saddr/daddr
3714           ip->src_address = f->rewrite.daddr;
3715           ip->dst_address = f->rewrite.saddr;
3716           udp->src_port = f->rewrite.dport;
3717           udp->dst_port = f->rewrite.sport;
3718         }
3719
3720       if (IP_PROTOCOL_TCP == proto)
3721         {
3722           ip_csum_t tcp_sum = tcp->checksum;
3723           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3724           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3725           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3726           tcp->checksum = ip_csum_fold (tcp_sum);
3727         }
3728       else if (IP_PROTOCOL_UDP == proto && udp->checksum)
3729         {
3730           ip_csum_t udp_sum = udp->checksum;
3731           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3732           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3733           udp->checksum = ip_csum_fold (udp_sum);
3734         }
3735     }
3736   else
3737     {
3738       if (!is_icmp_inner_ip4)
3739         { // regular case
3740           if (!skip_saddr_rewrite)
3741             {
3742               ip->src_address = f->rewrite.saddr;
3743             }
3744           ip->dst_address = f->rewrite.daddr;
3745         }
3746       else
3747         { // icmp inner ip4 - reversed saddr/daddr
3748           ip->src_address = f->rewrite.daddr;
3749           ip->dst_address = f->rewrite.saddr;
3750         }
3751     }
3752
3753   if (skip_saddr_rewrite)
3754     {
3755       ip->checksum = ip4_header_checksum (ip);
3756     }
3757   else
3758     {
3759       ip_csum_t ip_sum = ip->checksum;
3760       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3761       ip->checksum = ip_csum_fold (ip_sum);
3762     }
3763   if (0xffff == ip->checksum)
3764     ip->checksum = 0;
3765   ASSERT (ip4_header_checksum_is_valid (ip));
3766 }
3767
3768 static_always_inline int
3769 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3770 {
3771   int result = ((u8 *) object + size <=
3772                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3773                vlib_object_within_buffer_data (vm, b, object, size);
3774   return result;
3775 }
3776
3777 static_always_inline int
3778 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3779                             ip4_header_t *ip, nat_6t_flow_t *f)
3780 {
3781   if (IP_PROTOCOL_ICMP != ip->protocol)
3782     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3783
3784   icmp46_header_t *icmp = ip4_next_header (ip);
3785   nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3786
3787   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3788     {
3789       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3790         {
3791           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3792         }
3793
3794       if (!icmp_type_is_error_message (icmp->type))
3795         {
3796           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3797               (f->rewrite.icmp_id != echo->identifier))
3798             {
3799               ip_csum_t sum = icmp->checksum;
3800               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3801                                     nat_icmp_echo_header_t,
3802                                     identifier /* changed member */);
3803               echo->identifier = f->rewrite.icmp_id;
3804               icmp->checksum = ip_csum_fold (sum);
3805             }
3806         }
3807       else
3808         {
3809           ip_csum_t sum = ip_incremental_checksum (
3810             0, icmp,
3811             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3812           sum = (u16) ~ip_csum_fold (sum);
3813           if (sum != 0)
3814             {
3815               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3816             }
3817
3818           // errors are not fragmented
3819           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3820
3821           if (!ip4_header_checksum_is_valid (inner_ip))
3822             {
3823               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3824             }
3825
3826           ip_protocol_t inner_proto = inner_ip->protocol;
3827
3828           ip_csum_t old_icmp_sum = icmp->checksum;
3829           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3830           ip_csum_t old_udp_sum;
3831           ip_csum_t old_tcp_sum;
3832           ip_csum_t new_icmp_sum;
3833           udp_header_t *udp;
3834           tcp_header_t *tcp;
3835
3836           switch (inner_proto)
3837             {
3838             case IP_PROTOCOL_UDP:
3839               udp = (udp_header_t *) (inner_ip + 1);
3840               if (!it_fits (vm, b, udp, sizeof (*udp)))
3841                 {
3842                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3843                 }
3844               old_udp_sum = udp->checksum;
3845               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3846                                          1 /* is_icmp_inner_ip4 */,
3847                                          0 /* skip_saddr_rewrite */);
3848               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3849               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3850               new_icmp_sum =
3851                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3852                                 inner_ip->checksum, ip4_header_t, checksum);
3853               new_icmp_sum =
3854                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3855                                 udp_header_t, checksum);
3856               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3857               icmp->checksum = new_icmp_sum;
3858               break;
3859             case IP_PROTOCOL_TCP:
3860               tcp = (tcp_header_t *) (inner_ip + 1);
3861               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3862                 {
3863                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3864                 }
3865               old_tcp_sum = tcp->checksum;
3866               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3867                                          1 /* is_icmp_inner_ip4 */,
3868                                          0 /* skip_saddr_rewrite */);
3869               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3870               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3871               new_icmp_sum =
3872                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3873                                 inner_ip->checksum, ip4_header_t, checksum);
3874               new_icmp_sum =
3875                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3876                                 tcp_header_t, checksum);
3877               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3878               icmp->checksum = new_icmp_sum;
3879               break;
3880             case IP_PROTOCOL_ICMP:
3881               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3882                                          1 /* is_icmp_inner_ip4 */,
3883                                          0 /* skip_saddr_rewrite */);
3884               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3885                 {
3886                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3887                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3888                     {
3889                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3890                     }
3891                   nat_icmp_echo_header_t *inner_echo =
3892                     (nat_icmp_echo_header_t *) (inner_icmp + 1);
3893                   if (f->rewrite.icmp_id != inner_echo->identifier)
3894                     {
3895                       ip_csum_t sum = icmp->checksum;
3896                       sum = ip_csum_update (sum, inner_echo->identifier,
3897                                             f->rewrite.icmp_id,
3898                                             nat_icmp_echo_header_t,
3899                                             identifier /* changed member */);
3900                       icmp->checksum = ip_csum_fold (sum);
3901                       ip_csum_t inner_sum = inner_icmp->checksum;
3902                       inner_sum = ip_csum_update (
3903                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3904                         nat_icmp_echo_header_t,
3905                         identifier /* changed member */);
3906                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3907                       inner_echo->identifier = f->rewrite.icmp_id;
3908                     }
3909                 }
3910               break;
3911             default:
3912               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3913               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3914             }
3915         }
3916     }
3917
3918   return NAT_ED_TRNSL_ERR_SUCCESS;
3919 }
3920
3921 static_always_inline nat_translation_error_e
3922 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3923                            ip4_header_t *ip, nat_6t_flow_t *f,
3924                            ip_protocol_t proto, int is_output_feature,
3925                            int is_i2o)
3926 {
3927   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3928     {
3929       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3930     }
3931
3932   if (IP_PROTOCOL_ICMP == proto)
3933     {
3934       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3935         {
3936           // packet is returned from a router, not from destination
3937           // skip source address rewrite if in o2i path
3938           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3939                                      0 /* is_icmp_inner_ip4 */,
3940                                      !is_i2o /* skip_saddr_rewrite */);
3941         }
3942       else
3943         {
3944           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3945                                      0 /* is_icmp_inner_ip4 */,
3946                                      0 /* skip_saddr_rewrite */);
3947         }
3948       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
3949     }
3950
3951   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
3952                              0 /* skip_saddr_rewrite */);
3953
3954   return NAT_ED_TRNSL_ERR_SUCCESS;
3955 }
3956
3957 nat_translation_error_e
3958 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
3959                                vlib_buffer_t *b, ip4_header_t *ip,
3960                                nat_6t_flow_t *f, ip_protocol_t proto,
3961                                int is_output_feature)
3962 {
3963   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3964                                     1 /* is_i2o */);
3965 }
3966
3967 nat_translation_error_e
3968 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
3969                                vlib_buffer_t *b, ip4_header_t *ip,
3970                                nat_6t_flow_t *f, ip_protocol_t proto,
3971                                int is_output_feature)
3972 {
3973   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3974                                     0 /* is_i2o */);
3975 }
3976
3977 static_always_inline void
3978 nat_syslog_nat44_sess (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
3979                        u16 isport, ip4_address_t *xsaddr, u16 xsport,
3980                        ip4_address_t *idaddr, u16 idport,
3981                        ip4_address_t *xdaddr, u16 xdport, u8 proto, u8 is_add,
3982                        u8 is_twicenat)
3983 {
3984   syslog_msg_t syslog_msg;
3985   fib_table_t *fib;
3986
3987   if (!syslog_is_enabled ())
3988     return;
3989
3990   if (syslog_severity_filter_block (SADD_SDEL_SEVERITY))
3991     return;
3992
3993   fib = fib_table_get (sfibix, FIB_PROTOCOL_IP4);
3994
3995   syslog_msg_init (&syslog_msg, NAT_FACILITY, SADD_SDEL_SEVERITY, NAT_APPNAME,
3996                    is_add ? SADD_MSGID : SDEL_MSGID);
3997
3998   syslog_msg_sd_init (&syslog_msg, NSESS_SDID);
3999   syslog_msg_add_sd_param (&syslog_msg, SSUBIX_SDPARAM_NAME, "%d", ssubix);
4000   syslog_msg_add_sd_param (&syslog_msg, SVLAN_SDPARAM_NAME, "%d",
4001                            fib->ft_table_id);
4002   syslog_msg_add_sd_param (&syslog_msg, IATYP_SDPARAM_NAME, IATYP_IPV4);
4003   syslog_msg_add_sd_param (&syslog_msg, ISADDR_SDPARAM_NAME, "%U",
4004                            format_ip4_address, isaddr);
4005   syslog_msg_add_sd_param (&syslog_msg, ISPORT_SDPARAM_NAME, "%d",
4006                            clib_net_to_host_u16 (isport));
4007   syslog_msg_add_sd_param (&syslog_msg, XATYP_SDPARAM_NAME, IATYP_IPV4);
4008   syslog_msg_add_sd_param (&syslog_msg, XSADDR_SDPARAM_NAME, "%U",
4009                            format_ip4_address, xsaddr);
4010   syslog_msg_add_sd_param (&syslog_msg, XSPORT_SDPARAM_NAME, "%d",
4011                            clib_net_to_host_u16 (xsport));
4012   syslog_msg_add_sd_param (&syslog_msg, PROTO_SDPARAM_NAME, "%d", proto);
4013   syslog_msg_add_sd_param (&syslog_msg, XDADDR_SDPARAM_NAME, "%U",
4014                            format_ip4_address, xdaddr);
4015   syslog_msg_add_sd_param (&syslog_msg, XDPORT_SDPARAM_NAME, "%d",
4016                            clib_net_to_host_u16 (xdport));
4017   if (is_twicenat)
4018     {
4019       syslog_msg_add_sd_param (&syslog_msg, IDADDR_SDPARAM_NAME, "%U",
4020                                format_ip4_address, idaddr);
4021       syslog_msg_add_sd_param (&syslog_msg, IDPORT_SDPARAM_NAME, "%d",
4022                                clib_net_to_host_u16 (idport));
4023     }
4024
4025   syslog_msg_send (&syslog_msg);
4026 }
4027
4028 void
4029 nat_syslog_nat44_sadd (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4030                        u16 isport, ip4_address_t *idaddr, u16 idport,
4031                        ip4_address_t *xsaddr, u16 xsport,
4032                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4033                        u8 is_twicenat)
4034 {
4035   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4036                          idaddr, idport, xdaddr, xdport, proto, 1,
4037                          is_twicenat);
4038 }
4039
4040 void
4041 nat_syslog_nat44_sdel (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4042                        u16 isport, ip4_address_t *idaddr, u16 idport,
4043                        ip4_address_t *xsaddr, u16 xsport,
4044                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4045                        u8 is_twicenat)
4046 {
4047   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4048                          idaddr, idport, xdaddr, xdport, proto, 0,
4049                          is_twicenat);
4050 }
4051
4052 /*
4053  * fd.io coding-style-patch-verification: ON
4054  *
4055  * Local Variables:
4056  * eval: (c-set-style "gnu")
4057  * End:
4058  */