src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vpp/app/version.h>
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/ip/ip_table.h>
  22 #include <vnet/ip/reass/ip4_sv_reass.h>
  23 #include <vnet/fib/fib_table.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/plugin/plugin.h>
  26 #include <vppinfra/bihash_16_8.h>
  27
  28 #include <nat/lib/log.h>
  29 #include <nat/lib/nat_inlines.h>
  30 #include <nat/lib/ipfix_logging.h>
  31 #include <vnet/syslog/syslog.h>
  32 #include <nat/lib/nat_syslog_constants.h>
  33 #include <nat/lib/nat_syslog.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 #include <vlib/stats/stats.h>
  40
  41 snat_main_t snat_main;
  42
  43 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  44                                                            u32 sw_if_index);
  45
  46 #define skip_if_disabled()                                                    \
  47   do                                                                          \
  48     {                                                                         \
  49       snat_main_t *sm = &snat_main;                                           \
  50       if (PREDICT_FALSE (!sm->enabled))                                       \
  51         return;                                                               \
  52     }                                                                         \
  53   while (0)
  54
  55 #define fail_if_enabled()                                                     \
  56   do                                                                          \
  57     {                                                                         \
  58       snat_main_t *sm = &snat_main;                                           \
  59       if (PREDICT_FALSE (sm->enabled))                                        \
  60         {                                                                     \
  61           nat_log_err ("plugin enabled");                                     \
  62           return 1;                                                           \
  63         }                                                                     \
  64     }                                                                         \
  65   while (0)
  66
  67 #define fail_if_disabled()                                                    \
  68   do                                                                          \
  69     {                                                                         \
  70       snat_main_t *sm = &snat_main;                                           \
  71       if (PREDICT_FALSE (!sm->enabled))                                       \
  72         {                                                                     \
  73           nat_log_err ("plugin disabled");                                    \
  74           return 1;                                                           \
  75         }                                                                     \
  76     }                                                                         \
  77   while (0)
  78
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-ed-classify",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  96                                "ip4-sv-reassembly-feature"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-handoff-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 102                                "ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-worker-handoff",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 108 };
 109 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-worker-handoff",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-in2out",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-out2in",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 124                                "ip4-dhcp-client-detect"),
 125 };
 126 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 127   .arc_name = "ip4-output",
 128   .node_name = "nat-pre-in2out-output",
 129   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 130   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 136   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-ed-in2out-output",
 141   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 142   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 143 };
 144
 145 VLIB_PLUGIN_REGISTER () = {
 146     .version = VPP_BUILD_VER,
 147     .description = "Network Address Translation (NAT)",
 148 };
 149
 150 static void nat44_ed_db_init (u32 translations, u32 translation_buckets);
 151 static void nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm);
 152
 153 static int nat44_ed_add_static_mapping_internal (
 154   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 155   ip_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 156   ip4_address_t pool_addr, u8 *tag);
 157 static int nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
 158                                                  ip4_address_t e_addr,
 159                                                  u16 l_port, u16 e_port,
 160                                                  ip_protocol_t proto,
 161                                                  u32 vrf_id, u32 flags);
 162
 163 u32 nat_calc_bihash_buckets (u32 n_elts);
 164
 165 static_always_inline int
 166 nat44_ed_sm_i2o_add (snat_main_t *sm, snat_static_mapping_t *m,
 167                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 168 {
 169   ASSERT (!pool_is_free (sm->static_mappings, m));
 170   clib_bihash_kv_16_8_t kv;
 171   nat44_ed_sm_init_i2o_kv (&kv, addr.as_u32, port, fib_index, proto,
 172                            m - sm->static_mappings);
 173   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 174 }
 175
 176 static_always_inline int
 177 nat44_ed_sm_i2o_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 178                      u32 fib_index, u8 proto)
 179 {
 180   clib_bihash_kv_16_8_t kv;
 181   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 182   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 183 }
 184
 185 static_always_inline int
 186 nat44_ed_sm_o2i_add (snat_main_t *sm, snat_static_mapping_t *m,
 187                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 188 {
 189   ASSERT (!pool_is_free (sm->static_mappings, m));
 190   clib_bihash_kv_16_8_t kv;
 191   nat44_ed_sm_init_o2i_kv (&kv, addr.as_u32, port, fib_index, proto,
 192                            m - sm->static_mappings);
 193   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 194 }
 195
 196 static_always_inline int
 197 nat44_ed_sm_o2i_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 198                      u32 fib_index, u8 proto)
 199 {
 200   clib_bihash_kv_16_8_t kv;
 201   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 202   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 203 }
 204
 205 void
 206 nat44_ed_free_session_data (snat_main_t *sm, snat_session_t *s,
 207                             u32 thread_index, u8 is_ha)
 208 {
 209   per_vrf_sessions_unregister_session (s, thread_index);
 210
 211   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 212     nat_elog_warn (sm, "flow hash del failed");
 213
 214   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 215     nat_elog_warn (sm, "flow hash del failed");
 216
 217   if (na44_ed_is_fwd_bypass_session (s))
 218     {
 219       return;
 220     }
 221
 222   if (nat44_ed_is_affinity_session (s))
 223     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, s->proto,
 224                          s->out2in.port);
 225
 226   if (!is_ha)
 227     nat_syslog_nat44_sdel (0, s->in2out.fib_index, &s->in2out.addr,
 228                            s->in2out.port, &s->ext_host_nat_addr,
 229                            s->ext_host_nat_port, &s->out2in.addr,
 230                            s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 231                            s->proto, nat44_ed_is_twice_nat_session (s));
 232
 233   if (!is_ha)
 234     {
 235       /* log NAT event */
 236       nat_ipfix_logging_nat44_ses_delete (
 237         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 238         s->in2out.port, s->out2in.port, s->in2out.fib_index);
 239     }
 240 }
 241
 242 static ip_interface_address_t *
 243 nat44_ed_get_ip_interface_address (u32 sw_if_index, ip4_address_t addr)
 244 {
 245   snat_main_t *sm = &snat_main;
 246
 247   ip_lookup_main_t *lm = &sm->ip4_main->lookup_main;
 248   ip_interface_address_t *ia;
 249   ip4_address_t *ip4a;
 250
 251   foreach_ip_interface_address (
 252     lm, ia, sw_if_index, 1, ({
 253       ip4a = ip_interface_address_get_address (lm, ia);
 254       nat_log_debug ("sw_if_idx: %u addr: %U ? %U", sw_if_index,
 255                      format_ip4_address, ip4a, format_ip4_address, &addr);
 256       if (ip4a->as_u32 == addr.as_u32)
 257         {
 258           return ia;
 259         }
 260     }));
 261   return NULL;
 262 }
 263
 264 static int
 265 nat44_ed_resolve_nat_addr_len (snat_address_t *ap,
 266                                snat_interface_t *interfaces)
 267 {
 268   ip_interface_address_t *ia;
 269   snat_interface_t *i;
 270   u32 fib_index;
 271
 272   pool_foreach (i, interfaces)
 273     {
 274       if (!nat44_ed_is_interface_outside (i))
 275         {
 276           continue;
 277         }
 278
 279       fib_index = ip4_fib_table_get_index_for_sw_if_index (i->sw_if_index);
 280       if (fib_index != ap->fib_index)
 281         {
 282           continue;
 283         }
 284
 285       if ((ia = nat44_ed_get_ip_interface_address (i->sw_if_index, ap->addr)))
 286         {
 287           ap->addr_len = ia->address_length;
 288           ap->sw_if_index = i->sw_if_index;
 289           ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
 290                            << (32 - ap->addr_len);
 291
 292           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 293                          format_ip4_address, &ap->addr, ap->sw_if_index,
 294                          format_ip4_address, &ap->net, ap->addr_len);
 295           return 0;
 296         }
 297     }
 298   return 1;
 299 }
 300
 301 static void
 302 nat44_ed_update_outside_if_addresses (snat_address_t *ap)
 303 {
 304   snat_main_t *sm = &snat_main;
 305
 306   if (!nat44_ed_resolve_nat_addr_len (ap, sm->interfaces))
 307     {
 308       return;
 309     }
 310
 311   if (!nat44_ed_resolve_nat_addr_len (ap, sm->output_feature_interfaces))
 312     {
 313       return;
 314     }
 315 }
 316
 317 static void
 318 nat44_ed_bind_if_addr_to_nat_addr (u32 sw_if_index)
 319 {
 320   snat_main_t *sm = &snat_main;
 321   ip_interface_address_t *ia;
 322   snat_address_t *ap;
 323
 324   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 325
 326   vec_foreach (ap, sm->addresses)
 327     {
 328       if (fib_index != ap->fib_index)
 329         {
 330           continue;
 331         }
 332
 333       if ((ia = nat44_ed_get_ip_interface_address (sw_if_index, ap->addr)))
 334         {
 335           ap->addr_len = ia->address_length;
 336           ap->sw_if_index = sw_if_index;
 337           ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
 338                            << (32 - ap->addr_len);
 339
 340           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 341                          format_ip4_address, &ap->addr, ap->sw_if_index,
 342                          format_ip4_address, &ap->net, ap->addr_len);
 343           return;
 344         }
 345     }
 346 }
 347
 348 static_always_inline snat_fib_entry_reg_t *
 349 nat44_ed_get_fib_entry_reg (ip4_address_t addr, u32 sw_if_index, int *out_idx)
 350 {
 351   snat_main_t *sm = &snat_main;
 352   snat_fib_entry_reg_t *fe;
 353   int i;
 354
 355   for (i = 0; i < vec_len (sm->fib_entry_reg); i++)
 356     {
 357       fe = sm->fib_entry_reg + i;
 358       if ((addr.as_u32 == fe->addr.as_u32) && (sw_if_index == fe->sw_if_index))
 359         {
 360           if (out_idx)
 361             {
 362               *out_idx = i;
 363             }
 364           return fe;
 365         }
 366     }
 367   return NULL;
 368 }
 369
 370 static void
 371 nat44_ed_add_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 372 {
 373   // Add the external NAT address to the FIB as receive entries. This ensures
 374   // that VPP will reply to ARP for this address and we don't need to enable
 375   // proxy ARP on the outside interface.
 376   snat_main_t *sm = &snat_main;
 377   snat_fib_entry_reg_t *fe;
 378
 379   if (!(fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, 0)))
 380     {
 381       fib_prefix_t prefix = {
 382         .fp_len = 32,
 383         .fp_proto = FIB_PROTOCOL_IP4,
 384         .fp_addr = {
 385                   .ip4.as_u32 = addr.as_u32,
 386                 },
 387       };
 388       u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 389       fib_table_entry_update_one_path (fib_index, &prefix, sm->fib_src_low,
 390                                        (FIB_ENTRY_FLAG_CONNECTED |
 391                                         FIB_ENTRY_FLAG_LOCAL |
 392                                         FIB_ENTRY_FLAG_EXCLUSIVE),
 393                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
 394                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 395
 396       vec_add2 (sm->fib_entry_reg, fe, 1);
 397       clib_memset (fe, 0, sizeof (*fe));
 398       fe->addr.as_u32 = addr.as_u32;
 399       fe->sw_if_index = sw_if_index;
 400     }
 401   fe->count++;
 402 }
 403
 404 static void
 405 nat44_ed_del_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 406 {
 407   snat_main_t *sm = &snat_main;
 408   snat_fib_entry_reg_t *fe;
 409   int i;
 410
 411   if ((fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, &i)))
 412     {
 413       fe->count--;
 414       if (0 == fe->count)
 415         {
 416           fib_prefix_t prefix = {
 417             .fp_len = 32,
 418             .fp_proto = FIB_PROTOCOL_IP4,
 419             .fp_addr = {
 420               .ip4.as_u32 = addr.as_u32,
 421                     },
 422           };
 423           u32 fib_index =
 424             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 425           fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 426           vec_del1 (sm->fib_entry_reg, i);
 427         }
 428     }
 429 }
 430
 431 static void
 432 nat44_ed_add_del_interface_fib_reg_entries (ip4_address_t addr, u8 is_add)
 433 {
 434   snat_main_t *sm = &snat_main;
 435   snat_interface_t *i;
 436
 437   pool_foreach (i, sm->interfaces)
 438     {
 439       if (nat44_ed_is_interface_outside (i))
 440         {
 441           if (is_add)
 442             {
 443               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 444             }
 445           else
 446             {
 447               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 448             }
 449         }
 450     }
 451   pool_foreach (i, sm->output_feature_interfaces)
 452     {
 453       if (nat44_ed_is_interface_outside (i))
 454         {
 455           if (is_add)
 456             {
 457               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 458             }
 459           else
 460             {
 461               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 462             }
 463         }
 464     }
 465 }
 466
 467 static_always_inline void
 468 nat44_ed_add_del_nat_addr_fib_reg_entries (u32 sw_if_index, u8 is_add)
 469 {
 470   snat_main_t *sm = &snat_main;
 471   snat_address_t *ap;
 472
 473   vec_foreach (ap, sm->addresses)
 474     {
 475       if (is_add)
 476         {
 477           nat44_ed_add_fib_entry_reg (ap->addr, sw_if_index);
 478         }
 479       else
 480         {
 481           nat44_ed_del_fib_entry_reg (ap->addr, sw_if_index);
 482         }
 483     }
 484 }
 485
 486 static_always_inline void
 487 nat44_ed_add_del_sm_fib_reg_entries (u32 sw_if_index, u8 is_add)
 488 {
 489   snat_main_t *sm = &snat_main;
 490   snat_static_mapping_t *m;
 491
 492   pool_foreach (m, sm->static_mappings)
 493     {
 494       if (is_add)
 495         {
 496           nat44_ed_add_fib_entry_reg (m->external_addr, sw_if_index);
 497         }
 498       else
 499         {
 500           nat44_ed_del_fib_entry_reg (m->external_addr, sw_if_index);
 501         }
 502     }
 503 }
 504
 505 int
 506 nat44_ed_add_address (ip4_address_t *addr, u32 vrf_id, u8 twice_nat)
 507 {
 508   snat_main_t *sm = &snat_main;
 509   snat_address_t *ap, *addresses;
 510
 511   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 512
 513   if (!sm->enabled)
 514     {
 515       return VNET_API_ERROR_UNSUPPORTED;
 516     }
 517
 518   // check if address already exists
 519   vec_foreach (ap, addresses)
 520     {
 521       if (ap->addr.as_u32 == addr->as_u32)
 522         {
 523           nat_log_err ("address exist");
 524           return VNET_API_ERROR_VALUE_EXIST;
 525         }
 526     }
 527
 528   if (twice_nat)
 529     {
 530       vec_add2 (sm->twice_nat_addresses, ap, 1);
 531     }
 532   else
 533     {
 534       vec_add2 (sm->addresses, ap, 1);
 535     }
 536
 537   ap->addr_len = ~0;
 538   ap->fib_index = ~0;
 539   ap->addr = *addr;
 540
 541   if (vrf_id != ~0)
 542     {
 543       ap->fib_index = fib_table_find_or_create_and_lock (
 544         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 545     }
 546
 547   if (!twice_nat)
 548     {
 549       // if we don't have enabled interface we don't add address
 550       // to fib
 551       nat44_ed_add_del_interface_fib_reg_entries (*addr, 1);
 552       nat44_ed_update_outside_if_addresses (ap);
 553     }
 554   return 0;
 555 }
 556
 557 int
 558 nat44_ed_del_address (ip4_address_t addr, u8 twice_nat)
 559 {
 560   snat_main_t *sm = &snat_main;
 561   snat_address_t *a = 0, *addresses;
 562   snat_session_t *ses;
 563   u32 *ses_to_be_removed = 0, *ses_index;
 564   snat_main_per_thread_data_t *tsm;
 565   int j;
 566
 567   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 568
 569   for (j = 0; j < vec_len (addresses); j++)
 570     {
 571       if (addresses[j].addr.as_u32 == addr.as_u32)
 572         {
 573           a = addresses + j;
 574           break;
 575         }
 576     }
 577   if (!a)
 578     {
 579       nat_log_err ("no such address");
 580       return VNET_API_ERROR_NO_SUCH_ENTRY;
 581     }
 582
 583   // delete dynamic sessions only
 584   vec_foreach (tsm, sm->per_thread_data)
 585     {
 586       pool_foreach (ses, tsm->sessions)
 587         {
 588           if (ses->flags & SNAT_SESSION_FLAG_STATIC_MAPPING)
 589             {
 590               continue;
 591             }
 592           if (ses->out2in.addr.as_u32 == addr.as_u32)
 593             {
 594               nat44_ed_free_session_data (sm, ses, tsm - sm->per_thread_data,
 595                                           0);
 596               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
 597             }
 598         }
 599       vec_foreach (ses_index, ses_to_be_removed)
 600         {
 601           ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
 602           nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
 603         }
 604       vec_free (ses_to_be_removed);
 605     }
 606
 607   if (!twice_nat)
 608     {
 609       nat44_ed_add_del_interface_fib_reg_entries (addr, 0);
 610     }
 611
 612   if (a->fib_index != ~0)
 613     {
 614       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 615     }
 616
 617   if (!twice_nat)
 618     {
 619       vec_del1 (sm->addresses, j);
 620     }
 621   else
 622     {
 623       vec_del1 (sm->twice_nat_addresses, j);
 624     }
 625
 626   return 0;
 627 }
 628
 629 vrf_table_t *
 630 nat44_ed_get_vrf_table (u32 table_vrf_id)
 631 {
 632   snat_main_t *sm = &snat_main;
 633   vrf_table_t *t;
 634
 635   pool_foreach (t, sm->vrf_tables)
 636     {
 637       if (table_vrf_id == t->table_vrf_id)
 638         {
 639           return t;
 640         }
 641     }
 642   return NULL;
 643 }
 644
 645 vrf_route_t *
 646 nat44_ed_get_vrf_route (vrf_table_t *t, u32 vrf_id)
 647 {
 648   vrf_route_t *r;
 649
 650   pool_foreach (r, t->routes)
 651     {
 652       if (vrf_id == r->vrf_id)
 653         {
 654           return r;
 655         }
 656     }
 657   return NULL;
 658 }
 659
 660 int
 661 nat44_ed_add_del_vrf_table (u32 table_vrf_id, bool is_add)
 662 {
 663   snat_main_t *sm = &snat_main;
 664   vrf_table_t *t;
 665   vrf_route_t *r;
 666
 667   t = nat44_ed_get_vrf_table (table_vrf_id);
 668   if (t)
 669     {
 670       if (is_add)
 671         {
 672           return VNET_API_ERROR_VALUE_EXIST;
 673         }
 674       pool_foreach (r, t->routes)
 675         {
 676           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 677         }
 678       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 679
 680       pool_free (t->routes);
 681       pool_put (sm->vrf_tables, t);
 682     }
 683   else
 684     {
 685       if (!is_add)
 686         {
 687           return VNET_API_ERROR_NO_SUCH_ENTRY;
 688         }
 689       pool_get (sm->vrf_tables, t);
 690       clib_memset (t, 0, sizeof (*t));
 691       t->table_vrf_id = table_vrf_id;
 692       t->table_fib_index = fib_table_find_or_create_and_lock (
 693         FIB_PROTOCOL_IP4, table_vrf_id, sm->fib_src_low);
 694     }
 695
 696   return 0;
 697 }
 698
 699 void
 700 nat44_ed_del_vrf_tables ()
 701 {
 702   snat_main_t *sm = &snat_main;
 703   vrf_table_t *t;
 704   vrf_route_t *r;
 705
 706   pool_foreach (t, sm->vrf_tables)
 707     {
 708       pool_foreach (r, t->routes)
 709         {
 710           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 711         }
 712       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 713       pool_free (t->routes);
 714     }
 715   pool_free (sm->vrf_tables);
 716 }
 717
 718 int
 719 nat44_ed_add_del_vrf_route (u32 table_vrf_id, u32 vrf_id, bool is_add)
 720 {
 721   snat_main_t *sm = &snat_main;
 722   vrf_table_t *t;
 723   vrf_route_t *r;
 724
 725   t = nat44_ed_get_vrf_table (table_vrf_id);
 726   if (!t)
 727     {
 728       return VNET_API_ERROR_NO_SUCH_ENTRY;
 729     }
 730
 731   r = nat44_ed_get_vrf_route (t, vrf_id);
 732   if (r)
 733     {
 734       if (is_add)
 735         {
 736           return VNET_API_ERROR_VALUE_EXIST;
 737         }
 738       fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 739       pool_put (t->routes, r);
 740     }
 741   else
 742     {
 743       if (!is_add)
 744         {
 745           return VNET_API_ERROR_NO_SUCH_ENTRY;
 746         }
 747       pool_get (t->routes, r);
 748       clib_memset (r, 0, sizeof (*r));
 749       r->vrf_id = vrf_id;
 750       r->fib_index = fib_table_find_or_create_and_lock (
 751         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 752     }
 753
 754   return 0;
 755 }
 756
 757 u32
 758 get_thread_idx_by_port (u16 e_port)
 759 {
 760   snat_main_t *sm = &snat_main;
 761   u32 thread_idx = sm->num_workers;
 762   if (sm->num_workers > 1)
 763     {
 764       thread_idx =
 765         sm->first_worker_index +
 766         sm->workers[(e_port - 1024) / sm->port_per_thread];
 767     }
 768   return thread_idx;
 769 }
 770
 771 void
 772 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 773                                     snat_main_per_thread_data_t * tsm,
 774                                     ip4_address_t l_addr,
 775                                     u16 l_port,
 776                                     u8 protocol,
 777                                     u32 fib_index, int addr_only,
 778                                     ip4_address_t e_addr, u16 e_port)
 779 {
 780   snat_session_t *s;
 781   u32 *indexes_to_free = NULL;
 782   pool_foreach (s, tsm->sessions) {
 783     if (s->in2out.fib_index != fib_index ||
 784         s->in2out.addr.as_u32 != l_addr.as_u32)
 785       {
 786         continue;
 787       }
 788     if (!addr_only)
 789       {
 790         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 791             s->out2in.port != e_port || s->in2out.port != l_port ||
 792             s->proto != protocol)
 793           continue;
 794       }
 795
 796     if (nat44_ed_is_lb_session (s))
 797       continue;
 798     if (!nat44_ed_is_session_static (s))
 799       continue;
 800     nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 801     vec_add1 (indexes_to_free, s - tsm->sessions);
 802     if (!addr_only)
 803       break;
 804   }
 805   u32 *ses_index;
 806   vec_foreach (ses_index, indexes_to_free)
 807   {
 808     s = pool_elt_at_index (tsm->sessions, *ses_index);
 809     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 810   }
 811   vec_free (indexes_to_free);
 812 }
 813
 814 static_always_inline snat_static_mapping_t *
 815 nat44_ed_sm_lookup (snat_main_t *sm, clib_bihash_kv_16_8_t *kv)
 816 {
 817   clib_bihash_kv_16_8_t v;
 818   int rc = clib_bihash_search_16_8 (&sm->flow_hash, kv, &v);
 819   if (!rc)
 820     {
 821       ASSERT (0 == ed_value_get_thread_index (&v));
 822       return pool_elt_at_index (sm->static_mappings,
 823                                 ed_value_get_session_index (&v));
 824     }
 825   return NULL;
 826 }
 827
 828 snat_static_mapping_t *
 829 nat44_ed_sm_o2i_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 830                         u32 fib_index, u8 proto)
 831 {
 832   clib_bihash_kv_16_8_t kv;
 833   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 834   return nat44_ed_sm_lookup (sm, &kv);
 835 }
 836
 837 snat_static_mapping_t *
 838 nat44_ed_sm_i2o_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 839                         u32 fib_index, u8 proto)
 840 {
 841   clib_bihash_kv_16_8_t kv;
 842   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 843   return nat44_ed_sm_lookup (sm, &kv);
 844 }
 845
 846 static snat_static_mapping_resolve_t *
 847 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 848                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 849                              u32 flags, int *out_idx)
 850 {
 851   snat_static_mapping_resolve_t *rp;
 852   snat_main_t *sm = &snat_main;
 853   int i;
 854
 855   for (i = 0; i < vec_len (sm->sm_to_resolve); i++)
 856     {
 857       rp = sm->sm_to_resolve + i;
 858
 859       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 860         {
 861           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 862             {
 863               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 864                 {
 865                   if (rp->e_port != e_port || rp->proto != proto)
 866                     {
 867                       continue;
 868                     }
 869                 }
 870             }
 871           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 872             {
 873               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 874                 {
 875                   if (rp->l_port != l_port || rp->e_port != e_port ||
 876                       rp->proto != proto)
 877                     {
 878                       continue;
 879                     }
 880                 }
 881             }
 882           else
 883             {
 884               continue;
 885             }
 886           if (out_idx)
 887             {
 888               *out_idx = i;
 889             }
 890           return rp;
 891         }
 892     }
 893   return NULL;
 894 }
 895
 896 static int
 897 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 898                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 899                              u32 flags)
 900 {
 901   snat_main_t *sm = &snat_main;
 902   int i;
 903   if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 904                                    sw_if_index, flags, &i))
 905     {
 906       vec_del1 (sm->sm_to_resolve, i);
 907       return 0;
 908     }
 909   return 1;
 910 }
 911
 912 static_always_inline int
 913 nat44_ed_validate_sm_input (u32 flags)
 914 {
 915   // identity nat can be initiated only from inside interface
 916   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 917     {
 918       return VNET_API_ERROR_UNSUPPORTED;
 919     }
 920
 921   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 922     {
 923       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 924         {
 925           return VNET_API_ERROR_UNSUPPORTED;
 926         }
 927     }
 928   return 0;
 929 }
 930
 931 int
 932 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 933                              u16 l_port, u16 e_port, ip_protocol_t proto,
 934                              u32 vrf_id, u32 sw_if_index, u32 flags,
 935                              ip4_address_t pool_addr, u8 *tag)
 936 {
 937   snat_static_mapping_resolve_t *rp;
 938   snat_main_t *sm = &snat_main;
 939   int rv;
 940
 941   if (!sm->enabled)
 942     {
 943       return VNET_API_ERROR_UNSUPPORTED;
 944     }
 945
 946   rv = nat44_ed_validate_sm_input (flags);
 947   if (rv != 0)
 948     {
 949       return rv;
 950     }
 951
 952   // interface bound mapping
 953   if (is_sm_switch_address (flags))
 954     {
 955       if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 956                                        sw_if_index, flags, 0))
 957         {
 958           return VNET_API_ERROR_VALUE_EXIST;
 959         }
 960
 961       vec_add2 (sm->sm_to_resolve, rp, 1);
 962       rp->l_addr.as_u32 = l_addr.as_u32;
 963       rp->l_port = l_port;
 964       rp->e_port = e_port;
 965       rp->sw_if_index = sw_if_index;
 966       rp->vrf_id = vrf_id;
 967       rp->proto = proto;
 968       rp->flags = flags;
 969       rp->pool_addr = pool_addr;
 970       rp->tag = vec_dup (tag);
 971       rp->is_resolved = 0;
 972
 973       ip4_address_t *first_int_addr =
 974         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 975       if (!first_int_addr)
 976         {
 977           return 0;
 978         }
 979
 980       e_addr.as_u32 = first_int_addr->as_u32;
 981       rp->is_resolved = 1;
 982     }
 983
 984   rv = nat44_ed_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
 985                                              proto, vrf_id, sw_if_index, flags,
 986                                              pool_addr, tag);
 987   if ((0 != rv) && is_sm_switch_address (flags))
 988     {
 989       nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 990                                    sw_if_index, flags);
 991     }
 992
 993   return rv;
 994 }
 995
 996 int
 997 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 998                              u16 l_port, u16 e_port, ip_protocol_t proto,
 999                              u32 vrf_id, u32 sw_if_index, u32 flags)
1000 {
1001   snat_main_t *sm = &snat_main;
1002   int rv;
1003
1004   if (!sm->enabled)
1005     {
1006       return VNET_API_ERROR_UNSUPPORTED;
1007     }
1008
1009   rv = nat44_ed_validate_sm_input (flags);
1010   if (rv != 0)
1011     {
1012       return rv;
1013     }
1014
1015   // interface bound mapping
1016   if (is_sm_switch_address (flags))
1017     {
1018       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
1019                                        sw_if_index, flags))
1020         {
1021           return VNET_API_ERROR_NO_SUCH_ENTRY;
1022         }
1023
1024       ip4_address_t *first_int_addr =
1025         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
1026       if (!first_int_addr)
1027         {
1028           // dhcp resolution required
1029           return 0;
1030         }
1031
1032       e_addr.as_u32 = first_int_addr->as_u32;
1033     }
1034
1035   return nat44_ed_del_static_mapping_internal (l_addr, e_addr, l_port, e_port,
1036                                                proto, vrf_id, flags);
1037 }
1038
1039 static int
1040 nat44_ed_add_static_mapping_internal (ip4_address_t l_addr,
1041                                       ip4_address_t e_addr, u16 l_port,
1042                                       u16 e_port, ip_protocol_t proto,
1043                                       u32 vrf_id, u32 sw_if_index, u32 flags,
1044                                       ip4_address_t pool_addr, u8 *tag)
1045 {
1046   snat_main_t *sm = &snat_main;
1047   nat44_lb_addr_port_t *local;
1048   snat_static_mapping_t *m;
1049   u32 fib_index = ~0;
1050
1051   if (is_sm_addr_only (flags))
1052     {
1053       e_port = l_port = proto = 0;
1054     }
1055
1056   if (is_sm_identity_nat (flags))
1057     {
1058       l_port = e_port;
1059       l_addr.as_u32 = e_addr.as_u32;
1060     }
1061
1062   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1063   if (m)
1064     {
1065       // case:
1066       // adding local identity nat record for different vrf table
1067
1068       if (!is_sm_identity_nat (m->flags))
1069         {
1070           return VNET_API_ERROR_VALUE_EXIST;
1071         }
1072
1073       pool_foreach (local, m->locals)
1074         {
1075           if (local->vrf_id == vrf_id)
1076             {
1077               return VNET_API_ERROR_VALUE_EXIST;
1078             }
1079         }
1080
1081       pool_get (m->locals, local);
1082
1083       local->vrf_id = vrf_id;
1084       local->fib_index = fib_table_find_or_create_and_lock (
1085         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
1086
1087       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port,
1088                            local->fib_index, m->proto);
1089
1090       return 0;
1091     }
1092
1093   if (vrf_id != ~0)
1094     {
1095       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1096                                                      sm->fib_src_low);
1097     }
1098   else
1099     {
1100       // fallback to default vrf
1101       vrf_id = sm->inside_vrf_id;
1102       fib_index = sm->inside_fib_index;
1103       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1104     }
1105
1106   // test if local mapping record doesn't exist
1107   // identity nat supports multiple records in local mapping
1108   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
1109     {
1110       if (nat44_ed_sm_i2o_lookup (sm, l_addr, l_port, fib_index, proto))
1111         {
1112           return VNET_API_ERROR_VALUE_EXIST;
1113         }
1114     }
1115
1116   pool_get (sm->static_mappings, m);
1117   clib_memset (m, 0, sizeof (*m));
1118
1119   m->flags = flags;
1120   m->local_addr = l_addr;
1121   m->external_addr = e_addr;
1122
1123   m->pool_addr = pool_addr;
1124   m->tag = vec_dup (tag);
1125
1126   if (!is_sm_addr_only (flags))
1127     {
1128       m->local_port = l_port;
1129       m->external_port = e_port;
1130       m->proto = proto;
1131     }
1132
1133   if (is_sm_identity_nat (flags))
1134     {
1135       pool_get (m->locals, local);
1136
1137       local->vrf_id = vrf_id;
1138       local->fib_index = fib_index;
1139     }
1140   else
1141     {
1142       m->vrf_id = vrf_id;
1143       m->fib_index = fib_index;
1144     }
1145
1146   if (!is_sm_out2in_only (flags))
1147     {
1148       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port, fib_index,
1149                            m->proto);
1150     }
1151
1152   nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0, m->proto);
1153
1154   if (sm->num_workers > 1)
1155     {
1156       // store worker index for this record
1157       ip4_header_t ip = {
1158         .src_address = m->local_addr,
1159       };
1160       u32 worker_index;
1161       worker_index =
1162         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
1163       vec_add1 (m->workers, worker_index);
1164     }
1165
1166   nat44_ed_add_del_interface_fib_reg_entries (e_addr, 1);
1167
1168   return 0;
1169 }
1170
1171 static int
1172 nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
1173                                       ip4_address_t e_addr, u16 l_port,
1174                                       u16 e_port, ip_protocol_t proto,
1175                                       u32 vrf_id, u32 flags)
1176 {
1177   snat_main_per_thread_data_t *tsm;
1178   snat_main_t *sm = &snat_main;
1179
1180   nat44_lb_addr_port_t *local;
1181   snat_static_mapping_t *m;
1182   u32 fib_index = ~0;
1183
1184   if (is_sm_addr_only (flags))
1185     {
1186       e_port = l_port = proto = 0;
1187     }
1188
1189   if (is_sm_identity_nat (flags))
1190     {
1191       l_port = e_port;
1192       l_addr.as_u32 = e_addr.as_u32;
1193     }
1194
1195   // fib index 0
1196   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1197   if (!m)
1198     {
1199       return VNET_API_ERROR_NO_SUCH_ENTRY;
1200     }
1201
1202   if (is_sm_identity_nat (flags))
1203     {
1204       u8 found = 0;
1205
1206       if (vrf_id == ~0)
1207         {
1208           vrf_id = sm->inside_vrf_id;
1209         }
1210
1211       pool_foreach (local, m->locals)
1212         {
1213           if (local->vrf_id == vrf_id)
1214             {
1215               local = pool_elt_at_index (m->locals, local - m->locals);
1216               fib_index = local->fib_index;
1217               pool_put (m->locals, local);
1218               found = 1;
1219             }
1220         }
1221
1222       if (!found)
1223         {
1224           return VNET_API_ERROR_NO_SUCH_ENTRY;
1225         }
1226     }
1227   else
1228     {
1229       fib_index = m->fib_index;
1230     }
1231
1232   if (!is_sm_out2in_only (flags))
1233     {
1234       nat44_ed_sm_i2o_del (sm, l_addr, l_port, fib_index, proto);
1235     }
1236
1237   // delete sessions for static mapping
1238   if (sm->num_workers > 1)
1239     {
1240       tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1241     }
1242   else
1243     {
1244       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1245     }
1246
1247   nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr, m->local_port,
1248                                       m->proto, fib_index,
1249                                       is_sm_addr_only (flags), e_addr, e_port);
1250
1251   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1252
1253   if (!pool_elts (m->locals))
1254     {
1255       // this is last record remove all required stuff
1256       // fib_index 0
1257       nat44_ed_sm_o2i_del (sm, e_addr, e_port, 0, proto);
1258
1259       vec_free (m->tag);
1260       vec_free (m->workers);
1261       pool_put (sm->static_mappings, m);
1262
1263       nat44_ed_add_del_interface_fib_reg_entries (e_addr, 0);
1264     }
1265
1266   return 0;
1267 }
1268
1269 int
1270 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1271                                 ip_protocol_t proto,
1272                                 nat44_lb_addr_port_t *locals, u32 flags,
1273                                 u8 *tag, u32 affinity)
1274 {
1275   snat_main_t *sm = &snat_main;
1276   snat_static_mapping_t *m;
1277   snat_address_t *a = 0;
1278
1279   nat44_lb_addr_port_t *local;
1280   uword *bitmap = 0;
1281   int rc = 0;
1282
1283   int i;
1284
1285   if (!sm->enabled)
1286     {
1287       return VNET_API_ERROR_UNSUPPORTED;
1288     }
1289
1290   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1291
1292   if (m)
1293     {
1294       return VNET_API_ERROR_VALUE_EXIST;
1295     }
1296
1297   if (vec_len (locals) < 2)
1298     {
1299       return VNET_API_ERROR_INVALID_VALUE;
1300     }
1301
1302   if (!is_sm_out2in_only (flags))
1303     {
1304       /* Find external address in allocated addresses and reserve port for
1305          address and port pair mapping when dynamic translations enabled */
1306       for (i = 0; i < vec_len (sm->addresses); i++)
1307         {
1308           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1309             {
1310               /* External port must be unused */
1311               a = sm->addresses + i;
1312               if (nat44_ed_sm_o2i_lookup (sm, a->addr, e_port, 0, proto))
1313                 {
1314                   return VNET_API_ERROR_VALUE_EXIST;
1315                 }
1316               break;
1317             }
1318         }
1319       // external address must be allocated
1320       if (!a)
1321         {
1322           return VNET_API_ERROR_NO_SUCH_ENTRY;
1323         }
1324     }
1325
1326   pool_get (sm->static_mappings, m);
1327   clib_memset (m, 0, sizeof (*m));
1328   m->tag = vec_dup (tag);
1329   m->external_addr = e_addr;
1330   m->external_port = e_port;
1331   m->affinity = affinity;
1332   m->proto = proto;
1333
1334   m->flags = flags;
1335   m->flags |= NAT_SM_FLAG_LB;
1336
1337   if (affinity)
1338     m->affinity_per_service_list_head_index =
1339       nat_affinity_get_per_service_list_head_index ();
1340   else
1341     m->affinity_per_service_list_head_index = ~0;
1342
1343   if (nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0,
1344                            m->proto))
1345     {
1346       nat_log_err ("sm o2i key add failed");
1347       return VNET_API_ERROR_UNSPECIFIED;
1348     }
1349
1350   for (i = 0; i < vec_len (locals); i++)
1351     {
1352       locals[i].fib_index = fib_table_find_or_create_and_lock (
1353         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1354       if (!is_sm_out2in_only (flags))
1355         {
1356           if (nat44_ed_sm_o2i_add (sm, m, e_addr, e_port, 0, proto))
1357             {
1358               nat_log_err ("sm o2i key add failed");
1359               rc = VNET_API_ERROR_UNSPECIFIED;
1360               // here we continue with add operation so that it can be safely
1361               // reversed in delete path - otherwise we'd have to track what
1362               // we've done and deal with partial cleanups and since bihash
1363               // adds are (extremely improbable) the only points of failure,
1364               // it's easier to just do it this way
1365             }
1366         }
1367       locals[i].prefix = (i == 0) ?
1368                            locals[i].probability :
1369                            (locals[i - 1].prefix + locals[i].probability);
1370       pool_get (m->locals, local);
1371       *local = locals[i];
1372       if (sm->num_workers > 1)
1373         {
1374           ip4_header_t ip = {
1375             .src_address = locals[i].addr,
1376           };
1377           bitmap = clib_bitmap_set (
1378             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1379             1);
1380         }
1381     }
1382
1383   /* Assign workers */
1384   if (sm->num_workers > 1)
1385     {
1386       clib_bitmap_foreach (i, bitmap)
1387         {
1388           vec_add1 (m->workers, i);
1389         }
1390     }
1391
1392   return rc;
1393 }
1394
1395 int
1396 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1397                                 ip_protocol_t proto, u32 flags)
1398 {
1399   snat_main_t *sm = &snat_main;
1400   snat_static_mapping_t *m;
1401
1402   nat44_lb_addr_port_t *local;
1403   snat_main_per_thread_data_t *tsm;
1404   snat_session_t *s;
1405
1406   if (!sm->enabled)
1407     {
1408       return VNET_API_ERROR_UNSUPPORTED;
1409     }
1410
1411   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1412   if (!m)
1413     return VNET_API_ERROR_NO_SUCH_ENTRY;
1414
1415   if (!is_sm_lb (m->flags))
1416     return VNET_API_ERROR_INVALID_VALUE;
1417
1418   if (nat44_ed_sm_o2i_del (sm, m->external_addr, m->external_port, 0,
1419                            m->proto))
1420     {
1421       nat_log_err ("sm o2i key del failed");
1422       return VNET_API_ERROR_UNSPECIFIED;
1423     }
1424
1425   pool_foreach (local, m->locals)
1426     {
1427       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1428       if (!is_sm_out2in_only (flags))
1429         {
1430           if (nat44_ed_sm_i2o_del (sm, local->addr, local->port,
1431                                    local->fib_index, m->proto))
1432             {
1433               nat_log_err ("sm i2o key del failed");
1434               return VNET_API_ERROR_UNSPECIFIED;
1435             }
1436         }
1437
1438       if (sm->num_workers > 1)
1439         {
1440           ip4_header_t ip = {
1441             .src_address = local->addr,
1442           };
1443           tsm = vec_elt_at_index (
1444             sm->per_thread_data,
1445             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1446         }
1447       else
1448         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1449
1450       /* Delete sessions */
1451       pool_foreach (s, tsm->sessions)
1452         {
1453           if (!(nat44_ed_is_lb_session (s)))
1454             continue;
1455
1456           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1457               s->in2out.port != local->port)
1458             continue;
1459
1460           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1461           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1462         }
1463     }
1464
1465   if (m->affinity)
1466     {
1467       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1468     }
1469
1470   pool_free (m->locals);
1471   vec_free (m->tag);
1472   vec_free (m->workers);
1473   pool_put (sm->static_mappings, m);
1474
1475   return 0;
1476 }
1477
1478 int
1479 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1480                                           ip4_address_t l_addr, u16 l_port,
1481                                           ip_protocol_t proto, u32 vrf_id,
1482                                           u8 probability, u8 is_add)
1483 {
1484   snat_main_t *sm = &snat_main;
1485   snat_static_mapping_t *m = 0;
1486   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1487   snat_main_per_thread_data_t *tsm;
1488   snat_session_t *s;
1489   u32 *locals = 0;
1490   uword *bitmap = 0;
1491   int i;
1492
1493   if (!sm->enabled)
1494     {
1495       return VNET_API_ERROR_UNSUPPORTED;
1496     }
1497
1498   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1499
1500   if (!m)
1501     {
1502       return VNET_API_ERROR_NO_SUCH_ENTRY;
1503     }
1504
1505   if (!is_sm_lb (m->flags))
1506     {
1507       return VNET_API_ERROR_INVALID_VALUE;
1508     }
1509
1510   pool_foreach (local, m->locals)
1511    {
1512     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1513         (local->vrf_id == vrf_id))
1514       {
1515         match_local = local;
1516         break;
1517       }
1518   }
1519
1520   if (is_add)
1521     {
1522       if (match_local)
1523         {
1524           return VNET_API_ERROR_VALUE_EXIST;
1525         }
1526
1527       pool_get (m->locals, local);
1528       clib_memset (local, 0, sizeof (*local));
1529       local->addr.as_u32 = l_addr.as_u32;
1530       local->port = l_port;
1531       local->probability = probability;
1532       local->vrf_id = vrf_id;
1533       local->fib_index =
1534         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1535                                            sm->fib_src_low);
1536
1537       if (!is_sm_out2in_only (m->flags))
1538         {
1539           if (nat44_ed_sm_i2o_add (sm, m, l_addr, l_port, local->fib_index,
1540                                    proto))
1541             {
1542               nat_log_err ("sm i2o key add failed");
1543               pool_put (m->locals, local);
1544               return VNET_API_ERROR_UNSPECIFIED;
1545             }
1546         }
1547     }
1548   else
1549     {
1550       if (!match_local)
1551         return VNET_API_ERROR_NO_SUCH_ENTRY;
1552
1553       if (pool_elts (m->locals) < 3)
1554         return VNET_API_ERROR_UNSPECIFIED;
1555
1556       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1557                         sm->fib_src_low);
1558
1559       if (!is_sm_out2in_only (m->flags))
1560         {
1561           if (nat44_ed_sm_i2o_del (sm, l_addr, l_port, match_local->fib_index,
1562                                    proto))
1563             nat_log_err ("sm i2o key del failed");
1564         }
1565
1566       if (sm->num_workers > 1)
1567         {
1568           ip4_header_t ip = {
1569             .src_address = local->addr,
1570           };
1571           tsm = vec_elt_at_index (
1572             sm->per_thread_data,
1573             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1574         }
1575       else
1576         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1577
1578       /* Delete sessions */
1579       pool_foreach (s, tsm->sessions) {
1580           if (!(nat44_ed_is_lb_session (s)))
1581             continue;
1582
1583           if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1584               s->in2out.port != match_local->port)
1585             continue;
1586
1587           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1588           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1589       }
1590
1591       pool_put (m->locals, match_local);
1592     }
1593
1594   vec_free (m->workers);
1595
1596   pool_foreach (local, m->locals)
1597    {
1598     vec_add1 (locals, local - m->locals);
1599     if (sm->num_workers > 1)
1600       {
1601         ip4_header_t ip;
1602         ip.src_address.as_u32 = local->addr.as_u32,
1603         bitmap = clib_bitmap_set (
1604           bitmap,
1605           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1606       }
1607   }
1608
1609   ASSERT (vec_len (locals) > 1);
1610
1611   local = pool_elt_at_index (m->locals, locals[0]);
1612   local->prefix = local->probability;
1613   for (i = 1; i < vec_len (locals); i++)
1614     {
1615       local = pool_elt_at_index (m->locals, locals[i]);
1616       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1617       local->prefix = local->probability + prev_local->prefix;
1618     }
1619
1620   /* Assign workers */
1621   if (sm->num_workers > 1)
1622     {
1623       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1624     }
1625
1626   return 0;
1627 }
1628
1629 void
1630 expire_per_vrf_sessions (u32 fib_index)
1631 {
1632   per_vrf_sessions_t *per_vrf_sessions;
1633   snat_main_per_thread_data_t *tsm;
1634   snat_main_t *sm = &snat_main;
1635
1636   vec_foreach (tsm, sm->per_thread_data)
1637     {
1638       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1639         {
1640           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1641               (per_vrf_sessions->tx_fib_index == fib_index))
1642             {
1643               per_vrf_sessions->expired = 1;
1644             }
1645         }
1646     }
1647 }
1648
1649 void
1650 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1651 {
1652   snat_main_t *sm = &snat_main;
1653   nat_fib_t *fib;
1654
1655   // we don't care if it is outside/inside fib
1656   // we just care about their ref_count
1657   // if it reaches 0 sessions should expire
1658   // because the fib isn't valid for NAT anymore
1659
1660   vec_foreach (fib, sm->fibs)
1661   {
1662     if (fib->fib_index == fib_index)
1663       {
1664         if (is_del)
1665           {
1666             fib->ref_count--;
1667             if (!fib->ref_count)
1668               {
1669                 vec_del1 (sm->fibs, fib - sm->fibs);
1670                 expire_per_vrf_sessions (fib_index);
1671               }
1672             return;
1673           }
1674         else
1675           fib->ref_count++;
1676       }
1677   }
1678   if (!is_del)
1679     {
1680       vec_add2 (sm->fibs, fib, 1);
1681       fib->ref_count = 1;
1682       fib->fib_index = fib_index;
1683     }
1684 }
1685
1686 static_always_inline nat_fib_t *
1687 nat44_ed_get_outside_fib (nat_fib_t *outside_fibs, u32 fib_index)
1688 {
1689   nat_fib_t *f;
1690   vec_foreach (f, outside_fibs)
1691     {
1692       if (f->fib_index == fib_index)
1693         {
1694           return f;
1695         }
1696     }
1697   return 0;
1698 }
1699
1700 static_always_inline snat_interface_t *
1701 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1702 {
1703   snat_interface_t *i;
1704   pool_foreach (i, interfaces)
1705     {
1706       if (i->sw_if_index == sw_if_index)
1707         {
1708           return i;
1709         }
1710     }
1711   return 0;
1712 }
1713
1714 int
1715 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1716 {
1717   const char *del_feature_name, *feature_name;
1718   snat_main_t *sm = &snat_main;
1719
1720   nat_fib_t *outside_fib;
1721   snat_interface_t *i;
1722   u32 fib_index;
1723   int rv;
1724
1725   if (!sm->enabled)
1726     {
1727       nat_log_err ("nat44 is disabled");
1728       return VNET_API_ERROR_UNSUPPORTED;
1729     }
1730
1731   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1732     {
1733       nat_log_err ("error interface already configured");
1734       return VNET_API_ERROR_VALUE_EXIST;
1735     }
1736
1737   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1738   if (i)
1739     {
1740       if ((nat44_ed_is_interface_inside (i) && is_inside) ||
1741           (nat44_ed_is_interface_outside (i) && !is_inside))
1742         {
1743           return 0;
1744         }
1745       if (sm->num_workers > 1)
1746         {
1747           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1748                                           "nat44-out2in-worker-handoff";
1749           feature_name = "nat44-handoff-classify";
1750         }
1751       else
1752         {
1753           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1754
1755           feature_name = "nat44-ed-classify";
1756         }
1757
1758       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1759       if (rv)
1760         return rv;
1761       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1762                                    sw_if_index, 0, 0, 0);
1763       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1764                                    0, 0);
1765     }
1766   else
1767     {
1768       if (sm->num_workers > 1)
1769         {
1770           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1771                                      "nat44-out2in-worker-handoff";
1772         }
1773       else
1774         {
1775           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1776         }
1777
1778       nat_validate_interface_counters (sm, sw_if_index);
1779       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1780       if (rv)
1781         return rv;
1782       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1783                                    0, 0);
1784
1785       pool_get (sm->interfaces, i);
1786       i->sw_if_index = sw_if_index;
1787       i->flags = 0;
1788     }
1789
1790   fib_index =
1791     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1792
1793   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1794
1795   if (!is_inside)
1796     {
1797       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1798
1799       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1800       if (outside_fib)
1801         {
1802           outside_fib->ref_count++;
1803         }
1804       else
1805         {
1806           vec_add2 (sm->outside_fibs, outside_fib, 1);
1807           outside_fib->fib_index = fib_index;
1808           outside_fib->ref_count = 1;
1809         }
1810
1811       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
1812       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
1813
1814       nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
1815     }
1816   else
1817     {
1818       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1819     }
1820
1821   return 0;
1822 }
1823
1824 int
1825 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1826 {
1827   const char *del_feature_name, *feature_name;
1828   snat_main_t *sm = &snat_main;
1829
1830   nat_fib_t *outside_fib;
1831   snat_interface_t *i;
1832   u32 fib_index;
1833   int rv;
1834
1835   if (!sm->enabled)
1836     {
1837       nat_log_err ("nat44 is disabled");
1838       return VNET_API_ERROR_UNSUPPORTED;
1839     }
1840
1841   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1842   if (i == 0)
1843     {
1844       nat_log_err ("error interface couldn't be found");
1845       return VNET_API_ERROR_NO_SUCH_ENTRY;
1846     }
1847
1848   if (nat44_ed_is_interface_inside (i) && nat44_ed_is_interface_outside (i))
1849     {
1850       if (sm->num_workers > 1)
1851         {
1852           del_feature_name = "nat44-handoff-classify";
1853           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1854                                       "nat44-out2in-worker-handoff";
1855         }
1856       else
1857         {
1858           del_feature_name = "nat44-ed-classify";
1859           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1860         }
1861
1862       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1863       if (rv)
1864         {
1865           return rv;
1866         }
1867       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1868                                    sw_if_index, 0, 0, 0);
1869       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1870                                    0, 0);
1871
1872       if (is_inside)
1873         {
1874           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1875         }
1876       else
1877         {
1878           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1879         }
1880     }
1881   else
1882     {
1883       if (sm->num_workers > 1)
1884         {
1885           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1886                                      "nat44-out2in-worker-handoff";
1887         }
1888       else
1889         {
1890           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1891         }
1892
1893       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1894       if (rv)
1895         {
1896           return rv;
1897         }
1898       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1899                                    0, 0);
1900
1901       // remove interface
1902       pool_put (sm->interfaces, i);
1903     }
1904
1905   fib_index =
1906     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1907
1908   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
1909
1910   if (!is_inside)
1911     {
1912       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1913       if (outside_fib)
1914         {
1915           outside_fib->ref_count--;
1916           if (!outside_fib->ref_count)
1917             {
1918               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1919             }
1920         }
1921
1922       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
1923       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
1924     }
1925
1926   return 0;
1927 }
1928
1929 int
1930 nat44_ed_add_output_interface (u32 sw_if_index)
1931 {
1932   snat_main_t *sm = &snat_main;
1933
1934   nat_fib_t *outside_fib;
1935   snat_interface_t *i;
1936   u32 fib_index;
1937   int rv;
1938
1939   if (!sm->enabled)
1940     {
1941       nat_log_err ("nat44 is disabled");
1942       return VNET_API_ERROR_UNSUPPORTED;
1943     }
1944
1945   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1946     {
1947       nat_log_err ("error interface already configured");
1948       return VNET_API_ERROR_VALUE_EXIST;
1949     }
1950
1951   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1952     {
1953       nat_log_err ("error interface already configured");
1954       return VNET_API_ERROR_VALUE_EXIST;
1955     }
1956
1957   if (sm->num_workers > 1)
1958     {
1959       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1960       if (rv)
1961         {
1962           return rv;
1963         }
1964
1965       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1966       if (rv)
1967         {
1968           return rv;
1969         }
1970
1971       vnet_feature_enable_disable (
1972         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1973       vnet_feature_enable_disable ("ip4-output",
1974                                    "nat44-in2out-output-worker-handoff",
1975                                    sw_if_index, 1, 0, 0);
1976     }
1977   else
1978     {
1979       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1980       if (rv)
1981         {
1982           return rv;
1983         }
1984
1985       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1986       if (rv)
1987         {
1988           return rv;
1989         }
1990
1991       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1992                                    sw_if_index, 1, 0, 0);
1993       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1994                                    sw_if_index, 1, 0, 0);
1995     }
1996
1997   nat_validate_interface_counters (sm, sw_if_index);
1998
1999   pool_get (sm->output_feature_interfaces, i);
2000   i->sw_if_index = sw_if_index;
2001   i->flags = 0;
2002   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2003   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2004
2005   fib_index =
2006     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2007   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
2008
2009   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2010   if (outside_fib)
2011     {
2012       outside_fib->ref_count++;
2013     }
2014   else
2015     {
2016       vec_add2 (sm->outside_fibs, outside_fib, 1);
2017       outside_fib->fib_index = fib_index;
2018       outside_fib->ref_count = 1;
2019     }
2020
2021   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
2022   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
2023
2024   nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
2025
2026   return 0;
2027 }
2028
2029 int
2030 nat44_ed_del_output_interface (u32 sw_if_index)
2031 {
2032   snat_main_t *sm = &snat_main;
2033
2034   nat_fib_t *outside_fib;
2035   snat_interface_t *i;
2036   u32 fib_index;
2037   int rv;
2038
2039   if (!sm->enabled)
2040     {
2041       nat_log_err ("nat44 is disabled");
2042       return VNET_API_ERROR_UNSUPPORTED;
2043     }
2044
2045   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2046   if (!i)
2047     {
2048       nat_log_err ("error interface couldn't be found");
2049       return VNET_API_ERROR_NO_SUCH_ENTRY;
2050     }
2051
2052   if (sm->num_workers > 1)
2053     {
2054       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2055       if (rv)
2056         {
2057           return rv;
2058         }
2059
2060       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2061       if (rv)
2062         {
2063           return rv;
2064         }
2065
2066       vnet_feature_enable_disable (
2067         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
2068       vnet_feature_enable_disable ("ip4-output",
2069                                    "nat44-in2out-output-worker-handoff",
2070                                    sw_if_index, 0, 0, 0);
2071     }
2072   else
2073     {
2074       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2075       if (rv)
2076         {
2077           return rv;
2078         }
2079
2080       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2081       if (rv)
2082         {
2083           return rv;
2084         }
2085
2086       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2087                                    sw_if_index, 0, 0, 0);
2088       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2089                                    sw_if_index, 0, 0, 0);
2090     }
2091
2092   // remove interface
2093   pool_put (sm->output_feature_interfaces, i);
2094
2095   fib_index =
2096     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2097   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
2098
2099   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2100   if (outside_fib)
2101     {
2102       outside_fib->ref_count--;
2103       if (!outside_fib->ref_count)
2104         {
2105           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2106         }
2107     }
2108
2109   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
2110   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
2111
2112   return 0;
2113 }
2114
2115 int
2116 snat_set_workers (uword * bitmap)
2117 {
2118   snat_main_t *sm = &snat_main;
2119   int i, j = 0;
2120
2121   if (sm->num_workers < 2)
2122     return VNET_API_ERROR_FEATURE_DISABLED;
2123
2124   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2125     return VNET_API_ERROR_INVALID_WORKER;
2126
2127   vec_free (sm->workers);
2128   clib_bitmap_foreach (i, bitmap)
2129     {
2130       vec_add1(sm->workers, i);
2131       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2132       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2133       j++;
2134     }
2135
2136   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2137
2138   return 0;
2139 }
2140
2141 int
2142 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2143 {
2144   fail_if_enabled ();
2145   snat_main_t *sm = &snat_main;
2146
2147   if ((sm->fq_in2out_index != ~0) || (sm->fq_out2in_index != ~0) ||
2148       (sm->fq_in2out_output_index != ~0))
2149     {
2150       // frame queu nelts can be set only before first
2151       // call to nat44_plugin_enable after that it
2152       // doesn't make sense
2153       nat_log_err ("Frame queue was already initialized. "
2154                    "Change is not possible");
2155       return 1;
2156     }
2157
2158   sm->frame_queue_nelts = frame_queue_nelts;
2159   return 0;
2160 }
2161
2162 static void
2163 nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque, u32 sw_if_index,
2164                                 u32 new_fib_index, u32 old_fib_index)
2165 {
2166   snat_main_t *sm = &snat_main;
2167   nat_fib_t *outside_fib;
2168   snat_interface_t *i;
2169   u8 is_add = 1;
2170   u8 match = 0;
2171
2172   if (!sm->enabled || (new_fib_index == old_fib_index)
2173       || (!vec_len (sm->outside_fibs)))
2174     {
2175       return;
2176     }
2177
2178   pool_foreach (i, sm->interfaces)
2179     {
2180       if (i->sw_if_index == sw_if_index)
2181         {
2182           if (!(nat44_ed_is_interface_outside (i)))
2183             return;
2184           match = 1;
2185         }
2186     }
2187
2188   pool_foreach (i, sm->output_feature_interfaces)
2189     {
2190       if (i->sw_if_index == sw_if_index)
2191         {
2192           if (!(nat44_ed_is_interface_outside (i)))
2193             return;
2194           match = 1;
2195         }
2196     }
2197
2198   if (!match)
2199     return;
2200
2201   vec_foreach (outside_fib, sm->outside_fibs)
2202     {
2203       if (outside_fib->fib_index == old_fib_index)
2204         {
2205           outside_fib->ref_count--;
2206           if (!outside_fib->ref_count)
2207             vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2208           break;
2209         }
2210     }
2211
2212   vec_foreach (outside_fib, sm->outside_fibs)
2213     {
2214       if (outside_fib->fib_index == new_fib_index)
2215         {
2216           outside_fib->ref_count++;
2217           is_add = 0;
2218           break;
2219         }
2220     }
2221
2222   if (is_add)
2223     {
2224       vec_add2 (sm->outside_fibs, outside_fib, 1);
2225       outside_fib->ref_count = 1;
2226       outside_fib->fib_index = new_fib_index;
2227     }
2228 }
2229
2230 static void nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque,
2231                                             u32 sw_if_index, u32 new_fib_index,
2232                                             u32 old_fib_index);
2233
2234 static void nat44_ed_add_del_interface_address_cb (
2235   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2236   u32 address_length, u32 if_address_index, u32 is_delete);
2237
2238 static void nat44_ed_add_del_static_mapping_cb (
2239   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2240   u32 address_length, u32 if_address_index, u32 is_delete);
2241
2242 void
2243 test_key_calc_split ()
2244 {
2245   ip4_address_t l_addr;
2246   l_addr.as_u8[0] = 1;
2247   l_addr.as_u8[1] = 1;
2248   l_addr.as_u8[2] = 1;
2249   l_addr.as_u8[3] = 1;
2250   ip4_address_t r_addr;
2251   r_addr.as_u8[0] = 2;
2252   r_addr.as_u8[1] = 2;
2253   r_addr.as_u8[2] = 2;
2254   r_addr.as_u8[3] = 2;
2255   u16 l_port = 40001;
2256   u16 r_port = 40301;
2257   u8 proto = 9;
2258   u32 fib_index = 9000001;
2259   u32 thread_index = 3000000001;
2260   u32 session_index = 3000000221;
2261   clib_bihash_kv_16_8_t kv;
2262   init_ed_kv (&kv, l_addr.as_u32, l_port, r_addr.as_u32, r_port, fib_index,
2263               proto, thread_index, session_index);
2264   ip4_address_t l_addr2;
2265   ip4_address_t r_addr2;
2266   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2267   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2268   u16 l_port2 = 0;
2269   u16 r_port2 = 0;
2270   u8 proto2 = 0;
2271   u32 fib_index2 = 0;
2272   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2273                &r_port2);
2274   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2275   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2276   ASSERT (l_port == l_port2);
2277   ASSERT (r_port == r_port2);
2278   ASSERT (proto == proto2);
2279   ASSERT (fib_index == fib_index2);
2280   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2281   ASSERT (session_index == ed_value_get_session_index (&kv));
2282 }
2283
2284 static clib_error_t *
2285 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2286 {
2287   u32 fib_index;
2288   if (!is_add)
2289     {
2290       fib_index = ip4_fib_index_from_table_id (table_id);
2291       if (fib_index != ~0)
2292         {
2293           expire_per_vrf_sessions (fib_index);
2294         }
2295     }
2296   return 0;
2297 }
2298
2299 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2300
2301 #define nat_validate_simple_counter(c, i)                                     \
2302   do                                                                          \
2303     {                                                                         \
2304       vlib_validate_simple_counter (&c, i);                                   \
2305       vlib_zero_simple_counter (&c, i);                                       \
2306     }                                                                         \
2307   while (0);
2308
2309 #define nat_init_simple_counter(c, n, sn)                                     \
2310   do                                                                          \
2311     {                                                                         \
2312       c.name = n;                                                             \
2313       c.stat_segment_name = sn;                                               \
2314       nat_validate_simple_counter (c, 0);                                     \
2315     }                                                                         \
2316   while (0);
2317
2318 static_always_inline void
2319 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2320 {
2321 #define _(x)                                                                  \
2322   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2323   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2324   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2325   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2326   foreach_nat_counter;
2327 #undef _
2328   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2329 }
2330
2331 static clib_error_t *
2332 nat_init (vlib_main_t * vm)
2333 {
2334   snat_main_t *sm = &snat_main;
2335   vlib_thread_main_t *tm = vlib_get_thread_main ();
2336   vlib_thread_registration_t *tr;
2337   ip4_add_del_interface_address_callback_t cbi = { 0 };
2338   ip4_table_bind_callback_t cbt = { 0 };
2339   u32 i, num_threads = 0;
2340   uword *p, *bitmap = 0;
2341
2342   clib_memset (sm, 0, sizeof (*sm));
2343
2344   // required
2345   sm->vnet_main = vnet_get_main ();
2346   // convenience
2347   sm->ip4_main = &ip4_main;
2348
2349   // frame queue indices used for handoff
2350   sm->fq_out2in_index = ~0;
2351   sm->fq_in2out_index = ~0;
2352   sm->fq_in2out_output_index = ~0;
2353
2354   sm->log_level = NAT_LOG_ERROR;
2355
2356   sm->log_class = vlib_log_register_class ("nat", 0);
2357   nat_ipfix_logging_init (vm);
2358
2359   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2360                            "/nat44-ed/total-sessions");
2361   sm->max_cfg_sessions_gauge =
2362     vlib_stats_add_gauge ("/nat44-ed/max-cfg-sessions");
2363
2364 #define _(x)                                                                  \
2365   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2366                            "/nat44-ed/in2out/fastpath/" #x);                  \
2367   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2368                            "/nat44-ed/out2in/fastpath/" #x);                  \
2369   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2370                            "/nat44-ed/in2out/slowpath/" #x);                  \
2371   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2372                            "/nat44-ed/out2in/slowpath/" #x);
2373   foreach_nat_counter;
2374 #undef _
2375   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2376                            "/nat44-ed/hairpinning");
2377
2378   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2379   if (p)
2380     {
2381       tr = (vlib_thread_registration_t *) p[0];
2382       if (tr)
2383         {
2384           sm->num_workers = tr->count;
2385           sm->first_worker_index = tr->first_index;
2386         }
2387     }
2388   num_threads = tm->n_vlib_mains - 1;
2389   sm->port_per_thread = 0xffff - 1024;
2390   vec_validate (sm->per_thread_data, num_threads);
2391
2392   /* Use all available workers by default */
2393   if (sm->num_workers > 1)
2394     {
2395       for (i = 0; i < sm->num_workers; i++)
2396         bitmap = clib_bitmap_set (bitmap, i, 1);
2397       snat_set_workers (bitmap);
2398       clib_bitmap_free (bitmap);
2399     }
2400   else
2401     {
2402       sm->per_thread_data[0].snat_thread_index = 0;
2403     }
2404
2405   /* callbacks to call when interface address changes. */
2406   cbi.function = nat44_ed_add_del_interface_address_cb;
2407   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2408   cbi.function = nat44_ed_add_del_static_mapping_cb;
2409   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2410
2411   /* callbacks to call when interface to table biding changes */
2412   cbt.function = nat44_ed_update_outside_fib_cb;
2413   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2414
2415   sm->fib_src_low =
2416     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2417                          FIB_SOURCE_BH_SIMPLE);
2418   sm->fib_src_hi =
2419     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2420                          FIB_SOURCE_BH_SIMPLE);
2421
2422   nat_affinity_init (vm);
2423   test_key_calc_split ();
2424
2425   return nat44_api_hookup (vm);
2426 }
2427
2428 VLIB_INIT_FUNCTION (nat_init);
2429
2430 int
2431 nat44_plugin_enable (nat44_config_t c)
2432 {
2433   snat_main_t *sm = &snat_main;
2434
2435   fail_if_enabled ();
2436
2437   sm->forwarding_enabled = 0;
2438   sm->mss_clamping = 0;
2439
2440   if (!c.sessions)
2441     c.sessions = 63 * 1024;
2442
2443   sm->max_translations_per_thread = c.sessions;
2444   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
2445                         sm->max_translations_per_thread);
2446   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2447
2448   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2449
2450   sm->inside_vrf_id = c.inside_vrf;
2451   sm->inside_fib_index =
2452     fib_table_find_or_create_and_lock
2453     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2454
2455   sm->outside_vrf_id = c.outside_vrf;
2456   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2457     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2458
2459   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets);
2460
2461   nat44_ed_init_tcp_state_stable (sm);
2462
2463   nat_affinity_enable ();
2464
2465   nat_reset_timeouts (&sm->timeouts);
2466
2467   vlib_zero_simple_counter (&sm->total_sessions, 0);
2468
2469   if (!sm->frame_queue_nelts)
2470     {
2471       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2472     }
2473
2474   if (sm->num_workers > 1)
2475     {
2476       vlib_main_t *vm = vlib_get_main ();
2477       vlib_node_t *node;
2478
2479       if (sm->fq_in2out_index == ~0)
2480         {
2481           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2482           sm->fq_in2out_index =
2483             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2484         }
2485       if (sm->fq_out2in_index == ~0)
2486         {
2487           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2488           sm->fq_out2in_index =
2489             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2490         }
2491       if (sm->fq_in2out_output_index == ~0)
2492         {
2493           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2494           sm->fq_in2out_output_index =
2495             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2496         }
2497     }
2498
2499   sm->enabled = 1;
2500   sm->rconfig = c;
2501
2502   return 0;
2503 }
2504
2505 int
2506 nat44_ed_del_addresses ()
2507 {
2508   snat_main_t *sm = &snat_main;
2509   snat_address_t *a, *vec;
2510   int error = 0;
2511
2512   vec = vec_dup (sm->addresses);
2513   vec_foreach (a, vec)
2514     {
2515       error = nat44_ed_del_address (a->addr, 0);
2516       if (error)
2517         {
2518           nat_log_err ("error occurred while removing adderess");
2519         }
2520     }
2521   vec_free (vec);
2522   vec_free (sm->addresses);
2523   sm->addresses = 0;
2524
2525   vec = vec_dup (sm->twice_nat_addresses);
2526   vec_foreach (a, vec)
2527     {
2528       error = nat44_ed_del_address (a->addr, 1);
2529       if (error)
2530         {
2531           nat_log_err ("error occurred while removing adderess");
2532         }
2533     }
2534   vec_free (vec);
2535   vec_free (sm->twice_nat_addresses);
2536   sm->twice_nat_addresses = 0;
2537
2538   vec_free (sm->addr_to_resolve);
2539   sm->addr_to_resolve = 0;
2540
2541   return error;
2542 }
2543
2544 int
2545 nat44_ed_del_interfaces ()
2546 {
2547   snat_main_t *sm = &snat_main;
2548   snat_interface_t *i, *pool;
2549   int error = 0;
2550
2551   pool = pool_dup (sm->interfaces);
2552   pool_foreach (i, pool)
2553     {
2554       if (nat44_ed_is_interface_inside (i))
2555         {
2556           error = nat44_ed_del_interface (i->sw_if_index, 1);
2557         }
2558       if (nat44_ed_is_interface_outside (i))
2559         {
2560           error = nat44_ed_del_interface (i->sw_if_index, 0);
2561         }
2562
2563       if (error)
2564         {
2565           nat_log_err ("error occurred while removing interface");
2566         }
2567     }
2568   pool_free (pool);
2569   pool_free (sm->interfaces);
2570   sm->interfaces = 0;
2571   return error;
2572 }
2573
2574 int
2575 nat44_ed_del_output_interfaces ()
2576 {
2577   snat_main_t *sm = &snat_main;
2578   snat_interface_t *i, *pool;
2579   int error = 0;
2580
2581   pool = pool_dup (sm->output_feature_interfaces);
2582   pool_foreach (i, pool)
2583     {
2584       error = nat44_ed_del_output_interface (i->sw_if_index);
2585       if (error)
2586         {
2587           nat_log_err ("error occurred while removing output interface");
2588         }
2589     }
2590   pool_free (pool);
2591   pool_free (sm->output_feature_interfaces);
2592   sm->output_feature_interfaces = 0;
2593   return error;
2594 }
2595
2596 int
2597 nat44_ed_del_static_mappings ()
2598 {
2599   snat_main_t *sm = &snat_main;
2600   snat_static_mapping_t *m, *pool;
2601   int error = 0;
2602
2603   pool = pool_dup (sm->static_mappings);
2604   pool_foreach (m, pool)
2605     {
2606       error = nat44_ed_del_static_mapping_internal (
2607         m->local_addr, m->external_addr, m->local_port, m->external_port,
2608         m->proto, m->vrf_id, m->flags);
2609       if (error)
2610         {
2611           nat_log_err ("error occurred while removing mapping");
2612         }
2613     }
2614   pool_free (pool);
2615   pool_free (sm->static_mappings);
2616   sm->static_mappings = 0;
2617
2618   vec_free (sm->sm_to_resolve);
2619   sm->sm_to_resolve = 0;
2620
2621   return error;
2622 }
2623
2624 int
2625 nat44_plugin_disable ()
2626 {
2627   snat_main_per_thread_data_t *tsm;
2628   snat_main_t *sm = &snat_main;
2629   int rc, error = 0;
2630
2631   fail_if_disabled ();
2632
2633   rc = nat44_ed_del_static_mappings ();
2634   if (rc)
2635     error = 1;
2636
2637   rc = nat44_ed_del_addresses ();
2638   if (rc)
2639     error = 1;
2640
2641   rc = nat44_ed_del_interfaces ();
2642   if (rc)
2643     error = 1;
2644
2645   rc = nat44_ed_del_output_interfaces ();
2646   if (rc)
2647     error = 1;
2648
2649   nat44_ed_del_vrf_tables ();
2650
2651   vec_free (sm->max_translations_per_fib);
2652   sm->max_translations_per_fib = 0;
2653
2654   clib_bihash_free_16_8 (&sm->flow_hash);
2655
2656   vec_foreach (tsm, sm->per_thread_data)
2657     {
2658       nat44_ed_worker_db_free (tsm);
2659     }
2660
2661   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2662
2663   nat_affinity_disable ();
2664
2665   sm->forwarding_enabled = 0;
2666   sm->enabled = 0;
2667
2668   return error;
2669 }
2670
2671 void
2672 nat44_ed_forwarding_enable_disable (u8 is_enable)
2673 {
2674   snat_main_per_thread_data_t *tsm;
2675   snat_main_t *sm = &snat_main;
2676   snat_session_t *s;
2677
2678   u32 *ses_to_be_removed = 0, *ses_index;
2679
2680   sm->forwarding_enabled = is_enable != 0;
2681
2682   if (!sm->enabled || is_enable)
2683     {
2684       return;
2685     }
2686
2687   vec_foreach (tsm, sm->per_thread_data)
2688     {
2689       pool_foreach (s, tsm->sessions)
2690         {
2691           if (na44_ed_is_fwd_bypass_session (s))
2692             {
2693               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2694             }
2695         }
2696       vec_foreach (ses_index, ses_to_be_removed)
2697         {
2698           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2699           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2700           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2701         }
2702
2703       vec_free (ses_to_be_removed);
2704     }
2705 }
2706
2707 static_always_inline snat_static_mapping_t *
2708 nat44_ed_sm_match (snat_main_t *sm, ip4_address_t match_addr, u16 match_port,
2709                    u32 match_fib_index, ip_protocol_t match_protocol,
2710                    int by_external)
2711 {
2712   snat_static_mapping_t *m;
2713   if (!by_external)
2714     {
2715       m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port, match_fib_index,
2716                                   match_protocol);
2717       if (m)
2718         return m;
2719
2720       // try address only mapping
2721       m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, match_fib_index, 0);
2722       if (m)
2723         return m;
2724
2725       // default static mapping fib index (based on configuration)
2726       if (sm->inside_fib_index != match_fib_index)
2727         {
2728           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2729                                       sm->inside_fib_index, match_protocol);
2730           if (m)
2731             return m;
2732
2733           // try address only mapping
2734           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->inside_fib_index,
2735                                       0);
2736           if (m)
2737             return m;
2738         }
2739       // TODO: this specific use case may be deprecated (needs testing)
2740       if (sm->outside_fib_index != match_fib_index)
2741         {
2742           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2743                                       sm->outside_fib_index, match_protocol);
2744           if (m)
2745             return m;
2746
2747           // try address only mapping
2748           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->outside_fib_index,
2749                                       0);
2750           if (m)
2751             return m;
2752         }
2753     }
2754   else
2755     {
2756       m =
2757         nat44_ed_sm_o2i_lookup (sm, match_addr, match_port, 0, match_protocol);
2758       if (m)
2759         return m;
2760
2761       // try address only mapping
2762       m = nat44_ed_sm_o2i_lookup (sm, match_addr, 0, 0, 0);
2763       if (m)
2764         return m;
2765     }
2766   return 0;
2767 }
2768
2769 int
2770 snat_static_mapping_match (vlib_main_t *vm, ip4_address_t match_addr,
2771                            u16 match_port, u32 match_fib_index,
2772                            ip_protocol_t match_protocol,
2773                            ip4_address_t *mapping_addr, u16 *mapping_port,
2774                            u32 *mapping_fib_index, int by_external,
2775                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2776                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2777                            u8 *is_identity_nat, snat_static_mapping_t **out)
2778 {
2779   snat_main_t *sm = &snat_main;
2780   snat_static_mapping_t *m;
2781   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2782   nat44_lb_addr_port_t *local;
2783   u8 backend_index;
2784
2785   m = nat44_ed_sm_match (sm, match_addr, match_port, match_fib_index,
2786                          match_protocol, by_external);
2787   if (!m)
2788     {
2789       return 1;
2790     }
2791
2792   if (by_external)
2793     {
2794       if (is_sm_lb (m->flags))
2795         {
2796           if (PREDICT_FALSE (lb != 0))
2797             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2798           if (m->affinity && !nat_affinity_find_and_lock (
2799                                vm, ext_host_addr[0], match_addr,
2800                                match_protocol, match_port, &backend_index))
2801             {
2802               local = pool_elt_at_index (m->locals, backend_index);
2803               *mapping_addr = local->addr;
2804               *mapping_port = local->port;
2805               *mapping_fib_index = local->fib_index;
2806               goto end;
2807             }
2808           // pick locals matching this worker
2809           if (PREDICT_FALSE (sm->num_workers > 1))
2810             {
2811               u32 thread_index = vlib_get_thread_index ();
2812               pool_foreach_index (i, m->locals)
2813                {
2814                 local = pool_elt_at_index (m->locals, i);
2815
2816                 ip4_header_t ip = {
2817                   .src_address = local->addr,
2818                 };
2819
2820                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2821                                                       0) == thread_index)
2822                   {
2823                     vec_add1 (tmp, i);
2824                   }
2825                }
2826               ASSERT (vec_len (tmp) != 0);
2827             }
2828           else
2829             {
2830               pool_foreach_index (i, m->locals)
2831                {
2832                 vec_add1 (tmp, i);
2833               }
2834             }
2835           hi = vec_len (tmp) - 1;
2836           local = pool_elt_at_index (m->locals, tmp[hi]);
2837           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2838           while (lo < hi)
2839             {
2840               mid = ((hi - lo) >> 1) + lo;
2841               local = pool_elt_at_index (m->locals, tmp[mid]);
2842               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2843             }
2844           local = pool_elt_at_index (m->locals, tmp[lo]);
2845           if (!(local->prefix >= rand))
2846             return 1;
2847           *mapping_addr = local->addr;
2848           *mapping_port = local->port;
2849           *mapping_fib_index = local->fib_index;
2850           if (m->affinity)
2851             {
2852               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2853                                                 match_protocol, match_port,
2854                                                 tmp[lo], m->affinity,
2855                                                 m->affinity_per_service_list_head_index))
2856                 nat_elog_info (sm, "create affinity record failed");
2857             }
2858           vec_free (tmp);
2859         }
2860       else
2861         {
2862           if (PREDICT_FALSE (lb != 0))
2863             *lb = NO_LB_NAT;
2864           *mapping_fib_index = m->fib_index;
2865           *mapping_addr = m->local_addr;
2866           /* Address only mapping doesn't change port */
2867           *mapping_port =
2868             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2869         }
2870     }
2871   else
2872     {
2873       *mapping_addr = m->external_addr;
2874       /* Address only mapping doesn't change port */
2875       *mapping_port =
2876         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2877       *mapping_fib_index = sm->outside_fib_index;
2878     }
2879
2880 end:
2881   if (PREDICT_FALSE (is_addr_only != 0))
2882     *is_addr_only = is_sm_addr_only (m->flags);
2883
2884   if (PREDICT_FALSE (twice_nat != 0))
2885     {
2886       *twice_nat = TWICE_NAT_DISABLED;
2887
2888       if (is_sm_twice_nat (m->flags))
2889         {
2890           *twice_nat = TWICE_NAT;
2891         }
2892       else if (is_sm_self_twice_nat (m->flags))
2893         {
2894           *twice_nat = TWICE_NAT_SELF;
2895         }
2896     }
2897
2898   if (PREDICT_FALSE (is_identity_nat != 0))
2899     *is_identity_nat = is_sm_identity_nat (m->flags);
2900
2901   if (out != 0)
2902     *out = m;
2903
2904   return 0;
2905 }
2906
2907 u32
2908 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2909                                   u32 rx_fib_index, u8 is_output)
2910 {
2911   snat_main_t *sm = &snat_main;
2912   u32 next_worker_index = sm->first_worker_index;
2913   u32 hash;
2914
2915   clib_bihash_kv_16_8_t kv16, value16;
2916
2917   u32 fib_index = rx_fib_index;
2918   if (b)
2919     {
2920       if (PREDICT_FALSE (is_output))
2921         {
2922           fib_index = sm->outside_fib_index;
2923           nat_fib_t *outside_fib;
2924           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2925           fib_prefix_t pfx = {
2926                   .fp_proto = FIB_PROTOCOL_IP4,
2927                   .fp_len = 32,
2928                   .fp_addr = {
2929                           .ip4.as_u32 = ip->dst_address.as_u32,
2930                   } ,
2931           };
2932
2933           switch (vec_len (sm->outside_fibs))
2934             {
2935             case 0:
2936               fib_index = sm->outside_fib_index;
2937               break;
2938             case 1:
2939               fib_index = sm->outside_fibs[0].fib_index;
2940               break;
2941             default:
2942               vec_foreach (outside_fib, sm->outside_fibs)
2943                 {
2944                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2945                   if (FIB_NODE_INDEX_INVALID != fei)
2946                     {
2947                       if (fib_entry_get_resolving_interface (fei) != ~0)
2948                         {
2949                           fib_index = outside_fib->fib_index;
2950                           break;
2951                         }
2952                     }
2953                 }
2954               break;
2955             }
2956         }
2957
2958       if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2959         {
2960           ip4_address_t lookup_saddr, lookup_daddr;
2961           u16 lookup_sport, lookup_dport;
2962           u8 lookup_protocol;
2963
2964           if (!nat_get_icmp_session_lookup_values (
2965                 b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr,
2966                 &lookup_dport, &lookup_protocol))
2967             {
2968               init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
2969                          lookup_daddr.as_u32, lookup_dport, rx_fib_index,
2970                          lookup_protocol);
2971               if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2972                 {
2973                   next_worker_index = ed_value_get_thread_index (&value16);
2974                   vnet_buffer2 (b)->nat.cached_session_index =
2975                     ed_value_get_session_index (&value16);
2976                   goto out;
2977                 }
2978             }
2979         }
2980
2981       init_ed_k (&kv16, ip->src_address.as_u32,
2982                  vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
2983                  vnet_buffer (b)->ip.reass.l4_dst_port, fib_index,
2984                  ip->protocol);
2985
2986       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2987         {
2988           next_worker_index = ed_value_get_thread_index (&value16);
2989           vnet_buffer2 (b)->nat.cached_session_index =
2990             ed_value_get_session_index (&value16);
2991           goto out;
2992         }
2993
2994       // dst NAT
2995       init_ed_k (&kv16, ip->dst_address.as_u32,
2996                  vnet_buffer (b)->ip.reass.l4_dst_port, ip->src_address.as_u32,
2997                  vnet_buffer (b)->ip.reass.l4_src_port, rx_fib_index,
2998                  ip->protocol);
2999       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3000         {
3001           next_worker_index = ed_value_get_thread_index (&value16);
3002           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
3003             ed_value_get_session_index (&value16);
3004           goto out;
3005         }
3006     }
3007
3008   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3009     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3010
3011   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3012     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3013   else
3014     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3015
3016 out:
3017   if (PREDICT_TRUE (!is_output))
3018     {
3019       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
3020                               rx_fib_index,
3021                               clib_net_to_host_u32 (ip->src_address.as_u32),
3022                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3023     }
3024   else
3025     {
3026       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
3027                               next_worker_index, rx_fib_index,
3028                               clib_net_to_host_u32 (ip->src_address.as_u32),
3029                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3030     }
3031
3032   return next_worker_index;
3033 }
3034
3035 u32
3036 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
3037                                   u32 rx_fib_index, u8 is_output)
3038 {
3039   snat_main_t *sm = &snat_main;
3040   clib_bihash_kv_16_8_t kv16, value16;
3041
3042   u8 proto, next_worker_index = 0;
3043   u16 port;
3044   snat_static_mapping_t *m;
3045   u32 hash;
3046
3047   proto = ip->protocol;
3048
3049   if (PREDICT_FALSE (IP_PROTOCOL_ICMP == proto))
3050     {
3051       ip4_address_t lookup_saddr, lookup_daddr;
3052       u16 lookup_sport, lookup_dport;
3053       u8 lookup_protocol;
3054       if (!nat_get_icmp_session_lookup_values (
3055             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3056             &lookup_protocol))
3057         {
3058           init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3059                      lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3060                      lookup_protocol);
3061           if (PREDICT_TRUE (
3062                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3063             {
3064               next_worker_index = ed_value_get_thread_index (&value16);
3065               nat_elog_debug_handoff (
3066                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
3067                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
3068                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3069               return next_worker_index;
3070             }
3071         }
3072     }
3073
3074   init_ed_k (&kv16, ip->src_address.as_u32,
3075              vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3076              vnet_buffer (b)->ip.reass.l4_dst_port, rx_fib_index,
3077              ip->protocol);
3078
3079   if (PREDICT_TRUE (
3080         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3081     {
3082       vnet_buffer2 (b)->nat.cached_session_index =
3083         ed_value_get_session_index (&value16);
3084       next_worker_index = ed_value_get_thread_index (&value16);
3085       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
3086                               next_worker_index, rx_fib_index,
3087                               clib_net_to_host_u32 (ip->src_address.as_u32),
3088                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3089       return next_worker_index;
3090     }
3091
3092   /* first try static mappings without port */
3093   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3094     {
3095       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, proto);
3096       if (m)
3097         {
3098           {
3099             next_worker_index = m->workers[0];
3100             goto done;
3101           }
3102         }
3103     }
3104
3105   /* unknown protocol */
3106   if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto)))
3107     {
3108       /* use current thread */
3109       next_worker_index = vlib_get_thread_index ();
3110       goto done;
3111     }
3112
3113   port = vnet_buffer (b)->ip.reass.l4_dst_port;
3114
3115   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3116     {
3117       udp_header_t *udp = ip4_next_header (ip);
3118       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3119       nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3120       if (!icmp_type_is_error_message
3121           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3122         port = vnet_buffer (b)->ip.reass.l4_src_port;
3123       else
3124         {
3125           /* if error message, then it's not fragmented and we can access it */
3126           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3127           proto = inner_ip->protocol;
3128           void *l4_header = ip4_next_header (inner_ip);
3129           switch (proto)
3130             {
3131             case IP_PROTOCOL_ICMP:
3132               icmp = (icmp46_header_t *) l4_header;
3133               echo = (nat_icmp_echo_header_t *) (icmp + 1);
3134               port = echo->identifier;
3135               break;
3136             case IP_PROTOCOL_UDP:
3137               /* breakthrough */
3138             case IP_PROTOCOL_TCP:
3139               port = ((nat_tcp_udp_header_t *) l4_header)->src_port;
3140               break;
3141             default:
3142               next_worker_index = vlib_get_thread_index ();
3143               goto done;
3144             }
3145         }
3146     }
3147
3148   /* try static mappings with port */
3149   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3150     {
3151       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, port, 0, proto);
3152       if (m)
3153         {
3154           if (!is_sm_lb (m->flags))
3155             {
3156               next_worker_index = m->workers[0];
3157               goto done;
3158             }
3159
3160           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3161             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3162
3163           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3164             next_worker_index =
3165               m->workers[hash & (_vec_len (m->workers) - 1)];
3166           else
3167             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3168           goto done;
3169         }
3170     }
3171
3172   /* worker by outside port */
3173   next_worker_index = sm->first_worker_index;
3174   next_worker_index +=
3175     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3176
3177 done:
3178   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3179                           rx_fib_index,
3180                           clib_net_to_host_u32 (ip->src_address.as_u32),
3181                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3182   return next_worker_index;
3183 }
3184
3185 u32
3186 nat44_get_max_session_limit ()
3187 {
3188   snat_main_t *sm = &snat_main;
3189   u32 max_limit = 0, len = 0;
3190
3191   for (; len < vec_len (sm->max_translations_per_fib); len++)
3192     {
3193       if (max_limit < sm->max_translations_per_fib[len])
3194         max_limit = sm->max_translations_per_fib[len];
3195     }
3196   return max_limit;
3197 }
3198
3199 int
3200 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3201 {
3202   snat_main_t *sm = &snat_main;
3203   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3204   u32 len = vec_len (sm->max_translations_per_fib);
3205
3206   if (len <= fib_index)
3207     {
3208       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3209
3210       for (; len < vec_len (sm->max_translations_per_fib); len++)
3211         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3212     }
3213
3214   sm->max_translations_per_fib[fib_index] = session_limit;
3215   return 0;
3216 }
3217
3218 int
3219 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3220 {
3221   snat_main_t *sm = &snat_main;
3222
3223   if (nat44_set_session_limit (session_limit, vrf_id))
3224     return 1;
3225   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3226
3227   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
3228                         sm->max_translations_per_thread);
3229
3230   sm->translation_buckets =
3231     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3232
3233   nat44_ed_sessions_clear ();
3234   return 0;
3235 }
3236
3237 static void
3238 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3239                          u32 translation_buckets)
3240 {
3241   dlist_elt_t *head;
3242
3243   pool_alloc (tsm->sessions, translations);
3244   pool_alloc (tsm->lru_pool, translations);
3245
3246   pool_get (tsm->lru_pool, head);
3247   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3248   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3249
3250   pool_get (tsm->lru_pool, head);
3251   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3252   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3253
3254   pool_get (tsm->lru_pool, head);
3255   tsm->udp_lru_head_index = head - tsm->lru_pool;
3256   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3257
3258   pool_get (tsm->lru_pool, head);
3259   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3260   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3261
3262   pool_get (tsm->lru_pool, head);
3263   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3264   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3265 }
3266
3267 static void
3268 reinit_ed_flow_hash ()
3269 {
3270   snat_main_t *sm = &snat_main;
3271   // we expect 2 flows per session, so multiply translation_buckets by 2
3272   clib_bihash_init_16_8 (
3273     &sm->flow_hash, "ed-flow-hash",
3274     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3275   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3276 }
3277
3278 static void
3279 nat44_ed_db_init (u32 translations, u32 translation_buckets)
3280 {
3281   snat_main_t *sm = &snat_main;
3282   snat_main_per_thread_data_t *tsm;
3283
3284   reinit_ed_flow_hash ();
3285
3286   vec_foreach (tsm, sm->per_thread_data)
3287     {
3288       nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3289                                sm->translation_buckets);
3290     }
3291 }
3292
3293 static void
3294 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3295 {
3296   pool_free (tsm->lru_pool);
3297   pool_free (tsm->sessions);
3298   vec_free (tsm->per_vrf_sessions_vec);
3299 }
3300
3301 void
3302 nat44_ed_sessions_clear ()
3303 {
3304   snat_main_t *sm = &snat_main;
3305   snat_main_per_thread_data_t *tsm;
3306
3307   reinit_ed_flow_hash ();
3308
3309   vec_foreach (tsm, sm->per_thread_data)
3310     {
3311       nat44_ed_worker_db_free (tsm);
3312       nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3313                                sm->translation_buckets);
3314     }
3315   vlib_zero_simple_counter (&sm->total_sessions, 0);
3316 }
3317
3318 static void
3319 nat44_ed_add_del_static_mapping_cb (ip4_main_t *im, uword opaque,
3320                                     u32 sw_if_index, ip4_address_t *address,
3321                                     u32 address_length, u32 if_address_index,
3322                                     u32 is_delete)
3323 {
3324   snat_static_mapping_resolve_t *rp;
3325   snat_main_t *sm = &snat_main;
3326   int rv = 0;
3327
3328   if (!sm->enabled)
3329     {
3330       return;
3331     }
3332
3333   vec_foreach (rp, sm->sm_to_resolve)
3334     {
3335       if (sw_if_index == rp->sw_if_index)
3336         {
3337           if (is_delete)
3338             {
3339               if (rp->is_resolved)
3340                 {
3341                   rv = nat44_ed_del_static_mapping_internal (
3342                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3343                     rp->vrf_id, rp->flags);
3344                   if (rv)
3345                     {
3346                       nat_log_err ("ed del static mapping failed");
3347                     }
3348                   else
3349                     {
3350                       rp->is_resolved = 0;
3351                     }
3352                 }
3353             }
3354           else
3355             {
3356               if (!rp->is_resolved)
3357                 {
3358                   rv = nat44_ed_add_static_mapping_internal (
3359                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3360                     rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3361                   if (rv)
3362                     {
3363                       nat_log_err ("ed add static mapping failed");
3364                     }
3365                   else
3366                     {
3367                       rp->is_resolved = 1;
3368                     }
3369                 }
3370             }
3371         }
3372     }
3373 }
3374
3375 static int
3376 nat44_ed_get_addr_resolve_record (u32 sw_if_index, u8 twice_nat, int *out)
3377 {
3378   snat_main_t *sm = &snat_main;
3379   snat_address_resolve_t *rp;
3380   int i;
3381
3382   for (i = 0; i < vec_len (sm->addr_to_resolve); i++)
3383     {
3384       rp = sm->addr_to_resolve + i;
3385
3386       if ((rp->sw_if_index == sw_if_index) && (rp->is_twice_nat == twice_nat))
3387         {
3388           if (out)
3389             {
3390               *out = i;
3391             }
3392           return 0;
3393         }
3394     }
3395   return 1;
3396 }
3397 static int
3398 nat44_ed_del_addr_resolve_record (u32 sw_if_index, u8 twice_nat)
3399 {
3400   snat_main_t *sm = &snat_main;
3401   int i;
3402   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3403     {
3404       vec_del1 (sm->addr_to_resolve, i);
3405       return 0;
3406     }
3407   return 1;
3408 }
3409
3410 static void
3411 nat44_ed_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3412                                        u32 sw_if_index, ip4_address_t *address,
3413                                        u32 address_length,
3414                                        u32 if_address_index, u32 is_delete)
3415 {
3416   snat_main_t *sm = &snat_main;
3417   snat_address_resolve_t *arp;
3418   snat_address_t *ap;
3419   u8 twice_nat = 0;
3420   int i, rv;
3421
3422   if (!sm->enabled)
3423     {
3424       return;
3425     }
3426
3427   if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3428     {
3429       twice_nat = 1;
3430       if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3431         {
3432           u32 fib_index =
3433             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
3434           vec_foreach (ap, sm->addresses)
3435             {
3436               if ((fib_index == ap->fib_index) &&
3437                   (address->as_u32 == ap->addr.as_u32))
3438                 {
3439                   if (!is_delete)
3440                     {
3441                       ap->addr_len = address_length;
3442                       ap->sw_if_index = sw_if_index;
3443                       ap->net.as_u32 = (ap->addr.as_u32 >> (32 - ap->addr_len))
3444                                        << (32 - ap->addr_len);
3445
3446                       nat_log_debug (
3447                         "pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
3448                         format_ip4_address, &ap->addr, ap->sw_if_index,
3449                         format_ip4_address, &ap->net, ap->addr_len);
3450                     }
3451                   else
3452                     {
3453                       ap->addr_len = ~0;
3454                     }
3455                   break;
3456                 }
3457             }
3458           return;
3459         }
3460     }
3461
3462   arp = sm->addr_to_resolve + i;
3463
3464   if (!is_delete)
3465     {
3466       if (arp->is_resolved)
3467         {
3468           return;
3469         }
3470
3471       rv = nat44_ed_add_address (address, ~0, arp->is_twice_nat);
3472       if (0 == rv)
3473         {
3474           arp->is_resolved = 1;
3475         }
3476     }
3477   else
3478     {
3479       if (!arp->is_resolved)
3480         {
3481           return;
3482         }
3483
3484       rv = nat44_ed_del_address (address[0], arp->is_twice_nat);
3485       if (0 == rv)
3486         {
3487           arp->is_resolved = 0;
3488         }
3489     }
3490 }
3491
3492 int
3493 nat44_ed_add_interface_address (u32 sw_if_index, u8 twice_nat)
3494 {
3495   snat_main_t *sm = &snat_main;
3496   ip4_main_t *ip4_main = sm->ip4_main;
3497   ip4_address_t *first_int_addr;
3498   snat_address_resolve_t *ap;
3499   int rv;
3500
3501   if (!sm->enabled)
3502     {
3503       return VNET_API_ERROR_UNSUPPORTED;
3504     }
3505
3506   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, 0))
3507     {
3508       return VNET_API_ERROR_VALUE_EXIST;
3509     }
3510
3511   vec_add2 (sm->addr_to_resolve, ap, 1);
3512   ap->sw_if_index = sw_if_index;
3513   ap->is_twice_nat = twice_nat;
3514   ap->is_resolved = 0;
3515
3516   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3517   if (first_int_addr)
3518     {
3519       rv = nat44_ed_add_address (first_int_addr, ~0, twice_nat);
3520       if (0 != rv)
3521         {
3522           nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat);
3523           return rv;
3524         }
3525       ap->is_resolved = 1;
3526     }
3527
3528   return 0;
3529 }
3530
3531 int
3532 nat44_ed_del_interface_address (u32 sw_if_index, u8 twice_nat)
3533 {
3534   snat_main_t *sm = &snat_main;
3535   ip4_main_t *ip4_main = sm->ip4_main;
3536   ip4_address_t *first_int_addr;
3537
3538   if (!sm->enabled)
3539     {
3540       return VNET_API_ERROR_UNSUPPORTED;
3541     }
3542
3543   if (nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat))
3544     {
3545       return VNET_API_ERROR_NO_SUCH_ENTRY;
3546     }
3547
3548   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3549   if (first_int_addr)
3550     {
3551       return nat44_ed_del_address (first_int_addr[0], twice_nat);
3552     }
3553
3554   return 0;
3555 }
3556
3557 int
3558 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3559                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3560                       u32 vrf_id, int is_in)
3561 {
3562   ip4_header_t ip;
3563   clib_bihash_kv_16_8_t kv, value;
3564   u32 fib_index;
3565   snat_session_t *s;
3566   snat_main_per_thread_data_t *tsm;
3567
3568   if (!sm->enabled)
3569     {
3570       return VNET_API_ERROR_UNSUPPORTED;
3571     }
3572
3573   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3574   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3575   if (sm->num_workers > 1)
3576     tsm = vec_elt_at_index (
3577       sm->per_thread_data,
3578       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3579   else
3580     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3581
3582   init_ed_k (&kv, addr->as_u32, port, eh_addr->as_u32, eh_port, fib_index,
3583              proto);
3584   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3585     {
3586       return VNET_API_ERROR_NO_SUCH_ENTRY;
3587     }
3588
3589   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3590     return VNET_API_ERROR_UNSPECIFIED;
3591   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3592   nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3593   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3594   return 0;
3595 }
3596
3597 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3598                                  vlib_node_runtime_t * node,
3599                                  vlib_frame_t * frame)
3600 {
3601   return 0;
3602 }
3603
3604 VLIB_REGISTER_NODE (nat_default_node) = {
3605   .name = "nat-default",
3606   .vector_size = sizeof (u32),
3607   .format_trace = 0,
3608   .type = VLIB_NODE_TYPE_INTERNAL,
3609   .n_errors = 0,
3610   .n_next_nodes = NAT_N_NEXT,
3611   .next_nodes = {
3612     [NAT_NEXT_DROP] = "error-drop",
3613     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3614     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3615     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3616     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3617     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3618     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3619     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3620     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3621     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3622   },
3623 };
3624
3625 void
3626 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3627 {
3628   f->l3_csum_delta = 0;
3629   f->l4_csum_delta = 0;
3630   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3631       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3632     {
3633       f->l3_csum_delta =
3634         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3635       f->l3_csum_delta =
3636         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3637     }
3638   else
3639     {
3640       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3641     }
3642   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3643       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3644     {
3645       f->l3_csum_delta =
3646         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3647       f->l3_csum_delta =
3648         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3649     }
3650   else
3651     {
3652       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3653     }
3654   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3655     {
3656       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3657       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3658     }
3659   else
3660     {
3661       f->rewrite.sport = f->match.sport;
3662     }
3663   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3664     {
3665       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3666       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3667     }
3668   else
3669     {
3670       f->rewrite.dport = f->match.dport;
3671     }
3672   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3673       f->rewrite.icmp_id != f->match.sport)
3674     {
3675       f->l4_csum_delta =
3676         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3677       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3678     }
3679   else
3680     {
3681       f->rewrite.icmp_id = f->match.sport;
3682     }
3683   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3684     {
3685     }
3686   else
3687     {
3688       f->rewrite.fib_index = f->match.fib_index;
3689     }
3690 }
3691
3692 static_always_inline int
3693 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3694                             ip4_header_t *ip, nat_6t_flow_t *f);
3695
3696 static_always_inline void
3697 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3698                            nat_6t_flow_t *f, ip_protocol_t proto,
3699                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3700 {
3701   udp_header_t *udp = ip4_next_header (ip);
3702   tcp_header_t *tcp = (tcp_header_t *) udp;
3703
3704   if ((IP_PROTOCOL_TCP == proto || IP_PROTOCOL_UDP == proto) &&
3705       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3706     {
3707       if (!is_icmp_inner_ip4)
3708         { // regular case
3709           ip->src_address = f->rewrite.saddr;
3710           ip->dst_address = f->rewrite.daddr;
3711           udp->src_port = f->rewrite.sport;
3712           udp->dst_port = f->rewrite.dport;
3713         }
3714       else
3715         { // icmp inner ip4 - reversed saddr/daddr
3716           ip->src_address = f->rewrite.daddr;
3717           ip->dst_address = f->rewrite.saddr;
3718           udp->src_port = f->rewrite.dport;
3719           udp->dst_port = f->rewrite.sport;
3720         }
3721
3722       if (IP_PROTOCOL_TCP == proto)
3723         {
3724           ip_csum_t tcp_sum = tcp->checksum;
3725           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3726           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3727           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3728           tcp->checksum = ip_csum_fold (tcp_sum);
3729         }
3730       else if (IP_PROTOCOL_UDP == proto && udp->checksum)
3731         {
3732           ip_csum_t udp_sum = udp->checksum;
3733           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3734           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3735           udp->checksum = ip_csum_fold (udp_sum);
3736         }
3737     }
3738   else
3739     {
3740       if (!is_icmp_inner_ip4)
3741         { // regular case
3742           if (!skip_saddr_rewrite)
3743             {
3744               ip->src_address = f->rewrite.saddr;
3745             }
3746           ip->dst_address = f->rewrite.daddr;
3747         }
3748       else
3749         { // icmp inner ip4 - reversed saddr/daddr
3750           ip->src_address = f->rewrite.daddr;
3751           ip->dst_address = f->rewrite.saddr;
3752         }
3753     }
3754
3755   if (skip_saddr_rewrite)
3756     {
3757       ip->checksum = ip4_header_checksum (ip);
3758     }
3759   else
3760     {
3761       ip_csum_t ip_sum = ip->checksum;
3762       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3763       ip->checksum = ip_csum_fold (ip_sum);
3764     }
3765   if (0xffff == ip->checksum)
3766     ip->checksum = 0;
3767   ASSERT (ip4_header_checksum_is_valid (ip));
3768 }
3769
3770 static_always_inline int
3771 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3772 {
3773   int result = ((u8 *) object + size <=
3774                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3775                vlib_object_within_buffer_data (vm, b, object, size);
3776   return result;
3777 }
3778
3779 static_always_inline int
3780 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3781                             ip4_header_t *ip, nat_6t_flow_t *f)
3782 {
3783   if (IP_PROTOCOL_ICMP != ip->protocol)
3784     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3785
3786   icmp46_header_t *icmp = ip4_next_header (ip);
3787   nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3788
3789   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3790     {
3791       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3792         {
3793           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3794         }
3795
3796       if (!icmp_type_is_error_message (icmp->type))
3797         {
3798           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3799               (f->rewrite.icmp_id != echo->identifier))
3800             {
3801               ip_csum_t sum = icmp->checksum;
3802               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3803                                     nat_icmp_echo_header_t,
3804                                     identifier /* changed member */);
3805               echo->identifier = f->rewrite.icmp_id;
3806               icmp->checksum = ip_csum_fold (sum);
3807             }
3808         }
3809       else
3810         {
3811           ip_csum_t sum = ip_incremental_checksum (
3812             0, icmp,
3813             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3814           sum = (u16) ~ip_csum_fold (sum);
3815           if (sum != 0)
3816             {
3817               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3818             }
3819
3820           // errors are not fragmented
3821           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3822
3823           if (!ip4_header_checksum_is_valid (inner_ip))
3824             {
3825               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3826             }
3827
3828           ip_protocol_t inner_proto = inner_ip->protocol;
3829
3830           ip_csum_t old_icmp_sum = icmp->checksum;
3831           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3832           ip_csum_t old_udp_sum;
3833           ip_csum_t old_tcp_sum;
3834           ip_csum_t new_icmp_sum;
3835           udp_header_t *udp;
3836           tcp_header_t *tcp;
3837
3838           switch (inner_proto)
3839             {
3840             case IP_PROTOCOL_UDP:
3841               udp = (udp_header_t *) (inner_ip + 1);
3842               if (!it_fits (vm, b, udp, sizeof (*udp)))
3843                 {
3844                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3845                 }
3846               old_udp_sum = udp->checksum;
3847               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3848                                          1 /* is_icmp_inner_ip4 */,
3849                                          0 /* skip_saddr_rewrite */);
3850               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3851               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3852               new_icmp_sum =
3853                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3854                                 inner_ip->checksum, ip4_header_t, checksum);
3855               new_icmp_sum =
3856                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3857                                 udp_header_t, checksum);
3858               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3859               icmp->checksum = new_icmp_sum;
3860               break;
3861             case IP_PROTOCOL_TCP:
3862               tcp = (tcp_header_t *) (inner_ip + 1);
3863               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3864                 {
3865                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3866                 }
3867               old_tcp_sum = tcp->checksum;
3868               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3869                                          1 /* is_icmp_inner_ip4 */,
3870                                          0 /* skip_saddr_rewrite */);
3871               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3872               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3873               new_icmp_sum =
3874                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3875                                 inner_ip->checksum, ip4_header_t, checksum);
3876               new_icmp_sum =
3877                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3878                                 tcp_header_t, checksum);
3879               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3880               icmp->checksum = new_icmp_sum;
3881               break;
3882             case IP_PROTOCOL_ICMP:
3883               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3884                                          1 /* is_icmp_inner_ip4 */,
3885                                          0 /* skip_saddr_rewrite */);
3886               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3887                 {
3888                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3889                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3890                     {
3891                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3892                     }
3893                   nat_icmp_echo_header_t *inner_echo =
3894                     (nat_icmp_echo_header_t *) (inner_icmp + 1);
3895                   if (f->rewrite.icmp_id != inner_echo->identifier)
3896                     {
3897                       ip_csum_t sum = icmp->checksum;
3898                       sum = ip_csum_update (sum, inner_echo->identifier,
3899                                             f->rewrite.icmp_id,
3900                                             nat_icmp_echo_header_t,
3901                                             identifier /* changed member */);
3902                       icmp->checksum = ip_csum_fold (sum);
3903                       ip_csum_t inner_sum = inner_icmp->checksum;
3904                       inner_sum = ip_csum_update (
3905                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3906                         nat_icmp_echo_header_t,
3907                         identifier /* changed member */);
3908                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3909                       inner_echo->identifier = f->rewrite.icmp_id;
3910                     }
3911                 }
3912               break;
3913             default:
3914               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3915               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3916             }
3917         }
3918     }
3919
3920   return NAT_ED_TRNSL_ERR_SUCCESS;
3921 }
3922
3923 static_always_inline nat_translation_error_e
3924 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3925                            ip4_header_t *ip, nat_6t_flow_t *f,
3926                            ip_protocol_t proto, int is_output_feature,
3927                            int is_i2o)
3928 {
3929   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3930     {
3931       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3932     }
3933
3934   if (IP_PROTOCOL_ICMP == proto)
3935     {
3936       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3937         {
3938           // packet is returned from a router, not from destination
3939           // skip source address rewrite if in o2i path
3940           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3941                                      0 /* is_icmp_inner_ip4 */,
3942                                      !is_i2o /* skip_saddr_rewrite */);
3943         }
3944       else
3945         {
3946           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3947                                      0 /* is_icmp_inner_ip4 */,
3948                                      0 /* skip_saddr_rewrite */);
3949         }
3950       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
3951     }
3952
3953   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
3954                              0 /* skip_saddr_rewrite */);
3955
3956   return NAT_ED_TRNSL_ERR_SUCCESS;
3957 }
3958
3959 nat_translation_error_e
3960 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
3961                                vlib_buffer_t *b, ip4_header_t *ip,
3962                                nat_6t_flow_t *f, ip_protocol_t proto,
3963                                int is_output_feature)
3964 {
3965   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3966                                     1 /* is_i2o */);
3967 }
3968
3969 nat_translation_error_e
3970 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
3971                                vlib_buffer_t *b, ip4_header_t *ip,
3972                                nat_6t_flow_t *f, ip_protocol_t proto,
3973                                int is_output_feature)
3974 {
3975   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3976                                     0 /* is_i2o */);
3977 }
3978
3979 static_always_inline void
3980 nat_syslog_nat44_sess (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
3981                        u16 isport, ip4_address_t *xsaddr, u16 xsport,
3982                        ip4_address_t *idaddr, u16 idport,
3983                        ip4_address_t *xdaddr, u16 xdport, u8 proto, u8 is_add,
3984                        u8 is_twicenat)
3985 {
3986   syslog_msg_t syslog_msg;
3987   fib_table_t *fib;
3988
3989   if (!syslog_is_enabled ())
3990     return;
3991
3992   if (syslog_severity_filter_block (SADD_SDEL_SEVERITY))
3993     return;
3994
3995   fib = fib_table_get (sfibix, FIB_PROTOCOL_IP4);
3996
3997   syslog_msg_init (&syslog_msg, NAT_FACILITY, SADD_SDEL_SEVERITY, NAT_APPNAME,
3998                    is_add ? SADD_MSGID : SDEL_MSGID);
3999
4000   syslog_msg_sd_init (&syslog_msg, NSESS_SDID);
4001   syslog_msg_add_sd_param (&syslog_msg, SSUBIX_SDPARAM_NAME, "%d", ssubix);
4002   syslog_msg_add_sd_param (&syslog_msg, SVLAN_SDPARAM_NAME, "%d",
4003                            fib->ft_table_id);
4004   syslog_msg_add_sd_param (&syslog_msg, IATYP_SDPARAM_NAME, IATYP_IPV4);
4005   syslog_msg_add_sd_param (&syslog_msg, ISADDR_SDPARAM_NAME, "%U",
4006                            format_ip4_address, isaddr);
4007   syslog_msg_add_sd_param (&syslog_msg, ISPORT_SDPARAM_NAME, "%d",
4008                            clib_net_to_host_u16 (isport));
4009   syslog_msg_add_sd_param (&syslog_msg, XATYP_SDPARAM_NAME, IATYP_IPV4);
4010   syslog_msg_add_sd_param (&syslog_msg, XSADDR_SDPARAM_NAME, "%U",
4011                            format_ip4_address, xsaddr);
4012   syslog_msg_add_sd_param (&syslog_msg, XSPORT_SDPARAM_NAME, "%d",
4013                            clib_net_to_host_u16 (xsport));
4014   syslog_msg_add_sd_param (&syslog_msg, PROTO_SDPARAM_NAME, "%d", proto);
4015   syslog_msg_add_sd_param (&syslog_msg, XDADDR_SDPARAM_NAME, "%U",
4016                            format_ip4_address, xdaddr);
4017   syslog_msg_add_sd_param (&syslog_msg, XDPORT_SDPARAM_NAME, "%d",
4018                            clib_net_to_host_u16 (xdport));
4019   if (is_twicenat)
4020     {
4021       syslog_msg_add_sd_param (&syslog_msg, IDADDR_SDPARAM_NAME, "%U",
4022                                format_ip4_address, idaddr);
4023       syslog_msg_add_sd_param (&syslog_msg, IDPORT_SDPARAM_NAME, "%d",
4024                                clib_net_to_host_u16 (idport));
4025     }
4026
4027   syslog_msg_send (&syslog_msg);
4028 }
4029
4030 void
4031 nat_syslog_nat44_sadd (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4032                        u16 isport, ip4_address_t *idaddr, u16 idport,
4033                        ip4_address_t *xsaddr, u16 xsport,
4034                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4035                        u8 is_twicenat)
4036 {
4037   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4038                          idaddr, idport, xdaddr, xdport, proto, 1,
4039                          is_twicenat);
4040 }
4041
4042 void
4043 nat_syslog_nat44_sdel (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4044                        u16 isport, ip4_address_t *idaddr, u16 idport,
4045                        ip4_address_t *xsaddr, u16 xsport,
4046                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4047                        u8 is_twicenat)
4048 {
4049   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4050                          idaddr, idport, xdaddr, xdport, proto, 0,
4051                          is_twicenat);
4052 }
4053
4054 /*
4055  * fd.io coding-style-patch-verification: ON
4056  *
4057  * Local Variables:
4058  * eval: (c-set-style "gnu")
4059  * End:
4060  */