src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vpp/app/version.h>
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/ip/ip_table.h>
  22 #include <vnet/ip/reass/ip4_sv_reass.h>
  23 #include <vnet/fib/fib_table.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/plugin/plugin.h>
  26 #include <vppinfra/bihash_16_8.h>
  27
  28 #include <nat/lib/log.h>
  29 #include <nat/lib/nat_inlines.h>
  30 #include <nat/lib/ipfix_logging.h>
  31 #include <vnet/syslog/syslog.h>
  32 #include <nat/lib/nat_syslog_constants.h>
  33 #include <nat/lib/nat_syslog.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 #include <vlib/stats/stats.h>
  40
  41 snat_main_t snat_main;
  42
  43 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  44                                                            u32 sw_if_index);
  45
  46 #define skip_if_disabled()                                                    \
  47   do                                                                          \
  48     {                                                                         \
  49       snat_main_t *sm = &snat_main;                                           \
  50       if (PREDICT_FALSE (!sm->enabled))                                       \
  51         return;                                                               \
  52     }                                                                         \
  53   while (0)
  54
  55 #define fail_if_enabled()                                                     \
  56   do                                                                          \
  57     {                                                                         \
  58       snat_main_t *sm = &snat_main;                                           \
  59       if (PREDICT_FALSE (sm->enabled))                                        \
  60         {                                                                     \
  61           nat_log_err ("plugin enabled");                                     \
  62           return VNET_API_ERROR_FEATURE_ALREADY_ENABLED;                      \
  63         }                                                                     \
  64     }                                                                         \
  65   while (0)
  66
  67 #define fail_if_disabled()                                                    \
  68   do                                                                          \
  69     {                                                                         \
  70       snat_main_t *sm = &snat_main;                                           \
  71       if (PREDICT_FALSE (!sm->enabled))                                       \
  72         {                                                                     \
  73           nat_log_err ("plugin disabled");                                    \
  74           return VNET_API_ERROR_FEATURE_ALREADY_DISABLED;                     \
  75         }                                                                     \
  76     }                                                                         \
  77   while (0)
  78
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-ed-classify",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  96                                "ip4-sv-reassembly-feature"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-handoff-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 102                                "ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-worker-handoff",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 108 };
 109 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-worker-handoff",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-in2out",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-out2in",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 124                                "ip4-dhcp-client-detect"),
 125 };
 126 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 127   .arc_name = "ip4-output",
 128   .node_name = "nat-pre-in2out-output",
 129   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 130   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 136   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-ed-in2out-output",
 141   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 142   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 143 };
 144
 145 VLIB_PLUGIN_REGISTER () = {
 146     .version = VPP_BUILD_VER,
 147     .description = "Network Address Translation (NAT)",
 148 };
 149
 150 static void nat44_ed_db_init ();
 151 static void nat44_ed_db_free ();
 152 static void nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm);
 153
 154 static int nat44_ed_add_static_mapping_internal (
 155   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 156   ip_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 157   ip4_address_t pool_addr, u8 *tag);
 158 static int nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
 159                                                  ip4_address_t e_addr,
 160                                                  u16 l_port, u16 e_port,
 161                                                  ip_protocol_t proto,
 162                                                  u32 vrf_id, u32 flags);
 163
 164 u32 nat_calc_bihash_buckets (u32 n_elts);
 165
 166 static_always_inline int
 167 nat44_ed_sm_i2o_add (snat_main_t *sm, snat_static_mapping_t *m,
 168                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 169 {
 170   ASSERT (!pool_is_free (sm->static_mappings, m));
 171   clib_bihash_kv_16_8_t kv;
 172   nat44_ed_sm_init_i2o_kv (&kv, addr.as_u32, port, fib_index, proto,
 173                            m - sm->static_mappings);
 174   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 175 }
 176
 177 static_always_inline int
 178 nat44_ed_sm_i2o_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 179                      u32 fib_index, u8 proto)
 180 {
 181   clib_bihash_kv_16_8_t kv;
 182   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 183   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 184 }
 185
 186 static_always_inline int
 187 nat44_ed_sm_o2i_add (snat_main_t *sm, snat_static_mapping_t *m,
 188                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 189 {
 190   ASSERT (!pool_is_free (sm->static_mappings, m));
 191   clib_bihash_kv_16_8_t kv;
 192   nat44_ed_sm_init_o2i_kv (&kv, addr.as_u32, port, fib_index, proto,
 193                            m - sm->static_mappings);
 194   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 195 }
 196
 197 static_always_inline int
 198 nat44_ed_sm_o2i_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 199                      u32 fib_index, u8 proto)
 200 {
 201   clib_bihash_kv_16_8_t kv;
 202   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 203   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 204 }
 205
 206 void
 207 nat44_ed_free_session_data (snat_main_t *sm, snat_session_t *s,
 208                             u32 thread_index, u8 is_ha)
 209 {
 210   per_vrf_sessions_unregister_session (s, thread_index);
 211
 212   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 213     nat_elog_warn (sm, "flow hash del failed");
 214
 215   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 216     nat_elog_warn (sm, "flow hash del failed");
 217
 218   if (na44_ed_is_fwd_bypass_session (s))
 219     {
 220       return;
 221     }
 222
 223   if (nat44_ed_is_affinity_session (s))
 224     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, s->proto,
 225                          s->out2in.port);
 226
 227   if (!is_ha)
 228     nat_syslog_nat44_sdel (0, s->in2out.fib_index, &s->in2out.addr,
 229                            s->in2out.port, &s->ext_host_nat_addr,
 230                            s->ext_host_nat_port, &s->out2in.addr,
 231                            s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 232                            s->proto, nat44_ed_is_twice_nat_session (s));
 233
 234   if (!is_ha)
 235     {
 236       /* log NAT event */
 237       nat_ipfix_logging_nat44_ses_delete (
 238         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 239         s->in2out.port, s->out2in.port, s->in2out.fib_index);
 240     }
 241 }
 242
 243 static ip_interface_address_t *
 244 nat44_ed_get_ip_interface_address (u32 sw_if_index, ip4_address_t addr)
 245 {
 246   snat_main_t *sm = &snat_main;
 247
 248   ip_lookup_main_t *lm = &sm->ip4_main->lookup_main;
 249   ip_interface_address_t *ia;
 250   ip4_address_t *ip4a;
 251
 252   foreach_ip_interface_address (
 253     lm, ia, sw_if_index, 1, ({
 254       ip4a = ip_interface_address_get_address (lm, ia);
 255       nat_log_debug ("sw_if_idx: %u addr: %U ? %U", sw_if_index,
 256                      format_ip4_address, ip4a, format_ip4_address, &addr);
 257       if (ip4a->as_u32 == addr.as_u32)
 258         {
 259           return ia;
 260         }
 261     }));
 262   return NULL;
 263 }
 264
 265 static int
 266 nat44_ed_resolve_nat_addr_len (snat_address_t *ap,
 267                                snat_interface_t *interfaces)
 268 {
 269   ip_interface_address_t *ia;
 270   snat_interface_t *i;
 271   u32 fib_index;
 272
 273   pool_foreach (i, interfaces)
 274     {
 275       if (!nat44_ed_is_interface_outside (i))
 276         {
 277           continue;
 278         }
 279
 280       fib_index = ip4_fib_table_get_index_for_sw_if_index (i->sw_if_index);
 281       if (fib_index != ap->fib_index)
 282         {
 283           continue;
 284         }
 285
 286       if ((ia = nat44_ed_get_ip_interface_address (i->sw_if_index, ap->addr)))
 287         {
 288           ap->addr_len = ia->address_length;
 289           ap->sw_if_index = i->sw_if_index;
 290           ap->net.as_u32 = ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
 291
 292           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 293                          format_ip4_address, &ap->addr, ap->sw_if_index,
 294                          format_ip4_address, &ap->net, ap->addr_len);
 295           return 0;
 296         }
 297     }
 298   return 1;
 299 }
 300
 301 static void
 302 nat44_ed_update_outside_if_addresses (snat_address_t *ap)
 303 {
 304   snat_main_t *sm = &snat_main;
 305
 306   if (!nat44_ed_resolve_nat_addr_len (ap, sm->interfaces))
 307     {
 308       return;
 309     }
 310
 311   if (!nat44_ed_resolve_nat_addr_len (ap, sm->output_feature_interfaces))
 312     {
 313       return;
 314     }
 315 }
 316
 317 static void
 318 nat44_ed_bind_if_addr_to_nat_addr (u32 sw_if_index)
 319 {
 320   snat_main_t *sm = &snat_main;
 321   ip_interface_address_t *ia;
 322   snat_address_t *ap;
 323
 324   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 325
 326   vec_foreach (ap, sm->addresses)
 327     {
 328       if (fib_index != ap->fib_index)
 329         {
 330           continue;
 331         }
 332
 333       if ((ia = nat44_ed_get_ip_interface_address (sw_if_index, ap->addr)))
 334         {
 335           ap->addr_len = ia->address_length;
 336           ap->sw_if_index = sw_if_index;
 337           ap->net.as_u32 = ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
 338
 339           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 340                          format_ip4_address, &ap->addr, ap->sw_if_index,
 341                          format_ip4_address, &ap->net, ap->addr_len);
 342           return;
 343         }
 344     }
 345 }
 346
 347 static_always_inline snat_fib_entry_reg_t *
 348 nat44_ed_get_fib_entry_reg (ip4_address_t addr, u32 sw_if_index, int *out_idx)
 349 {
 350   snat_main_t *sm = &snat_main;
 351   snat_fib_entry_reg_t *fe;
 352   int i;
 353
 354   for (i = 0; i < vec_len (sm->fib_entry_reg); i++)
 355     {
 356       fe = sm->fib_entry_reg + i;
 357       if ((addr.as_u32 == fe->addr.as_u32) && (sw_if_index == fe->sw_if_index))
 358         {
 359           if (out_idx)
 360             {
 361               *out_idx = i;
 362             }
 363           return fe;
 364         }
 365     }
 366   return NULL;
 367 }
 368
 369 static void
 370 nat44_ed_add_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 371 {
 372   // Add the external NAT address to the FIB as receive entries. This ensures
 373   // that VPP will reply to ARP for this address and we don't need to enable
 374   // proxy ARP on the outside interface.
 375   snat_main_t *sm = &snat_main;
 376   snat_fib_entry_reg_t *fe;
 377
 378   if (!(fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, 0)))
 379     {
 380       fib_prefix_t prefix = {
 381         .fp_len = 32,
 382         .fp_proto = FIB_PROTOCOL_IP4,
 383         .fp_addr = {
 384                   .ip4.as_u32 = addr.as_u32,
 385                 },
 386       };
 387       u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 388       fib_table_entry_update_one_path (fib_index, &prefix, sm->fib_src_low,
 389                                        (FIB_ENTRY_FLAG_CONNECTED |
 390                                         FIB_ENTRY_FLAG_LOCAL |
 391                                         FIB_ENTRY_FLAG_EXCLUSIVE),
 392                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
 393                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 394
 395       vec_add2 (sm->fib_entry_reg, fe, 1);
 396       clib_memset (fe, 0, sizeof (*fe));
 397       fe->addr.as_u32 = addr.as_u32;
 398       fe->sw_if_index = sw_if_index;
 399     }
 400   fe->count++;
 401 }
 402
 403 static void
 404 nat44_ed_del_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 405 {
 406   snat_main_t *sm = &snat_main;
 407   snat_fib_entry_reg_t *fe;
 408   int i;
 409
 410   if ((fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, &i)))
 411     {
 412       fe->count--;
 413       if (0 == fe->count)
 414         {
 415           fib_prefix_t prefix = {
 416             .fp_len = 32,
 417             .fp_proto = FIB_PROTOCOL_IP4,
 418             .fp_addr = {
 419               .ip4.as_u32 = addr.as_u32,
 420                     },
 421           };
 422           u32 fib_index =
 423             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 424           fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 425           vec_del1 (sm->fib_entry_reg, i);
 426         }
 427     }
 428 }
 429
 430 static void
 431 nat44_ed_add_del_interface_fib_reg_entries (ip4_address_t addr, u8 is_add)
 432 {
 433   snat_main_t *sm = &snat_main;
 434   snat_interface_t *i;
 435
 436   pool_foreach (i, sm->interfaces)
 437     {
 438       if (nat44_ed_is_interface_outside (i))
 439         {
 440           if (is_add)
 441             {
 442               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 443             }
 444           else
 445             {
 446               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 447             }
 448         }
 449     }
 450   pool_foreach (i, sm->output_feature_interfaces)
 451     {
 452       if (nat44_ed_is_interface_outside (i))
 453         {
 454           if (is_add)
 455             {
 456               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 457             }
 458           else
 459             {
 460               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 461             }
 462         }
 463     }
 464 }
 465
 466 static_always_inline void
 467 nat44_ed_add_del_nat_addr_fib_reg_entries (u32 sw_if_index, u8 is_add)
 468 {
 469   snat_main_t *sm = &snat_main;
 470   snat_address_t *ap;
 471
 472   vec_foreach (ap, sm->addresses)
 473     {
 474       if (is_add)
 475         {
 476           nat44_ed_add_fib_entry_reg (ap->addr, sw_if_index);
 477         }
 478       else
 479         {
 480           nat44_ed_del_fib_entry_reg (ap->addr, sw_if_index);
 481         }
 482     }
 483 }
 484
 485 static_always_inline void
 486 nat44_ed_add_del_sm_fib_reg_entries (u32 sw_if_index, u8 is_add)
 487 {
 488   snat_main_t *sm = &snat_main;
 489   snat_static_mapping_t *m;
 490
 491   pool_foreach (m, sm->static_mappings)
 492     {
 493       if (is_add)
 494         {
 495           nat44_ed_add_fib_entry_reg (m->external_addr, sw_if_index);
 496         }
 497       else
 498         {
 499           nat44_ed_del_fib_entry_reg (m->external_addr, sw_if_index);
 500         }
 501     }
 502 }
 503
 504 int
 505 nat44_ed_add_address (ip4_address_t *addr, u32 vrf_id, u8 twice_nat)
 506 {
 507   snat_main_t *sm = &snat_main;
 508   snat_address_t *ap, *addresses;
 509
 510   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 511
 512   if (!sm->enabled)
 513     {
 514       return VNET_API_ERROR_UNSUPPORTED;
 515     }
 516
 517   // check if address already exists
 518   vec_foreach (ap, addresses)
 519     {
 520       if (ap->addr.as_u32 == addr->as_u32)
 521         {
 522           nat_log_err ("address exist");
 523           return VNET_API_ERROR_VALUE_EXIST;
 524         }
 525     }
 526
 527   if (twice_nat)
 528     {
 529       vec_add2 (sm->twice_nat_addresses, ap, 1);
 530     }
 531   else
 532     {
 533       vec_add2 (sm->addresses, ap, 1);
 534     }
 535
 536   ap->addr_len = ~0;
 537   ap->fib_index = ~0;
 538   ap->addr = *addr;
 539
 540   if (vrf_id != ~0)
 541     {
 542       ap->fib_index = fib_table_find_or_create_and_lock (
 543         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 544     }
 545
 546   if (!twice_nat)
 547     {
 548       // if we don't have enabled interface we don't add address
 549       // to fib
 550       nat44_ed_add_del_interface_fib_reg_entries (*addr, 1);
 551       nat44_ed_update_outside_if_addresses (ap);
 552     }
 553   return 0;
 554 }
 555
 556 int
 557 nat44_ed_del_address (ip4_address_t addr, u8 twice_nat)
 558 {
 559   snat_main_t *sm = &snat_main;
 560   snat_address_t *a = 0, *addresses;
 561   snat_session_t *ses;
 562   u32 *ses_to_be_removed = 0, *ses_index;
 563   snat_main_per_thread_data_t *tsm;
 564   int j;
 565
 566   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 567
 568   for (j = 0; j < vec_len (addresses); j++)
 569     {
 570       if (addresses[j].addr.as_u32 == addr.as_u32)
 571         {
 572           a = addresses + j;
 573           break;
 574         }
 575     }
 576   if (!a)
 577     {
 578       nat_log_err ("no such address");
 579       return VNET_API_ERROR_NO_SUCH_ENTRY;
 580     }
 581
 582   // delete dynamic sessions only
 583   vec_foreach (tsm, sm->per_thread_data)
 584     {
 585       pool_foreach (ses, tsm->sessions)
 586         {
 587           if (ses->flags & SNAT_SESSION_FLAG_STATIC_MAPPING)
 588             {
 589               continue;
 590             }
 591           if (ses->out2in.addr.as_u32 == addr.as_u32)
 592             {
 593               nat44_ed_free_session_data (sm, ses, tsm - sm->per_thread_data,
 594                                           0);
 595               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
 596             }
 597         }
 598       vec_foreach (ses_index, ses_to_be_removed)
 599         {
 600           ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
 601           nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
 602         }
 603       vec_free (ses_to_be_removed);
 604     }
 605
 606   if (!twice_nat)
 607     {
 608       nat44_ed_add_del_interface_fib_reg_entries (addr, 0);
 609     }
 610
 611   if (a->fib_index != ~0)
 612     {
 613       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 614     }
 615
 616   if (!twice_nat)
 617     {
 618       vec_del1 (sm->addresses, j);
 619     }
 620   else
 621     {
 622       vec_del1 (sm->twice_nat_addresses, j);
 623     }
 624
 625   return 0;
 626 }
 627
 628 vrf_table_t *
 629 nat44_ed_get_vrf_table (u32 table_vrf_id)
 630 {
 631   snat_main_t *sm = &snat_main;
 632   vrf_table_t *t;
 633
 634   pool_foreach (t, sm->vrf_tables)
 635     {
 636       if (table_vrf_id == t->table_vrf_id)
 637         {
 638           return t;
 639         }
 640     }
 641   return NULL;
 642 }
 643
 644 vrf_route_t *
 645 nat44_ed_get_vrf_route (vrf_table_t *t, u32 vrf_id)
 646 {
 647   vrf_route_t *r;
 648
 649   pool_foreach (r, t->routes)
 650     {
 651       if (vrf_id == r->vrf_id)
 652         {
 653           return r;
 654         }
 655     }
 656   return NULL;
 657 }
 658
 659 int
 660 nat44_ed_add_del_vrf_table (u32 table_vrf_id, bool is_add)
 661 {
 662   snat_main_t *sm = &snat_main;
 663   vrf_table_t *t;
 664   vrf_route_t *r;
 665
 666   t = nat44_ed_get_vrf_table (table_vrf_id);
 667   if (t)
 668     {
 669       if (is_add)
 670         {
 671           return VNET_API_ERROR_VALUE_EXIST;
 672         }
 673       pool_foreach (r, t->routes)
 674         {
 675           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 676         }
 677       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 678
 679       pool_free (t->routes);
 680       pool_put (sm->vrf_tables, t);
 681     }
 682   else
 683     {
 684       if (!is_add)
 685         {
 686           return VNET_API_ERROR_NO_SUCH_ENTRY;
 687         }
 688       pool_get (sm->vrf_tables, t);
 689       clib_memset (t, 0, sizeof (*t));
 690       t->table_vrf_id = table_vrf_id;
 691       t->table_fib_index = fib_table_find_or_create_and_lock (
 692         FIB_PROTOCOL_IP4, table_vrf_id, sm->fib_src_low);
 693     }
 694
 695   return 0;
 696 }
 697
 698 void
 699 nat44_ed_del_vrf_tables ()
 700 {
 701   snat_main_t *sm = &snat_main;
 702   vrf_table_t *t;
 703   vrf_route_t *r;
 704
 705   pool_foreach (t, sm->vrf_tables)
 706     {
 707       pool_foreach (r, t->routes)
 708         {
 709           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 710         }
 711       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 712       pool_free (t->routes);
 713     }
 714   pool_free (sm->vrf_tables);
 715 }
 716
 717 int
 718 nat44_ed_add_del_vrf_route (u32 table_vrf_id, u32 vrf_id, bool is_add)
 719 {
 720   snat_main_t *sm = &snat_main;
 721   vrf_table_t *t;
 722   vrf_route_t *r;
 723
 724   t = nat44_ed_get_vrf_table (table_vrf_id);
 725   if (!t)
 726     {
 727       return VNET_API_ERROR_NO_SUCH_ENTRY;
 728     }
 729
 730   r = nat44_ed_get_vrf_route (t, vrf_id);
 731   if (r)
 732     {
 733       if (is_add)
 734         {
 735           return VNET_API_ERROR_VALUE_EXIST;
 736         }
 737       fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 738       pool_put (t->routes, r);
 739     }
 740   else
 741     {
 742       if (!is_add)
 743         {
 744           return VNET_API_ERROR_NO_SUCH_ENTRY;
 745         }
 746       pool_get (t->routes, r);
 747       clib_memset (r, 0, sizeof (*r));
 748       r->vrf_id = vrf_id;
 749       r->fib_index = fib_table_find_or_create_and_lock (
 750         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 751     }
 752
 753   return 0;
 754 }
 755
 756 u32
 757 get_thread_idx_by_port (u16 e_port)
 758 {
 759   snat_main_t *sm = &snat_main;
 760   u32 thread_idx = sm->num_workers;
 761   if (sm->num_workers > 1)
 762     {
 763       thread_idx = sm->first_worker_index +
 764                    sm->workers[(e_port - ED_USER_PORT_OFFSET) /
 765                                sm->port_per_thread % _vec_len (sm->workers)];
 766     }
 767   return thread_idx;
 768 }
 769
 770 void
 771 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 772                                     snat_main_per_thread_data_t * tsm,
 773                                     ip4_address_t l_addr,
 774                                     u16 l_port,
 775                                     u8 protocol,
 776                                     u32 fib_index, int addr_only,
 777                                     ip4_address_t e_addr, u16 e_port)
 778 {
 779   snat_session_t *s;
 780   u32 *indexes_to_free = NULL;
 781   pool_foreach (s, tsm->sessions) {
 782     if (s->in2out.fib_index != fib_index ||
 783         s->in2out.addr.as_u32 != l_addr.as_u32)
 784       {
 785         continue;
 786       }
 787     if (!addr_only)
 788       {
 789         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 790             s->out2in.port != e_port || s->in2out.port != l_port ||
 791             s->proto != protocol)
 792           continue;
 793       }
 794
 795     if (nat44_ed_is_lb_session (s))
 796       continue;
 797     if (!nat44_ed_is_session_static (s))
 798       continue;
 799     nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 800     vec_add1 (indexes_to_free, s - tsm->sessions);
 801     if (!addr_only)
 802       break;
 803   }
 804   u32 *ses_index;
 805   vec_foreach (ses_index, indexes_to_free)
 806   {
 807     s = pool_elt_at_index (tsm->sessions, *ses_index);
 808     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 809   }
 810   vec_free (indexes_to_free);
 811 }
 812
 813 static_always_inline snat_static_mapping_t *
 814 nat44_ed_sm_lookup (snat_main_t *sm, clib_bihash_kv_16_8_t *kv)
 815 {
 816   clib_bihash_kv_16_8_t v;
 817   int rc = clib_bihash_search_16_8 (&sm->flow_hash, kv, &v);
 818   if (!rc)
 819     {
 820       ASSERT (0 == ed_value_get_thread_index (&v));
 821       return pool_elt_at_index (sm->static_mappings,
 822                                 ed_value_get_session_index (&v));
 823     }
 824   return NULL;
 825 }
 826
 827 snat_static_mapping_t *
 828 nat44_ed_sm_o2i_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 829                         u32 fib_index, u8 proto)
 830 {
 831   clib_bihash_kv_16_8_t kv;
 832   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 833   return nat44_ed_sm_lookup (sm, &kv);
 834 }
 835
 836 snat_static_mapping_t *
 837 nat44_ed_sm_i2o_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 838                         u32 fib_index, u8 proto)
 839 {
 840   clib_bihash_kv_16_8_t kv;
 841   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 842   return nat44_ed_sm_lookup (sm, &kv);
 843 }
 844
 845 static snat_static_mapping_resolve_t *
 846 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 847                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 848                              u32 flags, int *out_idx)
 849 {
 850   snat_static_mapping_resolve_t *rp;
 851   snat_main_t *sm = &snat_main;
 852   int i;
 853
 854   for (i = 0; i < vec_len (sm->sm_to_resolve); i++)
 855     {
 856       rp = sm->sm_to_resolve + i;
 857
 858       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 859         {
 860           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 861             {
 862               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 863                 {
 864                   if (rp->e_port != e_port || rp->proto != proto)
 865                     {
 866                       continue;
 867                     }
 868                 }
 869             }
 870           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 871             {
 872               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 873                 {
 874                   if (rp->l_port != l_port || rp->e_port != e_port ||
 875                       rp->proto != proto)
 876                     {
 877                       continue;
 878                     }
 879                 }
 880             }
 881           else
 882             {
 883               continue;
 884             }
 885           if (out_idx)
 886             {
 887               *out_idx = i;
 888             }
 889           return rp;
 890         }
 891     }
 892   return NULL;
 893 }
 894
 895 static int
 896 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 897                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 898                              u32 flags)
 899 {
 900   snat_main_t *sm = &snat_main;
 901   int i;
 902   if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 903                                    sw_if_index, flags, &i))
 904     {
 905       vec_del1 (sm->sm_to_resolve, i);
 906       return 0;
 907     }
 908   return 1;
 909 }
 910
 911 static_always_inline int
 912 nat44_ed_validate_sm_input (u32 flags)
 913 {
 914   // identity nat can be initiated only from inside interface
 915   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 916     {
 917       return VNET_API_ERROR_UNSUPPORTED;
 918     }
 919
 920   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 921     {
 922       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 923         {
 924           return VNET_API_ERROR_UNSUPPORTED;
 925         }
 926     }
 927   return 0;
 928 }
 929
 930 int
 931 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 932                              u16 l_port, u16 e_port, ip_protocol_t proto,
 933                              u32 vrf_id, u32 sw_if_index, u32 flags,
 934                              ip4_address_t pool_addr, u8 *tag)
 935 {
 936   snat_static_mapping_resolve_t *rp;
 937   snat_main_t *sm = &snat_main;
 938   int rv;
 939
 940   if (!sm->enabled)
 941     {
 942       return VNET_API_ERROR_UNSUPPORTED;
 943     }
 944
 945   rv = nat44_ed_validate_sm_input (flags);
 946   if (rv != 0)
 947     {
 948       return rv;
 949     }
 950
 951   // interface bound mapping
 952   if (is_sm_switch_address (flags))
 953     {
 954       if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 955                                        sw_if_index, flags, 0))
 956         {
 957           return VNET_API_ERROR_VALUE_EXIST;
 958         }
 959
 960       vec_add2 (sm->sm_to_resolve, rp, 1);
 961       rp->l_addr.as_u32 = l_addr.as_u32;
 962       rp->l_port = l_port;
 963       rp->e_port = e_port;
 964       rp->sw_if_index = sw_if_index;
 965       rp->vrf_id = vrf_id;
 966       rp->proto = proto;
 967       rp->flags = flags;
 968       rp->pool_addr = pool_addr;
 969       rp->tag = vec_dup (tag);
 970       rp->is_resolved = 0;
 971
 972       ip4_address_t *first_int_addr =
 973         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 974       if (!first_int_addr)
 975         {
 976           return 0;
 977         }
 978
 979       e_addr.as_u32 = first_int_addr->as_u32;
 980       rp->is_resolved = 1;
 981     }
 982
 983   rv = nat44_ed_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
 984                                              proto, vrf_id, sw_if_index, flags,
 985                                              pool_addr, tag);
 986   if ((0 != rv) && is_sm_switch_address (flags))
 987     {
 988       nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 989                                    sw_if_index, flags);
 990     }
 991
 992   return rv;
 993 }
 994
 995 int
 996 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 997                              u16 l_port, u16 e_port, ip_protocol_t proto,
 998                              u32 vrf_id, u32 sw_if_index, u32 flags)
 999 {
1000   snat_main_t *sm = &snat_main;
1001   int rv;
1002
1003   if (!sm->enabled)
1004     {
1005       return VNET_API_ERROR_UNSUPPORTED;
1006     }
1007
1008   rv = nat44_ed_validate_sm_input (flags);
1009   if (rv != 0)
1010     {
1011       return rv;
1012     }
1013
1014   // interface bound mapping
1015   if (is_sm_switch_address (flags))
1016     {
1017       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
1018                                        sw_if_index, flags))
1019         {
1020           return VNET_API_ERROR_NO_SUCH_ENTRY;
1021         }
1022
1023       ip4_address_t *first_int_addr =
1024         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
1025       if (!first_int_addr)
1026         {
1027           // dhcp resolution required
1028           return 0;
1029         }
1030
1031       e_addr.as_u32 = first_int_addr->as_u32;
1032     }
1033
1034   return nat44_ed_del_static_mapping_internal (l_addr, e_addr, l_port, e_port,
1035                                                proto, vrf_id, flags);
1036 }
1037
1038 static int
1039 nat44_ed_add_static_mapping_internal (ip4_address_t l_addr,
1040                                       ip4_address_t e_addr, u16 l_port,
1041                                       u16 e_port, ip_protocol_t proto,
1042                                       u32 vrf_id, u32 sw_if_index, u32 flags,
1043                                       ip4_address_t pool_addr, u8 *tag)
1044 {
1045   snat_main_t *sm = &snat_main;
1046   nat44_lb_addr_port_t *local;
1047   snat_static_mapping_t *m;
1048   u32 fib_index = ~0;
1049
1050   if (is_sm_addr_only (flags))
1051     {
1052       e_port = l_port = proto = 0;
1053     }
1054
1055   if (is_sm_identity_nat (flags))
1056     {
1057       l_port = e_port;
1058       l_addr.as_u32 = e_addr.as_u32;
1059     }
1060
1061   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1062   if (m)
1063     {
1064       // case:
1065       // adding local identity nat record for different vrf table
1066
1067       if (!is_sm_identity_nat (m->flags))
1068         {
1069           return VNET_API_ERROR_VALUE_EXIST;
1070         }
1071
1072       pool_foreach (local, m->locals)
1073         {
1074           if (local->vrf_id == vrf_id)
1075             {
1076               return VNET_API_ERROR_VALUE_EXIST;
1077             }
1078         }
1079
1080       pool_get (m->locals, local);
1081
1082       local->vrf_id = vrf_id;
1083       local->fib_index = fib_table_find_or_create_and_lock (
1084         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
1085
1086       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port,
1087                            local->fib_index, m->proto);
1088
1089       return 0;
1090     }
1091
1092   if (vrf_id != ~0)
1093     {
1094       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1095                                                      sm->fib_src_low);
1096     }
1097   else
1098     {
1099       // fallback to default vrf
1100       vrf_id = sm->inside_vrf_id;
1101       fib_index = sm->inside_fib_index;
1102       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1103     }
1104
1105   // test if local mapping record doesn't exist
1106   // identity nat supports multiple records in local mapping
1107   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
1108     {
1109       if (nat44_ed_sm_i2o_lookup (sm, l_addr, l_port, fib_index, proto))
1110         {
1111           return VNET_API_ERROR_VALUE_EXIST;
1112         }
1113     }
1114
1115   pool_get (sm->static_mappings, m);
1116   clib_memset (m, 0, sizeof (*m));
1117
1118   m->flags = flags;
1119   m->local_addr = l_addr;
1120   m->external_addr = e_addr;
1121
1122   m->pool_addr = pool_addr;
1123   m->tag = vec_dup (tag);
1124
1125   if (!is_sm_addr_only (flags))
1126     {
1127       m->local_port = l_port;
1128       m->external_port = e_port;
1129       m->proto = proto;
1130     }
1131
1132   if (is_sm_identity_nat (flags))
1133     {
1134       pool_get (m->locals, local);
1135
1136       local->vrf_id = vrf_id;
1137       local->fib_index = fib_index;
1138     }
1139   else
1140     {
1141       m->vrf_id = vrf_id;
1142       m->fib_index = fib_index;
1143     }
1144
1145   if (!is_sm_out2in_only (flags))
1146     {
1147       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port, fib_index,
1148                            m->proto);
1149     }
1150
1151   nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0, m->proto);
1152
1153   if (sm->num_workers > 1)
1154     {
1155       // store worker index for this record
1156       ip4_header_t ip = {
1157         .src_address = m->local_addr,
1158       };
1159       u32 worker_index;
1160       worker_index =
1161         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
1162       vec_add1 (m->workers, worker_index);
1163     }
1164
1165   nat44_ed_add_del_interface_fib_reg_entries (e_addr, 1);
1166
1167   return 0;
1168 }
1169
1170 static int
1171 nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
1172                                       ip4_address_t e_addr, u16 l_port,
1173                                       u16 e_port, ip_protocol_t proto,
1174                                       u32 vrf_id, u32 flags)
1175 {
1176   snat_main_per_thread_data_t *tsm;
1177   snat_main_t *sm = &snat_main;
1178
1179   nat44_lb_addr_port_t *local;
1180   snat_static_mapping_t *m;
1181   u32 fib_index = ~0;
1182
1183   if (is_sm_addr_only (flags))
1184     {
1185       e_port = l_port = proto = 0;
1186     }
1187
1188   if (is_sm_identity_nat (flags))
1189     {
1190       l_port = e_port;
1191       l_addr.as_u32 = e_addr.as_u32;
1192     }
1193
1194   // fib index 0
1195   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1196   if (!m)
1197     {
1198       return VNET_API_ERROR_NO_SUCH_ENTRY;
1199     }
1200
1201   if (is_sm_identity_nat (flags))
1202     {
1203       u8 found = 0;
1204
1205       if (vrf_id == ~0)
1206         {
1207           vrf_id = sm->inside_vrf_id;
1208         }
1209
1210       pool_foreach (local, m->locals)
1211         {
1212           if (local->vrf_id == vrf_id)
1213             {
1214               local = pool_elt_at_index (m->locals, local - m->locals);
1215               fib_index = local->fib_index;
1216               pool_put (m->locals, local);
1217               found = 1;
1218             }
1219         }
1220
1221       if (!found)
1222         {
1223           return VNET_API_ERROR_NO_SUCH_ENTRY;
1224         }
1225     }
1226   else
1227     {
1228       fib_index = m->fib_index;
1229     }
1230
1231   if (!is_sm_out2in_only (flags))
1232     {
1233       nat44_ed_sm_i2o_del (sm, l_addr, l_port, fib_index, proto);
1234     }
1235
1236   // delete sessions for static mapping
1237   if (sm->num_workers > 1)
1238     {
1239       tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1240     }
1241   else
1242     {
1243       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1244     }
1245
1246   nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr, m->local_port,
1247                                       m->proto, fib_index,
1248                                       is_sm_addr_only (flags), e_addr, e_port);
1249
1250   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1251
1252   if (!pool_elts (m->locals))
1253     {
1254       // this is last record remove all required stuff
1255       // fib_index 0
1256       nat44_ed_sm_o2i_del (sm, e_addr, e_port, 0, proto);
1257
1258       vec_free (m->tag);
1259       vec_free (m->workers);
1260       pool_put (sm->static_mappings, m);
1261
1262       nat44_ed_add_del_interface_fib_reg_entries (e_addr, 0);
1263     }
1264
1265   return 0;
1266 }
1267
1268 int
1269 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1270                                 ip_protocol_t proto,
1271                                 nat44_lb_addr_port_t *locals, u32 flags,
1272                                 u8 *tag, u32 affinity)
1273 {
1274   snat_main_t *sm = &snat_main;
1275   snat_static_mapping_t *m;
1276   snat_address_t *a = 0;
1277
1278   nat44_lb_addr_port_t *local;
1279   uword *bitmap = 0;
1280   int rc = 0;
1281
1282   int i;
1283
1284   if (!sm->enabled)
1285     {
1286       return VNET_API_ERROR_UNSUPPORTED;
1287     }
1288
1289   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1290
1291   if (m)
1292     {
1293       return VNET_API_ERROR_VALUE_EXIST;
1294     }
1295
1296   if (vec_len (locals) < 2)
1297     {
1298       return VNET_API_ERROR_INVALID_VALUE;
1299     }
1300
1301   if (!is_sm_out2in_only (flags))
1302     {
1303       /* Find external address in allocated addresses and reserve port for
1304          address and port pair mapping when dynamic translations enabled */
1305       for (i = 0; i < vec_len (sm->addresses); i++)
1306         {
1307           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1308             {
1309               /* External port must be unused */
1310               a = sm->addresses + i;
1311               if (nat44_ed_sm_o2i_lookup (sm, a->addr, e_port, 0, proto))
1312                 {
1313                   return VNET_API_ERROR_VALUE_EXIST;
1314                 }
1315               break;
1316             }
1317         }
1318       // external address must be allocated
1319       if (!a)
1320         {
1321           return VNET_API_ERROR_NO_SUCH_ENTRY;
1322         }
1323     }
1324
1325   pool_get (sm->static_mappings, m);
1326   clib_memset (m, 0, sizeof (*m));
1327   m->tag = vec_dup (tag);
1328   m->external_addr = e_addr;
1329   m->external_port = e_port;
1330   m->affinity = affinity;
1331   m->proto = proto;
1332
1333   m->flags = flags;
1334   m->flags |= NAT_SM_FLAG_LB;
1335
1336   if (affinity)
1337     m->affinity_per_service_list_head_index =
1338       nat_affinity_get_per_service_list_head_index ();
1339   else
1340     m->affinity_per_service_list_head_index = ~0;
1341
1342   if (nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0,
1343                            m->proto))
1344     {
1345       nat_log_err ("sm o2i key add failed");
1346       return VNET_API_ERROR_UNSPECIFIED;
1347     }
1348
1349   for (i = 0; i < vec_len (locals); i++)
1350     {
1351       locals[i].fib_index = fib_table_find_or_create_and_lock (
1352         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1353       if (!is_sm_out2in_only (flags))
1354         {
1355           if (nat44_ed_sm_o2i_add (sm, m, e_addr, e_port, 0, proto))
1356             {
1357               nat_log_err ("sm o2i key add failed");
1358               rc = VNET_API_ERROR_UNSPECIFIED;
1359               // here we continue with add operation so that it can be safely
1360               // reversed in delete path - otherwise we'd have to track what
1361               // we've done and deal with partial cleanups and since bihash
1362               // adds are (extremely improbable) the only points of failure,
1363               // it's easier to just do it this way
1364             }
1365         }
1366       locals[i].prefix = (i == 0) ?
1367                            locals[i].probability :
1368                            (locals[i - 1].prefix + locals[i].probability);
1369       pool_get (m->locals, local);
1370       *local = locals[i];
1371       if (sm->num_workers > 1)
1372         {
1373           ip4_header_t ip = {
1374             .src_address = locals[i].addr,
1375           };
1376           bitmap = clib_bitmap_set (
1377             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1378             1);
1379         }
1380     }
1381
1382   /* Assign workers */
1383   if (sm->num_workers > 1)
1384     {
1385       clib_bitmap_foreach (i, bitmap)
1386         {
1387           vec_add1 (m->workers, i);
1388         }
1389     }
1390
1391   return rc;
1392 }
1393
1394 int
1395 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1396                                 ip_protocol_t proto, u32 flags)
1397 {
1398   snat_main_t *sm = &snat_main;
1399   snat_static_mapping_t *m;
1400
1401   nat44_lb_addr_port_t *local;
1402   snat_main_per_thread_data_t *tsm;
1403   snat_session_t *s;
1404
1405   if (!sm->enabled)
1406     {
1407       return VNET_API_ERROR_UNSUPPORTED;
1408     }
1409
1410   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1411   if (!m)
1412     return VNET_API_ERROR_NO_SUCH_ENTRY;
1413
1414   if (!is_sm_lb (m->flags))
1415     return VNET_API_ERROR_INVALID_VALUE;
1416
1417   if (nat44_ed_sm_o2i_del (sm, m->external_addr, m->external_port, 0,
1418                            m->proto))
1419     {
1420       nat_log_err ("sm o2i key del failed");
1421       return VNET_API_ERROR_UNSPECIFIED;
1422     }
1423
1424   pool_foreach (local, m->locals)
1425     {
1426       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1427       if (!is_sm_out2in_only (flags))
1428         {
1429           if (nat44_ed_sm_i2o_del (sm, local->addr, local->port,
1430                                    local->fib_index, m->proto))
1431             {
1432               nat_log_err ("sm i2o key del failed");
1433               return VNET_API_ERROR_UNSPECIFIED;
1434             }
1435         }
1436
1437       if (sm->num_workers > 1)
1438         {
1439           ip4_header_t ip = {
1440             .src_address = local->addr,
1441           };
1442           tsm = vec_elt_at_index (
1443             sm->per_thread_data,
1444             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1445         }
1446       else
1447         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1448
1449       /* Delete sessions */
1450       pool_foreach (s, tsm->sessions)
1451         {
1452           if (!(nat44_ed_is_lb_session (s)))
1453             continue;
1454
1455           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1456               s->in2out.port != local->port)
1457             continue;
1458
1459           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1460           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1461         }
1462     }
1463
1464   if (m->affinity)
1465     {
1466       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1467     }
1468
1469   pool_free (m->locals);
1470   vec_free (m->tag);
1471   vec_free (m->workers);
1472   pool_put (sm->static_mappings, m);
1473
1474   return 0;
1475 }
1476
1477 int
1478 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1479                                           ip4_address_t l_addr, u16 l_port,
1480                                           ip_protocol_t proto, u32 vrf_id,
1481                                           u8 probability, u8 is_add)
1482 {
1483   snat_main_t *sm = &snat_main;
1484   snat_static_mapping_t *m = 0;
1485   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1486   snat_main_per_thread_data_t *tsm;
1487   snat_session_t *s;
1488   u32 *locals = 0;
1489   uword *bitmap = 0;
1490   int i;
1491
1492   if (!sm->enabled)
1493     {
1494       return VNET_API_ERROR_UNSUPPORTED;
1495     }
1496
1497   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1498
1499   if (!m)
1500     {
1501       return VNET_API_ERROR_NO_SUCH_ENTRY;
1502     }
1503
1504   if (!is_sm_lb (m->flags))
1505     {
1506       return VNET_API_ERROR_INVALID_VALUE;
1507     }
1508
1509   pool_foreach (local, m->locals)
1510    {
1511     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1512         (local->vrf_id == vrf_id))
1513       {
1514         match_local = local;
1515         break;
1516       }
1517   }
1518
1519   if (is_add)
1520     {
1521       if (match_local)
1522         {
1523           return VNET_API_ERROR_VALUE_EXIST;
1524         }
1525
1526       pool_get (m->locals, local);
1527       clib_memset (local, 0, sizeof (*local));
1528       local->addr.as_u32 = l_addr.as_u32;
1529       local->port = l_port;
1530       local->probability = probability;
1531       local->vrf_id = vrf_id;
1532       local->fib_index =
1533         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1534                                            sm->fib_src_low);
1535
1536       if (!is_sm_out2in_only (m->flags))
1537         {
1538           if (nat44_ed_sm_i2o_add (sm, m, l_addr, l_port, local->fib_index,
1539                                    proto))
1540             {
1541               nat_log_err ("sm i2o key add failed");
1542               pool_put (m->locals, local);
1543               return VNET_API_ERROR_UNSPECIFIED;
1544             }
1545         }
1546     }
1547   else
1548     {
1549       if (!match_local)
1550         return VNET_API_ERROR_NO_SUCH_ENTRY;
1551
1552       if (pool_elts (m->locals) < 3)
1553         return VNET_API_ERROR_UNSPECIFIED;
1554
1555       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1556                         sm->fib_src_low);
1557
1558       if (!is_sm_out2in_only (m->flags))
1559         {
1560           if (nat44_ed_sm_i2o_del (sm, l_addr, l_port, match_local->fib_index,
1561                                    proto))
1562             nat_log_err ("sm i2o key del failed");
1563         }
1564
1565       if (sm->num_workers > 1)
1566         {
1567           ip4_header_t ip = {
1568             .src_address = local->addr,
1569           };
1570           tsm = vec_elt_at_index (
1571             sm->per_thread_data,
1572             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1573         }
1574       else
1575         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1576
1577       /* Delete sessions */
1578       pool_foreach (s, tsm->sessions) {
1579           if (!(nat44_ed_is_lb_session (s)))
1580             continue;
1581
1582           if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1583               s->in2out.port != match_local->port)
1584             continue;
1585
1586           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1587           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1588       }
1589
1590       pool_put (m->locals, match_local);
1591     }
1592
1593   vec_free (m->workers);
1594
1595   pool_foreach (local, m->locals)
1596    {
1597     vec_add1 (locals, local - m->locals);
1598     if (sm->num_workers > 1)
1599       {
1600         ip4_header_t ip;
1601         ip.src_address.as_u32 = local->addr.as_u32,
1602         bitmap = clib_bitmap_set (
1603           bitmap,
1604           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1605       }
1606   }
1607
1608   ASSERT (vec_len (locals) > 1);
1609
1610   local = pool_elt_at_index (m->locals, locals[0]);
1611   local->prefix = local->probability;
1612   for (i = 1; i < vec_len (locals); i++)
1613     {
1614       local = pool_elt_at_index (m->locals, locals[i]);
1615       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1616       local->prefix = local->probability + prev_local->prefix;
1617     }
1618
1619   /* Assign workers */
1620   if (sm->num_workers > 1)
1621     {
1622       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1623     }
1624
1625   return 0;
1626 }
1627
1628 void
1629 expire_per_vrf_sessions (u32 fib_index)
1630 {
1631   per_vrf_sessions_t *per_vrf_sessions;
1632   snat_main_per_thread_data_t *tsm;
1633   snat_main_t *sm = &snat_main;
1634
1635   vec_foreach (tsm, sm->per_thread_data)
1636     {
1637       pool_foreach (per_vrf_sessions, tsm->per_vrf_sessions_pool)
1638         {
1639           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1640               (per_vrf_sessions->tx_fib_index == fib_index))
1641             {
1642               per_vrf_sessions->expired = 1;
1643             }
1644         }
1645     }
1646 }
1647
1648 void
1649 update_per_vrf_sessions_pool (u32 fib_index, int is_del)
1650 {
1651   snat_main_t *sm = &snat_main;
1652   nat_fib_t *fib;
1653
1654   // we don't care if it is outside/inside fib
1655   // we just care about their ref_count
1656   // if it reaches 0 sessions should expire
1657   // because the fib isn't valid for NAT anymore
1658
1659   vec_foreach (fib, sm->fibs)
1660   {
1661     if (fib->fib_index == fib_index)
1662       {
1663         if (is_del)
1664           {
1665             fib->ref_count--;
1666             if (!fib->ref_count)
1667               {
1668                 vec_del1 (sm->fibs, fib - sm->fibs);
1669                 expire_per_vrf_sessions (fib_index);
1670               }
1671             return;
1672           }
1673         else
1674           fib->ref_count++;
1675       }
1676   }
1677   if (!is_del)
1678     {
1679       vec_add2 (sm->fibs, fib, 1);
1680       fib->ref_count = 1;
1681       fib->fib_index = fib_index;
1682     }
1683 }
1684
1685 static_always_inline nat_fib_t *
1686 nat44_ed_get_outside_fib (nat_fib_t *outside_fibs, u32 fib_index)
1687 {
1688   nat_fib_t *f;
1689   vec_foreach (f, outside_fibs)
1690     {
1691       if (f->fib_index == fib_index)
1692         {
1693           return f;
1694         }
1695     }
1696   return 0;
1697 }
1698
1699 static_always_inline snat_interface_t *
1700 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1701 {
1702   snat_interface_t *i;
1703   pool_foreach (i, interfaces)
1704     {
1705       if (i->sw_if_index == sw_if_index)
1706         {
1707           return i;
1708         }
1709     }
1710   return 0;
1711 }
1712
1713 int
1714 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1715 {
1716   const char *del_feature_name, *feature_name;
1717   snat_main_t *sm = &snat_main;
1718
1719   nat_fib_t *outside_fib;
1720   snat_interface_t *i;
1721   u32 fib_index;
1722   int rv;
1723
1724   if (!sm->enabled)
1725     {
1726       nat_log_err ("nat44 is disabled");
1727       return VNET_API_ERROR_UNSUPPORTED;
1728     }
1729
1730   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1731     {
1732       nat_log_err ("error interface already configured");
1733       return VNET_API_ERROR_VALUE_EXIST;
1734     }
1735
1736   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1737   if (i)
1738     {
1739       if ((nat44_ed_is_interface_inside (i) && is_inside) ||
1740           (nat44_ed_is_interface_outside (i) && !is_inside))
1741         {
1742           return 0;
1743         }
1744       if (sm->num_workers > 1)
1745         {
1746           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1747                                           "nat44-out2in-worker-handoff";
1748           feature_name = "nat44-handoff-classify";
1749         }
1750       else
1751         {
1752           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1753
1754           feature_name = "nat44-ed-classify";
1755         }
1756
1757       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1758       if (rv)
1759         return rv;
1760       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1761                                    sw_if_index, 0, 0, 0);
1762       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1763                                    0, 0);
1764     }
1765   else
1766     {
1767       if (sm->num_workers > 1)
1768         {
1769           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1770                                      "nat44-out2in-worker-handoff";
1771         }
1772       else
1773         {
1774           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1775         }
1776
1777       nat_validate_interface_counters (sm, sw_if_index);
1778       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1779       if (rv)
1780         return rv;
1781       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1782                                    0, 0);
1783
1784       pool_get (sm->interfaces, i);
1785       i->sw_if_index = sw_if_index;
1786       i->flags = 0;
1787     }
1788
1789   fib_index =
1790     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1791
1792   update_per_vrf_sessions_pool (fib_index, 0 /*is_del*/);
1793
1794   if (!is_inside)
1795     {
1796       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1797
1798       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1799       if (outside_fib)
1800         {
1801           outside_fib->ref_count++;
1802         }
1803       else
1804         {
1805           vec_add2 (sm->outside_fibs, outside_fib, 1);
1806           outside_fib->fib_index = fib_index;
1807           outside_fib->ref_count = 1;
1808         }
1809
1810       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
1811       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
1812
1813       nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
1814     }
1815   else
1816     {
1817       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1818     }
1819
1820   return 0;
1821 }
1822
1823 int
1824 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1825 {
1826   const char *del_feature_name, *feature_name;
1827   snat_main_t *sm = &snat_main;
1828
1829   nat_fib_t *outside_fib;
1830   snat_interface_t *i;
1831   u32 fib_index;
1832   int rv;
1833
1834   if (!sm->enabled)
1835     {
1836       nat_log_err ("nat44 is disabled");
1837       return VNET_API_ERROR_UNSUPPORTED;
1838     }
1839
1840   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1841   if (i == 0)
1842     {
1843       nat_log_err ("error interface couldn't be found");
1844       return VNET_API_ERROR_NO_SUCH_ENTRY;
1845     }
1846
1847   if (nat44_ed_is_interface_inside (i) && nat44_ed_is_interface_outside (i))
1848     {
1849       if (sm->num_workers > 1)
1850         {
1851           del_feature_name = "nat44-handoff-classify";
1852           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1853                                       "nat44-out2in-worker-handoff";
1854         }
1855       else
1856         {
1857           del_feature_name = "nat44-ed-classify";
1858           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1859         }
1860
1861       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1862       if (rv)
1863         {
1864           return rv;
1865         }
1866       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1867                                    sw_if_index, 0, 0, 0);
1868       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1869                                    0, 0);
1870
1871       if (is_inside)
1872         {
1873           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1874         }
1875       else
1876         {
1877           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1878         }
1879     }
1880   else
1881     {
1882       if (sm->num_workers > 1)
1883         {
1884           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1885                                      "nat44-out2in-worker-handoff";
1886         }
1887       else
1888         {
1889           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1890         }
1891
1892       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1893       if (rv)
1894         {
1895           return rv;
1896         }
1897       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1898                                    0, 0);
1899
1900       // remove interface
1901       pool_put (sm->interfaces, i);
1902     }
1903
1904   fib_index =
1905     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1906
1907   update_per_vrf_sessions_pool (fib_index, 1 /*is_del*/);
1908
1909   if (!is_inside)
1910     {
1911       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1912       if (outside_fib)
1913         {
1914           outside_fib->ref_count--;
1915           if (!outside_fib->ref_count)
1916             {
1917               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1918             }
1919         }
1920
1921       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
1922       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
1923     }
1924
1925   return 0;
1926 }
1927
1928 int
1929 nat44_ed_add_output_interface (u32 sw_if_index)
1930 {
1931   snat_main_t *sm = &snat_main;
1932
1933   nat_fib_t *outside_fib;
1934   snat_interface_t *i;
1935   u32 fib_index;
1936   int rv;
1937
1938   if (!sm->enabled)
1939     {
1940       nat_log_err ("nat44 is disabled");
1941       return VNET_API_ERROR_UNSUPPORTED;
1942     }
1943
1944   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1945     {
1946       nat_log_err ("error interface already configured");
1947       return VNET_API_ERROR_VALUE_EXIST;
1948     }
1949
1950   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1951     {
1952       nat_log_err ("error interface already configured");
1953       return VNET_API_ERROR_VALUE_EXIST;
1954     }
1955
1956   if (sm->num_workers > 1)
1957     {
1958       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1959       if (rv)
1960         {
1961           return rv;
1962         }
1963
1964       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1965       if (rv)
1966         {
1967           return rv;
1968         }
1969
1970       vnet_feature_enable_disable (
1971         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1972       vnet_feature_enable_disable ("ip4-output",
1973                                    "nat44-in2out-output-worker-handoff",
1974                                    sw_if_index, 1, 0, 0);
1975     }
1976   else
1977     {
1978       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1979       if (rv)
1980         {
1981           return rv;
1982         }
1983
1984       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1985       if (rv)
1986         {
1987           return rv;
1988         }
1989
1990       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1991                                    sw_if_index, 1, 0, 0);
1992       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1993                                    sw_if_index, 1, 0, 0);
1994     }
1995
1996   nat_validate_interface_counters (sm, sw_if_index);
1997
1998   pool_get (sm->output_feature_interfaces, i);
1999   i->sw_if_index = sw_if_index;
2000   i->flags = 0;
2001   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2002   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2003
2004   fib_index =
2005     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2006   update_per_vrf_sessions_pool (fib_index, 0 /*is_del*/);
2007
2008   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2009   if (outside_fib)
2010     {
2011       outside_fib->ref_count++;
2012     }
2013   else
2014     {
2015       vec_add2 (sm->outside_fibs, outside_fib, 1);
2016       outside_fib->fib_index = fib_index;
2017       outside_fib->ref_count = 1;
2018     }
2019
2020   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
2021   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
2022
2023   nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
2024
2025   return 0;
2026 }
2027
2028 int
2029 nat44_ed_del_output_interface (u32 sw_if_index)
2030 {
2031   snat_main_t *sm = &snat_main;
2032
2033   nat_fib_t *outside_fib;
2034   snat_interface_t *i;
2035   u32 fib_index;
2036   int rv;
2037
2038   if (!sm->enabled)
2039     {
2040       nat_log_err ("nat44 is disabled");
2041       return VNET_API_ERROR_UNSUPPORTED;
2042     }
2043
2044   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2045   if (!i)
2046     {
2047       nat_log_err ("error interface couldn't be found");
2048       return VNET_API_ERROR_NO_SUCH_ENTRY;
2049     }
2050
2051   if (sm->num_workers > 1)
2052     {
2053       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2054       if (rv)
2055         {
2056           return rv;
2057         }
2058
2059       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2060       if (rv)
2061         {
2062           return rv;
2063         }
2064
2065       vnet_feature_enable_disable (
2066         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
2067       vnet_feature_enable_disable ("ip4-output",
2068                                    "nat44-in2out-output-worker-handoff",
2069                                    sw_if_index, 0, 0, 0);
2070     }
2071   else
2072     {
2073       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2074       if (rv)
2075         {
2076           return rv;
2077         }
2078
2079       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2080       if (rv)
2081         {
2082           return rv;
2083         }
2084
2085       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2086                                    sw_if_index, 0, 0, 0);
2087       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2088                                    sw_if_index, 0, 0, 0);
2089     }
2090
2091   // remove interface
2092   pool_put (sm->output_feature_interfaces, i);
2093
2094   fib_index =
2095     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2096   update_per_vrf_sessions_pool (fib_index, 1 /*is_del*/);
2097
2098   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2099   if (outside_fib)
2100     {
2101       outside_fib->ref_count--;
2102       if (!outside_fib->ref_count)
2103         {
2104           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2105         }
2106     }
2107
2108   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
2109   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
2110
2111   return 0;
2112 }
2113
2114 int
2115 snat_set_workers (uword * bitmap)
2116 {
2117   snat_main_t *sm = &snat_main;
2118   int i, j = 0;
2119
2120   if (sm->num_workers < 2)
2121     return VNET_API_ERROR_FEATURE_DISABLED;
2122
2123   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2124     return VNET_API_ERROR_INVALID_WORKER;
2125
2126   vec_free (sm->workers);
2127   clib_bitmap_foreach (i, bitmap)
2128     {
2129       vec_add1(sm->workers, i);
2130       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2131       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2132       j++;
2133     }
2134
2135   sm->port_per_thread = (65536 - ED_USER_PORT_OFFSET) / _vec_len (sm->workers);
2136
2137   return 0;
2138 }
2139
2140 int
2141 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2142 {
2143   fail_if_enabled ();
2144   snat_main_t *sm = &snat_main;
2145
2146   if ((sm->fq_in2out_index != ~0) || (sm->fq_out2in_index != ~0) ||
2147       (sm->fq_in2out_output_index != ~0))
2148     {
2149       // frame queu nelts can be set only before first
2150       // call to nat44_plugin_enable after that it
2151       // doesn't make sense
2152       nat_log_err ("Frame queue was already initialized. "
2153                    "Change is not possible");
2154       return 1;
2155     }
2156
2157   sm->frame_queue_nelts = frame_queue_nelts;
2158   return 0;
2159 }
2160
2161 static void
2162 nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque, u32 sw_if_index,
2163                                 u32 new_fib_index, u32 old_fib_index)
2164 {
2165   snat_main_t *sm = &snat_main;
2166   nat_fib_t *outside_fib;
2167   snat_interface_t *i;
2168   u8 is_add = 1;
2169   u8 match = 0;
2170
2171   if (!sm->enabled || (new_fib_index == old_fib_index)
2172       || (!vec_len (sm->outside_fibs)))
2173     {
2174       return;
2175     }
2176
2177   pool_foreach (i, sm->interfaces)
2178     {
2179       if (i->sw_if_index == sw_if_index)
2180         {
2181           if (!(nat44_ed_is_interface_outside (i)))
2182             return;
2183           match = 1;
2184         }
2185     }
2186
2187   pool_foreach (i, sm->output_feature_interfaces)
2188     {
2189       if (i->sw_if_index == sw_if_index)
2190         {
2191           if (!(nat44_ed_is_interface_outside (i)))
2192             return;
2193           match = 1;
2194         }
2195     }
2196
2197   if (!match)
2198     return;
2199
2200   vec_foreach (outside_fib, sm->outside_fibs)
2201     {
2202       if (outside_fib->fib_index == old_fib_index)
2203         {
2204           outside_fib->ref_count--;
2205           if (!outside_fib->ref_count)
2206             vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2207           break;
2208         }
2209     }
2210
2211   vec_foreach (outside_fib, sm->outside_fibs)
2212     {
2213       if (outside_fib->fib_index == new_fib_index)
2214         {
2215           outside_fib->ref_count++;
2216           is_add = 0;
2217           break;
2218         }
2219     }
2220
2221   if (is_add)
2222     {
2223       vec_add2 (sm->outside_fibs, outside_fib, 1);
2224       outside_fib->ref_count = 1;
2225       outside_fib->fib_index = new_fib_index;
2226     }
2227 }
2228
2229 static void nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque,
2230                                             u32 sw_if_index, u32 new_fib_index,
2231                                             u32 old_fib_index);
2232
2233 static void nat44_ed_add_del_interface_address_cb (
2234   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2235   u32 address_length, u32 if_address_index, u32 is_delete);
2236
2237 static void nat44_ed_add_del_static_mapping_cb (
2238   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2239   u32 address_length, u32 if_address_index, u32 is_delete);
2240
2241 void
2242 test_key_calc_split ()
2243 {
2244   ip4_address_t l_addr;
2245   l_addr.as_u8[0] = 1;
2246   l_addr.as_u8[1] = 1;
2247   l_addr.as_u8[2] = 1;
2248   l_addr.as_u8[3] = 1;
2249   ip4_address_t r_addr;
2250   r_addr.as_u8[0] = 2;
2251   r_addr.as_u8[1] = 2;
2252   r_addr.as_u8[2] = 2;
2253   r_addr.as_u8[3] = 2;
2254   u16 l_port = 40001;
2255   u16 r_port = 40301;
2256   u8 proto = 9;
2257   u32 fib_index = 9000001;
2258   u32 thread_index = 3000000001;
2259   u32 session_index = 3000000221;
2260   clib_bihash_kv_16_8_t kv;
2261   init_ed_kv (&kv, l_addr.as_u32, l_port, r_addr.as_u32, r_port, fib_index,
2262               proto, thread_index, session_index);
2263   ip4_address_t l_addr2;
2264   ip4_address_t r_addr2;
2265   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2266   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2267   u16 l_port2 = 0;
2268   u16 r_port2 = 0;
2269   u8 proto2 = 0;
2270   u32 fib_index2 = 0;
2271   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2272                &r_port2);
2273   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2274   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2275   ASSERT (l_port == l_port2);
2276   ASSERT (r_port == r_port2);
2277   ASSERT (proto == proto2);
2278   ASSERT (fib_index == fib_index2);
2279   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2280   ASSERT (session_index == ed_value_get_session_index (&kv));
2281 }
2282
2283 static clib_error_t *
2284 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2285 {
2286   u32 fib_index;
2287   if (!is_add)
2288     {
2289       fib_index = ip4_fib_index_from_table_id (table_id);
2290       if (fib_index != ~0)
2291         {
2292           expire_per_vrf_sessions (fib_index);
2293         }
2294     }
2295   return 0;
2296 }
2297
2298 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2299
2300 #define nat_validate_simple_counter(c, i)                                     \
2301   do                                                                          \
2302     {                                                                         \
2303       vlib_validate_simple_counter (&c, i);                                   \
2304       vlib_zero_simple_counter (&c, i);                                       \
2305     }                                                                         \
2306   while (0);
2307
2308 #define nat_init_simple_counter(c, n, sn)                                     \
2309   do                                                                          \
2310     {                                                                         \
2311       c.name = n;                                                             \
2312       c.stat_segment_name = sn;                                               \
2313       nat_validate_simple_counter (c, 0);                                     \
2314     }                                                                         \
2315   while (0);
2316
2317 static_always_inline void
2318 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2319 {
2320 #define _(x)                                                                  \
2321   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2322   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2323   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2324   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2325   foreach_nat_counter;
2326 #undef _
2327   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2328 }
2329
2330 static clib_error_t *
2331 nat_init (vlib_main_t * vm)
2332 {
2333   snat_main_t *sm = &snat_main;
2334   vlib_thread_main_t *tm = vlib_get_thread_main ();
2335   vlib_thread_registration_t *tr;
2336   ip4_add_del_interface_address_callback_t cbi = { 0 };
2337   ip4_table_bind_callback_t cbt = { 0 };
2338   u32 i, num_threads = 0;
2339   uword *p, *bitmap = 0;
2340
2341   clib_memset (sm, 0, sizeof (*sm));
2342
2343   // convenience
2344   sm->ip4_main = &ip4_main;
2345
2346   // frame queue indices used for handoff
2347   sm->fq_out2in_index = ~0;
2348   sm->fq_in2out_index = ~0;
2349   sm->fq_in2out_output_index = ~0;
2350
2351   sm->log_level = NAT_LOG_ERROR;
2352
2353   sm->log_class = vlib_log_register_class ("nat", 0);
2354   nat_ipfix_logging_init (vm);
2355
2356   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2357                            "/nat44-ed/total-sessions");
2358   sm->max_cfg_sessions_gauge =
2359     vlib_stats_add_gauge ("/nat44-ed/max-cfg-sessions");
2360
2361 #define _(x)                                                                  \
2362   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2363                            "/nat44-ed/in2out/fastpath/" #x);                  \
2364   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2365                            "/nat44-ed/out2in/fastpath/" #x);                  \
2366   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2367                            "/nat44-ed/in2out/slowpath/" #x);                  \
2368   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2369                            "/nat44-ed/out2in/slowpath/" #x);
2370   foreach_nat_counter;
2371 #undef _
2372   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2373                            "/nat44-ed/hairpinning");
2374
2375   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2376   if (p)
2377     {
2378       tr = (vlib_thread_registration_t *) p[0];
2379       if (tr)
2380         {
2381           sm->num_workers = tr->count;
2382           sm->first_worker_index = tr->first_index;
2383         }
2384     }
2385   num_threads = tm->n_vlib_mains - 1;
2386   sm->port_per_thread = 65536 - ED_USER_PORT_OFFSET;
2387   vec_validate (sm->per_thread_data, num_threads);
2388
2389   /* Use all available workers by default */
2390   if (sm->num_workers > 1)
2391     {
2392       for (i = 0; i < sm->num_workers; i++)
2393         bitmap = clib_bitmap_set (bitmap, i, 1);
2394       snat_set_workers (bitmap);
2395       clib_bitmap_free (bitmap);
2396     }
2397   else
2398     {
2399       sm->per_thread_data[0].snat_thread_index = 0;
2400     }
2401
2402   /* callbacks to call when interface address changes. */
2403   cbi.function = nat44_ed_add_del_interface_address_cb;
2404   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2405   cbi.function = nat44_ed_add_del_static_mapping_cb;
2406   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2407
2408   /* callbacks to call when interface to table biding changes */
2409   cbt.function = nat44_ed_update_outside_fib_cb;
2410   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2411
2412   sm->fib_src_low =
2413     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2414                          FIB_SOURCE_BH_SIMPLE);
2415   sm->fib_src_hi =
2416     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2417                          FIB_SOURCE_BH_SIMPLE);
2418
2419   nat_affinity_init (vm);
2420   test_key_calc_split ();
2421
2422   return nat44_api_hookup (vm);
2423 }
2424
2425 VLIB_INIT_FUNCTION (nat_init);
2426
2427 int
2428 nat44_plugin_enable (nat44_config_t c)
2429 {
2430   snat_main_t *sm = &snat_main;
2431
2432   fail_if_enabled ();
2433
2434   sm->forwarding_enabled = 0;
2435   sm->mss_clamping = 0;
2436
2437   if (!c.sessions)
2438     c.sessions = 63 * 1024;
2439
2440   sm->max_translations_per_thread = c.sessions;
2441   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
2442                         sm->max_translations_per_thread);
2443   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2444
2445   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2446
2447   sm->inside_vrf_id = c.inside_vrf;
2448   sm->inside_fib_index =
2449     fib_table_find_or_create_and_lock
2450     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2451
2452   sm->outside_vrf_id = c.outside_vrf;
2453   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2454     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2455
2456   nat44_ed_db_init ();
2457
2458   nat_affinity_enable ();
2459
2460   nat_reset_timeouts (&sm->timeouts);
2461
2462   vlib_zero_simple_counter (&sm->total_sessions, 0);
2463
2464   if (!sm->frame_queue_nelts)
2465     {
2466       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2467     }
2468
2469   if (sm->num_workers > 1)
2470     {
2471       vlib_main_t *vm = vlib_get_main ();
2472       vlib_node_t *node;
2473
2474       if (sm->fq_in2out_index == ~0)
2475         {
2476           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2477           sm->fq_in2out_index =
2478             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2479         }
2480       if (sm->fq_out2in_index == ~0)
2481         {
2482           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2483           sm->fq_out2in_index =
2484             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2485         }
2486       if (sm->fq_in2out_output_index == ~0)
2487         {
2488           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2489           sm->fq_in2out_output_index =
2490             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2491         }
2492     }
2493
2494   sm->enabled = 1;
2495   sm->rconfig = c;
2496
2497   return 0;
2498 }
2499
2500 int
2501 nat44_ed_del_addresses ()
2502 {
2503   snat_main_t *sm = &snat_main;
2504   snat_address_t *a, *vec;
2505   int error = 0;
2506
2507   vec = vec_dup (sm->addresses);
2508   vec_foreach (a, vec)
2509     {
2510       error = nat44_ed_del_address (a->addr, 0);
2511       if (error)
2512         {
2513           nat_log_err ("error occurred while removing adderess");
2514         }
2515     }
2516   vec_free (vec);
2517   vec_free (sm->addresses);
2518   sm->addresses = 0;
2519
2520   vec = vec_dup (sm->twice_nat_addresses);
2521   vec_foreach (a, vec)
2522     {
2523       error = nat44_ed_del_address (a->addr, 1);
2524       if (error)
2525         {
2526           nat_log_err ("error occurred while removing adderess");
2527         }
2528     }
2529   vec_free (vec);
2530   vec_free (sm->twice_nat_addresses);
2531   sm->twice_nat_addresses = 0;
2532
2533   vec_free (sm->addr_to_resolve);
2534   sm->addr_to_resolve = 0;
2535
2536   return error;
2537 }
2538
2539 int
2540 nat44_ed_del_interfaces ()
2541 {
2542   snat_main_t *sm = &snat_main;
2543   snat_interface_t *i, *pool;
2544   int error = 0;
2545
2546   pool = pool_dup (sm->interfaces);
2547   pool_foreach (i, pool)
2548     {
2549       if (nat44_ed_is_interface_inside (i))
2550         {
2551           error = nat44_ed_del_interface (i->sw_if_index, 1);
2552         }
2553       if (nat44_ed_is_interface_outside (i))
2554         {
2555           error = nat44_ed_del_interface (i->sw_if_index, 0);
2556         }
2557
2558       if (error)
2559         {
2560           nat_log_err ("error occurred while removing interface");
2561         }
2562     }
2563   pool_free (pool);
2564   pool_free (sm->interfaces);
2565   sm->interfaces = 0;
2566   return error;
2567 }
2568
2569 int
2570 nat44_ed_del_output_interfaces ()
2571 {
2572   snat_main_t *sm = &snat_main;
2573   snat_interface_t *i, *pool;
2574   int error = 0;
2575
2576   pool = pool_dup (sm->output_feature_interfaces);
2577   pool_foreach (i, pool)
2578     {
2579       error = nat44_ed_del_output_interface (i->sw_if_index);
2580       if (error)
2581         {
2582           nat_log_err ("error occurred while removing output interface");
2583         }
2584     }
2585   pool_free (pool);
2586   pool_free (sm->output_feature_interfaces);
2587   sm->output_feature_interfaces = 0;
2588   return error;
2589 }
2590
2591 static clib_error_t *
2592 nat44_ed_sw_interface_add_del (vnet_main_t *vnm, u32 sw_if_index, u32 is_add)
2593 {
2594   snat_main_t *sm = &snat_main;
2595   snat_interface_t *i;
2596   int error = 0;
2597
2598   if (is_add)
2599     return 0;
2600
2601   if (!sm->enabled)
2602     return 0;
2603
2604   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
2605   if (i)
2606     {
2607       bool is_inside = nat44_ed_is_interface_inside (i);
2608       bool is_outside = nat44_ed_is_interface_outside (i);
2609
2610       if (is_inside)
2611         {
2612           error |= nat44_ed_del_interface (sw_if_index, 1);
2613         }
2614       if (is_outside)
2615         {
2616           error |= nat44_ed_del_interface (sw_if_index, 0);
2617         }
2618
2619       if (error)
2620         {
2621           nat_log_err ("error occurred while removing interface");
2622         }
2623     }
2624
2625   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2626   if (i)
2627     {
2628       error = nat44_ed_del_output_interface (sw_if_index);
2629       if (error)
2630         {
2631           nat_log_err ("error occurred while removing output interface");
2632         }
2633     }
2634
2635   return 0;
2636 }
2637
2638 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (nat44_ed_sw_interface_add_del);
2639
2640 int
2641 nat44_ed_del_static_mappings ()
2642 {
2643   snat_main_t *sm = &snat_main;
2644   snat_static_mapping_t *m, *pool;
2645   int error = 0;
2646
2647   pool = pool_dup (sm->static_mappings);
2648   pool_foreach (m, pool)
2649     {
2650       error = nat44_ed_del_static_mapping_internal (
2651         m->local_addr, m->external_addr, m->local_port, m->external_port,
2652         m->proto, m->vrf_id, m->flags);
2653       if (error)
2654         {
2655           nat_log_err ("error occurred while removing mapping");
2656         }
2657     }
2658   pool_free (pool);
2659   pool_free (sm->static_mappings);
2660   sm->static_mappings = 0;
2661
2662   vec_free (sm->sm_to_resolve);
2663   sm->sm_to_resolve = 0;
2664
2665   return error;
2666 }
2667
2668 int
2669 nat44_plugin_disable ()
2670 {
2671   snat_main_t *sm = &snat_main;
2672   int rc, error = 0;
2673
2674   fail_if_disabled ();
2675
2676   rc = nat44_ed_del_static_mappings ();
2677   if (rc)
2678     error = VNET_API_ERROR_BUG;
2679
2680   rc = nat44_ed_del_addresses ();
2681   if (rc)
2682     error = VNET_API_ERROR_BUG;
2683
2684   rc = nat44_ed_del_interfaces ();
2685   if (rc)
2686     error = VNET_API_ERROR_BUG;
2687
2688   rc = nat44_ed_del_output_interfaces ();
2689   if (rc)
2690     error = VNET_API_ERROR_BUG;
2691
2692   nat44_ed_del_vrf_tables ();
2693
2694   vec_free (sm->max_translations_per_fib);
2695   sm->max_translations_per_fib = 0;
2696
2697   nat44_ed_db_free ();
2698
2699   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2700
2701   nat_affinity_disable ();
2702
2703   sm->forwarding_enabled = 0;
2704   sm->enabled = 0;
2705
2706   return error;
2707 }
2708
2709 void
2710 nat44_ed_forwarding_enable_disable (u8 is_enable)
2711 {
2712   snat_main_per_thread_data_t *tsm;
2713   snat_main_t *sm = &snat_main;
2714   snat_session_t *s;
2715
2716   u32 *ses_to_be_removed = 0, *ses_index;
2717
2718   sm->forwarding_enabled = is_enable != 0;
2719
2720   if (!sm->enabled || is_enable)
2721     {
2722       return;
2723     }
2724
2725   vec_foreach (tsm, sm->per_thread_data)
2726     {
2727       pool_foreach (s, tsm->sessions)
2728         {
2729           if (na44_ed_is_fwd_bypass_session (s))
2730             {
2731               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2732             }
2733         }
2734       vec_foreach (ses_index, ses_to_be_removed)
2735         {
2736           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2737           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2738           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2739         }
2740
2741       vec_free (ses_to_be_removed);
2742     }
2743 }
2744
2745 static_always_inline snat_static_mapping_t *
2746 nat44_ed_sm_match (snat_main_t *sm, ip4_address_t match_addr, u16 match_port,
2747                    u32 match_fib_index, ip_protocol_t match_protocol,
2748                    int by_external)
2749 {
2750   snat_static_mapping_t *m;
2751   if (!by_external)
2752     {
2753       m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port, match_fib_index,
2754                                   match_protocol);
2755       if (m)
2756         return m;
2757
2758       // try address only mapping
2759       m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, match_fib_index, 0);
2760       if (m)
2761         return m;
2762
2763       // default static mapping fib index (based on configuration)
2764       if (sm->inside_fib_index != match_fib_index)
2765         {
2766           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2767                                       sm->inside_fib_index, match_protocol);
2768           if (m)
2769             return m;
2770
2771           // try address only mapping
2772           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->inside_fib_index,
2773                                       0);
2774           if (m)
2775             return m;
2776         }
2777       // TODO: this specific use case may be deprecated (needs testing)
2778       if (sm->outside_fib_index != match_fib_index)
2779         {
2780           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2781                                       sm->outside_fib_index, match_protocol);
2782           if (m)
2783             return m;
2784
2785           // try address only mapping
2786           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->outside_fib_index,
2787                                       0);
2788           if (m)
2789             return m;
2790         }
2791     }
2792   else
2793     {
2794       m =
2795         nat44_ed_sm_o2i_lookup (sm, match_addr, match_port, 0, match_protocol);
2796       if (m)
2797         return m;
2798
2799       // try address only mapping
2800       m = nat44_ed_sm_o2i_lookup (sm, match_addr, 0, 0, 0);
2801       if (m)
2802         return m;
2803     }
2804   return 0;
2805 }
2806
2807 int
2808 snat_static_mapping_match (vlib_main_t *vm, ip4_address_t match_addr,
2809                            u16 match_port, u32 match_fib_index,
2810                            ip_protocol_t match_protocol,
2811                            ip4_address_t *mapping_addr, u16 *mapping_port,
2812                            u32 *mapping_fib_index, int by_external,
2813                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2814                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2815                            u8 *is_identity_nat, snat_static_mapping_t **out)
2816 {
2817   snat_main_t *sm = &snat_main;
2818   snat_static_mapping_t *m;
2819   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2820   nat44_lb_addr_port_t *local;
2821   u8 backend_index;
2822
2823   m = nat44_ed_sm_match (sm, match_addr, match_port, match_fib_index,
2824                          match_protocol, by_external);
2825   if (!m)
2826     {
2827       return 1;
2828     }
2829
2830   if (by_external)
2831     {
2832       if (is_sm_lb (m->flags))
2833         {
2834           if (PREDICT_FALSE (lb != 0))
2835             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2836           if (m->affinity && !nat_affinity_find_and_lock (
2837                                vm, ext_host_addr[0], match_addr,
2838                                match_protocol, match_port, &backend_index))
2839             {
2840               local = pool_elt_at_index (m->locals, backend_index);
2841               *mapping_addr = local->addr;
2842               *mapping_port = local->port;
2843               *mapping_fib_index = local->fib_index;
2844               goto end;
2845             }
2846           // pick locals matching this worker
2847           if (PREDICT_FALSE (sm->num_workers > 1))
2848             {
2849               u32 thread_index = vlib_get_thread_index ();
2850               pool_foreach_index (i, m->locals)
2851                {
2852                 local = pool_elt_at_index (m->locals, i);
2853
2854                 ip4_header_t ip = {
2855                   .src_address = local->addr,
2856                 };
2857
2858                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2859                                                       0) == thread_index)
2860                   {
2861                     vec_add1 (tmp, i);
2862                   }
2863                }
2864               ASSERT (vec_len (tmp) != 0);
2865             }
2866           else
2867             {
2868               pool_foreach_index (i, m->locals)
2869                {
2870                 vec_add1 (tmp, i);
2871               }
2872             }
2873           hi = vec_len (tmp) - 1;
2874           local = pool_elt_at_index (m->locals, tmp[hi]);
2875           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2876           while (lo < hi)
2877             {
2878               mid = ((hi - lo) >> 1) + lo;
2879               local = pool_elt_at_index (m->locals, tmp[mid]);
2880               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2881             }
2882           local = pool_elt_at_index (m->locals, tmp[lo]);
2883           if (!(local->prefix >= rand))
2884             return 1;
2885           *mapping_addr = local->addr;
2886           *mapping_port = local->port;
2887           *mapping_fib_index = local->fib_index;
2888           if (m->affinity)
2889             {
2890               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2891                                                 match_protocol, match_port,
2892                                                 tmp[lo], m->affinity,
2893                                                 m->affinity_per_service_list_head_index))
2894                 nat_elog_info (sm, "create affinity record failed");
2895             }
2896           vec_free (tmp);
2897         }
2898       else
2899         {
2900           if (PREDICT_FALSE (lb != 0))
2901             *lb = NO_LB_NAT;
2902           *mapping_fib_index = m->fib_index;
2903           *mapping_addr = m->local_addr;
2904           /* Address only mapping doesn't change port */
2905           *mapping_port =
2906             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2907         }
2908     }
2909   else
2910     {
2911       *mapping_addr = m->external_addr;
2912       /* Address only mapping doesn't change port */
2913       *mapping_port =
2914         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2915       *mapping_fib_index = sm->outside_fib_index;
2916     }
2917
2918 end:
2919   if (PREDICT_FALSE (is_addr_only != 0))
2920     *is_addr_only = is_sm_addr_only (m->flags);
2921
2922   if (PREDICT_FALSE (twice_nat != 0))
2923     {
2924       *twice_nat = TWICE_NAT_DISABLED;
2925
2926       if (is_sm_twice_nat (m->flags))
2927         {
2928           *twice_nat = TWICE_NAT;
2929         }
2930       else if (is_sm_self_twice_nat (m->flags))
2931         {
2932           *twice_nat = TWICE_NAT_SELF;
2933         }
2934     }
2935
2936   if (PREDICT_FALSE (is_identity_nat != 0))
2937     *is_identity_nat = is_sm_identity_nat (m->flags);
2938
2939   if (out != 0)
2940     *out = m;
2941
2942   return 0;
2943 }
2944
2945 u32
2946 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2947                                   u32 rx_fib_index, u8 is_output)
2948 {
2949   snat_main_t *sm = &snat_main;
2950   u32 next_worker_index = sm->first_worker_index;
2951   u32 hash;
2952
2953   clib_bihash_kv_16_8_t kv16, value16;
2954
2955   u32 fib_index = rx_fib_index;
2956   if (b)
2957     {
2958       if (PREDICT_FALSE (is_output))
2959         {
2960           fib_index = sm->outside_fib_index;
2961           nat_fib_t *outside_fib;
2962           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2963           fib_prefix_t pfx = {
2964                   .fp_proto = FIB_PROTOCOL_IP4,
2965                   .fp_len = 32,
2966                   .fp_addr = {
2967                           .ip4.as_u32 = ip->dst_address.as_u32,
2968                   } ,
2969           };
2970
2971           switch (vec_len (sm->outside_fibs))
2972             {
2973             case 0:
2974               fib_index = sm->outside_fib_index;
2975               break;
2976             case 1:
2977               fib_index = sm->outside_fibs[0].fib_index;
2978               break;
2979             default:
2980               vec_foreach (outside_fib, sm->outside_fibs)
2981                 {
2982                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2983                   if (FIB_NODE_INDEX_INVALID != fei)
2984                     {
2985                       if (fib_entry_get_resolving_interface (fei) != ~0)
2986                         {
2987                           fib_index = outside_fib->fib_index;
2988                           break;
2989                         }
2990                     }
2991                 }
2992               break;
2993             }
2994         }
2995
2996       if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2997         {
2998           ip4_address_t lookup_saddr, lookup_daddr;
2999           u16 lookup_sport, lookup_dport;
3000           u8 lookup_protocol;
3001
3002           if (!nat_get_icmp_session_lookup_values (
3003                 b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr,
3004                 &lookup_dport, &lookup_protocol))
3005             {
3006               init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3007                          lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3008                          lookup_protocol);
3009               if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3010                 {
3011                   next_worker_index = ed_value_get_thread_index (&value16);
3012                   vnet_buffer2 (b)->nat.cached_session_index =
3013                     ed_value_get_session_index (&value16);
3014                   goto out;
3015                 }
3016             }
3017         }
3018
3019       init_ed_k (&kv16, ip->src_address.as_u32,
3020                  vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3021                  vnet_buffer (b)->ip.reass.l4_dst_port, fib_index,
3022                  ip->protocol);
3023
3024       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3025         {
3026           next_worker_index = ed_value_get_thread_index (&value16);
3027           vnet_buffer2 (b)->nat.cached_session_index =
3028             ed_value_get_session_index (&value16);
3029           goto out;
3030         }
3031
3032       // dst NAT
3033       init_ed_k (&kv16, ip->dst_address.as_u32,
3034                  vnet_buffer (b)->ip.reass.l4_dst_port, ip->src_address.as_u32,
3035                  vnet_buffer (b)->ip.reass.l4_src_port, rx_fib_index,
3036                  ip->protocol);
3037       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3038         {
3039           next_worker_index = ed_value_get_thread_index (&value16);
3040           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
3041             ed_value_get_session_index (&value16);
3042           goto out;
3043         }
3044     }
3045
3046   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3047          (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24) +
3048          rx_fib_index + (rx_fib_index >> 8) + (rx_fib_index >> 16) +
3049          (rx_fib_index >> 24);
3050
3051   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3052     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3053   else
3054     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3055
3056 out:
3057   if (PREDICT_TRUE (!is_output))
3058     {
3059       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
3060                               rx_fib_index,
3061                               clib_net_to_host_u32 (ip->src_address.as_u32),
3062                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3063     }
3064   else
3065     {
3066       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
3067                               next_worker_index, rx_fib_index,
3068                               clib_net_to_host_u32 (ip->src_address.as_u32),
3069                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3070     }
3071
3072   return next_worker_index;
3073 }
3074
3075 u32
3076 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
3077                                   u32 rx_fib_index, u8 is_output)
3078 {
3079   snat_main_t *sm = &snat_main;
3080   clib_bihash_kv_16_8_t kv16, value16;
3081
3082   u8 proto, next_worker_index = 0;
3083   u16 port;
3084   snat_static_mapping_t *m;
3085   u32 hash;
3086
3087   proto = ip->protocol;
3088
3089   if (PREDICT_FALSE (IP_PROTOCOL_ICMP == proto))
3090     {
3091       ip4_address_t lookup_saddr, lookup_daddr;
3092       u16 lookup_sport, lookup_dport;
3093       u8 lookup_protocol;
3094       if (!nat_get_icmp_session_lookup_values (
3095             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3096             &lookup_protocol))
3097         {
3098           init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3099                      lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3100                      lookup_protocol);
3101           if (PREDICT_TRUE (
3102                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3103             {
3104               next_worker_index = ed_value_get_thread_index (&value16);
3105               nat_elog_debug_handoff (
3106                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
3107                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
3108                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3109               return next_worker_index;
3110             }
3111         }
3112     }
3113
3114   init_ed_k (&kv16, ip->src_address.as_u32,
3115              vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3116              vnet_buffer (b)->ip.reass.l4_dst_port, rx_fib_index,
3117              ip->protocol);
3118
3119   if (PREDICT_TRUE (
3120         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3121     {
3122       vnet_buffer2 (b)->nat.cached_session_index =
3123         ed_value_get_session_index (&value16);
3124       next_worker_index = ed_value_get_thread_index (&value16);
3125       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
3126                               next_worker_index, rx_fib_index,
3127                               clib_net_to_host_u32 (ip->src_address.as_u32),
3128                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3129       return next_worker_index;
3130     }
3131
3132   /* first try static mappings without port */
3133   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3134     {
3135       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, proto);
3136       if (m)
3137         {
3138           {
3139             next_worker_index = m->workers[0];
3140             goto done;
3141           }
3142         }
3143     }
3144
3145   /* unknown protocol */
3146   if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto)))
3147     {
3148       /* use current thread */
3149       next_worker_index = vlib_get_thread_index ();
3150       goto done;
3151     }
3152
3153   port = vnet_buffer (b)->ip.reass.l4_dst_port;
3154
3155   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3156     {
3157       udp_header_t *udp = ip4_next_header (ip);
3158       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3159       nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3160       if (!icmp_type_is_error_message
3161           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3162         port = vnet_buffer (b)->ip.reass.l4_src_port;
3163       else
3164         {
3165           /* if error message, then it's not fragmented and we can access it */
3166           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3167           proto = inner_ip->protocol;
3168           void *l4_header = ip4_next_header (inner_ip);
3169           switch (proto)
3170             {
3171             case IP_PROTOCOL_ICMP:
3172               icmp = (icmp46_header_t *) l4_header;
3173               echo = (nat_icmp_echo_header_t *) (icmp + 1);
3174               port = echo->identifier;
3175               break;
3176             case IP_PROTOCOL_UDP:
3177               /* breakthrough */
3178             case IP_PROTOCOL_TCP:
3179               port = ((nat_tcp_udp_header_t *) l4_header)->src_port;
3180               break;
3181             default:
3182               next_worker_index = vlib_get_thread_index ();
3183               goto done;
3184             }
3185         }
3186     }
3187
3188   /* try static mappings with port */
3189   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3190     {
3191       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, port, 0, proto);
3192       if (m)
3193         {
3194           if (!is_sm_lb (m->flags))
3195             {
3196               next_worker_index = m->workers[0];
3197               goto done;
3198             }
3199
3200           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3201             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3202
3203           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3204             next_worker_index =
3205               m->workers[hash & (_vec_len (m->workers) - 1)];
3206           else
3207             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3208           goto done;
3209         }
3210     }
3211
3212   /* worker by outside port */
3213   next_worker_index = get_thread_idx_by_port (clib_net_to_host_u16 (port));
3214
3215 done:
3216   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3217                           rx_fib_index,
3218                           clib_net_to_host_u32 (ip->src_address.as_u32),
3219                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3220   return next_worker_index;
3221 }
3222
3223 u32
3224 nat44_get_max_session_limit ()
3225 {
3226   snat_main_t *sm = &snat_main;
3227   u32 max_limit = 0, len = 0;
3228
3229   for (; len < vec_len (sm->max_translations_per_fib); len++)
3230     {
3231       if (max_limit < sm->max_translations_per_fib[len])
3232         max_limit = sm->max_translations_per_fib[len];
3233     }
3234   return max_limit;
3235 }
3236
3237 int
3238 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3239 {
3240   snat_main_t *sm = &snat_main;
3241   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3242   u32 len = vec_len (sm->max_translations_per_fib);
3243
3244   if (len <= fib_index)
3245     {
3246       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3247
3248       for (; len < vec_len (sm->max_translations_per_fib); len++)
3249         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3250     }
3251
3252   sm->max_translations_per_fib[fib_index] = session_limit;
3253   return 0;
3254 }
3255
3256 int
3257 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3258 {
3259   snat_main_t *sm = &snat_main;
3260
3261   if (nat44_set_session_limit (session_limit, vrf_id))
3262     return 1;
3263   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3264
3265   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
3266                         sm->max_translations_per_thread);
3267
3268   sm->translation_buckets =
3269     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3270
3271   nat44_ed_sessions_clear ();
3272   return 0;
3273 }
3274
3275 static void
3276 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations)
3277 {
3278   dlist_elt_t *head;
3279
3280   pool_alloc (tsm->per_vrf_sessions_pool, translations);
3281   pool_alloc (tsm->sessions, translations);
3282   pool_alloc (tsm->lru_pool, translations);
3283
3284   pool_get (tsm->lru_pool, head);
3285   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3286   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3287
3288   pool_get (tsm->lru_pool, head);
3289   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3290   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3291
3292   pool_get (tsm->lru_pool, head);
3293   tsm->udp_lru_head_index = head - tsm->lru_pool;
3294   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3295
3296   pool_get (tsm->lru_pool, head);
3297   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3298   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3299
3300   pool_get (tsm->lru_pool, head);
3301   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3302   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3303 }
3304
3305 static void
3306 nat44_ed_flow_hash_init ()
3307 {
3308   snat_main_t *sm = &snat_main;
3309   // we expect 2 flows per session, so multiply translation_buckets by 2
3310   clib_bihash_init_16_8 (
3311     &sm->flow_hash, "ed-flow-hash",
3312     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3313   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3314 }
3315
3316 static void
3317 nat44_ed_db_init ()
3318 {
3319   snat_main_t *sm = &snat_main;
3320   snat_main_per_thread_data_t *tsm;
3321
3322   nat44_ed_flow_hash_init ();
3323
3324   vec_foreach (tsm, sm->per_thread_data)
3325     {
3326       nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread);
3327     }
3328 }
3329
3330 static void
3331 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3332 {
3333   pool_free (tsm->lru_pool);
3334   pool_free (tsm->sessions);
3335   pool_free (tsm->per_vrf_sessions_pool);
3336 }
3337
3338 static void
3339 nat44_ed_flow_hash_free ()
3340 {
3341   snat_main_t *sm = &snat_main;
3342
3343   clib_bihash_free_16_8 (&sm->flow_hash);
3344 }
3345
3346 static void
3347 nat44_ed_db_free ()
3348 {
3349   snat_main_t *sm = &snat_main;
3350   snat_main_per_thread_data_t *tsm;
3351
3352   vec_foreach (tsm, sm->per_thread_data)
3353     {
3354       nat44_ed_worker_db_free (tsm);
3355     }
3356
3357   nat44_ed_flow_hash_free ();
3358 }
3359
3360 void
3361 nat44_ed_sessions_clear ()
3362 {
3363   snat_main_t *sm = &snat_main;
3364
3365   nat44_ed_db_free ();
3366   nat44_ed_db_init ();
3367   vlib_zero_simple_counter (&sm->total_sessions, 0);
3368 }
3369
3370 static void
3371 nat44_ed_add_del_static_mapping_cb (ip4_main_t *im, uword opaque,
3372                                     u32 sw_if_index, ip4_address_t *address,
3373                                     u32 address_length, u32 if_address_index,
3374                                     u32 is_delete)
3375 {
3376   snat_static_mapping_resolve_t *rp;
3377   snat_main_t *sm = &snat_main;
3378   int rv = 0;
3379
3380   if (!sm->enabled)
3381     {
3382       return;
3383     }
3384
3385   vec_foreach (rp, sm->sm_to_resolve)
3386     {
3387       if (sw_if_index == rp->sw_if_index)
3388         {
3389           if (is_delete)
3390             {
3391               if (rp->is_resolved)
3392                 {
3393                   rv = nat44_ed_del_static_mapping_internal (
3394                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3395                     rp->vrf_id, rp->flags);
3396                   if (rv)
3397                     {
3398                       nat_log_err ("ed del static mapping failed");
3399                     }
3400                   else
3401                     {
3402                       rp->is_resolved = 0;
3403                     }
3404                 }
3405             }
3406           else
3407             {
3408               if (!rp->is_resolved)
3409                 {
3410                   rv = nat44_ed_add_static_mapping_internal (
3411                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3412                     rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3413                   if (rv)
3414                     {
3415                       nat_log_err ("ed add static mapping failed");
3416                     }
3417                   else
3418                     {
3419                       rp->is_resolved = 1;
3420                     }
3421                 }
3422             }
3423         }
3424     }
3425 }
3426
3427 static int
3428 nat44_ed_get_addr_resolve_record (u32 sw_if_index, u8 twice_nat, int *out)
3429 {
3430   snat_main_t *sm = &snat_main;
3431   snat_address_resolve_t *rp;
3432   int i;
3433
3434   for (i = 0; i < vec_len (sm->addr_to_resolve); i++)
3435     {
3436       rp = sm->addr_to_resolve + i;
3437
3438       if ((rp->sw_if_index == sw_if_index) && (rp->is_twice_nat == twice_nat))
3439         {
3440           if (out)
3441             {
3442               *out = i;
3443             }
3444           return 0;
3445         }
3446     }
3447   return 1;
3448 }
3449 static int
3450 nat44_ed_del_addr_resolve_record (u32 sw_if_index, u8 twice_nat)
3451 {
3452   snat_main_t *sm = &snat_main;
3453   int i;
3454   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3455     {
3456       vec_del1 (sm->addr_to_resolve, i);
3457       return 0;
3458     }
3459   return 1;
3460 }
3461
3462 static void
3463 nat44_ed_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3464                                        u32 sw_if_index, ip4_address_t *address,
3465                                        u32 address_length,
3466                                        u32 if_address_index, u32 is_delete)
3467 {
3468   snat_main_t *sm = &snat_main;
3469   snat_address_resolve_t *arp;
3470   snat_address_t *ap;
3471   u8 twice_nat = 0;
3472   int i, rv;
3473
3474   if (!sm->enabled)
3475     {
3476       return;
3477     }
3478
3479   if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3480     {
3481       twice_nat = 1;
3482       if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3483         {
3484           u32 fib_index =
3485             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
3486           vec_foreach (ap, sm->addresses)
3487             {
3488               if ((fib_index == ap->fib_index) &&
3489                   (address->as_u32 == ap->addr.as_u32))
3490                 {
3491                   if (!is_delete)
3492                     {
3493                       ap->addr_len = address_length;
3494                       ap->sw_if_index = sw_if_index;
3495                       ap->net.as_u32 =
3496                         ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
3497
3498                       nat_log_debug (
3499                         "pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
3500                         format_ip4_address, &ap->addr, ap->sw_if_index,
3501                         format_ip4_address, &ap->net, ap->addr_len);
3502                     }
3503                   else
3504                     {
3505                       ap->addr_len = ~0;
3506                     }
3507                   break;
3508                 }
3509             }
3510           return;
3511         }
3512     }
3513
3514   arp = sm->addr_to_resolve + i;
3515
3516   if (!is_delete)
3517     {
3518       if (arp->is_resolved)
3519         {
3520           return;
3521         }
3522
3523       rv = nat44_ed_add_address (address, ~0, arp->is_twice_nat);
3524       if (0 == rv)
3525         {
3526           arp->is_resolved = 1;
3527         }
3528     }
3529   else
3530     {
3531       if (!arp->is_resolved)
3532         {
3533           return;
3534         }
3535
3536       rv = nat44_ed_del_address (address[0], arp->is_twice_nat);
3537       if (0 == rv)
3538         {
3539           arp->is_resolved = 0;
3540         }
3541     }
3542 }
3543
3544 int
3545 nat44_ed_add_interface_address (u32 sw_if_index, u8 twice_nat)
3546 {
3547   snat_main_t *sm = &snat_main;
3548   ip4_main_t *ip4_main = sm->ip4_main;
3549   ip4_address_t *first_int_addr;
3550   snat_address_resolve_t *ap;
3551   int rv;
3552
3553   if (!sm->enabled)
3554     {
3555       nat_log_err ("nat44 is disabled");
3556       return VNET_API_ERROR_UNSUPPORTED;
3557     }
3558
3559   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, 0))
3560     {
3561       return VNET_API_ERROR_VALUE_EXIST;
3562     }
3563
3564   vec_add2 (sm->addr_to_resolve, ap, 1);
3565   ap->sw_if_index = sw_if_index;
3566   ap->is_twice_nat = twice_nat;
3567   ap->is_resolved = 0;
3568
3569   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3570   if (first_int_addr)
3571     {
3572       rv = nat44_ed_add_address (first_int_addr, ~0, twice_nat);
3573       if (0 != rv)
3574         {
3575           nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat);
3576           return rv;
3577         }
3578       ap->is_resolved = 1;
3579     }
3580
3581   return 0;
3582 }
3583
3584 int
3585 nat44_ed_del_interface_address (u32 sw_if_index, u8 twice_nat)
3586 {
3587   snat_main_t *sm = &snat_main;
3588   ip4_main_t *ip4_main = sm->ip4_main;
3589   ip4_address_t *first_int_addr;
3590
3591   if (!sm->enabled)
3592     {
3593       nat_log_err ("nat44 is disabled");
3594       return VNET_API_ERROR_UNSUPPORTED;
3595     }
3596
3597   if (nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat))
3598     {
3599       return VNET_API_ERROR_NO_SUCH_ENTRY;
3600     }
3601
3602   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3603   if (first_int_addr)
3604     {
3605       return nat44_ed_del_address (first_int_addr[0], twice_nat);
3606     }
3607
3608   return 0;
3609 }
3610
3611 int
3612 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3613                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3614                       u32 vrf_id, int is_in)
3615 {
3616   ip4_header_t ip;
3617   clib_bihash_kv_16_8_t kv, value;
3618   u32 fib_index;
3619   snat_session_t *s;
3620   snat_main_per_thread_data_t *tsm;
3621
3622   if (!sm->enabled)
3623     {
3624       return VNET_API_ERROR_UNSUPPORTED;
3625     }
3626
3627   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3628   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3629   if (sm->num_workers > 1)
3630     tsm = vec_elt_at_index (
3631       sm->per_thread_data,
3632       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3633   else
3634     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3635
3636   init_ed_k (&kv, addr->as_u32, port, eh_addr->as_u32, eh_port, fib_index,
3637              proto);
3638   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3639     {
3640       return VNET_API_ERROR_NO_SUCH_ENTRY;
3641     }
3642
3643   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3644     return VNET_API_ERROR_UNSPECIFIED;
3645   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3646   nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3647   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3648   return 0;
3649 }
3650
3651 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3652                                  vlib_node_runtime_t * node,
3653                                  vlib_frame_t * frame)
3654 {
3655   return 0;
3656 }
3657
3658 VLIB_REGISTER_NODE (nat_default_node) = {
3659   .name = "nat-default",
3660   .vector_size = sizeof (u32),
3661   .format_trace = 0,
3662   .type = VLIB_NODE_TYPE_INTERNAL,
3663   .n_errors = 0,
3664   .n_next_nodes = NAT_N_NEXT,
3665   .next_nodes = {
3666     [NAT_NEXT_DROP] = "error-drop",
3667     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3668     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3669     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3670     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3671     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3672     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3673     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3674     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3675     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3676   },
3677 };
3678
3679 void
3680 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3681 {
3682   f->l3_csum_delta = 0;
3683   f->l4_csum_delta = 0;
3684   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3685       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3686     {
3687       f->l3_csum_delta =
3688         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3689       f->l3_csum_delta =
3690         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3691     }
3692   else
3693     {
3694       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3695     }
3696   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3697       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3698     {
3699       f->l3_csum_delta =
3700         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3701       f->l3_csum_delta =
3702         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3703     }
3704   else
3705     {
3706       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3707     }
3708   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3709     {
3710       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3711       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3712     }
3713   else
3714     {
3715       f->rewrite.sport = f->match.sport;
3716     }
3717   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3718     {
3719       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3720       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3721     }
3722   else
3723     {
3724       f->rewrite.dport = f->match.dport;
3725     }
3726   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3727       f->rewrite.icmp_id != f->match.sport)
3728     {
3729       f->l4_csum_delta =
3730         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3731       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3732     }
3733   else
3734     {
3735       f->rewrite.icmp_id = f->match.sport;
3736     }
3737   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3738     {
3739     }
3740   else
3741     {
3742       f->rewrite.fib_index = f->match.fib_index;
3743     }
3744 }
3745
3746 static_always_inline int
3747 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3748                             ip4_header_t *ip, nat_6t_flow_t *f);
3749
3750 static_always_inline void
3751 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3752                            nat_6t_flow_t *f, ip_protocol_t proto,
3753                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3754 {
3755   udp_header_t *udp = ip4_next_header (ip);
3756   tcp_header_t *tcp = (tcp_header_t *) udp;
3757
3758   if ((IP_PROTOCOL_TCP == proto || IP_PROTOCOL_UDP == proto) &&
3759       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3760     {
3761       if (!is_icmp_inner_ip4)
3762         { // regular case
3763           ip->src_address = f->rewrite.saddr;
3764           ip->dst_address = f->rewrite.daddr;
3765           udp->src_port = f->rewrite.sport;
3766           udp->dst_port = f->rewrite.dport;
3767         }
3768       else
3769         { // icmp inner ip4 - reversed saddr/daddr
3770           ip->src_address = f->rewrite.daddr;
3771           ip->dst_address = f->rewrite.saddr;
3772           udp->src_port = f->rewrite.dport;
3773           udp->dst_port = f->rewrite.sport;
3774         }
3775
3776       if (IP_PROTOCOL_TCP == proto)
3777         {
3778           ip_csum_t tcp_sum = tcp->checksum;
3779           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3780           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3781           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3782           tcp->checksum = ip_csum_fold (tcp_sum);
3783         }
3784       else if (IP_PROTOCOL_UDP == proto && udp->checksum)
3785         {
3786           ip_csum_t udp_sum = udp->checksum;
3787           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3788           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3789           udp->checksum = ip_csum_fold (udp_sum);
3790         }
3791     }
3792   else
3793     {
3794       if (!is_icmp_inner_ip4)
3795         { // regular case
3796           if (!skip_saddr_rewrite)
3797             {
3798               ip->src_address = f->rewrite.saddr;
3799             }
3800           ip->dst_address = f->rewrite.daddr;
3801         }
3802       else
3803         { // icmp inner ip4 - reversed saddr/daddr
3804           ip->src_address = f->rewrite.daddr;
3805           ip->dst_address = f->rewrite.saddr;
3806         }
3807     }
3808
3809   if (skip_saddr_rewrite)
3810     {
3811       ip->checksum = ip4_header_checksum (ip);
3812     }
3813   else
3814     {
3815       ip_csum_t ip_sum = ip->checksum;
3816       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3817       ip->checksum = ip_csum_fold (ip_sum);
3818     }
3819   if (0xffff == ip->checksum)
3820     ip->checksum = 0;
3821   ASSERT (ip4_header_checksum_is_valid (ip));
3822 }
3823
3824 static_always_inline int
3825 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3826 {
3827   int result = ((u8 *) object + size <=
3828                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3829                vlib_object_within_buffer_data (vm, b, object, size);
3830   return result;
3831 }
3832
3833 static_always_inline int
3834 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3835                             ip4_header_t *ip, nat_6t_flow_t *f)
3836 {
3837   if (IP_PROTOCOL_ICMP != ip->protocol)
3838     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3839
3840   icmp46_header_t *icmp = ip4_next_header (ip);
3841   nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3842
3843   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3844     {
3845       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3846         {
3847           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3848         }
3849
3850       if (!icmp_type_is_error_message (icmp->type))
3851         {
3852           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3853               (f->rewrite.icmp_id != echo->identifier))
3854             {
3855               ip_csum_t sum = icmp->checksum;
3856               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3857                                     nat_icmp_echo_header_t,
3858                                     identifier /* changed member */);
3859               echo->identifier = f->rewrite.icmp_id;
3860               icmp->checksum = ip_csum_fold (sum);
3861             }
3862         }
3863       else
3864         {
3865           ip_csum_t sum = ip_incremental_checksum (
3866             0, icmp,
3867             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3868           sum = (u16) ~ip_csum_fold (sum);
3869           if (sum != 0)
3870             {
3871               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3872             }
3873
3874           // errors are not fragmented
3875           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3876
3877           if (!ip4_header_checksum_is_valid (inner_ip))
3878             {
3879               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3880             }
3881
3882           ip_protocol_t inner_proto = inner_ip->protocol;
3883
3884           ip_csum_t old_icmp_sum = icmp->checksum;
3885           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3886           ip_csum_t old_udp_sum;
3887           ip_csum_t old_tcp_sum;
3888           ip_csum_t new_icmp_sum;
3889           udp_header_t *udp;
3890           tcp_header_t *tcp;
3891
3892           switch (inner_proto)
3893             {
3894             case IP_PROTOCOL_UDP:
3895               udp = (udp_header_t *) (inner_ip + 1);
3896               if (!it_fits (vm, b, udp, sizeof (*udp)))
3897                 {
3898                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3899                 }
3900               old_udp_sum = udp->checksum;
3901               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3902                                          1 /* is_icmp_inner_ip4 */,
3903                                          0 /* skip_saddr_rewrite */);
3904               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3905               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3906               new_icmp_sum =
3907                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3908                                 inner_ip->checksum, ip4_header_t, checksum);
3909               new_icmp_sum =
3910                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3911                                 udp_header_t, checksum);
3912               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3913               icmp->checksum = new_icmp_sum;
3914               break;
3915             case IP_PROTOCOL_TCP:
3916               tcp = (tcp_header_t *) (inner_ip + 1);
3917               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3918                 {
3919                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3920                 }
3921               old_tcp_sum = tcp->checksum;
3922               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3923                                          1 /* is_icmp_inner_ip4 */,
3924                                          0 /* skip_saddr_rewrite */);
3925               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3926               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3927               new_icmp_sum =
3928                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3929                                 inner_ip->checksum, ip4_header_t, checksum);
3930               new_icmp_sum =
3931                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3932                                 tcp_header_t, checksum);
3933               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3934               icmp->checksum = new_icmp_sum;
3935               break;
3936             case IP_PROTOCOL_ICMP:
3937               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3938                                          1 /* is_icmp_inner_ip4 */,
3939                                          0 /* skip_saddr_rewrite */);
3940               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3941                 {
3942                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3943                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3944                     {
3945                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3946                     }
3947                   nat_icmp_echo_header_t *inner_echo =
3948                     (nat_icmp_echo_header_t *) (inner_icmp + 1);
3949                   if (f->rewrite.icmp_id != inner_echo->identifier)
3950                     {
3951                       ip_csum_t sum = icmp->checksum;
3952                       sum = ip_csum_update (sum, inner_echo->identifier,
3953                                             f->rewrite.icmp_id,
3954                                             nat_icmp_echo_header_t,
3955                                             identifier /* changed member */);
3956                       icmp->checksum = ip_csum_fold (sum);
3957                       ip_csum_t inner_sum = inner_icmp->checksum;
3958                       inner_sum = ip_csum_update (
3959                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3960                         nat_icmp_echo_header_t,
3961                         identifier /* changed member */);
3962                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3963                       inner_echo->identifier = f->rewrite.icmp_id;
3964                     }
3965                 }
3966               break;
3967             default:
3968               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3969               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3970             }
3971         }
3972     }
3973
3974   return NAT_ED_TRNSL_ERR_SUCCESS;
3975 }
3976
3977 static_always_inline nat_translation_error_e
3978 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3979                            ip4_header_t *ip, nat_6t_flow_t *f,
3980                            ip_protocol_t proto, int is_output_feature,
3981                            int is_i2o)
3982 {
3983   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3984     {
3985       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3986     }
3987
3988   if (IP_PROTOCOL_ICMP == proto)
3989     {
3990       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3991         {
3992           // packet is returned from a router, not from destination
3993           // skip source address rewrite if in o2i path
3994           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3995                                      0 /* is_icmp_inner_ip4 */,
3996                                      !is_i2o /* skip_saddr_rewrite */);
3997         }
3998       else
3999         {
4000           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
4001                                      0 /* is_icmp_inner_ip4 */,
4002                                      0 /* skip_saddr_rewrite */);
4003         }
4004       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
4005     }
4006
4007   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
4008                              0 /* skip_saddr_rewrite */);
4009
4010   return NAT_ED_TRNSL_ERR_SUCCESS;
4011 }
4012
4013 nat_translation_error_e
4014 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
4015                                vlib_buffer_t *b, ip4_header_t *ip,
4016                                nat_6t_flow_t *f, ip_protocol_t proto,
4017                                int is_output_feature)
4018 {
4019   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
4020                                     1 /* is_i2o */);
4021 }
4022
4023 nat_translation_error_e
4024 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
4025                                vlib_buffer_t *b, ip4_header_t *ip,
4026                                nat_6t_flow_t *f, ip_protocol_t proto,
4027                                int is_output_feature)
4028 {
4029   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
4030                                     0 /* is_i2o */);
4031 }
4032
4033 static_always_inline void
4034 nat_syslog_nat44_sess (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4035                        u16 isport, ip4_address_t *xsaddr, u16 xsport,
4036                        ip4_address_t *idaddr, u16 idport,
4037                        ip4_address_t *xdaddr, u16 xdport, u8 proto, u8 is_add,
4038                        u8 is_twicenat)
4039 {
4040   syslog_msg_t syslog_msg;
4041   fib_table_t *fib;
4042
4043   if (!syslog_is_enabled ())
4044     return;
4045
4046   if (syslog_severity_filter_block (SADD_SDEL_SEVERITY))
4047     return;
4048
4049   fib = fib_table_get (sfibix, FIB_PROTOCOL_IP4);
4050
4051   syslog_msg_init (&syslog_msg, NAT_FACILITY, SADD_SDEL_SEVERITY, NAT_APPNAME,
4052                    is_add ? SADD_MSGID : SDEL_MSGID);
4053
4054   syslog_msg_sd_init (&syslog_msg, NSESS_SDID);
4055   syslog_msg_add_sd_param (&syslog_msg, SSUBIX_SDPARAM_NAME, "%d", ssubix);
4056   syslog_msg_add_sd_param (&syslog_msg, SVLAN_SDPARAM_NAME, "%d",
4057                            fib->ft_table_id);
4058   syslog_msg_add_sd_param (&syslog_msg, IATYP_SDPARAM_NAME, IATYP_IPV4);
4059   syslog_msg_add_sd_param (&syslog_msg, ISADDR_SDPARAM_NAME, "%U",
4060                            format_ip4_address, isaddr);
4061   syslog_msg_add_sd_param (&syslog_msg, ISPORT_SDPARAM_NAME, "%d",
4062                            clib_net_to_host_u16 (isport));
4063   syslog_msg_add_sd_param (&syslog_msg, XATYP_SDPARAM_NAME, IATYP_IPV4);
4064   syslog_msg_add_sd_param (&syslog_msg, XSADDR_SDPARAM_NAME, "%U",
4065                            format_ip4_address, xsaddr);
4066   syslog_msg_add_sd_param (&syslog_msg, XSPORT_SDPARAM_NAME, "%d",
4067                            clib_net_to_host_u16 (xsport));
4068   syslog_msg_add_sd_param (&syslog_msg, PROTO_SDPARAM_NAME, "%d", proto);
4069   syslog_msg_add_sd_param (&syslog_msg, XDADDR_SDPARAM_NAME, "%U",
4070                            format_ip4_address, xdaddr);
4071   syslog_msg_add_sd_param (&syslog_msg, XDPORT_SDPARAM_NAME, "%d",
4072                            clib_net_to_host_u16 (xdport));
4073   if (is_twicenat)
4074     {
4075       syslog_msg_add_sd_param (&syslog_msg, IDADDR_SDPARAM_NAME, "%U",
4076                                format_ip4_address, idaddr);
4077       syslog_msg_add_sd_param (&syslog_msg, IDPORT_SDPARAM_NAME, "%d",
4078                                clib_net_to_host_u16 (idport));
4079     }
4080
4081   syslog_msg_send (&syslog_msg);
4082 }
4083
4084 void
4085 nat_syslog_nat44_sadd (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4086                        u16 isport, ip4_address_t *idaddr, u16 idport,
4087                        ip4_address_t *xsaddr, u16 xsport,
4088                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4089                        u8 is_twicenat)
4090 {
4091   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4092                          idaddr, idport, xdaddr, xdport, proto, 1,
4093                          is_twicenat);
4094 }
4095
4096 void
4097 nat_syslog_nat44_sdel (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4098                        u16 isport, ip4_address_t *idaddr, u16 idport,
4099                        ip4_address_t *xsaddr, u16 xsport,
4100                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4101                        u8 is_twicenat)
4102 {
4103   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4104                          idaddr, idport, xdaddr, xdport, proto, 0,
4105                          is_twicenat);
4106 }
4107 __clib_export void
4108 nat44_original_dst_lookup (ip4_address_t *i2o_src, u16 i2o_src_port,
4109                            ip4_address_t *i2o_dst, u16 i2o_dst_port,
4110                            ip_protocol_t proto, u32 *original_dst,
4111                            u16 *original_dst_port)
4112 {
4113   snat_main_per_thread_data_t *tsm;
4114   snat_main_t *sm = &snat_main;
4115   u32 fib_index = 0;
4116   snat_session_t *s;
4117   ip4_header_t ip;
4118
4119   ip.src_address.as_u32 = i2o_src->as_u32;
4120   fib_index = fib_table_find (FIB_PROTOCOL_IP4, 0);
4121
4122   if (sm->num_workers > 1)
4123     {
4124       tsm = vec_elt_at_index (
4125         sm->per_thread_data,
4126         nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
4127     }
4128   else
4129     {
4130       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4131     }
4132
4133   /* query */
4134   clib_bihash_kv_16_8_t kv = { 0 }, value;
4135   init_ed_k (&kv, i2o_src->as_u32, i2o_src_port, i2o_dst->as_u32, i2o_dst_port,
4136              fib_index, proto);
4137   if (tsm->sessions == NULL ||
4138       clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
4139     {
4140       return;
4141     }
4142   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
4143   if (s)
4144     {
4145       *original_dst = s->i2o.rewrite.saddr.as_u32;
4146       *original_dst_port = s->i2o.rewrite.sport;
4147     }
4148   return;
4149 }
4150 /*
4151  * fd.io coding-style-patch-verification: ON
4152  *
4153  * Local Variables:
4154  * eval: (c-set-style "gnu")
4155  * End:
4156  */