src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vpp/app/version.h>
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/ip/ip_table.h>
  22 #include <vnet/ip/reass/ip4_sv_reass.h>
  23 #include <vnet/fib/fib_table.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/plugin/plugin.h>
  26 #include <vppinfra/bihash_16_8.h>
  27
  28 #include <nat/lib/log.h>
  29 #include <nat/lib/nat_inlines.h>
  30 #include <nat/lib/ipfix_logging.h>
  31 #include <vnet/syslog/syslog.h>
  32 #include <nat/lib/nat_syslog_constants.h>
  33 #include <nat/lib/nat_syslog.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 #include <vlib/stats/stats.h>
  40
  41 snat_main_t snat_main;
  42
  43 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  44                                                            u32 sw_if_index);
  45
  46 #define skip_if_disabled()                                                    \
  47   do                                                                          \
  48     {                                                                         \
  49       snat_main_t *sm = &snat_main;                                           \
  50       if (PREDICT_FALSE (!sm->enabled))                                       \
  51         return;                                                               \
  52     }                                                                         \
  53   while (0)
  54
  55 #define fail_if_enabled()                                                     \
  56   do                                                                          \
  57     {                                                                         \
  58       snat_main_t *sm = &snat_main;                                           \
  59       if (PREDICT_FALSE (sm->enabled))                                        \
  60         {                                                                     \
  61           nat_log_err ("plugin enabled");                                     \
  62           return VNET_API_ERROR_FEATURE_ALREADY_ENABLED;                      \
  63         }                                                                     \
  64     }                                                                         \
  65   while (0)
  66
  67 #define fail_if_disabled()                                                    \
  68   do                                                                          \
  69     {                                                                         \
  70       snat_main_t *sm = &snat_main;                                           \
  71       if (PREDICT_FALSE (!sm->enabled))                                       \
  72         {                                                                     \
  73           nat_log_err ("plugin disabled");                                    \
  74           return VNET_API_ERROR_FEATURE_ALREADY_DISABLED;                     \
  75         }                                                                     \
  76     }                                                                         \
  77   while (0)
  78
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-ed-classify",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  96                                "ip4-sv-reassembly-feature"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-handoff-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 102                                "ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-worker-handoff",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 108 };
 109 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-worker-handoff",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-in2out",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-out2in",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 124                                "ip4-dhcp-client-detect"),
 125 };
 126 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 127   .arc_name = "ip4-output",
 128   .node_name = "nat-pre-in2out-output",
 129   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 130   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 136   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-ed-in2out-output",
 141   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 142   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 143 };
 144
 145 VLIB_PLUGIN_REGISTER () = {
 146     .version = VPP_BUILD_VER,
 147     .description = "Network Address Translation (NAT)",
 148 };
 149
 150 static void nat44_ed_db_init ();
 151 static void nat44_ed_db_free ();
 152 static void nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm);
 153
 154 static int nat44_ed_add_static_mapping_internal (
 155   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 156   ip_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 157   ip4_address_t pool_addr, u8 *tag);
 158 static int nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
 159                                                  ip4_address_t e_addr,
 160                                                  u16 l_port, u16 e_port,
 161                                                  ip_protocol_t proto,
 162                                                  u32 vrf_id, u32 flags);
 163
 164 u32 nat_calc_bihash_buckets (u32 n_elts);
 165
 166 static_always_inline int
 167 nat44_ed_sm_i2o_add (snat_main_t *sm, snat_static_mapping_t *m,
 168                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 169 {
 170   ASSERT (!pool_is_free (sm->static_mappings, m));
 171   clib_bihash_kv_16_8_t kv;
 172   nat44_ed_sm_init_i2o_kv (&kv, addr.as_u32, port, fib_index, proto,
 173                            m - sm->static_mappings);
 174   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 175 }
 176
 177 static_always_inline int
 178 nat44_ed_sm_i2o_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 179                      u32 fib_index, u8 proto)
 180 {
 181   clib_bihash_kv_16_8_t kv;
 182   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 183   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 184 }
 185
 186 static_always_inline int
 187 nat44_ed_sm_o2i_add (snat_main_t *sm, snat_static_mapping_t *m,
 188                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 189 {
 190   ASSERT (!pool_is_free (sm->static_mappings, m));
 191   clib_bihash_kv_16_8_t kv;
 192   nat44_ed_sm_init_o2i_kv (&kv, addr.as_u32, port, fib_index, proto,
 193                            m - sm->static_mappings);
 194   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 195 }
 196
 197 static_always_inline int
 198 nat44_ed_sm_o2i_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 199                      u32 fib_index, u8 proto)
 200 {
 201   clib_bihash_kv_16_8_t kv;
 202   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 203   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 204 }
 205
 206 void
 207 nat44_ed_free_session_data (snat_main_t *sm, snat_session_t *s,
 208                             u32 thread_index, u8 is_ha)
 209 {
 210   per_vrf_sessions_unregister_session (s, thread_index);
 211
 212   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 213     nat_elog_warn (sm, "flow hash del failed");
 214
 215   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 216     nat_elog_warn (sm, "flow hash del failed");
 217
 218   if (na44_ed_is_fwd_bypass_session (s))
 219     {
 220       return;
 221     }
 222
 223   if (nat44_ed_is_affinity_session (s))
 224     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, s->proto,
 225                          s->out2in.port);
 226
 227   if (!is_ha)
 228     nat_syslog_nat44_sdel (0, s->in2out.fib_index, &s->in2out.addr,
 229                            s->in2out.port, &s->ext_host_nat_addr,
 230                            s->ext_host_nat_port, &s->out2in.addr,
 231                            s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 232                            s->proto, nat44_ed_is_twice_nat_session (s));
 233
 234   if (!is_ha)
 235     {
 236       /* log NAT event */
 237       nat_ipfix_logging_nat44_ses_delete (
 238         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 239         s->in2out.port, s->out2in.port, s->in2out.fib_index);
 240     }
 241 }
 242
 243 static ip_interface_address_t *
 244 nat44_ed_get_ip_interface_address (u32 sw_if_index, ip4_address_t addr)
 245 {
 246   snat_main_t *sm = &snat_main;
 247
 248   ip_lookup_main_t *lm = &sm->ip4_main->lookup_main;
 249   ip_interface_address_t *ia;
 250   ip4_address_t *ip4a;
 251
 252   foreach_ip_interface_address (
 253     lm, ia, sw_if_index, 1, ({
 254       ip4a = ip_interface_address_get_address (lm, ia);
 255       nat_log_debug ("sw_if_idx: %u addr: %U ? %U", sw_if_index,
 256                      format_ip4_address, ip4a, format_ip4_address, &addr);
 257       if (ip4a->as_u32 == addr.as_u32)
 258         {
 259           return ia;
 260         }
 261     }));
 262   return NULL;
 263 }
 264
 265 static int
 266 nat44_ed_resolve_nat_addr_len (snat_address_t *ap,
 267                                snat_interface_t *interfaces)
 268 {
 269   ip_interface_address_t *ia;
 270   snat_interface_t *i;
 271   u32 fib_index;
 272
 273   pool_foreach (i, interfaces)
 274     {
 275       if (!nat44_ed_is_interface_outside (i))
 276         {
 277           continue;
 278         }
 279
 280       fib_index = ip4_fib_table_get_index_for_sw_if_index (i->sw_if_index);
 281       if (fib_index != ap->fib_index)
 282         {
 283           continue;
 284         }
 285
 286       if ((ia = nat44_ed_get_ip_interface_address (i->sw_if_index, ap->addr)))
 287         {
 288           ap->addr_len = ia->address_length;
 289           ap->sw_if_index = i->sw_if_index;
 290           ap->net.as_u32 = ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
 291
 292           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 293                          format_ip4_address, &ap->addr, ap->sw_if_index,
 294                          format_ip4_address, &ap->net, ap->addr_len);
 295           return 0;
 296         }
 297     }
 298   return 1;
 299 }
 300
 301 static void
 302 nat44_ed_update_outside_if_addresses (snat_address_t *ap)
 303 {
 304   snat_main_t *sm = &snat_main;
 305
 306   if (!nat44_ed_resolve_nat_addr_len (ap, sm->interfaces))
 307     {
 308       return;
 309     }
 310
 311   if (!nat44_ed_resolve_nat_addr_len (ap, sm->output_feature_interfaces))
 312     {
 313       return;
 314     }
 315 }
 316
 317 static void
 318 nat44_ed_bind_if_addr_to_nat_addr (u32 sw_if_index)
 319 {
 320   snat_main_t *sm = &snat_main;
 321   ip_interface_address_t *ia;
 322   snat_address_t *ap;
 323
 324   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 325
 326   vec_foreach (ap, sm->addresses)
 327     {
 328       if (fib_index != ap->fib_index)
 329         {
 330           continue;
 331         }
 332
 333       if ((ia = nat44_ed_get_ip_interface_address (sw_if_index, ap->addr)))
 334         {
 335           ap->addr_len = ia->address_length;
 336           ap->sw_if_index = sw_if_index;
 337           ap->net.as_u32 = ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
 338
 339           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 340                          format_ip4_address, &ap->addr, ap->sw_if_index,
 341                          format_ip4_address, &ap->net, ap->addr_len);
 342           return;
 343         }
 344     }
 345 }
 346
 347 static_always_inline snat_fib_entry_reg_t *
 348 nat44_ed_get_fib_entry_reg (ip4_address_t addr, u32 sw_if_index, int *out_idx)
 349 {
 350   snat_main_t *sm = &snat_main;
 351   snat_fib_entry_reg_t *fe;
 352   int i;
 353
 354   for (i = 0; i < vec_len (sm->fib_entry_reg); i++)
 355     {
 356       fe = sm->fib_entry_reg + i;
 357       if ((addr.as_u32 == fe->addr.as_u32) && (sw_if_index == fe->sw_if_index))
 358         {
 359           if (out_idx)
 360             {
 361               *out_idx = i;
 362             }
 363           return fe;
 364         }
 365     }
 366   return NULL;
 367 }
 368
 369 static void
 370 nat44_ed_add_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 371 {
 372   // Add the external NAT address to the FIB as receive entries. This ensures
 373   // that VPP will reply to ARP for this address and we don't need to enable
 374   // proxy ARP on the outside interface.
 375   snat_main_t *sm = &snat_main;
 376   snat_fib_entry_reg_t *fe;
 377
 378   if (!(fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, 0)))
 379     {
 380       fib_prefix_t prefix = {
 381         .fp_len = 32,
 382         .fp_proto = FIB_PROTOCOL_IP4,
 383         .fp_addr = {
 384                   .ip4.as_u32 = addr.as_u32,
 385                 },
 386       };
 387       u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 388       fib_table_entry_update_one_path (fib_index, &prefix, sm->fib_src_low,
 389                                        (FIB_ENTRY_FLAG_CONNECTED |
 390                                         FIB_ENTRY_FLAG_LOCAL |
 391                                         FIB_ENTRY_FLAG_EXCLUSIVE),
 392                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
 393                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 394
 395       vec_add2 (sm->fib_entry_reg, fe, 1);
 396       clib_memset (fe, 0, sizeof (*fe));
 397       fe->addr.as_u32 = addr.as_u32;
 398       fe->sw_if_index = sw_if_index;
 399     }
 400   fe->count++;
 401 }
 402
 403 static void
 404 nat44_ed_del_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 405 {
 406   snat_main_t *sm = &snat_main;
 407   snat_fib_entry_reg_t *fe;
 408   int i;
 409
 410   if ((fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, &i)))
 411     {
 412       fe->count--;
 413       if (0 == fe->count)
 414         {
 415           fib_prefix_t prefix = {
 416             .fp_len = 32,
 417             .fp_proto = FIB_PROTOCOL_IP4,
 418             .fp_addr = {
 419               .ip4.as_u32 = addr.as_u32,
 420                     },
 421           };
 422           u32 fib_index =
 423             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 424           fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 425           vec_del1 (sm->fib_entry_reg, i);
 426         }
 427     }
 428 }
 429
 430 static void
 431 nat44_ed_add_del_interface_fib_reg_entries (ip4_address_t addr, u8 is_add)
 432 {
 433   snat_main_t *sm = &snat_main;
 434   snat_interface_t *i;
 435
 436   pool_foreach (i, sm->interfaces)
 437     {
 438       if (nat44_ed_is_interface_outside (i))
 439         {
 440           if (is_add)
 441             {
 442               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 443             }
 444           else
 445             {
 446               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 447             }
 448         }
 449     }
 450   pool_foreach (i, sm->output_feature_interfaces)
 451     {
 452       if (nat44_ed_is_interface_outside (i))
 453         {
 454           if (is_add)
 455             {
 456               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 457             }
 458           else
 459             {
 460               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 461             }
 462         }
 463     }
 464 }
 465
 466 static_always_inline void
 467 nat44_ed_add_del_nat_addr_fib_reg_entries (u32 sw_if_index, u8 is_add)
 468 {
 469   snat_main_t *sm = &snat_main;
 470   snat_address_t *ap;
 471
 472   vec_foreach (ap, sm->addresses)
 473     {
 474       if (is_add)
 475         {
 476           nat44_ed_add_fib_entry_reg (ap->addr, sw_if_index);
 477         }
 478       else
 479         {
 480           nat44_ed_del_fib_entry_reg (ap->addr, sw_if_index);
 481         }
 482     }
 483 }
 484
 485 static_always_inline void
 486 nat44_ed_add_del_sm_fib_reg_entries (u32 sw_if_index, u8 is_add)
 487 {
 488   snat_main_t *sm = &snat_main;
 489   snat_static_mapping_t *m;
 490
 491   pool_foreach (m, sm->static_mappings)
 492     {
 493       if (is_add)
 494         {
 495           nat44_ed_add_fib_entry_reg (m->external_addr, sw_if_index);
 496         }
 497       else
 498         {
 499           nat44_ed_del_fib_entry_reg (m->external_addr, sw_if_index);
 500         }
 501     }
 502 }
 503
 504 int
 505 nat44_ed_add_address (ip4_address_t *addr, u32 vrf_id, u8 twice_nat)
 506 {
 507   snat_main_t *sm = &snat_main;
 508   snat_address_t *ap, *addresses;
 509
 510   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 511
 512   if (!sm->enabled)
 513     {
 514       return VNET_API_ERROR_UNSUPPORTED;
 515     }
 516
 517   // check if address already exists
 518   vec_foreach (ap, addresses)
 519     {
 520       if (ap->addr.as_u32 == addr->as_u32)
 521         {
 522           nat_log_err ("address exist");
 523           return VNET_API_ERROR_VALUE_EXIST;
 524         }
 525     }
 526
 527   if (twice_nat)
 528     {
 529       vec_add2 (sm->twice_nat_addresses, ap, 1);
 530     }
 531   else
 532     {
 533       vec_add2 (sm->addresses, ap, 1);
 534     }
 535
 536   ap->addr_len = ~0;
 537   ap->fib_index = ~0;
 538   ap->addr = *addr;
 539
 540   if (vrf_id != ~0)
 541     {
 542       ap->fib_index = fib_table_find_or_create_and_lock (
 543         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 544     }
 545
 546   if (!twice_nat)
 547     {
 548       // if we don't have enabled interface we don't add address
 549       // to fib
 550       nat44_ed_add_del_interface_fib_reg_entries (*addr, 1);
 551       nat44_ed_update_outside_if_addresses (ap);
 552     }
 553   return 0;
 554 }
 555
 556 int
 557 nat44_ed_del_address (ip4_address_t addr, u8 twice_nat)
 558 {
 559   snat_main_t *sm = &snat_main;
 560   snat_address_t *a = 0, *addresses;
 561   snat_session_t *ses;
 562   u32 *ses_to_be_removed = 0, *ses_index;
 563   snat_main_per_thread_data_t *tsm;
 564   int j;
 565
 566   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 567
 568   for (j = 0; j < vec_len (addresses); j++)
 569     {
 570       if (addresses[j].addr.as_u32 == addr.as_u32)
 571         {
 572           a = addresses + j;
 573           break;
 574         }
 575     }
 576   if (!a)
 577     {
 578       nat_log_err ("no such address");
 579       return VNET_API_ERROR_NO_SUCH_ENTRY;
 580     }
 581
 582   // delete dynamic sessions only
 583   vec_foreach (tsm, sm->per_thread_data)
 584     {
 585       pool_foreach (ses, tsm->sessions)
 586         {
 587           if (ses->flags & SNAT_SESSION_FLAG_STATIC_MAPPING)
 588             {
 589               continue;
 590             }
 591           if (ses->out2in.addr.as_u32 == addr.as_u32)
 592             {
 593               nat44_ed_free_session_data (sm, ses, tsm - sm->per_thread_data,
 594                                           0);
 595               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
 596             }
 597         }
 598       vec_foreach (ses_index, ses_to_be_removed)
 599         {
 600           ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
 601           nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
 602         }
 603       vec_free (ses_to_be_removed);
 604     }
 605
 606   if (!twice_nat)
 607     {
 608       nat44_ed_add_del_interface_fib_reg_entries (addr, 0);
 609     }
 610
 611   if (a->fib_index != ~0)
 612     {
 613       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 614     }
 615
 616   if (!twice_nat)
 617     {
 618       vec_del1 (sm->addresses, j);
 619     }
 620   else
 621     {
 622       vec_del1 (sm->twice_nat_addresses, j);
 623     }
 624
 625   return 0;
 626 }
 627
 628 vrf_table_t *
 629 nat44_ed_get_vrf_table (u32 table_vrf_id)
 630 {
 631   snat_main_t *sm = &snat_main;
 632   vrf_table_t *t;
 633
 634   pool_foreach (t, sm->vrf_tables)
 635     {
 636       if (table_vrf_id == t->table_vrf_id)
 637         {
 638           return t;
 639         }
 640     }
 641   return NULL;
 642 }
 643
 644 vrf_route_t *
 645 nat44_ed_get_vrf_route (vrf_table_t *t, u32 vrf_id)
 646 {
 647   vrf_route_t *r;
 648
 649   pool_foreach (r, t->routes)
 650     {
 651       if (vrf_id == r->vrf_id)
 652         {
 653           return r;
 654         }
 655     }
 656   return NULL;
 657 }
 658
 659 int
 660 nat44_ed_add_del_vrf_table (u32 table_vrf_id, bool is_add)
 661 {
 662   snat_main_t *sm = &snat_main;
 663   vrf_table_t *t;
 664   vrf_route_t *r;
 665
 666   t = nat44_ed_get_vrf_table (table_vrf_id);
 667   if (t)
 668     {
 669       if (is_add)
 670         {
 671           return VNET_API_ERROR_VALUE_EXIST;
 672         }
 673       pool_foreach (r, t->routes)
 674         {
 675           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 676         }
 677       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 678
 679       pool_free (t->routes);
 680       pool_put (sm->vrf_tables, t);
 681     }
 682   else
 683     {
 684       if (!is_add)
 685         {
 686           return VNET_API_ERROR_NO_SUCH_ENTRY;
 687         }
 688       pool_get (sm->vrf_tables, t);
 689       clib_memset (t, 0, sizeof (*t));
 690       t->table_vrf_id = table_vrf_id;
 691       t->table_fib_index = fib_table_find_or_create_and_lock (
 692         FIB_PROTOCOL_IP4, table_vrf_id, sm->fib_src_low);
 693     }
 694
 695   return 0;
 696 }
 697
 698 void
 699 nat44_ed_del_vrf_tables ()
 700 {
 701   snat_main_t *sm = &snat_main;
 702   vrf_table_t *t;
 703   vrf_route_t *r;
 704
 705   pool_foreach (t, sm->vrf_tables)
 706     {
 707       pool_foreach (r, t->routes)
 708         {
 709           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 710         }
 711       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 712       pool_free (t->routes);
 713     }
 714   pool_free (sm->vrf_tables);
 715 }
 716
 717 int
 718 nat44_ed_add_del_vrf_route (u32 table_vrf_id, u32 vrf_id, bool is_add)
 719 {
 720   snat_main_t *sm = &snat_main;
 721   vrf_table_t *t;
 722   vrf_route_t *r;
 723
 724   t = nat44_ed_get_vrf_table (table_vrf_id);
 725   if (!t)
 726     {
 727       return VNET_API_ERROR_NO_SUCH_ENTRY;
 728     }
 729
 730   r = nat44_ed_get_vrf_route (t, vrf_id);
 731   if (r)
 732     {
 733       if (is_add)
 734         {
 735           return VNET_API_ERROR_VALUE_EXIST;
 736         }
 737       fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 738       pool_put (t->routes, r);
 739     }
 740   else
 741     {
 742       if (!is_add)
 743         {
 744           return VNET_API_ERROR_NO_SUCH_ENTRY;
 745         }
 746       pool_get (t->routes, r);
 747       clib_memset (r, 0, sizeof (*r));
 748       r->vrf_id = vrf_id;
 749       r->fib_index = fib_table_find_or_create_and_lock (
 750         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 751     }
 752
 753   return 0;
 754 }
 755
 756 u32
 757 get_thread_idx_by_port (u16 e_port)
 758 {
 759   snat_main_t *sm = &snat_main;
 760   u32 thread_idx = sm->num_workers;
 761   if (sm->num_workers > 1)
 762     {
 763       thread_idx = sm->first_worker_index +
 764                    sm->workers[(e_port - ED_USER_PORT_OFFSET) /
 765                                sm->port_per_thread % _vec_len (sm->workers)];
 766     }
 767   return thread_idx;
 768 }
 769
 770 void
 771 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 772                                     snat_main_per_thread_data_t * tsm,
 773                                     ip4_address_t l_addr,
 774                                     u16 l_port,
 775                                     u8 protocol,
 776                                     u32 fib_index, int addr_only,
 777                                     ip4_address_t e_addr, u16 e_port)
 778 {
 779   snat_session_t *s;
 780   u32 *indexes_to_free = NULL;
 781   pool_foreach (s, tsm->sessions) {
 782     if (s->in2out.fib_index != fib_index ||
 783         s->in2out.addr.as_u32 != l_addr.as_u32)
 784       {
 785         continue;
 786       }
 787     if (!addr_only)
 788       {
 789         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 790             s->out2in.port != e_port || s->in2out.port != l_port ||
 791             s->proto != protocol)
 792           continue;
 793       }
 794
 795     if (nat44_ed_is_lb_session (s))
 796       continue;
 797     if (!nat44_ed_is_session_static (s))
 798       continue;
 799     nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 800     vec_add1 (indexes_to_free, s - tsm->sessions);
 801     if (!addr_only)
 802       break;
 803   }
 804   u32 *ses_index;
 805   vec_foreach (ses_index, indexes_to_free)
 806   {
 807     s = pool_elt_at_index (tsm->sessions, *ses_index);
 808     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 809   }
 810   vec_free (indexes_to_free);
 811 }
 812
 813 static_always_inline snat_static_mapping_t *
 814 nat44_ed_sm_lookup (snat_main_t *sm, clib_bihash_kv_16_8_t *kv)
 815 {
 816   clib_bihash_kv_16_8_t v;
 817   int rc = clib_bihash_search_16_8 (&sm->flow_hash, kv, &v);
 818   if (!rc)
 819     {
 820       ASSERT (0 == ed_value_get_thread_index (&v));
 821       return pool_elt_at_index (sm->static_mappings,
 822                                 ed_value_get_session_index (&v));
 823     }
 824   return NULL;
 825 }
 826
 827 snat_static_mapping_t *
 828 nat44_ed_sm_o2i_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 829                         u32 fib_index, u8 proto)
 830 {
 831   clib_bihash_kv_16_8_t kv;
 832   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 833   return nat44_ed_sm_lookup (sm, &kv);
 834 }
 835
 836 snat_static_mapping_t *
 837 nat44_ed_sm_i2o_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 838                         u32 fib_index, u8 proto)
 839 {
 840   clib_bihash_kv_16_8_t kv;
 841   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 842   return nat44_ed_sm_lookup (sm, &kv);
 843 }
 844
 845 static snat_static_mapping_resolve_t *
 846 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 847                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 848                              u32 flags, int *out_idx)
 849 {
 850   snat_static_mapping_resolve_t *rp;
 851   snat_main_t *sm = &snat_main;
 852   int i;
 853
 854   for (i = 0; i < vec_len (sm->sm_to_resolve); i++)
 855     {
 856       rp = sm->sm_to_resolve + i;
 857
 858       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 859         {
 860           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 861             {
 862               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 863                 {
 864                   if (rp->e_port != e_port || rp->proto != proto)
 865                     {
 866                       continue;
 867                     }
 868                 }
 869             }
 870           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 871             {
 872               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 873                 {
 874                   if (rp->l_port != l_port || rp->e_port != e_port ||
 875                       rp->proto != proto)
 876                     {
 877                       continue;
 878                     }
 879                 }
 880             }
 881           else
 882             {
 883               continue;
 884             }
 885           if (out_idx)
 886             {
 887               *out_idx = i;
 888             }
 889           return rp;
 890         }
 891     }
 892   return NULL;
 893 }
 894
 895 static int
 896 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 897                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 898                              u32 flags)
 899 {
 900   snat_main_t *sm = &snat_main;
 901   int i;
 902   if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 903                                    sw_if_index, flags, &i))
 904     {
 905       vec_del1 (sm->sm_to_resolve, i);
 906       return 0;
 907     }
 908   return 1;
 909 }
 910
 911 static_always_inline int
 912 nat44_ed_validate_sm_input (u32 flags)
 913 {
 914   // identity nat can be initiated only from inside interface
 915   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 916     {
 917       return VNET_API_ERROR_UNSUPPORTED;
 918     }
 919
 920   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 921     {
 922       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 923         {
 924           return VNET_API_ERROR_UNSUPPORTED;
 925         }
 926     }
 927   return 0;
 928 }
 929
 930 int
 931 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 932                              u16 l_port, u16 e_port, ip_protocol_t proto,
 933                              u32 vrf_id, u32 sw_if_index, u32 flags,
 934                              ip4_address_t pool_addr, u8 *tag)
 935 {
 936   snat_static_mapping_resolve_t *rp;
 937   snat_main_t *sm = &snat_main;
 938   int rv;
 939
 940   if (!sm->enabled)
 941     {
 942       return VNET_API_ERROR_UNSUPPORTED;
 943     }
 944
 945   rv = nat44_ed_validate_sm_input (flags);
 946   if (rv != 0)
 947     {
 948       return rv;
 949     }
 950
 951   // interface bound mapping
 952   if (is_sm_switch_address (flags))
 953     {
 954       if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 955                                        sw_if_index, flags, 0))
 956         {
 957           return VNET_API_ERROR_VALUE_EXIST;
 958         }
 959
 960       vec_add2 (sm->sm_to_resolve, rp, 1);
 961       rp->l_addr.as_u32 = l_addr.as_u32;
 962       rp->l_port = l_port;
 963       rp->e_port = e_port;
 964       rp->sw_if_index = sw_if_index;
 965       rp->vrf_id = vrf_id;
 966       rp->proto = proto;
 967       rp->flags = flags;
 968       rp->pool_addr = pool_addr;
 969       rp->tag = vec_dup (tag);
 970       rp->is_resolved = 0;
 971
 972       ip4_address_t *first_int_addr =
 973         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 974       if (!first_int_addr)
 975         {
 976           return 0;
 977         }
 978
 979       e_addr.as_u32 = first_int_addr->as_u32;
 980       rp->is_resolved = 1;
 981     }
 982
 983   rv = nat44_ed_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
 984                                              proto, vrf_id, sw_if_index, flags,
 985                                              pool_addr, tag);
 986   if ((0 != rv) && is_sm_switch_address (flags))
 987     {
 988       nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 989                                    sw_if_index, flags);
 990     }
 991
 992   return rv;
 993 }
 994
 995 int
 996 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 997                              u16 l_port, u16 e_port, ip_protocol_t proto,
 998                              u32 vrf_id, u32 sw_if_index, u32 flags)
 999 {
1000   snat_main_t *sm = &snat_main;
1001   int rv;
1002
1003   if (!sm->enabled)
1004     {
1005       return VNET_API_ERROR_UNSUPPORTED;
1006     }
1007
1008   rv = nat44_ed_validate_sm_input (flags);
1009   if (rv != 0)
1010     {
1011       return rv;
1012     }
1013
1014   // interface bound mapping
1015   if (is_sm_switch_address (flags))
1016     {
1017       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
1018                                        sw_if_index, flags))
1019         {
1020           return VNET_API_ERROR_NO_SUCH_ENTRY;
1021         }
1022
1023       ip4_address_t *first_int_addr =
1024         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
1025       if (!first_int_addr)
1026         {
1027           // dhcp resolution required
1028           return 0;
1029         }
1030
1031       e_addr.as_u32 = first_int_addr->as_u32;
1032     }
1033
1034   return nat44_ed_del_static_mapping_internal (l_addr, e_addr, l_port, e_port,
1035                                                proto, vrf_id, flags);
1036 }
1037
1038 static int
1039 nat44_ed_add_static_mapping_internal (ip4_address_t l_addr,
1040                                       ip4_address_t e_addr, u16 l_port,
1041                                       u16 e_port, ip_protocol_t proto,
1042                                       u32 vrf_id, u32 sw_if_index, u32 flags,
1043                                       ip4_address_t pool_addr, u8 *tag)
1044 {
1045   snat_main_t *sm = &snat_main;
1046   nat44_lb_addr_port_t *local;
1047   snat_static_mapping_t *m;
1048   u32 fib_index = ~0;
1049
1050   if (is_sm_addr_only (flags))
1051     {
1052       e_port = l_port = proto = 0;
1053     }
1054
1055   if (is_sm_identity_nat (flags))
1056     {
1057       l_port = e_port;
1058       l_addr.as_u32 = e_addr.as_u32;
1059     }
1060
1061   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1062   if (m)
1063     {
1064       // case:
1065       // adding local identity nat record for different vrf table
1066
1067       if (!is_sm_identity_nat (m->flags))
1068         {
1069           return VNET_API_ERROR_VALUE_EXIST;
1070         }
1071
1072       pool_foreach (local, m->locals)
1073         {
1074           if (local->vrf_id == vrf_id)
1075             {
1076               return VNET_API_ERROR_VALUE_EXIST;
1077             }
1078         }
1079
1080       pool_get (m->locals, local);
1081
1082       local->vrf_id = vrf_id;
1083       local->fib_index = fib_table_find_or_create_and_lock (
1084         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
1085
1086       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port,
1087                            local->fib_index, m->proto);
1088
1089       return 0;
1090     }
1091
1092   if (vrf_id != ~0)
1093     {
1094       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1095                                                      sm->fib_src_low);
1096     }
1097   else
1098     {
1099       // fallback to default vrf
1100       vrf_id = sm->inside_vrf_id;
1101       fib_index = sm->inside_fib_index;
1102       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1103     }
1104
1105   // test if local mapping record doesn't exist
1106   // identity nat supports multiple records in local mapping
1107   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
1108     {
1109       if (nat44_ed_sm_i2o_lookup (sm, l_addr, l_port, fib_index, proto))
1110         {
1111           return VNET_API_ERROR_VALUE_EXIST;
1112         }
1113     }
1114
1115   pool_get (sm->static_mappings, m);
1116   clib_memset (m, 0, sizeof (*m));
1117
1118   m->flags = flags;
1119   m->local_addr = l_addr;
1120   m->external_addr = e_addr;
1121
1122   m->pool_addr = pool_addr;
1123   m->tag = vec_dup (tag);
1124
1125   if (!is_sm_addr_only (flags))
1126     {
1127       m->local_port = l_port;
1128       m->external_port = e_port;
1129       m->proto = proto;
1130     }
1131
1132   if (is_sm_identity_nat (flags))
1133     {
1134       pool_get (m->locals, local);
1135
1136       local->vrf_id = vrf_id;
1137       local->fib_index = fib_index;
1138     }
1139   else
1140     {
1141       m->vrf_id = vrf_id;
1142       m->fib_index = fib_index;
1143     }
1144
1145   if (!is_sm_out2in_only (flags))
1146     {
1147       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port, fib_index,
1148                            m->proto);
1149     }
1150
1151   nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0, m->proto);
1152
1153   if (sm->num_workers > 1)
1154     {
1155       // store worker index for this record
1156       ip4_header_t ip = {
1157         .src_address = m->local_addr,
1158       };
1159       u32 worker_index;
1160       worker_index =
1161         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
1162       vec_add1 (m->workers, worker_index);
1163     }
1164
1165   nat44_ed_add_del_interface_fib_reg_entries (e_addr, 1);
1166
1167   return 0;
1168 }
1169
1170 static int
1171 nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
1172                                       ip4_address_t e_addr, u16 l_port,
1173                                       u16 e_port, ip_protocol_t proto,
1174                                       u32 vrf_id, u32 flags)
1175 {
1176   snat_main_per_thread_data_t *tsm;
1177   snat_main_t *sm = &snat_main;
1178
1179   nat44_lb_addr_port_t *local;
1180   snat_static_mapping_t *m;
1181   u32 fib_index = ~0;
1182
1183   if (is_sm_addr_only (flags))
1184     {
1185       e_port = l_port = proto = 0;
1186     }
1187
1188   if (is_sm_identity_nat (flags))
1189     {
1190       l_port = e_port;
1191       l_addr.as_u32 = e_addr.as_u32;
1192     }
1193
1194   // fib index 0
1195   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1196   if (!m)
1197     {
1198       return VNET_API_ERROR_NO_SUCH_ENTRY;
1199     }
1200
1201   if (is_sm_identity_nat (flags))
1202     {
1203       u8 found = 0;
1204
1205       if (vrf_id == ~0)
1206         {
1207           vrf_id = sm->inside_vrf_id;
1208         }
1209
1210       pool_foreach (local, m->locals)
1211         {
1212           if (local->vrf_id == vrf_id)
1213             {
1214               local = pool_elt_at_index (m->locals, local - m->locals);
1215               fib_index = local->fib_index;
1216               pool_put (m->locals, local);
1217               found = 1;
1218             }
1219         }
1220
1221       if (!found)
1222         {
1223           return VNET_API_ERROR_NO_SUCH_ENTRY;
1224         }
1225     }
1226   else
1227     {
1228       fib_index = m->fib_index;
1229     }
1230
1231   if (!is_sm_out2in_only (flags))
1232     {
1233       nat44_ed_sm_i2o_del (sm, l_addr, l_port, fib_index, proto);
1234     }
1235
1236   // delete sessions for static mapping
1237   if (sm->num_workers > 1)
1238     {
1239       tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1240     }
1241   else
1242     {
1243       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1244     }
1245
1246   nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr, m->local_port,
1247                                       m->proto, fib_index,
1248                                       is_sm_addr_only (flags), e_addr, e_port);
1249
1250   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1251
1252   if (!pool_elts (m->locals))
1253     {
1254       // this is last record remove all required stuff
1255       // fib_index 0
1256       nat44_ed_sm_o2i_del (sm, e_addr, e_port, 0, proto);
1257
1258       vec_free (m->tag);
1259       vec_free (m->workers);
1260       pool_put (sm->static_mappings, m);
1261
1262       nat44_ed_add_del_interface_fib_reg_entries (e_addr, 0);
1263     }
1264
1265   return 0;
1266 }
1267
1268 int
1269 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1270                                 ip_protocol_t proto,
1271                                 nat44_lb_addr_port_t *locals, u32 flags,
1272                                 u8 *tag, u32 affinity)
1273 {
1274   snat_main_t *sm = &snat_main;
1275   snat_static_mapping_t *m;
1276   snat_address_t *a = 0;
1277
1278   nat44_lb_addr_port_t *local;
1279   uword *bitmap = 0;
1280   int rc = 0;
1281
1282   int i;
1283
1284   if (!sm->enabled)
1285     {
1286       return VNET_API_ERROR_UNSUPPORTED;
1287     }
1288
1289   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1290
1291   if (m)
1292     {
1293       return VNET_API_ERROR_VALUE_EXIST;
1294     }
1295
1296   if (vec_len (locals) < 2)
1297     {
1298       return VNET_API_ERROR_INVALID_VALUE;
1299     }
1300
1301   if (!is_sm_out2in_only (flags))
1302     {
1303       /* Find external address in allocated addresses and reserve port for
1304          address and port pair mapping when dynamic translations enabled */
1305       for (i = 0; i < vec_len (sm->addresses); i++)
1306         {
1307           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1308             {
1309               /* External port must be unused */
1310               a = sm->addresses + i;
1311               if (nat44_ed_sm_o2i_lookup (sm, a->addr, e_port, 0, proto))
1312                 {
1313                   return VNET_API_ERROR_VALUE_EXIST;
1314                 }
1315               break;
1316             }
1317         }
1318       // external address must be allocated
1319       if (!a)
1320         {
1321           return VNET_API_ERROR_NO_SUCH_ENTRY;
1322         }
1323     }
1324
1325   pool_get (sm->static_mappings, m);
1326   clib_memset (m, 0, sizeof (*m));
1327   m->tag = vec_dup (tag);
1328   m->external_addr = e_addr;
1329   m->external_port = e_port;
1330   m->affinity = affinity;
1331   m->proto = proto;
1332
1333   m->flags = flags;
1334   m->flags |= NAT_SM_FLAG_LB;
1335
1336   if (affinity)
1337     m->affinity_per_service_list_head_index =
1338       nat_affinity_get_per_service_list_head_index ();
1339   else
1340     m->affinity_per_service_list_head_index = ~0;
1341
1342   if (nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0,
1343                            m->proto))
1344     {
1345       nat_log_err ("sm o2i key add failed");
1346       return VNET_API_ERROR_UNSPECIFIED;
1347     }
1348
1349   for (i = 0; i < vec_len (locals); i++)
1350     {
1351       locals[i].fib_index = fib_table_find_or_create_and_lock (
1352         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1353       if (!is_sm_out2in_only (flags))
1354         {
1355           if (nat44_ed_sm_i2o_add (sm, m, locals[i].addr, locals[i].port, 0,
1356                                    proto))
1357             {
1358               nat_log_err ("sm i2o key add failed");
1359               rc = VNET_API_ERROR_UNSPECIFIED;
1360               // here we continue with add operation so that it can be safely
1361               // reversed in delete path - otherwise we'd have to track what
1362               // we've done and deal with partial cleanups and since bihash
1363               // adds are (extremely improbable) the only points of failure,
1364               // it's easier to just do it this way
1365             }
1366         }
1367       locals[i].prefix = (i == 0) ?
1368                            locals[i].probability :
1369                            (locals[i - 1].prefix + locals[i].probability);
1370       pool_get (m->locals, local);
1371       *local = locals[i];
1372       if (sm->num_workers > 1)
1373         {
1374           ip4_header_t ip = {
1375             .src_address = locals[i].addr,
1376           };
1377           bitmap = clib_bitmap_set (
1378             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1379             1);
1380         }
1381     }
1382
1383   /* Assign workers */
1384   if (sm->num_workers > 1)
1385     {
1386       clib_bitmap_foreach (i, bitmap)
1387         {
1388           vec_add1 (m->workers, i);
1389         }
1390     }
1391
1392   return rc;
1393 }
1394
1395 int
1396 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1397                                 ip_protocol_t proto, u32 flags)
1398 {
1399   snat_main_t *sm = &snat_main;
1400   snat_static_mapping_t *m;
1401
1402   nat44_lb_addr_port_t *local;
1403   snat_main_per_thread_data_t *tsm;
1404   snat_session_t *s;
1405
1406   if (!sm->enabled)
1407     {
1408       return VNET_API_ERROR_UNSUPPORTED;
1409     }
1410
1411   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1412   if (!m)
1413     return VNET_API_ERROR_NO_SUCH_ENTRY;
1414
1415   if (!is_sm_lb (m->flags))
1416     return VNET_API_ERROR_INVALID_VALUE;
1417
1418   if (nat44_ed_sm_o2i_del (sm, m->external_addr, m->external_port, 0,
1419                            m->proto))
1420     {
1421       nat_log_err ("sm o2i key del failed");
1422       return VNET_API_ERROR_UNSPECIFIED;
1423     }
1424
1425   pool_foreach (local, m->locals)
1426     {
1427       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1428       if (!is_sm_out2in_only (flags))
1429         {
1430           if (nat44_ed_sm_i2o_del (sm, local->addr, local->port,
1431                                    local->fib_index, m->proto))
1432             {
1433               nat_log_err ("sm i2o key del failed");
1434               // For the same reasons as above
1435             }
1436         }
1437
1438       if (sm->num_workers > 1)
1439         {
1440           ip4_header_t ip = {
1441             .src_address = local->addr,
1442           };
1443           tsm = vec_elt_at_index (
1444             sm->per_thread_data,
1445             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1446         }
1447       else
1448         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1449
1450       /* Delete sessions */
1451       pool_foreach (s, tsm->sessions)
1452         {
1453           if (!(nat44_ed_is_lb_session (s)))
1454             continue;
1455
1456           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1457               s->in2out.port != local->port)
1458             continue;
1459
1460           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1461           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1462         }
1463     }
1464
1465   if (m->affinity)
1466     {
1467       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1468     }
1469
1470   pool_free (m->locals);
1471   vec_free (m->tag);
1472   vec_free (m->workers);
1473   pool_put (sm->static_mappings, m);
1474
1475   return 0;
1476 }
1477
1478 int
1479 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1480                                           ip4_address_t l_addr, u16 l_port,
1481                                           ip_protocol_t proto, u32 vrf_id,
1482                                           u8 probability, u8 is_add)
1483 {
1484   snat_main_t *sm = &snat_main;
1485   snat_static_mapping_t *m = 0;
1486   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1487   snat_main_per_thread_data_t *tsm;
1488   snat_session_t *s;
1489   u32 *locals = 0;
1490   uword *bitmap = 0;
1491   int i;
1492
1493   if (!sm->enabled)
1494     {
1495       return VNET_API_ERROR_UNSUPPORTED;
1496     }
1497
1498   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1499
1500   if (!m)
1501     {
1502       return VNET_API_ERROR_NO_SUCH_ENTRY;
1503     }
1504
1505   if (!is_sm_lb (m->flags))
1506     {
1507       return VNET_API_ERROR_INVALID_VALUE;
1508     }
1509
1510   pool_foreach (local, m->locals)
1511    {
1512     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1513         (local->vrf_id == vrf_id))
1514       {
1515         match_local = local;
1516         break;
1517       }
1518   }
1519
1520   if (is_add)
1521     {
1522       if (match_local)
1523         {
1524           return VNET_API_ERROR_VALUE_EXIST;
1525         }
1526
1527       pool_get (m->locals, local);
1528       clib_memset (local, 0, sizeof (*local));
1529       local->addr.as_u32 = l_addr.as_u32;
1530       local->port = l_port;
1531       local->probability = probability;
1532       local->vrf_id = vrf_id;
1533       local->fib_index =
1534         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1535                                            sm->fib_src_low);
1536
1537       if (!is_sm_out2in_only (m->flags))
1538         {
1539           if (nat44_ed_sm_i2o_add (sm, m, l_addr, l_port, local->fib_index,
1540                                    proto))
1541             {
1542               nat_log_err ("sm i2o key add failed");
1543               pool_put (m->locals, local);
1544               return VNET_API_ERROR_UNSPECIFIED;
1545             }
1546         }
1547     }
1548   else
1549     {
1550       if (!match_local)
1551         return VNET_API_ERROR_NO_SUCH_ENTRY;
1552
1553       if (pool_elts (m->locals) < 3)
1554         return VNET_API_ERROR_UNSPECIFIED;
1555
1556       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1557                         sm->fib_src_low);
1558
1559       if (!is_sm_out2in_only (m->flags))
1560         {
1561           if (nat44_ed_sm_i2o_del (sm, l_addr, l_port, match_local->fib_index,
1562                                    proto))
1563             nat_log_err ("sm i2o key del failed");
1564         }
1565
1566       if (sm->num_workers > 1)
1567         {
1568           ip4_header_t ip = {
1569             .src_address = local->addr,
1570           };
1571           tsm = vec_elt_at_index (
1572             sm->per_thread_data,
1573             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1574         }
1575       else
1576         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1577
1578       /* Delete sessions */
1579       pool_foreach (s, tsm->sessions) {
1580           if (!(nat44_ed_is_lb_session (s)))
1581             continue;
1582
1583           if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1584               s->in2out.port != match_local->port)
1585             continue;
1586
1587           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1588           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1589       }
1590
1591       pool_put (m->locals, match_local);
1592     }
1593
1594   vec_free (m->workers);
1595
1596   pool_foreach (local, m->locals)
1597    {
1598     vec_add1 (locals, local - m->locals);
1599     if (sm->num_workers > 1)
1600       {
1601         ip4_header_t ip;
1602         ip.src_address.as_u32 = local->addr.as_u32,
1603         bitmap = clib_bitmap_set (
1604           bitmap,
1605           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1606       }
1607   }
1608
1609   ASSERT (vec_len (locals) > 1);
1610
1611   local = pool_elt_at_index (m->locals, locals[0]);
1612   local->prefix = local->probability;
1613   for (i = 1; i < vec_len (locals); i++)
1614     {
1615       local = pool_elt_at_index (m->locals, locals[i]);
1616       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1617       local->prefix = local->probability + prev_local->prefix;
1618     }
1619
1620   /* Assign workers */
1621   if (sm->num_workers > 1)
1622     {
1623       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1624     }
1625
1626   return 0;
1627 }
1628
1629 void
1630 expire_per_vrf_sessions (u32 fib_index)
1631 {
1632   per_vrf_sessions_t *per_vrf_sessions;
1633   snat_main_per_thread_data_t *tsm;
1634   snat_main_t *sm = &snat_main;
1635
1636   vec_foreach (tsm, sm->per_thread_data)
1637     {
1638       pool_foreach (per_vrf_sessions, tsm->per_vrf_sessions_pool)
1639         {
1640           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1641               (per_vrf_sessions->tx_fib_index == fib_index))
1642             {
1643               per_vrf_sessions->expired = 1;
1644             }
1645         }
1646     }
1647 }
1648
1649 void
1650 update_per_vrf_sessions_pool (u32 fib_index, int is_del)
1651 {
1652   snat_main_t *sm = &snat_main;
1653   nat_fib_t *fib;
1654
1655   // we don't care if it is outside/inside fib
1656   // we just care about their ref_count
1657   // if it reaches 0 sessions should expire
1658   // because the fib isn't valid for NAT anymore
1659
1660   vec_foreach (fib, sm->fibs)
1661   {
1662     if (fib->fib_index == fib_index)
1663       {
1664         if (is_del)
1665           {
1666             fib->ref_count--;
1667             if (!fib->ref_count)
1668               {
1669                 vec_del1 (sm->fibs, fib - sm->fibs);
1670                 expire_per_vrf_sessions (fib_index);
1671               }
1672             return;
1673           }
1674         else
1675           fib->ref_count++;
1676       }
1677   }
1678   if (!is_del)
1679     {
1680       vec_add2 (sm->fibs, fib, 1);
1681       fib->ref_count = 1;
1682       fib->fib_index = fib_index;
1683     }
1684 }
1685
1686 static_always_inline nat_fib_t *
1687 nat44_ed_get_outside_fib (nat_fib_t *outside_fibs, u32 fib_index)
1688 {
1689   nat_fib_t *f;
1690   vec_foreach (f, outside_fibs)
1691     {
1692       if (f->fib_index == fib_index)
1693         {
1694           return f;
1695         }
1696     }
1697   return 0;
1698 }
1699
1700 static_always_inline snat_interface_t *
1701 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1702 {
1703   snat_interface_t *i;
1704   pool_foreach (i, interfaces)
1705     {
1706       if (i->sw_if_index == sw_if_index)
1707         {
1708           return i;
1709         }
1710     }
1711   return 0;
1712 }
1713
1714 int
1715 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1716 {
1717   const char *del_feature_name, *feature_name;
1718   snat_main_t *sm = &snat_main;
1719
1720   nat_fib_t *outside_fib;
1721   snat_interface_t *i;
1722   u32 fib_index;
1723   int rv;
1724
1725   if (!sm->enabled)
1726     {
1727       nat_log_err ("nat44 is disabled");
1728       return VNET_API_ERROR_UNSUPPORTED;
1729     }
1730
1731   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1732     {
1733       nat_log_err ("error interface already configured");
1734       return VNET_API_ERROR_VALUE_EXIST;
1735     }
1736
1737   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1738   if (i)
1739     {
1740       if ((nat44_ed_is_interface_inside (i) && is_inside) ||
1741           (nat44_ed_is_interface_outside (i) && !is_inside))
1742         {
1743           return 0;
1744         }
1745       if (sm->num_workers > 1)
1746         {
1747           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1748                                           "nat44-out2in-worker-handoff";
1749           feature_name = "nat44-handoff-classify";
1750         }
1751       else
1752         {
1753           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1754
1755           feature_name = "nat44-ed-classify";
1756         }
1757
1758       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1759       if (rv)
1760         return rv;
1761       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1762                                    sw_if_index, 0, 0, 0);
1763       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1764                                    0, 0);
1765     }
1766   else
1767     {
1768       if (sm->num_workers > 1)
1769         {
1770           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1771                                      "nat44-out2in-worker-handoff";
1772         }
1773       else
1774         {
1775           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1776         }
1777
1778       nat_validate_interface_counters (sm, sw_if_index);
1779       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1780       if (rv)
1781         return rv;
1782       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1783                                    0, 0);
1784
1785       pool_get (sm->interfaces, i);
1786       i->sw_if_index = sw_if_index;
1787       i->flags = 0;
1788     }
1789
1790   fib_index =
1791     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1792
1793   update_per_vrf_sessions_pool (fib_index, 0 /*is_del*/);
1794
1795   if (!is_inside)
1796     {
1797       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1798
1799       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1800       if (outside_fib)
1801         {
1802           outside_fib->ref_count++;
1803         }
1804       else
1805         {
1806           vec_add2 (sm->outside_fibs, outside_fib, 1);
1807           outside_fib->fib_index = fib_index;
1808           outside_fib->ref_count = 1;
1809         }
1810
1811       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
1812       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
1813
1814       nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
1815     }
1816   else
1817     {
1818       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1819     }
1820
1821   return 0;
1822 }
1823
1824 int
1825 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1826 {
1827   const char *del_feature_name, *feature_name;
1828   snat_main_t *sm = &snat_main;
1829
1830   nat_fib_t *outside_fib;
1831   snat_interface_t *i;
1832   u32 fib_index;
1833   int rv;
1834
1835   if (!sm->enabled)
1836     {
1837       nat_log_err ("nat44 is disabled");
1838       return VNET_API_ERROR_UNSUPPORTED;
1839     }
1840
1841   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1842   if (i == 0)
1843     {
1844       nat_log_err ("error interface couldn't be found");
1845       return VNET_API_ERROR_NO_SUCH_ENTRY;
1846     }
1847
1848   if (nat44_ed_is_interface_inside (i) && nat44_ed_is_interface_outside (i))
1849     {
1850       if (sm->num_workers > 1)
1851         {
1852           del_feature_name = "nat44-handoff-classify";
1853           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1854                                       "nat44-out2in-worker-handoff";
1855         }
1856       else
1857         {
1858           del_feature_name = "nat44-ed-classify";
1859           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1860         }
1861
1862       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1863       if (rv)
1864         {
1865           return rv;
1866         }
1867       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1868                                    sw_if_index, 0, 0, 0);
1869       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1870                                    0, 0);
1871
1872       if (is_inside)
1873         {
1874           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1875         }
1876       else
1877         {
1878           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1879         }
1880     }
1881   else
1882     {
1883       if (sm->num_workers > 1)
1884         {
1885           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1886                                      "nat44-out2in-worker-handoff";
1887         }
1888       else
1889         {
1890           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1891         }
1892
1893       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1894       if (rv)
1895         {
1896           return rv;
1897         }
1898       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1899                                    0, 0);
1900
1901       // remove interface
1902       pool_put (sm->interfaces, i);
1903     }
1904
1905   fib_index =
1906     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1907
1908   update_per_vrf_sessions_pool (fib_index, 1 /*is_del*/);
1909
1910   if (!is_inside)
1911     {
1912       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1913       if (outside_fib)
1914         {
1915           outside_fib->ref_count--;
1916           if (!outside_fib->ref_count)
1917             {
1918               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1919             }
1920         }
1921
1922       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
1923       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
1924     }
1925
1926   return 0;
1927 }
1928
1929 int
1930 nat44_ed_add_output_interface (u32 sw_if_index)
1931 {
1932   snat_main_t *sm = &snat_main;
1933
1934   nat_fib_t *outside_fib;
1935   snat_interface_t *i;
1936   u32 fib_index;
1937   int rv;
1938
1939   if (!sm->enabled)
1940     {
1941       nat_log_err ("nat44 is disabled");
1942       return VNET_API_ERROR_UNSUPPORTED;
1943     }
1944
1945   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1946     {
1947       nat_log_err ("error interface already configured");
1948       return VNET_API_ERROR_VALUE_EXIST;
1949     }
1950
1951   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1952     {
1953       nat_log_err ("error interface already configured");
1954       return VNET_API_ERROR_VALUE_EXIST;
1955     }
1956
1957   if (sm->num_workers > 1)
1958     {
1959       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1960       if (rv)
1961         {
1962           return rv;
1963         }
1964
1965       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1966       if (rv)
1967         {
1968           return rv;
1969         }
1970
1971       vnet_feature_enable_disable (
1972         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1973       vnet_feature_enable_disable ("ip4-output",
1974                                    "nat44-in2out-output-worker-handoff",
1975                                    sw_if_index, 1, 0, 0);
1976     }
1977   else
1978     {
1979       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1980       if (rv)
1981         {
1982           return rv;
1983         }
1984
1985       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1986       if (rv)
1987         {
1988           return rv;
1989         }
1990
1991       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1992                                    sw_if_index, 1, 0, 0);
1993       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1994                                    sw_if_index, 1, 0, 0);
1995     }
1996
1997   nat_validate_interface_counters (sm, sw_if_index);
1998
1999   pool_get (sm->output_feature_interfaces, i);
2000   i->sw_if_index = sw_if_index;
2001   i->flags = 0;
2002   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2003   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2004
2005   fib_index =
2006     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2007   update_per_vrf_sessions_pool (fib_index, 0 /*is_del*/);
2008
2009   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2010   if (outside_fib)
2011     {
2012       outside_fib->ref_count++;
2013     }
2014   else
2015     {
2016       vec_add2 (sm->outside_fibs, outside_fib, 1);
2017       outside_fib->fib_index = fib_index;
2018       outside_fib->ref_count = 1;
2019     }
2020
2021   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
2022   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
2023
2024   nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
2025
2026   return 0;
2027 }
2028
2029 int
2030 nat44_ed_del_output_interface (u32 sw_if_index)
2031 {
2032   snat_main_t *sm = &snat_main;
2033
2034   nat_fib_t *outside_fib;
2035   snat_interface_t *i;
2036   u32 fib_index;
2037   int rv;
2038
2039   if (!sm->enabled)
2040     {
2041       nat_log_err ("nat44 is disabled");
2042       return VNET_API_ERROR_UNSUPPORTED;
2043     }
2044
2045   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2046   if (!i)
2047     {
2048       nat_log_err ("error interface couldn't be found");
2049       return VNET_API_ERROR_NO_SUCH_ENTRY;
2050     }
2051
2052   if (sm->num_workers > 1)
2053     {
2054       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2055       if (rv)
2056         {
2057           return rv;
2058         }
2059
2060       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2061       if (rv)
2062         {
2063           return rv;
2064         }
2065
2066       vnet_feature_enable_disable (
2067         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
2068       vnet_feature_enable_disable ("ip4-output",
2069                                    "nat44-in2out-output-worker-handoff",
2070                                    sw_if_index, 0, 0, 0);
2071     }
2072   else
2073     {
2074       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2075       if (rv)
2076         {
2077           return rv;
2078         }
2079
2080       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2081       if (rv)
2082         {
2083           return rv;
2084         }
2085
2086       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2087                                    sw_if_index, 0, 0, 0);
2088       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2089                                    sw_if_index, 0, 0, 0);
2090     }
2091
2092   // remove interface
2093   pool_put (sm->output_feature_interfaces, i);
2094
2095   fib_index =
2096     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2097   update_per_vrf_sessions_pool (fib_index, 1 /*is_del*/);
2098
2099   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2100   if (outside_fib)
2101     {
2102       outside_fib->ref_count--;
2103       if (!outside_fib->ref_count)
2104         {
2105           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2106         }
2107     }
2108
2109   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
2110   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
2111
2112   return 0;
2113 }
2114
2115 int
2116 snat_set_workers (uword * bitmap)
2117 {
2118   snat_main_t *sm = &snat_main;
2119   int i, j = 0;
2120
2121   if (sm->num_workers < 2)
2122     return VNET_API_ERROR_FEATURE_DISABLED;
2123
2124   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2125     return VNET_API_ERROR_INVALID_WORKER;
2126
2127   vec_free (sm->workers);
2128   clib_bitmap_foreach (i, bitmap)
2129     {
2130       vec_add1(sm->workers, i);
2131       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2132       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2133       j++;
2134     }
2135
2136   sm->port_per_thread = (65536 - ED_USER_PORT_OFFSET) / _vec_len (sm->workers);
2137
2138   return 0;
2139 }
2140
2141 int
2142 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2143 {
2144   fail_if_enabled ();
2145   snat_main_t *sm = &snat_main;
2146
2147   if ((sm->fq_in2out_index != ~0) || (sm->fq_out2in_index != ~0) ||
2148       (sm->fq_in2out_output_index != ~0))
2149     {
2150       // frame queu nelts can be set only before first
2151       // call to nat44_plugin_enable after that it
2152       // doesn't make sense
2153       nat_log_err ("Frame queue was already initialized. "
2154                    "Change is not possible");
2155       return 1;
2156     }
2157
2158   sm->frame_queue_nelts = frame_queue_nelts;
2159   return 0;
2160 }
2161
2162 static void
2163 nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque, u32 sw_if_index,
2164                                 u32 new_fib_index, u32 old_fib_index)
2165 {
2166   snat_main_t *sm = &snat_main;
2167   nat_fib_t *outside_fib;
2168   snat_interface_t *i;
2169   u8 is_add = 1;
2170   u8 match = 0;
2171
2172   if (!sm->enabled || (new_fib_index == old_fib_index)
2173       || (!vec_len (sm->outside_fibs)))
2174     {
2175       return;
2176     }
2177
2178   pool_foreach (i, sm->interfaces)
2179     {
2180       if (i->sw_if_index == sw_if_index)
2181         {
2182           if (!(nat44_ed_is_interface_outside (i)))
2183             return;
2184           match = 1;
2185         }
2186     }
2187
2188   pool_foreach (i, sm->output_feature_interfaces)
2189     {
2190       if (i->sw_if_index == sw_if_index)
2191         {
2192           if (!(nat44_ed_is_interface_outside (i)))
2193             return;
2194           match = 1;
2195         }
2196     }
2197
2198   if (!match)
2199     return;
2200
2201   vec_foreach (outside_fib, sm->outside_fibs)
2202     {
2203       if (outside_fib->fib_index == old_fib_index)
2204         {
2205           outside_fib->ref_count--;
2206           if (!outside_fib->ref_count)
2207             vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2208           break;
2209         }
2210     }
2211
2212   vec_foreach (outside_fib, sm->outside_fibs)
2213     {
2214       if (outside_fib->fib_index == new_fib_index)
2215         {
2216           outside_fib->ref_count++;
2217           is_add = 0;
2218           break;
2219         }
2220     }
2221
2222   if (is_add)
2223     {
2224       vec_add2 (sm->outside_fibs, outside_fib, 1);
2225       outside_fib->ref_count = 1;
2226       outside_fib->fib_index = new_fib_index;
2227     }
2228 }
2229
2230 static void nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque,
2231                                             u32 sw_if_index, u32 new_fib_index,
2232                                             u32 old_fib_index);
2233
2234 static void nat44_ed_add_del_interface_address_cb (
2235   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2236   u32 address_length, u32 if_address_index, u32 is_delete);
2237
2238 static void nat44_ed_add_del_static_mapping_cb (
2239   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2240   u32 address_length, u32 if_address_index, u32 is_delete);
2241
2242 void
2243 test_key_calc_split ()
2244 {
2245   ip4_address_t l_addr;
2246   l_addr.as_u8[0] = 1;
2247   l_addr.as_u8[1] = 1;
2248   l_addr.as_u8[2] = 1;
2249   l_addr.as_u8[3] = 1;
2250   ip4_address_t r_addr;
2251   r_addr.as_u8[0] = 2;
2252   r_addr.as_u8[1] = 2;
2253   r_addr.as_u8[2] = 2;
2254   r_addr.as_u8[3] = 2;
2255   u16 l_port = 40001;
2256   u16 r_port = 40301;
2257   u8 proto = 9;
2258   u32 fib_index = 9000001;
2259   u32 thread_index = 3000000001;
2260   u32 session_index = 3000000221;
2261   clib_bihash_kv_16_8_t kv;
2262   init_ed_kv (&kv, l_addr.as_u32, l_port, r_addr.as_u32, r_port, fib_index,
2263               proto, thread_index, session_index);
2264   ip4_address_t l_addr2;
2265   ip4_address_t r_addr2;
2266   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2267   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2268   u16 l_port2 = 0;
2269   u16 r_port2 = 0;
2270   u8 proto2 = 0;
2271   u32 fib_index2 = 0;
2272   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2273                &r_port2);
2274   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2275   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2276   ASSERT (l_port == l_port2);
2277   ASSERT (r_port == r_port2);
2278   ASSERT (proto == proto2);
2279   ASSERT (fib_index == fib_index2);
2280   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2281   ASSERT (session_index == ed_value_get_session_index (&kv));
2282 }
2283
2284 static clib_error_t *
2285 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2286 {
2287   u32 fib_index;
2288   if (!is_add)
2289     {
2290       fib_index = ip4_fib_index_from_table_id (table_id);
2291       if (fib_index != ~0)
2292         {
2293           expire_per_vrf_sessions (fib_index);
2294         }
2295     }
2296   return 0;
2297 }
2298
2299 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2300
2301 #define nat_validate_simple_counter(c, i)                                     \
2302   do                                                                          \
2303     {                                                                         \
2304       vlib_validate_simple_counter (&c, i);                                   \
2305       vlib_zero_simple_counter (&c, i);                                       \
2306     }                                                                         \
2307   while (0);
2308
2309 #define nat_init_simple_counter(c, n, sn)                                     \
2310   do                                                                          \
2311     {                                                                         \
2312       c.name = n;                                                             \
2313       c.stat_segment_name = sn;                                               \
2314       nat_validate_simple_counter (c, 0);                                     \
2315     }                                                                         \
2316   while (0);
2317
2318 static_always_inline void
2319 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2320 {
2321 #define _(x)                                                                  \
2322   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2323   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2324   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2325   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2326   foreach_nat_counter;
2327 #undef _
2328   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2329 }
2330
2331 static clib_error_t *
2332 nat_init (vlib_main_t * vm)
2333 {
2334   snat_main_t *sm = &snat_main;
2335   vlib_thread_main_t *tm = vlib_get_thread_main ();
2336   vlib_thread_registration_t *tr;
2337   ip4_add_del_interface_address_callback_t cbi = { 0 };
2338   ip4_table_bind_callback_t cbt = { 0 };
2339   u32 i, num_threads = 0;
2340   uword *p, *bitmap = 0;
2341
2342   clib_memset (sm, 0, sizeof (*sm));
2343
2344   // convenience
2345   sm->ip4_main = &ip4_main;
2346
2347   // frame queue indices used for handoff
2348   sm->fq_out2in_index = ~0;
2349   sm->fq_in2out_index = ~0;
2350   sm->fq_in2out_output_index = ~0;
2351
2352   sm->log_level = NAT_LOG_ERROR;
2353
2354   sm->log_class = vlib_log_register_class ("nat", 0);
2355   nat_ipfix_logging_init (vm);
2356
2357   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2358                            "/nat44-ed/total-sessions");
2359   sm->max_cfg_sessions_gauge =
2360     vlib_stats_add_gauge ("/nat44-ed/max-cfg-sessions");
2361
2362 #define _(x)                                                                  \
2363   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2364                            "/nat44-ed/in2out/fastpath/" #x);                  \
2365   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2366                            "/nat44-ed/out2in/fastpath/" #x);                  \
2367   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2368                            "/nat44-ed/in2out/slowpath/" #x);                  \
2369   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2370                            "/nat44-ed/out2in/slowpath/" #x);
2371   foreach_nat_counter;
2372 #undef _
2373   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2374                            "/nat44-ed/hairpinning");
2375
2376   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2377   if (p)
2378     {
2379       tr = (vlib_thread_registration_t *) p[0];
2380       if (tr)
2381         {
2382           sm->num_workers = tr->count;
2383           sm->first_worker_index = tr->first_index;
2384         }
2385     }
2386   num_threads = tm->n_vlib_mains - 1;
2387   sm->port_per_thread = 65536 - ED_USER_PORT_OFFSET;
2388   vec_validate (sm->per_thread_data, num_threads);
2389
2390   /* Use all available workers by default */
2391   if (sm->num_workers > 1)
2392     {
2393       for (i = 0; i < sm->num_workers; i++)
2394         bitmap = clib_bitmap_set (bitmap, i, 1);
2395       snat_set_workers (bitmap);
2396       clib_bitmap_free (bitmap);
2397     }
2398   else
2399     {
2400       sm->per_thread_data[0].snat_thread_index = 0;
2401     }
2402
2403   /* callbacks to call when interface address changes. */
2404   cbi.function = nat44_ed_add_del_interface_address_cb;
2405   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2406   cbi.function = nat44_ed_add_del_static_mapping_cb;
2407   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2408
2409   /* callbacks to call when interface to table biding changes */
2410   cbt.function = nat44_ed_update_outside_fib_cb;
2411   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2412
2413   sm->fib_src_low =
2414     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2415                          FIB_SOURCE_BH_SIMPLE);
2416   sm->fib_src_hi =
2417     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2418                          FIB_SOURCE_BH_SIMPLE);
2419
2420   nat_affinity_init (vm);
2421   test_key_calc_split ();
2422
2423   return nat44_api_hookup (vm);
2424 }
2425
2426 VLIB_INIT_FUNCTION (nat_init);
2427
2428 int
2429 nat44_plugin_enable (nat44_config_t c)
2430 {
2431   snat_main_t *sm = &snat_main;
2432
2433   fail_if_enabled ();
2434
2435   sm->forwarding_enabled = 0;
2436   sm->mss_clamping = 0;
2437
2438   if (!c.sessions)
2439     c.sessions = 63 * 1024;
2440
2441   sm->max_translations_per_thread = c.sessions;
2442   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
2443                         sm->max_translations_per_thread);
2444   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2445
2446   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2447
2448   sm->inside_vrf_id = c.inside_vrf;
2449   sm->inside_fib_index =
2450     fib_table_find_or_create_and_lock
2451     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2452
2453   sm->outside_vrf_id = c.outside_vrf;
2454   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2455     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2456
2457   nat44_ed_db_init ();
2458
2459   nat_affinity_enable ();
2460
2461   nat_reset_timeouts (&sm->timeouts);
2462
2463   vlib_zero_simple_counter (&sm->total_sessions, 0);
2464
2465   if (!sm->frame_queue_nelts)
2466     {
2467       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2468     }
2469
2470   if (sm->num_workers > 1)
2471     {
2472       vlib_main_t *vm = vlib_get_main ();
2473       vlib_node_t *node;
2474
2475       if (sm->fq_in2out_index == ~0)
2476         {
2477           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2478           sm->fq_in2out_index =
2479             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2480         }
2481       if (sm->fq_out2in_index == ~0)
2482         {
2483           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2484           sm->fq_out2in_index =
2485             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2486         }
2487       if (sm->fq_in2out_output_index == ~0)
2488         {
2489           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2490           sm->fq_in2out_output_index =
2491             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2492         }
2493     }
2494
2495   sm->enabled = 1;
2496   sm->rconfig = c;
2497
2498   return 0;
2499 }
2500
2501 int
2502 nat44_ed_del_addresses ()
2503 {
2504   snat_main_t *sm = &snat_main;
2505   snat_address_t *a, *vec;
2506   int error = 0;
2507
2508   vec = vec_dup (sm->addresses);
2509   vec_foreach (a, vec)
2510     {
2511       error = nat44_ed_del_address (a->addr, 0);
2512       if (error)
2513         {
2514           nat_log_err ("error occurred while removing adderess");
2515         }
2516     }
2517   vec_free (vec);
2518   vec_free (sm->addresses);
2519   sm->addresses = 0;
2520
2521   vec = vec_dup (sm->twice_nat_addresses);
2522   vec_foreach (a, vec)
2523     {
2524       error = nat44_ed_del_address (a->addr, 1);
2525       if (error)
2526         {
2527           nat_log_err ("error occurred while removing adderess");
2528         }
2529     }
2530   vec_free (vec);
2531   vec_free (sm->twice_nat_addresses);
2532   sm->twice_nat_addresses = 0;
2533
2534   vec_free (sm->addr_to_resolve);
2535   sm->addr_to_resolve = 0;
2536
2537   return error;
2538 }
2539
2540 int
2541 nat44_ed_del_interfaces ()
2542 {
2543   snat_main_t *sm = &snat_main;
2544   snat_interface_t *i, *pool;
2545   int error = 0;
2546
2547   pool = pool_dup (sm->interfaces);
2548   pool_foreach (i, pool)
2549     {
2550       if (nat44_ed_is_interface_inside (i))
2551         {
2552           error = nat44_ed_del_interface (i->sw_if_index, 1);
2553         }
2554       if (nat44_ed_is_interface_outside (i))
2555         {
2556           error = nat44_ed_del_interface (i->sw_if_index, 0);
2557         }
2558
2559       if (error)
2560         {
2561           nat_log_err ("error occurred while removing interface");
2562         }
2563     }
2564   pool_free (pool);
2565   pool_free (sm->interfaces);
2566   sm->interfaces = 0;
2567   return error;
2568 }
2569
2570 int
2571 nat44_ed_del_output_interfaces ()
2572 {
2573   snat_main_t *sm = &snat_main;
2574   snat_interface_t *i, *pool;
2575   int error = 0;
2576
2577   pool = pool_dup (sm->output_feature_interfaces);
2578   pool_foreach (i, pool)
2579     {
2580       error = nat44_ed_del_output_interface (i->sw_if_index);
2581       if (error)
2582         {
2583           nat_log_err ("error occurred while removing output interface");
2584         }
2585     }
2586   pool_free (pool);
2587   pool_free (sm->output_feature_interfaces);
2588   sm->output_feature_interfaces = 0;
2589   return error;
2590 }
2591
2592 static clib_error_t *
2593 nat44_ed_sw_interface_add_del (vnet_main_t *vnm, u32 sw_if_index, u32 is_add)
2594 {
2595   snat_main_t *sm = &snat_main;
2596   snat_interface_t *i;
2597   int error = 0;
2598
2599   if (is_add)
2600     return 0;
2601
2602   if (!sm->enabled)
2603     return 0;
2604
2605   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
2606   if (i)
2607     {
2608       bool is_inside = nat44_ed_is_interface_inside (i);
2609       bool is_outside = nat44_ed_is_interface_outside (i);
2610
2611       if (is_inside)
2612         {
2613           error |= nat44_ed_del_interface (sw_if_index, 1);
2614         }
2615       if (is_outside)
2616         {
2617           error |= nat44_ed_del_interface (sw_if_index, 0);
2618         }
2619
2620       if (error)
2621         {
2622           nat_log_err ("error occurred while removing interface");
2623         }
2624     }
2625
2626   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2627   if (i)
2628     {
2629       error = nat44_ed_del_output_interface (sw_if_index);
2630       if (error)
2631         {
2632           nat_log_err ("error occurred while removing output interface");
2633         }
2634     }
2635
2636   return 0;
2637 }
2638
2639 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (nat44_ed_sw_interface_add_del);
2640
2641 int
2642 nat44_ed_del_static_mappings ()
2643 {
2644   snat_main_t *sm = &snat_main;
2645   snat_static_mapping_t *m, *pool;
2646   int error = 0;
2647
2648   pool = pool_dup (sm->static_mappings);
2649   pool_foreach (m, pool)
2650     {
2651       error = nat44_ed_del_static_mapping_internal (
2652         m->local_addr, m->external_addr, m->local_port, m->external_port,
2653         m->proto, m->vrf_id, m->flags);
2654       if (error)
2655         {
2656           nat_log_err ("error occurred while removing mapping");
2657         }
2658     }
2659   pool_free (pool);
2660   pool_free (sm->static_mappings);
2661   sm->static_mappings = 0;
2662
2663   vec_free (sm->sm_to_resolve);
2664   sm->sm_to_resolve = 0;
2665
2666   return error;
2667 }
2668
2669 int
2670 nat44_plugin_disable ()
2671 {
2672   snat_main_t *sm = &snat_main;
2673   int rc, error = 0;
2674
2675   fail_if_disabled ();
2676
2677   rc = nat44_ed_del_static_mappings ();
2678   if (rc)
2679     error = VNET_API_ERROR_BUG;
2680
2681   rc = nat44_ed_del_addresses ();
2682   if (rc)
2683     error = VNET_API_ERROR_BUG;
2684
2685   rc = nat44_ed_del_interfaces ();
2686   if (rc)
2687     error = VNET_API_ERROR_BUG;
2688
2689   rc = nat44_ed_del_output_interfaces ();
2690   if (rc)
2691     error = VNET_API_ERROR_BUG;
2692
2693   nat44_ed_del_vrf_tables ();
2694
2695   vec_free (sm->max_translations_per_fib);
2696   sm->max_translations_per_fib = 0;
2697
2698   nat44_ed_db_free ();
2699
2700   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2701
2702   nat_affinity_disable ();
2703
2704   sm->forwarding_enabled = 0;
2705   sm->enabled = 0;
2706
2707   return error;
2708 }
2709
2710 void
2711 nat44_ed_forwarding_enable_disable (u8 is_enable)
2712 {
2713   snat_main_per_thread_data_t *tsm;
2714   snat_main_t *sm = &snat_main;
2715   snat_session_t *s;
2716
2717   u32 *ses_to_be_removed = 0, *ses_index;
2718
2719   sm->forwarding_enabled = is_enable != 0;
2720
2721   if (!sm->enabled || is_enable)
2722     {
2723       return;
2724     }
2725
2726   vec_foreach (tsm, sm->per_thread_data)
2727     {
2728       pool_foreach (s, tsm->sessions)
2729         {
2730           if (na44_ed_is_fwd_bypass_session (s))
2731             {
2732               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2733             }
2734         }
2735       vec_foreach (ses_index, ses_to_be_removed)
2736         {
2737           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2738           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2739           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2740         }
2741
2742       vec_free (ses_to_be_removed);
2743     }
2744 }
2745
2746 static_always_inline snat_static_mapping_t *
2747 nat44_ed_sm_match (snat_main_t *sm, ip4_address_t match_addr, u16 match_port,
2748                    u32 match_fib_index, ip_protocol_t match_protocol,
2749                    int by_external)
2750 {
2751   snat_static_mapping_t *m;
2752   if (!by_external)
2753     {
2754       m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port, match_fib_index,
2755                                   match_protocol);
2756       if (m)
2757         return m;
2758
2759       // try address only mapping
2760       m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, match_fib_index, 0);
2761       if (m)
2762         return m;
2763
2764       // default static mapping fib index (based on configuration)
2765       if (sm->inside_fib_index != match_fib_index)
2766         {
2767           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2768                                       sm->inside_fib_index, match_protocol);
2769           if (m)
2770             return m;
2771
2772           // try address only mapping
2773           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->inside_fib_index,
2774                                       0);
2775           if (m)
2776             return m;
2777         }
2778       // TODO: this specific use case may be deprecated (needs testing)
2779       if (sm->outside_fib_index != match_fib_index)
2780         {
2781           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2782                                       sm->outside_fib_index, match_protocol);
2783           if (m)
2784             return m;
2785
2786           // try address only mapping
2787           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->outside_fib_index,
2788                                       0);
2789           if (m)
2790             return m;
2791         }
2792     }
2793   else
2794     {
2795       m =
2796         nat44_ed_sm_o2i_lookup (sm, match_addr, match_port, 0, match_protocol);
2797       if (m)
2798         return m;
2799
2800       // try address only mapping
2801       m = nat44_ed_sm_o2i_lookup (sm, match_addr, 0, 0, 0);
2802       if (m)
2803         return m;
2804     }
2805   return 0;
2806 }
2807
2808 int
2809 snat_static_mapping_match (vlib_main_t *vm, ip4_address_t match_addr,
2810                            u16 match_port, u32 match_fib_index,
2811                            ip_protocol_t match_protocol,
2812                            ip4_address_t *mapping_addr, u16 *mapping_port,
2813                            u32 *mapping_fib_index, int by_external,
2814                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2815                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2816                            u8 *is_identity_nat, snat_static_mapping_t **out)
2817 {
2818   snat_main_t *sm = &snat_main;
2819   snat_static_mapping_t *m;
2820   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2821   nat44_lb_addr_port_t *local;
2822   u8 backend_index;
2823
2824   m = nat44_ed_sm_match (sm, match_addr, match_port, match_fib_index,
2825                          match_protocol, by_external);
2826   if (!m)
2827     {
2828       return 1;
2829     }
2830
2831   if (by_external)
2832     {
2833       if (is_sm_lb (m->flags))
2834         {
2835           if (PREDICT_FALSE (lb != 0))
2836             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2837           if (m->affinity && !nat_affinity_find_and_lock (
2838                                vm, ext_host_addr[0], match_addr,
2839                                match_protocol, match_port, &backend_index))
2840             {
2841               local = pool_elt_at_index (m->locals, backend_index);
2842               *mapping_addr = local->addr;
2843               *mapping_port = local->port;
2844               *mapping_fib_index = local->fib_index;
2845               goto end;
2846             }
2847           // pick locals matching this worker
2848           if (PREDICT_FALSE (sm->num_workers > 1))
2849             {
2850               u32 thread_index = vlib_get_thread_index ();
2851               pool_foreach_index (i, m->locals)
2852                {
2853                 local = pool_elt_at_index (m->locals, i);
2854
2855                 ip4_header_t ip = {
2856                   .src_address = local->addr,
2857                 };
2858
2859                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2860                                                       0) == thread_index)
2861                   {
2862                     vec_add1 (tmp, i);
2863                   }
2864                }
2865               ASSERT (vec_len (tmp) != 0);
2866             }
2867           else
2868             {
2869               pool_foreach_index (i, m->locals)
2870                {
2871                 vec_add1 (tmp, i);
2872               }
2873             }
2874           hi = vec_len (tmp) - 1;
2875           local = pool_elt_at_index (m->locals, tmp[hi]);
2876           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2877           while (lo < hi)
2878             {
2879               mid = ((hi - lo) >> 1) + lo;
2880               local = pool_elt_at_index (m->locals, tmp[mid]);
2881               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2882             }
2883           local = pool_elt_at_index (m->locals, tmp[lo]);
2884           if (!(local->prefix >= rand))
2885             return 1;
2886           *mapping_addr = local->addr;
2887           *mapping_port = local->port;
2888           *mapping_fib_index = local->fib_index;
2889           if (m->affinity)
2890             {
2891               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2892                                                 match_protocol, match_port,
2893                                                 tmp[lo], m->affinity,
2894                                                 m->affinity_per_service_list_head_index))
2895                 nat_elog_info (sm, "create affinity record failed");
2896             }
2897           vec_free (tmp);
2898         }
2899       else
2900         {
2901           if (PREDICT_FALSE (lb != 0))
2902             *lb = NO_LB_NAT;
2903           *mapping_fib_index = m->fib_index;
2904           *mapping_addr = m->local_addr;
2905           /* Address only mapping doesn't change port */
2906           *mapping_port =
2907             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2908         }
2909     }
2910   else
2911     {
2912       *mapping_addr = m->external_addr;
2913       /* Address only mapping doesn't change port */
2914       *mapping_port =
2915         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2916       *mapping_fib_index = sm->outside_fib_index;
2917     }
2918
2919 end:
2920   if (PREDICT_FALSE (is_addr_only != 0))
2921     *is_addr_only = is_sm_addr_only (m->flags);
2922
2923   if (PREDICT_FALSE (twice_nat != 0))
2924     {
2925       *twice_nat = TWICE_NAT_DISABLED;
2926
2927       if (is_sm_twice_nat (m->flags))
2928         {
2929           *twice_nat = TWICE_NAT;
2930         }
2931       else if (is_sm_self_twice_nat (m->flags))
2932         {
2933           *twice_nat = TWICE_NAT_SELF;
2934         }
2935     }
2936
2937   if (PREDICT_FALSE (is_identity_nat != 0))
2938     *is_identity_nat = is_sm_identity_nat (m->flags);
2939
2940   if (out != 0)
2941     *out = m;
2942
2943   return 0;
2944 }
2945
2946 u32
2947 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2948                                   u32 rx_fib_index, u8 is_output)
2949 {
2950   snat_main_t *sm = &snat_main;
2951   u32 next_worker_index = sm->first_worker_index;
2952   u32 hash;
2953
2954   clib_bihash_kv_16_8_t kv16, value16;
2955
2956   u32 fib_index = rx_fib_index;
2957   if (b)
2958     {
2959       if (PREDICT_FALSE (is_output))
2960         {
2961           fib_index = sm->outside_fib_index;
2962           nat_fib_t *outside_fib;
2963           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2964           fib_prefix_t pfx = {
2965                   .fp_proto = FIB_PROTOCOL_IP4,
2966                   .fp_len = 32,
2967                   .fp_addr = {
2968                           .ip4.as_u32 = ip->dst_address.as_u32,
2969                   } ,
2970           };
2971
2972           switch (vec_len (sm->outside_fibs))
2973             {
2974             case 0:
2975               fib_index = sm->outside_fib_index;
2976               break;
2977             case 1:
2978               fib_index = sm->outside_fibs[0].fib_index;
2979               break;
2980             default:
2981               vec_foreach (outside_fib, sm->outside_fibs)
2982                 {
2983                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2984                   if (FIB_NODE_INDEX_INVALID != fei)
2985                     {
2986                       if (fib_entry_get_resolving_interface (fei) != ~0)
2987                         {
2988                           fib_index = outside_fib->fib_index;
2989                           break;
2990                         }
2991                     }
2992                 }
2993               break;
2994             }
2995         }
2996
2997       if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2998         {
2999           ip4_address_t lookup_saddr, lookup_daddr;
3000           u16 lookup_sport, lookup_dport;
3001           u8 lookup_protocol;
3002
3003           if (!nat_get_icmp_session_lookup_values (
3004                 b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr,
3005                 &lookup_dport, &lookup_protocol))
3006             {
3007               init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3008                          lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3009                          lookup_protocol);
3010               if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3011                 {
3012                   next_worker_index = ed_value_get_thread_index (&value16);
3013                   vnet_buffer2 (b)->nat.cached_session_index =
3014                     ed_value_get_session_index (&value16);
3015                   goto out;
3016                 }
3017             }
3018         }
3019
3020       init_ed_k (&kv16, ip->src_address.as_u32,
3021                  vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3022                  vnet_buffer (b)->ip.reass.l4_dst_port, fib_index,
3023                  ip->protocol);
3024
3025       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3026         {
3027           next_worker_index = ed_value_get_thread_index (&value16);
3028           vnet_buffer2 (b)->nat.cached_session_index =
3029             ed_value_get_session_index (&value16);
3030           goto out;
3031         }
3032
3033       // dst NAT
3034       init_ed_k (&kv16, ip->dst_address.as_u32,
3035                  vnet_buffer (b)->ip.reass.l4_dst_port, ip->src_address.as_u32,
3036                  vnet_buffer (b)->ip.reass.l4_src_port, rx_fib_index,
3037                  ip->protocol);
3038       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3039         {
3040           next_worker_index = ed_value_get_thread_index (&value16);
3041           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
3042             ed_value_get_session_index (&value16);
3043           goto out;
3044         }
3045     }
3046
3047   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3048          (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24) +
3049          rx_fib_index + (rx_fib_index >> 8) + (rx_fib_index >> 16) +
3050          (rx_fib_index >> 24);
3051
3052   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3053     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3054   else
3055     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3056
3057 out:
3058   if (PREDICT_TRUE (!is_output))
3059     {
3060       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
3061                               rx_fib_index,
3062                               clib_net_to_host_u32 (ip->src_address.as_u32),
3063                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3064     }
3065   else
3066     {
3067       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
3068                               next_worker_index, rx_fib_index,
3069                               clib_net_to_host_u32 (ip->src_address.as_u32),
3070                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3071     }
3072
3073   return next_worker_index;
3074 }
3075
3076 u32
3077 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
3078                                   u32 rx_fib_index, u8 is_output)
3079 {
3080   snat_main_t *sm = &snat_main;
3081   clib_bihash_kv_16_8_t kv16, value16;
3082
3083   u8 proto, next_worker_index = 0;
3084   u16 port;
3085   snat_static_mapping_t *m;
3086   u32 hash;
3087
3088   proto = ip->protocol;
3089
3090   if (PREDICT_FALSE (IP_PROTOCOL_ICMP == proto))
3091     {
3092       ip4_address_t lookup_saddr, lookup_daddr;
3093       u16 lookup_sport, lookup_dport;
3094       u8 lookup_protocol;
3095       if (!nat_get_icmp_session_lookup_values (
3096             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3097             &lookup_protocol))
3098         {
3099           init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3100                      lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3101                      lookup_protocol);
3102           if (PREDICT_TRUE (
3103                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3104             {
3105               next_worker_index = ed_value_get_thread_index (&value16);
3106               nat_elog_debug_handoff (
3107                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
3108                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
3109                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3110               return next_worker_index;
3111             }
3112         }
3113     }
3114
3115   init_ed_k (&kv16, ip->src_address.as_u32,
3116              vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3117              vnet_buffer (b)->ip.reass.l4_dst_port, rx_fib_index,
3118              ip->protocol);
3119
3120   if (PREDICT_TRUE (
3121         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3122     {
3123       vnet_buffer2 (b)->nat.cached_session_index =
3124         ed_value_get_session_index (&value16);
3125       next_worker_index = ed_value_get_thread_index (&value16);
3126       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
3127                               next_worker_index, rx_fib_index,
3128                               clib_net_to_host_u32 (ip->src_address.as_u32),
3129                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3130       return next_worker_index;
3131     }
3132
3133   /* first try static mappings without port */
3134   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3135     {
3136       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, proto);
3137       if (m)
3138         {
3139           {
3140             next_worker_index = m->workers[0];
3141             goto done;
3142           }
3143         }
3144     }
3145
3146   /* unknown protocol */
3147   if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto)))
3148     {
3149       /* use current thread */
3150       next_worker_index = vlib_get_thread_index ();
3151       goto done;
3152     }
3153
3154   port = vnet_buffer (b)->ip.reass.l4_dst_port;
3155
3156   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3157     {
3158       udp_header_t *udp = ip4_next_header (ip);
3159       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3160       nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3161       if (!icmp_type_is_error_message
3162           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3163         port = vnet_buffer (b)->ip.reass.l4_src_port;
3164       else
3165         {
3166           /* if error message, then it's not fragmented and we can access it */
3167           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3168           proto = inner_ip->protocol;
3169           void *l4_header = ip4_next_header (inner_ip);
3170           switch (proto)
3171             {
3172             case IP_PROTOCOL_ICMP:
3173               icmp = (icmp46_header_t *) l4_header;
3174               echo = (nat_icmp_echo_header_t *) (icmp + 1);
3175               port = echo->identifier;
3176               break;
3177             case IP_PROTOCOL_UDP:
3178               /* breakthrough */
3179             case IP_PROTOCOL_TCP:
3180               port = ((nat_tcp_udp_header_t *) l4_header)->src_port;
3181               break;
3182             default:
3183               next_worker_index = vlib_get_thread_index ();
3184               goto done;
3185             }
3186         }
3187     }
3188
3189   /* try static mappings with port */
3190   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3191     {
3192       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, port, 0, proto);
3193       if (m)
3194         {
3195           if (!is_sm_lb (m->flags))
3196             {
3197               next_worker_index = m->workers[0];
3198               goto done;
3199             }
3200
3201           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3202             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3203
3204           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3205             next_worker_index =
3206               m->workers[hash & (_vec_len (m->workers) - 1)];
3207           else
3208             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3209           goto done;
3210         }
3211     }
3212
3213   /* worker by outside port */
3214   next_worker_index = get_thread_idx_by_port (clib_net_to_host_u16 (port));
3215
3216 done:
3217   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3218                           rx_fib_index,
3219                           clib_net_to_host_u32 (ip->src_address.as_u32),
3220                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3221   return next_worker_index;
3222 }
3223
3224 u32
3225 nat44_get_max_session_limit ()
3226 {
3227   snat_main_t *sm = &snat_main;
3228   u32 max_limit = 0, len = 0;
3229
3230   for (; len < vec_len (sm->max_translations_per_fib); len++)
3231     {
3232       if (max_limit < sm->max_translations_per_fib[len])
3233         max_limit = sm->max_translations_per_fib[len];
3234     }
3235   return max_limit;
3236 }
3237
3238 int
3239 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3240 {
3241   snat_main_t *sm = &snat_main;
3242   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3243
3244   if (~0 == fib_index)
3245     return -1;
3246
3247   vec_validate_init_empty (sm->max_translations_per_fib, fib_index,
3248                            sm->max_translations_per_thread);
3249   sm->max_translations_per_fib[fib_index] = session_limit;
3250   return 0;
3251 }
3252
3253 int
3254 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3255 {
3256   snat_main_t *sm = &snat_main;
3257
3258   if (nat44_set_session_limit (session_limit, vrf_id))
3259     return 1;
3260   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3261
3262   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
3263                         sm->max_translations_per_thread);
3264
3265   sm->translation_buckets =
3266     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3267
3268   nat44_ed_sessions_clear ();
3269   return 0;
3270 }
3271
3272 static void
3273 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations)
3274 {
3275   dlist_elt_t *head;
3276
3277   pool_alloc (tsm->per_vrf_sessions_pool, translations);
3278   pool_alloc (tsm->sessions, translations);
3279   pool_alloc (tsm->lru_pool, translations);
3280
3281   pool_get (tsm->lru_pool, head);
3282   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3283   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3284
3285   pool_get (tsm->lru_pool, head);
3286   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3287   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3288
3289   pool_get (tsm->lru_pool, head);
3290   tsm->udp_lru_head_index = head - tsm->lru_pool;
3291   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3292
3293   pool_get (tsm->lru_pool, head);
3294   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3295   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3296
3297   pool_get (tsm->lru_pool, head);
3298   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3299   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3300 }
3301
3302 static void
3303 nat44_ed_flow_hash_init ()
3304 {
3305   snat_main_t *sm = &snat_main;
3306   // we expect 2 flows per session, so multiply translation_buckets by 2
3307   clib_bihash_init_16_8 (
3308     &sm->flow_hash, "ed-flow-hash",
3309     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3310   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3311 }
3312
3313 static void
3314 nat44_ed_db_init ()
3315 {
3316   snat_main_t *sm = &snat_main;
3317   snat_main_per_thread_data_t *tsm;
3318
3319   nat44_ed_flow_hash_init ();
3320
3321   vec_foreach (tsm, sm->per_thread_data)
3322     {
3323       nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread);
3324     }
3325 }
3326
3327 static void
3328 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3329 {
3330   pool_free (tsm->lru_pool);
3331   pool_free (tsm->sessions);
3332   pool_free (tsm->per_vrf_sessions_pool);
3333 }
3334
3335 static void
3336 nat44_ed_flow_hash_free ()
3337 {
3338   snat_main_t *sm = &snat_main;
3339
3340   clib_bihash_free_16_8 (&sm->flow_hash);
3341 }
3342
3343 static void
3344 nat44_ed_db_free ()
3345 {
3346   snat_main_t *sm = &snat_main;
3347   snat_main_per_thread_data_t *tsm;
3348
3349   vec_foreach (tsm, sm->per_thread_data)
3350     {
3351       nat44_ed_worker_db_free (tsm);
3352     }
3353
3354   nat44_ed_flow_hash_free ();
3355 }
3356
3357 void
3358 nat44_ed_sessions_clear ()
3359 {
3360   snat_main_t *sm = &snat_main;
3361
3362   nat44_ed_db_free ();
3363   nat44_ed_db_init ();
3364   vlib_zero_simple_counter (&sm->total_sessions, 0);
3365 }
3366
3367 static void
3368 nat44_ed_add_del_static_mapping_cb (ip4_main_t *im, uword opaque,
3369                                     u32 sw_if_index, ip4_address_t *address,
3370                                     u32 address_length, u32 if_address_index,
3371                                     u32 is_delete)
3372 {
3373   snat_static_mapping_resolve_t *rp;
3374   snat_main_t *sm = &snat_main;
3375   int rv = 0;
3376
3377   if (!sm->enabled)
3378     {
3379       return;
3380     }
3381
3382   vec_foreach (rp, sm->sm_to_resolve)
3383     {
3384       if (sw_if_index == rp->sw_if_index)
3385         {
3386           if (is_delete)
3387             {
3388               if (rp->is_resolved)
3389                 {
3390                   rv = nat44_ed_del_static_mapping_internal (
3391                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3392                     rp->vrf_id, rp->flags);
3393                   if (rv)
3394                     {
3395                       nat_log_err ("ed del static mapping failed");
3396                     }
3397                   else
3398                     {
3399                       rp->is_resolved = 0;
3400                     }
3401                 }
3402             }
3403           else
3404             {
3405               if (!rp->is_resolved)
3406                 {
3407                   rv = nat44_ed_add_static_mapping_internal (
3408                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3409                     rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3410                   if (rv)
3411                     {
3412                       nat_log_err ("ed add static mapping failed");
3413                     }
3414                   else
3415                     {
3416                       rp->is_resolved = 1;
3417                     }
3418                 }
3419             }
3420         }
3421     }
3422 }
3423
3424 static int
3425 nat44_ed_get_addr_resolve_record (u32 sw_if_index, u8 twice_nat, int *out)
3426 {
3427   snat_main_t *sm = &snat_main;
3428   snat_address_resolve_t *rp;
3429   int i;
3430
3431   for (i = 0; i < vec_len (sm->addr_to_resolve); i++)
3432     {
3433       rp = sm->addr_to_resolve + i;
3434
3435       if ((rp->sw_if_index == sw_if_index) && (rp->is_twice_nat == twice_nat))
3436         {
3437           if (out)
3438             {
3439               *out = i;
3440             }
3441           return 0;
3442         }
3443     }
3444   return 1;
3445 }
3446 static int
3447 nat44_ed_del_addr_resolve_record (u32 sw_if_index, u8 twice_nat)
3448 {
3449   snat_main_t *sm = &snat_main;
3450   int i;
3451   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3452     {
3453       vec_del1 (sm->addr_to_resolve, i);
3454       return 0;
3455     }
3456   return 1;
3457 }
3458
3459 static void
3460 nat44_ed_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3461                                        u32 sw_if_index, ip4_address_t *address,
3462                                        u32 address_length,
3463                                        u32 if_address_index, u32 is_delete)
3464 {
3465   snat_main_t *sm = &snat_main;
3466   snat_address_resolve_t *arp;
3467   snat_address_t *ap;
3468   u8 twice_nat = 0;
3469   int i, rv;
3470
3471   if (!sm->enabled)
3472     {
3473       return;
3474     }
3475
3476   if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3477     {
3478       twice_nat = 1;
3479       if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3480         {
3481           u32 fib_index =
3482             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
3483           vec_foreach (ap, sm->addresses)
3484             {
3485               if ((fib_index == ap->fib_index) &&
3486                   (address->as_u32 == ap->addr.as_u32))
3487                 {
3488                   if (!is_delete)
3489                     {
3490                       ap->addr_len = address_length;
3491                       ap->sw_if_index = sw_if_index;
3492                       ap->net.as_u32 =
3493                         ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
3494
3495                       nat_log_debug (
3496                         "pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
3497                         format_ip4_address, &ap->addr, ap->sw_if_index,
3498                         format_ip4_address, &ap->net, ap->addr_len);
3499                     }
3500                   else
3501                     {
3502                       ap->addr_len = ~0;
3503                     }
3504                   break;
3505                 }
3506             }
3507           return;
3508         }
3509     }
3510
3511   arp = sm->addr_to_resolve + i;
3512
3513   if (!is_delete)
3514     {
3515       if (arp->is_resolved)
3516         {
3517           return;
3518         }
3519
3520       rv = nat44_ed_add_address (address, ~0, arp->is_twice_nat);
3521       if (0 == rv)
3522         {
3523           arp->is_resolved = 1;
3524         }
3525     }
3526   else
3527     {
3528       if (!arp->is_resolved)
3529         {
3530           return;
3531         }
3532
3533       rv = nat44_ed_del_address (address[0], arp->is_twice_nat);
3534       if (0 == rv)
3535         {
3536           arp->is_resolved = 0;
3537         }
3538     }
3539 }
3540
3541 int
3542 nat44_ed_add_interface_address (u32 sw_if_index, u8 twice_nat)
3543 {
3544   snat_main_t *sm = &snat_main;
3545   ip4_main_t *ip4_main = sm->ip4_main;
3546   ip4_address_t *first_int_addr;
3547   snat_address_resolve_t *ap;
3548   int rv;
3549
3550   if (!sm->enabled)
3551     {
3552       nat_log_err ("nat44 is disabled");
3553       return VNET_API_ERROR_UNSUPPORTED;
3554     }
3555
3556   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, 0))
3557     {
3558       return VNET_API_ERROR_VALUE_EXIST;
3559     }
3560
3561   vec_add2 (sm->addr_to_resolve, ap, 1);
3562   ap->sw_if_index = sw_if_index;
3563   ap->is_twice_nat = twice_nat;
3564   ap->is_resolved = 0;
3565
3566   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3567   if (first_int_addr)
3568     {
3569       rv = nat44_ed_add_address (first_int_addr, ~0, twice_nat);
3570       if (0 != rv)
3571         {
3572           nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat);
3573           return rv;
3574         }
3575       ap->is_resolved = 1;
3576     }
3577
3578   return 0;
3579 }
3580
3581 int
3582 nat44_ed_del_interface_address (u32 sw_if_index, u8 twice_nat)
3583 {
3584   snat_main_t *sm = &snat_main;
3585   ip4_main_t *ip4_main = sm->ip4_main;
3586   ip4_address_t *first_int_addr;
3587
3588   if (!sm->enabled)
3589     {
3590       nat_log_err ("nat44 is disabled");
3591       return VNET_API_ERROR_UNSUPPORTED;
3592     }
3593
3594   if (nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat))
3595     {
3596       return VNET_API_ERROR_NO_SUCH_ENTRY;
3597     }
3598
3599   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3600   if (first_int_addr)
3601     {
3602       return nat44_ed_del_address (first_int_addr[0], twice_nat);
3603     }
3604
3605   return 0;
3606 }
3607
3608 int
3609 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3610                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3611                       u32 vrf_id, int is_in)
3612 {
3613   ip4_header_t ip;
3614   clib_bihash_kv_16_8_t kv, value;
3615   u32 fib_index;
3616   snat_session_t *s;
3617   snat_main_per_thread_data_t *tsm;
3618
3619   if (!sm->enabled)
3620     {
3621       return VNET_API_ERROR_UNSUPPORTED;
3622     }
3623
3624   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3625   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3626   if (sm->num_workers > 1)
3627     tsm = vec_elt_at_index (
3628       sm->per_thread_data,
3629       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3630   else
3631     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3632
3633   init_ed_k (&kv, addr->as_u32, port, eh_addr->as_u32, eh_port, fib_index,
3634              proto);
3635   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3636     {
3637       return VNET_API_ERROR_NO_SUCH_ENTRY;
3638     }
3639
3640   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3641     return VNET_API_ERROR_UNSPECIFIED;
3642   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3643   nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3644   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3645   return 0;
3646 }
3647
3648 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3649                                  vlib_node_runtime_t * node,
3650                                  vlib_frame_t * frame)
3651 {
3652   return 0;
3653 }
3654
3655 VLIB_REGISTER_NODE (nat_default_node) = {
3656   .name = "nat-default",
3657   .vector_size = sizeof (u32),
3658   .format_trace = 0,
3659   .type = VLIB_NODE_TYPE_INTERNAL,
3660   .n_errors = 0,
3661   .n_next_nodes = NAT_N_NEXT,
3662   .next_nodes = {
3663     [NAT_NEXT_DROP] = "error-drop",
3664     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3665     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3666     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3667     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3668     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3669     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3670     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3671     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3672     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3673   },
3674 };
3675
3676 void
3677 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3678 {
3679   f->l3_csum_delta = 0;
3680   f->l4_csum_delta = 0;
3681   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3682       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3683     {
3684       f->l3_csum_delta =
3685         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3686       f->l3_csum_delta =
3687         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3688     }
3689   else
3690     {
3691       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3692     }
3693   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3694       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3695     {
3696       f->l3_csum_delta =
3697         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3698       f->l3_csum_delta =
3699         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3700     }
3701   else
3702     {
3703       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3704     }
3705   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3706     {
3707       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3708       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3709     }
3710   else
3711     {
3712       f->rewrite.sport = f->match.sport;
3713     }
3714   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3715     {
3716       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3717       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3718     }
3719   else
3720     {
3721       f->rewrite.dport = f->match.dport;
3722     }
3723   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3724       f->rewrite.icmp_id != f->match.sport)
3725     {
3726       f->l4_csum_delta =
3727         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3728       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3729     }
3730   else
3731     {
3732       f->rewrite.icmp_id = f->match.sport;
3733     }
3734   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3735     {
3736     }
3737   else
3738     {
3739       f->rewrite.fib_index = f->match.fib_index;
3740     }
3741 }
3742
3743 static_always_inline int
3744 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3745                             ip4_header_t *ip, nat_6t_flow_t *f);
3746
3747 static_always_inline void
3748 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3749                            nat_6t_flow_t *f, ip_protocol_t proto,
3750                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3751 {
3752   udp_header_t *udp = ip4_next_header (ip);
3753   tcp_header_t *tcp = (tcp_header_t *) udp;
3754
3755   if ((IP_PROTOCOL_TCP == proto || IP_PROTOCOL_UDP == proto) &&
3756       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3757     {
3758       if (!is_icmp_inner_ip4)
3759         { // regular case
3760           ip->src_address = f->rewrite.saddr;
3761           ip->dst_address = f->rewrite.daddr;
3762           udp->src_port = f->rewrite.sport;
3763           udp->dst_port = f->rewrite.dport;
3764         }
3765       else
3766         { // icmp inner ip4 - reversed saddr/daddr
3767           ip->src_address = f->rewrite.daddr;
3768           ip->dst_address = f->rewrite.saddr;
3769           udp->src_port = f->rewrite.dport;
3770           udp->dst_port = f->rewrite.sport;
3771         }
3772
3773       if (IP_PROTOCOL_TCP == proto)
3774         {
3775           ip_csum_t tcp_sum = tcp->checksum;
3776           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3777           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3778           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3779           tcp->checksum = ip_csum_fold (tcp_sum);
3780         }
3781       else if (IP_PROTOCOL_UDP == proto && udp->checksum)
3782         {
3783           ip_csum_t udp_sum = udp->checksum;
3784           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3785           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3786           udp->checksum = ip_csum_fold (udp_sum);
3787         }
3788     }
3789   else
3790     {
3791       if (!is_icmp_inner_ip4)
3792         { // regular case
3793           if (!skip_saddr_rewrite)
3794             {
3795               ip->src_address = f->rewrite.saddr;
3796             }
3797           ip->dst_address = f->rewrite.daddr;
3798         }
3799       else
3800         { // icmp inner ip4 - reversed saddr/daddr
3801           ip->src_address = f->rewrite.daddr;
3802           ip->dst_address = f->rewrite.saddr;
3803         }
3804     }
3805
3806   if (skip_saddr_rewrite)
3807     {
3808       ip->checksum = ip4_header_checksum (ip);
3809     }
3810   else
3811     {
3812       ip_csum_t ip_sum = ip->checksum;
3813       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3814       ip->checksum = ip_csum_fold (ip_sum);
3815     }
3816   if (0xffff == ip->checksum)
3817     ip->checksum = 0;
3818   ASSERT (ip4_header_checksum_is_valid (ip));
3819 }
3820
3821 static_always_inline int
3822 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3823 {
3824   int result = ((u8 *) object + size <=
3825                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3826                vlib_object_within_buffer_data (vm, b, object, size);
3827   return result;
3828 }
3829
3830 static_always_inline int
3831 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3832                             ip4_header_t *ip, nat_6t_flow_t *f)
3833 {
3834   if (IP_PROTOCOL_ICMP != ip->protocol)
3835     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3836
3837   icmp46_header_t *icmp = ip4_next_header (ip);
3838   nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3839
3840   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3841     {
3842       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3843         {
3844           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3845         }
3846
3847       if (!icmp_type_is_error_message (icmp->type))
3848         {
3849           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3850               (f->rewrite.icmp_id != echo->identifier))
3851             {
3852               ip_csum_t sum = icmp->checksum;
3853               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3854                                     nat_icmp_echo_header_t,
3855                                     identifier /* changed member */);
3856               echo->identifier = f->rewrite.icmp_id;
3857               icmp->checksum = ip_csum_fold (sum);
3858             }
3859         }
3860       else
3861         {
3862           ip_csum_t sum = ip_incremental_checksum (
3863             0, icmp,
3864             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3865           sum = (u16) ~ip_csum_fold (sum);
3866           if (sum != 0)
3867             {
3868               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3869             }
3870
3871           // errors are not fragmented
3872           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3873
3874           if (!ip4_header_checksum_is_valid (inner_ip))
3875             {
3876               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3877             }
3878
3879           ip_protocol_t inner_proto = inner_ip->protocol;
3880
3881           ip_csum_t old_icmp_sum = icmp->checksum;
3882           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3883           ip_csum_t old_udp_sum;
3884           ip_csum_t old_tcp_sum;
3885           ip_csum_t new_icmp_sum;
3886           udp_header_t *udp;
3887           tcp_header_t *tcp;
3888
3889           switch (inner_proto)
3890             {
3891             case IP_PROTOCOL_UDP:
3892               udp = (udp_header_t *) (inner_ip + 1);
3893               if (!it_fits (vm, b, udp, sizeof (*udp)))
3894                 {
3895                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3896                 }
3897               old_udp_sum = udp->checksum;
3898               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3899                                          1 /* is_icmp_inner_ip4 */,
3900                                          0 /* skip_saddr_rewrite */);
3901               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3902               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3903               new_icmp_sum =
3904                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3905                                 inner_ip->checksum, ip4_header_t, checksum);
3906               new_icmp_sum =
3907                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3908                                 udp_header_t, checksum);
3909               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3910               icmp->checksum = new_icmp_sum;
3911               break;
3912             case IP_PROTOCOL_TCP:
3913               tcp = (tcp_header_t *) (inner_ip + 1);
3914               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3915                 {
3916                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3917                 }
3918               old_tcp_sum = tcp->checksum;
3919               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3920                                          1 /* is_icmp_inner_ip4 */,
3921                                          0 /* skip_saddr_rewrite */);
3922               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3923               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3924               new_icmp_sum =
3925                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3926                                 inner_ip->checksum, ip4_header_t, checksum);
3927               new_icmp_sum =
3928                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3929                                 tcp_header_t, checksum);
3930               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3931               icmp->checksum = new_icmp_sum;
3932               break;
3933             case IP_PROTOCOL_ICMP:
3934               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3935                                          1 /* is_icmp_inner_ip4 */,
3936                                          0 /* skip_saddr_rewrite */);
3937               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3938                 {
3939                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3940                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3941                     {
3942                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3943                     }
3944                   nat_icmp_echo_header_t *inner_echo =
3945                     (nat_icmp_echo_header_t *) (inner_icmp + 1);
3946                   if (f->rewrite.icmp_id != inner_echo->identifier)
3947                     {
3948                       ip_csum_t sum = icmp->checksum;
3949                       sum = ip_csum_update (sum, inner_echo->identifier,
3950                                             f->rewrite.icmp_id,
3951                                             nat_icmp_echo_header_t,
3952                                             identifier /* changed member */);
3953                       icmp->checksum = ip_csum_fold (sum);
3954                       ip_csum_t inner_sum = inner_icmp->checksum;
3955                       inner_sum = ip_csum_update (
3956                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3957                         nat_icmp_echo_header_t,
3958                         identifier /* changed member */);
3959                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3960                       inner_echo->identifier = f->rewrite.icmp_id;
3961                     }
3962                 }
3963               break;
3964             default:
3965               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3966               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3967             }
3968         }
3969     }
3970
3971   return NAT_ED_TRNSL_ERR_SUCCESS;
3972 }
3973
3974 static_always_inline nat_translation_error_e
3975 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3976                            ip4_header_t *ip, nat_6t_flow_t *f,
3977                            ip_protocol_t proto, int is_output_feature,
3978                            int is_i2o)
3979 {
3980   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3981     {
3982       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3983     }
3984
3985   if (IP_PROTOCOL_ICMP == proto)
3986     {
3987       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3988         {
3989           // packet is returned from a router, not from destination
3990           // skip source address rewrite if in o2i path
3991           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3992                                      0 /* is_icmp_inner_ip4 */,
3993                                      !is_i2o /* skip_saddr_rewrite */);
3994         }
3995       else
3996         {
3997           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3998                                      0 /* is_icmp_inner_ip4 */,
3999                                      0 /* skip_saddr_rewrite */);
4000         }
4001       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
4002     }
4003
4004   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
4005                              0 /* skip_saddr_rewrite */);
4006
4007   return NAT_ED_TRNSL_ERR_SUCCESS;
4008 }
4009
4010 nat_translation_error_e
4011 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
4012                                vlib_buffer_t *b, ip4_header_t *ip,
4013                                nat_6t_flow_t *f, ip_protocol_t proto,
4014                                int is_output_feature)
4015 {
4016   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
4017                                     1 /* is_i2o */);
4018 }
4019
4020 nat_translation_error_e
4021 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
4022                                vlib_buffer_t *b, ip4_header_t *ip,
4023                                nat_6t_flow_t *f, ip_protocol_t proto,
4024                                int is_output_feature)
4025 {
4026   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
4027                                     0 /* is_i2o */);
4028 }
4029
4030 static_always_inline void
4031 nat_syslog_nat44_sess (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4032                        u16 isport, ip4_address_t *xsaddr, u16 xsport,
4033                        ip4_address_t *idaddr, u16 idport,
4034                        ip4_address_t *xdaddr, u16 xdport, u8 proto, u8 is_add,
4035                        u8 is_twicenat)
4036 {
4037   syslog_msg_t syslog_msg;
4038   fib_table_t *fib;
4039
4040   if (!syslog_is_enabled ())
4041     return;
4042
4043   if (syslog_severity_filter_block (SADD_SDEL_SEVERITY))
4044     return;
4045
4046   fib = fib_table_get (sfibix, FIB_PROTOCOL_IP4);
4047
4048   syslog_msg_init (&syslog_msg, NAT_FACILITY, SADD_SDEL_SEVERITY, NAT_APPNAME,
4049                    is_add ? SADD_MSGID : SDEL_MSGID);
4050
4051   syslog_msg_sd_init (&syslog_msg, NSESS_SDID);
4052   syslog_msg_add_sd_param (&syslog_msg, SSUBIX_SDPARAM_NAME, "%d", ssubix);
4053   syslog_msg_add_sd_param (&syslog_msg, SVLAN_SDPARAM_NAME, "%d",
4054                            fib->ft_table_id);
4055   syslog_msg_add_sd_param (&syslog_msg, IATYP_SDPARAM_NAME, IATYP_IPV4);
4056   syslog_msg_add_sd_param (&syslog_msg, ISADDR_SDPARAM_NAME, "%U",
4057                            format_ip4_address, isaddr);
4058   syslog_msg_add_sd_param (&syslog_msg, ISPORT_SDPARAM_NAME, "%d",
4059                            clib_net_to_host_u16 (isport));
4060   syslog_msg_add_sd_param (&syslog_msg, XATYP_SDPARAM_NAME, IATYP_IPV4);
4061   syslog_msg_add_sd_param (&syslog_msg, XSADDR_SDPARAM_NAME, "%U",
4062                            format_ip4_address, xsaddr);
4063   syslog_msg_add_sd_param (&syslog_msg, XSPORT_SDPARAM_NAME, "%d",
4064                            clib_net_to_host_u16 (xsport));
4065   syslog_msg_add_sd_param (&syslog_msg, PROTO_SDPARAM_NAME, "%d", proto);
4066   syslog_msg_add_sd_param (&syslog_msg, XDADDR_SDPARAM_NAME, "%U",
4067                            format_ip4_address, xdaddr);
4068   syslog_msg_add_sd_param (&syslog_msg, XDPORT_SDPARAM_NAME, "%d",
4069                            clib_net_to_host_u16 (xdport));
4070   if (is_twicenat)
4071     {
4072       syslog_msg_add_sd_param (&syslog_msg, IDADDR_SDPARAM_NAME, "%U",
4073                                format_ip4_address, idaddr);
4074       syslog_msg_add_sd_param (&syslog_msg, IDPORT_SDPARAM_NAME, "%d",
4075                                clib_net_to_host_u16 (idport));
4076     }
4077
4078   syslog_msg_send (&syslog_msg);
4079 }
4080
4081 void
4082 nat_syslog_nat44_sadd (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4083                        u16 isport, ip4_address_t *idaddr, u16 idport,
4084                        ip4_address_t *xsaddr, u16 xsport,
4085                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4086                        u8 is_twicenat)
4087 {
4088   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4089                          idaddr, idport, xdaddr, xdport, proto, 1,
4090                          is_twicenat);
4091 }
4092
4093 void
4094 nat_syslog_nat44_sdel (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4095                        u16 isport, ip4_address_t *idaddr, u16 idport,
4096                        ip4_address_t *xsaddr, u16 xsport,
4097                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4098                        u8 is_twicenat)
4099 {
4100   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4101                          idaddr, idport, xdaddr, xdport, proto, 0,
4102                          is_twicenat);
4103 }
4104 __clib_export void
4105 nat44_original_dst_lookup (ip4_address_t *i2o_src, u16 i2o_src_port,
4106                            ip4_address_t *i2o_dst, u16 i2o_dst_port,
4107                            ip_protocol_t proto, u32 *original_dst,
4108                            u16 *original_dst_port)
4109 {
4110   snat_main_per_thread_data_t *tsm;
4111   snat_main_t *sm = &snat_main;
4112   u32 fib_index = 0;
4113   snat_session_t *s;
4114   ip4_header_t ip;
4115
4116   ip.src_address.as_u32 = i2o_src->as_u32;
4117   fib_index = fib_table_find (FIB_PROTOCOL_IP4, 0);
4118
4119   if (sm->num_workers > 1)
4120     {
4121       tsm = vec_elt_at_index (
4122         sm->per_thread_data,
4123         nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
4124     }
4125   else
4126     {
4127       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4128     }
4129
4130   /* query */
4131   clib_bihash_kv_16_8_t kv = { 0 }, value;
4132   init_ed_k (&kv, i2o_src->as_u32, i2o_src_port, i2o_dst->as_u32, i2o_dst_port,
4133              fib_index, proto);
4134   if (tsm->sessions == NULL ||
4135       clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
4136     {
4137       return;
4138     }
4139   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
4140   if (s)
4141     {
4142       *original_dst = s->i2o.rewrite.saddr.as_u32;
4143       *original_dst_port = s->i2o.rewrite.sport;
4144     }
4145   return;
4146 }
4147 /*
4148  * fd.io coding-style-patch-verification: ON
4149  *
4150  * Local Variables:
4151  * eval: (c-set-style "gnu")
4152  * End:
4153  */