src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vpp/app/version.h>
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip4.h>
  21 #include <vnet/ip/ip_table.h>
  22 #include <vnet/ip/reass/ip4_sv_reass.h>
  23 #include <vnet/fib/fib_table.h>
  24 #include <vnet/fib/ip4_fib.h>
  25 #include <vnet/plugin/plugin.h>
  26 #include <vppinfra/bihash_16_8.h>
  27
  28 #include <nat/lib/log.h>
  29 #include <nat/lib/nat_inlines.h>
  30 #include <nat/lib/ipfix_logging.h>
  31 #include <vnet/syslog/syslog.h>
  32 #include <nat/lib/nat_syslog_constants.h>
  33 #include <nat/lib/nat_syslog.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 #include <vlib/stats/stats.h>
  40
  41 snat_main_t snat_main;
  42
  43 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  44                                                            u32 sw_if_index);
  45
  46 #define skip_if_disabled()                                                    \
  47   do                                                                          \
  48     {                                                                         \
  49       snat_main_t *sm = &snat_main;                                           \
  50       if (PREDICT_FALSE (!sm->enabled))                                       \
  51         return;                                                               \
  52     }                                                                         \
  53   while (0)
  54
  55 #define fail_if_enabled()                                                     \
  56   do                                                                          \
  57     {                                                                         \
  58       snat_main_t *sm = &snat_main;                                           \
  59       if (PREDICT_FALSE (sm->enabled))                                        \
  60         {                                                                     \
  61           nat_log_err ("plugin enabled");                                     \
  62           return VNET_API_ERROR_FEATURE_ALREADY_ENABLED;                      \
  63         }                                                                     \
  64     }                                                                         \
  65   while (0)
  66
  67 #define fail_if_disabled()                                                    \
  68   do                                                                          \
  69     {                                                                         \
  70       snat_main_t *sm = &snat_main;                                           \
  71       if (PREDICT_FALSE (!sm->enabled))                                       \
  72         {                                                                     \
  73           nat_log_err ("plugin disabled");                                    \
  74           return VNET_API_ERROR_FEATURE_ALREADY_DISABLED;                     \
  75         }                                                                     \
  76     }                                                                         \
  77   while (0)
  78
  79 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  80   .arc_name = "ip4-unicast",
  81   .node_name = "nat-pre-in2out",
  82   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  83                                "ip4-sv-reassembly-feature"),
  84 };
  85 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  86   .arc_name = "ip4-unicast",
  87   .node_name = "nat-pre-out2in",
  88   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  89                                "ip4-dhcp-client-detect",
  90                                "ip4-sv-reassembly-feature"),
  91 };
  92 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
  93   .arc_name = "ip4-unicast",
  94   .node_name = "nat44-ed-classify",
  95   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  96                                "ip4-sv-reassembly-feature"),
  97 };
  98 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-handoff-classify",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 102                                "ip4-sv-reassembly-feature"),
 103 };
 104 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out-worker-handoff",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 108 };
 109 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in-worker-handoff",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-in2out",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-out2in",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 124                                "ip4-dhcp-client-detect"),
 125 };
 126 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 127   .arc_name = "ip4-output",
 128   .node_name = "nat-pre-in2out-output",
 129   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 130   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 131 };
 132 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 133   .arc_name = "ip4-output",
 134   .node_name = "nat44-in2out-output-worker-handoff",
 135   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 136   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 137 };
 138 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 139   .arc_name = "ip4-output",
 140   .node_name = "nat44-ed-in2out-output",
 141   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 142   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 143 };
 144
 145 VLIB_PLUGIN_REGISTER () = {
 146     .version = VPP_BUILD_VER,
 147     .description = "Network Address Translation (NAT)",
 148 };
 149
 150 static void nat44_ed_db_init ();
 151 static void nat44_ed_db_free ();
 152 static void nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm);
 153
 154 static int nat44_ed_add_static_mapping_internal (
 155   ip4_address_t l_addr, ip4_address_t e_addr, u16 l_port, u16 e_port,
 156   ip_protocol_t proto, u32 vrf_id, u32 sw_if_index, u32 flags,
 157   ip4_address_t pool_addr, u8 *tag);
 158 static int nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
 159                                                  ip4_address_t e_addr,
 160                                                  u16 l_port, u16 e_port,
 161                                                  ip_protocol_t proto,
 162                                                  u32 vrf_id, u32 flags);
 163
 164 u32 nat_calc_bihash_buckets (u32 n_elts);
 165
 166 static_always_inline int
 167 nat44_ed_sm_i2o_add (snat_main_t *sm, snat_static_mapping_t *m,
 168                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 169 {
 170   ASSERT (!pool_is_free (sm->static_mappings, m));
 171   clib_bihash_kv_16_8_t kv;
 172   nat44_ed_sm_init_i2o_kv (&kv, addr.as_u32, port, fib_index, proto,
 173                            m - sm->static_mappings);
 174   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 175 }
 176
 177 static_always_inline int
 178 nat44_ed_sm_i2o_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 179                      u32 fib_index, u8 proto)
 180 {
 181   clib_bihash_kv_16_8_t kv;
 182   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 183   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 184 }
 185
 186 static_always_inline int
 187 nat44_ed_sm_o2i_add (snat_main_t *sm, snat_static_mapping_t *m,
 188                      ip4_address_t addr, u16 port, u32 fib_index, u8 proto)
 189 {
 190   ASSERT (!pool_is_free (sm->static_mappings, m));
 191   clib_bihash_kv_16_8_t kv;
 192   nat44_ed_sm_init_o2i_kv (&kv, addr.as_u32, port, fib_index, proto,
 193                            m - sm->static_mappings);
 194   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 1 /*is_add*/);
 195 }
 196
 197 static_always_inline int
 198 nat44_ed_sm_o2i_del (snat_main_t *sm, ip4_address_t addr, u16 port,
 199                      u32 fib_index, u8 proto)
 200 {
 201   clib_bihash_kv_16_8_t kv;
 202   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 203   return clib_bihash_add_del_16_8 (&sm->flow_hash, &kv, 0 /*is_add*/);
 204 }
 205
 206 void
 207 nat44_ed_free_session_data (snat_main_t *sm, snat_session_t *s,
 208                             u32 thread_index, u8 is_ha)
 209 {
 210   per_vrf_sessions_unregister_session (s, thread_index);
 211
 212   if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 213     nat_elog_warn (sm, "flow hash del failed");
 214
 215   if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 216     nat_elog_warn (sm, "flow hash del failed");
 217
 218   if (na44_ed_is_fwd_bypass_session (s))
 219     {
 220       return;
 221     }
 222
 223   if (nat44_ed_is_affinity_session (s))
 224     nat_affinity_unlock (s->ext_host_addr, s->out2in.addr, s->proto,
 225                          s->out2in.port);
 226
 227   if (!is_ha)
 228     nat_syslog_nat44_sdel (0, s->in2out.fib_index, &s->in2out.addr,
 229                            s->in2out.port, &s->ext_host_nat_addr,
 230                            s->ext_host_nat_port, &s->out2in.addr,
 231                            s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 232                            s->proto, nat44_ed_is_twice_nat_session (s));
 233
 234   if (!is_ha)
 235     {
 236       /* log NAT event */
 237       nat_ipfix_logging_nat44_ses_delete (
 238         thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->proto,
 239         s->in2out.port, s->out2in.port, s->in2out.fib_index);
 240     }
 241 }
 242
 243 static ip_interface_address_t *
 244 nat44_ed_get_ip_interface_address (u32 sw_if_index, ip4_address_t addr)
 245 {
 246   snat_main_t *sm = &snat_main;
 247
 248   ip_lookup_main_t *lm = &sm->ip4_main->lookup_main;
 249   ip_interface_address_t *ia;
 250   ip4_address_t *ip4a;
 251
 252   foreach_ip_interface_address (
 253     lm, ia, sw_if_index, 1, ({
 254       ip4a = ip_interface_address_get_address (lm, ia);
 255       nat_log_debug ("sw_if_idx: %u addr: %U ? %U", sw_if_index,
 256                      format_ip4_address, ip4a, format_ip4_address, &addr);
 257       if (ip4a->as_u32 == addr.as_u32)
 258         {
 259           return ia;
 260         }
 261     }));
 262   return NULL;
 263 }
 264
 265 static int
 266 nat44_ed_resolve_nat_addr_len (snat_address_t *ap,
 267                                snat_interface_t *interfaces)
 268 {
 269   ip_interface_address_t *ia;
 270   snat_interface_t *i;
 271   u32 fib_index;
 272
 273   pool_foreach (i, interfaces)
 274     {
 275       if (!nat44_ed_is_interface_outside (i))
 276         {
 277           continue;
 278         }
 279
 280       fib_index = ip4_fib_table_get_index_for_sw_if_index (i->sw_if_index);
 281       if (fib_index != ap->fib_index)
 282         {
 283           continue;
 284         }
 285
 286       if ((ia = nat44_ed_get_ip_interface_address (i->sw_if_index, ap->addr)))
 287         {
 288           ap->addr_len = ia->address_length;
 289           ap->sw_if_index = i->sw_if_index;
 290           ap->net.as_u32 = ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
 291
 292           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 293                          format_ip4_address, &ap->addr, ap->sw_if_index,
 294                          format_ip4_address, &ap->net, ap->addr_len);
 295           return 0;
 296         }
 297     }
 298   return 1;
 299 }
 300
 301 static void
 302 nat44_ed_update_outside_if_addresses (snat_address_t *ap)
 303 {
 304   snat_main_t *sm = &snat_main;
 305
 306   if (!nat44_ed_resolve_nat_addr_len (ap, sm->interfaces))
 307     {
 308       return;
 309     }
 310
 311   if (!nat44_ed_resolve_nat_addr_len (ap, sm->output_feature_interfaces))
 312     {
 313       return;
 314     }
 315 }
 316
 317 static void
 318 nat44_ed_bind_if_addr_to_nat_addr (u32 sw_if_index)
 319 {
 320   snat_main_t *sm = &snat_main;
 321   ip_interface_address_t *ia;
 322   snat_address_t *ap;
 323
 324   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 325
 326   vec_foreach (ap, sm->addresses)
 327     {
 328       if (fib_index != ap->fib_index)
 329         {
 330           continue;
 331         }
 332
 333       if ((ia = nat44_ed_get_ip_interface_address (sw_if_index, ap->addr)))
 334         {
 335           ap->addr_len = ia->address_length;
 336           ap->sw_if_index = sw_if_index;
 337           ap->net.as_u32 = ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
 338
 339           nat_log_debug ("pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
 340                          format_ip4_address, &ap->addr, ap->sw_if_index,
 341                          format_ip4_address, &ap->net, ap->addr_len);
 342           return;
 343         }
 344     }
 345 }
 346
 347 static_always_inline snat_fib_entry_reg_t *
 348 nat44_ed_get_fib_entry_reg (ip4_address_t addr, u32 sw_if_index, int *out_idx)
 349 {
 350   snat_main_t *sm = &snat_main;
 351   snat_fib_entry_reg_t *fe;
 352   int i;
 353
 354   for (i = 0; i < vec_len (sm->fib_entry_reg); i++)
 355     {
 356       fe = sm->fib_entry_reg + i;
 357       if ((addr.as_u32 == fe->addr.as_u32) && (sw_if_index == fe->sw_if_index))
 358         {
 359           if (out_idx)
 360             {
 361               *out_idx = i;
 362             }
 363           return fe;
 364         }
 365     }
 366   return NULL;
 367 }
 368
 369 static void
 370 nat44_ed_add_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 371 {
 372   // Add the external NAT address to the FIB as receive entries. This ensures
 373   // that VPP will reply to ARP for this address and we don't need to enable
 374   // proxy ARP on the outside interface.
 375   snat_main_t *sm = &snat_main;
 376   snat_fib_entry_reg_t *fe;
 377
 378   if (!(fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, 0)))
 379     {
 380       fib_prefix_t prefix = {
 381         .fp_len = 32,
 382         .fp_proto = FIB_PROTOCOL_IP4,
 383         .fp_addr = {
 384                   .ip4.as_u32 = addr.as_u32,
 385                 },
 386       };
 387       u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 388       fib_table_entry_update_one_path (fib_index, &prefix, sm->fib_src_low,
 389                                        (FIB_ENTRY_FLAG_CONNECTED |
 390                                         FIB_ENTRY_FLAG_LOCAL |
 391                                         FIB_ENTRY_FLAG_EXCLUSIVE),
 392                                        DPO_PROTO_IP4, NULL, sw_if_index, ~0, 1,
 393                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 394
 395       vec_add2 (sm->fib_entry_reg, fe, 1);
 396       clib_memset (fe, 0, sizeof (*fe));
 397       fe->addr.as_u32 = addr.as_u32;
 398       fe->sw_if_index = sw_if_index;
 399     }
 400   fe->count++;
 401 }
 402
 403 static void
 404 nat44_ed_del_fib_entry_reg (ip4_address_t addr, u32 sw_if_index)
 405 {
 406   snat_main_t *sm = &snat_main;
 407   snat_fib_entry_reg_t *fe;
 408   int i;
 409
 410   if ((fe = nat44_ed_get_fib_entry_reg (addr, sw_if_index, &i)))
 411     {
 412       fe->count--;
 413       if (0 == fe->count)
 414         {
 415           fib_prefix_t prefix = {
 416             .fp_len = 32,
 417             .fp_proto = FIB_PROTOCOL_IP4,
 418             .fp_addr = {
 419               .ip4.as_u32 = addr.as_u32,
 420                     },
 421           };
 422           u32 fib_index =
 423             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 424           fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 425           vec_del1 (sm->fib_entry_reg, i);
 426         }
 427     }
 428 }
 429
 430 static void
 431 nat44_ed_add_del_interface_fib_reg_entries (ip4_address_t addr, u8 is_add)
 432 {
 433   snat_main_t *sm = &snat_main;
 434   snat_interface_t *i;
 435
 436   pool_foreach (i, sm->interfaces)
 437     {
 438       if (nat44_ed_is_interface_outside (i))
 439         {
 440           if (is_add)
 441             {
 442               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 443             }
 444           else
 445             {
 446               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 447             }
 448         }
 449     }
 450   pool_foreach (i, sm->output_feature_interfaces)
 451     {
 452       if (nat44_ed_is_interface_outside (i))
 453         {
 454           if (is_add)
 455             {
 456               nat44_ed_add_fib_entry_reg (addr, i->sw_if_index);
 457             }
 458           else
 459             {
 460               nat44_ed_del_fib_entry_reg (addr, i->sw_if_index);
 461             }
 462         }
 463     }
 464 }
 465
 466 static_always_inline void
 467 nat44_ed_add_del_nat_addr_fib_reg_entries (u32 sw_if_index, u8 is_add)
 468 {
 469   snat_main_t *sm = &snat_main;
 470   snat_address_t *ap;
 471
 472   vec_foreach (ap, sm->addresses)
 473     {
 474       if (is_add)
 475         {
 476           nat44_ed_add_fib_entry_reg (ap->addr, sw_if_index);
 477         }
 478       else
 479         {
 480           nat44_ed_del_fib_entry_reg (ap->addr, sw_if_index);
 481         }
 482     }
 483 }
 484
 485 static_always_inline void
 486 nat44_ed_add_del_sm_fib_reg_entries (u32 sw_if_index, u8 is_add)
 487 {
 488   snat_main_t *sm = &snat_main;
 489   snat_static_mapping_t *m;
 490
 491   pool_foreach (m, sm->static_mappings)
 492     {
 493       if (is_add)
 494         {
 495           nat44_ed_add_fib_entry_reg (m->external_addr, sw_if_index);
 496         }
 497       else
 498         {
 499           nat44_ed_del_fib_entry_reg (m->external_addr, sw_if_index);
 500         }
 501     }
 502 }
 503
 504 int
 505 nat44_ed_add_address (ip4_address_t *addr, u32 vrf_id, u8 twice_nat)
 506 {
 507   snat_main_t *sm = &snat_main;
 508   snat_address_t *ap, *addresses;
 509
 510   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 511
 512   if (!sm->enabled)
 513     {
 514       return VNET_API_ERROR_UNSUPPORTED;
 515     }
 516
 517   // check if address already exists
 518   vec_foreach (ap, addresses)
 519     {
 520       if (ap->addr.as_u32 == addr->as_u32)
 521         {
 522           nat_log_err ("address exist");
 523           return VNET_API_ERROR_VALUE_EXIST;
 524         }
 525     }
 526
 527   if (twice_nat)
 528     {
 529       vec_add2 (sm->twice_nat_addresses, ap, 1);
 530     }
 531   else
 532     {
 533       vec_add2 (sm->addresses, ap, 1);
 534     }
 535
 536   ap->addr_len = ~0;
 537   ap->fib_index = ~0;
 538   ap->addr = *addr;
 539
 540   if (vrf_id != ~0)
 541     {
 542       ap->fib_index = fib_table_find_or_create_and_lock (
 543         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 544     }
 545
 546   if (!twice_nat)
 547     {
 548       // if we don't have enabled interface we don't add address
 549       // to fib
 550       nat44_ed_add_del_interface_fib_reg_entries (*addr, 1);
 551       nat44_ed_update_outside_if_addresses (ap);
 552     }
 553   return 0;
 554 }
 555
 556 int
 557 nat44_ed_del_address (ip4_address_t addr, u8 twice_nat)
 558 {
 559   snat_main_t *sm = &snat_main;
 560   snat_address_t *a = 0, *addresses;
 561   snat_session_t *ses;
 562   u32 *ses_to_be_removed = 0, *ses_index;
 563   snat_main_per_thread_data_t *tsm;
 564   int j;
 565
 566   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
 567
 568   for (j = 0; j < vec_len (addresses); j++)
 569     {
 570       if (addresses[j].addr.as_u32 == addr.as_u32)
 571         {
 572           a = addresses + j;
 573           break;
 574         }
 575     }
 576   if (!a)
 577     {
 578       nat_log_err ("no such address");
 579       return VNET_API_ERROR_NO_SUCH_ENTRY;
 580     }
 581
 582   // delete dynamic sessions only
 583   vec_foreach (tsm, sm->per_thread_data)
 584     {
 585       pool_foreach (ses, tsm->sessions)
 586         {
 587           if (ses->flags & SNAT_SESSION_FLAG_STATIC_MAPPING)
 588             {
 589               continue;
 590             }
 591           if (ses->out2in.addr.as_u32 == addr.as_u32)
 592             {
 593               nat44_ed_free_session_data (sm, ses, tsm - sm->per_thread_data,
 594                                           0);
 595               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
 596             }
 597         }
 598       vec_foreach (ses_index, ses_to_be_removed)
 599         {
 600           ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
 601           nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
 602         }
 603       vec_free (ses_to_be_removed);
 604     }
 605
 606   if (!twice_nat)
 607     {
 608       nat44_ed_add_del_interface_fib_reg_entries (addr, 0);
 609     }
 610
 611   if (a->fib_index != ~0)
 612     {
 613       fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 614     }
 615
 616   if (!twice_nat)
 617     {
 618       vec_del1 (sm->addresses, j);
 619     }
 620   else
 621     {
 622       vec_del1 (sm->twice_nat_addresses, j);
 623     }
 624
 625   return 0;
 626 }
 627
 628 vrf_table_t *
 629 nat44_ed_get_vrf_table (u32 table_vrf_id)
 630 {
 631   snat_main_t *sm = &snat_main;
 632   vrf_table_t *t;
 633
 634   pool_foreach (t, sm->vrf_tables)
 635     {
 636       if (table_vrf_id == t->table_vrf_id)
 637         {
 638           return t;
 639         }
 640     }
 641   return NULL;
 642 }
 643
 644 vrf_route_t *
 645 nat44_ed_get_vrf_route (vrf_table_t *t, u32 vrf_id)
 646 {
 647   vrf_route_t *r;
 648
 649   pool_foreach (r, t->routes)
 650     {
 651       if (vrf_id == r->vrf_id)
 652         {
 653           return r;
 654         }
 655     }
 656   return NULL;
 657 }
 658
 659 int
 660 nat44_ed_add_del_vrf_table (u32 table_vrf_id, bool is_add)
 661 {
 662   snat_main_t *sm = &snat_main;
 663   vrf_table_t *t;
 664   vrf_route_t *r;
 665
 666   t = nat44_ed_get_vrf_table (table_vrf_id);
 667   if (t)
 668     {
 669       if (is_add)
 670         {
 671           return VNET_API_ERROR_VALUE_EXIST;
 672         }
 673       pool_foreach (r, t->routes)
 674         {
 675           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 676         }
 677       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 678
 679       pool_free (t->routes);
 680       pool_put (sm->vrf_tables, t);
 681     }
 682   else
 683     {
 684       if (!is_add)
 685         {
 686           return VNET_API_ERROR_NO_SUCH_ENTRY;
 687         }
 688       pool_get (sm->vrf_tables, t);
 689       clib_memset (t, 0, sizeof (*t));
 690       t->table_vrf_id = table_vrf_id;
 691       t->table_fib_index = fib_table_find_or_create_and_lock (
 692         FIB_PROTOCOL_IP4, table_vrf_id, sm->fib_src_low);
 693     }
 694
 695   return 0;
 696 }
 697
 698 void
 699 nat44_ed_del_vrf_tables ()
 700 {
 701   snat_main_t *sm = &snat_main;
 702   vrf_table_t *t;
 703   vrf_route_t *r;
 704
 705   pool_foreach (t, sm->vrf_tables)
 706     {
 707       pool_foreach (r, t->routes)
 708         {
 709           fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 710         }
 711       fib_table_unlock (t->table_fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 712       pool_free (t->routes);
 713     }
 714   pool_free (sm->vrf_tables);
 715 }
 716
 717 int
 718 nat44_ed_add_del_vrf_route (u32 table_vrf_id, u32 vrf_id, bool is_add)
 719 {
 720   snat_main_t *sm = &snat_main;
 721   vrf_table_t *t;
 722   vrf_route_t *r;
 723
 724   t = nat44_ed_get_vrf_table (table_vrf_id);
 725   if (!t)
 726     {
 727       return VNET_API_ERROR_NO_SUCH_ENTRY;
 728     }
 729
 730   r = nat44_ed_get_vrf_route (t, vrf_id);
 731   if (r)
 732     {
 733       if (is_add)
 734         {
 735           return VNET_API_ERROR_VALUE_EXIST;
 736         }
 737       fib_table_unlock (r->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 738       pool_put (t->routes, r);
 739     }
 740   else
 741     {
 742       if (!is_add)
 743         {
 744           return VNET_API_ERROR_NO_SUCH_ENTRY;
 745         }
 746       pool_get (t->routes, r);
 747       clib_memset (r, 0, sizeof (*r));
 748       r->vrf_id = vrf_id;
 749       r->fib_index = fib_table_find_or_create_and_lock (
 750         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 751     }
 752
 753   return 0;
 754 }
 755
 756 u32
 757 get_thread_idx_by_port (u16 e_port)
 758 {
 759   snat_main_t *sm = &snat_main;
 760   u32 thread_idx = sm->num_workers;
 761   if (sm->num_workers > 1)
 762     {
 763       thread_idx = sm->first_worker_index +
 764                    sm->workers[(e_port - ED_USER_PORT_OFFSET) /
 765                                sm->port_per_thread % _vec_len (sm->workers)];
 766     }
 767   return thread_idx;
 768 }
 769
 770 void
 771 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 772                                     snat_main_per_thread_data_t * tsm,
 773                                     ip4_address_t l_addr,
 774                                     u16 l_port,
 775                                     u8 protocol,
 776                                     u32 fib_index, int addr_only,
 777                                     ip4_address_t e_addr, u16 e_port)
 778 {
 779   snat_session_t *s;
 780   u32 *indexes_to_free = NULL;
 781   pool_foreach (s, tsm->sessions) {
 782     if (s->in2out.fib_index != fib_index ||
 783         s->in2out.addr.as_u32 != l_addr.as_u32)
 784       {
 785         continue;
 786       }
 787     if (!addr_only)
 788       {
 789         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 790             s->out2in.port != e_port || s->in2out.port != l_port ||
 791             s->proto != protocol)
 792           continue;
 793       }
 794
 795     if (nat44_ed_is_lb_session (s))
 796       continue;
 797     if (!nat44_ed_is_session_static (s))
 798       continue;
 799     nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 800     vec_add1 (indexes_to_free, s - tsm->sessions);
 801     if (!addr_only)
 802       break;
 803   }
 804   u32 *ses_index;
 805   vec_foreach (ses_index, indexes_to_free)
 806   {
 807     s = pool_elt_at_index (tsm->sessions, *ses_index);
 808     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 809   }
 810   vec_free (indexes_to_free);
 811 }
 812
 813 static_always_inline snat_static_mapping_t *
 814 nat44_ed_sm_lookup (snat_main_t *sm, clib_bihash_kv_16_8_t *kv)
 815 {
 816   clib_bihash_kv_16_8_t v;
 817   int rc = clib_bihash_search_16_8 (&sm->flow_hash, kv, &v);
 818   if (!rc)
 819     {
 820       ASSERT (0 == ed_value_get_thread_index (&v));
 821       return pool_elt_at_index (sm->static_mappings,
 822                                 ed_value_get_session_index (&v));
 823     }
 824   return NULL;
 825 }
 826
 827 snat_static_mapping_t *
 828 nat44_ed_sm_o2i_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 829                         u32 fib_index, u8 proto)
 830 {
 831   clib_bihash_kv_16_8_t kv;
 832   nat44_ed_sm_init_o2i_k (&kv, addr.as_u32, port, fib_index, proto);
 833   return nat44_ed_sm_lookup (sm, &kv);
 834 }
 835
 836 snat_static_mapping_t *
 837 nat44_ed_sm_i2o_lookup (snat_main_t *sm, ip4_address_t addr, u16 port,
 838                         u32 fib_index, u8 proto)
 839 {
 840   clib_bihash_kv_16_8_t kv;
 841   nat44_ed_sm_init_i2o_k (&kv, addr.as_u32, port, fib_index, proto);
 842   return nat44_ed_sm_lookup (sm, &kv);
 843 }
 844
 845 static snat_static_mapping_resolve_t *
 846 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 847                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 848                              u32 flags, int *out_idx)
 849 {
 850   snat_static_mapping_resolve_t *rp;
 851   snat_main_t *sm = &snat_main;
 852   int i;
 853
 854   for (i = 0; i < vec_len (sm->sm_to_resolve); i++)
 855     {
 856       rp = sm->sm_to_resolve + i;
 857
 858       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 859         {
 860           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 861             {
 862               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 863                 {
 864                   if (rp->e_port != e_port || rp->proto != proto)
 865                     {
 866                       continue;
 867                     }
 868                 }
 869             }
 870           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 871             {
 872               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 873                 {
 874                   if (rp->l_port != l_port || rp->e_port != e_port ||
 875                       rp->proto != proto)
 876                     {
 877                       continue;
 878                     }
 879                 }
 880             }
 881           else
 882             {
 883               continue;
 884             }
 885           if (out_idx)
 886             {
 887               *out_idx = i;
 888             }
 889           return rp;
 890         }
 891     }
 892   return NULL;
 893 }
 894
 895 static int
 896 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 897                              ip_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 898                              u32 flags)
 899 {
 900   snat_main_t *sm = &snat_main;
 901   int i;
 902   if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 903                                    sw_if_index, flags, &i))
 904     {
 905       vec_del1 (sm->sm_to_resolve, i);
 906       return 0;
 907     }
 908   return 1;
 909 }
 910
 911 static_always_inline int
 912 nat44_ed_validate_sm_input (u32 flags)
 913 {
 914   // identity nat can be initiated only from inside interface
 915   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 916     {
 917       return VNET_API_ERROR_UNSUPPORTED;
 918     }
 919
 920   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 921     {
 922       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 923         {
 924           return VNET_API_ERROR_UNSUPPORTED;
 925         }
 926     }
 927   return 0;
 928 }
 929
 930 int
 931 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 932                              u16 l_port, u16 e_port, ip_protocol_t proto,
 933                              u32 vrf_id, u32 sw_if_index, u32 flags,
 934                              ip4_address_t pool_addr, u8 *tag)
 935 {
 936   snat_static_mapping_resolve_t *rp;
 937   snat_main_t *sm = &snat_main;
 938   int rv;
 939
 940   if (!sm->enabled)
 941     {
 942       return VNET_API_ERROR_UNSUPPORTED;
 943     }
 944
 945   rv = nat44_ed_validate_sm_input (flags);
 946   if (rv != 0)
 947     {
 948       return rv;
 949     }
 950
 951   // interface bound mapping
 952   if (is_sm_switch_address (flags))
 953     {
 954       if (nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 955                                        sw_if_index, flags, 0))
 956         {
 957           return VNET_API_ERROR_VALUE_EXIST;
 958         }
 959
 960       vec_add2 (sm->sm_to_resolve, rp, 1);
 961       rp->l_addr.as_u32 = l_addr.as_u32;
 962       rp->l_port = l_port;
 963       rp->e_port = e_port;
 964       rp->sw_if_index = sw_if_index;
 965       rp->vrf_id = vrf_id;
 966       rp->proto = proto;
 967       rp->flags = flags;
 968       rp->pool_addr = pool_addr;
 969       rp->tag = vec_dup (tag);
 970       rp->is_resolved = 0;
 971
 972       ip4_address_t *first_int_addr =
 973         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 974       if (!first_int_addr)
 975         {
 976           return 0;
 977         }
 978
 979       e_addr.as_u32 = first_int_addr->as_u32;
 980       rp->is_resolved = 1;
 981     }
 982
 983   rv = nat44_ed_add_static_mapping_internal (l_addr, e_addr, l_port, e_port,
 984                                              proto, vrf_id, sw_if_index, flags,
 985                                              pool_addr, tag);
 986   if ((0 != rv) && is_sm_switch_address (flags))
 987     {
 988       nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 989                                    sw_if_index, flags);
 990     }
 991
 992   return rv;
 993 }
 994
 995 int
 996 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 997                              u16 l_port, u16 e_port, ip_protocol_t proto,
 998                              u32 vrf_id, u32 sw_if_index, u32 flags)
 999 {
1000   snat_main_t *sm = &snat_main;
1001   int rv;
1002
1003   if (!sm->enabled)
1004     {
1005       return VNET_API_ERROR_UNSUPPORTED;
1006     }
1007
1008   rv = nat44_ed_validate_sm_input (flags);
1009   if (rv != 0)
1010     {
1011       return rv;
1012     }
1013
1014   // interface bound mapping
1015   if (is_sm_switch_address (flags))
1016     {
1017       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
1018                                        sw_if_index, flags))
1019         {
1020           return VNET_API_ERROR_NO_SUCH_ENTRY;
1021         }
1022
1023       ip4_address_t *first_int_addr =
1024         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
1025       if (!first_int_addr)
1026         {
1027           // dhcp resolution required
1028           return 0;
1029         }
1030
1031       e_addr.as_u32 = first_int_addr->as_u32;
1032     }
1033
1034   return nat44_ed_del_static_mapping_internal (l_addr, e_addr, l_port, e_port,
1035                                                proto, vrf_id, flags);
1036 }
1037
1038 static int
1039 nat44_ed_add_static_mapping_internal (ip4_address_t l_addr,
1040                                       ip4_address_t e_addr, u16 l_port,
1041                                       u16 e_port, ip_protocol_t proto,
1042                                       u32 vrf_id, u32 sw_if_index, u32 flags,
1043                                       ip4_address_t pool_addr, u8 *tag)
1044 {
1045   snat_main_t *sm = &snat_main;
1046   nat44_lb_addr_port_t *local;
1047   snat_static_mapping_t *m;
1048   u32 fib_index = ~0;
1049
1050   if (is_sm_addr_only (flags))
1051     {
1052       e_port = l_port = proto = 0;
1053     }
1054
1055   if (is_sm_identity_nat (flags))
1056     {
1057       l_port = e_port;
1058       l_addr.as_u32 = e_addr.as_u32;
1059     }
1060
1061   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1062   if (m)
1063     {
1064       // case:
1065       // adding local identity nat record for different vrf table
1066
1067       if (!is_sm_identity_nat (m->flags))
1068         {
1069           return VNET_API_ERROR_VALUE_EXIST;
1070         }
1071
1072       pool_foreach (local, m->locals)
1073         {
1074           if (local->vrf_id == vrf_id)
1075             {
1076               return VNET_API_ERROR_VALUE_EXIST;
1077             }
1078         }
1079
1080       pool_get (m->locals, local);
1081
1082       local->vrf_id = vrf_id;
1083       local->fib_index = fib_table_find_or_create_and_lock (
1084         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
1085
1086       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port,
1087                            local->fib_index, m->proto);
1088
1089       return 0;
1090     }
1091
1092   if (vrf_id != ~0)
1093     {
1094       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1095                                                      sm->fib_src_low);
1096     }
1097   else
1098     {
1099       // fallback to default vrf
1100       vrf_id = sm->inside_vrf_id;
1101       fib_index = sm->inside_fib_index;
1102       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1103     }
1104
1105   // test if local mapping record doesn't exist
1106   // identity nat supports multiple records in local mapping
1107   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
1108     {
1109       if (nat44_ed_sm_i2o_lookup (sm, l_addr, l_port, fib_index, proto))
1110         {
1111           return VNET_API_ERROR_VALUE_EXIST;
1112         }
1113     }
1114
1115   pool_get (sm->static_mappings, m);
1116   clib_memset (m, 0, sizeof (*m));
1117
1118   m->flags = flags;
1119   m->local_addr = l_addr;
1120   m->external_addr = e_addr;
1121
1122   m->pool_addr = pool_addr;
1123   m->tag = vec_dup (tag);
1124
1125   if (!is_sm_addr_only (flags))
1126     {
1127       m->local_port = l_port;
1128       m->external_port = e_port;
1129       m->proto = proto;
1130     }
1131
1132   if (is_sm_identity_nat (flags))
1133     {
1134       pool_get (m->locals, local);
1135
1136       local->vrf_id = vrf_id;
1137       local->fib_index = fib_index;
1138     }
1139   else
1140     {
1141       m->vrf_id = vrf_id;
1142       m->fib_index = fib_index;
1143     }
1144
1145   if (!is_sm_out2in_only (flags))
1146     {
1147       nat44_ed_sm_i2o_add (sm, m, m->local_addr, m->local_port, fib_index,
1148                            m->proto);
1149     }
1150
1151   nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0, m->proto);
1152
1153   if (sm->num_workers > 1)
1154     {
1155       // store worker index for this record
1156       ip4_header_t ip = {
1157         .src_address = m->local_addr,
1158       };
1159       u32 worker_index;
1160       worker_index =
1161         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
1162       vec_add1 (m->workers, worker_index);
1163     }
1164
1165   nat44_ed_add_del_interface_fib_reg_entries (e_addr, 1);
1166
1167   return 0;
1168 }
1169
1170 static int
1171 nat44_ed_del_static_mapping_internal (ip4_address_t l_addr,
1172                                       ip4_address_t e_addr, u16 l_port,
1173                                       u16 e_port, ip_protocol_t proto,
1174                                       u32 vrf_id, u32 flags)
1175 {
1176   snat_main_per_thread_data_t *tsm;
1177   snat_main_t *sm = &snat_main;
1178
1179   nat44_lb_addr_port_t *local;
1180   snat_static_mapping_t *m;
1181   u32 fib_index = ~0;
1182
1183   if (is_sm_addr_only (flags))
1184     {
1185       e_port = l_port = proto = 0;
1186     }
1187
1188   if (is_sm_identity_nat (flags))
1189     {
1190       l_port = e_port;
1191       l_addr.as_u32 = e_addr.as_u32;
1192     }
1193
1194   // fib index 0
1195   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1196   if (!m)
1197     {
1198       return VNET_API_ERROR_NO_SUCH_ENTRY;
1199     }
1200
1201   if (is_sm_identity_nat (flags))
1202     {
1203       u8 found = 0;
1204
1205       if (vrf_id == ~0)
1206         {
1207           vrf_id = sm->inside_vrf_id;
1208         }
1209
1210       pool_foreach (local, m->locals)
1211         {
1212           if (local->vrf_id == vrf_id)
1213             {
1214               local = pool_elt_at_index (m->locals, local - m->locals);
1215               fib_index = local->fib_index;
1216               pool_put (m->locals, local);
1217               found = 1;
1218             }
1219         }
1220
1221       if (!found)
1222         {
1223           return VNET_API_ERROR_NO_SUCH_ENTRY;
1224         }
1225     }
1226   else
1227     {
1228       fib_index = m->fib_index;
1229     }
1230
1231   if (!is_sm_out2in_only (flags))
1232     {
1233       nat44_ed_sm_i2o_del (sm, l_addr, l_port, fib_index, proto);
1234     }
1235
1236   // delete sessions for static mapping
1237   if (sm->num_workers > 1)
1238     {
1239       tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
1240     }
1241   else
1242     {
1243       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1244     }
1245
1246   nat_ed_static_mapping_del_sessions (sm, tsm, m->local_addr, m->local_port,
1247                                       m->proto, fib_index,
1248                                       is_sm_addr_only (flags), e_addr, e_port);
1249
1250   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1251
1252   if (!pool_elts (m->locals))
1253     {
1254       // this is last record remove all required stuff
1255       // fib_index 0
1256       nat44_ed_sm_o2i_del (sm, e_addr, e_port, 0, proto);
1257
1258       vec_free (m->tag);
1259       vec_free (m->workers);
1260       pool_put (sm->static_mappings, m);
1261
1262       nat44_ed_add_del_interface_fib_reg_entries (e_addr, 0);
1263     }
1264
1265   return 0;
1266 }
1267
1268 int
1269 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1270                                 ip_protocol_t proto,
1271                                 nat44_lb_addr_port_t *locals, u32 flags,
1272                                 u8 *tag, u32 affinity)
1273 {
1274   snat_main_t *sm = &snat_main;
1275   snat_static_mapping_t *m;
1276   snat_address_t *a = 0;
1277
1278   nat44_lb_addr_port_t *local;
1279   uword *bitmap = 0;
1280   int rc = 0;
1281
1282   int i;
1283
1284   if (!sm->enabled)
1285     {
1286       return VNET_API_ERROR_UNSUPPORTED;
1287     }
1288
1289   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1290
1291   if (m)
1292     {
1293       return VNET_API_ERROR_VALUE_EXIST;
1294     }
1295
1296   if (vec_len (locals) < 2)
1297     {
1298       return VNET_API_ERROR_INVALID_VALUE;
1299     }
1300
1301   if (!is_sm_out2in_only (flags))
1302     {
1303       /* Find external address in allocated addresses and reserve port for
1304          address and port pair mapping when dynamic translations enabled */
1305       for (i = 0; i < vec_len (sm->addresses); i++)
1306         {
1307           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1308             {
1309               /* External port must be unused */
1310               a = sm->addresses + i;
1311               if (nat44_ed_sm_o2i_lookup (sm, a->addr, e_port, 0, proto))
1312                 {
1313                   return VNET_API_ERROR_VALUE_EXIST;
1314                 }
1315               break;
1316             }
1317         }
1318       // external address must be allocated
1319       if (!a)
1320         {
1321           return VNET_API_ERROR_NO_SUCH_ENTRY;
1322         }
1323     }
1324
1325   pool_get (sm->static_mappings, m);
1326   clib_memset (m, 0, sizeof (*m));
1327   m->tag = vec_dup (tag);
1328   m->external_addr = e_addr;
1329   m->external_port = e_port;
1330   m->affinity = affinity;
1331   m->proto = proto;
1332
1333   m->flags = flags;
1334   m->flags |= NAT_SM_FLAG_LB;
1335
1336   if (affinity)
1337     m->affinity_per_service_list_head_index =
1338       nat_affinity_get_per_service_list_head_index ();
1339   else
1340     m->affinity_per_service_list_head_index = ~0;
1341
1342   if (nat44_ed_sm_o2i_add (sm, m, m->external_addr, m->external_port, 0,
1343                            m->proto))
1344     {
1345       nat_log_err ("sm o2i key add failed");
1346       return VNET_API_ERROR_UNSPECIFIED;
1347     }
1348
1349   for (i = 0; i < vec_len (locals); i++)
1350     {
1351       locals[i].fib_index = fib_table_find_or_create_and_lock (
1352         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1353       if (!is_sm_out2in_only (flags))
1354         {
1355           if (nat44_ed_sm_i2o_add (sm, m, locals[i].addr, locals[i].port, 0,
1356                                    proto))
1357             {
1358               nat_log_err ("sm i2o key add failed");
1359               rc = VNET_API_ERROR_UNSPECIFIED;
1360               // here we continue with add operation so that it can be safely
1361               // reversed in delete path - otherwise we'd have to track what
1362               // we've done and deal with partial cleanups and since bihash
1363               // adds are (extremely improbable) the only points of failure,
1364               // it's easier to just do it this way
1365             }
1366         }
1367       locals[i].prefix = (i == 0) ?
1368                            locals[i].probability :
1369                            (locals[i - 1].prefix + locals[i].probability);
1370       pool_get (m->locals, local);
1371       *local = locals[i];
1372       if (sm->num_workers > 1)
1373         {
1374           ip4_header_t ip = {
1375             .src_address = locals[i].addr,
1376           };
1377           bitmap = clib_bitmap_set (
1378             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1379             1);
1380         }
1381     }
1382
1383   /* Assign workers */
1384   if (sm->num_workers > 1)
1385     {
1386       clib_bitmap_foreach (i, bitmap)
1387         {
1388           vec_add1 (m->workers, i);
1389         }
1390     }
1391
1392   return rc;
1393 }
1394
1395 int
1396 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1397                                 ip_protocol_t proto, u32 flags)
1398 {
1399   snat_main_t *sm = &snat_main;
1400   snat_static_mapping_t *m;
1401
1402   nat44_lb_addr_port_t *local;
1403   snat_main_per_thread_data_t *tsm;
1404   snat_session_t *s;
1405
1406   if (!sm->enabled)
1407     {
1408       return VNET_API_ERROR_UNSUPPORTED;
1409     }
1410
1411   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1412   if (!m)
1413     return VNET_API_ERROR_NO_SUCH_ENTRY;
1414
1415   if (!is_sm_lb (m->flags))
1416     return VNET_API_ERROR_INVALID_VALUE;
1417
1418   if (nat44_ed_sm_o2i_del (sm, m->external_addr, m->external_port, 0,
1419                            m->proto))
1420     {
1421       nat_log_err ("sm o2i key del failed");
1422       return VNET_API_ERROR_UNSPECIFIED;
1423     }
1424
1425   pool_foreach (local, m->locals)
1426     {
1427       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1428       if (!is_sm_out2in_only (flags))
1429         {
1430           if (nat44_ed_sm_i2o_del (sm, local->addr, local->port,
1431                                    local->fib_index, m->proto))
1432             {
1433               nat_log_err ("sm i2o key del failed");
1434               // For the same reasons as above
1435             }
1436         }
1437
1438       if (sm->num_workers > 1)
1439         {
1440           ip4_header_t ip = {
1441             .src_address = local->addr,
1442           };
1443           tsm = vec_elt_at_index (
1444             sm->per_thread_data,
1445             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1446         }
1447       else
1448         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1449
1450       /* Delete sessions */
1451       pool_foreach (s, tsm->sessions)
1452         {
1453           if (!(nat44_ed_is_lb_session (s)))
1454             continue;
1455
1456           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1457               s->in2out.port != local->port)
1458             continue;
1459
1460           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1461           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1462         }
1463     }
1464
1465   if (m->affinity)
1466     {
1467       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1468     }
1469
1470   pool_free (m->locals);
1471   vec_free (m->tag);
1472   vec_free (m->workers);
1473   pool_put (sm->static_mappings, m);
1474
1475   return 0;
1476 }
1477
1478 int
1479 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1480                                           ip4_address_t l_addr, u16 l_port,
1481                                           ip_protocol_t proto, u32 vrf_id,
1482                                           u8 probability, u8 is_add)
1483 {
1484   snat_main_t *sm = &snat_main;
1485   snat_static_mapping_t *m = 0;
1486   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1487   snat_main_per_thread_data_t *tsm;
1488   snat_session_t *s;
1489   u32 *locals = 0;
1490   uword *bitmap = 0;
1491   int i;
1492
1493   if (!sm->enabled)
1494     {
1495       return VNET_API_ERROR_UNSUPPORTED;
1496     }
1497
1498   m = nat44_ed_sm_o2i_lookup (sm, e_addr, e_port, 0, proto);
1499
1500   if (!m)
1501     {
1502       return VNET_API_ERROR_NO_SUCH_ENTRY;
1503     }
1504
1505   if (!is_sm_lb (m->flags))
1506     {
1507       return VNET_API_ERROR_INVALID_VALUE;
1508     }
1509
1510   pool_foreach (local, m->locals)
1511    {
1512     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1513         (local->vrf_id == vrf_id))
1514       {
1515         match_local = local;
1516         break;
1517       }
1518   }
1519
1520   if (is_add)
1521     {
1522       if (match_local)
1523         {
1524           return VNET_API_ERROR_VALUE_EXIST;
1525         }
1526
1527       pool_get (m->locals, local);
1528       clib_memset (local, 0, sizeof (*local));
1529       local->addr.as_u32 = l_addr.as_u32;
1530       local->port = l_port;
1531       local->probability = probability;
1532       local->vrf_id = vrf_id;
1533       local->fib_index =
1534         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1535                                            sm->fib_src_low);
1536
1537       if (!is_sm_out2in_only (m->flags))
1538         {
1539           if (nat44_ed_sm_i2o_add (sm, m, l_addr, l_port, local->fib_index,
1540                                    proto))
1541             {
1542               nat_log_err ("sm i2o key add failed");
1543               pool_put (m->locals, local);
1544               return VNET_API_ERROR_UNSPECIFIED;
1545             }
1546         }
1547     }
1548   else
1549     {
1550       if (!match_local)
1551         return VNET_API_ERROR_NO_SUCH_ENTRY;
1552
1553       if (pool_elts (m->locals) < 3)
1554         return VNET_API_ERROR_UNSPECIFIED;
1555
1556       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1557                         sm->fib_src_low);
1558
1559       if (!is_sm_out2in_only (m->flags))
1560         {
1561           if (nat44_ed_sm_i2o_del (sm, l_addr, l_port, match_local->fib_index,
1562                                    proto))
1563             nat_log_err ("sm i2o key del failed");
1564         }
1565
1566       if (sm->num_workers > 1)
1567         {
1568           ip4_header_t ip = {
1569             .src_address = local->addr,
1570           };
1571           tsm = vec_elt_at_index (
1572             sm->per_thread_data,
1573             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1574         }
1575       else
1576         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1577
1578       /* Delete sessions */
1579       pool_foreach (s, tsm->sessions) {
1580           if (!(nat44_ed_is_lb_session (s)))
1581             continue;
1582
1583           if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1584               s->in2out.port != match_local->port)
1585             continue;
1586
1587           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1588           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1589       }
1590
1591       pool_put (m->locals, match_local);
1592     }
1593
1594   vec_free (m->workers);
1595
1596   pool_foreach (local, m->locals)
1597    {
1598     vec_add1 (locals, local - m->locals);
1599     if (sm->num_workers > 1)
1600       {
1601         ip4_header_t ip;
1602         ip.src_address.as_u32 = local->addr.as_u32,
1603         bitmap = clib_bitmap_set (
1604           bitmap,
1605           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1606       }
1607   }
1608
1609   ASSERT (vec_len (locals) > 1);
1610
1611   local = pool_elt_at_index (m->locals, locals[0]);
1612   local->prefix = local->probability;
1613   for (i = 1; i < vec_len (locals); i++)
1614     {
1615       local = pool_elt_at_index (m->locals, locals[i]);
1616       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1617       local->prefix = local->probability + prev_local->prefix;
1618     }
1619
1620   /* Assign workers */
1621   if (sm->num_workers > 1)
1622     {
1623       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1624     }
1625
1626   return 0;
1627 }
1628
1629 void
1630 expire_per_vrf_sessions (u32 fib_index)
1631 {
1632   per_vrf_sessions_t *per_vrf_sessions;
1633   snat_main_per_thread_data_t *tsm;
1634   snat_main_t *sm = &snat_main;
1635
1636   vec_foreach (tsm, sm->per_thread_data)
1637     {
1638       pool_foreach (per_vrf_sessions, tsm->per_vrf_sessions_pool)
1639         {
1640           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1641               (per_vrf_sessions->tx_fib_index == fib_index))
1642             {
1643               per_vrf_sessions->expired = 1;
1644             }
1645         }
1646     }
1647 }
1648
1649 void
1650 update_per_vrf_sessions_pool (u32 fib_index, int is_del)
1651 {
1652   snat_main_t *sm = &snat_main;
1653   nat_fib_t *fib;
1654
1655   // we don't care if it is outside/inside fib
1656   // we just care about their ref_count
1657   // if it reaches 0 sessions should expire
1658   // because the fib isn't valid for NAT anymore
1659
1660   vec_foreach (fib, sm->fibs)
1661   {
1662     if (fib->fib_index == fib_index)
1663       {
1664         if (is_del)
1665           {
1666             fib->ref_count--;
1667             if (!fib->ref_count)
1668               {
1669                 vec_del1 (sm->fibs, fib - sm->fibs);
1670                 expire_per_vrf_sessions (fib_index);
1671               }
1672             return;
1673           }
1674         else
1675           fib->ref_count++;
1676       }
1677   }
1678   if (!is_del)
1679     {
1680       vec_add2 (sm->fibs, fib, 1);
1681       fib->ref_count = 1;
1682       fib->fib_index = fib_index;
1683     }
1684 }
1685
1686 static_always_inline nat_fib_t *
1687 nat44_ed_get_outside_fib (nat_fib_t *outside_fibs, u32 fib_index)
1688 {
1689   nat_fib_t *f;
1690   vec_foreach (f, outside_fibs)
1691     {
1692       if (f->fib_index == fib_index)
1693         {
1694           return f;
1695         }
1696     }
1697   return 0;
1698 }
1699
1700 static_always_inline snat_interface_t *
1701 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1702 {
1703   snat_interface_t *i;
1704   pool_foreach (i, interfaces)
1705     {
1706       if (i->sw_if_index == sw_if_index)
1707         {
1708           return i;
1709         }
1710     }
1711   return 0;
1712 }
1713
1714 int
1715 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1716 {
1717   const char *del_feature_name, *feature_name;
1718   snat_main_t *sm = &snat_main;
1719
1720   nat_fib_t *outside_fib;
1721   snat_interface_t *i;
1722   u32 fib_index;
1723   int rv;
1724
1725   if (!sm->enabled)
1726     {
1727       nat_log_err ("nat44 is disabled");
1728       return VNET_API_ERROR_UNSUPPORTED;
1729     }
1730
1731   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1732     {
1733       nat_log_err ("error interface already configured");
1734       return VNET_API_ERROR_VALUE_EXIST;
1735     }
1736
1737   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1738   if (i)
1739     {
1740       if ((nat44_ed_is_interface_inside (i) && is_inside) ||
1741           (nat44_ed_is_interface_outside (i) && !is_inside))
1742         {
1743           return 0;
1744         }
1745       if (sm->num_workers > 1)
1746         {
1747           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1748                                           "nat44-out2in-worker-handoff";
1749           feature_name = "nat44-handoff-classify";
1750         }
1751       else
1752         {
1753           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1754
1755           feature_name = "nat44-ed-classify";
1756         }
1757
1758       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1759       if (rv)
1760         return rv;
1761       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1762                                    sw_if_index, 0, 0, 0);
1763       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1764                                    0, 0);
1765     }
1766   else
1767     {
1768       if (sm->num_workers > 1)
1769         {
1770           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1771                                      "nat44-out2in-worker-handoff";
1772         }
1773       else
1774         {
1775           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1776         }
1777
1778       nat_validate_interface_counters (sm, sw_if_index);
1779       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1780       if (rv)
1781         return rv;
1782       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1783                                    0, 0);
1784
1785       pool_get (sm->interfaces, i);
1786       i->sw_if_index = sw_if_index;
1787       i->flags = 0;
1788     }
1789
1790   fib_index =
1791     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1792
1793   update_per_vrf_sessions_pool (fib_index, 0 /*is_del*/);
1794
1795   if (!is_inside)
1796     {
1797       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1798
1799       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1800       if (outside_fib)
1801         {
1802           outside_fib->ref_count++;
1803         }
1804       else
1805         {
1806           vec_add2 (sm->outside_fibs, outside_fib, 1);
1807           outside_fib->fib_index = fib_index;
1808           outside_fib->ref_count = 1;
1809         }
1810
1811       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
1812       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
1813
1814       nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
1815     }
1816   else
1817     {
1818       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1819     }
1820
1821   return 0;
1822 }
1823
1824 int
1825 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1826 {
1827   const char *del_feature_name, *feature_name;
1828   snat_main_t *sm = &snat_main;
1829
1830   nat_fib_t *outside_fib;
1831   snat_interface_t *i;
1832   u32 fib_index;
1833   int rv;
1834
1835   if (!sm->enabled)
1836     {
1837       nat_log_err ("nat44 is disabled");
1838       return VNET_API_ERROR_UNSUPPORTED;
1839     }
1840
1841   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1842   if (i == 0)
1843     {
1844       nat_log_err ("error interface couldn't be found");
1845       return VNET_API_ERROR_NO_SUCH_ENTRY;
1846     }
1847
1848   if (nat44_ed_is_interface_inside (i) && nat44_ed_is_interface_outside (i))
1849     {
1850       if (sm->num_workers > 1)
1851         {
1852           del_feature_name = "nat44-handoff-classify";
1853           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1854                                       "nat44-out2in-worker-handoff";
1855         }
1856       else
1857         {
1858           del_feature_name = "nat44-ed-classify";
1859           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1860         }
1861
1862       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1863       if (rv)
1864         {
1865           return rv;
1866         }
1867       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1868                                    sw_if_index, 0, 0, 0);
1869       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1870                                    0, 0);
1871
1872       if (is_inside)
1873         {
1874           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1875         }
1876       else
1877         {
1878           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1879         }
1880     }
1881   else
1882     {
1883       if (sm->num_workers > 1)
1884         {
1885           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1886                                      "nat44-out2in-worker-handoff";
1887         }
1888       else
1889         {
1890           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1891         }
1892
1893       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1894       if (rv)
1895         {
1896           return rv;
1897         }
1898       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1899                                    0, 0);
1900
1901       // remove interface
1902       pool_put (sm->interfaces, i);
1903     }
1904
1905   fib_index =
1906     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1907
1908   update_per_vrf_sessions_pool (fib_index, 1 /*is_del*/);
1909
1910   if (!is_inside)
1911     {
1912       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1913       if (outside_fib)
1914         {
1915           outside_fib->ref_count--;
1916           if (!outside_fib->ref_count)
1917             {
1918               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1919             }
1920         }
1921
1922       nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
1923       nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
1924     }
1925
1926   return 0;
1927 }
1928
1929 int
1930 nat44_ed_add_output_interface (u32 sw_if_index)
1931 {
1932   snat_main_t *sm = &snat_main;
1933
1934   nat_fib_t *outside_fib;
1935   snat_interface_t *i;
1936   u32 fib_index;
1937   int rv;
1938
1939   if (!sm->enabled)
1940     {
1941       nat_log_err ("nat44 is disabled");
1942       return VNET_API_ERROR_UNSUPPORTED;
1943     }
1944
1945   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1946     {
1947       nat_log_err ("error interface already configured");
1948       return VNET_API_ERROR_VALUE_EXIST;
1949     }
1950
1951   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1952     {
1953       nat_log_err ("error interface already configured");
1954       return VNET_API_ERROR_VALUE_EXIST;
1955     }
1956
1957   if (sm->num_workers > 1)
1958     {
1959       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1960       if (rv)
1961         {
1962           return rv;
1963         }
1964
1965       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1966       if (rv)
1967         {
1968           return rv;
1969         }
1970
1971       vnet_feature_enable_disable (
1972         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1973       vnet_feature_enable_disable ("ip4-output",
1974                                    "nat44-in2out-output-worker-handoff",
1975                                    sw_if_index, 1, 0, 0);
1976     }
1977   else
1978     {
1979       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1980       if (rv)
1981         {
1982           return rv;
1983         }
1984
1985       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1986       if (rv)
1987         {
1988           return rv;
1989         }
1990
1991       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1992                                    sw_if_index, 1, 0, 0);
1993       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1994                                    sw_if_index, 1, 0, 0);
1995     }
1996
1997   nat_validate_interface_counters (sm, sw_if_index);
1998
1999   pool_get (sm->output_feature_interfaces, i);
2000   i->sw_if_index = sw_if_index;
2001   i->flags = 0;
2002   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
2003   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
2004
2005   fib_index =
2006     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2007   update_per_vrf_sessions_pool (fib_index, 0 /*is_del*/);
2008
2009   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2010   if (outside_fib)
2011     {
2012       outside_fib->ref_count++;
2013     }
2014   else
2015     {
2016       vec_add2 (sm->outside_fibs, outside_fib, 1);
2017       outside_fib->fib_index = fib_index;
2018       outside_fib->ref_count = 1;
2019     }
2020
2021   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 1);
2022   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 1);
2023
2024   nat44_ed_bind_if_addr_to_nat_addr (sw_if_index);
2025
2026   return 0;
2027 }
2028
2029 int
2030 nat44_ed_del_output_interface (u32 sw_if_index)
2031 {
2032   snat_main_t *sm = &snat_main;
2033
2034   nat_fib_t *outside_fib;
2035   snat_interface_t *i;
2036   u32 fib_index;
2037   int rv;
2038
2039   if (!sm->enabled)
2040     {
2041       nat_log_err ("nat44 is disabled");
2042       return VNET_API_ERROR_UNSUPPORTED;
2043     }
2044
2045   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2046   if (!i)
2047     {
2048       nat_log_err ("error interface couldn't be found");
2049       return VNET_API_ERROR_NO_SUCH_ENTRY;
2050     }
2051
2052   if (sm->num_workers > 1)
2053     {
2054       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2055       if (rv)
2056         {
2057           return rv;
2058         }
2059
2060       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2061       if (rv)
2062         {
2063           return rv;
2064         }
2065
2066       vnet_feature_enable_disable (
2067         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
2068       vnet_feature_enable_disable ("ip4-output",
2069                                    "nat44-in2out-output-worker-handoff",
2070                                    sw_if_index, 0, 0, 0);
2071     }
2072   else
2073     {
2074       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2075       if (rv)
2076         {
2077           return rv;
2078         }
2079
2080       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2081       if (rv)
2082         {
2083           return rv;
2084         }
2085
2086       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2087                                    sw_if_index, 0, 0, 0);
2088       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2089                                    sw_if_index, 0, 0, 0);
2090     }
2091
2092   // remove interface
2093   pool_put (sm->output_feature_interfaces, i);
2094
2095   fib_index =
2096     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2097   update_per_vrf_sessions_pool (fib_index, 1 /*is_del*/);
2098
2099   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2100   if (outside_fib)
2101     {
2102       outside_fib->ref_count--;
2103       if (!outside_fib->ref_count)
2104         {
2105           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2106         }
2107     }
2108
2109   nat44_ed_add_del_nat_addr_fib_reg_entries (sw_if_index, 0);
2110   nat44_ed_add_del_sm_fib_reg_entries (sw_if_index, 0);
2111
2112   return 0;
2113 }
2114
2115 int
2116 snat_set_workers (uword * bitmap)
2117 {
2118   snat_main_t *sm = &snat_main;
2119   int i, j = 0;
2120
2121   if (sm->num_workers < 2)
2122     return VNET_API_ERROR_FEATURE_DISABLED;
2123
2124   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2125     return VNET_API_ERROR_INVALID_WORKER;
2126
2127   vec_free (sm->workers);
2128   clib_bitmap_foreach (i, bitmap)
2129     {
2130       vec_add1(sm->workers, i);
2131       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2132       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2133       j++;
2134     }
2135
2136   sm->port_per_thread = (65536 - ED_USER_PORT_OFFSET) / _vec_len (sm->workers);
2137
2138   return 0;
2139 }
2140
2141 int
2142 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2143 {
2144   fail_if_enabled ();
2145   snat_main_t *sm = &snat_main;
2146
2147   if ((sm->fq_in2out_index != ~0) || (sm->fq_out2in_index != ~0) ||
2148       (sm->fq_in2out_output_index != ~0))
2149     {
2150       // frame queu nelts can be set only before first
2151       // call to nat44_plugin_enable after that it
2152       // doesn't make sense
2153       nat_log_err ("Frame queue was already initialized. "
2154                    "Change is not possible");
2155       return 1;
2156     }
2157
2158   sm->frame_queue_nelts = frame_queue_nelts;
2159   return 0;
2160 }
2161
2162 static void
2163 nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque, u32 sw_if_index,
2164                                 u32 new_fib_index, u32 old_fib_index)
2165 {
2166   snat_main_t *sm = &snat_main;
2167   nat_fib_t *outside_fib;
2168   snat_interface_t *i;
2169   u8 is_add = 1;
2170   u8 match = 0;
2171
2172   if (!sm->enabled || (new_fib_index == old_fib_index)
2173       || (!vec_len (sm->outside_fibs)))
2174     {
2175       return;
2176     }
2177
2178   pool_foreach (i, sm->interfaces)
2179     {
2180       if (i->sw_if_index == sw_if_index)
2181         {
2182           if (!(nat44_ed_is_interface_outside (i)))
2183             return;
2184           match = 1;
2185         }
2186     }
2187
2188   pool_foreach (i, sm->output_feature_interfaces)
2189     {
2190       if (i->sw_if_index == sw_if_index)
2191         {
2192           if (!(nat44_ed_is_interface_outside (i)))
2193             return;
2194           match = 1;
2195         }
2196     }
2197
2198   if (!match)
2199     return;
2200
2201   vec_foreach (outside_fib, sm->outside_fibs)
2202     {
2203       if (outside_fib->fib_index == old_fib_index)
2204         {
2205           outside_fib->ref_count--;
2206           if (!outside_fib->ref_count)
2207             vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2208           break;
2209         }
2210     }
2211
2212   vec_foreach (outside_fib, sm->outside_fibs)
2213     {
2214       if (outside_fib->fib_index == new_fib_index)
2215         {
2216           outside_fib->ref_count++;
2217           is_add = 0;
2218           break;
2219         }
2220     }
2221
2222   if (is_add)
2223     {
2224       vec_add2 (sm->outside_fibs, outside_fib, 1);
2225       outside_fib->ref_count = 1;
2226       outside_fib->fib_index = new_fib_index;
2227     }
2228 }
2229
2230 static void nat44_ed_update_outside_fib_cb (ip4_main_t *im, uword opaque,
2231                                             u32 sw_if_index, u32 new_fib_index,
2232                                             u32 old_fib_index);
2233
2234 static void nat44_ed_add_del_interface_address_cb (
2235   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2236   u32 address_length, u32 if_address_index, u32 is_delete);
2237
2238 static void nat44_ed_add_del_static_mapping_cb (
2239   ip4_main_t *im, uword opaque, u32 sw_if_index, ip4_address_t *address,
2240   u32 address_length, u32 if_address_index, u32 is_delete);
2241
2242 void
2243 test_key_calc_split ()
2244 {
2245   ip4_address_t l_addr;
2246   l_addr.as_u8[0] = 1;
2247   l_addr.as_u8[1] = 1;
2248   l_addr.as_u8[2] = 1;
2249   l_addr.as_u8[3] = 1;
2250   ip4_address_t r_addr;
2251   r_addr.as_u8[0] = 2;
2252   r_addr.as_u8[1] = 2;
2253   r_addr.as_u8[2] = 2;
2254   r_addr.as_u8[3] = 2;
2255   u16 l_port = 40001;
2256   u16 r_port = 40301;
2257   u8 proto = 9;
2258   u32 fib_index = 9000001;
2259   u32 thread_index = 3000000001;
2260   u32 session_index = 3000000221;
2261   clib_bihash_kv_16_8_t kv;
2262   init_ed_kv (&kv, l_addr.as_u32, l_port, r_addr.as_u32, r_port, fib_index,
2263               proto, thread_index, session_index);
2264   ip4_address_t l_addr2;
2265   ip4_address_t r_addr2;
2266   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2267   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2268   u16 l_port2 = 0;
2269   u16 r_port2 = 0;
2270   u8 proto2 = 0;
2271   u32 fib_index2 = 0;
2272   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2273                &r_port2);
2274   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2275   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2276   ASSERT (l_port == l_port2);
2277   ASSERT (r_port == r_port2);
2278   ASSERT (proto == proto2);
2279   ASSERT (fib_index == fib_index2);
2280   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2281   ASSERT (session_index == ed_value_get_session_index (&kv));
2282 }
2283
2284 static clib_error_t *
2285 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2286 {
2287   u32 fib_index;
2288   if (!is_add)
2289     {
2290       fib_index = ip4_fib_index_from_table_id (table_id);
2291       if (fib_index != ~0)
2292         {
2293           expire_per_vrf_sessions (fib_index);
2294         }
2295     }
2296   return 0;
2297 }
2298
2299 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2300
2301 #define nat_validate_simple_counter(c, i)                                     \
2302   do                                                                          \
2303     {                                                                         \
2304       vlib_validate_simple_counter (&c, i);                                   \
2305       vlib_zero_simple_counter (&c, i);                                       \
2306     }                                                                         \
2307   while (0);
2308
2309 #define nat_init_simple_counter(c, n, sn)                                     \
2310   do                                                                          \
2311     {                                                                         \
2312       c.name = n;                                                             \
2313       c.stat_segment_name = sn;                                               \
2314       nat_validate_simple_counter (c, 0);                                     \
2315     }                                                                         \
2316   while (0);
2317
2318 static_always_inline void
2319 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2320 {
2321 #define _(x)                                                                  \
2322   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2323   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2324   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2325   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2326   foreach_nat_counter;
2327 #undef _
2328   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2329 }
2330
2331 static clib_error_t *
2332 nat_init (vlib_main_t * vm)
2333 {
2334   snat_main_t *sm = &snat_main;
2335   vlib_thread_main_t *tm = vlib_get_thread_main ();
2336   vlib_thread_registration_t *tr;
2337   ip4_add_del_interface_address_callback_t cbi = { 0 };
2338   ip4_table_bind_callback_t cbt = { 0 };
2339   u32 i, num_threads = 0;
2340   uword *p, *bitmap = 0;
2341
2342   clib_memset (sm, 0, sizeof (*sm));
2343
2344   // convenience
2345   sm->ip4_main = &ip4_main;
2346
2347   // frame queue indices used for handoff
2348   sm->fq_out2in_index = ~0;
2349   sm->fq_in2out_index = ~0;
2350   sm->fq_in2out_output_index = ~0;
2351
2352   sm->log_level = NAT_LOG_ERROR;
2353
2354   sm->log_class = vlib_log_register_class ("nat", 0);
2355   nat_ipfix_logging_init (vm);
2356
2357   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2358                            "/nat44-ed/total-sessions");
2359   sm->max_cfg_sessions_gauge =
2360     vlib_stats_add_gauge ("/nat44-ed/max-cfg-sessions");
2361
2362 #define _(x)                                                                  \
2363   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2364                            "/nat44-ed/in2out/fastpath/" #x);                  \
2365   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2366                            "/nat44-ed/out2in/fastpath/" #x);                  \
2367   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2368                            "/nat44-ed/in2out/slowpath/" #x);                  \
2369   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2370                            "/nat44-ed/out2in/slowpath/" #x);
2371   foreach_nat_counter;
2372 #undef _
2373   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2374                            "/nat44-ed/hairpinning");
2375
2376   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2377   if (p)
2378     {
2379       tr = (vlib_thread_registration_t *) p[0];
2380       if (tr)
2381         {
2382           sm->num_workers = tr->count;
2383           sm->first_worker_index = tr->first_index;
2384         }
2385     }
2386   num_threads = tm->n_vlib_mains - 1;
2387   sm->port_per_thread = 65536 - ED_USER_PORT_OFFSET;
2388   vec_validate (sm->per_thread_data, num_threads);
2389
2390   /* Use all available workers by default */
2391   if (sm->num_workers > 1)
2392     {
2393       for (i = 0; i < sm->num_workers; i++)
2394         bitmap = clib_bitmap_set (bitmap, i, 1);
2395       snat_set_workers (bitmap);
2396       clib_bitmap_free (bitmap);
2397     }
2398   else
2399     {
2400       sm->per_thread_data[0].snat_thread_index = 0;
2401     }
2402
2403   /* callbacks to call when interface address changes. */
2404   cbi.function = nat44_ed_add_del_interface_address_cb;
2405   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2406   cbi.function = nat44_ed_add_del_static_mapping_cb;
2407   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2408
2409   /* callbacks to call when interface to table biding changes */
2410   cbt.function = nat44_ed_update_outside_fib_cb;
2411   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2412
2413   sm->fib_src_low =
2414     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2415                          FIB_SOURCE_BH_SIMPLE);
2416   sm->fib_src_hi =
2417     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2418                          FIB_SOURCE_BH_SIMPLE);
2419
2420   nat_affinity_init (vm);
2421   test_key_calc_split ();
2422
2423   return nat44_api_hookup (vm);
2424 }
2425
2426 VLIB_INIT_FUNCTION (nat_init);
2427
2428 int
2429 nat44_plugin_enable (nat44_config_t c)
2430 {
2431   snat_main_t *sm = &snat_main;
2432
2433   fail_if_enabled ();
2434
2435   sm->forwarding_enabled = 0;
2436   sm->mss_clamping = 0;
2437
2438   if (!c.sessions)
2439     c.sessions = 63 * 1024;
2440
2441   sm->max_translations_per_thread = c.sessions;
2442   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
2443                         sm->max_translations_per_thread);
2444   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2445
2446   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2447
2448   sm->inside_vrf_id = c.inside_vrf;
2449   sm->inside_fib_index =
2450     fib_table_find_or_create_and_lock
2451     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2452
2453   sm->outside_vrf_id = c.outside_vrf;
2454   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2455     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2456
2457   nat44_ed_db_init ();
2458
2459   nat_affinity_enable ();
2460
2461   nat_reset_timeouts (&sm->timeouts);
2462
2463   vlib_zero_simple_counter (&sm->total_sessions, 0);
2464
2465   if (!sm->frame_queue_nelts)
2466     {
2467       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2468     }
2469
2470   if (sm->num_workers > 1)
2471     {
2472       vlib_main_t *vm = vlib_get_main ();
2473       vlib_node_t *node;
2474
2475       if (sm->fq_in2out_index == ~0)
2476         {
2477           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2478           sm->fq_in2out_index =
2479             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2480         }
2481       if (sm->fq_out2in_index == ~0)
2482         {
2483           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2484           sm->fq_out2in_index =
2485             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2486         }
2487       if (sm->fq_in2out_output_index == ~0)
2488         {
2489           node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2490           sm->fq_in2out_output_index =
2491             vlib_frame_queue_main_init (node->index, sm->frame_queue_nelts);
2492         }
2493     }
2494
2495   sm->enabled = 1;
2496   sm->rconfig = c;
2497
2498   return 0;
2499 }
2500
2501 int
2502 nat44_ed_del_addresses ()
2503 {
2504   snat_main_t *sm = &snat_main;
2505   snat_address_t *a, *vec;
2506   int error = 0;
2507
2508   vec = vec_dup (sm->addresses);
2509   vec_foreach (a, vec)
2510     {
2511       error = nat44_ed_del_address (a->addr, 0);
2512       if (error)
2513         {
2514           nat_log_err ("error occurred while removing adderess");
2515         }
2516     }
2517   vec_free (vec);
2518   vec_free (sm->addresses);
2519   sm->addresses = 0;
2520
2521   vec = vec_dup (sm->twice_nat_addresses);
2522   vec_foreach (a, vec)
2523     {
2524       error = nat44_ed_del_address (a->addr, 1);
2525       if (error)
2526         {
2527           nat_log_err ("error occurred while removing adderess");
2528         }
2529     }
2530   vec_free (vec);
2531   vec_free (sm->twice_nat_addresses);
2532   sm->twice_nat_addresses = 0;
2533
2534   vec_free (sm->addr_to_resolve);
2535   sm->addr_to_resolve = 0;
2536
2537   return error;
2538 }
2539
2540 int
2541 nat44_ed_del_interfaces ()
2542 {
2543   snat_main_t *sm = &snat_main;
2544   snat_interface_t *i, *pool;
2545   int error = 0;
2546
2547   pool = pool_dup (sm->interfaces);
2548   pool_foreach (i, pool)
2549     {
2550       if (nat44_ed_is_interface_inside (i))
2551         {
2552           error = nat44_ed_del_interface (i->sw_if_index, 1);
2553         }
2554       if (nat44_ed_is_interface_outside (i))
2555         {
2556           error = nat44_ed_del_interface (i->sw_if_index, 0);
2557         }
2558
2559       if (error)
2560         {
2561           nat_log_err ("error occurred while removing interface");
2562         }
2563     }
2564   pool_free (pool);
2565   pool_free (sm->interfaces);
2566   sm->interfaces = 0;
2567   return error;
2568 }
2569
2570 int
2571 nat44_ed_del_output_interfaces ()
2572 {
2573   snat_main_t *sm = &snat_main;
2574   snat_interface_t *i, *pool;
2575   int error = 0;
2576
2577   pool = pool_dup (sm->output_feature_interfaces);
2578   pool_foreach (i, pool)
2579     {
2580       error = nat44_ed_del_output_interface (i->sw_if_index);
2581       if (error)
2582         {
2583           nat_log_err ("error occurred while removing output interface");
2584         }
2585     }
2586   pool_free (pool);
2587   pool_free (sm->output_feature_interfaces);
2588   sm->output_feature_interfaces = 0;
2589   return error;
2590 }
2591
2592 static clib_error_t *
2593 nat44_ed_sw_interface_add_del (vnet_main_t *vnm, u32 sw_if_index, u32 is_add)
2594 {
2595   snat_main_t *sm = &snat_main;
2596   snat_interface_t *i;
2597   int error = 0;
2598
2599   if (is_add)
2600     return 0;
2601
2602   if (!sm->enabled)
2603     return 0;
2604
2605   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
2606   if (i)
2607     {
2608       bool is_inside = nat44_ed_is_interface_inside (i);
2609       bool is_outside = nat44_ed_is_interface_outside (i);
2610
2611       if (is_inside)
2612         {
2613           error |= nat44_ed_del_interface (sw_if_index, 1);
2614         }
2615       if (is_outside)
2616         {
2617           error |= nat44_ed_del_interface (sw_if_index, 0);
2618         }
2619
2620       if (error)
2621         {
2622           nat_log_err ("error occurred while removing interface");
2623         }
2624     }
2625
2626   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2627   if (i)
2628     {
2629       error = nat44_ed_del_output_interface (sw_if_index);
2630       if (error)
2631         {
2632           nat_log_err ("error occurred while removing output interface");
2633         }
2634     }
2635
2636   return 0;
2637 }
2638
2639 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (nat44_ed_sw_interface_add_del);
2640
2641 int
2642 nat44_ed_del_static_mappings ()
2643 {
2644   snat_main_t *sm = &snat_main;
2645   snat_static_mapping_t *m, *pool;
2646   int error = 0;
2647
2648   pool = pool_dup (sm->static_mappings);
2649   pool_foreach (m, pool)
2650     {
2651       error = nat44_ed_del_static_mapping_internal (
2652         m->local_addr, m->external_addr, m->local_port, m->external_port,
2653         m->proto, m->vrf_id, m->flags);
2654       if (error)
2655         {
2656           nat_log_err ("error occurred while removing mapping");
2657         }
2658     }
2659   pool_free (pool);
2660   pool_free (sm->static_mappings);
2661   sm->static_mappings = 0;
2662
2663   vec_free (sm->sm_to_resolve);
2664   sm->sm_to_resolve = 0;
2665
2666   return error;
2667 }
2668
2669 int
2670 nat44_plugin_disable ()
2671 {
2672   snat_main_t *sm = &snat_main;
2673   int rc, error = 0;
2674
2675   fail_if_disabled ();
2676
2677   rc = nat44_ed_del_static_mappings ();
2678   if (rc)
2679     error = VNET_API_ERROR_BUG;
2680
2681   rc = nat44_ed_del_addresses ();
2682   if (rc)
2683     error = VNET_API_ERROR_BUG;
2684
2685   rc = nat44_ed_del_interfaces ();
2686   if (rc)
2687     error = VNET_API_ERROR_BUG;
2688
2689   rc = nat44_ed_del_output_interfaces ();
2690   if (rc)
2691     error = VNET_API_ERROR_BUG;
2692
2693   nat44_ed_del_vrf_tables ();
2694
2695   vec_free (sm->max_translations_per_fib);
2696   sm->max_translations_per_fib = 0;
2697
2698   nat44_ed_db_free ();
2699
2700   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2701
2702   nat_affinity_disable ();
2703
2704   sm->forwarding_enabled = 0;
2705   sm->enabled = 0;
2706
2707   return error;
2708 }
2709
2710 void
2711 nat44_ed_forwarding_enable_disable (u8 is_enable)
2712 {
2713   snat_main_per_thread_data_t *tsm;
2714   snat_main_t *sm = &snat_main;
2715   snat_session_t *s;
2716
2717   u32 *ses_to_be_removed = 0, *ses_index;
2718
2719   sm->forwarding_enabled = is_enable != 0;
2720
2721   if (!sm->enabled || is_enable)
2722     {
2723       return;
2724     }
2725
2726   vec_foreach (tsm, sm->per_thread_data)
2727     {
2728       pool_foreach (s, tsm->sessions)
2729         {
2730           if (na44_ed_is_fwd_bypass_session (s))
2731             {
2732               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2733             }
2734         }
2735       vec_foreach (ses_index, ses_to_be_removed)
2736         {
2737           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2738           nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2739           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2740         }
2741
2742       vec_free (ses_to_be_removed);
2743     }
2744 }
2745
2746 static_always_inline snat_static_mapping_t *
2747 nat44_ed_sm_match (snat_main_t *sm, ip4_address_t match_addr, u16 match_port,
2748                    u32 match_fib_index, ip_protocol_t match_protocol,
2749                    int by_external)
2750 {
2751   snat_static_mapping_t *m;
2752   if (!by_external)
2753     {
2754       m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port, match_fib_index,
2755                                   match_protocol);
2756       if (m)
2757         return m;
2758
2759       // try address only mapping
2760       m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, match_fib_index, 0);
2761       if (m)
2762         return m;
2763
2764       // default static mapping fib index (based on configuration)
2765       if (sm->inside_fib_index != match_fib_index)
2766         {
2767           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2768                                       sm->inside_fib_index, match_protocol);
2769           if (m)
2770             return m;
2771
2772           // try address only mapping
2773           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->inside_fib_index,
2774                                       0);
2775           if (m)
2776             return m;
2777         }
2778       // TODO: this specific use case may be deprecated (needs testing)
2779       if (sm->outside_fib_index != match_fib_index)
2780         {
2781           m = nat44_ed_sm_i2o_lookup (sm, match_addr, match_port,
2782                                       sm->outside_fib_index, match_protocol);
2783           if (m)
2784             return m;
2785
2786           // try address only mapping
2787           m = nat44_ed_sm_i2o_lookup (sm, match_addr, 0, sm->outside_fib_index,
2788                                       0);
2789           if (m)
2790             return m;
2791         }
2792     }
2793   else
2794     {
2795       m =
2796         nat44_ed_sm_o2i_lookup (sm, match_addr, match_port, 0, match_protocol);
2797       if (m)
2798         return m;
2799
2800       // try address only mapping
2801       m = nat44_ed_sm_o2i_lookup (sm, match_addr, 0, 0, 0);
2802       if (m)
2803         return m;
2804     }
2805   return 0;
2806 }
2807
2808 int
2809 snat_static_mapping_match (vlib_main_t *vm, ip4_address_t match_addr,
2810                            u16 match_port, u32 match_fib_index,
2811                            ip_protocol_t match_protocol,
2812                            ip4_address_t *mapping_addr, u16 *mapping_port,
2813                            u32 *mapping_fib_index, int by_external,
2814                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2815                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2816                            u8 *is_identity_nat, snat_static_mapping_t **out)
2817 {
2818   snat_main_t *sm = &snat_main;
2819   snat_static_mapping_t *m;
2820   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2821   nat44_lb_addr_port_t *local;
2822   u8 backend_index;
2823
2824   m = nat44_ed_sm_match (sm, match_addr, match_port, match_fib_index,
2825                          match_protocol, by_external);
2826   if (!m)
2827     {
2828       return 1;
2829     }
2830
2831   if (by_external)
2832     {
2833       if (is_sm_lb (m->flags))
2834         {
2835           if (PREDICT_FALSE (lb != 0))
2836             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2837           if (m->affinity && !nat_affinity_find_and_lock (
2838                                vm, ext_host_addr[0], match_addr,
2839                                match_protocol, match_port, &backend_index))
2840             {
2841               local = pool_elt_at_index (m->locals, backend_index);
2842               *mapping_addr = local->addr;
2843               *mapping_port = local->port;
2844               *mapping_fib_index = local->fib_index;
2845               goto end;
2846             }
2847           // pick locals matching this worker
2848           if (PREDICT_FALSE (sm->num_workers > 1))
2849             {
2850               u32 thread_index = vlib_get_thread_index ();
2851               pool_foreach_index (i, m->locals)
2852                {
2853                 local = pool_elt_at_index (m->locals, i);
2854
2855                 ip4_header_t ip = {
2856                   .src_address = local->addr,
2857                 };
2858
2859                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2860                                                       0) == thread_index)
2861                   {
2862                     vec_add1 (tmp, i);
2863                   }
2864                }
2865               ASSERT (vec_len (tmp) != 0);
2866             }
2867           else
2868             {
2869               pool_foreach_index (i, m->locals)
2870                {
2871                 vec_add1 (tmp, i);
2872               }
2873             }
2874           hi = vec_len (tmp) - 1;
2875           local = pool_elt_at_index (m->locals, tmp[hi]);
2876           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2877           while (lo < hi)
2878             {
2879               mid = ((hi - lo) >> 1) + lo;
2880               local = pool_elt_at_index (m->locals, tmp[mid]);
2881               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2882             }
2883           local = pool_elt_at_index (m->locals, tmp[lo]);
2884           if (!(local->prefix >= rand))
2885             return 1;
2886           *mapping_addr = local->addr;
2887           *mapping_port = local->port;
2888           *mapping_fib_index = local->fib_index;
2889           if (m->affinity)
2890             {
2891               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2892                                                 match_protocol, match_port,
2893                                                 tmp[lo], m->affinity,
2894                                                 m->affinity_per_service_list_head_index))
2895                 nat_elog_info (sm, "create affinity record failed");
2896             }
2897           vec_free (tmp);
2898         }
2899       else
2900         {
2901           if (PREDICT_FALSE (lb != 0))
2902             *lb = NO_LB_NAT;
2903           *mapping_fib_index = m->fib_index;
2904           *mapping_addr = m->local_addr;
2905           /* Address only mapping doesn't change port */
2906           *mapping_port =
2907             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2908         }
2909     }
2910   else
2911     {
2912       *mapping_addr = m->external_addr;
2913       /* Address only mapping doesn't change port */
2914       *mapping_port =
2915         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2916       *mapping_fib_index = sm->outside_fib_index;
2917     }
2918
2919 end:
2920   if (PREDICT_FALSE (is_addr_only != 0))
2921     *is_addr_only = is_sm_addr_only (m->flags);
2922
2923   if (PREDICT_FALSE (twice_nat != 0))
2924     {
2925       *twice_nat = TWICE_NAT_DISABLED;
2926
2927       if (is_sm_twice_nat (m->flags))
2928         {
2929           *twice_nat = TWICE_NAT;
2930         }
2931       else if (is_sm_self_twice_nat (m->flags))
2932         {
2933           *twice_nat = TWICE_NAT_SELF;
2934         }
2935     }
2936
2937   if (PREDICT_FALSE (is_identity_nat != 0))
2938     *is_identity_nat = is_sm_identity_nat (m->flags);
2939
2940   if (out != 0)
2941     *out = m;
2942
2943   return 0;
2944 }
2945
2946 u32
2947 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2948                                   u32 rx_fib_index, u8 is_output)
2949 {
2950   snat_main_t *sm = &snat_main;
2951   u32 next_worker_index = sm->first_worker_index;
2952   u32 hash;
2953
2954   clib_bihash_kv_16_8_t kv16, value16;
2955
2956   u32 fib_index = rx_fib_index;
2957   if (b)
2958     {
2959       if (PREDICT_FALSE (is_output))
2960         {
2961           fib_index = sm->outside_fib_index;
2962           nat_fib_t *outside_fib;
2963           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2964           fib_prefix_t pfx = {
2965                   .fp_proto = FIB_PROTOCOL_IP4,
2966                   .fp_len = 32,
2967                   .fp_addr = {
2968                           .ip4.as_u32 = ip->dst_address.as_u32,
2969                   } ,
2970           };
2971
2972           switch (vec_len (sm->outside_fibs))
2973             {
2974             case 0:
2975               fib_index = sm->outside_fib_index;
2976               break;
2977             case 1:
2978               fib_index = sm->outside_fibs[0].fib_index;
2979               break;
2980             default:
2981               vec_foreach (outside_fib, sm->outside_fibs)
2982                 {
2983                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2984                   if (FIB_NODE_INDEX_INVALID != fei)
2985                     {
2986                       if (fib_entry_get_resolving_interface (fei) != ~0)
2987                         {
2988                           fib_index = outside_fib->fib_index;
2989                           break;
2990                         }
2991                     }
2992                 }
2993               break;
2994             }
2995         }
2996
2997       if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
2998         {
2999           ip4_address_t lookup_saddr, lookup_daddr;
3000           u16 lookup_sport, lookup_dport;
3001           u8 lookup_protocol;
3002
3003           if (!nat_get_icmp_session_lookup_values (
3004                 b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr,
3005                 &lookup_dport, &lookup_protocol))
3006             {
3007               init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3008                          lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3009                          lookup_protocol);
3010               if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3011                 {
3012                   next_worker_index = ed_value_get_thread_index (&value16);
3013                   vnet_buffer2 (b)->nat.cached_session_index =
3014                     ed_value_get_session_index (&value16);
3015                   goto out;
3016                 }
3017             }
3018         }
3019
3020       init_ed_k (&kv16, ip->src_address.as_u32,
3021                  vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3022                  vnet_buffer (b)->ip.reass.l4_dst_port, fib_index,
3023                  ip->protocol);
3024
3025       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3026         {
3027           next_worker_index = ed_value_get_thread_index (&value16);
3028           vnet_buffer2 (b)->nat.cached_session_index =
3029             ed_value_get_session_index (&value16);
3030           goto out;
3031         }
3032
3033       // dst NAT
3034       init_ed_k (&kv16, ip->dst_address.as_u32,
3035                  vnet_buffer (b)->ip.reass.l4_dst_port, ip->src_address.as_u32,
3036                  vnet_buffer (b)->ip.reass.l4_src_port, rx_fib_index,
3037                  ip->protocol);
3038       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
3039         {
3040           next_worker_index = ed_value_get_thread_index (&value16);
3041           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
3042             ed_value_get_session_index (&value16);
3043           goto out;
3044         }
3045     }
3046
3047   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3048          (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24) +
3049          rx_fib_index + (rx_fib_index >> 8) + (rx_fib_index >> 16) +
3050          (rx_fib_index >> 24);
3051
3052   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
3053     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
3054   else
3055     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
3056
3057 out:
3058   if (PREDICT_TRUE (!is_output))
3059     {
3060       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
3061                               rx_fib_index,
3062                               clib_net_to_host_u32 (ip->src_address.as_u32),
3063                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3064     }
3065   else
3066     {
3067       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
3068                               next_worker_index, rx_fib_index,
3069                               clib_net_to_host_u32 (ip->src_address.as_u32),
3070                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3071     }
3072
3073   return next_worker_index;
3074 }
3075
3076 u32
3077 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
3078                                   u32 rx_fib_index, u8 is_output)
3079 {
3080   snat_main_t *sm = &snat_main;
3081   clib_bihash_kv_16_8_t kv16, value16;
3082
3083   u8 proto, next_worker_index = 0;
3084   u16 port;
3085   snat_static_mapping_t *m;
3086   u32 hash;
3087
3088   proto = ip->protocol;
3089
3090   if (PREDICT_FALSE (IP_PROTOCOL_ICMP == proto))
3091     {
3092       ip4_address_t lookup_saddr, lookup_daddr;
3093       u16 lookup_sport, lookup_dport;
3094       u8 lookup_protocol;
3095       if (!nat_get_icmp_session_lookup_values (
3096             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
3097             &lookup_protocol))
3098         {
3099           init_ed_k (&kv16, lookup_saddr.as_u32, lookup_sport,
3100                      lookup_daddr.as_u32, lookup_dport, rx_fib_index,
3101                      lookup_protocol);
3102           if (PREDICT_TRUE (
3103                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3104             {
3105               next_worker_index = ed_value_get_thread_index (&value16);
3106               nat_elog_debug_handoff (
3107                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
3108                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
3109                 clib_net_to_host_u32 (ip->dst_address.as_u32));
3110               return next_worker_index;
3111             }
3112         }
3113     }
3114
3115   init_ed_k (&kv16, ip->src_address.as_u32,
3116              vnet_buffer (b)->ip.reass.l4_src_port, ip->dst_address.as_u32,
3117              vnet_buffer (b)->ip.reass.l4_dst_port, rx_fib_index,
3118              ip->protocol);
3119
3120   if (PREDICT_TRUE (
3121         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
3122     {
3123       vnet_buffer2 (b)->nat.cached_session_index =
3124         ed_value_get_session_index (&value16);
3125       next_worker_index = ed_value_get_thread_index (&value16);
3126       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
3127                               next_worker_index, rx_fib_index,
3128                               clib_net_to_host_u32 (ip->src_address.as_u32),
3129                               clib_net_to_host_u32 (ip->dst_address.as_u32));
3130       return next_worker_index;
3131     }
3132
3133   /* first try static mappings without port */
3134   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3135     {
3136       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, 0, 0, proto);
3137       if (m)
3138         {
3139           {
3140             next_worker_index = m->workers[0];
3141             goto done;
3142           }
3143         }
3144     }
3145
3146   /* unknown protocol */
3147   if (PREDICT_FALSE (nat44_ed_is_unk_proto (proto)))
3148     {
3149       /* use current thread */
3150       next_worker_index = vlib_get_thread_index ();
3151       goto done;
3152     }
3153
3154   port = vnet_buffer (b)->ip.reass.l4_dst_port;
3155
3156   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3157     {
3158       udp_header_t *udp = ip4_next_header (ip);
3159       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3160       nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3161       if (!icmp_type_is_error_message
3162           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3163         port = vnet_buffer (b)->ip.reass.l4_src_port;
3164       else
3165         {
3166           /* if error message, then it's not fragmented and we can access it */
3167           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3168           proto = inner_ip->protocol;
3169           void *l4_header = ip4_next_header (inner_ip);
3170           switch (proto)
3171             {
3172             case IP_PROTOCOL_ICMP:
3173               icmp = (icmp46_header_t *) l4_header;
3174               echo = (nat_icmp_echo_header_t *) (icmp + 1);
3175               port = echo->identifier;
3176               break;
3177             case IP_PROTOCOL_UDP:
3178               /* breakthrough */
3179             case IP_PROTOCOL_TCP:
3180               port = ((nat_tcp_udp_header_t *) l4_header)->src_port;
3181               break;
3182             default:
3183               next_worker_index = vlib_get_thread_index ();
3184               goto done;
3185             }
3186         }
3187     }
3188
3189   /* try static mappings with port */
3190   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3191     {
3192       m = nat44_ed_sm_o2i_lookup (sm, ip->dst_address, port, 0, proto);
3193       if (m)
3194         {
3195           if (!is_sm_lb (m->flags))
3196             {
3197               next_worker_index = m->workers[0];
3198               goto done;
3199             }
3200
3201           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3202             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3203
3204           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3205             next_worker_index =
3206               m->workers[hash & (_vec_len (m->workers) - 1)];
3207           else
3208             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3209           goto done;
3210         }
3211     }
3212
3213   /* worker by outside port */
3214   next_worker_index = get_thread_idx_by_port (clib_net_to_host_u16 (port));
3215
3216 done:
3217   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3218                           rx_fib_index,
3219                           clib_net_to_host_u32 (ip->src_address.as_u32),
3220                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3221   return next_worker_index;
3222 }
3223
3224 u32
3225 nat44_get_max_session_limit ()
3226 {
3227   snat_main_t *sm = &snat_main;
3228   u32 max_limit = 0, len = 0;
3229
3230   for (; len < vec_len (sm->max_translations_per_fib); len++)
3231     {
3232       if (max_limit < sm->max_translations_per_fib[len])
3233         max_limit = sm->max_translations_per_fib[len];
3234     }
3235   return max_limit;
3236 }
3237
3238 int
3239 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3240 {
3241   snat_main_t *sm = &snat_main;
3242   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3243   u32 len = vec_len (sm->max_translations_per_fib);
3244
3245   if (len <= fib_index)
3246     {
3247       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3248
3249       for (; len < vec_len (sm->max_translations_per_fib); len++)
3250         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3251     }
3252
3253   sm->max_translations_per_fib[fib_index] = session_limit;
3254   return 0;
3255 }
3256
3257 int
3258 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3259 {
3260   snat_main_t *sm = &snat_main;
3261
3262   if (nat44_set_session_limit (session_limit, vrf_id))
3263     return 1;
3264   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3265
3266   vlib_stats_set_gauge (sm->max_cfg_sessions_gauge,
3267                         sm->max_translations_per_thread);
3268
3269   sm->translation_buckets =
3270     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3271
3272   nat44_ed_sessions_clear ();
3273   return 0;
3274 }
3275
3276 static void
3277 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations)
3278 {
3279   dlist_elt_t *head;
3280
3281   pool_alloc (tsm->per_vrf_sessions_pool, translations);
3282   pool_alloc (tsm->sessions, translations);
3283   pool_alloc (tsm->lru_pool, translations);
3284
3285   pool_get (tsm->lru_pool, head);
3286   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3287   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3288
3289   pool_get (tsm->lru_pool, head);
3290   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3291   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3292
3293   pool_get (tsm->lru_pool, head);
3294   tsm->udp_lru_head_index = head - tsm->lru_pool;
3295   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3296
3297   pool_get (tsm->lru_pool, head);
3298   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3299   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3300
3301   pool_get (tsm->lru_pool, head);
3302   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3303   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3304 }
3305
3306 static void
3307 nat44_ed_flow_hash_init ()
3308 {
3309   snat_main_t *sm = &snat_main;
3310   // we expect 2 flows per session, so multiply translation_buckets by 2
3311   clib_bihash_init_16_8 (
3312     &sm->flow_hash, "ed-flow-hash",
3313     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3314   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3315 }
3316
3317 static void
3318 nat44_ed_db_init ()
3319 {
3320   snat_main_t *sm = &snat_main;
3321   snat_main_per_thread_data_t *tsm;
3322
3323   nat44_ed_flow_hash_init ();
3324
3325   vec_foreach (tsm, sm->per_thread_data)
3326     {
3327       nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread);
3328     }
3329 }
3330
3331 static void
3332 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3333 {
3334   pool_free (tsm->lru_pool);
3335   pool_free (tsm->sessions);
3336   pool_free (tsm->per_vrf_sessions_pool);
3337 }
3338
3339 static void
3340 nat44_ed_flow_hash_free ()
3341 {
3342   snat_main_t *sm = &snat_main;
3343
3344   clib_bihash_free_16_8 (&sm->flow_hash);
3345 }
3346
3347 static void
3348 nat44_ed_db_free ()
3349 {
3350   snat_main_t *sm = &snat_main;
3351   snat_main_per_thread_data_t *tsm;
3352
3353   vec_foreach (tsm, sm->per_thread_data)
3354     {
3355       nat44_ed_worker_db_free (tsm);
3356     }
3357
3358   nat44_ed_flow_hash_free ();
3359 }
3360
3361 void
3362 nat44_ed_sessions_clear ()
3363 {
3364   snat_main_t *sm = &snat_main;
3365
3366   nat44_ed_db_free ();
3367   nat44_ed_db_init ();
3368   vlib_zero_simple_counter (&sm->total_sessions, 0);
3369 }
3370
3371 static void
3372 nat44_ed_add_del_static_mapping_cb (ip4_main_t *im, uword opaque,
3373                                     u32 sw_if_index, ip4_address_t *address,
3374                                     u32 address_length, u32 if_address_index,
3375                                     u32 is_delete)
3376 {
3377   snat_static_mapping_resolve_t *rp;
3378   snat_main_t *sm = &snat_main;
3379   int rv = 0;
3380
3381   if (!sm->enabled)
3382     {
3383       return;
3384     }
3385
3386   vec_foreach (rp, sm->sm_to_resolve)
3387     {
3388       if (sw_if_index == rp->sw_if_index)
3389         {
3390           if (is_delete)
3391             {
3392               if (rp->is_resolved)
3393                 {
3394                   rv = nat44_ed_del_static_mapping_internal (
3395                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3396                     rp->vrf_id, rp->flags);
3397                   if (rv)
3398                     {
3399                       nat_log_err ("ed del static mapping failed");
3400                     }
3401                   else
3402                     {
3403                       rp->is_resolved = 0;
3404                     }
3405                 }
3406             }
3407           else
3408             {
3409               if (!rp->is_resolved)
3410                 {
3411                   rv = nat44_ed_add_static_mapping_internal (
3412                     rp->l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3413                     rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3414                   if (rv)
3415                     {
3416                       nat_log_err ("ed add static mapping failed");
3417                     }
3418                   else
3419                     {
3420                       rp->is_resolved = 1;
3421                     }
3422                 }
3423             }
3424         }
3425     }
3426 }
3427
3428 static int
3429 nat44_ed_get_addr_resolve_record (u32 sw_if_index, u8 twice_nat, int *out)
3430 {
3431   snat_main_t *sm = &snat_main;
3432   snat_address_resolve_t *rp;
3433   int i;
3434
3435   for (i = 0; i < vec_len (sm->addr_to_resolve); i++)
3436     {
3437       rp = sm->addr_to_resolve + i;
3438
3439       if ((rp->sw_if_index == sw_if_index) && (rp->is_twice_nat == twice_nat))
3440         {
3441           if (out)
3442             {
3443               *out = i;
3444             }
3445           return 0;
3446         }
3447     }
3448   return 1;
3449 }
3450 static int
3451 nat44_ed_del_addr_resolve_record (u32 sw_if_index, u8 twice_nat)
3452 {
3453   snat_main_t *sm = &snat_main;
3454   int i;
3455   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3456     {
3457       vec_del1 (sm->addr_to_resolve, i);
3458       return 0;
3459     }
3460   return 1;
3461 }
3462
3463 static void
3464 nat44_ed_add_del_interface_address_cb (ip4_main_t *im, uword opaque,
3465                                        u32 sw_if_index, ip4_address_t *address,
3466                                        u32 address_length,
3467                                        u32 if_address_index, u32 is_delete)
3468 {
3469   snat_main_t *sm = &snat_main;
3470   snat_address_resolve_t *arp;
3471   snat_address_t *ap;
3472   u8 twice_nat = 0;
3473   int i, rv;
3474
3475   if (!sm->enabled)
3476     {
3477       return;
3478     }
3479
3480   if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3481     {
3482       twice_nat = 1;
3483       if (nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, &i))
3484         {
3485           u32 fib_index =
3486             ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
3487           vec_foreach (ap, sm->addresses)
3488             {
3489               if ((fib_index == ap->fib_index) &&
3490                   (address->as_u32 == ap->addr.as_u32))
3491                 {
3492                   if (!is_delete)
3493                     {
3494                       ap->addr_len = address_length;
3495                       ap->sw_if_index = sw_if_index;
3496                       ap->net.as_u32 =
3497                         ap->addr.as_u32 & ip4_main.fib_masks[ap->addr_len];
3498
3499                       nat_log_debug (
3500                         "pool addr %U binds to -> sw_if_idx: %u net: %U/%u",
3501                         format_ip4_address, &ap->addr, ap->sw_if_index,
3502                         format_ip4_address, &ap->net, ap->addr_len);
3503                     }
3504                   else
3505                     {
3506                       ap->addr_len = ~0;
3507                     }
3508                   break;
3509                 }
3510             }
3511           return;
3512         }
3513     }
3514
3515   arp = sm->addr_to_resolve + i;
3516
3517   if (!is_delete)
3518     {
3519       if (arp->is_resolved)
3520         {
3521           return;
3522         }
3523
3524       rv = nat44_ed_add_address (address, ~0, arp->is_twice_nat);
3525       if (0 == rv)
3526         {
3527           arp->is_resolved = 1;
3528         }
3529     }
3530   else
3531     {
3532       if (!arp->is_resolved)
3533         {
3534           return;
3535         }
3536
3537       rv = nat44_ed_del_address (address[0], arp->is_twice_nat);
3538       if (0 == rv)
3539         {
3540           arp->is_resolved = 0;
3541         }
3542     }
3543 }
3544
3545 int
3546 nat44_ed_add_interface_address (u32 sw_if_index, u8 twice_nat)
3547 {
3548   snat_main_t *sm = &snat_main;
3549   ip4_main_t *ip4_main = sm->ip4_main;
3550   ip4_address_t *first_int_addr;
3551   snat_address_resolve_t *ap;
3552   int rv;
3553
3554   if (!sm->enabled)
3555     {
3556       nat_log_err ("nat44 is disabled");
3557       return VNET_API_ERROR_UNSUPPORTED;
3558     }
3559
3560   if (!nat44_ed_get_addr_resolve_record (sw_if_index, twice_nat, 0))
3561     {
3562       return VNET_API_ERROR_VALUE_EXIST;
3563     }
3564
3565   vec_add2 (sm->addr_to_resolve, ap, 1);
3566   ap->sw_if_index = sw_if_index;
3567   ap->is_twice_nat = twice_nat;
3568   ap->is_resolved = 0;
3569
3570   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3571   if (first_int_addr)
3572     {
3573       rv = nat44_ed_add_address (first_int_addr, ~0, twice_nat);
3574       if (0 != rv)
3575         {
3576           nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat);
3577           return rv;
3578         }
3579       ap->is_resolved = 1;
3580     }
3581
3582   return 0;
3583 }
3584
3585 int
3586 nat44_ed_del_interface_address (u32 sw_if_index, u8 twice_nat)
3587 {
3588   snat_main_t *sm = &snat_main;
3589   ip4_main_t *ip4_main = sm->ip4_main;
3590   ip4_address_t *first_int_addr;
3591
3592   if (!sm->enabled)
3593     {
3594       nat_log_err ("nat44 is disabled");
3595       return VNET_API_ERROR_UNSUPPORTED;
3596     }
3597
3598   if (nat44_ed_del_addr_resolve_record (sw_if_index, twice_nat))
3599     {
3600       return VNET_API_ERROR_NO_SUCH_ENTRY;
3601     }
3602
3603   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3604   if (first_int_addr)
3605     {
3606       return nat44_ed_del_address (first_int_addr[0], twice_nat);
3607     }
3608
3609   return 0;
3610 }
3611
3612 int
3613 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3614                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3615                       u32 vrf_id, int is_in)
3616 {
3617   ip4_header_t ip;
3618   clib_bihash_kv_16_8_t kv, value;
3619   u32 fib_index;
3620   snat_session_t *s;
3621   snat_main_per_thread_data_t *tsm;
3622
3623   if (!sm->enabled)
3624     {
3625       return VNET_API_ERROR_UNSUPPORTED;
3626     }
3627
3628   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3629   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3630   if (sm->num_workers > 1)
3631     tsm = vec_elt_at_index (
3632       sm->per_thread_data,
3633       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3634   else
3635     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3636
3637   init_ed_k (&kv, addr->as_u32, port, eh_addr->as_u32, eh_port, fib_index,
3638              proto);
3639   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3640     {
3641       return VNET_API_ERROR_NO_SUCH_ENTRY;
3642     }
3643
3644   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3645     return VNET_API_ERROR_UNSPECIFIED;
3646   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3647   nat44_ed_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3648   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3649   return 0;
3650 }
3651
3652 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3653                                  vlib_node_runtime_t * node,
3654                                  vlib_frame_t * frame)
3655 {
3656   return 0;
3657 }
3658
3659 VLIB_REGISTER_NODE (nat_default_node) = {
3660   .name = "nat-default",
3661   .vector_size = sizeof (u32),
3662   .format_trace = 0,
3663   .type = VLIB_NODE_TYPE_INTERNAL,
3664   .n_errors = 0,
3665   .n_next_nodes = NAT_N_NEXT,
3666   .next_nodes = {
3667     [NAT_NEXT_DROP] = "error-drop",
3668     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3669     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3670     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3671     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3672     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3673     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3674     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3675     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3676     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3677   },
3678 };
3679
3680 void
3681 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3682 {
3683   f->l3_csum_delta = 0;
3684   f->l4_csum_delta = 0;
3685   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3686       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3687     {
3688       f->l3_csum_delta =
3689         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3690       f->l3_csum_delta =
3691         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3692     }
3693   else
3694     {
3695       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3696     }
3697   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3698       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3699     {
3700       f->l3_csum_delta =
3701         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3702       f->l3_csum_delta =
3703         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3704     }
3705   else
3706     {
3707       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3708     }
3709   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3710     {
3711       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3712       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3713     }
3714   else
3715     {
3716       f->rewrite.sport = f->match.sport;
3717     }
3718   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3719     {
3720       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3721       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3722     }
3723   else
3724     {
3725       f->rewrite.dport = f->match.dport;
3726     }
3727   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3728       f->rewrite.icmp_id != f->match.sport)
3729     {
3730       f->l4_csum_delta =
3731         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3732       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3733     }
3734   else
3735     {
3736       f->rewrite.icmp_id = f->match.sport;
3737     }
3738   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3739     {
3740     }
3741   else
3742     {
3743       f->rewrite.fib_index = f->match.fib_index;
3744     }
3745 }
3746
3747 static_always_inline int
3748 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3749                             ip4_header_t *ip, nat_6t_flow_t *f);
3750
3751 static_always_inline void
3752 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3753                            nat_6t_flow_t *f, ip_protocol_t proto,
3754                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3755 {
3756   udp_header_t *udp = ip4_next_header (ip);
3757   tcp_header_t *tcp = (tcp_header_t *) udp;
3758
3759   if ((IP_PROTOCOL_TCP == proto || IP_PROTOCOL_UDP == proto) &&
3760       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3761     {
3762       if (!is_icmp_inner_ip4)
3763         { // regular case
3764           ip->src_address = f->rewrite.saddr;
3765           ip->dst_address = f->rewrite.daddr;
3766           udp->src_port = f->rewrite.sport;
3767           udp->dst_port = f->rewrite.dport;
3768         }
3769       else
3770         { // icmp inner ip4 - reversed saddr/daddr
3771           ip->src_address = f->rewrite.daddr;
3772           ip->dst_address = f->rewrite.saddr;
3773           udp->src_port = f->rewrite.dport;
3774           udp->dst_port = f->rewrite.sport;
3775         }
3776
3777       if (IP_PROTOCOL_TCP == proto)
3778         {
3779           ip_csum_t tcp_sum = tcp->checksum;
3780           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3781           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3782           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3783           tcp->checksum = ip_csum_fold (tcp_sum);
3784         }
3785       else if (IP_PROTOCOL_UDP == proto && udp->checksum)
3786         {
3787           ip_csum_t udp_sum = udp->checksum;
3788           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3789           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3790           udp->checksum = ip_csum_fold (udp_sum);
3791         }
3792     }
3793   else
3794     {
3795       if (!is_icmp_inner_ip4)
3796         { // regular case
3797           if (!skip_saddr_rewrite)
3798             {
3799               ip->src_address = f->rewrite.saddr;
3800             }
3801           ip->dst_address = f->rewrite.daddr;
3802         }
3803       else
3804         { // icmp inner ip4 - reversed saddr/daddr
3805           ip->src_address = f->rewrite.daddr;
3806           ip->dst_address = f->rewrite.saddr;
3807         }
3808     }
3809
3810   if (skip_saddr_rewrite)
3811     {
3812       ip->checksum = ip4_header_checksum (ip);
3813     }
3814   else
3815     {
3816       ip_csum_t ip_sum = ip->checksum;
3817       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3818       ip->checksum = ip_csum_fold (ip_sum);
3819     }
3820   if (0xffff == ip->checksum)
3821     ip->checksum = 0;
3822   ASSERT (ip4_header_checksum_is_valid (ip));
3823 }
3824
3825 static_always_inline int
3826 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3827 {
3828   int result = ((u8 *) object + size <=
3829                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3830                vlib_object_within_buffer_data (vm, b, object, size);
3831   return result;
3832 }
3833
3834 static_always_inline int
3835 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3836                             ip4_header_t *ip, nat_6t_flow_t *f)
3837 {
3838   if (IP_PROTOCOL_ICMP != ip->protocol)
3839     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3840
3841   icmp46_header_t *icmp = ip4_next_header (ip);
3842   nat_icmp_echo_header_t *echo = (nat_icmp_echo_header_t *) (icmp + 1);
3843
3844   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3845     {
3846       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3847         {
3848           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3849         }
3850
3851       if (!icmp_type_is_error_message (icmp->type))
3852         {
3853           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3854               (f->rewrite.icmp_id != echo->identifier))
3855             {
3856               ip_csum_t sum = icmp->checksum;
3857               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3858                                     nat_icmp_echo_header_t,
3859                                     identifier /* changed member */);
3860               echo->identifier = f->rewrite.icmp_id;
3861               icmp->checksum = ip_csum_fold (sum);
3862             }
3863         }
3864       else
3865         {
3866           ip_csum_t sum = ip_incremental_checksum (
3867             0, icmp,
3868             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3869           sum = (u16) ~ip_csum_fold (sum);
3870           if (sum != 0)
3871             {
3872               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3873             }
3874
3875           // errors are not fragmented
3876           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3877
3878           if (!ip4_header_checksum_is_valid (inner_ip))
3879             {
3880               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3881             }
3882
3883           ip_protocol_t inner_proto = inner_ip->protocol;
3884
3885           ip_csum_t old_icmp_sum = icmp->checksum;
3886           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3887           ip_csum_t old_udp_sum;
3888           ip_csum_t old_tcp_sum;
3889           ip_csum_t new_icmp_sum;
3890           udp_header_t *udp;
3891           tcp_header_t *tcp;
3892
3893           switch (inner_proto)
3894             {
3895             case IP_PROTOCOL_UDP:
3896               udp = (udp_header_t *) (inner_ip + 1);
3897               if (!it_fits (vm, b, udp, sizeof (*udp)))
3898                 {
3899                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3900                 }
3901               old_udp_sum = udp->checksum;
3902               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3903                                          1 /* is_icmp_inner_ip4 */,
3904                                          0 /* skip_saddr_rewrite */);
3905               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3906               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3907               new_icmp_sum =
3908                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3909                                 inner_ip->checksum, ip4_header_t, checksum);
3910               new_icmp_sum =
3911                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3912                                 udp_header_t, checksum);
3913               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3914               icmp->checksum = new_icmp_sum;
3915               break;
3916             case IP_PROTOCOL_TCP:
3917               tcp = (tcp_header_t *) (inner_ip + 1);
3918               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3919                 {
3920                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3921                 }
3922               old_tcp_sum = tcp->checksum;
3923               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3924                                          1 /* is_icmp_inner_ip4 */,
3925                                          0 /* skip_saddr_rewrite */);
3926               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3927               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3928               new_icmp_sum =
3929                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3930                                 inner_ip->checksum, ip4_header_t, checksum);
3931               new_icmp_sum =
3932                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3933                                 tcp_header_t, checksum);
3934               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3935               icmp->checksum = new_icmp_sum;
3936               break;
3937             case IP_PROTOCOL_ICMP:
3938               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3939                                          1 /* is_icmp_inner_ip4 */,
3940                                          0 /* skip_saddr_rewrite */);
3941               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3942                 {
3943                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3944                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3945                     {
3946                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3947                     }
3948                   nat_icmp_echo_header_t *inner_echo =
3949                     (nat_icmp_echo_header_t *) (inner_icmp + 1);
3950                   if (f->rewrite.icmp_id != inner_echo->identifier)
3951                     {
3952                       ip_csum_t sum = icmp->checksum;
3953                       sum = ip_csum_update (sum, inner_echo->identifier,
3954                                             f->rewrite.icmp_id,
3955                                             nat_icmp_echo_header_t,
3956                                             identifier /* changed member */);
3957                       icmp->checksum = ip_csum_fold (sum);
3958                       ip_csum_t inner_sum = inner_icmp->checksum;
3959                       inner_sum = ip_csum_update (
3960                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3961                         nat_icmp_echo_header_t,
3962                         identifier /* changed member */);
3963                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3964                       inner_echo->identifier = f->rewrite.icmp_id;
3965                     }
3966                 }
3967               break;
3968             default:
3969               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3970               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3971             }
3972         }
3973     }
3974
3975   return NAT_ED_TRNSL_ERR_SUCCESS;
3976 }
3977
3978 static_always_inline nat_translation_error_e
3979 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3980                            ip4_header_t *ip, nat_6t_flow_t *f,
3981                            ip_protocol_t proto, int is_output_feature,
3982                            int is_i2o)
3983 {
3984   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3985     {
3986       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3987     }
3988
3989   if (IP_PROTOCOL_ICMP == proto)
3990     {
3991       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3992         {
3993           // packet is returned from a router, not from destination
3994           // skip source address rewrite if in o2i path
3995           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3996                                      0 /* is_icmp_inner_ip4 */,
3997                                      !is_i2o /* skip_saddr_rewrite */);
3998         }
3999       else
4000         {
4001           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
4002                                      0 /* is_icmp_inner_ip4 */,
4003                                      0 /* skip_saddr_rewrite */);
4004         }
4005       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
4006     }
4007
4008   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
4009                              0 /* skip_saddr_rewrite */);
4010
4011   return NAT_ED_TRNSL_ERR_SUCCESS;
4012 }
4013
4014 nat_translation_error_e
4015 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
4016                                vlib_buffer_t *b, ip4_header_t *ip,
4017                                nat_6t_flow_t *f, ip_protocol_t proto,
4018                                int is_output_feature)
4019 {
4020   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
4021                                     1 /* is_i2o */);
4022 }
4023
4024 nat_translation_error_e
4025 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
4026                                vlib_buffer_t *b, ip4_header_t *ip,
4027                                nat_6t_flow_t *f, ip_protocol_t proto,
4028                                int is_output_feature)
4029 {
4030   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
4031                                     0 /* is_i2o */);
4032 }
4033
4034 static_always_inline void
4035 nat_syslog_nat44_sess (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4036                        u16 isport, ip4_address_t *xsaddr, u16 xsport,
4037                        ip4_address_t *idaddr, u16 idport,
4038                        ip4_address_t *xdaddr, u16 xdport, u8 proto, u8 is_add,
4039                        u8 is_twicenat)
4040 {
4041   syslog_msg_t syslog_msg;
4042   fib_table_t *fib;
4043
4044   if (!syslog_is_enabled ())
4045     return;
4046
4047   if (syslog_severity_filter_block (SADD_SDEL_SEVERITY))
4048     return;
4049
4050   fib = fib_table_get (sfibix, FIB_PROTOCOL_IP4);
4051
4052   syslog_msg_init (&syslog_msg, NAT_FACILITY, SADD_SDEL_SEVERITY, NAT_APPNAME,
4053                    is_add ? SADD_MSGID : SDEL_MSGID);
4054
4055   syslog_msg_sd_init (&syslog_msg, NSESS_SDID);
4056   syslog_msg_add_sd_param (&syslog_msg, SSUBIX_SDPARAM_NAME, "%d", ssubix);
4057   syslog_msg_add_sd_param (&syslog_msg, SVLAN_SDPARAM_NAME, "%d",
4058                            fib->ft_table_id);
4059   syslog_msg_add_sd_param (&syslog_msg, IATYP_SDPARAM_NAME, IATYP_IPV4);
4060   syslog_msg_add_sd_param (&syslog_msg, ISADDR_SDPARAM_NAME, "%U",
4061                            format_ip4_address, isaddr);
4062   syslog_msg_add_sd_param (&syslog_msg, ISPORT_SDPARAM_NAME, "%d",
4063                            clib_net_to_host_u16 (isport));
4064   syslog_msg_add_sd_param (&syslog_msg, XATYP_SDPARAM_NAME, IATYP_IPV4);
4065   syslog_msg_add_sd_param (&syslog_msg, XSADDR_SDPARAM_NAME, "%U",
4066                            format_ip4_address, xsaddr);
4067   syslog_msg_add_sd_param (&syslog_msg, XSPORT_SDPARAM_NAME, "%d",
4068                            clib_net_to_host_u16 (xsport));
4069   syslog_msg_add_sd_param (&syslog_msg, PROTO_SDPARAM_NAME, "%d", proto);
4070   syslog_msg_add_sd_param (&syslog_msg, XDADDR_SDPARAM_NAME, "%U",
4071                            format_ip4_address, xdaddr);
4072   syslog_msg_add_sd_param (&syslog_msg, XDPORT_SDPARAM_NAME, "%d",
4073                            clib_net_to_host_u16 (xdport));
4074   if (is_twicenat)
4075     {
4076       syslog_msg_add_sd_param (&syslog_msg, IDADDR_SDPARAM_NAME, "%U",
4077                                format_ip4_address, idaddr);
4078       syslog_msg_add_sd_param (&syslog_msg, IDPORT_SDPARAM_NAME, "%d",
4079                                clib_net_to_host_u16 (idport));
4080     }
4081
4082   syslog_msg_send (&syslog_msg);
4083 }
4084
4085 void
4086 nat_syslog_nat44_sadd (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4087                        u16 isport, ip4_address_t *idaddr, u16 idport,
4088                        ip4_address_t *xsaddr, u16 xsport,
4089                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4090                        u8 is_twicenat)
4091 {
4092   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4093                          idaddr, idport, xdaddr, xdport, proto, 1,
4094                          is_twicenat);
4095 }
4096
4097 void
4098 nat_syslog_nat44_sdel (u32 ssubix, u32 sfibix, ip4_address_t *isaddr,
4099                        u16 isport, ip4_address_t *idaddr, u16 idport,
4100                        ip4_address_t *xsaddr, u16 xsport,
4101                        ip4_address_t *xdaddr, u16 xdport, u8 proto,
4102                        u8 is_twicenat)
4103 {
4104   nat_syslog_nat44_sess (ssubix, sfibix, isaddr, isport, xsaddr, xsport,
4105                          idaddr, idport, xdaddr, xdport, proto, 0,
4106                          is_twicenat);
4107 }
4108 __clib_export void
4109 nat44_original_dst_lookup (ip4_address_t *i2o_src, u16 i2o_src_port,
4110                            ip4_address_t *i2o_dst, u16 i2o_dst_port,
4111                            ip_protocol_t proto, u32 *original_dst,
4112                            u16 *original_dst_port)
4113 {
4114   snat_main_per_thread_data_t *tsm;
4115   snat_main_t *sm = &snat_main;
4116   u32 fib_index = 0;
4117   snat_session_t *s;
4118   ip4_header_t ip;
4119
4120   ip.src_address.as_u32 = i2o_src->as_u32;
4121   fib_index = fib_table_find (FIB_PROTOCOL_IP4, 0);
4122
4123   if (sm->num_workers > 1)
4124     {
4125       tsm = vec_elt_at_index (
4126         sm->per_thread_data,
4127         nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
4128     }
4129   else
4130     {
4131       tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
4132     }
4133
4134   /* query */
4135   clib_bihash_kv_16_8_t kv = { 0 }, value;
4136   init_ed_k (&kv, i2o_src->as_u32, i2o_src_port, i2o_dst->as_u32, i2o_dst_port,
4137              fib_index, proto);
4138   if (tsm->sessions == NULL ||
4139       clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
4140     {
4141       return;
4142     }
4143   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
4144   if (s)
4145     {
4146       *original_dst = s->i2o.rewrite.saddr.as_u32;
4147       *original_dst_port = s->i2o.rewrite.sport;
4148     }
4149   return;
4150 }
4151 /*
4152  * fd.io coding-style-patch-verification: ON
4153  *
4154  * Local Variables:
4155  * eval: (c-set-style "gnu")
4156  * End:
4157  */