src/plugins/nat/nat44-ed/nat44_ed.c

   1 /*
   2  * snat.c - simple nat plugin
   3  *
   4  * Copyright (c) 2016 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vpp/app/version.h>
  19
  20 #include <vnet/vnet.h>
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ip/ip4.h>
  23 #include <vnet/ip/ip_table.h>
  24 #include <vnet/ip/reass/ip4_sv_reass.h>
  25 #include <vnet/fib/fib_table.h>
  26 #include <vnet/fib/ip4_fib.h>
  27 #include <vnet/plugin/plugin.h>
  28 #include <vppinfra/bihash_16_8.h>
  29
  30 #include <nat/lib/log.h>
  31 #include <nat/lib/nat_syslog.h>
  32 #include <nat/lib/nat_inlines.h>
  33 #include <nat/lib/ipfix_logging.h>
  34
  35 #include <nat/nat44-ed/nat44_ed.h>
  36 #include <nat/nat44-ed/nat44_ed_affinity.h>
  37 #include <nat/nat44-ed/nat44_ed_inlines.h>
  38
  39 #include <vpp/stats/stat_segment.h>
  40
  41 snat_main_t snat_main;
  42
  43 static_always_inline void nat_validate_interface_counters (snat_main_t *sm,
  44                                                            u32 sw_if_index);
  45
  46 #define skip_if_disabled()                                                    \
  47   do                                                                          \
  48     {                                                                         \
  49       snat_main_t *sm = &snat_main;                                           \
  50       if (PREDICT_FALSE (!sm->enabled))                                       \
  51         return;                                                               \
  52     }                                                                         \
  53   while (0)
  54
  55 #define fail_if_enabled()                                                     \
  56   do                                                                          \
  57     {                                                                         \
  58       snat_main_t *sm = &snat_main;                                           \
  59       if (PREDICT_FALSE (sm->enabled))                                        \
  60         {                                                                     \
  61           nat_log_err ("plugin enabled");                                     \
  62           return 1;                                                           \
  63         }                                                                     \
  64     }                                                                         \
  65   while (0)
  66
  67 #define fail_if_disabled()                                                    \
  68   do                                                                          \
  69     {                                                                         \
  70       snat_main_t *sm = &snat_main;                                           \
  71       if (PREDICT_FALSE (!sm->enabled))                                       \
  72         {                                                                     \
  73           nat_log_err ("plugin disabled");                                    \
  74           return 1;                                                           \
  75         }                                                                     \
  76     }                                                                         \
  77   while (0)
  78
  79 /* Hook up input features */
  80 VNET_FEATURE_INIT (nat_pre_in2out, static) = {
  81   .arc_name = "ip4-unicast",
  82   .node_name = "nat-pre-in2out",
  83   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  84                                "ip4-sv-reassembly-feature"),
  85 };
  86 VNET_FEATURE_INIT (nat_pre_out2in, static) = {
  87   .arc_name = "ip4-unicast",
  88   .node_name = "nat-pre-out2in",
  89   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
  90                                "ip4-dhcp-client-detect",
  91                                "ip4-sv-reassembly-feature"),
  92 };
  93 VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
  94   .arc_name = "ip4-unicast",
  95   .node_name = "nat44-in2out-worker-handoff",
  96   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
  97 };
  98 VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
  99   .arc_name = "ip4-unicast",
 100   .node_name = "nat44-out2in-worker-handoff",
 101   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
 102                                "ip4-dhcp-client-detect"),
 103 };
 104 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
 105   .arc_name = "ip4-unicast",
 106   .node_name = "nat44-in2out",
 107   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 108 };
 109 VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
 110   .arc_name = "ip4-unicast",
 111   .node_name = "nat44-out2in",
 112   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 113                                "ip4-dhcp-client-detect"),
 114 };
 115 VNET_FEATURE_INIT (ip4_nat44_ed_in2out, static) = {
 116   .arc_name = "ip4-unicast",
 117   .node_name = "nat44-ed-in2out",
 118   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 119 };
 120 VNET_FEATURE_INIT (ip4_nat44_ed_out2in, static) = {
 121   .arc_name = "ip4-unicast",
 122   .node_name = "nat44-ed-out2in",
 123   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 124                                "ip4-dhcp-client-detect"),
 125 };
 126 VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
 127   .arc_name = "ip4-unicast",
 128   .node_name = "nat44-ed-classify",
 129   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 130 };
 131 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
 132   .arc_name = "ip4-unicast",
 133   .node_name = "nat44-handoff-classify",
 134   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 135 };
 136 VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
 137   .arc_name = "ip4-unicast",
 138   .node_name = "nat44-in2out-fast",
 139   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 140 };
 141 VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
 142   .arc_name = "ip4-unicast",
 143   .node_name = "nat44-out2in-fast",
 144   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature",
 145                                "ip4-dhcp-client-detect"),
 146 };
 147
 148 /* Hook up output features */
 149 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
 150   .arc_name = "ip4-output",
 151   .node_name = "nat44-in2out-output",
 152   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 153   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 154 };
 155 VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
 156   .arc_name = "ip4-output",
 157   .node_name = "nat44-in2out-output-worker-handoff",
 158   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 159   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 160 };
 161 VNET_FEATURE_INIT (nat_pre_in2out_output, static) = {
 162   .arc_name = "ip4-output",
 163   .node_name = "nat-pre-in2out-output",
 164   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 165   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 166 };
 167 VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
 168   .arc_name = "ip4-output",
 169   .node_name = "nat44-ed-in2out-output",
 170   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
 171   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 172 };
 173
 174 VLIB_PLUGIN_REGISTER () = {
 175     .version = VPP_BUILD_VER,
 176     .description = "Network Address Translation (NAT)",
 177 };
 178
 179 static void nat44_ed_db_init (u32 translations, u32 translation_buckets);
 180
 181 static void nat44_ed_db_free ();
 182
 183 u32 nat_calc_bihash_buckets (u32 n_elts);
 184
 185 u8 *
 186 format_session_kvp (u8 * s, va_list * args)
 187 {
 188   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 189
 190   s = format (s, "%U thread-index %llu session-index %llu", format_snat_key,
 191               v->key, nat_value_get_thread_index (v),
 192               nat_value_get_session_index (v));
 193
 194   return s;
 195 }
 196
 197 u8 *
 198 format_static_mapping_kvp (u8 * s, va_list * args)
 199 {
 200   clib_bihash_kv_8_8_t *v = va_arg (*args, clib_bihash_kv_8_8_t *);
 201
 202   s = format (s, "%U static-mapping-index %llu",
 203               format_snat_key, v->key, v->value);
 204
 205   return s;
 206 }
 207
 208 u8 *
 209 format_ed_session_kvp (u8 * s, va_list * args)
 210 {
 211   clib_bihash_kv_16_8_t *v = va_arg (*args, clib_bihash_kv_16_8_t *);
 212
 213   u8 proto;
 214   u16 r_port, l_port;
 215   ip4_address_t l_addr, r_addr;
 216   u32 fib_index;
 217
 218   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
 219   s = format (s,
 220               "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
 221               "session-index %u",
 222               format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
 223               format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
 224               format_ip_protocol, proto, fib_index,
 225               ed_value_get_thread_index (v), ed_value_get_session_index (v));
 226
 227   return s;
 228 }
 229
 230 void
 231 nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 232                        u8 is_ha)
 233 {
 234       per_vrf_sessions_unregister_session (s, thread_index);
 235
 236       if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
 237         nat_elog_warn (sm, "flow hash del failed");
 238
 239       if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
 240         nat_elog_warn (sm, "flow hash del failed");
 241
 242   if (is_fwd_bypass_session (s))
 243     {
 244       return;
 245     }
 246
 247       if (is_affinity_sessions (s))
 248         nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 249                              s->nat_proto, s->out2in.port);
 250
 251       if (!is_ha)
 252         nat_syslog_nat44_sdel (
 253           0, s->in2out.fib_index, &s->in2out.addr, s->in2out.port,
 254           &s->ext_host_nat_addr, s->ext_host_nat_port, &s->out2in.addr,
 255           s->out2in.port, &s->ext_host_addr, s->ext_host_port, s->nat_proto,
 256           is_twice_nat_session (s));
 257
 258   if (snat_is_unk_proto_session (s))
 259     return;
 260
 261   if (!is_ha)
 262     {
 263       /* log NAT event */
 264       nat_ipfix_logging_nat44_ses_delete (thread_index,
 265                                           s->in2out.addr.as_u32,
 266                                           s->out2in.addr.as_u32,
 267                                           s->nat_proto,
 268                                           s->in2out.port,
 269                                           s->out2in.port,
 270                                           s->in2out.fib_index);
 271     }
 272
 273   /* Twice NAT address and port for external host */
 274   if (is_twice_nat_session (s))
 275     {
 276       snat_free_outside_address_and_port (sm->twice_nat_addresses,
 277                                           thread_index,
 278                                           &s->ext_host_nat_addr,
 279                                           s->ext_host_nat_port, s->nat_proto);
 280     }
 281
 282   if (snat_is_session_static (s))
 283     return;
 284
 285   snat_free_outside_address_and_port (sm->addresses, thread_index,
 286                                       &s->out2in.addr, s->out2in.port,
 287                                       s->nat_proto);
 288 }
 289
 290 void
 291 snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
 292                           int is_add)
 293 {
 294   snat_main_t *sm = &snat_main;
 295   fib_prefix_t prefix = {
 296     .fp_len = p_len,
 297     .fp_proto = FIB_PROTOCOL_IP4,
 298     .fp_addr = {
 299                 .ip4.as_u32 = addr->as_u32,
 300                 },
 301   };
 302   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 303
 304   if (is_add)
 305     fib_table_entry_update_one_path (fib_index,
 306                                      &prefix,
 307                                      sm->fib_src_low,
 308                                      (FIB_ENTRY_FLAG_CONNECTED |
 309                                       FIB_ENTRY_FLAG_LOCAL |
 310                                       FIB_ENTRY_FLAG_EXCLUSIVE),
 311                                      DPO_PROTO_IP4,
 312                                      NULL,
 313                                      sw_if_index,
 314                                      ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 315   else
 316     fib_table_entry_delete (fib_index, &prefix, sm->fib_src_low);
 317 }
 318
 319 int
 320 snat_add_address (snat_main_t * sm, ip4_address_t * addr, u32 vrf_id,
 321                   u8 twice_nat)
 322 {
 323   snat_address_t *ap;
 324   snat_interface_t *i;
 325   vlib_thread_main_t *tm = vlib_get_thread_main ();
 326
 327   if (!sm->enabled)
 328     {
 329       return VNET_API_ERROR_UNSUPPORTED;
 330     }
 331
 332   /* Check if address already exists */
 333   vec_foreach (ap, twice_nat ? sm->twice_nat_addresses : sm->addresses)
 334     {
 335       if (ap->addr.as_u32 == addr->as_u32)
 336         {
 337           nat_log_err ("address exist");
 338           return VNET_API_ERROR_VALUE_EXIST;
 339         }
 340     }
 341
 342   if (twice_nat)
 343     vec_add2 (sm->twice_nat_addresses, ap, 1);
 344   else
 345     vec_add2 (sm->addresses, ap, 1);
 346
 347   ap->addr = *addr;
 348   if (vrf_id != ~0)
 349     ap->fib_index =
 350       fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 351                                          sm->fib_src_low);
 352   else
 353     ap->fib_index = ~0;
 354
 355   #define _(N, i, n, s) \
 356     clib_memset(ap->busy_##n##_port_refcounts, 0, sizeof(ap->busy_##n##_port_refcounts));\
 357     ap->busy_##n##_ports = 0; \
 358     ap->busy_##n##_ports_per_thread = 0;\
 359     vec_validate_init_empty (ap->busy_##n##_ports_per_thread, tm->n_vlib_mains - 1, 0);
 360     foreach_nat_protocol
 361   #undef _
 362
 363   if (twice_nat)
 364     return 0;
 365
 366   /* Add external address to FIB */
 367   pool_foreach (i, sm->interfaces)
 368    {
 369      if (nat_interface_is_inside (i))
 370        continue;
 371
 372      snat_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
 373      break;
 374   }
 375   pool_foreach (i, sm->output_feature_interfaces)
 376    {
 377      if (nat_interface_is_inside (i))
 378        continue;
 379
 380      snat_add_del_addr_to_fib (addr, 32, i->sw_if_index, 1);
 381      break;
 382   }
 383
 384   return 0;
 385 }
 386
 387 static int
 388 is_snat_address_used_in_static_mapping (snat_main_t * sm, ip4_address_t addr)
 389 {
 390   snat_static_mapping_t *m;
 391   pool_foreach (m, sm->static_mappings)
 392    {
 393      if (is_sm_addr_only (m->flags) || is_sm_out2in_only (m->flags) ||
 394          is_sm_identity_nat (m->flags))
 395        continue;
 396      if (m->external_addr.as_u32 == addr.as_u32)
 397        return 1;
 398   }
 399   return 0;
 400 }
 401
 402 u32
 403 get_thread_idx_by_port (u16 e_port)
 404 {
 405   snat_main_t *sm = &snat_main;
 406   u32 thread_idx = sm->num_workers;
 407   if (sm->num_workers > 1)
 408     {
 409       thread_idx =
 410         sm->first_worker_index +
 411         sm->workers[(e_port - 1024) / sm->port_per_thread];
 412     }
 413   return thread_idx;
 414 }
 415
 416 void
 417 nat_ed_static_mapping_del_sessions (snat_main_t * sm,
 418                                     snat_main_per_thread_data_t * tsm,
 419                                     ip4_address_t l_addr,
 420                                     u16 l_port,
 421                                     u8 protocol,
 422                                     u32 fib_index, int addr_only,
 423                                     ip4_address_t e_addr, u16 e_port)
 424 {
 425   snat_session_t *s;
 426   u32 *indexes_to_free = NULL;
 427   pool_foreach (s, tsm->sessions) {
 428     if (s->in2out.fib_index != fib_index ||
 429         s->in2out.addr.as_u32 != l_addr.as_u32)
 430       {
 431         continue;
 432       }
 433     if (!addr_only)
 434       {
 435         if ((s->out2in.addr.as_u32 != e_addr.as_u32) ||
 436             s->out2in.port != e_port ||
 437             s->in2out.port != l_port ||
 438             s->nat_proto != protocol)
 439           continue;
 440       }
 441
 442     if (is_lb_session (s))
 443       continue;
 444     if (!snat_is_session_static (s))
 445       continue;
 446     nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
 447     vec_add1 (indexes_to_free, s - tsm->sessions);
 448     if (!addr_only)
 449       break;
 450   }
 451   u32 *ses_index;
 452   vec_foreach (ses_index, indexes_to_free)
 453   {
 454     s = pool_elt_at_index (tsm->sessions, *ses_index);
 455     nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
 456   }
 457   vec_free (indexes_to_free);
 458 }
 459
 460 int
 461 nat44_ed_reserve_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
 462 {
 463   u32 ti = get_thread_idx_by_port (port);
 464   snat_main_t *sm = &snat_main;
 465   snat_address_t *a = 0;
 466   int i;
 467
 468   for (i = 0; i < vec_len (sm->addresses); i++)
 469     {
 470       a = sm->addresses + i;
 471
 472       if (a->addr.as_u32 != addr.as_u32)
 473         continue;
 474
 475       switch (proto)
 476         {
 477 #define _(N, j, n, s)                                                         \
 478   case NAT_PROTOCOL_##N:                                                      \
 479     if (a->busy_##n##_port_refcounts[port])                                   \
 480       goto done;                                                              \
 481     ++a->busy_##n##_port_refcounts[port];                                     \
 482     if (port > 1024)                                                          \
 483       {                                                                       \
 484         a->busy_##n##_ports++;                                                \
 485         a->busy_##n##_ports_per_thread[ti]++;                                 \
 486       }                                                                       \
 487     break;
 488           foreach_nat_protocol
 489 #undef _
 490             default : nat_elog_info (sm, "unknown protocol");
 491           goto done;
 492         }
 493
 494       return 0;
 495     }
 496
 497 done:
 498   return 1;
 499 }
 500
 501 int
 502 nat44_ed_free_port (ip4_address_t addr, u16 port, nat_protocol_t proto)
 503 {
 504   u32 ti = get_thread_idx_by_port (port);
 505   snat_main_t *sm = &snat_main;
 506   snat_address_t *a = 0;
 507   int i;
 508
 509   for (i = 0; i < vec_len (sm->addresses); i++)
 510     {
 511       a = sm->addresses + i;
 512
 513       if (a->addr.as_u32 != addr.as_u32)
 514         continue;
 515
 516       switch (proto)
 517         {
 518 #define _(N, j, n, s)                                                         \
 519   case NAT_PROTOCOL_##N:                                                      \
 520     --a->busy_##n##_port_refcounts[port];                                     \
 521     if (port > 1024)                                                          \
 522       {                                                                       \
 523         a->busy_##n##_ports--;                                                \
 524         a->busy_##n##_ports_per_thread[ti]--;                                 \
 525       }                                                                       \
 526     break;
 527           foreach_nat_protocol
 528 #undef _
 529             default : nat_elog_info (sm, "unknown protocol");
 530           goto done;
 531         }
 532
 533       return 0;
 534     }
 535
 536 done:
 537   return 1;
 538 }
 539
 540 void
 541 nat44_ed_add_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 542                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 543                              u32 flags, ip4_address_t pool_addr, u8 *tag)
 544 {
 545   snat_static_map_resolve_t *rp;
 546   snat_main_t *sm = &snat_main;
 547
 548   vec_add2 (sm->to_resolve, rp, 1);
 549   rp->l_addr.as_u32 = l_addr.as_u32;
 550   rp->l_port = l_port;
 551   rp->e_port = e_port;
 552   rp->sw_if_index = sw_if_index;
 553   rp->vrf_id = vrf_id;
 554   rp->proto = proto;
 555   rp->flags = flags;
 556   rp->pool_addr = pool_addr;
 557   rp->tag = vec_dup (tag);
 558 }
 559
 560 int
 561 nat44_ed_get_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 562                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 563                              u32 flags, int *out)
 564 {
 565   snat_static_map_resolve_t *rp;
 566   snat_main_t *sm = &snat_main;
 567   int i;
 568
 569   for (i = 0; i < vec_len (sm->to_resolve); i++)
 570     {
 571       rp = sm->to_resolve + i;
 572
 573       if (rp->sw_if_index == sw_if_index && rp->vrf_id == vrf_id)
 574         {
 575           if (is_sm_identity_nat (rp->flags) && is_sm_identity_nat (flags))
 576             {
 577               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 578                 {
 579                   if (rp->e_port != e_port || rp->proto != proto)
 580                     {
 581                       continue;
 582                     }
 583                 }
 584             }
 585           else if (rp->l_addr.as_u32 == l_addr.as_u32)
 586             {
 587               if (!(is_sm_addr_only (rp->flags) && is_sm_addr_only (flags)))
 588                 {
 589                   if (rp->l_port != l_port || rp->e_port != e_port ||
 590                       rp->proto != proto)
 591                     {
 592                       continue;
 593                     }
 594                 }
 595             }
 596           else
 597             {
 598               continue;
 599             }
 600           if (out)
 601             {
 602               *out = i;
 603             }
 604           return 0;
 605         }
 606     }
 607   return 1;
 608 }
 609
 610 int
 611 nat44_ed_del_resolve_record (ip4_address_t l_addr, u16 l_port, u16 e_port,
 612                              nat_protocol_t proto, u32 vrf_id, u32 sw_if_index,
 613                              u32 flags)
 614 {
 615   snat_main_t *sm = &snat_main;
 616   int i;
 617   if (!nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 618                                     sw_if_index, flags, &i))
 619     {
 620       vec_del1 (sm->to_resolve, i);
 621       return 0;
 622     }
 623   return 1;
 624 }
 625
 626 static_always_inline int
 627 nat44_ed_validate_sm_input (u32 flags)
 628 {
 629   // identity nat can be initiated only from inside interface
 630   if (is_sm_identity_nat (flags) && is_sm_out2in_only (flags))
 631     {
 632       return VNET_API_ERROR_UNSUPPORTED;
 633     }
 634
 635   if (is_sm_twice_nat (flags) || is_sm_self_twice_nat (flags))
 636     {
 637       if (is_sm_addr_only (flags) || is_sm_identity_nat (flags))
 638         {
 639           return VNET_API_ERROR_UNSUPPORTED;
 640         }
 641     }
 642   return 0;
 643 }
 644
 645 int
 646 nat44_ed_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 647                              u16 l_port, u16 e_port, nat_protocol_t proto,
 648                              u32 vrf_id, u32 sw_if_index, u32 flags,
 649                              ip4_address_t pool_addr, u8 *tag)
 650 {
 651   snat_main_t *sm = &snat_main;
 652   clib_bihash_kv_8_8_t kv, value;
 653   snat_interface_t *interface;
 654   nat44_lb_addr_port_t *local;
 655   snat_static_mapping_t *m;
 656   u32 fib_index = ~0;
 657   int rv;
 658
 659   if (!sm->enabled)
 660     {
 661       return VNET_API_ERROR_UNSUPPORTED;
 662     }
 663
 664   rv = nat44_ed_validate_sm_input (flags);
 665   if (rv != 0)
 666     {
 667       return rv;
 668     }
 669
 670   if (is_sm_addr_only (flags))
 671     {
 672       e_port = l_port = proto = 0;
 673     }
 674
 675   if (is_sm_switch_address (flags))
 676     {
 677       // this mapping is interface bound
 678       ip4_address_t *first_int_addr;
 679
 680       // check if this record isn't registered for resolve
 681       if (!nat44_ed_get_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 682                                         sw_if_index, flags, 0))
 683         {
 684           return VNET_API_ERROR_VALUE_EXIST;
 685         }
 686       // register record for resolve
 687       nat44_ed_add_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 688                                    sw_if_index, flags, pool_addr, tag);
 689
 690       first_int_addr =
 691         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 692       if (!first_int_addr)
 693         {
 694           // dhcp resolution required
 695           return 0;
 696         }
 697
 698       e_addr.as_u32 = first_int_addr->as_u32;
 699     }
 700
 701   if (is_sm_identity_nat (flags))
 702     {
 703       l_port = e_port;
 704       l_addr.as_u32 = e_addr.as_u32;
 705     }
 706
 707   // fib index 0
 708   init_nat_k (&kv, e_addr, e_port, 0, proto);
 709
 710   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 711     {
 712       m = pool_elt_at_index (sm->static_mappings, value.value);
 713       if (!is_sm_identity_nat (m->flags))
 714         {
 715           return VNET_API_ERROR_VALUE_EXIST;
 716         }
 717
 718       // case:
 719       // adding local identity nat record for different vrf table
 720       pool_foreach (local, m->locals)
 721         {
 722           if (local->vrf_id == vrf_id)
 723             {
 724               return VNET_API_ERROR_VALUE_EXIST;
 725             }
 726         }
 727
 728       pool_get (m->locals, local);
 729
 730       local->vrf_id = vrf_id;
 731       local->fib_index = fib_table_find_or_create_and_lock (
 732         FIB_PROTOCOL_IP4, vrf_id, sm->fib_src_low);
 733
 734       init_nat_kv (&kv, m->local_addr, m->local_port, local->fib_index,
 735                    m->proto, 0, m - sm->static_mappings);
 736       clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 737
 738       return 0;
 739     }
 740
 741   if (vrf_id != ~0)
 742     {
 743       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
 744                                                      sm->fib_src_low);
 745     }
 746   else
 747     {
 748       // fallback to default vrf
 749       vrf_id = sm->inside_vrf_id;
 750       fib_index = sm->inside_fib_index;
 751       fib_table_lock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 752     }
 753
 754   // test if local mapping record doesn't exist
 755   // identity nat supports multiple records in local mapping
 756   if (!(is_sm_out2in_only (flags) || is_sm_identity_nat (flags)))
 757     {
 758       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
 759       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 760         {
 761           return VNET_API_ERROR_VALUE_EXIST;
 762         }
 763     }
 764
 765   if (!(is_sm_out2in_only (flags) || is_sm_addr_only (flags) ||
 766         sm->static_mapping_only))
 767     {
 768       if (nat44_ed_reserve_port (e_addr, e_port, proto))
 769         {
 770           // remove resolve record
 771           if (is_sm_switch_address (flags) && !is_sm_identity_nat (flags))
 772             {
 773               nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto,
 774                                            vrf_id, sw_if_index, flags);
 775             }
 776           return VNET_API_ERROR_NO_SUCH_ENTRY;
 777         }
 778     }
 779
 780   pool_get (sm->static_mappings, m);
 781   clib_memset (m, 0, sizeof (*m));
 782
 783   m->flags = flags;
 784   m->local_addr = l_addr;
 785   m->external_addr = e_addr;
 786
 787   m->tag = vec_dup (tag);
 788
 789   if (is_sm_exact_address (flags) && is_sm_twice_nat (flags))
 790     {
 791       m->pool_addr = pool_addr;
 792     }
 793
 794   if (!is_sm_addr_only (flags))
 795     {
 796       m->local_port = l_port;
 797       m->external_port = e_port;
 798       m->proto = proto;
 799     }
 800
 801   if (is_sm_identity_nat (flags))
 802     {
 803       pool_get (m->locals, local);
 804
 805       local->vrf_id = vrf_id;
 806       local->fib_index = fib_index;
 807     }
 808   else
 809     {
 810       m->vrf_id = vrf_id;
 811       m->fib_index = fib_index;
 812     }
 813
 814   if (!is_sm_out2in_only (flags))
 815     {
 816       init_nat_kv (&kv, m->local_addr, m->local_port, fib_index, m->proto, 0,
 817                    m - sm->static_mappings);
 818       clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
 819     }
 820
 821   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
 822                m - sm->static_mappings);
 823   clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1);
 824
 825   if (sm->num_workers > 1)
 826     {
 827       // store worker index for this record
 828       ip4_header_t ip = {
 829         .src_address = m->local_addr,
 830       };
 831       u32 worker_index;
 832       worker_index =
 833         nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0);
 834       vec_add1 (m->workers, worker_index);
 835     }
 836
 837   if (is_sm_identity_nat (flags) || !is_sm_addr_only (flags))
 838     return 0;
 839
 840   pool_foreach (interface, sm->interfaces)
 841     {
 842       if (nat_interface_is_inside (interface))
 843         continue;
 844
 845       snat_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index, 1);
 846       break;
 847     }
 848
 849   pool_foreach (interface, sm->output_feature_interfaces)
 850     {
 851       if (nat_interface_is_inside (interface))
 852         continue;
 853
 854       snat_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index, 1);
 855       break;
 856     }
 857
 858   return 0;
 859 }
 860
 861 int
 862 nat44_ed_del_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 863                              u16 l_port, u16 e_port, nat_protocol_t proto,
 864                              u32 vrf_id, u32 sw_if_index, u32 flags)
 865 {
 866   snat_main_per_thread_data_t *tsm;
 867   snat_main_t *sm = &snat_main;
 868
 869   clib_bihash_kv_8_8_t kv, value;
 870   snat_interface_t *interface;
 871   nat44_lb_addr_port_t *local;
 872   snat_static_mapping_t *m;
 873   u32 fib_index = ~0;
 874   int rv;
 875
 876   if (!sm->enabled)
 877     {
 878       return VNET_API_ERROR_UNSUPPORTED;
 879     }
 880
 881   rv = nat44_ed_validate_sm_input (flags);
 882   if (rv != 0)
 883     {
 884       return rv;
 885     }
 886
 887   if (is_sm_addr_only (flags))
 888     {
 889       e_port = l_port = proto = 0;
 890     }
 891
 892   if (is_sm_switch_address (flags))
 893     {
 894       // this mapping is interface bound
 895       ip4_address_t *first_int_addr;
 896
 897       // delete record registered for resolve
 898       if (nat44_ed_del_resolve_record (l_addr, l_port, e_port, proto, vrf_id,
 899                                        sw_if_index, flags))
 900         {
 901           return VNET_API_ERROR_NO_SUCH_ENTRY;
 902         }
 903
 904       first_int_addr =
 905         ip4_interface_first_address (sm->ip4_main, sw_if_index, 0);
 906       if (!first_int_addr)
 907         {
 908           // dhcp resolution required
 909           return 0;
 910         }
 911
 912       e_addr.as_u32 = first_int_addr->as_u32;
 913     }
 914
 915   if (is_sm_identity_nat (flags))
 916     {
 917       l_port = e_port;
 918       l_addr.as_u32 = e_addr.as_u32;
 919     }
 920
 921   // fib index 0
 922   init_nat_k (&kv, e_addr, e_port, 0, proto);
 923
 924   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 925     {
 926       if (is_sm_switch_address (flags))
 927         {
 928           return 0;
 929         }
 930       return VNET_API_ERROR_NO_SUCH_ENTRY;
 931     }
 932
 933   m = pool_elt_at_index (sm->static_mappings, value.value);
 934
 935   if (is_sm_identity_nat (flags))
 936     {
 937       u8 failure = 1;
 938
 939       if (!is_sm_switch_address (flags))
 940         {
 941           vrf_id = sm->inside_vrf_id;
 942         }
 943
 944       pool_foreach (local, m->locals)
 945         {
 946           if (local->vrf_id == vrf_id)
 947             {
 948               local = pool_elt_at_index (m->locals, local - m->locals);
 949               fib_index = local->fib_index;
 950               pool_put (m->locals, local);
 951               failure = 0;
 952             }
 953         }
 954
 955       if (failure)
 956         {
 957           return VNET_API_ERROR_NO_SUCH_ENTRY;
 958         }
 959     }
 960   else
 961     {
 962       fib_index = m->fib_index;
 963     }
 964
 965   if (!(is_sm_out2in_only (flags) || is_sm_addr_only (flags) ||
 966         sm->static_mapping_only))
 967     {
 968       if (nat44_ed_free_port (e_addr, e_port, proto))
 969         {
 970           return VNET_API_ERROR_INVALID_VALUE;
 971         }
 972     }
 973
 974   if (!is_sm_out2in_only (flags))
 975     {
 976       init_nat_k (&kv, l_addr, l_port, fib_index, proto);
 977       clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0);
 978     }
 979
 980   if (!sm->static_mapping_only || sm->static_mapping_connection_tracking)
 981     {
 982       // delete sessions for static mapping
 983       if (sm->num_workers > 1)
 984         tsm = vec_elt_at_index (sm->per_thread_data, m->workers[0]);
 985       else
 986         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 987
 988       nat_ed_static_mapping_del_sessions (
 989         sm, tsm, m->local_addr, m->local_port, m->proto, fib_index,
 990         is_sm_addr_only (flags), e_addr, e_port);
 991     }
 992
 993   fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
 994
 995   if (pool_elts (m->locals))
 996     return 0;
 997
 998   // fib_index 0
 999   init_nat_k (&kv, e_addr, e_port, 0, proto);
1000   clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0);
1001
1002   vec_free (m->tag);
1003   vec_free (m->workers);
1004   pool_put (sm->static_mappings, m);
1005
1006   if (is_sm_identity_nat (flags) || !is_sm_addr_only (flags))
1007     return 0;
1008
1009   pool_foreach (interface, sm->interfaces)
1010     {
1011       if (nat_interface_is_inside (interface))
1012         continue;
1013
1014       snat_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index, 0);
1015       break;
1016     }
1017
1018   pool_foreach (interface, sm->output_feature_interfaces)
1019     {
1020       if (nat_interface_is_inside (interface))
1021         continue;
1022
1023       snat_add_del_addr_to_fib (&e_addr, 32, interface->sw_if_index, 0);
1024       break;
1025     }
1026
1027   return 0;
1028 }
1029
1030 int
1031 nat44_ed_add_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1032                                 nat_protocol_t proto,
1033                                 nat44_lb_addr_port_t *locals, u32 flags,
1034                                 u8 *tag, u32 affinity)
1035 {
1036   snat_main_t *sm = &snat_main;
1037   snat_static_mapping_t *m;
1038   clib_bihash_kv_8_8_t kv, value;
1039   snat_address_t *a = 0;
1040
1041   nat44_lb_addr_port_t *local;
1042   uword *bitmap = 0;
1043
1044   int i;
1045
1046   if (!sm->enabled)
1047     {
1048       return VNET_API_ERROR_UNSUPPORTED;
1049     }
1050
1051   init_nat_k (&kv, e_addr, e_port, 0, proto);
1052   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1053     m = 0;
1054   else
1055     m = pool_elt_at_index (sm->static_mappings, value.value);
1056
1057   if (m)
1058     return VNET_API_ERROR_VALUE_EXIST;
1059
1060   if (vec_len (locals) < 2)
1061     return VNET_API_ERROR_INVALID_VALUE;
1062
1063   /* Find external address in allocated addresses and reserve port for
1064      address and port pair mapping when dynamic translations enabled */
1065   if (!(sm->static_mapping_only || is_sm_out2in_only (flags)))
1066     {
1067       for (i = 0; i < vec_len (sm->addresses); i++)
1068         {
1069           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1070             {
1071               a = sm->addresses + i;
1072               /* External port must be unused */
1073               switch (proto)
1074                 {
1075 #define _(N, j, n, s)                                                         \
1076   case NAT_PROTOCOL_##N:                                                      \
1077     if (a->busy_##n##_port_refcounts[e_port])                                 \
1078       return VNET_API_ERROR_INVALID_VALUE;                                    \
1079     ++a->busy_##n##_port_refcounts[e_port];                                   \
1080     if (e_port > 1024)                                                        \
1081       {                                                                       \
1082         a->busy_##n##_ports++;                                                \
1083         a->busy_##n##_ports_per_thread[get_thread_idx_by_port (e_port)]++;    \
1084       }                                                                       \
1085     break;
1086                   foreach_nat_protocol
1087 #undef _
1088                     default : nat_elog_info (sm, "unknown protocol");
1089                   return VNET_API_ERROR_INVALID_VALUE_2;
1090                 }
1091               break;
1092             }
1093         }
1094       /* External address must be allocated */
1095       if (!a)
1096         return VNET_API_ERROR_NO_SUCH_ENTRY;
1097     }
1098
1099   pool_get (sm->static_mappings, m);
1100   clib_memset (m, 0, sizeof (*m));
1101   m->tag = vec_dup (tag);
1102   m->external_addr = e_addr;
1103   m->external_port = e_port;
1104   m->affinity = affinity;
1105   m->proto = proto;
1106
1107   m->flags = flags;
1108   m->flags |= NAT_SM_FLAG_LB;
1109
1110   if (affinity)
1111     m->affinity_per_service_list_head_index =
1112       nat_affinity_get_per_service_list_head_index ();
1113   else
1114     m->affinity_per_service_list_head_index = ~0;
1115
1116   init_nat_kv (&kv, m->external_addr, m->external_port, 0, m->proto, 0,
1117                m - sm->static_mappings);
1118   if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 1))
1119     {
1120       nat_elog_err (sm, "static_mapping_by_external key add failed");
1121       return VNET_API_ERROR_UNSPECIFIED;
1122     }
1123
1124   for (i = 0; i < vec_len (locals); i++)
1125     {
1126       locals[i].fib_index = fib_table_find_or_create_and_lock (
1127         FIB_PROTOCOL_IP4, locals[i].vrf_id, sm->fib_src_low);
1128       if (!is_sm_out2in_only (flags))
1129         {
1130           init_nat_kv (&kv, locals[i].addr, locals[i].port,
1131                        locals[i].fib_index, m->proto, 0,
1132                        m - sm->static_mappings);
1133           clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1);
1134         }
1135       locals[i].prefix = (i == 0) ?
1136                            locals[i].probability :
1137                            (locals[i - 1].prefix + locals[i].probability);
1138       pool_get (m->locals, local);
1139       *local = locals[i];
1140       if (sm->num_workers > 1)
1141         {
1142           ip4_header_t ip = {
1143             .src_address = locals[i].addr,
1144           };
1145           bitmap = clib_bitmap_set (
1146             bitmap, nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0),
1147             1);
1148         }
1149     }
1150
1151   /* Assign workers */
1152   if (sm->num_workers > 1)
1153     {
1154       clib_bitmap_foreach (i, bitmap)
1155         {
1156           vec_add1 (m->workers, i);
1157         }
1158     }
1159
1160   return 0;
1161 }
1162
1163 int
1164 nat44_ed_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
1165                                 nat_protocol_t proto, u32 flags)
1166 {
1167   snat_main_t *sm = &snat_main;
1168   snat_static_mapping_t *m;
1169   clib_bihash_kv_8_8_t kv, value;
1170   snat_address_t *a = 0;
1171
1172   nat44_lb_addr_port_t *local;
1173   snat_main_per_thread_data_t *tsm;
1174   snat_session_t *s;
1175   int i;
1176
1177   if (!sm->enabled)
1178     {
1179       return VNET_API_ERROR_UNSUPPORTED;
1180     }
1181
1182   init_nat_k (&kv, e_addr, e_port, 0, proto);
1183   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1184     m = 0;
1185   else
1186     m = pool_elt_at_index (sm->static_mappings, value.value);
1187
1188   if (!m)
1189     return VNET_API_ERROR_NO_SUCH_ENTRY;
1190
1191   if (!is_sm_lb (m->flags))
1192     return VNET_API_ERROR_INVALID_VALUE;
1193
1194   /* Free external address port */
1195   if (!(sm->static_mapping_only || is_sm_out2in_only (flags)))
1196     {
1197       for (i = 0; i < vec_len (sm->addresses); i++)
1198         {
1199           if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
1200             {
1201               a = sm->addresses + i;
1202               switch (proto)
1203                 {
1204 #define _(N, j, n, s)                                                         \
1205   case NAT_PROTOCOL_##N:                                                      \
1206     --a->busy_##n##_port_refcounts[e_port];                                   \
1207     if (e_port > 1024)                                                        \
1208       {                                                                       \
1209         a->busy_##n##_ports--;                                                \
1210         a->busy_##n##_ports_per_thread[get_thread_idx_by_port (e_port)]--;    \
1211       }                                                                       \
1212     break;
1213                   foreach_nat_protocol
1214 #undef _
1215                     default : nat_elog_info (sm, "unknown protocol");
1216                   return VNET_API_ERROR_INVALID_VALUE_2;
1217                 }
1218               break;
1219             }
1220         }
1221     }
1222
1223   init_nat_k (&kv, m->external_addr, m->external_port, 0, m->proto);
1224   if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_external, &kv, 0))
1225     {
1226       nat_elog_err (sm, "static_mapping_by_external key del failed");
1227       return VNET_API_ERROR_UNSPECIFIED;
1228     }
1229
1230   pool_foreach (local, m->locals)
1231     {
1232       fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1233       if (!is_sm_out2in_only (flags))
1234         {
1235           init_nat_k (&kv, local->addr, local->port, local->fib_index,
1236                       m->proto);
1237           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1238             {
1239               nat_elog_err (sm, "static_mapping_by_local key del failed");
1240               return VNET_API_ERROR_UNSPECIFIED;
1241             }
1242         }
1243
1244       if (sm->num_workers > 1)
1245         {
1246           ip4_header_t ip = {
1247             .src_address = local->addr,
1248           };
1249           tsm = vec_elt_at_index (
1250             sm->per_thread_data,
1251             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1252         }
1253       else
1254         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1255
1256       /* Delete sessions */
1257       pool_foreach (s, tsm->sessions)
1258         {
1259           if (!(is_lb_session (s)))
1260             continue;
1261
1262           if ((s->in2out.addr.as_u32 != local->addr.as_u32) ||
1263               s->in2out.port != local->port)
1264             continue;
1265
1266           nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1267           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1268         }
1269     }
1270
1271   if (m->affinity)
1272     {
1273       nat_affinity_flush_service (m->affinity_per_service_list_head_index);
1274     }
1275
1276   pool_free (m->locals);
1277   vec_free (m->tag);
1278   vec_free (m->workers);
1279   pool_put (sm->static_mappings, m);
1280
1281   return 0;
1282 }
1283
1284 int
1285 nat44_ed_add_del_lb_static_mapping_local (ip4_address_t e_addr, u16 e_port,
1286                                           ip4_address_t l_addr, u16 l_port,
1287                                           nat_protocol_t proto, u32 vrf_id,
1288                                           u8 probability, u8 is_add)
1289 {
1290   snat_main_t *sm = &snat_main;
1291   snat_static_mapping_t *m = 0;
1292   clib_bihash_kv_8_8_t kv, value;
1293   nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
1294   snat_main_per_thread_data_t *tsm;
1295   snat_session_t *s;
1296   u32 *locals = 0;
1297   uword *bitmap = 0;
1298   int i;
1299
1300   if (!sm->enabled)
1301     {
1302       return VNET_API_ERROR_UNSUPPORTED;
1303     }
1304
1305   init_nat_k (&kv, e_addr, e_port, 0, proto);
1306   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1307     m = pool_elt_at_index (sm->static_mappings, value.value);
1308
1309   if (!m)
1310     return VNET_API_ERROR_NO_SUCH_ENTRY;
1311
1312   if (!is_sm_lb (m->flags))
1313     return VNET_API_ERROR_INVALID_VALUE;
1314
1315   pool_foreach (local, m->locals)
1316    {
1317     if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
1318         (local->vrf_id == vrf_id))
1319       {
1320         match_local = local;
1321         break;
1322       }
1323   }
1324
1325   if (is_add)
1326     {
1327       if (match_local)
1328         return VNET_API_ERROR_VALUE_EXIST;
1329
1330       pool_get (m->locals, local);
1331       clib_memset (local, 0, sizeof (*local));
1332       local->addr.as_u32 = l_addr.as_u32;
1333       local->port = l_port;
1334       local->probability = probability;
1335       local->vrf_id = vrf_id;
1336       local->fib_index =
1337         fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
1338                                            sm->fib_src_low);
1339
1340       if (!is_sm_out2in_only (m->flags))
1341         {
1342           init_nat_kv (&kv, l_addr, l_port, local->fib_index, proto, 0,
1343                        m - sm->static_mappings);
1344           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
1345             nat_elog_err (sm, "static_mapping_by_local key add failed");
1346         }
1347     }
1348   else
1349     {
1350       if (!match_local)
1351         return VNET_API_ERROR_NO_SUCH_ENTRY;
1352
1353       if (pool_elts (m->locals) < 3)
1354         return VNET_API_ERROR_UNSPECIFIED;
1355
1356       fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
1357                         sm->fib_src_low);
1358
1359       if (!is_sm_out2in_only (m->flags))
1360         {
1361           init_nat_k (&kv, l_addr, l_port, match_local->fib_index, proto);
1362           if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
1363             nat_elog_err (sm, "static_mapping_by_local key del failed");
1364         }
1365
1366       if (sm->num_workers > 1)
1367         {
1368           ip4_header_t ip = {
1369             .src_address = local->addr,
1370           };
1371           tsm = vec_elt_at_index (
1372             sm->per_thread_data,
1373             nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index, 0));
1374         }
1375       else
1376         tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
1377
1378       /* Delete sessions */
1379       pool_foreach (s, tsm->sessions) {
1380         if (!(is_lb_session (s)))
1381           continue;
1382
1383         if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
1384             s->in2out.port != match_local->port)
1385           continue;
1386
1387         nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
1388         nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
1389       }
1390
1391       pool_put (m->locals, match_local);
1392     }
1393
1394   vec_free (m->workers);
1395
1396   pool_foreach (local, m->locals)
1397    {
1398     vec_add1 (locals, local - m->locals);
1399     if (sm->num_workers > 1)
1400       {
1401         ip4_header_t ip;
1402         ip.src_address.as_u32 = local->addr.as_u32,
1403         bitmap = clib_bitmap_set (
1404           bitmap,
1405           nat44_ed_get_in2out_worker_index (0, &ip, local->fib_index, 0), 1);
1406       }
1407   }
1408
1409   ASSERT (vec_len (locals) > 1);
1410
1411   local = pool_elt_at_index (m->locals, locals[0]);
1412   local->prefix = local->probability;
1413   for (i = 1; i < vec_len (locals); i++)
1414     {
1415       local = pool_elt_at_index (m->locals, locals[i]);
1416       prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
1417       local->prefix = local->probability + prev_local->prefix;
1418     }
1419
1420   /* Assign workers */
1421   if (sm->num_workers > 1)
1422     {
1423       clib_bitmap_foreach (i, bitmap)  { vec_add1(m->workers, i); }
1424     }
1425
1426   return 0;
1427 }
1428
1429 int
1430 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
1431                   u8 twice_nat)
1432 {
1433   snat_address_t *a = 0;
1434   snat_session_t *ses;
1435   u32 *ses_to_be_removed = 0, *ses_index;
1436   snat_main_per_thread_data_t *tsm;
1437   snat_static_mapping_t *m;
1438   snat_interface_t *interface;
1439   snat_address_t *addresses;
1440   int i;
1441
1442   if (!sm->enabled)
1443     {
1444       return VNET_API_ERROR_UNSUPPORTED;
1445     }
1446
1447   addresses = twice_nat ? sm->twice_nat_addresses : sm->addresses;
1448
1449   /* Find SNAT address */
1450   for (i = 0; i < vec_len (addresses); i++)
1451     {
1452       if (addresses[i].addr.as_u32 == addr.as_u32)
1453         {
1454           a = addresses + i;
1455           break;
1456         }
1457     }
1458   if (!a)
1459     {
1460       nat_log_err ("no such address");
1461       return VNET_API_ERROR_NO_SUCH_ENTRY;
1462     }
1463
1464   if (delete_sm)
1465     {
1466       pool_foreach (m, sm->static_mappings)
1467        {
1468           if (m->external_addr.as_u32 == addr.as_u32)
1469             {
1470               nat44_ed_del_static_mapping (m->local_addr, m->external_addr,
1471                                            m->local_port, m->external_port,
1472                                            m->proto, m->vrf_id, ~0, m->flags);
1473             }
1474       }
1475     }
1476   else
1477     {
1478       /* Check if address is used in some static mapping */
1479       if (is_snat_address_used_in_static_mapping (sm, addr))
1480         {
1481           nat_log_err ("address used in static mapping");
1482           return VNET_API_ERROR_UNSPECIFIED;
1483         }
1484     }
1485
1486   if (a->fib_index != ~0)
1487     fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP4, sm->fib_src_low);
1488
1489   /* Delete sessions using address */
1490   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
1491     {
1492       vec_foreach (tsm, sm->per_thread_data)
1493       {
1494         pool_foreach (ses, tsm->sessions)  {
1495           if (ses->out2in.addr.as_u32 == addr.as_u32)
1496             {
1497               nat_free_session_data (sm, ses, tsm - sm->per_thread_data, 0);
1498               vec_add1 (ses_to_be_removed, ses - tsm->sessions);
1499             }
1500         }
1501
1502             vec_foreach (ses_index, ses_to_be_removed)
1503             {
1504               ses = pool_elt_at_index (tsm->sessions, ses_index[0]);
1505               nat_ed_session_delete (sm, ses, tsm - sm->per_thread_data, 1);
1506             }
1507
1508         vec_free (ses_to_be_removed);
1509       }
1510     }
1511
1512 #define _(N, i, n, s) \
1513   vec_free (a->busy_##n##_ports_per_thread);
1514   foreach_nat_protocol
1515 #undef _
1516
1517     if (twice_nat)
1518   {
1519     vec_del1 (sm->twice_nat_addresses, i);
1520     return 0;
1521   }
1522   else vec_del1 (sm->addresses, i);
1523
1524   /* Delete external address from FIB */
1525   pool_foreach (interface, sm->interfaces)
1526     {
1527       if (nat_interface_is_inside (interface))
1528         continue;
1529
1530       snat_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
1531       break;
1532     }
1533   pool_foreach (interface, sm->output_feature_interfaces)
1534    {
1535      if (nat_interface_is_inside (interface))
1536        continue;
1537
1538      snat_add_del_addr_to_fib (&addr, 32, interface->sw_if_index, 0);
1539      break;
1540   }
1541
1542   return 0;
1543 }
1544
1545 void
1546 expire_per_vrf_sessions (u32 fib_index)
1547 {
1548   per_vrf_sessions_t *per_vrf_sessions;
1549   snat_main_per_thread_data_t *tsm;
1550   snat_main_t *sm = &snat_main;
1551
1552   vec_foreach (tsm, sm->per_thread_data)
1553     {
1554       vec_foreach (per_vrf_sessions, tsm->per_vrf_sessions_vec)
1555         {
1556           if ((per_vrf_sessions->rx_fib_index == fib_index) ||
1557               (per_vrf_sessions->tx_fib_index == fib_index))
1558             {
1559               per_vrf_sessions->expired = 1;
1560             }
1561         }
1562     }
1563 }
1564
1565 void
1566 update_per_vrf_sessions_vec (u32 fib_index, int is_del)
1567 {
1568   snat_main_t *sm = &snat_main;
1569   nat_fib_t *fib;
1570
1571   // we don't care if it is outside/inside fib
1572   // we just care about their ref_count
1573   // if it reaches 0 sessions should expire
1574   // because the fib isn't valid for NAT anymore
1575
1576   vec_foreach (fib, sm->fibs)
1577   {
1578     if (fib->fib_index == fib_index)
1579       {
1580         if (is_del)
1581           {
1582             fib->ref_count--;
1583             if (!fib->ref_count)
1584               {
1585                 vec_del1 (sm->fibs, fib - sm->fibs);
1586                 expire_per_vrf_sessions (fib_index);
1587               }
1588             return;
1589           }
1590         else
1591           fib->ref_count++;
1592       }
1593   }
1594   if (!is_del)
1595     {
1596       vec_add2 (sm->fibs, fib, 1);
1597       fib->ref_count = 1;
1598       fib->fib_index = fib_index;
1599     }
1600 }
1601
1602 static_always_inline nat_outside_fib_t *
1603 nat44_ed_get_outside_fib (nat_outside_fib_t *outside_fibs, u32 fib_index)
1604 {
1605   nat_outside_fib_t *f;
1606   vec_foreach (f, outside_fibs)
1607     {
1608       if (f->fib_index == fib_index)
1609         {
1610           return f;
1611         }
1612     }
1613   return 0;
1614 }
1615
1616 static_always_inline snat_interface_t *
1617 nat44_ed_get_interface (snat_interface_t *interfaces, u32 sw_if_index)
1618 {
1619   snat_interface_t *i;
1620   pool_foreach (i, interfaces)
1621     {
1622       if (i->sw_if_index == sw_if_index)
1623         {
1624           return i;
1625         }
1626     }
1627   return 0;
1628 }
1629
1630 int
1631 nat44_ed_add_interface (u32 sw_if_index, u8 is_inside)
1632 {
1633   const char *del_feature_name, *feature_name;
1634   snat_main_t *sm = &snat_main;
1635
1636   nat_outside_fib_t *outside_fib;
1637   snat_static_mapping_t *m;
1638   snat_interface_t *i;
1639   snat_address_t *ap;
1640   u32 fib_index;
1641   int rv;
1642
1643   if (!sm->enabled)
1644     {
1645       nat_log_err ("nat44 is disabled");
1646       return VNET_API_ERROR_UNSUPPORTED;
1647     }
1648
1649   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1650     {
1651       nat_log_err ("error interface already configured");
1652       return VNET_API_ERROR_VALUE_EXIST;
1653     }
1654
1655   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1656   if (i)
1657     {
1658       if ((nat_interface_is_inside (i) && is_inside) ||
1659           (nat_interface_is_outside (i) && !is_inside))
1660         {
1661           return 0;
1662         }
1663       if (sm->num_workers > 1)
1664         {
1665           del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1666                                           "nat44-out2in-worker-handoff";
1667           feature_name = "nat44-handoff-classify";
1668         }
1669       else
1670         {
1671           del_feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1672
1673           feature_name = "nat44-ed-classify";
1674         }
1675
1676       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1677       if (rv)
1678         return rv;
1679       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1680                                    sw_if_index, 0, 0, 0);
1681       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1682                                    0, 0);
1683     }
1684   else
1685     {
1686       if (sm->num_workers > 1)
1687         {
1688           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1689                                      "nat44-out2in-worker-handoff";
1690         }
1691       else
1692         {
1693           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1694         }
1695
1696       nat_validate_interface_counters (sm, sw_if_index);
1697       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1698       if (rv)
1699         return rv;
1700       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1701                                    0, 0);
1702
1703       pool_get (sm->interfaces, i);
1704       i->sw_if_index = sw_if_index;
1705       i->flags = 0;
1706     }
1707
1708   fib_index =
1709     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1710
1711   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1712
1713   if (!is_inside)
1714     {
1715       i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1716
1717       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1718       if (outside_fib)
1719         {
1720           outside_fib->refcount++;
1721         }
1722       else
1723         {
1724           vec_add2 (sm->outside_fibs, outside_fib, 1);
1725           outside_fib->fib_index = fib_index;
1726           outside_fib->refcount = 1;
1727         }
1728
1729       vec_foreach (ap, sm->addresses)
1730         {
1731           snat_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, 1);
1732         }
1733       pool_foreach (m, sm->static_mappings)
1734         {
1735           if (!(is_sm_addr_only (m->flags)) ||
1736               (m->local_addr.as_u32 == m->external_addr.as_u32))
1737             {
1738               continue;
1739             }
1740           snat_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index, 1);
1741         }
1742     }
1743   else
1744     {
1745       i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1746     }
1747
1748   return 0;
1749 }
1750
1751 int
1752 nat44_ed_del_interface (u32 sw_if_index, u8 is_inside)
1753 {
1754   const char *del_feature_name, *feature_name;
1755   snat_main_t *sm = &snat_main;
1756
1757   nat_outside_fib_t *outside_fib;
1758   snat_static_mapping_t *m;
1759   snat_interface_t *i;
1760   snat_address_t *ap;
1761   u32 fib_index;
1762   int rv;
1763
1764   if (!sm->enabled)
1765     {
1766       nat_log_err ("nat44 is disabled");
1767       return VNET_API_ERROR_UNSUPPORTED;
1768     }
1769
1770   i = nat44_ed_get_interface (sm->interfaces, sw_if_index);
1771   if (i == 0)
1772     {
1773       nat_log_err ("error interface couldn't be found");
1774       return VNET_API_ERROR_NO_SUCH_ENTRY;
1775     }
1776
1777   if (nat_interface_is_inside (i) && nat_interface_is_outside (i))
1778     {
1779       if (sm->num_workers > 1)
1780         {
1781           del_feature_name = "nat44-handoff-classify";
1782           feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
1783                                       "nat44-out2in-worker-handoff";
1784         }
1785       else
1786         {
1787           del_feature_name = "nat44-ed-classify";
1788           feature_name = !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1789         }
1790
1791       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1792       if (rv)
1793         {
1794           return rv;
1795         }
1796       vnet_feature_enable_disable ("ip4-unicast", del_feature_name,
1797                                    sw_if_index, 0, 0, 0);
1798       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 1,
1799                                    0, 0);
1800
1801       if (is_inside)
1802         {
1803           i->flags &= ~NAT_INTERFACE_FLAG_IS_INSIDE;
1804         }
1805       else
1806         {
1807           i->flags &= ~NAT_INTERFACE_FLAG_IS_OUTSIDE;
1808         }
1809     }
1810   else
1811     {
1812       if (sm->num_workers > 1)
1813         {
1814           feature_name = is_inside ? "nat44-in2out-worker-handoff" :
1815                                      "nat44-out2in-worker-handoff";
1816         }
1817       else
1818         {
1819           feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
1820         }
1821
1822       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
1823       if (rv)
1824         {
1825           return rv;
1826         }
1827       vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, 0,
1828                                    0, 0);
1829
1830       // remove interface
1831       pool_put (sm->interfaces, i);
1832     }
1833
1834   fib_index =
1835     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1836
1837   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
1838
1839   if (!is_inside)
1840     {
1841       outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1842       if (outside_fib)
1843         {
1844           outside_fib->refcount--;
1845           if (!outside_fib->refcount)
1846             {
1847               vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
1848             }
1849         }
1850
1851       vec_foreach (ap, sm->addresses)
1852         {
1853           snat_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, 0);
1854         }
1855
1856       pool_foreach (m, sm->static_mappings)
1857         {
1858           if (!(is_sm_addr_only (m->flags)) ||
1859               (m->local_addr.as_u32 == m->external_addr.as_u32))
1860             {
1861               continue;
1862             }
1863           snat_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index, 0);
1864         }
1865     }
1866
1867   return 0;
1868 }
1869
1870 int
1871 nat44_ed_add_output_interface (u32 sw_if_index)
1872 {
1873   snat_main_t *sm = &snat_main;
1874
1875   nat_outside_fib_t *outside_fib;
1876   snat_static_mapping_t *m;
1877   snat_interface_t *i;
1878   snat_address_t *ap;
1879   u32 fib_index;
1880   int rv;
1881
1882   if (!sm->enabled)
1883     {
1884       nat_log_err ("nat44 is disabled");
1885       return VNET_API_ERROR_UNSUPPORTED;
1886     }
1887
1888   if (nat44_ed_get_interface (sm->interfaces, sw_if_index))
1889     {
1890       nat_log_err ("error interface already configured");
1891       return VNET_API_ERROR_VALUE_EXIST;
1892     }
1893
1894   if (nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index))
1895     {
1896       nat_log_err ("error interface already configured");
1897       return VNET_API_ERROR_VALUE_EXIST;
1898     }
1899
1900   if (sm->num_workers > 1)
1901     {
1902       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1903       if (rv)
1904         {
1905           return rv;
1906         }
1907
1908       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1909       if (rv)
1910         {
1911           return rv;
1912         }
1913
1914       vnet_feature_enable_disable (
1915         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 1, 0, 0);
1916       vnet_feature_enable_disable ("ip4-output",
1917                                    "nat44-in2out-output-worker-handoff",
1918                                    sw_if_index, 1, 0, 0);
1919     }
1920   else
1921     {
1922       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
1923       if (rv)
1924         {
1925           return rv;
1926         }
1927
1928       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 1);
1929       if (rv)
1930         {
1931           return rv;
1932         }
1933
1934       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
1935                                    sw_if_index, 1, 0, 0);
1936       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
1937                                    sw_if_index, 1, 0, 0);
1938     }
1939
1940   nat_validate_interface_counters (sm, sw_if_index);
1941
1942   pool_get (sm->output_feature_interfaces, i);
1943   i->sw_if_index = sw_if_index;
1944   i->flags = 0;
1945   i->flags |= NAT_INTERFACE_FLAG_IS_INSIDE;
1946   i->flags |= NAT_INTERFACE_FLAG_IS_OUTSIDE;
1947
1948   fib_index =
1949     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
1950   update_per_vrf_sessions_vec (fib_index, 0 /*is_del*/);
1951
1952   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
1953   if (outside_fib)
1954     {
1955       outside_fib->refcount++;
1956     }
1957   else
1958     {
1959       vec_add2 (sm->outside_fibs, outside_fib, 1);
1960       outside_fib->fib_index = fib_index;
1961       outside_fib->refcount = 1;
1962     }
1963
1964   vec_foreach (ap, sm->addresses)
1965     {
1966       snat_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, 1);
1967     }
1968
1969   pool_foreach (m, sm->static_mappings)
1970     {
1971       if (!((is_sm_addr_only (m->flags))) ||
1972           (m->local_addr.as_u32 == m->external_addr.as_u32))
1973         {
1974           continue;
1975         }
1976       snat_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index, 1);
1977     }
1978
1979   return 0;
1980 }
1981
1982 int
1983 nat44_ed_del_output_interface (u32 sw_if_index)
1984 {
1985   snat_main_t *sm = &snat_main;
1986
1987   nat_outside_fib_t *outside_fib;
1988   snat_static_mapping_t *m;
1989   snat_interface_t *i;
1990   snat_address_t *ap;
1991   u32 fib_index;
1992   int rv;
1993
1994   if (!sm->enabled)
1995     {
1996       nat_log_err ("nat44 is disabled");
1997       return VNET_API_ERROR_UNSUPPORTED;
1998     }
1999
2000   i = nat44_ed_get_interface (sm->output_feature_interfaces, sw_if_index);
2001   if (!i)
2002     {
2003       nat_log_err ("error interface couldn't be found");
2004       return VNET_API_ERROR_NO_SUCH_ENTRY;
2005     }
2006
2007   if (sm->num_workers > 1)
2008     {
2009       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2010       if (rv)
2011         {
2012           return rv;
2013         }
2014
2015       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2016       if (rv)
2017         {
2018           return rv;
2019         }
2020
2021       vnet_feature_enable_disable (
2022         "ip4-unicast", "nat44-out2in-worker-handoff", sw_if_index, 0, 0, 0);
2023       vnet_feature_enable_disable ("ip4-output",
2024                                    "nat44-in2out-output-worker-handoff",
2025                                    sw_if_index, 0, 0, 0);
2026     }
2027   else
2028     {
2029       rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
2030       if (rv)
2031         {
2032           return rv;
2033         }
2034
2035       rv = ip4_sv_reass_output_enable_disable_with_refcnt (sw_if_index, 0);
2036       if (rv)
2037         {
2038           return rv;
2039         }
2040
2041       vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
2042                                    sw_if_index, 0, 0, 0);
2043       vnet_feature_enable_disable ("ip4-output", "nat-pre-in2out-output",
2044                                    sw_if_index, 0, 0, 0);
2045     }
2046
2047   // remove interface
2048   pool_put (sm->output_feature_interfaces, i);
2049
2050   fib_index =
2051     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index);
2052   update_per_vrf_sessions_vec (fib_index, 1 /*is_del*/);
2053
2054   outside_fib = nat44_ed_get_outside_fib (sm->outside_fibs, fib_index);
2055   if (outside_fib)
2056     {
2057       outside_fib->refcount--;
2058       if (!outside_fib->refcount)
2059         {
2060           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2061         }
2062     }
2063
2064   vec_foreach (ap, sm->addresses)
2065     {
2066       snat_add_del_addr_to_fib (&ap->addr, 32, sw_if_index, 0);
2067     }
2068
2069   pool_foreach (m, sm->static_mappings)
2070     {
2071       if (!((is_sm_addr_only (m->flags))) ||
2072           (m->local_addr.as_u32 == m->external_addr.as_u32))
2073         {
2074           continue;
2075         }
2076       snat_add_del_addr_to_fib (&m->external_addr, 32, sw_if_index, 0);
2077     }
2078
2079   return 0;
2080 }
2081
2082 int
2083 snat_set_workers (uword * bitmap)
2084 {
2085   snat_main_t *sm = &snat_main;
2086   int i, j = 0;
2087
2088   if (sm->num_workers < 2)
2089     return VNET_API_ERROR_FEATURE_DISABLED;
2090
2091   if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
2092     return VNET_API_ERROR_INVALID_WORKER;
2093
2094   vec_free (sm->workers);
2095   clib_bitmap_foreach (i, bitmap)
2096     {
2097       vec_add1(sm->workers, i);
2098       sm->per_thread_data[sm->first_worker_index + i].snat_thread_index = j;
2099       sm->per_thread_data[sm->first_worker_index + i].thread_index = i;
2100       j++;
2101     }
2102
2103   sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
2104
2105   return 0;
2106 }
2107
2108 int
2109 nat44_ed_set_frame_queue_nelts (u32 frame_queue_nelts)
2110 {
2111   fail_if_enabled ();
2112   snat_main_t *sm = &snat_main;
2113   sm->frame_queue_nelts = frame_queue_nelts;
2114   return 0;
2115 }
2116
2117 static void
2118 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2119                          u32 sw_if_index, u32 new_fib_index,
2120                          u32 old_fib_index)
2121 {
2122   snat_main_t *sm = &snat_main;
2123   nat_outside_fib_t *outside_fib;
2124   snat_interface_t *i;
2125   u8 is_add = 1;
2126   u8 match = 0;
2127
2128   if (!sm->enabled || (new_fib_index == old_fib_index)
2129       || (!vec_len (sm->outside_fibs)))
2130     {
2131       return;
2132     }
2133
2134   pool_foreach (i, sm->interfaces)
2135      {
2136       if (i->sw_if_index == sw_if_index)
2137         {
2138           if (!(nat_interface_is_outside (i)))
2139             return;
2140           match = 1;
2141         }
2142     }
2143
2144   pool_foreach (i, sm->output_feature_interfaces)
2145      {
2146       if (i->sw_if_index == sw_if_index)
2147         {
2148           if (!(nat_interface_is_outside (i)))
2149             return;
2150           match = 1;
2151         }
2152     }
2153
2154   if (!match)
2155     return;
2156
2157   vec_foreach (outside_fib, sm->outside_fibs)
2158   {
2159     if (outside_fib->fib_index == old_fib_index)
2160       {
2161         outside_fib->refcount--;
2162         if (!outside_fib->refcount)
2163           vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
2164         break;
2165       }
2166   }
2167
2168   vec_foreach (outside_fib, sm->outside_fibs)
2169   {
2170     if (outside_fib->fib_index == new_fib_index)
2171       {
2172         outside_fib->refcount++;
2173         is_add = 0;
2174         break;
2175       }
2176   }
2177
2178   if (is_add)
2179     {
2180       vec_add2 (sm->outside_fibs, outside_fib, 1);
2181       outside_fib->refcount = 1;
2182       outside_fib->fib_index = new_fib_index;
2183     }
2184 }
2185
2186 static void
2187 snat_update_outside_fib (ip4_main_t * im, uword opaque,
2188                          u32 sw_if_index, u32 new_fib_index,
2189                          u32 old_fib_index);
2190
2191 static void
2192 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
2193                                        uword opaque,
2194                                        u32 sw_if_index,
2195                                        ip4_address_t * address,
2196                                        u32 address_length,
2197                                        u32 if_address_index, u32 is_delete);
2198
2199 static void
2200 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
2201                                  uword opaque,
2202                                  u32 sw_if_index,
2203                                  ip4_address_t * address,
2204                                  u32 address_length,
2205                                  u32 if_address_index, u32 is_delete);
2206
2207 void
2208 test_key_calc_split ()
2209 {
2210   ip4_address_t l_addr;
2211   l_addr.as_u8[0] = 1;
2212   l_addr.as_u8[1] = 1;
2213   l_addr.as_u8[2] = 1;
2214   l_addr.as_u8[3] = 1;
2215   ip4_address_t r_addr;
2216   r_addr.as_u8[0] = 2;
2217   r_addr.as_u8[1] = 2;
2218   r_addr.as_u8[2] = 2;
2219   r_addr.as_u8[3] = 2;
2220   u16 l_port = 40001;
2221   u16 r_port = 40301;
2222   u8 proto = 9;
2223   u32 fib_index = 9000001;
2224   u32 thread_index = 3000000001;
2225   u32 session_index = 3000000221;
2226   clib_bihash_kv_16_8_t kv;
2227   init_ed_kv (&kv, l_addr, l_port, r_addr, r_port, fib_index, proto,
2228               thread_index, session_index);
2229   ip4_address_t l_addr2;
2230   ip4_address_t r_addr2;
2231   clib_memset (&l_addr2, 0, sizeof (l_addr2));
2232   clib_memset (&r_addr2, 0, sizeof (r_addr2));
2233   u16 l_port2 = 0;
2234   u16 r_port2 = 0;
2235   u8 proto2 = 0;
2236   u32 fib_index2 = 0;
2237   split_ed_kv (&kv, &l_addr2, &r_addr2, &proto2, &fib_index2, &l_port2,
2238                &r_port2);
2239   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2240   ASSERT (r_addr.as_u32 == r_addr2.as_u32);
2241   ASSERT (l_port == l_port2);
2242   ASSERT (r_port == r_port2);
2243   ASSERT (proto == proto2);
2244   ASSERT (fib_index == fib_index2);
2245   ASSERT (thread_index == ed_value_get_thread_index (&kv));
2246   ASSERT (session_index == ed_value_get_session_index (&kv));
2247
2248   fib_index = 7001;
2249   proto = 5;
2250   nat_protocol_t proto3 = ~0;
2251   u64 key = calc_nat_key (l_addr, l_port, fib_index, proto);
2252   split_nat_key (key, &l_addr2, &l_port2, &fib_index2, &proto3);
2253   ASSERT (l_addr.as_u32 == l_addr2.as_u32);
2254   ASSERT (l_port == l_port2);
2255   ASSERT (proto == proto3);
2256   ASSERT (fib_index == fib_index2);
2257 }
2258
2259 static clib_error_t *
2260 nat_ip_table_add_del (vnet_main_t * vnm, u32 table_id, u32 is_add)
2261 {
2262   u32 fib_index;
2263   if (!is_add)
2264     {
2265       fib_index = ip4_fib_index_from_table_id (table_id);
2266       if (fib_index != ~0)
2267         {
2268           expire_per_vrf_sessions (fib_index);
2269         }
2270     }
2271   return 0;
2272 }
2273
2274 VNET_IP_TABLE_ADD_DEL_FUNCTION (nat_ip_table_add_del);
2275
2276 void
2277 nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
2278 {
2279   vlib_node_t *node;
2280
2281   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-out2in");
2282   sm->out2in_node_index = node->index;
2283
2284   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out");
2285   sm->in2out_node_index = node->index;
2286
2287   node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-in2out-output");
2288   sm->in2out_output_node_index = node->index;
2289 }
2290
2291 #define nat_validate_simple_counter(c, i)                                     \
2292   do                                                                          \
2293     {                                                                         \
2294       vlib_validate_simple_counter (&c, i);                                   \
2295       vlib_zero_simple_counter (&c, i);                                       \
2296     }                                                                         \
2297   while (0);
2298
2299 #define nat_init_simple_counter(c, n, sn)                                     \
2300   do                                                                          \
2301     {                                                                         \
2302       c.name = n;                                                             \
2303       c.stat_segment_name = sn;                                               \
2304       nat_validate_simple_counter (c, 0);                                     \
2305     }                                                                         \
2306   while (0);
2307
2308 static_always_inline void
2309 nat_validate_interface_counters (snat_main_t *sm, u32 sw_if_index)
2310 {
2311 #define _(x)                                                                  \
2312   nat_validate_simple_counter (sm->counters.fastpath.in2out.x, sw_if_index);  \
2313   nat_validate_simple_counter (sm->counters.fastpath.out2in.x, sw_if_index);  \
2314   nat_validate_simple_counter (sm->counters.slowpath.in2out.x, sw_if_index);  \
2315   nat_validate_simple_counter (sm->counters.slowpath.out2in.x, sw_if_index);
2316   foreach_nat_counter;
2317 #undef _
2318   nat_validate_simple_counter (sm->counters.hairpinning, sw_if_index);
2319 }
2320
2321 static clib_error_t *
2322 nat_init (vlib_main_t * vm)
2323 {
2324   snat_main_t *sm = &snat_main;
2325   vlib_thread_main_t *tm = vlib_get_thread_main ();
2326   vlib_thread_registration_t *tr;
2327   ip4_add_del_interface_address_callback_t cbi = { 0 };
2328   ip4_table_bind_callback_t cbt = { 0 };
2329   u32 i, num_threads = 0;
2330   uword *p, *bitmap = 0;
2331
2332   clib_memset (sm, 0, sizeof (*sm));
2333
2334   // required
2335   sm->vnet_main = vnet_get_main ();
2336   // convenience
2337   sm->ip4_main = &ip4_main;
2338   sm->api_main = vlibapi_get_main ();
2339   sm->ip4_lookup_main = &ip4_main.lookup_main;
2340
2341   // frame queue indices used for handoff
2342   sm->fq_out2in_index = ~0;
2343   sm->fq_in2out_index = ~0;
2344   sm->fq_in2out_output_index = ~0;
2345
2346   sm->log_level = NAT_LOG_ERROR;
2347
2348   nat44_set_node_indexes (sm, vm);
2349
2350   sm->log_class = vlib_log_register_class ("nat", 0);
2351   nat_ipfix_logging_init (vm);
2352
2353   nat_init_simple_counter (sm->total_sessions, "total-sessions",
2354                            "/nat44-ed/total-sessions");
2355   sm->max_cfg_sessions_gauge = stat_segment_new_entry (
2356     (u8 *) "/nat44-ed/max-cfg-sessions", STAT_DIR_TYPE_SCALAR_INDEX);
2357
2358 #define _(x)                                                                  \
2359   nat_init_simple_counter (sm->counters.fastpath.in2out.x, #x,                \
2360                            "/nat44-ed/in2out/fastpath/" #x);                  \
2361   nat_init_simple_counter (sm->counters.fastpath.out2in.x, #x,                \
2362                            "/nat44-ed/out2in/fastpath/" #x);                  \
2363   nat_init_simple_counter (sm->counters.slowpath.in2out.x, #x,                \
2364                            "/nat44-ed/in2out/slowpath/" #x);                  \
2365   nat_init_simple_counter (sm->counters.slowpath.out2in.x, #x,                \
2366                            "/nat44-ed/out2in/slowpath/" #x);
2367   foreach_nat_counter;
2368 #undef _
2369   nat_init_simple_counter (sm->counters.hairpinning, "hairpinning",
2370                            "/nat44-ed/hairpinning");
2371
2372   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
2373   if (p)
2374     {
2375       tr = (vlib_thread_registration_t *) p[0];
2376       if (tr)
2377         {
2378           sm->num_workers = tr->count;
2379           sm->first_worker_index = tr->first_index;
2380         }
2381     }
2382   num_threads = tm->n_vlib_mains - 1;
2383   sm->port_per_thread = 0xffff - 1024;
2384   vec_validate (sm->per_thread_data, num_threads);
2385
2386   /* Use all available workers by default */
2387   if (sm->num_workers > 1)
2388     {
2389       for (i = 0; i < sm->num_workers; i++)
2390         bitmap = clib_bitmap_set (bitmap, i, 1);
2391       snat_set_workers (bitmap);
2392       clib_bitmap_free (bitmap);
2393     }
2394   else
2395     {
2396       sm->per_thread_data[0].snat_thread_index = 0;
2397     }
2398
2399   /* callbacks to call when interface address changes. */
2400   cbi.function = snat_ip4_add_del_interface_address_cb;
2401   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2402   cbi.function = nat_ip4_add_del_addr_only_sm_cb;
2403   vec_add1 (sm->ip4_main->add_del_interface_address_callbacks, cbi);
2404
2405   /* callbacks to call when interface to table biding changes */
2406   cbt.function = snat_update_outside_fib;
2407   vec_add1 (sm->ip4_main->table_bind_callbacks, cbt);
2408
2409   sm->fib_src_low =
2410     fib_source_allocate ("nat-low", FIB_SOURCE_PRIORITY_LOW,
2411                          FIB_SOURCE_BH_SIMPLE);
2412   sm->fib_src_hi =
2413     fib_source_allocate ("nat-hi", FIB_SOURCE_PRIORITY_HI,
2414                          FIB_SOURCE_BH_SIMPLE);
2415
2416   nat_affinity_init (vm);
2417   test_key_calc_split ();
2418
2419   return nat44_api_hookup (vm);
2420 }
2421
2422 VLIB_INIT_FUNCTION (nat_init);
2423
2424 int
2425 nat44_plugin_enable (nat44_config_t c)
2426 {
2427   snat_main_t *sm = &snat_main;
2428
2429   fail_if_enabled ();
2430
2431   if (c.static_mapping_only && !c.connection_tracking)
2432     {
2433       nat_log_err ("unsupported combination of configuration");
2434       return 1;
2435     }
2436
2437   sm->static_mapping_only = c.static_mapping_only;
2438   sm->static_mapping_connection_tracking = c.connection_tracking;
2439
2440   sm->forwarding_enabled = 0;
2441   sm->mss_clamping = 0;
2442   sm->pat = (!c.static_mapping_only ||
2443              (c.static_mapping_only && c.connection_tracking));
2444
2445   if (!c.sessions)
2446     c.sessions = 63 * 1024;
2447
2448   sm->max_translations_per_thread = c.sessions;
2449   stat_segment_set_state_counter (sm->max_cfg_sessions_gauge,
2450                                   sm->max_translations_per_thread);
2451   sm->translation_buckets = nat_calc_bihash_buckets (c.sessions);
2452
2453   vec_add1 (sm->max_translations_per_fib, sm->max_translations_per_thread);
2454
2455   sm->inside_vrf_id = c.inside_vrf;
2456   sm->inside_fib_index =
2457     fib_table_find_or_create_and_lock
2458     (FIB_PROTOCOL_IP4, c.inside_vrf, sm->fib_src_hi);
2459
2460   sm->outside_vrf_id = c.outside_vrf;
2461   sm->outside_fib_index = fib_table_find_or_create_and_lock (
2462     FIB_PROTOCOL_IP4, c.outside_vrf, sm->fib_src_hi);
2463
2464   nat44_ed_db_init (sm->max_translations_per_thread, sm->translation_buckets);
2465
2466   nat_affinity_enable ();
2467
2468   nat_reset_timeouts (&sm->timeouts);
2469
2470   vlib_zero_simple_counter (&sm->total_sessions, 0);
2471
2472   if (!sm->frame_queue_nelts)
2473     {
2474       sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
2475     }
2476
2477   if (sm->num_workers > 1)
2478     {
2479       if (sm->fq_in2out_index == ~0)
2480         {
2481           sm->fq_in2out_index = vlib_frame_queue_main_init (
2482             sm->in2out_node_index, sm->frame_queue_nelts);
2483         }
2484       if (sm->fq_out2in_index == ~0)
2485         {
2486           sm->fq_out2in_index = vlib_frame_queue_main_init (
2487             sm->out2in_node_index, sm->frame_queue_nelts);
2488         }
2489       if (sm->fq_in2out_output_index == ~0)
2490         {
2491           sm->fq_in2out_output_index = vlib_frame_queue_main_init (
2492             sm->in2out_output_node_index, sm->frame_queue_nelts);
2493         }
2494     }
2495
2496   sm->enabled = 1;
2497   sm->rconfig = c;
2498
2499   return 0;
2500 }
2501
2502 void
2503 nat44_addresses_free (snat_address_t ** addresses)
2504 {
2505   snat_address_t *ap;
2506   vec_foreach (ap, *addresses)
2507     {
2508     #define _(N, i, n, s) \
2509       vec_free (ap->busy_##n##_ports_per_thread);
2510       foreach_nat_protocol
2511     #undef _
2512     }
2513   vec_free (*addresses);
2514   *addresses = 0;
2515 }
2516
2517 int
2518 nat44_plugin_disable ()
2519 {
2520   snat_main_t *sm = &snat_main;
2521   snat_interface_t *i, *pool;
2522   int error = 0;
2523
2524   fail_if_disabled ();
2525
2526   pool = pool_dup (sm->interfaces);
2527   pool_foreach (i, pool)
2528     {
2529       if (nat_interface_is_inside (i))
2530         {
2531           error = nat44_ed_del_interface (i->sw_if_index, 1);
2532         }
2533       if (nat_interface_is_outside (i))
2534         {
2535           error = nat44_ed_del_interface (i->sw_if_index, 0);
2536         }
2537       if (error)
2538         {
2539           nat_log_err ("error occurred while removing interface %u",
2540                        i->sw_if_index);
2541         }
2542     }
2543   pool_free (sm->interfaces);
2544   pool_free (pool);
2545   sm->interfaces = 0;
2546
2547   pool = pool_dup (sm->output_feature_interfaces);
2548   pool_foreach (i, pool)
2549     {
2550       error = nat44_ed_del_output_interface (i->sw_if_index);
2551       if (error)
2552         {
2553           nat_log_err ("error occurred while removing interface %u",
2554                        i->sw_if_index);
2555         }
2556     }
2557   pool_free (sm->output_feature_interfaces);
2558   pool_free (pool);
2559   sm->output_feature_interfaces = 0;
2560
2561   vec_free (sm->max_translations_per_fib);
2562
2563   nat44_ed_db_free ();
2564
2565   nat44_addresses_free (&sm->addresses);
2566   nat44_addresses_free (&sm->twice_nat_addresses);
2567
2568   vec_free (sm->to_resolve);
2569   vec_free (sm->auto_add_sw_if_indices);
2570   vec_free (sm->auto_add_sw_if_indices_twice_nat);
2571
2572   sm->to_resolve = 0;
2573   sm->auto_add_sw_if_indices = 0;
2574   sm->auto_add_sw_if_indices_twice_nat = 0;
2575
2576   sm->forwarding_enabled = 0;
2577
2578   sm->enabled = 0;
2579   clib_memset (&sm->rconfig, 0, sizeof (sm->rconfig));
2580
2581   return 0;
2582 }
2583
2584 void
2585 nat44_ed_forwarding_enable_disable (u8 is_enable)
2586 {
2587   snat_main_per_thread_data_t *tsm;
2588   snat_main_t *sm = &snat_main;
2589   snat_session_t *s;
2590
2591   u32 *ses_to_be_removed = 0, *ses_index;
2592
2593   sm->forwarding_enabled = is_enable != 0;
2594
2595   if (!sm->enabled || is_enable)
2596     {
2597       return;
2598     }
2599
2600   vec_foreach (tsm, sm->per_thread_data)
2601     {
2602       pool_foreach (s, tsm->sessions)
2603         {
2604           if (is_fwd_bypass_session (s))
2605             {
2606               vec_add1 (ses_to_be_removed, s - tsm->sessions);
2607             }
2608         }
2609       vec_foreach (ses_index, ses_to_be_removed)
2610         {
2611           s = pool_elt_at_index (tsm->sessions, ses_index[0]);
2612           nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
2613           nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
2614         }
2615
2616       vec_free (ses_to_be_removed);
2617     }
2618 }
2619
2620 void
2621 snat_free_outside_address_and_port (snat_address_t *addresses,
2622                                     u32 thread_index, ip4_address_t *addr,
2623                                     u16 port, nat_protocol_t protocol)
2624 {
2625   snat_main_t *sm = &snat_main;
2626   snat_address_t *a;
2627   u32 address_index;
2628   u16 port_host_byte_order = clib_net_to_host_u16 (port);
2629
2630   for (address_index = 0; address_index < vec_len (addresses);
2631        address_index++)
2632     {
2633       if (addresses[address_index].addr.as_u32 == addr->as_u32)
2634         break;
2635     }
2636
2637   ASSERT (address_index < vec_len (addresses));
2638
2639   a = addresses + address_index;
2640
2641   switch (protocol)
2642     {
2643 #define _(N, i, n, s) \
2644     case NAT_PROTOCOL_##N: \
2645       ASSERT (a->busy_##n##_port_refcounts[port_host_byte_order] >= 1); \
2646       --a->busy_##n##_port_refcounts[port_host_byte_order]; \
2647       a->busy_##n##_ports--; \
2648       a->busy_##n##_ports_per_thread[thread_index]--; \
2649       break;
2650       foreach_nat_protocol
2651 #undef _
2652         default : nat_elog_info (sm, "unknown protocol");
2653       return;
2654     }
2655 }
2656
2657 int
2658 snat_static_mapping_match (vlib_main_t *vm, snat_main_t *sm,
2659                            ip4_address_t match_addr, u16 match_port,
2660                            u32 match_fib_index, nat_protocol_t match_protocol,
2661                            ip4_address_t *mapping_addr, u16 *mapping_port,
2662                            u32 *mapping_fib_index, u8 by_external,
2663                            u8 *is_addr_only, twice_nat_type_t *twice_nat,
2664                            lb_nat_type_t *lb, ip4_address_t *ext_host_addr,
2665                            u8 *is_identity_nat, snat_static_mapping_t **out)
2666 {
2667   clib_bihash_kv_8_8_t kv, value;
2668   clib_bihash_8_8_t *mapping_hash;
2669   snat_static_mapping_t *m;
2670   u32 rand, lo = 0, hi, mid, *tmp = 0, i;
2671   nat44_lb_addr_port_t *local;
2672   u8 backend_index;
2673
2674   if (!by_external)
2675     {
2676       mapping_hash = &sm->static_mapping_by_local;
2677       init_nat_k (&kv, match_addr, match_port, match_fib_index,
2678                   match_protocol);
2679       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2680         {
2681           /* Try address only mapping */
2682           init_nat_k (&kv, match_addr, 0, match_fib_index, 0);
2683           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2684             return 1;
2685         }
2686     }
2687   else
2688     {
2689       mapping_hash = &sm->static_mapping_by_external;
2690       init_nat_k (&kv, match_addr, match_port, 0, match_protocol);
2691       if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2692         {
2693           /* Try address only mapping */
2694           init_nat_k (&kv, match_addr, 0, 0, 0);
2695           if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
2696             return 1;
2697         }
2698     }
2699
2700   m = pool_elt_at_index (sm->static_mappings, value.value);
2701
2702   if (by_external)
2703     {
2704       if (is_sm_lb (m->flags))
2705         {
2706           if (PREDICT_FALSE (lb != 0))
2707             *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
2708           if (m->affinity && !nat_affinity_find_and_lock (
2709                                vm, ext_host_addr[0], match_addr,
2710                                match_protocol, match_port, &backend_index))
2711             {
2712               local = pool_elt_at_index (m->locals, backend_index);
2713               *mapping_addr = local->addr;
2714               *mapping_port = local->port;
2715               *mapping_fib_index = local->fib_index;
2716               goto end;
2717             }
2718           // pick locals matching this worker
2719           if (PREDICT_FALSE (sm->num_workers > 1))
2720             {
2721               u32 thread_index = vlib_get_thread_index ();
2722               pool_foreach_index (i, m->locals)
2723                {
2724                 local = pool_elt_at_index (m->locals, i);
2725
2726                 ip4_header_t ip = {
2727                   .src_address = local->addr,
2728                 };
2729
2730                 if (nat44_ed_get_in2out_worker_index (0, &ip, m->fib_index,
2731                                                       0) == thread_index)
2732                   {
2733                     vec_add1 (tmp, i);
2734                   }
2735                }
2736               ASSERT (vec_len (tmp) != 0);
2737             }
2738           else
2739             {
2740               pool_foreach_index (i, m->locals)
2741                {
2742                 vec_add1 (tmp, i);
2743               }
2744             }
2745           hi = vec_len (tmp) - 1;
2746           local = pool_elt_at_index (m->locals, tmp[hi]);
2747           rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
2748           while (lo < hi)
2749             {
2750               mid = ((hi - lo) >> 1) + lo;
2751               local = pool_elt_at_index (m->locals, tmp[mid]);
2752               (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
2753             }
2754           local = pool_elt_at_index (m->locals, tmp[lo]);
2755           if (!(local->prefix >= rand))
2756             return 1;
2757           *mapping_addr = local->addr;
2758           *mapping_port = local->port;
2759           *mapping_fib_index = local->fib_index;
2760           if (m->affinity)
2761             {
2762               if (nat_affinity_create_and_lock (ext_host_addr[0], match_addr,
2763                                                 match_protocol, match_port,
2764                                                 tmp[lo], m->affinity,
2765                                                 m->affinity_per_service_list_head_index))
2766                 nat_elog_info (sm, "create affinity record failed");
2767             }
2768           vec_free (tmp);
2769         }
2770       else
2771         {
2772           if (PREDICT_FALSE (lb != 0))
2773             *lb = NO_LB_NAT;
2774           *mapping_fib_index = m->fib_index;
2775           *mapping_addr = m->local_addr;
2776           /* Address only mapping doesn't change port */
2777           *mapping_port =
2778             is_sm_addr_only (m->flags) ? match_port : m->local_port;
2779         }
2780     }
2781   else
2782     {
2783       *mapping_addr = m->external_addr;
2784       /* Address only mapping doesn't change port */
2785       *mapping_port =
2786         is_sm_addr_only (m->flags) ? match_port : m->external_port;
2787       *mapping_fib_index = sm->outside_fib_index;
2788     }
2789
2790 end:
2791   if (PREDICT_FALSE (is_addr_only != 0))
2792     *is_addr_only = is_sm_addr_only (m->flags);
2793
2794   if (PREDICT_FALSE (twice_nat != 0))
2795     {
2796       *twice_nat = TWICE_NAT_DISABLED;
2797
2798       if (is_sm_twice_nat (m->flags))
2799         {
2800           *twice_nat = TWICE_NAT;
2801         }
2802       else if (is_sm_self_twice_nat (m->flags))
2803         {
2804           *twice_nat = TWICE_NAT_SELF;
2805         }
2806     }
2807
2808   if (PREDICT_FALSE (is_identity_nat != 0))
2809     *is_identity_nat = is_sm_identity_nat (m->flags);
2810
2811   if (out != 0)
2812     *out = m;
2813
2814   return 0;
2815 }
2816
2817 u32
2818 nat44_ed_get_in2out_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2819                                   u32 rx_fib_index, u8 is_output)
2820 {
2821   snat_main_t *sm = &snat_main;
2822   u32 next_worker_index = sm->first_worker_index;
2823   u32 hash;
2824
2825   clib_bihash_kv_16_8_t kv16, value16;
2826
2827   u32 fib_index = rx_fib_index;
2828   if (b)
2829     {
2830       if (PREDICT_FALSE (is_output))
2831         {
2832           fib_index = sm->outside_fib_index;
2833           nat_outside_fib_t *outside_fib;
2834           fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2835           fib_prefix_t pfx = {
2836                   .fp_proto = FIB_PROTOCOL_IP4,
2837                   .fp_len = 32,
2838                   .fp_addr = {
2839                           .ip4.as_u32 = ip->dst_address.as_u32,
2840                   } ,
2841           };
2842
2843           switch (vec_len (sm->outside_fibs))
2844             {
2845             case 0:
2846               fib_index = sm->outside_fib_index;
2847               break;
2848             case 1:
2849               fib_index = sm->outside_fibs[0].fib_index;
2850               break;
2851             default:
2852               vec_foreach (outside_fib, sm->outside_fibs)
2853                 {
2854                   fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2855                   if (FIB_NODE_INDEX_INVALID != fei)
2856                     {
2857                       if (fib_entry_get_resolving_interface (fei) != ~0)
2858                         {
2859                           fib_index = outside_fib->fib_index;
2860                           break;
2861                         }
2862                     }
2863                 }
2864               break;
2865             }
2866         }
2867
2868       init_ed_k (&kv16, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2869                  ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2870                  fib_index, ip->protocol);
2871
2872       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2873         {
2874           next_worker_index = ed_value_get_thread_index (&value16);
2875           vnet_buffer2 (b)->nat.cached_session_index =
2876             ed_value_get_session_index (&value16);
2877           goto out;
2878         }
2879
2880       // dst NAT
2881       init_ed_k (&kv16, ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2882                  ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2883                  rx_fib_index, ip->protocol);
2884       if (!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16))
2885         {
2886           next_worker_index = ed_value_get_thread_index (&value16);
2887           vnet_buffer2 (b)->nat.cached_dst_nat_session_index =
2888             ed_value_get_session_index (&value16);
2889           goto out;
2890         }
2891     }
2892
2893   hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
2894     (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
2895
2896   if (PREDICT_TRUE (is_pow2 (_vec_len (sm->workers))))
2897     next_worker_index += sm->workers[hash & (_vec_len (sm->workers) - 1)];
2898   else
2899     next_worker_index += sm->workers[hash % _vec_len (sm->workers)];
2900
2901 out:
2902   if (PREDICT_TRUE (!is_output))
2903     {
2904       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT", next_worker_index,
2905                               rx_fib_index,
2906                               clib_net_to_host_u32 (ip->src_address.as_u32),
2907                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2908     }
2909   else
2910     {
2911       nat_elog_debug_handoff (sm, "HANDOFF IN2OUT-OUTPUT-FEATURE",
2912                               next_worker_index, rx_fib_index,
2913                               clib_net_to_host_u32 (ip->src_address.as_u32),
2914                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2915     }
2916
2917   return next_worker_index;
2918 }
2919
2920 u32
2921 nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
2922                                   u32 rx_fib_index, u8 is_output)
2923 {
2924   snat_main_t *sm = &snat_main;
2925   clib_bihash_kv_8_8_t kv, value;
2926   clib_bihash_kv_16_8_t kv16, value16;
2927
2928   u32 proto, next_worker_index = 0;
2929   u16 port;
2930   snat_static_mapping_t *m;
2931   u32 hash;
2932
2933   proto = ip_proto_to_nat_proto (ip->protocol);
2934
2935   if (PREDICT_FALSE (proto == NAT_PROTOCOL_ICMP))
2936     {
2937       ip4_address_t lookup_saddr, lookup_daddr;
2938       u16 lookup_sport, lookup_dport;
2939       u8 lookup_protocol;
2940       if (!nat_get_icmp_session_lookup_values (
2941             b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
2942             &lookup_protocol))
2943         {
2944           init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
2945                      lookup_dport, rx_fib_index, lookup_protocol);
2946           if (PREDICT_TRUE (
2947                 !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
2948             {
2949               next_worker_index = ed_value_get_thread_index (&value16);
2950               nat_elog_debug_handoff (
2951                 sm, "HANDOFF OUT2IN (session)", next_worker_index,
2952                 rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
2953                 clib_net_to_host_u32 (ip->dst_address.as_u32));
2954               return next_worker_index;
2955             }
2956         }
2957     }
2958
2959   init_ed_k (&kv16, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
2960              ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
2961              rx_fib_index, ip->protocol);
2962
2963   if (PREDICT_TRUE (
2964         !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
2965     {
2966       vnet_buffer2 (b)->nat.cached_session_index =
2967         ed_value_get_session_index (&value16);
2968       next_worker_index = ed_value_get_thread_index (&value16);
2969       nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
2970                               next_worker_index, rx_fib_index,
2971                               clib_net_to_host_u32 (ip->src_address.as_u32),
2972                               clib_net_to_host_u32 (ip->dst_address.as_u32));
2973       return next_worker_index;
2974     }
2975
2976   /* first try static mappings without port */
2977   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
2978     {
2979       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
2980       if (!clib_bihash_search_8_8
2981           (&sm->static_mapping_by_external, &kv, &value))
2982         {
2983           m = pool_elt_at_index (sm->static_mappings, value.value);
2984           next_worker_index = m->workers[0];
2985           goto done;
2986         }
2987     }
2988
2989   /* unknown protocol */
2990   if (PREDICT_FALSE (proto == NAT_PROTOCOL_OTHER))
2991     {
2992       /* use current thread */
2993       next_worker_index = vlib_get_thread_index ();
2994       goto done;
2995     }
2996
2997   port = vnet_buffer (b)->ip.reass.l4_dst_port;
2998
2999   if (PREDICT_FALSE (ip->protocol == IP_PROTOCOL_ICMP))
3000     {
3001       udp_header_t *udp = ip4_next_header (ip);
3002       icmp46_header_t *icmp = (icmp46_header_t *) udp;
3003       icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3004       if (!icmp_type_is_error_message
3005           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
3006         port = vnet_buffer (b)->ip.reass.l4_src_port;
3007       else
3008         {
3009           /* if error message, then it's not fragmented and we can access it */
3010           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3011           proto = ip_proto_to_nat_proto (inner_ip->protocol);
3012           void *l4_header = ip4_next_header (inner_ip);
3013           switch (proto)
3014             {
3015             case NAT_PROTOCOL_ICMP:
3016               icmp = (icmp46_header_t *) l4_header;
3017               echo = (icmp_echo_header_t *) (icmp + 1);
3018               port = echo->identifier;
3019               break;
3020             case NAT_PROTOCOL_UDP:
3021             case NAT_PROTOCOL_TCP:
3022               port = ((tcp_udp_header_t *) l4_header)->src_port;
3023               break;
3024             default:
3025               next_worker_index = vlib_get_thread_index ();
3026               goto done;
3027             }
3028         }
3029     }
3030
3031   /* try static mappings with port */
3032   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
3033     {
3034       init_nat_k (&kv, ip->dst_address, port, 0, proto);
3035       if (!clib_bihash_search_8_8
3036           (&sm->static_mapping_by_external, &kv, &value))
3037         {
3038           m = pool_elt_at_index (sm->static_mappings, value.value);
3039           if (!is_sm_lb (m->flags))
3040             {
3041               next_worker_index = m->workers[0];
3042               goto done;
3043             }
3044
3045           hash = ip->src_address.as_u32 + (ip->src_address.as_u32 >> 8) +
3046             (ip->src_address.as_u32 >> 16) + (ip->src_address.as_u32 >> 24);
3047
3048           if (PREDICT_TRUE (is_pow2 (_vec_len (m->workers))))
3049             next_worker_index =
3050               m->workers[hash & (_vec_len (m->workers) - 1)];
3051           else
3052             next_worker_index = m->workers[hash % _vec_len (m->workers)];
3053           goto done;
3054         }
3055     }
3056
3057   /* worker by outside port */
3058   next_worker_index = sm->first_worker_index;
3059   next_worker_index +=
3060     sm->workers[(clib_net_to_host_u16 (port) - 1024) / sm->port_per_thread];
3061
3062 done:
3063   nat_elog_debug_handoff (sm, "HANDOFF OUT2IN", next_worker_index,
3064                           rx_fib_index,
3065                           clib_net_to_host_u32 (ip->src_address.as_u32),
3066                           clib_net_to_host_u32 (ip->dst_address.as_u32));
3067   return next_worker_index;
3068 }
3069
3070 u32
3071 nat44_get_max_session_limit ()
3072 {
3073   snat_main_t *sm = &snat_main;
3074   u32 max_limit = 0, len = 0;
3075
3076   for (; len < vec_len (sm->max_translations_per_fib); len++)
3077     {
3078       if (max_limit < sm->max_translations_per_fib[len])
3079         max_limit = sm->max_translations_per_fib[len];
3080     }
3081   return max_limit;
3082 }
3083
3084 int
3085 nat44_set_session_limit (u32 session_limit, u32 vrf_id)
3086 {
3087   snat_main_t *sm = &snat_main;
3088   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3089   u32 len = vec_len (sm->max_translations_per_fib);
3090
3091   if (len <= fib_index)
3092     {
3093       vec_validate (sm->max_translations_per_fib, fib_index + 1);
3094
3095       for (; len < vec_len (sm->max_translations_per_fib); len++)
3096         sm->max_translations_per_fib[len] = sm->max_translations_per_thread;
3097     }
3098
3099   sm->max_translations_per_fib[fib_index] = session_limit;
3100   return 0;
3101 }
3102
3103 int
3104 nat44_update_session_limit (u32 session_limit, u32 vrf_id)
3105 {
3106   snat_main_t *sm = &snat_main;
3107
3108   if (nat44_set_session_limit (session_limit, vrf_id))
3109     return 1;
3110   sm->max_translations_per_thread = nat44_get_max_session_limit ();
3111
3112   stat_segment_set_state_counter (sm->max_cfg_sessions_gauge,
3113                                   sm->max_translations_per_thread);
3114
3115   sm->translation_buckets =
3116     nat_calc_bihash_buckets (sm->max_translations_per_thread);
3117
3118   nat44_ed_sessions_clear ();
3119   return 0;
3120 }
3121
3122 static void
3123 nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
3124                          u32 translation_buckets)
3125 {
3126   dlist_elt_t *head;
3127
3128   pool_alloc (tsm->sessions, translations);
3129   pool_alloc (tsm->lru_pool, translations);
3130
3131   pool_get (tsm->lru_pool, head);
3132   tsm->tcp_trans_lru_head_index = head - tsm->lru_pool;
3133   clib_dlist_init (tsm->lru_pool, tsm->tcp_trans_lru_head_index);
3134
3135   pool_get (tsm->lru_pool, head);
3136   tsm->tcp_estab_lru_head_index = head - tsm->lru_pool;
3137   clib_dlist_init (tsm->lru_pool, tsm->tcp_estab_lru_head_index);
3138
3139   pool_get (tsm->lru_pool, head);
3140   tsm->udp_lru_head_index = head - tsm->lru_pool;
3141   clib_dlist_init (tsm->lru_pool, tsm->udp_lru_head_index);
3142
3143   pool_get (tsm->lru_pool, head);
3144   tsm->icmp_lru_head_index = head - tsm->lru_pool;
3145   clib_dlist_init (tsm->lru_pool, tsm->icmp_lru_head_index);
3146
3147   pool_get (tsm->lru_pool, head);
3148   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
3149   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
3150 }
3151
3152 static void
3153 reinit_ed_flow_hash ()
3154 {
3155   snat_main_t *sm = &snat_main;
3156   // we expect 2 flows per session, so multiply translation_buckets by 2
3157   clib_bihash_init_16_8 (
3158     &sm->flow_hash, "ed-flow-hash",
3159     clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
3160   clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
3161 }
3162
3163 static void
3164 nat44_ed_db_init (u32 translations, u32 translation_buckets)
3165 {
3166   snat_main_t *sm = &snat_main;
3167   snat_main_per_thread_data_t *tsm;
3168   u32 static_mapping_buckets = 1024;
3169   u32 static_mapping_memory_size = 64 << 20;
3170
3171   reinit_ed_flow_hash ();
3172
3173   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
3174                         "static_mapping_by_local", static_mapping_buckets,
3175                         static_mapping_memory_size);
3176   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_local,
3177                                      format_static_mapping_kvp);
3178
3179   clib_bihash_init_8_8 (&sm->static_mapping_by_external,
3180                         "static_mapping_by_external", static_mapping_buckets,
3181                         static_mapping_memory_size);
3182   clib_bihash_set_kvp_format_fn_8_8 (&sm->static_mapping_by_external,
3183                                      format_static_mapping_kvp);
3184
3185   if (sm->pat)
3186     {
3187       vec_foreach (tsm, sm->per_thread_data)
3188         {
3189           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3190                                    sm->translation_buckets);
3191         }
3192     }
3193 }
3194
3195 static void
3196 nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
3197 {
3198   pool_free (tsm->lru_pool);
3199   pool_free (tsm->sessions);
3200   vec_free (tsm->per_vrf_sessions_vec);
3201 }
3202
3203 static void
3204 nat44_ed_db_free ()
3205 {
3206   snat_main_t *sm = &snat_main;
3207   snat_main_per_thread_data_t *tsm;
3208
3209   pool_free (sm->static_mappings);
3210   clib_bihash_free_16_8 (&sm->flow_hash);
3211   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
3212   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
3213
3214   if (sm->pat)
3215     {
3216       vec_foreach (tsm, sm->per_thread_data)
3217         {
3218           nat44_ed_worker_db_free (tsm);
3219         }
3220     }
3221 }
3222
3223 void
3224 nat44_ed_sessions_clear ()
3225 {
3226   snat_main_t *sm = &snat_main;
3227   snat_main_per_thread_data_t *tsm;
3228
3229   reinit_ed_flow_hash ();
3230
3231   if (sm->pat)
3232     {
3233       vec_foreach (tsm, sm->per_thread_data)
3234         {
3235
3236           nat44_ed_worker_db_free (tsm);
3237           nat44_ed_worker_db_init (tsm, sm->max_translations_per_thread,
3238                                    sm->translation_buckets);
3239         }
3240     }
3241   vlib_zero_simple_counter (&sm->total_sessions, 0);
3242 }
3243
3244 static void
3245 nat_ip4_add_del_addr_only_sm_cb (ip4_main_t * im,
3246                                  uword opaque,
3247                                  u32 sw_if_index,
3248                                  ip4_address_t * address,
3249                                  u32 address_length,
3250                                  u32 if_address_index, u32 is_delete)
3251 {
3252   snat_main_t *sm = &snat_main;
3253   snat_static_map_resolve_t *rp;
3254   snat_static_mapping_t *m;
3255   clib_bihash_kv_8_8_t kv, value;
3256   ip4_address_t l_addr;
3257   int i, rv;
3258
3259   if (!sm->enabled)
3260     return;
3261
3262   for (i = 0; i < vec_len (sm->to_resolve); i++)
3263     {
3264       rp = sm->to_resolve + i;
3265       if (rp->addr_only == 0)
3266         continue;
3267       if (rp->sw_if_index == sw_if_index)
3268         goto match;
3269     }
3270
3271   return;
3272
3273 match:
3274   init_nat_k (&kv, *address, rp->addr_only ? 0 : rp->e_port,
3275               sm->outside_fib_index, rp->addr_only ? 0 : rp->proto);
3276   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3277     m = 0;
3278   else
3279     m = pool_elt_at_index (sm->static_mappings, value.value);
3280
3281   if (!is_delete)
3282     {
3283       /* Don't trip over lease renewal, static config */
3284       if (m)
3285         return;
3286     }
3287   else
3288     {
3289       if (!m)
3290         return;
3291     }
3292
3293   /* Indetity mapping? */
3294   if (rp->l_addr.as_u32 == 0)
3295     l_addr.as_u32 = address[0].as_u32;
3296   else
3297     l_addr.as_u32 = rp->l_addr.as_u32;
3298
3299   if (is_delete)
3300     {
3301       rv = nat44_ed_del_static_mapping (l_addr, address[0], rp->l_port,
3302                                         rp->e_port, rp->proto, rp->vrf_id, ~0,
3303                                         rp->flags);
3304     }
3305   else
3306     {
3307       rv = nat44_ed_add_static_mapping (l_addr, address[0], rp->l_port,
3308                                         rp->e_port, rp->proto, rp->vrf_id, ~0,
3309                                         rp->flags, rp->pool_addr, rp->tag);
3310     }
3311   if (rv)
3312     {
3313       nat_elog_notice_X1 (sm, "add_static_mapping returned %d", "i4", rv);
3314     }
3315 }
3316
3317 static void
3318 snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
3319                                        uword opaque,
3320                                        u32 sw_if_index,
3321                                        ip4_address_t * address,
3322                                        u32 address_length,
3323                                        u32 if_address_index, u32 is_delete)
3324 {
3325   snat_main_t *sm = &snat_main;
3326   snat_static_map_resolve_t *rp;
3327   ip4_address_t l_addr;
3328   int i, j;
3329   int rv;
3330   u8 twice_nat = 0;
3331   snat_address_t *addresses = sm->addresses;
3332
3333   if (!sm->enabled)
3334     return;
3335
3336   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices); i++)
3337     {
3338       if (sw_if_index == sm->auto_add_sw_if_indices[i])
3339         goto match;
3340     }
3341
3342   for (i = 0; i < vec_len (sm->auto_add_sw_if_indices_twice_nat); i++)
3343     {
3344       twice_nat = 1;
3345       addresses = sm->twice_nat_addresses;
3346       if (sw_if_index == sm->auto_add_sw_if_indices_twice_nat[i])
3347         goto match;
3348     }
3349
3350   return;
3351
3352 match:
3353   if (!is_delete)
3354     {
3355       /* Don't trip over lease renewal, static config */
3356       for (j = 0; j < vec_len (addresses); j++)
3357         if (addresses[j].addr.as_u32 == address->as_u32)
3358           return;
3359
3360       (void) snat_add_address (sm, address, ~0, twice_nat);
3361       /* Scan static map resolution vector */
3362       for (j = 0; j < vec_len (sm->to_resolve); j++)
3363         {
3364           rp = sm->to_resolve + j;
3365           if (rp->addr_only)
3366             continue;
3367           /* On this interface? */
3368           if (rp->sw_if_index == sw_if_index)
3369             {
3370
3371               // TODO: remove if not needed (handled by function)
3372               /* Indetity mapping? */
3373               if (rp->l_addr.as_u32 == 0)
3374                 l_addr.as_u32 = address[0].as_u32;
3375               else
3376                 l_addr.as_u32 = rp->l_addr.as_u32;
3377
3378               /* Add the static mapping */
3379               rv = nat44_ed_add_static_mapping (
3380                 l_addr, address[0], rp->l_port, rp->e_port, rp->proto,
3381                 rp->vrf_id, ~0, rp->flags, rp->pool_addr, rp->tag);
3382               if (rv)
3383                 {
3384                   nat_elog_notice_X1 (sm, "add_static_mapping returned %d",
3385                                       "i4", rv);
3386                 }
3387             }
3388         }
3389       return;
3390     }
3391   else
3392     {
3393       (void) snat_del_address (sm, address[0], 1, twice_nat);
3394       return;
3395     }
3396 }
3397
3398 int
3399 snat_add_interface_address (snat_main_t * sm, u32 sw_if_index, int is_del,
3400                             u8 twice_nat)
3401 {
3402   ip4_main_t *ip4_main = sm->ip4_main;
3403   ip4_address_t *first_int_addr;
3404   snat_static_map_resolve_t *rp;
3405   u32 *indices_to_delete = 0;
3406   int i, j;
3407   u32 *auto_add_sw_if_indices;
3408
3409   if (!sm->enabled)
3410     {
3411       return VNET_API_ERROR_UNSUPPORTED;
3412     }
3413
3414   auto_add_sw_if_indices = twice_nat ? sm->auto_add_sw_if_indices_twice_nat :
3415                                        sm->auto_add_sw_if_indices;
3416
3417   first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index, 0);
3418
3419   for (i = 0; i < vec_len (auto_add_sw_if_indices); i++)
3420     {
3421       if (auto_add_sw_if_indices[i] == sw_if_index)
3422         {
3423           if (is_del)
3424             {
3425               /* if have address remove it */
3426               if (first_int_addr)
3427                 (void) snat_del_address (sm, first_int_addr[0], 1, twice_nat);
3428               else
3429                 {
3430                   for (j = 0; j < vec_len (sm->to_resolve); j++)
3431                     {
3432                       rp = sm->to_resolve + j;
3433                       if (rp->sw_if_index == sw_if_index)
3434                         vec_add1 (indices_to_delete, j);
3435                     }
3436                   if (vec_len (indices_to_delete))
3437                     {
3438                       for (j = vec_len (indices_to_delete) - 1; j >= 0; j--)
3439                         vec_del1 (sm->to_resolve, j);
3440                       vec_free (indices_to_delete);
3441                     }
3442                 }
3443               if (twice_nat)
3444                 vec_del1 (sm->auto_add_sw_if_indices_twice_nat, i);
3445               else
3446                 vec_del1 (sm->auto_add_sw_if_indices, i);
3447             }
3448           else
3449             return VNET_API_ERROR_VALUE_EXIST;
3450
3451           return 0;
3452         }
3453     }
3454
3455   if (is_del)
3456     return VNET_API_ERROR_NO_SUCH_ENTRY;
3457
3458   /* add to the auto-address list */
3459   if (twice_nat)
3460     vec_add1 (sm->auto_add_sw_if_indices_twice_nat, sw_if_index);
3461   else
3462     vec_add1 (sm->auto_add_sw_if_indices, sw_if_index);
3463
3464   /* If the address is already bound - or static - add it now */
3465   if (first_int_addr)
3466     (void) snat_add_address (sm, first_int_addr, ~0, twice_nat);
3467
3468   return 0;
3469 }
3470
3471 int
3472 nat44_ed_del_session (snat_main_t *sm, ip4_address_t *addr, u16 port,
3473                       ip4_address_t *eh_addr, u16 eh_port, u8 proto,
3474                       u32 vrf_id, int is_in)
3475 {
3476   ip4_header_t ip;
3477   clib_bihash_kv_16_8_t kv, value;
3478   u32 fib_index;
3479   snat_session_t *s;
3480   snat_main_per_thread_data_t *tsm;
3481
3482   if (!sm->enabled)
3483     {
3484       return VNET_API_ERROR_UNSUPPORTED;
3485     }
3486
3487   fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
3488   ip.dst_address.as_u32 = ip.src_address.as_u32 = addr->as_u32;
3489   if (sm->num_workers > 1)
3490     tsm = vec_elt_at_index (
3491       sm->per_thread_data,
3492       nat44_ed_get_in2out_worker_index (0, &ip, fib_index, 0));
3493   else
3494     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
3495
3496   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
3497   if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
3498     {
3499       return VNET_API_ERROR_NO_SUCH_ENTRY;
3500     }
3501
3502   if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
3503     return VNET_API_ERROR_UNSPECIFIED;
3504   s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
3505   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
3506   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
3507   return 0;
3508 }
3509
3510 VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
3511                                  vlib_node_runtime_t * node,
3512                                  vlib_frame_t * frame)
3513 {
3514   return 0;
3515 }
3516
3517 VLIB_REGISTER_NODE (nat_default_node) = {
3518   .name = "nat-default",
3519   .vector_size = sizeof (u32),
3520   .format_trace = 0,
3521   .type = VLIB_NODE_TYPE_INTERNAL,
3522   .n_errors = 0,
3523   .n_next_nodes = NAT_N_NEXT,
3524   .next_nodes = {
3525     [NAT_NEXT_DROP] = "error-drop",
3526     [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3527     [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
3528     [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3529     [NAT_NEXT_IN2OUT_ED_OUTPUT_FAST_PATH] = "nat44-ed-in2out-output",
3530     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3531     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
3532     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
3533     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
3534     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
3535   },
3536 };
3537
3538 void
3539 nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
3540 {
3541   f->l3_csum_delta = 0;
3542   f->l4_csum_delta = 0;
3543   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
3544       f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
3545     {
3546       f->l3_csum_delta =
3547         ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
3548       f->l3_csum_delta =
3549         ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
3550     }
3551   else
3552     {
3553       f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
3554     }
3555   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
3556       f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
3557     {
3558       f->l3_csum_delta =
3559         ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
3560       f->l3_csum_delta =
3561         ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
3562     }
3563   else
3564     {
3565       f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
3566     }
3567   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
3568     {
3569       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
3570       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3571     }
3572   else
3573     {
3574       f->rewrite.sport = f->match.sport;
3575     }
3576   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
3577     {
3578       f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
3579       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
3580     }
3581   else
3582     {
3583       f->rewrite.dport = f->match.dport;
3584     }
3585   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
3586       f->rewrite.icmp_id != f->match.sport)
3587     {
3588       f->l4_csum_delta =
3589         ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
3590       f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
3591     }
3592   else
3593     {
3594       f->rewrite.icmp_id = f->match.sport;
3595     }
3596   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3597     {
3598     }
3599   else
3600     {
3601       f->rewrite.fib_index = f->match.fib_index;
3602     }
3603 }
3604
3605 static_always_inline int
3606 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3607                             ip4_header_t *ip, nat_6t_flow_t *f);
3608
3609 static_always_inline void
3610 nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
3611                            nat_6t_flow_t *f, nat_protocol_t proto,
3612                            int is_icmp_inner_ip4, int skip_saddr_rewrite)
3613 {
3614   udp_header_t *udp = ip4_next_header (ip);
3615   tcp_header_t *tcp = (tcp_header_t *) udp;
3616
3617   if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
3618       !vnet_buffer (b)->ip.reass.is_non_first_fragment)
3619     {
3620       if (!is_icmp_inner_ip4)
3621         { // regular case
3622           ip->src_address = f->rewrite.saddr;
3623           ip->dst_address = f->rewrite.daddr;
3624           udp->src_port = f->rewrite.sport;
3625           udp->dst_port = f->rewrite.dport;
3626         }
3627       else
3628         { // icmp inner ip4 - reversed saddr/daddr
3629           ip->src_address = f->rewrite.daddr;
3630           ip->dst_address = f->rewrite.saddr;
3631           udp->src_port = f->rewrite.dport;
3632           udp->dst_port = f->rewrite.sport;
3633         }
3634
3635       if (NAT_PROTOCOL_TCP == proto)
3636         {
3637           ip_csum_t tcp_sum = tcp->checksum;
3638           tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
3639           tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
3640           mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
3641           tcp->checksum = ip_csum_fold (tcp_sum);
3642         }
3643       else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
3644         {
3645           ip_csum_t udp_sum = udp->checksum;
3646           udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
3647           udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
3648           udp->checksum = ip_csum_fold (udp_sum);
3649         }
3650     }
3651   else
3652     {
3653       if (!is_icmp_inner_ip4)
3654         { // regular case
3655           if (!skip_saddr_rewrite)
3656             {
3657               ip->src_address = f->rewrite.saddr;
3658             }
3659           ip->dst_address = f->rewrite.daddr;
3660         }
3661       else
3662         { // icmp inner ip4 - reversed saddr/daddr
3663           ip->src_address = f->rewrite.daddr;
3664           ip->dst_address = f->rewrite.saddr;
3665         }
3666     }
3667
3668   if (skip_saddr_rewrite)
3669     {
3670       ip->checksum = ip4_header_checksum (ip);
3671     }
3672   else
3673     {
3674       ip_csum_t ip_sum = ip->checksum;
3675       ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
3676       ip->checksum = ip_csum_fold (ip_sum);
3677     }
3678   if (0xffff == ip->checksum)
3679     ip->checksum = 0;
3680   ASSERT (ip4_header_checksum_is_valid (ip));
3681 }
3682
3683 static_always_inline int
3684 it_fits (vlib_main_t *vm, vlib_buffer_t *b, void *object, size_t size)
3685 {
3686   int result = ((u8 *) object + size <=
3687                 (u8 *) vlib_buffer_get_current (b) + b->current_length) &&
3688                vlib_object_within_buffer_data (vm, b, object, size);
3689   return result;
3690 }
3691
3692 static_always_inline int
3693 nat_6t_flow_icmp_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3694                             ip4_header_t *ip, nat_6t_flow_t *f)
3695 {
3696   if (IP_PROTOCOL_ICMP != ip->protocol)
3697     return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3698
3699   icmp46_header_t *icmp = ip4_next_header (ip);
3700   icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
3701
3702   if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
3703     {
3704       if (!it_fits (vm, b, icmp, sizeof (*icmp)))
3705         {
3706           return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3707         }
3708
3709       if (!icmp_type_is_error_message (icmp->type))
3710         {
3711           if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
3712               (f->rewrite.icmp_id != echo->identifier))
3713             {
3714               ip_csum_t sum = icmp->checksum;
3715               sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
3716                                     icmp_echo_header_t,
3717                                     identifier /* changed member */);
3718               echo->identifier = f->rewrite.icmp_id;
3719               icmp->checksum = ip_csum_fold (sum);
3720             }
3721         }
3722       else
3723         {
3724           ip_csum_t sum = ip_incremental_checksum (
3725             0, icmp,
3726             clib_net_to_host_u16 (ip->length) - ip4_header_bytes (ip));
3727           sum = (u16) ~ip_csum_fold (sum);
3728           if (sum != 0)
3729             {
3730               return NAT_ED_TRNSL_ERR_INVALID_CSUM;
3731             }
3732
3733           // errors are not fragmented
3734           ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
3735
3736           if (!ip4_header_checksum_is_valid (inner_ip))
3737             {
3738               return NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT;
3739             }
3740
3741           nat_protocol_t inner_proto =
3742             ip_proto_to_nat_proto (inner_ip->protocol);
3743
3744           ip_csum_t old_icmp_sum = icmp->checksum;
3745           ip_csum_t old_inner_ip_sum = inner_ip->checksum;
3746           ip_csum_t old_udp_sum;
3747           ip_csum_t old_tcp_sum;
3748           ip_csum_t new_icmp_sum;
3749           udp_header_t *udp;
3750           tcp_header_t *tcp;
3751
3752           switch (inner_proto)
3753             {
3754             case NAT_PROTOCOL_UDP:
3755               udp = (udp_header_t *) (inner_ip + 1);
3756               if (!it_fits (vm, b, udp, sizeof (*udp)))
3757                 {
3758                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3759                 }
3760               old_udp_sum = udp->checksum;
3761               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3762                                          1 /* is_icmp_inner_ip4 */,
3763                                          0 /* skip_saddr_rewrite */);
3764               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3765               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3766               new_icmp_sum =
3767                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3768                                 inner_ip->checksum, ip4_header_t, checksum);
3769               new_icmp_sum =
3770                 ip_csum_update (new_icmp_sum, old_udp_sum, udp->checksum,
3771                                 udp_header_t, checksum);
3772               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3773               icmp->checksum = new_icmp_sum;
3774               break;
3775             case NAT_PROTOCOL_TCP:
3776               tcp = (tcp_header_t *) (inner_ip + 1);
3777               if (!it_fits (vm, b, tcp, sizeof (*tcp)))
3778                 {
3779                   return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3780                 }
3781               old_tcp_sum = tcp->checksum;
3782               nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
3783                                          1 /* is_icmp_inner_ip4 */,
3784                                          0 /* skip_saddr_rewrite */);
3785               new_icmp_sum = ip_csum_sub_even (old_icmp_sum, f->l3_csum_delta);
3786               new_icmp_sum = ip_csum_sub_even (new_icmp_sum, f->l4_csum_delta);
3787               new_icmp_sum =
3788                 ip_csum_update (new_icmp_sum, old_inner_ip_sum,
3789                                 inner_ip->checksum, ip4_header_t, checksum);
3790               new_icmp_sum =
3791                 ip_csum_update (new_icmp_sum, old_tcp_sum, tcp->checksum,
3792                                 tcp_header_t, checksum);
3793               new_icmp_sum = ip_csum_fold (new_icmp_sum);
3794               icmp->checksum = new_icmp_sum;
3795               break;
3796             case NAT_PROTOCOL_ICMP:
3797               if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3798                 {
3799                   icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
3800                   if (!it_fits (vm, b, inner_icmp, sizeof (*inner_icmp)))
3801                     {
3802                       return NAT_ED_TRNSL_ERR_PACKET_TRUNCATED;
3803                     }
3804                   icmp_echo_header_t *inner_echo =
3805                     (icmp_echo_header_t *) (inner_icmp + 1);
3806                   if (f->rewrite.icmp_id != inner_echo->identifier)
3807                     {
3808                       ip_csum_t sum = icmp->checksum;
3809                       sum = ip_csum_update (
3810                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3811                         icmp_echo_header_t, identifier /* changed member */);
3812                       icmp->checksum = ip_csum_fold (sum);
3813                       ip_csum_t inner_sum = inner_icmp->checksum;
3814                       inner_sum = ip_csum_update (
3815                         sum, inner_echo->identifier, f->rewrite.icmp_id,
3816                         icmp_echo_header_t, identifier /* changed member */);
3817                       inner_icmp->checksum = ip_csum_fold (inner_sum);
3818                       inner_echo->identifier = f->rewrite.icmp_id;
3819                     }
3820                 }
3821               break;
3822             default:
3823               clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
3824               return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
3825             }
3826         }
3827     }
3828
3829   return NAT_ED_TRNSL_ERR_SUCCESS;
3830 }
3831
3832 static_always_inline nat_translation_error_e
3833 nat_6t_flow_buf_translate (vlib_main_t *vm, snat_main_t *sm, vlib_buffer_t *b,
3834                            ip4_header_t *ip, nat_6t_flow_t *f,
3835                            nat_protocol_t proto, int is_output_feature,
3836                            int is_i2o)
3837 {
3838   if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3839     {
3840       vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
3841     }
3842
3843   if (NAT_PROTOCOL_ICMP == proto)
3844     {
3845       if (ip->src_address.as_u32 != f->rewrite.saddr.as_u32)
3846         {
3847           // packet is returned from a router, not from destination
3848           // skip source address rewrite if in o2i path
3849           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3850                                      0 /* is_icmp_inner_ip4 */,
3851                                      !is_i2o /* skip_saddr_rewrite */);
3852         }
3853       else
3854         {
3855           nat_6t_flow_ip4_translate (sm, b, ip, f, proto,
3856                                      0 /* is_icmp_inner_ip4 */,
3857                                      0 /* skip_saddr_rewrite */);
3858         }
3859       return nat_6t_flow_icmp_translate (vm, sm, b, ip, f);
3860     }
3861
3862   nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */,
3863                              0 /* skip_saddr_rewrite */);
3864
3865   return NAT_ED_TRNSL_ERR_SUCCESS;
3866 }
3867
3868 nat_translation_error_e
3869 nat_6t_flow_buf_translate_i2o (vlib_main_t *vm, snat_main_t *sm,
3870                                vlib_buffer_t *b, ip4_header_t *ip,
3871                                nat_6t_flow_t *f, nat_protocol_t proto,
3872                                int is_output_feature)
3873 {
3874   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3875                                     1 /* is_i2o */);
3876 }
3877
3878 nat_translation_error_e
3879 nat_6t_flow_buf_translate_o2i (vlib_main_t *vm, snat_main_t *sm,
3880                                vlib_buffer_t *b, ip4_header_t *ip,
3881                                nat_6t_flow_t *f, nat_protocol_t proto,
3882                                int is_output_feature)
3883 {
3884   return nat_6t_flow_buf_translate (vm, sm, b, ip, f, proto, is_output_feature,
3885                                     0 /* is_i2o */);
3886 }
3887
3888 u8 *
3889 format_nat_6t (u8 *s, va_list *args)
3890 {
3891   nat_6t_t *t = va_arg (*args, nat_6t_t *);
3892
3893   s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
3894               format_ip4_address, t->saddr.as_u8,
3895               clib_net_to_host_u16 (t->sport), format_ip4_address,
3896               t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
3897               format_ip_protocol, t->proto, t->fib_index);
3898   return s;
3899 }
3900
3901 u8 *
3902 format_nat_ed_translation_error (u8 *s, va_list *args)
3903 {
3904   nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
3905
3906   switch (e)
3907     {
3908     case NAT_ED_TRNSL_ERR_SUCCESS:
3909       s = format (s, "success");
3910       break;
3911     case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
3912       s = format (s, "translation-failed");
3913       break;
3914     case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
3915       s = format (s, "flow-mismatch");
3916       break;
3917     case NAT_ED_TRNSL_ERR_PACKET_TRUNCATED:
3918       s = format (s, "packet-truncated");
3919       break;
3920     case NAT_ED_TRNSL_ERR_INNER_IP_CORRUPT:
3921       s = format (s, "inner-ip-corrupted");
3922       break;
3923     case NAT_ED_TRNSL_ERR_INVALID_CSUM:
3924       s = format (s, "invalid-checksum");
3925       break;
3926     }
3927   return s;
3928 }
3929
3930 u8 *
3931 format_nat_6t_flow (u8 *s, va_list *args)
3932 {
3933   nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
3934
3935   s = format (s, "match: %U ", format_nat_6t, &f->match);
3936   int r = 0;
3937   if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
3938     {
3939       s = format (s, "rewrite: saddr %U ", format_ip4_address,
3940                   f->rewrite.saddr.as_u8);
3941       r = 1;
3942     }
3943   if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
3944     {
3945       if (!r)
3946         {
3947           s = format (s, "rewrite: ");
3948           r = 1;
3949         }
3950       s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
3951     }
3952   if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
3953     {
3954       if (!r)
3955         {
3956           s = format (s, "rewrite: ");
3957           r = 1;
3958         }
3959       s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
3960     }
3961   if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
3962     {
3963       if (!r)
3964         {
3965           s = format (s, "rewrite: ");
3966           r = 1;
3967         }
3968       s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
3969     }
3970   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
3971     {
3972       if (!r)
3973         {
3974           s = format (s, "rewrite: ");
3975           r = 1;
3976         }
3977       s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
3978     }
3979   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
3980     {
3981       if (!r)
3982         {
3983           s = format (s, "rewrite: ");
3984           r = 1;
3985         }
3986       s = format (s, "txfib %u ", f->rewrite.fib_index);
3987     }
3988   return s;
3989 }
3990
3991 /*
3992  * fd.io coding-style-patch-verification: ON
3993  *
3994  * Local Variables:
3995  * eval: (c-set-style "gnu")
3996  * End:
3997  */