src/plugins/nat/in2out_ed.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 endpoint-dependent inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/pg/pg.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/fib/ip4_fib.h>
  26 #include <vnet/udp/udp.h>
  27 #include <vppinfra/error.h>
  28 #include <nat/nat.h>
  29 #include <nat/nat_ipfix_logging.h>
  30 #include <nat/nat_inlines.h>
  31 #include <nat/nat44/inlines.h>
  32 #include <nat/nat_syslog.h>
  33 #include <nat/nat_ha.h>
  34
  35 static char *nat_in2out_ed_error_strings[] = {
  36 #define _(sym,string) string,
  37   foreach_nat_in2out_ed_error
  38 #undef _
  39 };
  40
  41 typedef struct
  42 {
  43   u32 sw_if_index;
  44   u32 next_index;
  45   u32 session_index;
  46   u32 is_slow_path;
  47 } nat_in2out_ed_trace_t;
  48
  49 static u8 *
  50 format_nat_in2out_ed_trace (u8 * s, va_list * args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
  55   char *tag;
  56
  57   tag =
  58     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
  59     "NAT44_IN2OUT_ED_FAST_PATH";
  60
  61   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  62               t->sw_if_index, t->next_index, t->session_index);
  63
  64   return s;
  65 }
  66
  67 #ifndef CLIB_MARCH_VARIANT
  68 int
  69 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
  70 {
  71   snat_main_t *sm = &snat_main;
  72   nat44_is_idle_session_ctx_t *ctx = arg;
  73   snat_session_t *s;
  74   u64 sess_timeout_time;
  75   nat_ed_ses_key_t ed_key;
  76   clib_bihash_kv_16_8_t ed_kv;
  77   int i;
  78   snat_address_t *a;
  79   snat_session_key_t key;
  80   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
  81                                                        ctx->thread_index);
  82
  83   s = pool_elt_at_index (tsm->sessions, kv->value);
  84   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
  85   if (ctx->now >= sess_timeout_time)
  86     {
  87       if (is_fwd_bypass_session (s))
  88         goto delete;
  89
  90       ed_key.l_addr = s->out2in.addr;
  91       ed_key.r_addr = s->ext_host_addr;
  92       ed_key.fib_index = s->out2in.fib_index;
  93       if (snat_is_unk_proto_session (s))
  94         {
  95           ed_key.proto = s->in2out.port;
  96           ed_key.r_port = 0;
  97           ed_key.l_port = 0;
  98         }
  99       else
 100         {
 101           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
 102           ed_key.l_port = s->out2in.port;
 103           ed_key.r_port = s->ext_host_port;
 104         }
 105       ed_kv.key[0] = ed_key.as_u64[0];
 106       ed_kv.key[1] = ed_key.as_u64[1];
 107       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
 108         nat_elog_warn ("out2in_ed key del failed");
 109
 110       if (snat_is_unk_proto_session (s))
 111         goto delete;
 112
 113       snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 114                                            s->in2out.addr.as_u32,
 115                                            s->out2in.addr.as_u32,
 116                                            s->in2out.protocol,
 117                                            s->in2out.port,
 118                                            s->out2in.port,
 119                                            s->in2out.fib_index);
 120
 121       nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
 122                              &s->in2out.addr, s->in2out.port,
 123                              &s->ext_host_nat_addr, s->ext_host_nat_port,
 124                              &s->out2in.addr, s->out2in.port,
 125                              &s->ext_host_addr, s->ext_host_port,
 126                              s->in2out.protocol, is_twice_nat_session (s));
 127
 128       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 129                    s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
 130                    ctx->thread_index);
 131
 132       if (is_twice_nat_session (s))
 133         {
 134           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
 135             {
 136               key.protocol = s->in2out.protocol;
 137               key.port = s->ext_host_nat_port;
 138               a = sm->twice_nat_addresses + i;
 139               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
 140                 {
 141                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
 142                                                       ctx->thread_index,
 143                                                       &key);
 144                   break;
 145                 }
 146             }
 147         }
 148
 149       if (snat_is_session_static (s))
 150         goto delete;
 151
 152       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
 153                                           &s->out2in);
 154     delete:
 155       nat44_delete_session (sm, s, ctx->thread_index);
 156       return 1;
 157     }
 158
 159   return 0;
 160 }
 161 #endif
 162
 163 static inline u32
 164 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
 165                           ip4_header_t * ip0, icmp46_header_t * icmp0,
 166                           u32 sw_if_index0, u32 rx_fib_index0,
 167                           vlib_node_runtime_t * node, u32 next0, f64 now,
 168                           u32 thread_index, snat_session_t ** p_s0)
 169 {
 170   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
 171                        next0, thread_index, p_s0, 0);
 172   snat_session_t *s0 = *p_s0;
 173   if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
 174     {
 175       /* Accounting */
 176       nat44_session_update_counters (s0, now,
 177                                      vlib_buffer_length_in_chain
 178                                      (sm->vlib_main, b0), thread_index);
 179       /* Per-user LRU list maintenance */
 180       nat44_session_update_lru (sm, s0, thread_index);
 181     }
 182   return next0;
 183 }
 184
 185 static u32
 186 slow_path_ed (snat_main_t * sm,
 187               vlib_buffer_t * b,
 188               u32 rx_fib_index,
 189               clib_bihash_kv_16_8_t * kv,
 190               snat_session_t ** sessionp,
 191               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
 192 {
 193   snat_session_t *s = 0;
 194   snat_user_t *u;
 195   snat_session_key_t key0, key1;
 196   lb_nat_type_t lb = 0, is_sm = 0;
 197   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 198   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
 199   u32 proto = ip_proto_to_snat_proto (key->proto);
 200   nat_outside_fib_t *outside_fib;
 201   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 202   u8 identity_nat;
 203   fib_prefix_t pfx = {
 204     .fp_proto = FIB_PROTOCOL_IP4,
 205     .fp_len = 32,
 206     .fp_addr = {
 207                 .ip4.as_u32 = key->r_addr.as_u32,
 208                 },
 209   };
 210   nat44_is_idle_session_ctx_t ctx;
 211
 212   u32 cleared = 0;
 213
 214   if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 215     {
 216       if (PREDICT_FALSE
 217           (!(cleared = nat44_users_cleanup (thread_index, now))))
 218         {
 219           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 220           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 221           nat_elog_notice ("maximum sessions exceeded");
 222           return NAT_NEXT_DROP;
 223         }
 224     }
 225
 226   key0.addr = key->l_addr;
 227   key0.port = key->l_port;
 228   key1.protocol = key0.protocol = proto;
 229   key0.fib_index = rx_fib_index;
 230   key1.fib_index = sm->outside_fib_index;
 231
 232   /* First try to match static mapping by local address and port */
 233   if (snat_static_mapping_match
 234       (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
 235     {
 236       /* Try to create dynamic translation */
 237       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
 238                                                thread_index, &key1,
 239                                                sm->port_per_thread,
 240                                                tsm->snat_thread_index))
 241         {
 242           if (cleared || !nat44_out_of_ports_cleanup (thread_index, now) ||
 243               snat_alloc_outside_address_and_port (sm->addresses,
 244                                                    rx_fib_index, thread_index,
 245                                                    &key1, sm->port_per_thread,
 246                                                    tsm->snat_thread_index))
 247             {
 248               nat_elog_notice ("addresses exhausted");
 249               b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
 250               return NAT_NEXT_DROP;
 251             }
 252         }
 253     }
 254   else
 255     {
 256       if (PREDICT_FALSE (identity_nat))
 257         {
 258           *sessionp = s;
 259           return next;
 260         }
 261       is_sm = 1;
 262     }
 263
 264   if (PREDICT_TRUE (proto == SNAT_PROTOCOL_TCP))
 265     {
 266       if (PREDICT_FALSE
 267           (!tcp_flags_is_init
 268            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 269         {
 270           b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
 271           if (!is_sm)
 272             snat_free_outside_address_and_port (sm->addresses,
 273                                                 thread_index, &key1);
 274           return NAT_NEXT_DROP;
 275         }
 276     }
 277
 278   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
 279   if (!u)
 280     {
 281       nat_elog_warn ("create NAT user failed");
 282       if (!is_sm)
 283         snat_free_outside_address_and_port (sm->addresses,
 284                                             thread_index, &key1);
 285       b->error = node->errors[NAT_IN2OUT_ED_ERROR_CANNOT_CREATE_USER];
 286       return NAT_NEXT_DROP;
 287     }
 288
 289   s = nat_ed_session_alloc (sm, u, thread_index, now);
 290   if (!s)
 291     {
 292       nat44_delete_user_with_no_session (sm, u, thread_index);
 293       nat_elog_warn ("create NAT session failed");
 294       if (!is_sm)
 295         snat_free_outside_address_and_port (sm->addresses,
 296                                             thread_index, &key1);
 297       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 298       return NAT_NEXT_DROP;
 299     }
 300
 301   user_session_increment (sm, u, is_sm);
 302   if (is_sm)
 303     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 304   if (lb)
 305     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
 306   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 307   s->ext_host_addr = key->r_addr;
 308   s->ext_host_port = key->r_port;
 309   s->in2out = key0;
 310   s->out2in = key1;
 311   s->out2in.protocol = key0.protocol;
 312
 313   switch (vec_len (sm->outside_fibs))
 314     {
 315     case 0:
 316       s->out2in.fib_index = sm->outside_fib_index;
 317       break;
 318     case 1:
 319       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 320       break;
 321     default:
 322       /* *INDENT-OFF* */
 323       vec_foreach (outside_fib, sm->outside_fibs)
 324        {
 325           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 326           if (FIB_NODE_INDEX_INVALID != fei)
 327             {
 328               if (fib_entry_get_resolving_interface (fei) != ~0)
 329                 {
 330                   s->out2in.fib_index = outside_fib->fib_index;
 331                   break;
 332                 }
 333             }
 334         }
 335       /* *INDENT-ON* */
 336       break;
 337     }
 338
 339   /* Add to lookup tables */
 340   kv->value = s - tsm->sessions;
 341   ctx.now = now;
 342   ctx.thread_index = thread_index;
 343   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
 344                                                nat44_i2o_ed_is_idle_session_cb,
 345                                                &ctx))
 346     nat_elog_notice ("in2out-ed key add failed");
 347
 348   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
 349               key1.port, key->r_port);
 350   kv->value = s - tsm->sessions;
 351   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
 352                                                nat44_o2i_ed_is_idle_session_cb,
 353                                                &ctx))
 354     nat_elog_notice ("out2in-ed key add failed");
 355
 356   *sessionp = s;
 357
 358   /* log NAT event */
 359   snat_ipfix_logging_nat44_ses_create (thread_index,
 360                                        s->in2out.addr.as_u32,
 361                                        s->out2in.addr.as_u32,
 362                                        s->in2out.protocol,
 363                                        s->in2out.port,
 364                                        s->out2in.port, s->in2out.fib_index);
 365
 366   nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
 367                          &s->in2out.addr, s->in2out.port,
 368                          &s->ext_host_nat_addr, s->ext_host_nat_port,
 369                          &s->out2in.addr, s->out2in.port,
 370                          &s->ext_host_addr, s->ext_host_port,
 371                          s->in2out.protocol, 0);
 372
 373   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 374                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 375                &s->ext_host_nat_addr, s->ext_host_nat_port,
 376                s->in2out.protocol, s->in2out.fib_index, s->flags,
 377                thread_index, 0);
 378
 379   return next;
 380 }
 381
 382 static_always_inline int
 383 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
 384                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
 385                         u32 rx_fib_index, u32 thread_index)
 386 {
 387   udp_header_t *udp = ip4_next_header (ip);
 388   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 389   clib_bihash_kv_16_8_t kv, value;
 390   snat_session_key_t key0, key1;
 391
 392   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
 393               sm->outside_fib_index, udp->dst_port, udp->src_port);
 394
 395   /* NAT packet aimed at external address if */
 396   /* has active sessions */
 397   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 398     {
 399       key0.addr = ip->dst_address;
 400       key0.port = udp->dst_port;
 401       key0.protocol = proto;
 402       key0.fib_index = sm->outside_fib_index;
 403       /* or is static mappings */
 404       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
 405         return 0;
 406     }
 407   else
 408     return 0;
 409
 410   if (sm->forwarding_enabled)
 411     return 1;
 412
 413   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
 414                                   rx_fib_index);
 415 }
 416
 417 static_always_inline int
 418 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 419                                       u32 thread_index, f64 now,
 420                                       vlib_main_t * vm, vlib_buffer_t * b)
 421 {
 422   nat_ed_ses_key_t key;
 423   clib_bihash_kv_16_8_t kv, value;
 424   snat_session_t *s = 0;
 425   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 426
 427   if (!sm->forwarding_enabled)
 428     return 0;
 429
 430   if (ip->protocol == IP_PROTOCOL_ICMP)
 431     {
 432       key.as_u64[0] = key.as_u64[1] = 0;
 433       if (get_icmp_i2o_ed_key (b, ip, &key))
 434         return 0;
 435       key.fib_index = 0;
 436       kv.key[0] = key.as_u64[0];
 437       kv.key[1] = key.as_u64[1];
 438     }
 439   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 440     {
 441       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
 442                   vnet_buffer (b)->ip.reass.l4_src_port,
 443                   vnet_buffer (b)->ip.reass.l4_dst_port);
 444     }
 445   else
 446     {
 447       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
 448                   0);
 449     }
 450
 451   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 452     {
 453       s = pool_elt_at_index (tsm->sessions, value.value);
 454       if (is_fwd_bypass_session (s))
 455         {
 456           if (ip->protocol == IP_PROTOCOL_TCP)
 457             {
 458               if (nat44_set_tcp_session_state_i2o
 459                   (sm, now, s, b, thread_index))
 460                 return 1;
 461             }
 462           /* Accounting */
 463           nat44_session_update_counters (s, now,
 464                                          vlib_buffer_length_in_chain (vm, b),
 465                                          thread_index);
 466           /* Per-user LRU list maintenance */
 467           nat44_session_update_lru (sm, s, thread_index);
 468           return 1;
 469         }
 470       else
 471         return 0;
 472     }
 473
 474   return 0;
 475 }
 476
 477 static_always_inline int
 478 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
 479                                        u8 proto, u16 src_port, u16 dst_port,
 480                                        u32 thread_index, u32 rx_sw_if_index,
 481                                        u32 tx_sw_if_index)
 482 {
 483   clib_bihash_kv_16_8_t kv, value;
 484   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 485   snat_interface_t *i;
 486   snat_session_t *s;
 487   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
 488   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
 489
 490   /* src NAT check */
 491   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
 492               src_port, dst_port);
 493   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
 494     {
 495       s = pool_elt_at_index (tsm->sessions, value.value);
 496       if (nat44_is_ses_closed (s))
 497         {
 498           nat_free_session_data (sm, s, thread_index, 0);
 499           nat44_delete_session (sm, s, thread_index);
 500         }
 501       else
 502         s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
 503       return 1;
 504     }
 505
 506   /* dst NAT check */
 507   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
 508               dst_port, src_port);
 509   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 510     {
 511       s = pool_elt_at_index (tsm->sessions, value.value);
 512       if (is_fwd_bypass_session (s))
 513         return 0;
 514
 515       /* hairpinning */
 516       /* *INDENT-OFF* */
 517       pool_foreach (i, sm->output_feature_interfaces,
 518       ({
 519         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
 520            return 0;
 521       }));
 522       /* *INDENT-ON* */
 523       return 1;
 524     }
 525
 526   return 0;
 527 }
 528
 529 #ifndef CLIB_MARCH_VARIANT
 530 u32
 531 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 532                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
 533                       u8 * p_proto, snat_session_key_t * p_value,
 534                       u8 * p_dont_translate, void *d, void *e)
 535 {
 536   u32 sw_if_index;
 537   u32 rx_fib_index;
 538   nat_ed_ses_key_t key;
 539   snat_session_t *s = 0;
 540   u8 dont_translate = 0;
 541   clib_bihash_kv_16_8_t kv, value;
 542   u32 next = ~0;
 543   int err;
 544   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 545
 546   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 547   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 548
 549   key.as_u64[0] = key.as_u64[1] = 0;
 550   err = get_icmp_i2o_ed_key (b, ip, &key);
 551   if (err != 0)
 552     {
 553       b->error = node->errors[err];
 554       next = NAT_NEXT_DROP;
 555       goto out;
 556     }
 557   key.fib_index = rx_fib_index;
 558
 559   kv.key[0] = key.as_u64[0];
 560   kv.key[1] = key.as_u64[1];
 561
 562   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
 563     {
 564       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
 565         {
 566           if (PREDICT_FALSE
 567               (nat44_ed_not_translate_output_feature
 568                (sm, ip, key.proto, key.l_port, key.r_port, thread_index,
 569                 sw_if_index, vnet_buffer (b)->sw_if_index[VLIB_TX])))
 570             {
 571               dont_translate = 1;
 572               goto out;
 573             }
 574         }
 575       else
 576         {
 577           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
 578                                                      ip, SNAT_PROTOCOL_ICMP,
 579                                                      rx_fib_index,
 580                                                      thread_index)))
 581             {
 582               dont_translate = 1;
 583               goto out;
 584             }
 585         }
 586
 587       if (PREDICT_FALSE
 588           (icmp_type_is_error_message
 589            (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags)))
 590         {
 591           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 592           next = NAT_NEXT_DROP;
 593           goto out;
 594         }
 595
 596       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
 597                            thread_index, vlib_time_now (sm->vlib_main));
 598
 599       if (PREDICT_FALSE (next == NAT_NEXT_DROP))
 600         goto out;
 601
 602       if (!s)
 603         {
 604           dont_translate = 1;
 605           goto out;
 606         }
 607     }
 608   else
 609     {
 610       if (PREDICT_FALSE
 611           (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 612            ICMP4_echo_request
 613            && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
 614            ICMP4_echo_reply
 615            && !icmp_type_is_error_message (vnet_buffer (b)->ip.
 616                                            reass.icmp_type_or_tcp_flags)))
 617         {
 618           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
 619           next = NAT_NEXT_DROP;
 620           goto out;
 621         }
 622
 623       s = pool_elt_at_index (tsm->sessions, value.value);
 624     }
 625
 626   *p_proto = ip_proto_to_snat_proto (key.proto);
 627 out:
 628   if (s)
 629     *p_value = s->out2in;
 630   *p_dont_translate = dont_translate;
 631   if (d)
 632     *(snat_session_t **) d = s;
 633   return next;
 634 }
 635 #endif
 636
 637 static snat_session_t *
 638 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
 639                                vlib_buffer_t * b,
 640                                ip4_header_t * ip,
 641                                u32 rx_fib_index,
 642                                u32 thread_index,
 643                                f64 now,
 644                                vlib_main_t * vm, vlib_node_runtime_t * node)
 645 {
 646   clib_bihash_kv_8_8_t kv, value;
 647   clib_bihash_kv_16_8_t s_kv, s_value;
 648   snat_static_mapping_t *m;
 649   u32 old_addr, new_addr = 0;
 650   ip_csum_t sum;
 651   snat_user_t *u;
 652   dlist_elt_t *head, *elt;
 653   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 654   u32 elt_index, head_index, ses_index;
 655   snat_session_t *s;
 656   u32 outside_fib_index = sm->outside_fib_index;
 657   int i;
 658   u8 is_sm = 0;
 659   nat_outside_fib_t *outside_fib;
 660   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 661   fib_prefix_t pfx = {
 662     .fp_proto = FIB_PROTOCOL_IP4,
 663     .fp_len = 32,
 664     .fp_addr = {
 665                 .ip4.as_u32 = ip->dst_address.as_u32,
 666                 },
 667   };
 668
 669   switch (vec_len (sm->outside_fibs))
 670     {
 671     case 0:
 672       outside_fib_index = sm->outside_fib_index;
 673       break;
 674     case 1:
 675       outside_fib_index = sm->outside_fibs[0].fib_index;
 676       break;
 677     default:
 678       /* *INDENT-OFF* */
 679       vec_foreach (outside_fib, sm->outside_fibs)
 680         {
 681           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 682           if (FIB_NODE_INDEX_INVALID != fei)
 683             {
 684               if (fib_entry_get_resolving_interface (fei) != ~0)
 685                 {
 686                   outside_fib_index = outside_fib->fib_index;
 687                   break;
 688                 }
 689             }
 690         }
 691       /* *INDENT-ON* */
 692       break;
 693     }
 694   old_addr = ip->src_address.as_u32;
 695
 696   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
 697               rx_fib_index, 0, 0);
 698
 699   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
 700     {
 701       s = pool_elt_at_index (tsm->sessions, s_value.value);
 702       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 703     }
 704   else
 705     {
 706       if (PREDICT_FALSE (nat44_maximum_sessions_exceeded (sm, thread_index)))
 707         {
 708           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
 709           nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
 710           nat_elog_notice ("maximum sessions exceeded");
 711           return 0;
 712         }
 713
 714       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
 715                                   thread_index);
 716       if (!u)
 717         {
 718           b->error = node->errors[NAT_IN2OUT_ED_ERROR_CANNOT_CREATE_USER];
 719           nat_elog_warn ("create NAT user failed");
 720           return 0;
 721         }
 722
 723       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
 724
 725       /* Try to find static mapping first */
 726       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
 727         {
 728           m = pool_elt_at_index (sm->static_mappings, value.value);
 729           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 730           is_sm = 1;
 731           goto create_ses;
 732         }
 733       /* Fallback to 3-tuple key */
 734       else
 735         {
 736           /* Choose same out address as for TCP/UDP session to same destination */
 737           head_index = u->sessions_per_user_list_head_index;
 738           head = pool_elt_at_index (tsm->list_pool, head_index);
 739           elt_index = head->next;
 740           if (PREDICT_FALSE (elt_index == ~0))
 741             ses_index = ~0;
 742           else
 743             {
 744               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 745               ses_index = elt->value;
 746             }
 747
 748           while (ses_index != ~0)
 749             {
 750               s = pool_elt_at_index (tsm->sessions, ses_index);
 751               elt_index = elt->next;
 752               elt = pool_elt_at_index (tsm->list_pool, elt_index);
 753               ses_index = elt->value;
 754
 755               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
 756                 {
 757                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
 758
 759                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
 760                               ip->protocol, outside_fib_index, 0, 0);
 761                   if (clib_bihash_search_16_8
 762                       (&tsm->out2in_ed, &s_kv, &s_value))
 763                     goto create_ses;
 764
 765                   break;
 766                 }
 767             }
 768
 769           for (i = 0; i < vec_len (sm->addresses); i++)
 770             {
 771               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
 772                           ip->protocol, outside_fib_index, 0, 0);
 773               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 774                 {
 775                   new_addr = ip->src_address.as_u32 =
 776                     sm->addresses[i].addr.as_u32;
 777                   goto create_ses;
 778                 }
 779             }
 780           return 0;
 781         }
 782
 783     create_ses:
 784       s = nat_ed_session_alloc (sm, u, thread_index, now);
 785       if (!s)
 786         {
 787           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_USER_SESS_EXCEEDED];
 788           nat44_delete_user_with_no_session (sm, u, thread_index);
 789           nat_elog_warn ("create NAT session failed");
 790           return 0;
 791         }
 792
 793       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
 794       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
 795       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
 796       s->out2in.addr.as_u32 = new_addr;
 797       s->out2in.fib_index = outside_fib_index;
 798       s->in2out.addr.as_u32 = old_addr;
 799       s->in2out.fib_index = rx_fib_index;
 800       s->in2out.port = s->out2in.port = ip->protocol;
 801       if (is_sm)
 802         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 803       user_session_increment (sm, u, is_sm);
 804
 805       /* Add to lookup tables */
 806       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
 807                   rx_fib_index, 0, 0);
 808       s_kv.value = s - tsm->sessions;
 809       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
 810         nat_elog_notice ("in2out key add failed");
 811
 812       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
 813                   outside_fib_index, 0, 0);
 814       s_kv.value = s - tsm->sessions;
 815       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
 816         nat_elog_notice ("out2in key add failed");
 817     }
 818
 819   /* Update IP checksum */
 820   sum = ip->checksum;
 821   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 822   ip->checksum = ip_csum_fold (sum);
 823
 824   /* Accounting */
 825   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
 826                                  thread_index);
 827   /* Per-user LRU list maintenance */
 828   nat44_session_update_lru (sm, s, thread_index);
 829
 830   /* Hairpinning */
 831   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 832     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
 833
 834   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 835     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
 836
 837   return s;
 838 }
 839
 840 static inline uword
 841 nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 842                                           vlib_node_runtime_t * node,
 843                                           vlib_frame_t * frame,
 844                                           int is_output_feature)
 845 {
 846   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
 847   nat_next_t next_index;
 848   snat_main_t *sm = &snat_main;
 849   f64 now = vlib_time_now (vm);
 850   u32 thread_index = vm->thread_index;
 851   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 852   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 853     0, def_slow;
 854   u32 tcp_closed_drops = 0;
 855
 856   def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
 857     NAT_NEXT_IN2OUT_ED_SLOW_PATH;
 858
 859   stats_node_index = sm->ed_in2out_node_index;
 860
 861   from = vlib_frame_vector_args (frame);
 862   n_left_from = frame->n_vectors;
 863   next_index = node->cached_next_index;
 864
 865   while (n_left_from > 0)
 866     {
 867       u32 n_left_to_next;
 868
 869       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 870
 871       while (n_left_from > 0 && n_left_to_next > 0)
 872         {
 873           u32 bi0;
 874           vlib_buffer_t *b0;
 875           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
 876             new_addr0, old_addr0;
 877           u16 old_port0, new_port0;
 878           ip4_header_t *ip0;
 879           udp_header_t *udp0;
 880           tcp_header_t *tcp0;
 881           snat_session_t *s0 = 0;
 882           clib_bihash_kv_16_8_t kv0, value0;
 883           ip_csum_t sum0;
 884
 885           /* speculatively enqueue b0 to the current next frame */
 886           bi0 = from[0];
 887           to_next[0] = bi0;
 888           from += 1;
 889           to_next += 1;
 890           n_left_from -= 1;
 891           n_left_to_next -= 1;
 892
 893           b0 = vlib_get_buffer (vm, bi0);
 894
 895           if (is_output_feature)
 896             {
 897               vnet_feature_next (&nat_buffer_opaque (b0)->arc_next, b0);
 898               iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 899             }
 900
 901           next0 = nat_buffer_opaque (b0)->arc_next;
 902
 903           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 904                                   iph_offset0);
 905
 906           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 907           rx_fib_index0 =
 908             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 909                                                  sw_if_index0);
 910
 911           if (PREDICT_FALSE (ip0->ttl == 1))
 912             {
 913               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 914               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 915                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
 916                                            0);
 917               next0 = NAT_NEXT_ICMP_ERROR;
 918               goto trace0;
 919             }
 920
 921           udp0 = ip4_next_header (ip0);
 922           tcp0 = (tcp_header_t *) udp0;
 923           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 924
 925           if (PREDICT_FALSE (proto0 == ~0))
 926             {
 927               next0 = def_slow;
 928               goto trace0;
 929             }
 930
 931           if (is_output_feature)
 932             {
 933               if (PREDICT_FALSE (nat_not_translate_output_feature_fwd
 934                                  (sm, ip0, thread_index, now, vm, b0)))
 935                 goto trace0;
 936             }
 937
 938           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
 939             {
 940               next0 = def_slow;
 941               goto trace0;
 942             }
 943
 944           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
 945                       ip0->protocol, rx_fib_index0,
 946                       vnet_buffer (b0)->ip.reass.l4_src_port,
 947                       vnet_buffer (b0)->ip.reass.l4_dst_port);
 948
 949           // lookup for session
 950           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
 951             {
 952               // session does not exist go slow path
 953               next0 = def_slow;
 954               goto trace0;
 955             }
 956           s0 = pool_elt_at_index (tsm->sessions, value0.value);
 957
 958           if (s0->tcp_close_timestamp)
 959             {
 960               if (now >= s0->tcp_close_timestamp)
 961                 {
 962                   // session is closed, go slow path
 963                   next0 = def_slow;
 964                 }
 965               else
 966                 {
 967                   // session in transitory timeout, drop
 968                   ++tcp_closed_drops;
 969                   b0->error = node->errors[NAT_IN2OUT_ED_ERROR_TCP_CLOSED];
 970                   next0 = NAT_NEXT_DROP;
 971                 }
 972               goto trace0;
 973             }
 974
 975           // drop if session expired
 976           u64 sess_timeout_time;
 977           sess_timeout_time = s0->last_heard +
 978             (f64) nat44_session_get_timeout (sm, s0);
 979           if (now >= sess_timeout_time)
 980             {
 981               // delete session
 982               nat_free_session_data (sm, s0, thread_index, 0);
 983               nat44_delete_session (sm, s0, thread_index);
 984
 985               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_SESS_EXPIRED];
 986               next0 = NAT_NEXT_DROP;
 987               goto trace0;
 988             }
 989
 990           b0->flags |= VNET_BUFFER_F_IS_NATED;
 991
 992           if (!is_output_feature)
 993             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
 994
 995           old_addr0 = ip0->src_address.as_u32;
 996           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
 997           sum0 = ip0->checksum;
 998           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 999                                  src_address);
1000           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1001             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1002                                    s0->ext_host_addr.as_u32, ip4_header_t,
1003                                    dst_address);
1004           ip0->checksum = ip_csum_fold (sum0);
1005
1006           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1007
1008           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1009             {
1010               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1011                 {
1012                   new_port0 = udp0->src_port = s0->out2in.port;
1013                   sum0 = tcp0->checksum;
1014                   sum0 =
1015                     ip_csum_update (sum0, old_addr0, new_addr0,
1016                                     ip4_header_t, dst_address);
1017                   sum0 =
1018                     ip_csum_update (sum0, old_port0, new_port0,
1019                                     ip4_header_t, length);
1020                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1021                     {
1022                       sum0 =
1023                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1024                                         s0->ext_host_addr.as_u32,
1025                                         ip4_header_t, dst_address);
1026                       sum0 =
1027                         ip_csum_update (sum0,
1028                                         vnet_buffer (b0)->ip.
1029                                         reass.l4_dst_port, s0->ext_host_port,
1030                                         ip4_header_t, length);
1031                       tcp0->dst_port = s0->ext_host_port;
1032                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1033                     }
1034                   mss_clamping (sm, tcp0, &sum0);
1035                   tcp0->checksum = ip_csum_fold (sum0);
1036                 }
1037               tcp_packets++;
1038               if (nat44_set_tcp_session_state_i2o
1039                   (sm, now, s0, b0, thread_index))
1040                 goto trace0;
1041             }
1042           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1043                    && udp0->checksum)
1044             {
1045               new_port0 = udp0->src_port = s0->out2in.port;
1046               sum0 = udp0->checksum;
1047               sum0 =
1048                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1049                                 dst_address);
1050               sum0 =
1051                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1052                                 length);
1053               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1054                 {
1055                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1056                                          s0->ext_host_addr.as_u32,
1057                                          ip4_header_t, dst_address);
1058                   sum0 =
1059                     ip_csum_update (sum0,
1060                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1061                                     s0->ext_host_port, ip4_header_t, length);
1062                   udp0->dst_port = s0->ext_host_port;
1063                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1064                 }
1065               udp0->checksum = ip_csum_fold (sum0);
1066               udp_packets++;
1067             }
1068           else
1069             {
1070               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1071                 {
1072                   new_port0 = udp0->src_port = s0->out2in.port;
1073                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1074                     {
1075                       udp0->dst_port = s0->ext_host_port;
1076                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1077                     }
1078                   udp_packets++;
1079                 }
1080             }
1081
1082           /* Accounting */
1083           nat44_session_update_counters (s0, now,
1084                                          vlib_buffer_length_in_chain
1085                                          (vm, b0), thread_index);
1086           /* Per-user LRU list maintenance */
1087           nat44_session_update_lru (sm, s0, thread_index);
1088
1089         trace0:
1090           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1091                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1092             {
1093               nat_in2out_ed_trace_t *t =
1094                 vlib_add_trace (vm, node, b0, sizeof (*t));
1095               t->sw_if_index = sw_if_index0;
1096               t->next_index = next0;
1097               t->is_slow_path = 0;
1098
1099               if (s0)
1100                 t->session_index = s0 - tsm->sessions;
1101               else
1102                 t->session_index = ~0;
1103             }
1104
1105           pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1106           /* verify speculative enqueue, maybe switch current next frame */
1107           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1108                                            to_next, n_left_to_next,
1109                                            bi0, next0);
1110         }
1111
1112       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1113     }
1114
1115   vlib_node_increment_counter (vm, stats_node_index,
1116                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1117                                pkts_processed);
1118   vlib_node_increment_counter (vm, stats_node_index,
1119                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1120   vlib_node_increment_counter (vm, stats_node_index,
1121                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1122   vlib_node_increment_counter (vm, stats_node_index,
1123                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1124                                icmp_packets);
1125   vlib_node_increment_counter (vm, stats_node_index,
1126                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1127                                other_packets);
1128   return frame->n_vectors;
1129 }
1130
1131 static inline uword
1132 nat44_ed_in2out_slow_path_node_fn_inline (vlib_main_t * vm,
1133                                           vlib_node_runtime_t * node,
1134                                           vlib_frame_t * frame,
1135                                           int is_output_feature)
1136 {
1137   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
1138   nat_next_t next_index;
1139   snat_main_t *sm = &snat_main;
1140   f64 now = vlib_time_now (vm);
1141   u32 thread_index = vm->thread_index;
1142   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1143   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets = 0;
1144
1145   stats_node_index = sm->ed_in2out_slowpath_node_index;
1146
1147   from = vlib_frame_vector_args (frame);
1148   n_left_from = frame->n_vectors;
1149   next_index = node->cached_next_index;
1150
1151   while (n_left_from > 0)
1152     {
1153       u32 n_left_to_next;
1154
1155       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1156
1157       while (n_left_from > 0 && n_left_to_next > 0)
1158         {
1159           u32 bi0;
1160           vlib_buffer_t *b0;
1161           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1162             new_addr0, old_addr0;
1163           u16 old_port0, new_port0;
1164           ip4_header_t *ip0;
1165           udp_header_t *udp0;
1166           tcp_header_t *tcp0;
1167           icmp46_header_t *icmp0;
1168           snat_session_t *s0 = 0;
1169           clib_bihash_kv_16_8_t kv0, value0;
1170           ip_csum_t sum0;
1171
1172           /* speculatively enqueue b0 to the current next frame */
1173           bi0 = from[0];
1174           to_next[0] = bi0;
1175           from += 1;
1176           to_next += 1;
1177           n_left_from -= 1;
1178           n_left_to_next -= 1;
1179
1180           b0 = vlib_get_buffer (vm, bi0);
1181
1182           if (is_output_feature)
1183             iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1184
1185           next0 = nat_buffer_opaque (b0)->arc_next;
1186
1187           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1188                                   iph_offset0);
1189
1190           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1191           rx_fib_index0 =
1192             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1193                                                  sw_if_index0);
1194
1195           if (PREDICT_FALSE (ip0->ttl == 1))
1196             {
1197               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1198               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1199                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1200                                            0);
1201               next0 = NAT_NEXT_ICMP_ERROR;
1202               goto trace0;
1203             }
1204
1205           udp0 = ip4_next_header (ip0);
1206           tcp0 = (tcp_header_t *) udp0;
1207           icmp0 = (icmp46_header_t *) udp0;
1208           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1209
1210           if (PREDICT_FALSE (proto0 == ~0))
1211             {
1212               s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1213                                                   rx_fib_index0,
1214                                                   thread_index, now,
1215                                                   vm, node);
1216               if (!s0)
1217                 next0 = NAT_NEXT_DROP;
1218
1219               other_packets++;
1220               goto trace0;
1221             }
1222
1223           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1224             {
1225               next0 = icmp_in2out_ed_slow_path
1226                 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1227                  node, next0, now, thread_index, &s0);
1228               icmp_packets++;
1229               goto trace0;
1230             }
1231
1232           // move down
1233           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1234                       ip0->protocol, rx_fib_index0,
1235                       vnet_buffer (b0)->ip.reass.l4_src_port,
1236                       vnet_buffer (b0)->ip.reass.l4_dst_port);
1237
1238           if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1239             {
1240               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1241
1242               if (s0->tcp_close_timestamp && now >= s0->tcp_close_timestamp)
1243                 {
1244                   nat_free_session_data (sm, s0, thread_index, 0);
1245                   nat44_delete_session (sm, s0, thread_index);
1246                   s0 = NULL;
1247                 }
1248             }
1249
1250           if (!s0)
1251             {
1252               if (is_output_feature)
1253                 {
1254                   if (PREDICT_FALSE
1255                       (nat44_ed_not_translate_output_feature
1256                        (sm, ip0, ip0->protocol,
1257                         vnet_buffer (b0)->ip.reass.l4_src_port,
1258                         vnet_buffer (b0)->ip.reass.l4_dst_port,
1259                         thread_index, sw_if_index0,
1260                         vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1261                     goto trace0;
1262
1263                   /*
1264                    * Send DHCP packets to the ipv4 stack, or we won't
1265                    * be able to use dhcp client on the outside interface
1266                    */
1267                   if (PREDICT_FALSE
1268                       (proto0 == SNAT_PROTOCOL_UDP
1269                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1270                            clib_host_to_net_u16 (UDP_DST_PORT_dhcp_to_server))
1271                        && ip0->dst_address.as_u32 == 0xffffffff))
1272                     goto trace0;
1273                 }
1274               else
1275                 {
1276                   if (PREDICT_FALSE
1277                       (nat44_ed_not_translate
1278                        (sm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1279                         thread_index)))
1280                     goto trace0;
1281                 }
1282
1283               next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1284                                     next0, thread_index, now);
1285
1286               if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1287                 goto trace0;
1288
1289               if (PREDICT_FALSE (!s0))
1290                 goto trace0;
1291
1292             }
1293
1294           b0->flags |= VNET_BUFFER_F_IS_NATED;
1295
1296           if (!is_output_feature)
1297             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1298
1299           old_addr0 = ip0->src_address.as_u32;
1300           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1301           sum0 = ip0->checksum;
1302           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1303                                  src_address);
1304           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1305             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1306                                    s0->ext_host_addr.as_u32, ip4_header_t,
1307                                    dst_address);
1308           ip0->checksum = ip_csum_fold (sum0);
1309
1310           old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1311
1312           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1313             {
1314               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1315                 {
1316                   new_port0 = udp0->src_port = s0->out2in.port;
1317                   sum0 = tcp0->checksum;
1318                   sum0 =
1319                     ip_csum_update (sum0, old_addr0, new_addr0,
1320                                     ip4_header_t, dst_address);
1321                   sum0 =
1322                     ip_csum_update (sum0, old_port0, new_port0,
1323                                     ip4_header_t, length);
1324                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1325                     {
1326                       sum0 =
1327                         ip_csum_update (sum0, ip0->dst_address.as_u32,
1328                                         s0->ext_host_addr.as_u32,
1329                                         ip4_header_t, dst_address);
1330                       sum0 =
1331                         ip_csum_update (sum0,
1332                                         vnet_buffer (b0)->ip.
1333                                         reass.l4_dst_port, s0->ext_host_port,
1334                                         ip4_header_t, length);
1335                       tcp0->dst_port = s0->ext_host_port;
1336                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1337                     }
1338                   mss_clamping (sm, tcp0, &sum0);
1339                   tcp0->checksum = ip_csum_fold (sum0);
1340                 }
1341               tcp_packets++;
1342               if (nat44_set_tcp_session_state_i2o
1343                   (sm, now, s0, b0, thread_index))
1344                 goto trace0;
1345             }
1346           else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1347                    && udp0->checksum)
1348             {
1349               new_port0 = udp0->src_port = s0->out2in.port;
1350               sum0 = udp0->checksum;
1351               sum0 =
1352                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1353                                 dst_address);
1354               sum0 =
1355                 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1356                                 length);
1357               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1358                 {
1359                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1360                                          s0->ext_host_addr.as_u32,
1361                                          ip4_header_t, dst_address);
1362                   sum0 =
1363                     ip_csum_update (sum0,
1364                                     vnet_buffer (b0)->ip.reass.l4_dst_port,
1365                                     s0->ext_host_port, ip4_header_t, length);
1366                   udp0->dst_port = s0->ext_host_port;
1367                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1368                 }
1369               udp0->checksum = ip_csum_fold (sum0);
1370               udp_packets++;
1371             }
1372           else
1373             {
1374               if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1375                 {
1376                   new_port0 = udp0->src_port = s0->out2in.port;
1377                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1378                     {
1379                       udp0->dst_port = s0->ext_host_port;
1380                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1381                     }
1382                   udp_packets++;
1383                 }
1384             }
1385
1386           /* Accounting */
1387           nat44_session_update_counters (s0, now,
1388                                          vlib_buffer_length_in_chain
1389                                          (vm, b0), thread_index);
1390           /* Per-user LRU list maintenance */
1391           nat44_session_update_lru (sm, s0, thread_index);
1392
1393         trace0:
1394           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1395                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1396             {
1397               nat_in2out_ed_trace_t *t =
1398                 vlib_add_trace (vm, node, b0, sizeof (*t));
1399               t->sw_if_index = sw_if_index0;
1400               t->next_index = next0;
1401               t->is_slow_path = 1;
1402
1403               if (s0)
1404                 t->session_index = s0 - tsm->sessions;
1405               else
1406                 t->session_index = ~0;
1407             }
1408
1409           pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1410
1411           /* verify speculative enqueue, maybe switch current next frame */
1412           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1413                                            to_next, n_left_to_next,
1414                                            bi0, next0);
1415         }
1416
1417       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1418     }
1419
1420   vlib_node_increment_counter (vm, stats_node_index,
1421                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1422                                pkts_processed);
1423   vlib_node_increment_counter (vm, stats_node_index,
1424                                NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1425   vlib_node_increment_counter (vm, stats_node_index,
1426                                NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1427   vlib_node_increment_counter (vm, stats_node_index,
1428                                NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1429                                icmp_packets);
1430   vlib_node_increment_counter (vm, stats_node_index,
1431                                NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1432                                other_packets);
1433   return frame->n_vectors;
1434 }
1435
1436 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
1437                                      vlib_node_runtime_t * node,
1438                                      vlib_frame_t * frame)
1439 {
1440   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
1441 }
1442
1443 /* *INDENT-OFF* */
1444 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1445   .name = "nat44-ed-in2out",
1446   .vector_size = sizeof (u32),
1447   .sibling_of = "nat-default",
1448   .format_trace = format_nat_in2out_ed_trace,
1449   .type = VLIB_NODE_TYPE_INTERNAL,
1450   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1451   .error_strings = nat_in2out_ed_error_strings,
1452   .runtime_data_bytes = sizeof (snat_runtime_t),
1453 };
1454 /* *INDENT-ON* */
1455
1456 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
1457                                             vlib_node_runtime_t * node,
1458                                             vlib_frame_t * frame)
1459 {
1460   return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
1461 }
1462
1463 /* *INDENT-OFF* */
1464 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1465   .name = "nat44-ed-in2out-output",
1466   .vector_size = sizeof (u32),
1467   .sibling_of = "nat-default",
1468   .format_trace = format_nat_in2out_ed_trace,
1469   .type = VLIB_NODE_TYPE_INTERNAL,
1470   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1471   .error_strings = nat_in2out_ed_error_strings,
1472   .runtime_data_bytes = sizeof (snat_runtime_t),
1473 };
1474 /* *INDENT-ON* */
1475
1476 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
1477                                               vlib_node_runtime_t *
1478                                               node, vlib_frame_t * frame)
1479 {
1480   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 0);
1481 }
1482
1483 /* *INDENT-OFF* */
1484 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1485   .name = "nat44-ed-in2out-slowpath",
1486   .vector_size = sizeof (u32),
1487   .sibling_of = "nat-default",
1488   .format_trace = format_nat_in2out_ed_trace,
1489   .type = VLIB_NODE_TYPE_INTERNAL,
1490   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1491   .error_strings = nat_in2out_ed_error_strings,
1492   .runtime_data_bytes = sizeof (snat_runtime_t),
1493 };
1494 /* *INDENT-ON* */
1495
1496 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
1497                                                      vlib_node_runtime_t
1498                                                      * node,
1499                                                      vlib_frame_t * frame)
1500 {
1501   return nat44_ed_in2out_slow_path_node_fn_inline (vm, node, frame, 1);
1502 }
1503
1504 /* *INDENT-OFF* */
1505 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1506   .name = "nat44-ed-in2out-output-slowpath",
1507   .vector_size = sizeof (u32),
1508   .sibling_of = "nat-default",
1509   .format_trace = format_nat_in2out_ed_trace,
1510   .type = VLIB_NODE_TYPE_INTERNAL,
1511   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1512   .error_strings = nat_in2out_ed_error_strings,
1513   .runtime_data_bytes = sizeof (snat_runtime_t),
1514 };
1515 /* *INDENT-ON* */
1516
1517 static u8 *
1518 format_nat_pre_trace (u8 * s, va_list * args)
1519 {
1520   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1521   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1522   nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1523   return format (s, "in2out next_index %d", t->next_index);
1524 }
1525
1526 VLIB_NODE_FN (nat_pre_in2out_node)
1527   (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1528 {
1529   return nat_pre_node_fn_inline (vm, node, frame,
1530                                  NAT_NEXT_IN2OUT_ED_FAST_PATH);
1531 }
1532
1533 /* *INDENT-OFF* */
1534 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
1535   .name = "nat-pre-in2out",
1536   .vector_size = sizeof (u32),
1537   .sibling_of = "nat-default",
1538   .format_trace = format_nat_pre_trace,
1539   .type = VLIB_NODE_TYPE_INTERNAL,
1540   .n_errors = 0,
1541 };
1542 /* *INDENT-ON* */
1543
1544 /*
1545  * fd.io coding-style-patch-verification: ON
1546  *
1547  * Local Variables:
1548  * eval: (c-set-style "gnu")
1549  * End:
1550  */