src/plugins/nat/nat44_hairpinning.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 hairpinning
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/fib/ip4_fib.h>
  23 #include <nat/nat.h>
  24 #include <nat/nat_inlines.h>
  25
  26 typedef enum
  27 {
  28   SNAT_HAIRPIN_SRC_NEXT_DROP,
  29   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
  30   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
  31   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
  32   SNAT_HAIRPIN_SRC_N_NEXT,
  33 } snat_hairpin_src_next_t;
  34
  35 typedef enum
  36 {
  37   NAT_HAIRPIN_NEXT_LOOKUP,
  38   NAT_HAIRPIN_NEXT_DROP,
  39   NAT_HAIRPIN_N_NEXT,
  40 } nat_hairpin_next_t;
  41
  42 #define foreach_nat44_hairpin_error                       \
  43 _(PROCESSED, "NAT44 hairpinning packets processed")
  44
  45 typedef enum
  46 {
  47 #define _(sym,str) NAT44_HAIRPIN_ERROR_##sym,
  48   foreach_nat44_hairpin_error
  49 #undef _
  50     NAT44_HAIRPIN_N_ERROR,
  51 } nat44_hairpin_error_t;
  52
  53 static char *nat44_hairpin_error_strings[] = {
  54 #define _(sym,string) string,
  55   foreach_nat44_hairpin_error
  56 #undef _
  57 };
  58
  59 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
  60
  61 static_always_inline int
  62 is_hairpinning (snat_main_t * sm, ip4_address_t * dst_addr)
  63 {
  64   snat_address_t *ap;
  65   clib_bihash_kv_8_8_t kv, value;
  66   snat_session_key_t m_key;
  67
  68   /* *INDENT-OFF* */
  69   vec_foreach (ap, sm->addresses)
  70     {
  71       if (ap->addr.as_u32 == dst_addr->as_u32)
  72         return 1;
  73     }
  74   /* *INDENT-ON* */
  75
  76   m_key.addr.as_u32 = dst_addr->as_u32;
  77   m_key.fib_index = 0;
  78   m_key.port = 0;
  79   m_key.protocol = 0;
  80   kv.key = m_key.as_u64;
  81   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
  82     return 1;
  83
  84   return 0;
  85 }
  86
  87 #ifndef CLIB_MARCH_VARIANT
  88 int
  89 snat_hairpinning (snat_main_t * sm,
  90                   vlib_buffer_t * b0,
  91                   ip4_header_t * ip0,
  92                   udp_header_t * udp0,
  93                   tcp_header_t * tcp0, u32 proto0, int is_ed)
  94 {
  95   snat_session_key_t key0, sm0;
  96   snat_session_t *s0;
  97   clib_bihash_kv_8_8_t kv0, value0;
  98   ip_csum_t sum0;
  99   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 100   u16 new_dst_port0, old_dst_port0;
 101   int rv;
 102
 103   key0.addr = ip0->dst_address;
 104   key0.port = udp0->dst_port;
 105   key0.protocol = proto0;
 106   key0.fib_index = sm->outside_fib_index;
 107   kv0.key = key0.as_u64;
 108
 109   /* Check if destination is static mappings */
 110   if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 111     {
 112       new_dst_addr0 = sm0.addr.as_u32;
 113       new_dst_port0 = sm0.port;
 114       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 115     }
 116   /* or active session */
 117   else
 118     {
 119       if (sm->num_workers > 1)
 120         ti =
 121           (clib_net_to_host_u16 (udp0->dst_port) -
 122            1024) / sm->port_per_thread;
 123       else
 124         ti = sm->num_workers;
 125
 126       if (is_ed)
 127         {
 128           clib_bihash_kv_16_8_t ed_kv, ed_value;
 129           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 130                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 131                       udp0->src_port);
 132           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 133                                         &ed_kv, &ed_value);
 134           si = ed_value.value;
 135         }
 136       else
 137         {
 138           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 139                                        &value0);
 140           si = value0.value;
 141         }
 142       if (rv)
 143         return 0;
 144
 145       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 146       new_dst_addr0 = s0->in2out.addr.as_u32;
 147       new_dst_port0 = s0->in2out.port;
 148       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 149     }
 150
 151   /* Destination is behind the same NAT, use internal address and port */
 152   if (new_dst_addr0)
 153     {
 154       old_dst_addr0 = ip0->dst_address.as_u32;
 155       ip0->dst_address.as_u32 = new_dst_addr0;
 156       sum0 = ip0->checksum;
 157       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 158                              ip4_header_t, dst_address);
 159       ip0->checksum = ip_csum_fold (sum0);
 160
 161       old_dst_port0 = tcp0->dst;
 162       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
 163         {
 164           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 165             {
 166               tcp0->dst = new_dst_port0;
 167               sum0 = tcp0->checksum;
 168               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 169                                      ip4_header_t, dst_address);
 170               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 171                                      ip4_header_t /* cheat */ , length);
 172               tcp0->checksum = ip_csum_fold (sum0);
 173             }
 174           else
 175             {
 176               udp0->dst_port = new_dst_port0;
 177               udp0->checksum = 0;
 178             }
 179         }
 180       else
 181         {
 182           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 183             {
 184               sum0 = tcp0->checksum;
 185               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 186                                      ip4_header_t, dst_address);
 187               tcp0->checksum = ip_csum_fold (sum0);
 188             }
 189         }
 190       return 1;
 191     }
 192   return 0;
 193 }
 194 #endif
 195
 196 #ifndef CLIB_MARCH_VARIANT
 197 u32
 198 snat_icmp_hairpinning (snat_main_t * sm,
 199                        vlib_buffer_t * b0,
 200                        ip4_header_t * ip0, icmp46_header_t * icmp0, int is_ed)
 201 {
 202   snat_session_key_t key0;
 203   clib_bihash_kv_8_8_t kv0, value0;
 204   u32 old_dst_addr0, new_dst_addr0;
 205   u32 old_addr0, new_addr0;
 206   u16 old_port0, new_port0;
 207   u16 old_checksum0, new_checksum0;
 208   u32 si, ti = 0;
 209   ip_csum_t sum0;
 210   snat_session_t *s0;
 211   snat_static_mapping_t *m0;
 212
 213   if (icmp_type_is_error_message
 214       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 215     {
 216       ip4_header_t *inner_ip0 = 0;
 217       tcp_udp_header_t *l4_header = 0;
 218
 219       inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
 220       l4_header = ip4_next_header (inner_ip0);
 221       u32 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 222
 223       if (protocol != SNAT_PROTOCOL_TCP && protocol != SNAT_PROTOCOL_UDP)
 224         return 1;
 225
 226       if (is_ed)
 227         {
 228           clib_bihash_kv_16_8_t ed_kv, ed_value;
 229           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 230                       inner_ip0->protocol, sm->outside_fib_index,
 231                       l4_header->src_port, l4_header->dst_port);
 232           if (clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 233                                        &ed_kv, &ed_value))
 234             return 1;
 235           si = ed_value.value;
 236         }
 237       else
 238         {
 239           key0.addr = ip0->dst_address;
 240           key0.port = l4_header->src_port;
 241           key0.protocol = protocol;
 242           key0.fib_index = sm->outside_fib_index;
 243           kv0.key = key0.as_u64;
 244           if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 245                                       &value0))
 246             return 1;
 247           si = value0.value;
 248         }
 249       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 250       new_dst_addr0 = s0->in2out.addr.as_u32;
 251       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 252
 253       /* update inner source IP address */
 254       old_addr0 = inner_ip0->src_address.as_u32;
 255       inner_ip0->src_address.as_u32 = new_dst_addr0;
 256       new_addr0 = inner_ip0->src_address.as_u32;
 257       sum0 = icmp0->checksum;
 258       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 259                              src_address);
 260       icmp0->checksum = ip_csum_fold (sum0);
 261
 262       /* update inner IP header checksum */
 263       old_checksum0 = inner_ip0->checksum;
 264       sum0 = inner_ip0->checksum;
 265       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 266                              src_address);
 267       inner_ip0->checksum = ip_csum_fold (sum0);
 268       new_checksum0 = inner_ip0->checksum;
 269       sum0 = icmp0->checksum;
 270       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 271                              checksum);
 272       icmp0->checksum = ip_csum_fold (sum0);
 273
 274       /* update inner source port */
 275       old_port0 = l4_header->src_port;
 276       l4_header->src_port = s0->in2out.port;
 277       new_port0 = l4_header->src_port;
 278       sum0 = icmp0->checksum;
 279       sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
 280                              src_port);
 281       icmp0->checksum = ip_csum_fold (sum0);
 282     }
 283   else
 284     {
 285       key0.addr = ip0->dst_address;
 286       key0.port = 0;
 287       key0.protocol = 0;
 288       key0.fib_index = sm->outside_fib_index;
 289       kv0.key = key0.as_u64;
 290
 291       if (clib_bihash_search_8_8
 292           (&sm->static_mapping_by_external, &kv0, &value0))
 293         {
 294           if (!is_ed)
 295             {
 296               icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 297               u16 icmp_id0 = echo0->identifier;
 298               key0.addr = ip0->dst_address;
 299               key0.port = icmp_id0;
 300               key0.protocol = SNAT_PROTOCOL_ICMP;
 301               key0.fib_index = sm->outside_fib_index;
 302               kv0.key = key0.as_u64;
 303               if (sm->num_workers > 1)
 304                 ti =
 305                   (clib_net_to_host_u16 (icmp_id0) -
 306                    1024) / sm->port_per_thread;
 307               else
 308                 ti = sm->num_workers;
 309               int rv =
 310                 clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 311                                         &value0);
 312               if (!rv)
 313                 {
 314                   si = value0.value;
 315                   s0 =
 316                     pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 317                   new_dst_addr0 = s0->in2out.addr.as_u32;
 318                   vnet_buffer (b0)->sw_if_index[VLIB_TX] =
 319                     s0->in2out.fib_index;
 320                   echo0->identifier = s0->in2out.port;
 321                   sum0 = icmp0->checksum;
 322                   sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 323                                          icmp_echo_header_t, identifier);
 324                   icmp0->checksum = ip_csum_fold (sum0);
 325                   goto change_addr;
 326                 }
 327             }
 328
 329           return 1;
 330         }
 331
 332       m0 = pool_elt_at_index (sm->static_mappings, value0.value);
 333
 334       new_dst_addr0 = m0->local_addr.as_u32;
 335       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 336         vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
 337     }
 338 change_addr:
 339   /* Destination is behind the same NAT, use internal address and port */
 340   if (new_dst_addr0)
 341     {
 342       old_dst_addr0 = ip0->dst_address.as_u32;
 343       ip0->dst_address.as_u32 = new_dst_addr0;
 344       sum0 = ip0->checksum;
 345       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 346                              ip4_header_t, dst_address);
 347       ip0->checksum = ip_csum_fold (sum0);
 348     }
 349   return 0;
 350 }
 351 #endif
 352
 353 #ifndef CLIB_MARCH_VARIANT
 354 void
 355 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
 356                                   vlib_buffer_t * b, ip4_header_t * ip)
 357 {
 358   clib_bihash_kv_8_8_t kv, value;
 359   snat_static_mapping_t *m;
 360   u32 old_addr, new_addr;
 361   ip_csum_t sum;
 362
 363   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 364   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 365     return;
 366
 367   m = pool_elt_at_index (sm->static_mappings, value.value);
 368
 369   old_addr = ip->dst_address.as_u32;
 370   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 371   sum = ip->checksum;
 372   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 373   ip->checksum = ip_csum_fold (sum);
 374
 375   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 376     vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 377 }
 378 #endif
 379
 380 #ifndef CLIB_MARCH_VARIANT
 381 void
 382 nat44_ed_hairpinning_unknown_proto (snat_main_t * sm,
 383                                     vlib_buffer_t * b, ip4_header_t * ip)
 384 {
 385   u32 old_addr, new_addr = 0, ti = 0;
 386   clib_bihash_kv_8_8_t kv, value;
 387   clib_bihash_kv_16_8_t s_kv, s_value;
 388   snat_static_mapping_t *m;
 389   ip_csum_t sum;
 390   snat_session_t *s;
 391   snat_main_per_thread_data_t *tsm;
 392
 393   if (sm->num_workers > 1)
 394     ti = sm->worker_out2in_cb (b, ip, sm->outside_fib_index, 0);
 395   else
 396     ti = sm->num_workers;
 397   tsm = &sm->per_thread_data[ti];
 398
 399   old_addr = ip->dst_address.as_u32;
 400   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 401               sm->outside_fib_index, 0, 0);
 402   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 403     {
 404       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 405       if (clib_bihash_search_8_8
 406           (&sm->static_mapping_by_external, &kv, &value))
 407         return;
 408
 409       m = pool_elt_at_index (sm->static_mappings, value.value);
 410       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 411         vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 412       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 413     }
 414   else
 415     {
 416       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
 417       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 418         vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 419       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 420     }
 421   sum = ip->checksum;
 422   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 423   ip->checksum = ip_csum_fold (sum);
 424 }
 425 #endif
 426
 427 #ifndef CLIB_MARCH_VARIANT
 428 void
 429 nat44_reass_hairpinning (snat_main_t * sm,
 430                          vlib_buffer_t * b0,
 431                          ip4_header_t * ip0,
 432                          u16 sport, u16 dport, u32 proto0, int is_ed)
 433 {
 434   snat_session_key_t key0, sm0;
 435   snat_session_t *s0;
 436   clib_bihash_kv_8_8_t kv0, value0;
 437   ip_csum_t sum0;
 438   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 439   u16 new_dst_port0, old_dst_port0;
 440   udp_header_t *udp0;
 441   tcp_header_t *tcp0;
 442   int rv;
 443
 444   key0.addr = ip0->dst_address;
 445   key0.port = dport;
 446   key0.protocol = proto0;
 447   key0.fib_index = sm->outside_fib_index;
 448   kv0.key = key0.as_u64;
 449
 450   udp0 = ip4_next_header (ip0);
 451
 452   /* Check if destination is static mappings */
 453   if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 454     {
 455       new_dst_addr0 = sm0.addr.as_u32;
 456       new_dst_port0 = sm0.port;
 457       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 458     }
 459   /* or active sessions */
 460   else
 461     {
 462       if (sm->num_workers > 1)
 463         ti =
 464           (clib_net_to_host_u16 (udp0->dst_port) -
 465            1024) / sm->port_per_thread;
 466       else
 467         ti = sm->num_workers;
 468
 469       if (is_ed)
 470         {
 471           clib_bihash_kv_16_8_t ed_kv, ed_value;
 472           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 473                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 474                       udp0->src_port);
 475           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 476                                         &ed_kv, &ed_value);
 477           si = ed_value.value;
 478         }
 479       else
 480         {
 481           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 482                                        &value0);
 483           si = value0.value;
 484         }
 485       if (!rv)
 486         {
 487           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 488           new_dst_addr0 = s0->in2out.addr.as_u32;
 489           new_dst_port0 = s0->in2out.port;
 490           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 491         }
 492     }
 493
 494   /* Destination is behind the same NAT, use internal address and port */
 495   if (new_dst_addr0)
 496     {
 497       old_dst_addr0 = ip0->dst_address.as_u32;
 498       ip0->dst_address.as_u32 = new_dst_addr0;
 499       sum0 = ip0->checksum;
 500       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 501                              ip4_header_t, dst_address);
 502       ip0->checksum = ip_csum_fold (sum0);
 503
 504       old_dst_port0 = dport;
 505       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0 &&
 506                         ip4_is_first_fragment (ip0)))
 507         {
 508           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 509             {
 510               tcp0 = ip4_next_header (ip0);
 511               tcp0->dst = new_dst_port0;
 512               sum0 = tcp0->checksum;
 513               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 514                                      ip4_header_t, dst_address);
 515               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 516                                      ip4_header_t /* cheat */ , length);
 517               tcp0->checksum = ip_csum_fold (sum0);
 518             }
 519           else
 520             {
 521               udp0->dst_port = new_dst_port0;
 522               udp0->checksum = 0;
 523             }
 524         }
 525       else
 526         {
 527           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 528             {
 529               tcp0 = ip4_next_header (ip0);
 530               sum0 = tcp0->checksum;
 531               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 532                                      ip4_header_t, dst_address);
 533               tcp0->checksum = ip_csum_fold (sum0);
 534             }
 535         }
 536     }
 537 }
 538 #endif
 539
 540 static inline uword
 541 nat44_hairpinning_fn_inline (vlib_main_t * vm,
 542                              vlib_node_runtime_t * node,
 543                              vlib_frame_t * frame, int is_ed)
 544 {
 545   u32 n_left_from, *from, *to_next, stats_node_index;
 546   nat_hairpin_next_t next_index;
 547   u32 pkts_processed = 0;
 548   snat_main_t *sm = &snat_main;
 549   vnet_feature_main_t *fm = &feature_main;
 550   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
 551   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
 552
 553   stats_node_index = is_ed ? sm->ed_hairpinning_node_index :
 554     sm->hairpinning_node_index;
 555   from = vlib_frame_vector_args (frame);
 556   n_left_from = frame->n_vectors;
 557   next_index = node->cached_next_index;
 558
 559   while (n_left_from > 0)
 560     {
 561       u32 n_left_to_next;
 562
 563       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 564
 565       while (n_left_from > 0 && n_left_to_next > 0)
 566         {
 567           u32 bi0;
 568           vlib_buffer_t *b0;
 569           u32 next0;
 570           ip4_header_t *ip0;
 571           u32 proto0;
 572           udp_header_t *udp0;
 573           tcp_header_t *tcp0;
 574
 575           /* speculatively enqueue b0 to the current next frame */
 576           bi0 = from[0];
 577           to_next[0] = bi0;
 578           from += 1;
 579           to_next += 1;
 580           n_left_from -= 1;
 581           n_left_to_next -= 1;
 582
 583           b0 = vlib_get_buffer (vm, bi0);
 584           ip0 = vlib_buffer_get_current (b0);
 585           udp0 = ip4_next_header (ip0);
 586           tcp0 = (tcp_header_t *) udp0;
 587
 588           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 589
 590           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
 591                                 &next0, 0);
 592
 593           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
 594             next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 595
 596           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 597
 598           /* verify speculative enqueue, maybe switch current next frame */
 599           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 600                                            to_next, n_left_to_next,
 601                                            bi0, next0);
 602         }
 603
 604       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 605     }
 606
 607   vlib_node_increment_counter (vm, stats_node_index,
 608                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 609   return frame->n_vectors;
 610 }
 611
 612 VLIB_NODE_FN (nat44_hairpinning_node) (vlib_main_t * vm,
 613                                        vlib_node_runtime_t * node,
 614                                        vlib_frame_t * frame)
 615 {
 616   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
 617 }
 618
 619 /* *INDENT-OFF* */
 620 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
 621   .name = "nat44-hairpinning",
 622   .vector_size = sizeof (u32),
 623   .type = VLIB_NODE_TYPE_INTERNAL,
 624   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 625   .error_strings = nat44_hairpin_error_strings,
 626   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 627   .next_nodes = {
 628     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 629     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 630   },
 631 };
 632 /* *INDENT-ON* */
 633
 634 VLIB_NODE_FN (nat44_ed_hairpinning_node) (vlib_main_t * vm,
 635                                           vlib_node_runtime_t * node,
 636                                           vlib_frame_t * frame)
 637 {
 638   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
 639 }
 640
 641 /* *INDENT-OFF* */
 642 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
 643   .name = "nat44-ed-hairpinning",
 644   .vector_size = sizeof (u32),
 645   .type = VLIB_NODE_TYPE_INTERNAL,
 646   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 647   .error_strings = nat44_hairpin_error_strings,
 648   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 649   .next_nodes = {
 650     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 651     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 652   },
 653 };
 654 /* *INDENT-ON* */
 655
 656 static inline uword
 657 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
 658                             vlib_node_runtime_t * node,
 659                             vlib_frame_t * frame, int is_ed)
 660 {
 661   u32 n_left_from, *from, *to_next, stats_node_index;
 662   nat_hairpin_next_t next_index;
 663   u32 pkts_processed = 0;
 664   snat_main_t *sm = &snat_main;
 665
 666   stats_node_index = is_ed ? sm->ed_hairpin_dst_node_index :
 667     sm->hairpin_dst_node_index;
 668
 669   from = vlib_frame_vector_args (frame);
 670   n_left_from = frame->n_vectors;
 671   next_index = node->cached_next_index;
 672
 673   while (n_left_from > 0)
 674     {
 675       u32 n_left_to_next;
 676
 677       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 678
 679       while (n_left_from > 0 && n_left_to_next > 0)
 680         {
 681           u32 bi0;
 682           vlib_buffer_t *b0;
 683           u32 next0;
 684           ip4_header_t *ip0;
 685           u32 proto0;
 686
 687           /* speculatively enqueue b0 to the current next frame */
 688           bi0 = from[0];
 689           to_next[0] = bi0;
 690           from += 1;
 691           to_next += 1;
 692           n_left_from -= 1;
 693           n_left_to_next -= 1;
 694
 695           b0 = vlib_get_buffer (vm, bi0);
 696           next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 697           ip0 = vlib_buffer_get_current (b0);
 698
 699           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 700
 701           vnet_buffer (b0)->snat.flags = 0;
 702           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
 703             {
 704               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
 705                 {
 706                   udp_header_t *udp0 = ip4_next_header (ip0);
 707                   tcp_header_t *tcp0 = (tcp_header_t *) udp0;
 708
 709                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
 710                 }
 711               else if (proto0 == SNAT_PROTOCOL_ICMP)
 712                 {
 713                   icmp46_header_t *icmp0 = ip4_next_header (ip0);
 714
 715                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
 716                 }
 717               else
 718                 {
 719                   if (is_ed)
 720                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
 721                   else
 722                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
 723                 }
 724
 725               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
 726             }
 727
 728           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 729
 730           /* verify speculative enqueue, maybe switch current next frame */
 731           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 732                                            to_next, n_left_to_next,
 733                                            bi0, next0);
 734         }
 735
 736       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 737     }
 738
 739   vlib_node_increment_counter (vm, stats_node_index,
 740                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 741   return frame->n_vectors;
 742 }
 743
 744 VLIB_NODE_FN (snat_hairpin_dst_node) (vlib_main_t * vm,
 745                                       vlib_node_runtime_t * node,
 746                                       vlib_frame_t * frame)
 747 {
 748   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
 749 }
 750
 751 /* *INDENT-OFF* */
 752 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
 753   .name = "nat44-hairpin-dst",
 754   .vector_size = sizeof (u32),
 755   .type = VLIB_NODE_TYPE_INTERNAL,
 756   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 757   .error_strings = nat44_hairpin_error_strings,
 758   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 759   .next_nodes = {
 760     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 761     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 762   },
 763 };
 764 /* *INDENT-ON* */
 765
 766 VLIB_NODE_FN (nat44_ed_hairpin_dst_node) (vlib_main_t * vm,
 767                                           vlib_node_runtime_t * node,
 768                                           vlib_frame_t * frame)
 769 {
 770   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
 771 }
 772
 773 /* *INDENT-OFF* */
 774 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
 775   .name = "nat44-ed-hairpin-dst",
 776   .vector_size = sizeof (u32),
 777   .type = VLIB_NODE_TYPE_INTERNAL,
 778   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 779   .error_strings = nat44_hairpin_error_strings,
 780   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 781   .next_nodes = {
 782     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 783     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 784   },
 785 };
 786 /* *INDENT-ON* */
 787
 788 static inline uword
 789 snat_hairpin_src_fn_inline (vlib_main_t * vm,
 790                             vlib_node_runtime_t * node,
 791                             vlib_frame_t * frame, int is_ed)
 792 {
 793   u32 n_left_from, *from, *to_next, stats_node_index;
 794   snat_hairpin_src_next_t next_index;
 795   u32 pkts_processed = 0;
 796   snat_main_t *sm = &snat_main;
 797
 798   stats_node_index = is_ed ? sm->ed_hairpin_src_node_index :
 799     sm->hairpin_src_node_index;
 800
 801   from = vlib_frame_vector_args (frame);
 802   n_left_from = frame->n_vectors;
 803   next_index = node->cached_next_index;
 804
 805   while (n_left_from > 0)
 806     {
 807       u32 n_left_to_next;
 808
 809       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 810
 811       while (n_left_from > 0 && n_left_to_next > 0)
 812         {
 813           u32 bi0;
 814           vlib_buffer_t *b0;
 815           u32 next0;
 816           snat_interface_t *i;
 817           u32 sw_if_index0;
 818
 819           /* speculatively enqueue b0 to the current next frame */
 820           bi0 = from[0];
 821           to_next[0] = bi0;
 822           from += 1;
 823           to_next += 1;
 824           n_left_from -= 1;
 825           n_left_to_next -= 1;
 826
 827           b0 = vlib_get_buffer (vm, bi0);
 828           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 829           vnet_feature_next (&next0, b0);
 830
 831           /* *INDENT-OFF* */
 832           pool_foreach (i, sm->output_feature_interfaces,
 833           ({
 834             /* Only packets from NAT inside interface */
 835             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
 836               {
 837                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
 838                                     SNAT_FLAG_HAIRPINNING))
 839                   {
 840                     if (PREDICT_TRUE (sm->num_workers > 1))
 841                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
 842                     else
 843                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
 844                   }
 845                 break;
 846               }
 847           }));
 848           /* *INDENT-ON* */
 849
 850           pkts_processed += next0 != SNAT_HAIRPIN_SRC_NEXT_DROP;
 851
 852           /* verify speculative enqueue, maybe switch current next frame */
 853           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 854                                            to_next, n_left_to_next,
 855                                            bi0, next0);
 856         }
 857
 858       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 859     }
 860
 861   vlib_node_increment_counter (vm, stats_node_index,
 862                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 863   return frame->n_vectors;
 864 }
 865
 866 VLIB_NODE_FN (snat_hairpin_src_node) (vlib_main_t * vm,
 867                                       vlib_node_runtime_t * node,
 868                                       vlib_frame_t * frame)
 869 {
 870   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
 871 }
 872
 873 /* *INDENT-OFF* */
 874 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
 875   .name = "nat44-hairpin-src",
 876   .vector_size = sizeof (u32),
 877   .type = VLIB_NODE_TYPE_INTERNAL,
 878   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 879   .error_strings = nat44_hairpin_error_strings,
 880   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 881   .next_nodes = {
 882      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 883      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
 884      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 885      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 886   },
 887 };
 888 /* *INDENT-ON* */
 889
 890 VLIB_NODE_FN (nat44_ed_hairpin_src_node) (vlib_main_t * vm,
 891                                           vlib_node_runtime_t * node,
 892                                           vlib_frame_t * frame)
 893 {
 894   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
 895 }
 896
 897 /* *INDENT-OFF* */
 898 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
 899   .name = "nat44-ed-hairpin-src",
 900   .vector_size = sizeof (u32),
 901   .type = VLIB_NODE_TYPE_INTERNAL,
 902   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 903   .error_strings = nat44_hairpin_error_strings,
 904   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 905   .next_nodes = {
 906      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 907      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
 908      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 909      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 910   },
 911 };
 912 /* *INDENT-ON* */
 913
 914 /*
 915  * fd.io coding-style-patch-verification: ON
 916  *
 917  * Local Variables:
 918  * eval: (c-set-style "gnu")
 919  * End:
 920  */