src/plugins/nat/nat44_hairpinning.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 hairpinning
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/fib/ip4_fib.h>
  23 #include <nat/nat.h>
  24 #include <nat/nat_inlines.h>
  25 #include <nat/nat_reass.h>
  26
  27 typedef enum
  28 {
  29   SNAT_HAIRPIN_SRC_NEXT_DROP,
  30   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
  31   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
  32   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
  33   SNAT_HAIRPIN_SRC_N_NEXT,
  34 } snat_hairpin_src_next_t;
  35
  36 typedef enum
  37 {
  38   NAT_HAIRPIN_NEXT_LOOKUP,
  39   NAT_HAIRPIN_NEXT_DROP,
  40   NAT_HAIRPIN_N_NEXT,
  41 } nat_hairpin_next_t;
  42
  43 #define foreach_nat44_hairpin_error                       \
  44 _(PROCESSED, "NAT44 hairpinning packets processed")
  45
  46 typedef enum
  47 {
  48 #define _(sym,str) NAT44_HAIRPIN_ERROR_##sym,
  49   foreach_nat44_hairpin_error
  50 #undef _
  51     NAT44_HAIRPIN_N_ERROR,
  52 } nat44_hairpin_error_t;
  53
  54 static char *nat44_hairpin_error_strings[] = {
  55 #define _(sym,string) string,
  56   foreach_nat44_hairpin_error
  57 #undef _
  58 };
  59
  60 vlib_node_registration_t snat_hairpin_dst_node;
  61 vlib_node_registration_t snat_hairpin_src_node;
  62 vlib_node_registration_t nat44_hairpinning_node;
  63 vlib_node_registration_t nat44_ed_hairpin_dst_node;
  64 vlib_node_registration_t nat44_ed_hairpin_src_node;
  65 vlib_node_registration_t nat44_ed_hairpinning_node;
  66
  67 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
  68
  69 static_always_inline int
  70 is_hairpinning (snat_main_t * sm, ip4_address_t * dst_addr)
  71 {
  72   snat_address_t *ap;
  73   clib_bihash_kv_8_8_t kv, value;
  74   snat_session_key_t m_key;
  75
  76   /* *INDENT-OFF* */
  77   vec_foreach (ap, sm->addresses)
  78     {
  79       if (ap->addr.as_u32 == dst_addr->as_u32)
  80         return 1;
  81     }
  82   /* *INDENT-ON* */
  83
  84   m_key.addr.as_u32 = dst_addr->as_u32;
  85   m_key.fib_index = 0;
  86   m_key.port = 0;
  87   m_key.protocol = 0;
  88   kv.key = m_key.as_u64;
  89   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
  90     return 1;
  91
  92   return 0;
  93 }
  94
  95 int
  96 snat_hairpinning (snat_main_t * sm,
  97                   vlib_buffer_t * b0,
  98                   ip4_header_t * ip0,
  99                   udp_header_t * udp0,
 100                   tcp_header_t * tcp0, u32 proto0, int is_ed)
 101 {
 102   snat_session_key_t key0, sm0;
 103   snat_session_t *s0;
 104   clib_bihash_kv_8_8_t kv0, value0;
 105   ip_csum_t sum0;
 106   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 107   u16 new_dst_port0, old_dst_port0;
 108   int rv;
 109
 110   key0.addr = ip0->dst_address;
 111   key0.port = udp0->dst_port;
 112   key0.protocol = proto0;
 113   key0.fib_index = sm->outside_fib_index;
 114   kv0.key = key0.as_u64;
 115
 116   /* Check if destination is static mappings */
 117   if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 118     {
 119       new_dst_addr0 = sm0.addr.as_u32;
 120       new_dst_port0 = sm0.port;
 121       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 122     }
 123   /* or active session */
 124   else
 125     {
 126       if (sm->num_workers > 1)
 127         ti =
 128           (clib_net_to_host_u16 (udp0->dst_port) -
 129            1024) / sm->port_per_thread;
 130       else
 131         ti = sm->num_workers;
 132
 133       if (is_ed)
 134         {
 135           clib_bihash_kv_16_8_t ed_kv, ed_value;
 136           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 137                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 138                       udp0->src_port);
 139           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 140                                         &ed_kv, &ed_value);
 141           si = ed_value.value;
 142         }
 143       else
 144         {
 145           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 146                                        &value0);
 147           si = value0.value;
 148         }
 149       if (rv)
 150         return 0;
 151
 152       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 153       new_dst_addr0 = s0->in2out.addr.as_u32;
 154       new_dst_port0 = s0->in2out.port;
 155       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 156     }
 157
 158   /* Destination is behind the same NAT, use internal address and port */
 159   if (new_dst_addr0)
 160     {
 161       old_dst_addr0 = ip0->dst_address.as_u32;
 162       ip0->dst_address.as_u32 = new_dst_addr0;
 163       sum0 = ip0->checksum;
 164       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 165                              ip4_header_t, dst_address);
 166       ip0->checksum = ip_csum_fold (sum0);
 167
 168       old_dst_port0 = tcp0->dst;
 169       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
 170         {
 171           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 172             {
 173               tcp0->dst = new_dst_port0;
 174               sum0 = tcp0->checksum;
 175               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 176                                      ip4_header_t, dst_address);
 177               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 178                                      ip4_header_t /* cheat */ , length);
 179               tcp0->checksum = ip_csum_fold (sum0);
 180             }
 181           else
 182             {
 183               udp0->dst_port = new_dst_port0;
 184               udp0->checksum = 0;
 185             }
 186         }
 187       else
 188         {
 189           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 190             {
 191               sum0 = tcp0->checksum;
 192               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 193                                      ip4_header_t, dst_address);
 194               tcp0->checksum = ip_csum_fold (sum0);
 195             }
 196         }
 197       return 1;
 198     }
 199   return 0;
 200 }
 201
 202 u32
 203 snat_icmp_hairpinning (snat_main_t * sm,
 204                        vlib_buffer_t * b0,
 205                        ip4_header_t * ip0, icmp46_header_t * icmp0, int is_ed)
 206 {
 207   snat_session_key_t key0;
 208   clib_bihash_kv_8_8_t kv0, value0;
 209   u32 old_dst_addr0, new_dst_addr0;
 210   u32 old_addr0, new_addr0;
 211   u16 old_port0, new_port0;
 212   u16 old_checksum0, new_checksum0;
 213   u32 si, ti = 0;
 214   ip_csum_t sum0;
 215   snat_session_t *s0;
 216   snat_static_mapping_t *m0;
 217
 218   if (icmp_is_error_message (icmp0))
 219     {
 220       ip4_header_t *inner_ip0 = 0;
 221       tcp_udp_header_t *l4_header = 0;
 222
 223       inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
 224       l4_header = ip4_next_header (inner_ip0);
 225       u32 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 226
 227       if (protocol != SNAT_PROTOCOL_TCP && protocol != SNAT_PROTOCOL_UDP)
 228         return 1;
 229
 230       if (is_ed)
 231         {
 232           clib_bihash_kv_16_8_t ed_kv, ed_value;
 233           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 234                       inner_ip0->protocol, sm->outside_fib_index,
 235                       l4_header->src_port, l4_header->dst_port);
 236           if (clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 237                                        &ed_kv, &ed_value))
 238             return 1;
 239           si = ed_value.value;
 240         }
 241       else
 242         {
 243           key0.addr = ip0->dst_address;
 244           key0.port = l4_header->src_port;
 245           key0.protocol = protocol;
 246           key0.fib_index = sm->outside_fib_index;
 247           kv0.key = key0.as_u64;
 248           if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 249                                       &value0))
 250             return 1;
 251           si = value0.value;
 252         }
 253       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 254       new_dst_addr0 = s0->in2out.addr.as_u32;
 255       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 256
 257       /* update inner source IP address */
 258       old_addr0 = inner_ip0->src_address.as_u32;
 259       inner_ip0->src_address.as_u32 = new_dst_addr0;
 260       new_addr0 = inner_ip0->src_address.as_u32;
 261       sum0 = icmp0->checksum;
 262       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 263                              src_address);
 264       icmp0->checksum = ip_csum_fold (sum0);
 265
 266       /* update inner IP header checksum */
 267       old_checksum0 = inner_ip0->checksum;
 268       sum0 = inner_ip0->checksum;
 269       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 270                              src_address);
 271       inner_ip0->checksum = ip_csum_fold (sum0);
 272       new_checksum0 = inner_ip0->checksum;
 273       sum0 = icmp0->checksum;
 274       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 275                              checksum);
 276       icmp0->checksum = ip_csum_fold (sum0);
 277
 278       /* update inner source port */
 279       old_port0 = l4_header->src_port;
 280       l4_header->src_port = s0->in2out.port;
 281       new_port0 = l4_header->src_port;
 282       sum0 = icmp0->checksum;
 283       sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
 284                              src_port);
 285       icmp0->checksum = ip_csum_fold (sum0);
 286     }
 287   else
 288     {
 289       key0.addr = ip0->dst_address;
 290       key0.port = 0;
 291       key0.protocol = 0;
 292       key0.fib_index = sm->outside_fib_index;
 293       kv0.key = key0.as_u64;
 294
 295       if (clib_bihash_search_8_8
 296           (&sm->static_mapping_by_external, &kv0, &value0))
 297         {
 298           if (!is_ed)
 299             {
 300               icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 301               u16 icmp_id0 = echo0->identifier;
 302               key0.addr = ip0->dst_address;
 303               key0.port = icmp_id0;
 304               key0.protocol = SNAT_PROTOCOL_ICMP;
 305               key0.fib_index = sm->outside_fib_index;
 306               kv0.key = key0.as_u64;
 307               if (sm->num_workers > 1)
 308                 ti =
 309                   (clib_net_to_host_u16 (icmp_id0) -
 310                    1024) / sm->port_per_thread;
 311               else
 312                 ti = sm->num_workers;
 313               int rv =
 314                 clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 315                                         &value0);
 316               if (!rv)
 317                 {
 318                   si = value0.value;
 319                   s0 =
 320                     pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 321                   new_dst_addr0 = s0->in2out.addr.as_u32;
 322                   vnet_buffer (b0)->sw_if_index[VLIB_TX] =
 323                     s0->in2out.fib_index;
 324                   echo0->identifier = s0->in2out.port;
 325                   sum0 = icmp0->checksum;
 326                   sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 327                                          icmp_echo_header_t, identifier);
 328                   icmp0->checksum = ip_csum_fold (sum0);
 329                   goto change_addr;
 330                 }
 331             }
 332
 333           return 1;
 334         }
 335
 336       m0 = pool_elt_at_index (sm->static_mappings, value0.value);
 337
 338       new_dst_addr0 = m0->local_addr.as_u32;
 339       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 340         vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
 341     }
 342 change_addr:
 343   /* Destination is behind the same NAT, use internal address and port */
 344   if (new_dst_addr0)
 345     {
 346       old_dst_addr0 = ip0->dst_address.as_u32;
 347       ip0->dst_address.as_u32 = new_dst_addr0;
 348       sum0 = ip0->checksum;
 349       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 350                              ip4_header_t, dst_address);
 351       ip0->checksum = ip_csum_fold (sum0);
 352     }
 353   return 0;
 354 }
 355
 356 void
 357 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
 358                                   vlib_buffer_t * b, ip4_header_t * ip)
 359 {
 360   clib_bihash_kv_8_8_t kv, value;
 361   snat_static_mapping_t *m;
 362   u32 old_addr, new_addr;
 363   ip_csum_t sum;
 364
 365   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 366   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 367     return;
 368
 369   m = pool_elt_at_index (sm->static_mappings, value.value);
 370
 371   old_addr = ip->dst_address.as_u32;
 372   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 373   sum = ip->checksum;
 374   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 375   ip->checksum = ip_csum_fold (sum);
 376
 377   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 378     vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 379 }
 380
 381 void
 382 nat44_ed_hairpinning_unknown_proto (snat_main_t * sm,
 383                                     vlib_buffer_t * b, ip4_header_t * ip)
 384 {
 385   u32 old_addr, new_addr = 0, ti = 0;
 386   clib_bihash_kv_8_8_t kv, value;
 387   clib_bihash_kv_16_8_t s_kv, s_value;
 388   snat_static_mapping_t *m;
 389   ip_csum_t sum;
 390   snat_session_t *s;
 391   snat_main_per_thread_data_t *tsm;
 392
 393   if (sm->num_workers > 1)
 394     ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
 395   else
 396     ti = sm->num_workers;
 397   tsm = &sm->per_thread_data[ti];
 398
 399   old_addr = ip->dst_address.as_u32;
 400   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 401               sm->outside_fib_index, 0, 0);
 402   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 403     {
 404       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 405       if (clib_bihash_search_8_8
 406           (&sm->static_mapping_by_external, &kv, &value))
 407         return;
 408
 409       m = pool_elt_at_index (sm->static_mappings, value.value);
 410       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 411         vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 412       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 413     }
 414   else
 415     {
 416       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
 417       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 418         vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 419       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 420     }
 421   sum = ip->checksum;
 422   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 423   ip->checksum = ip_csum_fold (sum);
 424 }
 425
 426 void
 427 nat44_reass_hairpinning (snat_main_t * sm,
 428                          vlib_buffer_t * b0,
 429                          ip4_header_t * ip0,
 430                          u16 sport, u16 dport, u32 proto0, int is_ed)
 431 {
 432   snat_session_key_t key0, sm0;
 433   snat_session_t *s0;
 434   clib_bihash_kv_8_8_t kv0, value0;
 435   ip_csum_t sum0;
 436   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 437   u16 new_dst_port0, old_dst_port0;
 438   udp_header_t *udp0;
 439   tcp_header_t *tcp0;
 440   int rv;
 441
 442   key0.addr = ip0->dst_address;
 443   key0.port = dport;
 444   key0.protocol = proto0;
 445   key0.fib_index = sm->outside_fib_index;
 446   kv0.key = key0.as_u64;
 447
 448   udp0 = ip4_next_header (ip0);
 449
 450   /* Check if destination is static mappings */
 451   if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 452     {
 453       new_dst_addr0 = sm0.addr.as_u32;
 454       new_dst_port0 = sm0.port;
 455       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 456     }
 457   /* or active sessions */
 458   else
 459     {
 460       if (sm->num_workers > 1)
 461         ti =
 462           (clib_net_to_host_u16 (udp0->dst_port) -
 463            1024) / sm->port_per_thread;
 464       else
 465         ti = sm->num_workers;
 466
 467       if (is_ed)
 468         {
 469           clib_bihash_kv_16_8_t ed_kv, ed_value;
 470           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 471                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 472                       udp0->src_port);
 473           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 474                                         &ed_kv, &ed_value);
 475           si = ed_value.value;
 476         }
 477       else
 478         {
 479           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 480                                        &value0);
 481           si = value0.value;
 482         }
 483       if (!rv)
 484         {
 485           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 486           new_dst_addr0 = s0->in2out.addr.as_u32;
 487           new_dst_port0 = s0->in2out.port;
 488           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 489         }
 490     }
 491
 492   /* Destination is behind the same NAT, use internal address and port */
 493   if (new_dst_addr0)
 494     {
 495       old_dst_addr0 = ip0->dst_address.as_u32;
 496       ip0->dst_address.as_u32 = new_dst_addr0;
 497       sum0 = ip0->checksum;
 498       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 499                              ip4_header_t, dst_address);
 500       ip0->checksum = ip_csum_fold (sum0);
 501
 502       old_dst_port0 = dport;
 503       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0 &&
 504                         ip4_is_first_fragment (ip0)))
 505         {
 506           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 507             {
 508               tcp0 = ip4_next_header (ip0);
 509               tcp0->dst = new_dst_port0;
 510               sum0 = tcp0->checksum;
 511               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 512                                      ip4_header_t, dst_address);
 513               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 514                                      ip4_header_t /* cheat */ , length);
 515               tcp0->checksum = ip_csum_fold (sum0);
 516             }
 517           else
 518             {
 519               udp0->dst_port = new_dst_port0;
 520               udp0->checksum = 0;
 521             }
 522         }
 523       else
 524         {
 525           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 526             {
 527               tcp0 = ip4_next_header (ip0);
 528               sum0 = tcp0->checksum;
 529               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 530                                      ip4_header_t, dst_address);
 531               tcp0->checksum = ip_csum_fold (sum0);
 532             }
 533         }
 534     }
 535 }
 536
 537 static inline uword
 538 nat44_hairpinning_fn_inline (vlib_main_t * vm,
 539                              vlib_node_runtime_t * node,
 540                              vlib_frame_t * frame, int is_ed)
 541 {
 542   u32 n_left_from, *from, *to_next, stats_node_index;
 543   nat_hairpin_next_t next_index;
 544   u32 pkts_processed = 0;
 545   snat_main_t *sm = &snat_main;
 546   vnet_feature_main_t *fm = &feature_main;
 547   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
 548   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
 549
 550   stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
 551     nat44_hairpinning_node.index;
 552   from = vlib_frame_vector_args (frame);
 553   n_left_from = frame->n_vectors;
 554   next_index = node->cached_next_index;
 555
 556   while (n_left_from > 0)
 557     {
 558       u32 n_left_to_next;
 559
 560       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 561
 562       while (n_left_from > 0 && n_left_to_next > 0)
 563         {
 564           u32 bi0;
 565           vlib_buffer_t *b0;
 566           u32 next0;
 567           ip4_header_t *ip0;
 568           u32 proto0;
 569           udp_header_t *udp0;
 570           tcp_header_t *tcp0;
 571
 572           /* speculatively enqueue b0 to the current next frame */
 573           bi0 = from[0];
 574           to_next[0] = bi0;
 575           from += 1;
 576           to_next += 1;
 577           n_left_from -= 1;
 578           n_left_to_next -= 1;
 579
 580           b0 = vlib_get_buffer (vm, bi0);
 581           ip0 = vlib_buffer_get_current (b0);
 582           udp0 = ip4_next_header (ip0);
 583           tcp0 = (tcp_header_t *) udp0;
 584
 585           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 586
 587           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
 588                                 &next0, 0);
 589
 590           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
 591             next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 592
 593           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 594
 595           /* verify speculative enqueue, maybe switch current next frame */
 596           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 597                                            to_next, n_left_to_next,
 598                                            bi0, next0);
 599         }
 600
 601       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 602     }
 603
 604   vlib_node_increment_counter (vm, stats_node_index,
 605                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 606   return frame->n_vectors;
 607 }
 608
 609 static uword
 610 nat44_hairpinning_fn (vlib_main_t * vm,
 611                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 612 {
 613   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
 614 }
 615
 616 /* *INDENT-OFF* */
 617 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
 618   .function = nat44_hairpinning_fn,
 619   .name = "nat44-hairpinning",
 620   .vector_size = sizeof (u32),
 621   .type = VLIB_NODE_TYPE_INTERNAL,
 622   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 623   .error_strings = nat44_hairpin_error_strings,
 624   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 625   .next_nodes = {
 626     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 627     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 628   },
 629 };
 630 /* *INDENT-ON* */
 631
 632 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node, nat44_hairpinning_fn);
 633
 634 static uword
 635 nat44_ed_hairpinning_fn (vlib_main_t * vm,
 636                          vlib_node_runtime_t * node, vlib_frame_t * frame)
 637 {
 638   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
 639 }
 640
 641 /* *INDENT-OFF* */
 642 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
 643   .function = nat44_ed_hairpinning_fn,
 644   .name = "nat44-ed-hairpinning",
 645   .vector_size = sizeof (u32),
 646   .type = VLIB_NODE_TYPE_INTERNAL,
 647   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 648   .error_strings = nat44_hairpin_error_strings,
 649   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 650   .next_nodes = {
 651     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 652     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 653   },
 654 };
 655 /* *INDENT-ON* */
 656
 657 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
 658                               nat44_ed_hairpinning_fn);
 659
 660 static inline uword
 661 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
 662                             vlib_node_runtime_t * node,
 663                             vlib_frame_t * frame, int is_ed)
 664 {
 665   u32 n_left_from, *from, *to_next, stats_node_index;
 666   nat_hairpin_next_t next_index;
 667   u32 pkts_processed = 0;
 668   snat_main_t *sm = &snat_main;
 669
 670   stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
 671     snat_hairpin_dst_node.index;
 672
 673   from = vlib_frame_vector_args (frame);
 674   n_left_from = frame->n_vectors;
 675   next_index = node->cached_next_index;
 676
 677   while (n_left_from > 0)
 678     {
 679       u32 n_left_to_next;
 680
 681       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 682
 683       while (n_left_from > 0 && n_left_to_next > 0)
 684         {
 685           u32 bi0;
 686           vlib_buffer_t *b0;
 687           u32 next0;
 688           ip4_header_t *ip0;
 689           u32 proto0;
 690
 691           /* speculatively enqueue b0 to the current next frame */
 692           bi0 = from[0];
 693           to_next[0] = bi0;
 694           from += 1;
 695           to_next += 1;
 696           n_left_from -= 1;
 697           n_left_to_next -= 1;
 698
 699           b0 = vlib_get_buffer (vm, bi0);
 700           next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 701           ip0 = vlib_buffer_get_current (b0);
 702
 703           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 704
 705           vnet_buffer (b0)->snat.flags = 0;
 706           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
 707             {
 708               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
 709                 {
 710                   udp_header_t *udp0 = ip4_next_header (ip0);
 711                   tcp_header_t *tcp0 = (tcp_header_t *) udp0;
 712
 713                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
 714                 }
 715               else if (proto0 == SNAT_PROTOCOL_ICMP)
 716                 {
 717                   icmp46_header_t *icmp0 = ip4_next_header (ip0);
 718
 719                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
 720                 }
 721               else
 722                 {
 723                   if (is_ed)
 724                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
 725                   else
 726                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
 727                 }
 728
 729               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
 730             }
 731
 732           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 733
 734           /* verify speculative enqueue, maybe switch current next frame */
 735           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 736                                            to_next, n_left_to_next,
 737                                            bi0, next0);
 738         }
 739
 740       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 741     }
 742
 743   vlib_node_increment_counter (vm, stats_node_index,
 744                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 745   return frame->n_vectors;
 746 }
 747
 748 static uword
 749 snat_hairpin_dst_fn (vlib_main_t * vm,
 750                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 751 {
 752   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
 753 }
 754
 755 /* *INDENT-OFF* */
 756 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
 757   .function = snat_hairpin_dst_fn,
 758   .name = "nat44-hairpin-dst",
 759   .vector_size = sizeof (u32),
 760   .type = VLIB_NODE_TYPE_INTERNAL,
 761   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 762   .error_strings = nat44_hairpin_error_strings,
 763   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 764   .next_nodes = {
 765     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 766     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 767   },
 768 };
 769 /* *INDENT-ON* */
 770
 771 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node, snat_hairpin_dst_fn);
 772
 773 static uword
 774 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
 775                          vlib_node_runtime_t * node, vlib_frame_t * frame)
 776 {
 777   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
 778 }
 779
 780 /* *INDENT-OFF* */
 781 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
 782   .function = nat44_ed_hairpin_dst_fn,
 783   .name = "nat44-ed-hairpin-dst",
 784   .vector_size = sizeof (u32),
 785   .type = VLIB_NODE_TYPE_INTERNAL,
 786   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 787   .error_strings = nat44_hairpin_error_strings,
 788   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 789   .next_nodes = {
 790     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 791     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 792   },
 793 };
 794 /* *INDENT-ON* */
 795
 796 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
 797                               nat44_ed_hairpin_dst_fn);
 798
 799 static inline uword
 800 snat_hairpin_src_fn_inline (vlib_main_t * vm,
 801                             vlib_node_runtime_t * node,
 802                             vlib_frame_t * frame, int is_ed)
 803 {
 804   u32 n_left_from, *from, *to_next, stats_node_index;
 805   snat_hairpin_src_next_t next_index;
 806   u32 pkts_processed = 0;
 807   snat_main_t *sm = &snat_main;
 808
 809   stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
 810     snat_hairpin_src_node.index;
 811
 812   from = vlib_frame_vector_args (frame);
 813   n_left_from = frame->n_vectors;
 814   next_index = node->cached_next_index;
 815
 816   while (n_left_from > 0)
 817     {
 818       u32 n_left_to_next;
 819
 820       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 821
 822       while (n_left_from > 0 && n_left_to_next > 0)
 823         {
 824           u32 bi0;
 825           vlib_buffer_t *b0;
 826           u32 next0;
 827           snat_interface_t *i;
 828           u32 sw_if_index0;
 829
 830           /* speculatively enqueue b0 to the current next frame */
 831           bi0 = from[0];
 832           to_next[0] = bi0;
 833           from += 1;
 834           to_next += 1;
 835           n_left_from -= 1;
 836           n_left_to_next -= 1;
 837
 838           b0 = vlib_get_buffer (vm, bi0);
 839           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 840           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
 841
 842           /* *INDENT-OFF* */
 843           pool_foreach (i, sm->output_feature_interfaces,
 844           ({
 845             /* Only packets from NAT inside interface */
 846             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
 847               {
 848                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
 849                                     SNAT_FLAG_HAIRPINNING))
 850                   {
 851                     if (PREDICT_TRUE (sm->num_workers > 1))
 852                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
 853                     else
 854                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
 855                   }
 856                 break;
 857               }
 858           }));
 859           /* *INDENT-ON* */
 860
 861           pkts_processed += next0 != SNAT_HAIRPIN_SRC_NEXT_DROP;
 862
 863           /* verify speculative enqueue, maybe switch current next frame */
 864           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 865                                            to_next, n_left_to_next,
 866                                            bi0, next0);
 867         }
 868
 869       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 870     }
 871
 872   vlib_node_increment_counter (vm, stats_node_index,
 873                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 874   return frame->n_vectors;
 875 }
 876
 877 static uword
 878 snat_hairpin_src_fn (vlib_main_t * vm,
 879                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 880 {
 881   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
 882 }
 883
 884 /* *INDENT-OFF* */
 885 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
 886   .function = snat_hairpin_src_fn,
 887   .name = "nat44-hairpin-src",
 888   .vector_size = sizeof (u32),
 889   .type = VLIB_NODE_TYPE_INTERNAL,
 890   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 891   .error_strings = nat44_hairpin_error_strings,
 892   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 893   .next_nodes = {
 894      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 895      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
 896      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 897      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 898   },
 899 };
 900 /* *INDENT-ON* */
 901
 902 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node, snat_hairpin_src_fn);
 903
 904 static uword
 905 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
 906                          vlib_node_runtime_t * node, vlib_frame_t * frame)
 907 {
 908   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
 909 }
 910
 911 /* *INDENT-OFF* */
 912 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
 913   .function = nat44_ed_hairpin_src_fn,
 914   .name = "nat44-ed-hairpin-src",
 915   .vector_size = sizeof (u32),
 916   .type = VLIB_NODE_TYPE_INTERNAL,
 917   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 918   .error_strings = nat44_hairpin_error_strings,
 919   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 920   .next_nodes = {
 921      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 922      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
 923      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 924      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 925   },
 926 };
 927 /* *INDENT-ON* */
 928
 929 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
 930                               nat44_ed_hairpin_src_fn);
 931
 932 /*
 933  * fd.io coding-style-patch-verification: ON
 934  *
 935  * Local Variables:
 936  * eval: (c-set-style "gnu")
 937  * End:
 938  */