src/plugins/nat/nat44_hairpinning.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 hairpinning
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/fib/ip4_fib.h>
  23 #include <nat/nat.h>
  24 #include <nat/nat_inlines.h>
  25 #include <nat/nat_reass.h>
  26
  27 typedef enum
  28 {
  29   SNAT_HAIRPIN_SRC_NEXT_DROP,
  30   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
  31   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
  32   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
  33   SNAT_HAIRPIN_SRC_N_NEXT,
  34 } snat_hairpin_src_next_t;
  35
  36 typedef enum
  37 {
  38   NAT_HAIRPIN_NEXT_LOOKUP,
  39   NAT_HAIRPIN_NEXT_DROP,
  40   NAT_HAIRPIN_N_NEXT,
  41 } nat_hairpin_next_t;
  42
  43 #define foreach_nat44_hairpin_error                       \
  44 _(PROCESSED, "NAT44 hairpinning packets processed")
  45
  46 typedef enum
  47 {
  48 #define _(sym,str) NAT44_HAIRPIN_ERROR_##sym,
  49   foreach_nat44_hairpin_error
  50 #undef _
  51     NAT44_HAIRPIN_N_ERROR,
  52 } nat44_hairpin_error_t;
  53
  54 static char *nat44_hairpin_error_strings[] = {
  55 #define _(sym,string) string,
  56   foreach_nat44_hairpin_error
  57 #undef _
  58 };
  59
  60 vlib_node_registration_t snat_hairpin_dst_node;
  61 vlib_node_registration_t snat_hairpin_src_node;
  62 vlib_node_registration_t nat44_hairpinning_node;
  63 vlib_node_registration_t nat44_ed_hairpin_dst_node;
  64 vlib_node_registration_t nat44_ed_hairpin_src_node;
  65 vlib_node_registration_t nat44_ed_hairpinning_node;
  66
  67 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
  68
  69 static_always_inline int
  70 is_hairpinning (snat_main_t * sm, ip4_address_t * dst_addr)
  71 {
  72   snat_address_t *ap;
  73   clib_bihash_kv_8_8_t kv, value;
  74   snat_session_key_t m_key;
  75
  76   /* *INDENT-OFF* */
  77   vec_foreach (ap, sm->addresses)
  78     {
  79       if (ap->addr.as_u32 == dst_addr->as_u32)
  80         return 1;
  81     }
  82   /* *INDENT-ON* */
  83
  84   m_key.addr.as_u32 = dst_addr->as_u32;
  85   m_key.fib_index = 0;
  86   m_key.port = 0;
  87   m_key.protocol = 0;
  88   kv.key = m_key.as_u64;
  89   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
  90     return 1;
  91
  92   return 0;
  93 }
  94
  95 int
  96 snat_hairpinning (snat_main_t * sm,
  97                   vlib_buffer_t * b0,
  98                   ip4_header_t * ip0,
  99                   udp_header_t * udp0,
 100                   tcp_header_t * tcp0, u32 proto0, int is_ed)
 101 {
 102   snat_session_key_t key0, sm0;
 103   snat_session_t *s0;
 104   clib_bihash_kv_8_8_t kv0, value0;
 105   ip_csum_t sum0;
 106   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 107   u16 new_dst_port0, old_dst_port0;
 108   int rv;
 109
 110   key0.addr = ip0->dst_address;
 111   key0.port = udp0->dst_port;
 112   key0.protocol = proto0;
 113   key0.fib_index = sm->outside_fib_index;
 114   kv0.key = key0.as_u64;
 115
 116   /* Check if destination is static mappings */
 117   if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
 118     {
 119       new_dst_addr0 = sm0.addr.as_u32;
 120       new_dst_port0 = sm0.port;
 121       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 122     }
 123   /* or active session */
 124   else
 125     {
 126       if (sm->num_workers > 1)
 127         ti =
 128           (clib_net_to_host_u16 (udp0->dst_port) -
 129            1024) / sm->port_per_thread;
 130       else
 131         ti = sm->num_workers;
 132
 133       if (is_ed)
 134         {
 135           clib_bihash_kv_16_8_t ed_kv, ed_value;
 136           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 137                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 138                       udp0->src_port);
 139           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 140                                         &ed_kv, &ed_value);
 141           si = ed_value.value;
 142         }
 143       else
 144         {
 145           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 146                                        &value0);
 147           si = value0.value;
 148         }
 149       if (rv)
 150         return 0;
 151
 152       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 153       new_dst_addr0 = s0->in2out.addr.as_u32;
 154       new_dst_port0 = s0->in2out.port;
 155       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 156     }
 157
 158   /* Destination is behind the same NAT, use internal address and port */
 159   if (new_dst_addr0)
 160     {
 161       old_dst_addr0 = ip0->dst_address.as_u32;
 162       ip0->dst_address.as_u32 = new_dst_addr0;
 163       sum0 = ip0->checksum;
 164       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 165                              ip4_header_t, dst_address);
 166       ip0->checksum = ip_csum_fold (sum0);
 167
 168       old_dst_port0 = tcp0->dst;
 169       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
 170         {
 171           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 172             {
 173               tcp0->dst = new_dst_port0;
 174               sum0 = tcp0->checksum;
 175               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 176                                      ip4_header_t, dst_address);
 177               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 178                                      ip4_header_t /* cheat */ , length);
 179               tcp0->checksum = ip_csum_fold (sum0);
 180             }
 181           else
 182             {
 183               udp0->dst_port = new_dst_port0;
 184               udp0->checksum = 0;
 185             }
 186         }
 187       else
 188         {
 189           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 190             {
 191               sum0 = tcp0->checksum;
 192               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 193                                      ip4_header_t, dst_address);
 194               tcp0->checksum = ip_csum_fold (sum0);
 195             }
 196         }
 197       return 1;
 198     }
 199   return 0;
 200 }
 201
 202 u32
 203 snat_icmp_hairpinning (snat_main_t * sm,
 204                        vlib_buffer_t * b0,
 205                        ip4_header_t * ip0, icmp46_header_t * icmp0, int is_ed)
 206 {
 207   snat_session_key_t key0;
 208   clib_bihash_kv_8_8_t kv0, value0;
 209   u32 old_dst_addr0, new_dst_addr0;
 210   u32 old_addr0, new_addr0;
 211   u16 old_port0, new_port0;
 212   u16 old_checksum0, new_checksum0;
 213   u32 si, ti = 0;
 214   ip_csum_t sum0;
 215   snat_session_t *s0;
 216   snat_static_mapping_t *m0;
 217
 218   if (icmp_is_error_message (icmp0))
 219     {
 220       ip4_header_t *inner_ip0 = 0;
 221       tcp_udp_header_t *l4_header = 0;
 222
 223       inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
 224       l4_header = ip4_next_header (inner_ip0);
 225       u32 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 226
 227       if (protocol != SNAT_PROTOCOL_TCP && protocol != SNAT_PROTOCOL_UDP)
 228         return 1;
 229
 230       if (is_ed)
 231         {
 232           clib_bihash_kv_16_8_t ed_kv, ed_value;
 233           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 234                       inner_ip0->protocol, sm->outside_fib_index,
 235                       l4_header->src_port, l4_header->dst_port);
 236           if (clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 237                                        &ed_kv, &ed_value))
 238             return 1;
 239           si = ed_value.value;
 240         }
 241       else
 242         {
 243           key0.addr = ip0->dst_address;
 244           key0.port = l4_header->src_port;
 245           key0.protocol = protocol;
 246           key0.fib_index = sm->outside_fib_index;
 247           kv0.key = key0.as_u64;
 248           if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 249                                       &value0))
 250             return 1;
 251           si = value0.value;
 252         }
 253       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 254       new_dst_addr0 = s0->in2out.addr.as_u32;
 255       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 256
 257       /* update inner source IP address */
 258       old_addr0 = inner_ip0->src_address.as_u32;
 259       inner_ip0->src_address.as_u32 = new_dst_addr0;
 260       new_addr0 = inner_ip0->src_address.as_u32;
 261       sum0 = icmp0->checksum;
 262       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 263                              src_address);
 264       icmp0->checksum = ip_csum_fold (sum0);
 265
 266       /* update inner IP header checksum */
 267       old_checksum0 = inner_ip0->checksum;
 268       sum0 = inner_ip0->checksum;
 269       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 270                              src_address);
 271       inner_ip0->checksum = ip_csum_fold (sum0);
 272       new_checksum0 = inner_ip0->checksum;
 273       sum0 = icmp0->checksum;
 274       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 275                              checksum);
 276       icmp0->checksum = ip_csum_fold (sum0);
 277
 278       /* update inner source port */
 279       old_port0 = l4_header->src_port;
 280       l4_header->src_port = s0->in2out.port;
 281       new_port0 = l4_header->src_port;
 282       sum0 = icmp0->checksum;
 283       sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
 284                              src_port);
 285       icmp0->checksum = ip_csum_fold (sum0);
 286     }
 287   else
 288     {
 289       if (!is_ed)
 290         {
 291           icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 292           u16 icmp_id0 = echo0->identifier;
 293           key0.addr = ip0->dst_address;
 294           key0.port = icmp_id0;
 295           key0.protocol = SNAT_PROTOCOL_ICMP;
 296           key0.fib_index = sm->outside_fib_index;
 297           kv0.key = key0.as_u64;
 298           if (sm->num_workers > 1)
 299             ti =
 300               (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 301           else
 302             ti = sm->num_workers;
 303           int rv =
 304             clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 305                                     &value0);
 306           if (!rv)
 307             {
 308               si = value0.value;
 309               s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 310               new_dst_addr0 = s0->in2out.addr.as_u32;
 311               vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 312               echo0->identifier = s0->in2out.port;
 313               sum0 = icmp0->checksum;
 314               sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 315                                      icmp_echo_header_t, identifier);
 316               icmp0->checksum = ip_csum_fold (sum0);
 317               goto change_addr;
 318             }
 319           ti = 0;
 320         }
 321
 322       key0.addr = ip0->dst_address;
 323       key0.port = 0;
 324       key0.protocol = 0;
 325       key0.fib_index = sm->outside_fib_index;
 326       kv0.key = key0.as_u64;
 327
 328       if (clib_bihash_search_8_8
 329           (&sm->static_mapping_by_external, &kv0, &value0))
 330         return 1;
 331
 332       m0 = pool_elt_at_index (sm->static_mappings, value0.value);
 333
 334       new_dst_addr0 = m0->local_addr.as_u32;
 335       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 336         vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
 337     }
 338 change_addr:
 339   /* Destination is behind the same NAT, use internal address and port */
 340   if (new_dst_addr0)
 341     {
 342       old_dst_addr0 = ip0->dst_address.as_u32;
 343       ip0->dst_address.as_u32 = new_dst_addr0;
 344       sum0 = ip0->checksum;
 345       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 346                              ip4_header_t, dst_address);
 347       ip0->checksum = ip_csum_fold (sum0);
 348     }
 349   return 0;
 350 }
 351
 352 void
 353 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
 354                                   vlib_buffer_t * b, ip4_header_t * ip)
 355 {
 356   clib_bihash_kv_8_8_t kv, value;
 357   snat_static_mapping_t *m;
 358   u32 old_addr, new_addr;
 359   ip_csum_t sum;
 360
 361   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 362   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 363     return;
 364
 365   m = pool_elt_at_index (sm->static_mappings, value.value);
 366
 367   old_addr = ip->dst_address.as_u32;
 368   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 369   sum = ip->checksum;
 370   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 371   ip->checksum = ip_csum_fold (sum);
 372
 373   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 374     vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 375 }
 376
 377 void
 378 nat44_ed_hairpinning_unknown_proto (snat_main_t * sm,
 379                                     vlib_buffer_t * b, ip4_header_t * ip)
 380 {
 381   u32 old_addr, new_addr = 0, ti = 0;
 382   clib_bihash_kv_8_8_t kv, value;
 383   clib_bihash_kv_16_8_t s_kv, s_value;
 384   snat_static_mapping_t *m;
 385   ip_csum_t sum;
 386   snat_session_t *s;
 387   snat_main_per_thread_data_t *tsm;
 388
 389   if (sm->num_workers > 1)
 390     ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
 391   else
 392     ti = sm->num_workers;
 393   tsm = &sm->per_thread_data[ti];
 394
 395   old_addr = ip->dst_address.as_u32;
 396   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
 397               sm->outside_fib_index, 0, 0);
 398   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 399     {
 400       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 401       if (clib_bihash_search_8_8
 402           (&sm->static_mapping_by_external, &kv, &value))
 403         return;
 404
 405       m = pool_elt_at_index (sm->static_mappings, value.value);
 406       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 407         vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 408       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 409     }
 410   else
 411     {
 412       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
 413       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 414         vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 415       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 416     }
 417   sum = ip->checksum;
 418   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 419   ip->checksum = ip_csum_fold (sum);
 420 }
 421
 422 void
 423 nat44_reass_hairpinning (snat_main_t * sm,
 424                          vlib_buffer_t * b0,
 425                          ip4_header_t * ip0,
 426                          u16 sport, u16 dport, u32 proto0, int is_ed)
 427 {
 428   snat_session_key_t key0, sm0;
 429   snat_session_t *s0;
 430   clib_bihash_kv_8_8_t kv0, value0;
 431   ip_csum_t sum0;
 432   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 433   u16 new_dst_port0, old_dst_port0;
 434   udp_header_t *udp0;
 435   tcp_header_t *tcp0;
 436   int rv;
 437
 438   key0.addr = ip0->dst_address;
 439   key0.port = dport;
 440   key0.protocol = proto0;
 441   key0.fib_index = sm->outside_fib_index;
 442   kv0.key = key0.as_u64;
 443
 444   udp0 = ip4_next_header (ip0);
 445
 446   /* Check if destination is static mappings */
 447   if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0))
 448     {
 449       new_dst_addr0 = sm0.addr.as_u32;
 450       new_dst_port0 = sm0.port;
 451       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 452     }
 453   /* or active sessions */
 454   else
 455     {
 456       if (sm->num_workers > 1)
 457         ti =
 458           (clib_net_to_host_u16 (udp0->dst_port) -
 459            1024) / sm->port_per_thread;
 460       else
 461         ti = sm->num_workers;
 462
 463       if (is_ed)
 464         {
 465           clib_bihash_kv_16_8_t ed_kv, ed_value;
 466           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 467                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 468                       udp0->src_port);
 469           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 470                                         &ed_kv, &ed_value);
 471           si = ed_value.value;
 472         }
 473       else
 474         {
 475           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 476                                        &value0);
 477           si = value0.value;
 478         }
 479       if (!rv)
 480         {
 481           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 482           new_dst_addr0 = s0->in2out.addr.as_u32;
 483           new_dst_port0 = s0->in2out.port;
 484           vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 485         }
 486     }
 487
 488   /* Destination is behind the same NAT, use internal address and port */
 489   if (new_dst_addr0)
 490     {
 491       old_dst_addr0 = ip0->dst_address.as_u32;
 492       ip0->dst_address.as_u32 = new_dst_addr0;
 493       sum0 = ip0->checksum;
 494       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 495                              ip4_header_t, dst_address);
 496       ip0->checksum = ip_csum_fold (sum0);
 497
 498       old_dst_port0 = dport;
 499       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0 &&
 500                         ip4_is_first_fragment (ip0)))
 501         {
 502           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 503             {
 504               tcp0 = ip4_next_header (ip0);
 505               tcp0->dst = new_dst_port0;
 506               sum0 = tcp0->checksum;
 507               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 508                                      ip4_header_t, dst_address);
 509               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 510                                      ip4_header_t /* cheat */ , length);
 511               tcp0->checksum = ip_csum_fold (sum0);
 512             }
 513           else
 514             {
 515               udp0->dst_port = new_dst_port0;
 516               udp0->checksum = 0;
 517             }
 518         }
 519       else
 520         {
 521           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
 522             {
 523               tcp0 = ip4_next_header (ip0);
 524               sum0 = tcp0->checksum;
 525               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 526                                      ip4_header_t, dst_address);
 527               tcp0->checksum = ip_csum_fold (sum0);
 528             }
 529         }
 530     }
 531 }
 532
 533 static inline uword
 534 nat44_hairpinning_fn_inline (vlib_main_t * vm,
 535                              vlib_node_runtime_t * node,
 536                              vlib_frame_t * frame, int is_ed)
 537 {
 538   u32 n_left_from, *from, *to_next, stats_node_index;
 539   nat_hairpin_next_t next_index;
 540   u32 pkts_processed = 0;
 541   snat_main_t *sm = &snat_main;
 542   vnet_feature_main_t *fm = &feature_main;
 543   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
 544   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
 545
 546   stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
 547     nat44_hairpinning_node.index;
 548   from = vlib_frame_vector_args (frame);
 549   n_left_from = frame->n_vectors;
 550   next_index = node->cached_next_index;
 551
 552   while (n_left_from > 0)
 553     {
 554       u32 n_left_to_next;
 555
 556       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 557
 558       while (n_left_from > 0 && n_left_to_next > 0)
 559         {
 560           u32 bi0;
 561           vlib_buffer_t *b0;
 562           u32 next0;
 563           ip4_header_t *ip0;
 564           u32 proto0;
 565           udp_header_t *udp0;
 566           tcp_header_t *tcp0;
 567
 568           /* speculatively enqueue b0 to the current next frame */
 569           bi0 = from[0];
 570           to_next[0] = bi0;
 571           from += 1;
 572           to_next += 1;
 573           n_left_from -= 1;
 574           n_left_to_next -= 1;
 575
 576           b0 = vlib_get_buffer (vm, bi0);
 577           ip0 = vlib_buffer_get_current (b0);
 578           udp0 = ip4_next_header (ip0);
 579           tcp0 = (tcp_header_t *) udp0;
 580
 581           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 582
 583           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
 584                                 &next0, 0);
 585
 586           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
 587             next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 588
 589           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 590
 591           /* verify speculative enqueue, maybe switch current next frame */
 592           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 593                                            to_next, n_left_to_next,
 594                                            bi0, next0);
 595         }
 596
 597       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 598     }
 599
 600   vlib_node_increment_counter (vm, stats_node_index,
 601                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 602   return frame->n_vectors;
 603 }
 604
 605 static uword
 606 nat44_hairpinning_fn (vlib_main_t * vm,
 607                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 608 {
 609   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
 610 }
 611
 612 /* *INDENT-OFF* */
 613 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
 614   .function = nat44_hairpinning_fn,
 615   .name = "nat44-hairpinning",
 616   .vector_size = sizeof (u32),
 617   .type = VLIB_NODE_TYPE_INTERNAL,
 618   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 619   .error_strings = nat44_hairpin_error_strings,
 620   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 621   .next_nodes = {
 622     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 623     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 624   },
 625 };
 626 /* *INDENT-ON* */
 627
 628 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node, nat44_hairpinning_fn);
 629
 630 static uword
 631 nat44_ed_hairpinning_fn (vlib_main_t * vm,
 632                          vlib_node_runtime_t * node, vlib_frame_t * frame)
 633 {
 634   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
 635 }
 636
 637 /* *INDENT-OFF* */
 638 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
 639   .function = nat44_ed_hairpinning_fn,
 640   .name = "nat44-ed-hairpinning",
 641   .vector_size = sizeof (u32),
 642   .type = VLIB_NODE_TYPE_INTERNAL,
 643   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 644   .error_strings = nat44_hairpin_error_strings,
 645   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 646   .next_nodes = {
 647     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 648     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 649   },
 650 };
 651 /* *INDENT-ON* */
 652
 653 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
 654                               nat44_ed_hairpinning_fn);
 655
 656 static inline uword
 657 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
 658                             vlib_node_runtime_t * node,
 659                             vlib_frame_t * frame, int is_ed)
 660 {
 661   u32 n_left_from, *from, *to_next, stats_node_index;
 662   nat_hairpin_next_t next_index;
 663   u32 pkts_processed = 0;
 664   snat_main_t *sm = &snat_main;
 665
 666   stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
 667     snat_hairpin_dst_node.index;
 668
 669   from = vlib_frame_vector_args (frame);
 670   n_left_from = frame->n_vectors;
 671   next_index = node->cached_next_index;
 672
 673   while (n_left_from > 0)
 674     {
 675       u32 n_left_to_next;
 676
 677       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 678
 679       while (n_left_from > 0 && n_left_to_next > 0)
 680         {
 681           u32 bi0;
 682           vlib_buffer_t *b0;
 683           u32 next0;
 684           ip4_header_t *ip0;
 685           u32 proto0;
 686
 687           /* speculatively enqueue b0 to the current next frame */
 688           bi0 = from[0];
 689           to_next[0] = bi0;
 690           from += 1;
 691           to_next += 1;
 692           n_left_from -= 1;
 693           n_left_to_next -= 1;
 694
 695           b0 = vlib_get_buffer (vm, bi0);
 696           next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 697           ip0 = vlib_buffer_get_current (b0);
 698
 699           proto0 = ip_proto_to_snat_proto (ip0->protocol);
 700
 701           vnet_buffer (b0)->snat.flags = 0;
 702           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
 703             {
 704               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
 705                 {
 706                   udp_header_t *udp0 = ip4_next_header (ip0);
 707                   tcp_header_t *tcp0 = (tcp_header_t *) udp0;
 708
 709                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
 710                 }
 711               else if (proto0 == SNAT_PROTOCOL_ICMP)
 712                 {
 713                   icmp46_header_t *icmp0 = ip4_next_header (ip0);
 714
 715                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
 716                 }
 717               else
 718                 {
 719                   if (is_ed)
 720                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
 721                   else
 722                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
 723                 }
 724
 725               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
 726             }
 727
 728           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 729
 730           /* verify speculative enqueue, maybe switch current next frame */
 731           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 732                                            to_next, n_left_to_next,
 733                                            bi0, next0);
 734         }
 735
 736       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 737     }
 738
 739   vlib_node_increment_counter (vm, stats_node_index,
 740                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 741   return frame->n_vectors;
 742 }
 743
 744 static uword
 745 snat_hairpin_dst_fn (vlib_main_t * vm,
 746                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 747 {
 748   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
 749 }
 750
 751 /* *INDENT-OFF* */
 752 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
 753   .function = snat_hairpin_dst_fn,
 754   .name = "nat44-hairpin-dst",
 755   .vector_size = sizeof (u32),
 756   .type = VLIB_NODE_TYPE_INTERNAL,
 757   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 758   .error_strings = nat44_hairpin_error_strings,
 759   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 760   .next_nodes = {
 761     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 762     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 763   },
 764 };
 765 /* *INDENT-ON* */
 766
 767 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node, snat_hairpin_dst_fn);
 768
 769 static uword
 770 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
 771                          vlib_node_runtime_t * node, vlib_frame_t * frame)
 772 {
 773   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
 774 }
 775
 776 /* *INDENT-OFF* */
 777 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
 778   .function = nat44_ed_hairpin_dst_fn,
 779   .name = "nat44-ed-hairpin-dst",
 780   .vector_size = sizeof (u32),
 781   .type = VLIB_NODE_TYPE_INTERNAL,
 782   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 783   .error_strings = nat44_hairpin_error_strings,
 784   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 785   .next_nodes = {
 786     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 787     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 788   },
 789 };
 790 /* *INDENT-ON* */
 791
 792 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
 793                               nat44_ed_hairpin_dst_fn);
 794
 795 static inline uword
 796 snat_hairpin_src_fn_inline (vlib_main_t * vm,
 797                             vlib_node_runtime_t * node,
 798                             vlib_frame_t * frame, int is_ed)
 799 {
 800   u32 n_left_from, *from, *to_next, stats_node_index;
 801   snat_hairpin_src_next_t next_index;
 802   u32 pkts_processed = 0;
 803   snat_main_t *sm = &snat_main;
 804
 805   stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
 806     snat_hairpin_src_node.index;
 807
 808   from = vlib_frame_vector_args (frame);
 809   n_left_from = frame->n_vectors;
 810   next_index = node->cached_next_index;
 811
 812   while (n_left_from > 0)
 813     {
 814       u32 n_left_to_next;
 815
 816       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 817
 818       while (n_left_from > 0 && n_left_to_next > 0)
 819         {
 820           u32 bi0;
 821           vlib_buffer_t *b0;
 822           u32 next0;
 823           snat_interface_t *i;
 824           u32 sw_if_index0;
 825
 826           /* speculatively enqueue b0 to the current next frame */
 827           bi0 = from[0];
 828           to_next[0] = bi0;
 829           from += 1;
 830           to_next += 1;
 831           n_left_from -= 1;
 832           n_left_to_next -= 1;
 833
 834           b0 = vlib_get_buffer (vm, bi0);
 835           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 836           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
 837
 838           /* *INDENT-OFF* */
 839           pool_foreach (i, sm->output_feature_interfaces,
 840           ({
 841             /* Only packets from NAT inside interface */
 842             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
 843               {
 844                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
 845                                     SNAT_FLAG_HAIRPINNING))
 846                   {
 847                     if (PREDICT_TRUE (sm->num_workers > 1))
 848                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
 849                     else
 850                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
 851                   }
 852                 break;
 853               }
 854           }));
 855           /* *INDENT-ON* */
 856
 857           pkts_processed += next0 != SNAT_HAIRPIN_SRC_NEXT_DROP;
 858
 859           /* verify speculative enqueue, maybe switch current next frame */
 860           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 861                                            to_next, n_left_to_next,
 862                                            bi0, next0);
 863         }
 864
 865       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 866     }
 867
 868   vlib_node_increment_counter (vm, stats_node_index,
 869                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 870   return frame->n_vectors;
 871 }
 872
 873 static uword
 874 snat_hairpin_src_fn (vlib_main_t * vm,
 875                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 876 {
 877   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
 878 }
 879
 880 /* *INDENT-OFF* */
 881 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
 882   .function = snat_hairpin_src_fn,
 883   .name = "nat44-hairpin-src",
 884   .vector_size = sizeof (u32),
 885   .type = VLIB_NODE_TYPE_INTERNAL,
 886   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 887   .error_strings = nat44_hairpin_error_strings,
 888   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 889   .next_nodes = {
 890      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 891      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
 892      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 893      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 894   },
 895 };
 896 /* *INDENT-ON* */
 897
 898 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node, snat_hairpin_src_fn);
 899
 900 static uword
 901 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
 902                          vlib_node_runtime_t * node, vlib_frame_t * frame)
 903 {
 904   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
 905 }
 906
 907 /* *INDENT-OFF* */
 908 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
 909   .function = nat44_ed_hairpin_src_fn,
 910   .name = "nat44-ed-hairpin-src",
 911   .vector_size = sizeof (u32),
 912   .type = VLIB_NODE_TYPE_INTERNAL,
 913   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 914   .error_strings = nat44_hairpin_error_strings,
 915   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 916   .next_nodes = {
 917      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 918      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
 919      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 920      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 921   },
 922 };
 923 /* *INDENT-ON* */
 924
 925 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
 926                               nat44_ed_hairpin_src_fn);
 927
 928 /*
 929  * fd.io coding-style-patch-verification: ON
 930  *
 931  * Local Variables:
 932  * eval: (c-set-style "gnu")
 933  * End:
 934  */