src/plugins/nat/nat44_hairpinning.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 hairpinning
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/fib/ip4_fib.h>
  23 #include <nat/nat.h>
  24 #include <nat/nat_inlines.h>
  25
  26 typedef enum
  27 {
  28   SNAT_HAIRPIN_SRC_NEXT_DROP,
  29   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
  30   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
  31   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
  32   SNAT_HAIRPIN_SRC_N_NEXT,
  33 } snat_hairpin_src_next_t;
  34
  35 typedef enum
  36 {
  37   NAT_HAIRPIN_NEXT_LOOKUP,
  38   NAT_HAIRPIN_NEXT_DROP,
  39   NAT_HAIRPIN_N_NEXT,
  40 } nat_hairpin_next_t;
  41
  42 #define foreach_nat44_hairpin_error                       \
  43 _(PROCESSED, "NAT44 hairpinning packets processed")
  44
  45 typedef enum
  46 {
  47 #define _(sym,str) NAT44_HAIRPIN_ERROR_##sym,
  48   foreach_nat44_hairpin_error
  49 #undef _
  50     NAT44_HAIRPIN_N_ERROR,
  51 } nat44_hairpin_error_t;
  52
  53 static char *nat44_hairpin_error_strings[] = {
  54 #define _(sym,string) string,
  55   foreach_nat44_hairpin_error
  56 #undef _
  57 };
  58
  59 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
  60
  61 static_always_inline int
  62 is_hairpinning (snat_main_t * sm, ip4_address_t * dst_addr)
  63 {
  64   snat_address_t *ap;
  65   clib_bihash_kv_8_8_t kv, value;
  66   snat_session_key_t m_key;
  67
  68   /* *INDENT-OFF* */
  69   vec_foreach (ap, sm->addresses)
  70     {
  71       if (ap->addr.as_u32 == dst_addr->as_u32)
  72         return 1;
  73     }
  74   /* *INDENT-ON* */
  75
  76   m_key.addr.as_u32 = dst_addr->as_u32;
  77   m_key.fib_index = 0;
  78   m_key.port = 0;
  79   m_key.protocol = 0;
  80   kv.key = m_key.as_u64;
  81   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
  82     return 1;
  83
  84   return 0;
  85 }
  86
  87 #ifndef CLIB_MARCH_VARIANT
  88 int
  89 snat_hairpinning (snat_main_t * sm,
  90                   vlib_buffer_t * b0,
  91                   ip4_header_t * ip0,
  92                   udp_header_t * udp0,
  93                   tcp_header_t * tcp0, u32 proto0, int is_ed)
  94 {
  95   snat_session_key_t key0, sm0;
  96   snat_session_t *s0;
  97   clib_bihash_kv_8_8_t kv0, value0;
  98   ip_csum_t sum0;
  99   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 100   u16 new_dst_port0, old_dst_port0;
 101   int rv;
 102
 103   key0.addr = ip0->dst_address;
 104   key0.port = udp0->dst_port;
 105   key0.protocol = proto0;
 106   key0.fib_index = sm->outside_fib_index;
 107   kv0.key = key0.as_u64;
 108
 109   /* Check if destination is static mappings */
 110   if (!snat_static_mapping_match (sm, key0, &sm0, 1, 0, 0, 0, 0, 0))
 111     {
 112       new_dst_addr0 = sm0.addr.as_u32;
 113       new_dst_port0 = sm0.port;
 114       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 115     }
 116   /* or active session */
 117   else
 118     {
 119       if (sm->num_workers > 1)
 120         ti =
 121           (clib_net_to_host_u16 (udp0->dst_port) -
 122            1024) / sm->port_per_thread;
 123       else
 124         ti = sm->num_workers;
 125
 126       if (is_ed)
 127         {
 128           clib_bihash_kv_16_8_t ed_kv, ed_value;
 129           make_ed_kv (&ip0->dst_address, &ip0->src_address,
 130                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 131                       udp0->src_port, ~0ULL, &ed_kv);
 132           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 133                                         &ed_kv, &ed_value);
 134           si = ed_value.value;
 135         }
 136       else
 137         {
 138           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 139                                        &value0);
 140           si = value0.value;
 141         }
 142       if (rv)
 143         return 0;
 144
 145       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 146       new_dst_addr0 = s0->in2out.addr.as_u32;
 147       new_dst_port0 = s0->in2out.port;
 148       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 149     }
 150
 151   /* Destination is behind the same NAT, use internal address and port */
 152   if (new_dst_addr0)
 153     {
 154       old_dst_addr0 = ip0->dst_address.as_u32;
 155       ip0->dst_address.as_u32 = new_dst_addr0;
 156       sum0 = ip0->checksum;
 157       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 158                              ip4_header_t, dst_address);
 159       ip0->checksum = ip_csum_fold (sum0);
 160
 161       old_dst_port0 = tcp0->dst;
 162       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
 163         {
 164           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 165             {
 166               tcp0->dst = new_dst_port0;
 167               sum0 = tcp0->checksum;
 168               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 169                                      ip4_header_t, dst_address);
 170               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 171                                      ip4_header_t /* cheat */ , length);
 172               tcp0->checksum = ip_csum_fold (sum0);
 173             }
 174           else
 175             {
 176               udp0->dst_port = new_dst_port0;
 177               udp0->checksum = 0;
 178             }
 179         }
 180       else
 181         {
 182           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 183             {
 184               sum0 = tcp0->checksum;
 185               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 186                                      ip4_header_t, dst_address);
 187               tcp0->checksum = ip_csum_fold (sum0);
 188             }
 189         }
 190       return 1;
 191     }
 192   return 0;
 193 }
 194 #endif
 195
 196 #ifndef CLIB_MARCH_VARIANT
 197 u32
 198 snat_icmp_hairpinning (snat_main_t * sm,
 199                        vlib_buffer_t * b0,
 200                        ip4_header_t * ip0, icmp46_header_t * icmp0, int is_ed)
 201 {
 202   snat_session_key_t key0;
 203   clib_bihash_kv_8_8_t kv0, value0;
 204   u32 old_dst_addr0, new_dst_addr0;
 205   u32 old_addr0, new_addr0;
 206   u16 old_port0, new_port0;
 207   u16 old_checksum0, new_checksum0;
 208   u32 si, ti = 0;
 209   ip_csum_t sum0;
 210   snat_session_t *s0;
 211   snat_static_mapping_t *m0;
 212
 213   if (icmp_type_is_error_message
 214       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 215     {
 216       ip4_header_t *inner_ip0 = 0;
 217       tcp_udp_header_t *l4_header = 0;
 218
 219       inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
 220       l4_header = ip4_next_header (inner_ip0);
 221       u32 protocol = ip_proto_to_nat_proto (inner_ip0->protocol);
 222
 223       if (protocol != NAT_PROTOCOL_TCP && protocol != NAT_PROTOCOL_UDP)
 224         return 1;
 225
 226       if (is_ed)
 227         {
 228           clib_bihash_kv_16_8_t ed_kv, ed_value;
 229           make_ed_kv (&ip0->dst_address, &ip0->src_address,
 230                       inner_ip0->protocol, sm->outside_fib_index,
 231                       l4_header->src_port, l4_header->dst_port, ~0ULL,
 232                       &ed_kv);
 233           if (clib_bihash_search_16_8
 234               (&sm->per_thread_data[ti].out2in_ed, &ed_kv, &ed_value))
 235             return 1;
 236           si = ed_value.value;
 237         }
 238       else
 239         {
 240           key0.addr = ip0->dst_address;
 241           key0.port = l4_header->src_port;
 242           key0.protocol = protocol;
 243           key0.fib_index = sm->outside_fib_index;
 244           kv0.key = key0.as_u64;
 245           if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 246                                       &value0))
 247             return 1;
 248           si = value0.value;
 249         }
 250       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 251       new_dst_addr0 = s0->in2out.addr.as_u32;
 252       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 253
 254       /* update inner source IP address */
 255       old_addr0 = inner_ip0->src_address.as_u32;
 256       inner_ip0->src_address.as_u32 = new_dst_addr0;
 257       new_addr0 = inner_ip0->src_address.as_u32;
 258       sum0 = icmp0->checksum;
 259       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 260                              src_address);
 261       icmp0->checksum = ip_csum_fold (sum0);
 262
 263       /* update inner IP header checksum */
 264       old_checksum0 = inner_ip0->checksum;
 265       sum0 = inner_ip0->checksum;
 266       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 267                              src_address);
 268       inner_ip0->checksum = ip_csum_fold (sum0);
 269       new_checksum0 = inner_ip0->checksum;
 270       sum0 = icmp0->checksum;
 271       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 272                              checksum);
 273       icmp0->checksum = ip_csum_fold (sum0);
 274
 275       /* update inner source port */
 276       old_port0 = l4_header->src_port;
 277       l4_header->src_port = s0->in2out.port;
 278       new_port0 = l4_header->src_port;
 279       sum0 = icmp0->checksum;
 280       sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
 281                              src_port);
 282       icmp0->checksum = ip_csum_fold (sum0);
 283     }
 284   else
 285     {
 286       key0.addr = ip0->dst_address;
 287       key0.port = 0;
 288       key0.protocol = 0;
 289       key0.fib_index = sm->outside_fib_index;
 290       kv0.key = key0.as_u64;
 291
 292       if (clib_bihash_search_8_8
 293           (&sm->static_mapping_by_external, &kv0, &value0))
 294         {
 295           if (!is_ed)
 296             {
 297               icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 298               u16 icmp_id0 = echo0->identifier;
 299               key0.addr = ip0->dst_address;
 300               key0.port = icmp_id0;
 301               key0.protocol = NAT_PROTOCOL_ICMP;
 302               key0.fib_index = sm->outside_fib_index;
 303               kv0.key = key0.as_u64;
 304               if (sm->num_workers > 1)
 305                 ti =
 306                   (clib_net_to_host_u16 (icmp_id0) -
 307                    1024) / sm->port_per_thread;
 308               else
 309                 ti = sm->num_workers;
 310               int rv =
 311                 clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 312                                         &value0);
 313               if (!rv)
 314                 {
 315                   si = value0.value;
 316                   s0 =
 317                     pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 318                   new_dst_addr0 = s0->in2out.addr.as_u32;
 319                   vnet_buffer (b0)->sw_if_index[VLIB_TX] =
 320                     s0->in2out.fib_index;
 321                   echo0->identifier = s0->in2out.port;
 322                   sum0 = icmp0->checksum;
 323                   sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 324                                          icmp_echo_header_t, identifier);
 325                   icmp0->checksum = ip_csum_fold (sum0);
 326                   goto change_addr;
 327                 }
 328             }
 329
 330           return 1;
 331         }
 332
 333       m0 = pool_elt_at_index (sm->static_mappings, value0.value);
 334
 335       new_dst_addr0 = m0->local_addr.as_u32;
 336       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 337         vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
 338     }
 339 change_addr:
 340   /* Destination is behind the same NAT, use internal address and port */
 341   if (new_dst_addr0)
 342     {
 343       old_dst_addr0 = ip0->dst_address.as_u32;
 344       ip0->dst_address.as_u32 = new_dst_addr0;
 345       sum0 = ip0->checksum;
 346       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 347                              ip4_header_t, dst_address);
 348       ip0->checksum = ip_csum_fold (sum0);
 349     }
 350   return 0;
 351 }
 352 #endif
 353
 354 #ifndef CLIB_MARCH_VARIANT
 355 void
 356 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
 357                                   vlib_buffer_t * b, ip4_header_t * ip)
 358 {
 359   clib_bihash_kv_8_8_t kv, value;
 360   snat_static_mapping_t *m;
 361   u32 old_addr, new_addr;
 362   ip_csum_t sum;
 363
 364   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 365   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 366     return;
 367
 368   m = pool_elt_at_index (sm->static_mappings, value.value);
 369
 370   old_addr = ip->dst_address.as_u32;
 371   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 372   sum = ip->checksum;
 373   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 374   ip->checksum = ip_csum_fold (sum);
 375
 376   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 377     vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 378 }
 379 #endif
 380
 381 #ifndef CLIB_MARCH_VARIANT
 382 void
 383 nat44_ed_hairpinning_unknown_proto (snat_main_t * sm,
 384                                     vlib_buffer_t * b, ip4_header_t * ip)
 385 {
 386   u32 old_addr, new_addr = 0, ti = 0;
 387   clib_bihash_kv_8_8_t kv, value;
 388   clib_bihash_kv_16_8_t s_kv, s_value;
 389   snat_static_mapping_t *m;
 390   ip_csum_t sum;
 391   snat_session_t *s;
 392   snat_main_per_thread_data_t *tsm;
 393
 394   if (sm->num_workers > 1)
 395     ti = sm->worker_out2in_cb (b, ip, sm->outside_fib_index, 0);
 396   else
 397     ti = sm->num_workers;
 398   tsm = &sm->per_thread_data[ti];
 399
 400   old_addr = ip->dst_address.as_u32;
 401   make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
 402               sm->outside_fib_index, 0, 0, ~0ULL, &s_kv);
 403   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
 404     {
 405       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
 406       if (clib_bihash_search_8_8
 407           (&sm->static_mapping_by_external, &kv, &value))
 408         return;
 409
 410       m = pool_elt_at_index (sm->static_mappings, value.value);
 411       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 412         vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 413       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 414     }
 415   else
 416     {
 417       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
 418       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 419         vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 420       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 421     }
 422   sum = ip->checksum;
 423   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 424   ip->checksum = ip_csum_fold (sum);
 425 }
 426 #endif
 427
 428 static inline uword
 429 nat44_hairpinning_fn_inline (vlib_main_t * vm,
 430                              vlib_node_runtime_t * node,
 431                              vlib_frame_t * frame, int is_ed)
 432 {
 433   u32 n_left_from, *from, *to_next, stats_node_index;
 434   nat_hairpin_next_t next_index;
 435   u32 pkts_processed = 0;
 436   snat_main_t *sm = &snat_main;
 437   vnet_feature_main_t *fm = &feature_main;
 438   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
 439   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
 440
 441   stats_node_index = is_ed ? sm->ed_hairpinning_node_index :
 442     sm->hairpinning_node_index;
 443   from = vlib_frame_vector_args (frame);
 444   n_left_from = frame->n_vectors;
 445   next_index = node->cached_next_index;
 446
 447   while (n_left_from > 0)
 448     {
 449       u32 n_left_to_next;
 450
 451       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 452
 453       while (n_left_from > 0 && n_left_to_next > 0)
 454         {
 455           u32 bi0;
 456           vlib_buffer_t *b0;
 457           u32 next0;
 458           ip4_header_t *ip0;
 459           u32 proto0;
 460           udp_header_t *udp0;
 461           tcp_header_t *tcp0;
 462
 463           /* speculatively enqueue b0 to the current next frame */
 464           bi0 = from[0];
 465           to_next[0] = bi0;
 466           from += 1;
 467           to_next += 1;
 468           n_left_from -= 1;
 469           n_left_to_next -= 1;
 470
 471           b0 = vlib_get_buffer (vm, bi0);
 472           ip0 = vlib_buffer_get_current (b0);
 473           udp0 = ip4_next_header (ip0);
 474           tcp0 = (tcp_header_t *) udp0;
 475
 476           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 477
 478           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
 479                                 &next0, 0);
 480
 481           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
 482             next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 483
 484           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 485
 486           /* verify speculative enqueue, maybe switch current next frame */
 487           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 488                                            to_next, n_left_to_next,
 489                                            bi0, next0);
 490         }
 491
 492       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 493     }
 494
 495   vlib_node_increment_counter (vm, stats_node_index,
 496                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 497   return frame->n_vectors;
 498 }
 499
 500 VLIB_NODE_FN (nat44_hairpinning_node) (vlib_main_t * vm,
 501                                        vlib_node_runtime_t * node,
 502                                        vlib_frame_t * frame)
 503 {
 504   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
 505 }
 506
 507 /* *INDENT-OFF* */
 508 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
 509   .name = "nat44-hairpinning",
 510   .vector_size = sizeof (u32),
 511   .type = VLIB_NODE_TYPE_INTERNAL,
 512   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 513   .error_strings = nat44_hairpin_error_strings,
 514   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 515   .next_nodes = {
 516     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 517     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 518   },
 519 };
 520 /* *INDENT-ON* */
 521
 522 VLIB_NODE_FN (nat44_ed_hairpinning_node) (vlib_main_t * vm,
 523                                           vlib_node_runtime_t * node,
 524                                           vlib_frame_t * frame)
 525 {
 526   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
 527 }
 528
 529 /* *INDENT-OFF* */
 530 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
 531   .name = "nat44-ed-hairpinning",
 532   .vector_size = sizeof (u32),
 533   .type = VLIB_NODE_TYPE_INTERNAL,
 534   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 535   .error_strings = nat44_hairpin_error_strings,
 536   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 537   .next_nodes = {
 538     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 539     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 540   },
 541 };
 542 /* *INDENT-ON* */
 543
 544 static inline uword
 545 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
 546                             vlib_node_runtime_t * node,
 547                             vlib_frame_t * frame, int is_ed)
 548 {
 549   u32 n_left_from, *from, *to_next, stats_node_index;
 550   nat_hairpin_next_t next_index;
 551   u32 pkts_processed = 0;
 552   snat_main_t *sm = &snat_main;
 553
 554   stats_node_index = is_ed ? sm->ed_hairpin_dst_node_index :
 555     sm->hairpin_dst_node_index;
 556
 557   from = vlib_frame_vector_args (frame);
 558   n_left_from = frame->n_vectors;
 559   next_index = node->cached_next_index;
 560
 561   while (n_left_from > 0)
 562     {
 563       u32 n_left_to_next;
 564
 565       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 566
 567       while (n_left_from > 0 && n_left_to_next > 0)
 568         {
 569           u32 bi0;
 570           vlib_buffer_t *b0;
 571           u32 next0;
 572           ip4_header_t *ip0;
 573           u32 proto0;
 574
 575           /* speculatively enqueue b0 to the current next frame */
 576           bi0 = from[0];
 577           to_next[0] = bi0;
 578           from += 1;
 579           to_next += 1;
 580           n_left_from -= 1;
 581           n_left_to_next -= 1;
 582
 583           b0 = vlib_get_buffer (vm, bi0);
 584           next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 585           ip0 = vlib_buffer_get_current (b0);
 586
 587           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 588
 589           vnet_buffer (b0)->snat.flags = 0;
 590           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
 591             {
 592               if (proto0 == NAT_PROTOCOL_TCP || proto0 == NAT_PROTOCOL_UDP)
 593                 {
 594                   udp_header_t *udp0 = ip4_next_header (ip0);
 595                   tcp_header_t *tcp0 = (tcp_header_t *) udp0;
 596
 597                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
 598                 }
 599               else if (proto0 == NAT_PROTOCOL_ICMP)
 600                 {
 601                   icmp46_header_t *icmp0 = ip4_next_header (ip0);
 602
 603                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
 604                 }
 605               else
 606                 {
 607                   if (is_ed)
 608                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
 609                   else
 610                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
 611                 }
 612
 613               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
 614             }
 615
 616           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 617
 618           /* verify speculative enqueue, maybe switch current next frame */
 619           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 620                                            to_next, n_left_to_next,
 621                                            bi0, next0);
 622         }
 623
 624       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 625     }
 626
 627   vlib_node_increment_counter (vm, stats_node_index,
 628                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 629   return frame->n_vectors;
 630 }
 631
 632 VLIB_NODE_FN (snat_hairpin_dst_node) (vlib_main_t * vm,
 633                                       vlib_node_runtime_t * node,
 634                                       vlib_frame_t * frame)
 635 {
 636   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
 637 }
 638
 639 /* *INDENT-OFF* */
 640 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
 641   .name = "nat44-hairpin-dst",
 642   .vector_size = sizeof (u32),
 643   .type = VLIB_NODE_TYPE_INTERNAL,
 644   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 645   .error_strings = nat44_hairpin_error_strings,
 646   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 647   .next_nodes = {
 648     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 649     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 650   },
 651 };
 652 /* *INDENT-ON* */
 653
 654 VLIB_NODE_FN (nat44_ed_hairpin_dst_node) (vlib_main_t * vm,
 655                                           vlib_node_runtime_t * node,
 656                                           vlib_frame_t * frame)
 657 {
 658   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
 659 }
 660
 661 /* *INDENT-OFF* */
 662 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
 663   .name = "nat44-ed-hairpin-dst",
 664   .vector_size = sizeof (u32),
 665   .type = VLIB_NODE_TYPE_INTERNAL,
 666   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 667   .error_strings = nat44_hairpin_error_strings,
 668   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 669   .next_nodes = {
 670     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 671     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 672   },
 673 };
 674 /* *INDENT-ON* */
 675
 676 static inline uword
 677 snat_hairpin_src_fn_inline (vlib_main_t * vm,
 678                             vlib_node_runtime_t * node,
 679                             vlib_frame_t * frame, int is_ed)
 680 {
 681   u32 n_left_from, *from, *to_next, stats_node_index;
 682   snat_hairpin_src_next_t next_index;
 683   u32 pkts_processed = 0;
 684   snat_main_t *sm = &snat_main;
 685
 686   stats_node_index = is_ed ? sm->ed_hairpin_src_node_index :
 687     sm->hairpin_src_node_index;
 688
 689   from = vlib_frame_vector_args (frame);
 690   n_left_from = frame->n_vectors;
 691   next_index = node->cached_next_index;
 692
 693   while (n_left_from > 0)
 694     {
 695       u32 n_left_to_next;
 696
 697       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 698
 699       while (n_left_from > 0 && n_left_to_next > 0)
 700         {
 701           u32 bi0;
 702           vlib_buffer_t *b0;
 703           u32 next0;
 704           snat_interface_t *i;
 705           u32 sw_if_index0;
 706
 707           /* speculatively enqueue b0 to the current next frame */
 708           bi0 = from[0];
 709           to_next[0] = bi0;
 710           from += 1;
 711           to_next += 1;
 712           n_left_from -= 1;
 713           n_left_to_next -= 1;
 714
 715           b0 = vlib_get_buffer (vm, bi0);
 716           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 717           vnet_feature_next (&next0, b0);
 718
 719           /* *INDENT-OFF* */
 720           pool_foreach (i, sm->output_feature_interfaces,
 721           ({
 722             /* Only packets from NAT inside interface */
 723             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
 724               {
 725                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
 726                                     SNAT_FLAG_HAIRPINNING))
 727                   {
 728                     if (PREDICT_TRUE (sm->num_workers > 1))
 729                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
 730                     else
 731                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
 732                   }
 733                 break;
 734               }
 735           }));
 736           /* *INDENT-ON* */
 737
 738           pkts_processed += next0 != SNAT_HAIRPIN_SRC_NEXT_DROP;
 739
 740           /* verify speculative enqueue, maybe switch current next frame */
 741           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 742                                            to_next, n_left_to_next,
 743                                            bi0, next0);
 744         }
 745
 746       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 747     }
 748
 749   vlib_node_increment_counter (vm, stats_node_index,
 750                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 751   return frame->n_vectors;
 752 }
 753
 754 VLIB_NODE_FN (snat_hairpin_src_node) (vlib_main_t * vm,
 755                                       vlib_node_runtime_t * node,
 756                                       vlib_frame_t * frame)
 757 {
 758   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
 759 }
 760
 761 /* *INDENT-OFF* */
 762 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
 763   .name = "nat44-hairpin-src",
 764   .vector_size = sizeof (u32),
 765   .type = VLIB_NODE_TYPE_INTERNAL,
 766   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 767   .error_strings = nat44_hairpin_error_strings,
 768   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 769   .next_nodes = {
 770      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 771      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
 772      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 773      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 774   },
 775 };
 776 /* *INDENT-ON* */
 777
 778 VLIB_NODE_FN (nat44_ed_hairpin_src_node) (vlib_main_t * vm,
 779                                           vlib_node_runtime_t * node,
 780                                           vlib_frame_t * frame)
 781 {
 782   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
 783 }
 784
 785 /* *INDENT-OFF* */
 786 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
 787   .name = "nat44-ed-hairpin-src",
 788   .vector_size = sizeof (u32),
 789   .type = VLIB_NODE_TYPE_INTERNAL,
 790   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 791   .error_strings = nat44_hairpin_error_strings,
 792   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 793   .next_nodes = {
 794      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 795      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
 796      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 797      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 798   },
 799 };
 800 /* *INDENT-ON* */
 801
 802 /*
 803  * fd.io coding-style-patch-verification: ON
 804  *
 805  * Local Variables:
 806  * eval: (c-set-style "gnu")
 807  * End:
 808  */