src/plugins/nat/nat44_hairpinning.c

   1 /*
   2  * Copyright (c) 2018 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 hairpinning
  18  */
  19
  20 #include <vlib/vlib.h>
  21 #include <vnet/vnet.h>
  22 #include <vnet/fib/ip4_fib.h>
  23 #include <nat/nat.h>
  24 #include <nat/nat_inlines.h>
  25
  26 typedef enum
  27 {
  28   SNAT_HAIRPIN_SRC_NEXT_DROP,
  29   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
  30   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
  31   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
  32   SNAT_HAIRPIN_SRC_N_NEXT,
  33 } snat_hairpin_src_next_t;
  34
  35 typedef enum
  36 {
  37   NAT_HAIRPIN_NEXT_LOOKUP,
  38   NAT_HAIRPIN_NEXT_DROP,
  39   NAT_HAIRPIN_N_NEXT,
  40 } nat_hairpin_next_t;
  41
  42 #define foreach_nat44_hairpin_error                       \
  43 _(PROCESSED, "NAT44 hairpinning packets processed")
  44
  45 typedef enum
  46 {
  47 #define _(sym,str) NAT44_HAIRPIN_ERROR_##sym,
  48   foreach_nat44_hairpin_error
  49 #undef _
  50     NAT44_HAIRPIN_N_ERROR,
  51 } nat44_hairpin_error_t;
  52
  53 static char *nat44_hairpin_error_strings[] = {
  54 #define _(sym,string) string,
  55   foreach_nat44_hairpin_error
  56 #undef _
  57 };
  58
  59 typedef struct
  60 {
  61   ip4_address_t addr;
  62   u16 port;
  63   u32 fib_index;
  64   u32 session_index;
  65 } nat_hairpin_trace_t;
  66
  67 static u8 *
  68 format_nat_hairpin_trace (u8 * s, va_list * args)
  69 {
  70   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  71   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  72   nat_hairpin_trace_t *t = va_arg (*args, nat_hairpin_trace_t *);
  73
  74   s =
  75     format (s, "new dst addr %U port %u fib-index %u", format_ip4_address,
  76             &t->addr, clib_net_to_host_u16 (t->port), t->fib_index);
  77   if (~0 == t->session_index)
  78     {
  79       s = format (s, " is-static-mapping");
  80     }
  81   else
  82     {
  83       s = format (s, " session-index %u", t->session_index);
  84     }
  85
  86   return s;
  87 }
  88
  89 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
  90
  91 static_always_inline int
  92 is_hairpinning (snat_main_t * sm, ip4_address_t * dst_addr)
  93 {
  94   snat_address_t *ap;
  95   clib_bihash_kv_8_8_t kv, value;
  96
  97   /* *INDENT-OFF* */
  98   vec_foreach (ap, sm->addresses)
  99     {
 100       if (ap->addr.as_u32 == dst_addr->as_u32)
 101         return 1;
 102     }
 103   /* *INDENT-ON* */
 104
 105   init_nat_k (&kv, *dst_addr, 0, 0, 0);
 106   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 107     return 1;
 108
 109   return 0;
 110 }
 111
 112 #ifndef CLIB_MARCH_VARIANT
 113 int
 114 snat_hairpinning (vlib_main_t * vm, vlib_node_runtime_t * node,
 115                   snat_main_t * sm, vlib_buffer_t * b0, ip4_header_t * ip0,
 116                   udp_header_t * udp0, tcp_header_t * tcp0, u32 proto0,
 117                   int is_ed, int do_trace)
 118 {
 119   snat_session_t *s0 = NULL;
 120   clib_bihash_kv_8_8_t kv0, value0;
 121   ip_csum_t sum0;
 122   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si = ~0;
 123   u16 new_dst_port0 = ~0, old_dst_port0;
 124   int rv;
 125   ip4_address_t sm0_addr;
 126   u16 sm0_port;
 127   u32 sm0_fib_index;
 128   /* Check if destination is static mappings */
 129   if (!snat_static_mapping_match
 130       (sm, ip0->dst_address, udp0->dst_port, sm->outside_fib_index, proto0,
 131        &sm0_addr, &sm0_port, &sm0_fib_index, 1, 0, 0, 0, 0, 0))
 132     {
 133       new_dst_addr0 = sm0_addr.as_u32;
 134       new_dst_port0 = sm0_port;
 135       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
 136     }
 137   /* or active session */
 138   else
 139     {
 140       if (sm->num_workers > 1)
 141         ti =
 142           (clib_net_to_host_u16 (udp0->dst_port) -
 143            1024) / sm->port_per_thread;
 144       else
 145         ti = sm->num_workers;
 146
 147       if (is_ed)
 148         {
 149           clib_bihash_kv_16_8_t ed_kv, ed_value;
 150           init_ed_k (&ed_kv, ip0->dst_address, udp0->dst_port,
 151                      ip0->src_address, udp0->src_port, sm->outside_fib_index,
 152                      ip0->protocol);
 153           rv = clib_bihash_search_16_8 (&sm->out2in_ed, &ed_kv, &ed_value);
 154           ASSERT (ti == ed_value_get_thread_index (&ed_value));
 155           si = ed_value_get_session_index (&ed_value);
 156         }
 157       else
 158         {
 159
 160           init_nat_k (&kv0, ip0->dst_address, udp0->dst_port,
 161                       sm->outside_fib_index, proto0);
 162           rv =
 163             clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 164                                     &value0);
 165           si = value0.value;
 166         }
 167       if (rv)
 168         {
 169           rv = 0;
 170           goto trace;
 171         }
 172
 173       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 174       new_dst_addr0 = s0->in2out.addr.as_u32;
 175       new_dst_port0 = s0->in2out.port;
 176       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 177     }
 178
 179   /* Destination is behind the same NAT, use internal address and port */
 180   if (new_dst_addr0)
 181     {
 182       old_dst_addr0 = ip0->dst_address.as_u32;
 183       ip0->dst_address.as_u32 = new_dst_addr0;
 184       sum0 = ip0->checksum;
 185       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 186                              ip4_header_t, dst_address);
 187       ip0->checksum = ip_csum_fold (sum0);
 188
 189       old_dst_port0 = tcp0->dst;
 190       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
 191         {
 192           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 193             {
 194               tcp0->dst = new_dst_port0;
 195               sum0 = tcp0->checksum;
 196               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 197                                      ip4_header_t, dst_address);
 198               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 199                                      ip4_header_t /* cheat */ , length);
 200               tcp0->checksum = ip_csum_fold (sum0);
 201             }
 202           else
 203             {
 204               udp0->dst_port = new_dst_port0;
 205               udp0->checksum = 0;
 206             }
 207         }
 208       else
 209         {
 210           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
 211             {
 212               sum0 = tcp0->checksum;
 213               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 214                                      ip4_header_t, dst_address);
 215               tcp0->checksum = ip_csum_fold (sum0);
 216             }
 217         }
 218       rv = 1;
 219       goto trace;
 220     }
 221   rv = 0;
 222 trace:
 223   if (do_trace && PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 224                                  && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 225     {
 226       nat_hairpin_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
 227       t->addr.as_u32 = new_dst_addr0;
 228       t->port = new_dst_port0;
 229       t->fib_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
 230       if (s0)
 231         {
 232           t->session_index = si;
 233         }
 234       else
 235         {
 236           t->session_index = ~0;
 237         }
 238     }
 239   return rv;
 240 }
 241 #endif
 242
 243 #ifndef CLIB_MARCH_VARIANT
 244 u32
 245 snat_icmp_hairpinning (snat_main_t * sm,
 246                        vlib_buffer_t * b0,
 247                        ip4_header_t * ip0, icmp46_header_t * icmp0, int is_ed)
 248 {
 249   clib_bihash_kv_8_8_t kv0, value0;
 250   u32 old_dst_addr0, new_dst_addr0;
 251   u32 old_addr0, new_addr0;
 252   u16 old_port0, new_port0;
 253   u16 old_checksum0, new_checksum0;
 254   u32 si, ti = 0;
 255   ip_csum_t sum0;
 256   snat_session_t *s0;
 257   snat_static_mapping_t *m0;
 258
 259   if (icmp_type_is_error_message
 260       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 261     {
 262       ip4_header_t *inner_ip0 = 0;
 263       tcp_udp_header_t *l4_header = 0;
 264
 265       inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
 266       l4_header = ip4_next_header (inner_ip0);
 267       u32 protocol = ip_proto_to_nat_proto (inner_ip0->protocol);
 268
 269       if (protocol != NAT_PROTOCOL_TCP && protocol != NAT_PROTOCOL_UDP)
 270         return 1;
 271
 272       if (is_ed)
 273         {
 274           clib_bihash_kv_16_8_t ed_kv, ed_value;
 275           init_ed_k (&ed_kv, ip0->dst_address, l4_header->src_port,
 276                      ip0->src_address, l4_header->dst_port,
 277                      sm->outside_fib_index, inner_ip0->protocol);
 278           if (clib_bihash_search_16_8 (&sm->out2in_ed, &ed_kv, &ed_value))
 279             return 1;
 280           ASSERT (ti == ed_value_get_thread_index (&ed_value));
 281           si = ed_value_get_session_index (&ed_value);
 282         }
 283       else
 284         {
 285           init_nat_k (&kv0, ip0->dst_address, l4_header->src_port,
 286                       sm->outside_fib_index, protocol);
 287           if (clib_bihash_search_8_8
 288               (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 289             return 1;
 290           si = value0.value;
 291         }
 292       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 293       new_dst_addr0 = s0->in2out.addr.as_u32;
 294       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 295
 296       /* update inner source IP address */
 297       old_addr0 = inner_ip0->src_address.as_u32;
 298       inner_ip0->src_address.as_u32 = new_dst_addr0;
 299       new_addr0 = inner_ip0->src_address.as_u32;
 300       sum0 = icmp0->checksum;
 301       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 302                              src_address);
 303       icmp0->checksum = ip_csum_fold (sum0);
 304
 305       /* update inner IP header checksum */
 306       old_checksum0 = inner_ip0->checksum;
 307       sum0 = inner_ip0->checksum;
 308       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 309                              src_address);
 310       inner_ip0->checksum = ip_csum_fold (sum0);
 311       new_checksum0 = inner_ip0->checksum;
 312       sum0 = icmp0->checksum;
 313       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 314                              checksum);
 315       icmp0->checksum = ip_csum_fold (sum0);
 316
 317       /* update inner source port */
 318       old_port0 = l4_header->src_port;
 319       l4_header->src_port = s0->in2out.port;
 320       new_port0 = l4_header->src_port;
 321       sum0 = icmp0->checksum;
 322       sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
 323                              src_port);
 324       icmp0->checksum = ip_csum_fold (sum0);
 325     }
 326   else
 327     {
 328       init_nat_k (&kv0, ip0->dst_address, 0, sm->outside_fib_index, 0);
 329       if (clib_bihash_search_8_8
 330           (&sm->static_mapping_by_external, &kv0, &value0))
 331         {
 332           if (!is_ed)
 333             {
 334               icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 335               u16 icmp_id0 = echo0->identifier;
 336               init_nat_k (&kv0, ip0->dst_address, icmp_id0,
 337                           sm->outside_fib_index, NAT_PROTOCOL_ICMP);
 338               if (sm->num_workers > 1)
 339                 ti =
 340                   (clib_net_to_host_u16 (icmp_id0) -
 341                    1024) / sm->port_per_thread;
 342               else
 343                 ti = sm->num_workers;
 344               int rv =
 345                 clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 346                                         &value0);
 347               if (!rv)
 348                 {
 349                   si = value0.value;
 350                   s0 =
 351                     pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 352                   new_dst_addr0 = s0->in2out.addr.as_u32;
 353                   vnet_buffer (b0)->sw_if_index[VLIB_TX] =
 354                     s0->in2out.fib_index;
 355                   echo0->identifier = s0->in2out.port;
 356                   sum0 = icmp0->checksum;
 357                   sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 358                                          icmp_echo_header_t, identifier);
 359                   icmp0->checksum = ip_csum_fold (sum0);
 360                   goto change_addr;
 361                 }
 362             }
 363
 364           return 1;
 365         }
 366
 367       m0 = pool_elt_at_index (sm->static_mappings, value0.value);
 368
 369       new_dst_addr0 = m0->local_addr.as_u32;
 370       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 371         vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
 372     }
 373 change_addr:
 374   /* Destination is behind the same NAT, use internal address and port */
 375   if (new_dst_addr0)
 376     {
 377       old_dst_addr0 = ip0->dst_address.as_u32;
 378       ip0->dst_address.as_u32 = new_dst_addr0;
 379       sum0 = ip0->checksum;
 380       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 381                              ip4_header_t, dst_address);
 382       ip0->checksum = ip_csum_fold (sum0);
 383     }
 384   return 0;
 385 }
 386 #endif
 387
 388 #ifndef CLIB_MARCH_VARIANT
 389 void
 390 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
 391                                   vlib_buffer_t * b, ip4_header_t * ip)
 392 {
 393   clib_bihash_kv_8_8_t kv, value;
 394   snat_static_mapping_t *m;
 395   u32 old_addr, new_addr;
 396   ip_csum_t sum;
 397
 398   init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 399   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
 400     return;
 401
 402   m = pool_elt_at_index (sm->static_mappings, value.value);
 403
 404   old_addr = ip->dst_address.as_u32;
 405   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 406   sum = ip->checksum;
 407   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 408   ip->checksum = ip_csum_fold (sum);
 409
 410   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 411     vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 412 }
 413 #endif
 414
 415 #ifndef CLIB_MARCH_VARIANT
 416 void
 417 nat44_ed_hairpinning_unknown_proto (snat_main_t * sm,
 418                                     vlib_buffer_t * b, ip4_header_t * ip)
 419 {
 420   u32 old_addr, new_addr = 0, ti = 0;
 421   clib_bihash_kv_8_8_t kv, value;
 422   clib_bihash_kv_16_8_t s_kv, s_value;
 423   snat_static_mapping_t *m;
 424   ip_csum_t sum;
 425   snat_session_t *s;
 426
 427   if (sm->num_workers > 1)
 428     ti = sm->worker_out2in_cb (b, ip, sm->outside_fib_index, 0);
 429   else
 430     ti = sm->num_workers;
 431
 432   old_addr = ip->dst_address.as_u32;
 433   init_ed_k (&s_kv, ip->dst_address, 0, ip->src_address, 0,
 434              sm->outside_fib_index, ip->protocol);
 435   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
 436     {
 437       init_nat_k (&kv, ip->dst_address, 0, 0, 0);
 438       if (clib_bihash_search_8_8
 439           (&sm->static_mapping_by_external, &kv, &value))
 440         return;
 441
 442       m = pool_elt_at_index (sm->static_mappings, value.value);
 443       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 444         vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 445       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
 446     }
 447   else
 448     {
 449       ASSERT (ti == ed_value_get_thread_index (&s_value));
 450       s =
 451         pool_elt_at_index (sm->per_thread_data[ti].sessions,
 452                            ed_value_get_session_index (&s_value));
 453       if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 454         vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
 455       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
 456     }
 457   sum = ip->checksum;
 458   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
 459   ip->checksum = ip_csum_fold (sum);
 460 }
 461 #endif
 462
 463 static inline uword
 464 nat44_hairpinning_fn_inline (vlib_main_t * vm,
 465                              vlib_node_runtime_t * node,
 466                              vlib_frame_t * frame, int is_ed)
 467 {
 468   u32 n_left_from, *from, *to_next, stats_node_index;
 469   nat_hairpin_next_t next_index;
 470   u32 pkts_processed = 0;
 471   snat_main_t *sm = &snat_main;
 472   vnet_feature_main_t *fm = &feature_main;
 473   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
 474   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
 475
 476   stats_node_index = is_ed ? sm->ed_hairpinning_node_index :
 477     sm->hairpinning_node_index;
 478   from = vlib_frame_vector_args (frame);
 479   n_left_from = frame->n_vectors;
 480   next_index = node->cached_next_index;
 481
 482   while (n_left_from > 0)
 483     {
 484       u32 n_left_to_next;
 485
 486       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 487
 488       while (n_left_from > 0 && n_left_to_next > 0)
 489         {
 490           u32 bi0;
 491           vlib_buffer_t *b0;
 492           u32 next0;
 493           ip4_header_t *ip0;
 494           u32 proto0;
 495           udp_header_t *udp0;
 496           tcp_header_t *tcp0;
 497
 498           /* speculatively enqueue b0 to the current next frame */
 499           bi0 = from[0];
 500           to_next[0] = bi0;
 501           from += 1;
 502           to_next += 1;
 503           n_left_from -= 1;
 504           n_left_to_next -= 1;
 505
 506           b0 = vlib_get_buffer (vm, bi0);
 507           ip0 = vlib_buffer_get_current (b0);
 508           udp0 = ip4_next_header (ip0);
 509           tcp0 = (tcp_header_t *) udp0;
 510
 511           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 512
 513           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
 514                                 &next0, 0);
 515
 516           if (snat_hairpinning
 517               (vm, node, sm, b0, ip0, udp0, tcp0, proto0, is_ed,
 518                1 /* do_trace */ ))
 519             next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 520
 521           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 522
 523           /* verify speculative enqueue, maybe switch current next frame */
 524           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 525                                            to_next, n_left_to_next,
 526                                            bi0, next0);
 527         }
 528
 529       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 530     }
 531
 532   vlib_node_increment_counter (vm, stats_node_index,
 533                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 534   return frame->n_vectors;
 535 }
 536
 537 VLIB_NODE_FN (nat44_hairpinning_node) (vlib_main_t * vm,
 538                                        vlib_node_runtime_t * node,
 539                                        vlib_frame_t * frame)
 540 {
 541   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
 542 }
 543
 544 /* *INDENT-OFF* */
 545 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
 546   .name = "nat44-hairpinning",
 547   .vector_size = sizeof (u32),
 548   .type = VLIB_NODE_TYPE_INTERNAL,
 549   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 550   .format_trace = format_nat_hairpin_trace,
 551   .error_strings = nat44_hairpin_error_strings,
 552   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 553   .next_nodes = {
 554     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 555     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 556   },
 557 };
 558 /* *INDENT-ON* */
 559
 560 VLIB_NODE_FN (nat44_ed_hairpinning_node) (vlib_main_t * vm,
 561                                           vlib_node_runtime_t * node,
 562                                           vlib_frame_t * frame)
 563 {
 564   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
 565 }
 566
 567 /* *INDENT-OFF* */
 568 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
 569   .name = "nat44-ed-hairpinning",
 570   .vector_size = sizeof (u32),
 571   .type = VLIB_NODE_TYPE_INTERNAL,
 572   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 573   .error_strings = nat44_hairpin_error_strings,
 574   .format_trace = format_nat_hairpin_trace,
 575   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 576   .next_nodes = {
 577     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 578     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 579   },
 580 };
 581 /* *INDENT-ON* */
 582
 583 static inline uword
 584 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
 585                             vlib_node_runtime_t * node,
 586                             vlib_frame_t * frame, int is_ed)
 587 {
 588   u32 n_left_from, *from, *to_next, stats_node_index;
 589   nat_hairpin_next_t next_index;
 590   u32 pkts_processed = 0;
 591   snat_main_t *sm = &snat_main;
 592
 593   stats_node_index = is_ed ? sm->ed_hairpin_dst_node_index :
 594     sm->hairpin_dst_node_index;
 595
 596   from = vlib_frame_vector_args (frame);
 597   n_left_from = frame->n_vectors;
 598   next_index = node->cached_next_index;
 599
 600   while (n_left_from > 0)
 601     {
 602       u32 n_left_to_next;
 603
 604       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 605
 606       while (n_left_from > 0 && n_left_to_next > 0)
 607         {
 608           u32 bi0;
 609           vlib_buffer_t *b0;
 610           u32 next0;
 611           ip4_header_t *ip0;
 612           u32 proto0;
 613
 614           /* speculatively enqueue b0 to the current next frame */
 615           bi0 = from[0];
 616           to_next[0] = bi0;
 617           from += 1;
 618           to_next += 1;
 619           n_left_from -= 1;
 620           n_left_to_next -= 1;
 621
 622           b0 = vlib_get_buffer (vm, bi0);
 623           next0 = NAT_HAIRPIN_NEXT_LOOKUP;
 624           ip0 = vlib_buffer_get_current (b0);
 625
 626           proto0 = ip_proto_to_nat_proto (ip0->protocol);
 627
 628           vnet_buffer (b0)->snat.flags = 0;
 629           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
 630             {
 631               if (proto0 == NAT_PROTOCOL_TCP || proto0 == NAT_PROTOCOL_UDP)
 632                 {
 633                   udp_header_t *udp0 = ip4_next_header (ip0);
 634                   tcp_header_t *tcp0 = (tcp_header_t *) udp0;
 635
 636                   snat_hairpinning (vm, node, sm, b0, ip0, udp0, tcp0, proto0,
 637                                     is_ed, 1 /* do_trace */ );
 638                 }
 639               else if (proto0 == NAT_PROTOCOL_ICMP)
 640                 {
 641                   icmp46_header_t *icmp0 = ip4_next_header (ip0);
 642
 643                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
 644                 }
 645               else
 646                 {
 647                   if (is_ed)
 648                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
 649                   else
 650                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
 651                 }
 652
 653               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
 654             }
 655
 656           pkts_processed += next0 != NAT_HAIRPIN_NEXT_DROP;
 657
 658           /* verify speculative enqueue, maybe switch current next frame */
 659           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 660                                            to_next, n_left_to_next,
 661                                            bi0, next0);
 662         }
 663
 664       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 665     }
 666
 667   vlib_node_increment_counter (vm, stats_node_index,
 668                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 669   return frame->n_vectors;
 670 }
 671
 672 VLIB_NODE_FN (snat_hairpin_dst_node) (vlib_main_t * vm,
 673                                       vlib_node_runtime_t * node,
 674                                       vlib_frame_t * frame)
 675 {
 676   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
 677 }
 678
 679 /* *INDENT-OFF* */
 680 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
 681   .name = "nat44-hairpin-dst",
 682   .vector_size = sizeof (u32),
 683   .type = VLIB_NODE_TYPE_INTERNAL,
 684   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 685   .format_trace = format_nat_hairpin_trace,
 686   .error_strings = nat44_hairpin_error_strings,
 687   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 688   .next_nodes = {
 689     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 690     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 691   },
 692 };
 693 /* *INDENT-ON* */
 694
 695 VLIB_NODE_FN (nat44_ed_hairpin_dst_node) (vlib_main_t * vm,
 696                                           vlib_node_runtime_t * node,
 697                                           vlib_frame_t * frame)
 698 {
 699   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
 700 }
 701
 702 /* *INDENT-OFF* */
 703 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
 704   .name = "nat44-ed-hairpin-dst",
 705   .vector_size = sizeof (u32),
 706   .type = VLIB_NODE_TYPE_INTERNAL,
 707   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 708   .format_trace = format_nat_hairpin_trace,
 709   .error_strings = nat44_hairpin_error_strings,
 710   .n_next_nodes = NAT_HAIRPIN_N_NEXT,
 711   .next_nodes = {
 712     [NAT_HAIRPIN_NEXT_DROP] = "error-drop",
 713     [NAT_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
 714   },
 715 };
 716 /* *INDENT-ON* */
 717
 718 static inline uword
 719 snat_hairpin_src_fn_inline (vlib_main_t * vm,
 720                             vlib_node_runtime_t * node,
 721                             vlib_frame_t * frame, int is_ed)
 722 {
 723   u32 n_left_from, *from, *to_next, stats_node_index;
 724   snat_hairpin_src_next_t next_index;
 725   u32 pkts_processed = 0;
 726   snat_main_t *sm = &snat_main;
 727
 728   stats_node_index = is_ed ? sm->ed_hairpin_src_node_index :
 729     sm->hairpin_src_node_index;
 730
 731   from = vlib_frame_vector_args (frame);
 732   n_left_from = frame->n_vectors;
 733   next_index = node->cached_next_index;
 734
 735   while (n_left_from > 0)
 736     {
 737       u32 n_left_to_next;
 738
 739       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 740
 741       while (n_left_from > 0 && n_left_to_next > 0)
 742         {
 743           u32 bi0;
 744           vlib_buffer_t *b0;
 745           u32 next0;
 746           snat_interface_t *i;
 747           u32 sw_if_index0;
 748
 749           /* speculatively enqueue b0 to the current next frame */
 750           bi0 = from[0];
 751           to_next[0] = bi0;
 752           from += 1;
 753           to_next += 1;
 754           n_left_from -= 1;
 755           n_left_to_next -= 1;
 756
 757           b0 = vlib_get_buffer (vm, bi0);
 758           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 759           vnet_feature_next (&next0, b0);
 760
 761           /* *INDENT-OFF* */
 762           pool_foreach (i, sm->output_feature_interfaces,
 763           ({
 764             /* Only packets from NAT inside interface */
 765             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
 766               {
 767                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
 768                                     SNAT_FLAG_HAIRPINNING))
 769                   {
 770                     if (PREDICT_TRUE (sm->num_workers > 1))
 771                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
 772                     else
 773                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
 774                   }
 775                 break;
 776               }
 777           }));
 778           /* *INDENT-ON* */
 779
 780           pkts_processed += next0 != SNAT_HAIRPIN_SRC_NEXT_DROP;
 781
 782           /* verify speculative enqueue, maybe switch current next frame */
 783           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 784                                            to_next, n_left_to_next,
 785                                            bi0, next0);
 786         }
 787
 788       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 789     }
 790
 791   vlib_node_increment_counter (vm, stats_node_index,
 792                                NAT44_HAIRPIN_ERROR_PROCESSED, pkts_processed);
 793   return frame->n_vectors;
 794 }
 795
 796 VLIB_NODE_FN (snat_hairpin_src_node) (vlib_main_t * vm,
 797                                       vlib_node_runtime_t * node,
 798                                       vlib_frame_t * frame)
 799 {
 800   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
 801 }
 802
 803 /* *INDENT-OFF* */
 804 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
 805   .name = "nat44-hairpin-src",
 806   .vector_size = sizeof (u32),
 807   .type = VLIB_NODE_TYPE_INTERNAL,
 808   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 809   .error_strings = nat44_hairpin_error_strings,
 810   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 811   .next_nodes = {
 812      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 813      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
 814      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 815      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 816   },
 817 };
 818 /* *INDENT-ON* */
 819
 820 VLIB_NODE_FN (nat44_ed_hairpin_src_node) (vlib_main_t * vm,
 821                                           vlib_node_runtime_t * node,
 822                                           vlib_frame_t * frame)
 823 {
 824   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
 825 }
 826
 827 /* *INDENT-OFF* */
 828 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
 829   .name = "nat44-ed-hairpin-src",
 830   .vector_size = sizeof (u32),
 831   .type = VLIB_NODE_TYPE_INTERNAL,
 832   .n_errors = ARRAY_LEN(nat44_hairpin_error_strings),
 833   .error_strings = nat44_hairpin_error_strings,
 834   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
 835   .next_nodes = {
 836      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
 837      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
 838      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
 839      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
 840   },
 841 };
 842 /* *INDENT-ON* */
 843
 844 /*
 845  * fd.io coding-style-patch-verification: ON
 846  *
 847  * Local Variables:
 848  * eval: (c-set-style "gnu")
 849  * End:
 850  */