nat: optimize prefetching
[vpp.git] / src / plugins / nat / nat_det_in2out.c
index 1366c2f..74b4149 100644 (file)
@@ -26,6 +26,7 @@
 #include <nat/nat.h>
 #include <nat/nat_det.h>
 #include <nat/nat_inlines.h>
+#include <nat/lib/nat_inlines.h>
 
 typedef struct
 {
@@ -63,8 +64,6 @@ static char *nat_det_in2out_error_strings[] = {
 #undef _
 };
 
-vlib_node_registration_t snat_det_in2out_node;
-
 static u8 *
 format_nat_det_in2out_trace (u8 * s, va_list * args)
 {
@@ -78,6 +77,7 @@ format_nat_det_in2out_trace (u8 * s, va_list * args)
   return s;
 }
 
+#ifndef CLIB_MARCH_VARIANT
 /**
  * Get address and port values to be used for ICMP packet translation
  * and create session if needed
@@ -86,6 +86,7 @@ format_nat_det_in2out_trace (u8 * s, va_list * args)
  * @param[in,out] node           NAT node runtime
  * @param[in] thread_index       thread index
  * @param[in,out] b0             buffer containing packet to be translated
+ * @param[in,out] ip0            ip header
  * @param[out] p_proto           protocol used for matching
  * @param[out] p_value           address and port after NAT translation
  * @param[out] p_dont_translate  if packet should not be translated
@@ -95,16 +96,17 @@ format_nat_det_in2out_trace (u8 * s, va_list * args)
 u32
 icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
                       u32 thread_index, vlib_buffer_t * b0,
-                      ip4_header_t * ip0, u8 * p_proto,
-                      snat_session_key_t * p_value,
-                      u8 * p_dont_translate, void *d, void *e)
+                      ip4_header_t * ip0, ip4_address_t * addr,
+                      u16 * port, u32 * fib_index,
+                      nat_protocol_t * proto, void *d, void *e,
+                      u8 * dont_translate)
 {
+  vlib_main_t *vm = vlib_get_main ();
   icmp46_header_t *icmp0;
   u32 sw_if_index0;
   u32 rx_fib_index0;
-  u8 protocol;
+  nat_protocol_t protocol;
   snat_det_out_key_t key0;
-  u8 dont_translate = 0;
   u32 next0 = ~0;
   icmp_echo_header_t *echo0, *inner_echo0 = 0;
   ip4_header_t *inner_ip0;
@@ -116,33 +118,36 @@ icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
   snat_det_session_t *ses0 = 0;
   ip4_address_t in_addr;
   u16 in_port;
+  *dont_translate = 0;
 
   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 
-  if (!icmp_is_error_message (icmp0))
+  if (!icmp_type_is_error_message
+      (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
     {
-      protocol = SNAT_PROTOCOL_ICMP;
+      protocol = NAT_PROTOCOL_ICMP;
       in_addr = ip0->src_address;
-      in_port = echo0->identifier;
+      in_port = vnet_buffer (b0)->ip.reass.l4_src_port;
     }
   else
     {
+      /* if error message, then it's not fragmented and we can access it */
       inner_ip0 = (ip4_header_t *) (echo0 + 1);
       l4_header = ip4_next_header (inner_ip0);
-      protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
+      protocol = ip_proto_to_nat_proto (inner_ip0->protocol);
       in_addr = inner_ip0->dst_address;
       switch (protocol)
        {
-       case SNAT_PROTOCOL_ICMP:
+       case NAT_PROTOCOL_ICMP:
          inner_icmp0 = (icmp46_header_t *) l4_header;
          inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
          in_port = inner_echo0->identifier;
          break;
-       case SNAT_PROTOCOL_UDP:
-       case SNAT_PROTOCOL_TCP:
+       case NAT_PROTOCOL_UDP:
+       case NAT_PROTOCOL_TCP:
          in_port = ((tcp_udp_header_t *) l4_header)->dst_port;
          break;
        default:
@@ -161,7 +166,7 @@ icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
                                                  IP_PROTOCOL_ICMP,
                                                  rx_fib_index0)))
        {
-         dont_translate = 1;
+         *dont_translate = 1;
          goto out;
        }
       next0 = NAT_DET_IN2OUT_NEXT_DROP;
@@ -181,7 +186,7 @@ icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
                                                  IP_PROTOCOL_ICMP,
                                                  rx_fib_index0)))
        {
-         dont_translate = 1;
+         *dont_translate = 1;
          goto out;
        }
       if (icmp0->type != ICMP4_echo_request)
@@ -202,7 +207,8 @@ icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
            continue;
 
          ses0 =
-           snat_det_ses_create (dm0, &in_addr, echo0->identifier, &key0);
+           snat_det_ses_create (thread_index, dm0,
+                                &in_addr, echo0->identifier, &key0);
          break;
        }
       if (PREDICT_FALSE (!ses0))
@@ -213,38 +219,40 @@ icmp_match_in2out_det (snat_main_t * sm, vlib_node_runtime_t * node,
        }
     }
 
-  if (PREDICT_FALSE (icmp0->type != ICMP4_echo_request &&
-                    !icmp_is_error_message (icmp0)))
+  if (PREDICT_FALSE
+      (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
+       && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
+                                      reass.icmp_type_or_tcp_flags)))
     {
       b0->error = node->errors[NAT_DET_IN2OUT_ERROR_BAD_ICMP_TYPE];
       next0 = NAT_DET_IN2OUT_NEXT_DROP;
       goto out;
     }
 
-  u32 now = (u32) vlib_time_now (sm->vlib_main);
+  u32 now = (u32) vlib_time_now (vm);
 
   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
   ses0->expire = now + sm->icmp_timeout;
 
 out:
-  *p_proto = protocol;
+  *proto = protocol;
   if (ses0)
     {
-      p_value->addr = new_addr0;
-      p_value->fib_index = sm->outside_fib_index;
-      p_value->port = ses0->out.out_port;
+      *addr = new_addr0;
+      *fib_index = sm->outside_fib_index;
+      *port = ses0->out.out_port;
     }
-  *p_dont_translate = dont_translate;
   if (d)
     *(snat_det_session_t **) d = ses0;
   if (e)
     *(snat_det_map_t **) e = dm0;
   return next0;
 }
+#endif
 
-static uword
-snat_det_in2out_node_fn (vlib_main_t * vm,
-                        vlib_node_runtime_t * node, vlib_frame_t * frame)
+VLIB_NODE_FN (snat_det_in2out_node) (vlib_main_t * vm,
+                                    vlib_node_runtime_t * node,
+                                    vlib_frame_t * frame)
 {
   u32 n_left_from, *from, *to_next;
   nat_det_in2out_next_t next_index;
@@ -293,8 +301,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
+           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
          }
 
          /* speculatively enqueue b0 and b1 to the current next frame */
@@ -327,9 +335,9 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
              goto trace0;
            }
 
-         proto0 = ip_proto_to_snat_proto (ip0->protocol);
+         proto0 = ip_proto_to_nat_proto (ip0->protocol);
 
-         if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+         if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
            {
              rx_fib_index0 =
                ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
@@ -374,8 +382,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                    continue;
 
                  ses0 =
-                   snat_det_ses_create (dm0, &ip0->src_address, tcp0->src,
-                                        &key0);
+                   snat_det_ses_create (thread_index, dm0, &ip0->src_address,
+                                        tcp0->src, &key0);
                  break;
                }
              if (PREDICT_FALSE (!ses0))
@@ -391,7 +399,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                }
            }
 
-         new_port0 = ses0->out.out_port;
+         old_port0 = udp0->src_port;
+         udp0->src_port = new_port0 = ses0->out.out_port;
 
          old_addr0.as_u32 = ip0->src_address.as_u32;
          ip0->src_address.as_u32 = new_addr0.as_u32;
@@ -403,7 +412,7 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                                 src_address /* changed member */ );
          ip0->checksum = ip_csum_fold (sum0);
 
-         if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
+         if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
            {
              if (tcp0->flags & TCP_FLAG_SYN)
                ses0->state = SNAT_SESSION_TCP_SYN_SENT;
@@ -423,9 +432,6 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                       && ses0->state == SNAT_SESSION_UNKNOWN)
                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
 
-             old_port0 = tcp0->src;
-             tcp0->src = new_port0;
-
              sum0 = tcp0->checksum;
              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
                                     ip4_header_t,
@@ -433,15 +439,26 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
              sum0 = ip_csum_update (sum0, old_port0, new_port0,
                                     ip4_header_t /* cheat */ ,
                                     length /* changed member */ );
-             mss_clamping (sm, tcp0, &sum0);
+             mss_clamping (sm->mss_clamping, tcp0, &sum0);
              tcp0->checksum = ip_csum_fold (sum0);
            }
          else
            {
              ses0->state = SNAT_SESSION_UDP_ACTIVE;
-             old_port0 = udp0->src_port;
-             udp0->src_port = new_port0;
-             udp0->checksum = 0;
+
+             if (PREDICT_FALSE (udp0->checksum))
+               {
+                 sum0 = udp0->checksum;
+                 sum0 =
+                   ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                   ip4_header_t,
+                                   dst_address /* changed member */ );
+                 sum0 =
+                   ip_csum_update (sum0, old_port0, new_port0,
+                                   ip4_header_t /* cheat */ ,
+                                   length /* changed member */ );
+                 udp0->checksum = ip_csum_fold (sum0);
+               }
            }
 
          switch (ses0->state)
@@ -491,9 +508,9 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
              goto trace1;
            }
 
-         proto1 = ip_proto_to_snat_proto (ip1->protocol);
+         proto1 = ip_proto_to_nat_proto (ip1->protocol);
 
-         if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
+         if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
            {
              rx_fib_index1 =
                ip4_fib_table_get_index_for_sw_if_index (sw_if_index1);
@@ -538,8 +555,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                    continue;
 
                  ses1 =
-                   snat_det_ses_create (dm1, &ip1->src_address, tcp1->src,
-                                        &key1);
+                   snat_det_ses_create (thread_index, dm1, &ip1->src_address,
+                                        tcp1->src, &key1);
                  break;
                }
              if (PREDICT_FALSE (!ses1))
@@ -555,7 +572,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                }
            }
 
-         new_port1 = ses1->out.out_port;
+         old_port1 = udp1->src_port;
+         udp1->src_port = new_port1 = ses1->out.out_port;
 
          old_addr1.as_u32 = ip1->src_address.as_u32;
          ip1->src_address.as_u32 = new_addr1.as_u32;
@@ -567,7 +585,7 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                                 src_address /* changed member */ );
          ip1->checksum = ip_csum_fold (sum1);
 
-         if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
+         if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
            {
              if (tcp1->flags & TCP_FLAG_SYN)
                ses1->state = SNAT_SESSION_TCP_SYN_SENT;
@@ -587,9 +605,6 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                       && ses1->state == SNAT_SESSION_UNKNOWN)
                ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
 
-             old_port1 = tcp1->src;
-             tcp1->src = new_port1;
-
              sum1 = tcp1->checksum;
              sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
                                     ip4_header_t,
@@ -597,15 +612,26 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
              sum1 = ip_csum_update (sum1, old_port1, new_port1,
                                     ip4_header_t /* cheat */ ,
                                     length /* changed member */ );
-             mss_clamping (sm, tcp1, &sum1);
+             mss_clamping (sm->mss_clamping, tcp1, &sum1);
              tcp1->checksum = ip_csum_fold (sum1);
            }
          else
            {
              ses1->state = SNAT_SESSION_UDP_ACTIVE;
-             old_port1 = udp1->src_port;
-             udp1->src_port = new_port1;
-             udp1->checksum = 0;
+
+             if (PREDICT_FALSE (udp1->checksum))
+               {
+                 sum1 = udp1->checksum;
+                 sum1 =
+                   ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
+                                   ip4_header_t,
+                                   dst_address /* changed member */ );
+                 sum1 =
+                   ip_csum_update (sum1, old_port1, new_port1,
+                                   ip4_header_t /* cheat */ ,
+                                   length /* changed member */ );
+                 udp1->checksum = ip_csum_fold (sum1);
+               }
            }
 
          switch (ses1->state)
@@ -691,9 +717,9 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
              goto trace00;
            }
 
-         proto0 = ip_proto_to_snat_proto (ip0->protocol);
+         proto0 = ip_proto_to_nat_proto (ip0->protocol);
 
-         if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+         if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
            {
              rx_fib_index0 =
                ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
@@ -738,8 +764,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                    continue;
 
                  ses0 =
-                   snat_det_ses_create (dm0, &ip0->src_address, tcp0->src,
-                                        &key0);
+                   snat_det_ses_create (thread_index, dm0, &ip0->src_address,
+                                        tcp0->src, &key0);
                  break;
                }
              if (PREDICT_FALSE (!ses0))
@@ -755,7 +781,8 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                }
            }
 
-         new_port0 = ses0->out.out_port;
+         old_port0 = udp0->src_port;
+         udp0->src_port = new_port0 = ses0->out.out_port;
 
          old_addr0.as_u32 = ip0->src_address.as_u32;
          ip0->src_address.as_u32 = new_addr0.as_u32;
@@ -767,7 +794,7 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                                 src_address /* changed member */ );
          ip0->checksum = ip_csum_fold (sum0);
 
-         if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
+         if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
            {
              if (tcp0->flags & TCP_FLAG_SYN)
                ses0->state = SNAT_SESSION_TCP_SYN_SENT;
@@ -787,9 +814,6 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
                       && ses0->state == SNAT_SESSION_UNKNOWN)
                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
 
-             old_port0 = tcp0->src;
-             tcp0->src = new_port0;
-
              sum0 = tcp0->checksum;
              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
                                     ip4_header_t,
@@ -797,15 +821,26 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
              sum0 = ip_csum_update (sum0, old_port0, new_port0,
                                     ip4_header_t /* cheat */ ,
                                     length /* changed member */ );
-             mss_clamping (sm, tcp0, &sum0);
+             mss_clamping (sm->mss_clamping, tcp0, &sum0);
              tcp0->checksum = ip_csum_fold (sum0);
            }
          else
            {
              ses0->state = SNAT_SESSION_UDP_ACTIVE;
-             old_port0 = udp0->src_port;
-             udp0->src_port = new_port0;
-             udp0->checksum = 0;
+
+             if (PREDICT_FALSE (udp0->checksum))
+               {
+                 sum0 = udp0->checksum;
+                 sum0 =
+                   ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                   ip4_header_t,
+                                   dst_address /* changed member */ );
+                 sum0 =
+                   ip_csum_update (sum0, old_port0, new_port0,
+                                   ip4_header_t /* cheat */ ,
+                                   length /* changed member */ );
+                 udp0->checksum = ip_csum_fold (sum0);
+               }
            }
 
          switch (ses0->state)
@@ -848,7 +883,7 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
 
-  vlib_node_increment_counter (vm, snat_det_in2out_node.index,
+  vlib_node_increment_counter (vm, sm->det_in2out_node_index,
                               NAT_DET_IN2OUT_ERROR_IN2OUT_PACKETS,
                               pkts_processed);
   return frame->n_vectors;
@@ -856,7 +891,6 @@ snat_det_in2out_node_fn (vlib_main_t * vm,
 
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
-  .function = snat_det_in2out_node_fn,
   .name = "nat44-det-in2out",
   .vector_size = sizeof (u32),
   .format_trace = format_nat_det_in2out_trace,
@@ -873,8 +907,6 @@ VLIB_REGISTER_NODE (snat_det_in2out_node) = {
 };
 /* *INDENT-ON* */
 
-VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
-
 /*
  * fd.io coding-style-patch-verification: ON
  *