dpdk: introduce AVX512 variants of node functions

[vpp.git] / src / plugins / dpdk / device / node.c

diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
index 8297821..1240b8d 100644 (file)
--- a/src/plugins/dpdk/device/node.c
+++ b/src/plugins/dpdk/device/node.c
@@ -28,11 +28,13 @@
 
 #include <dpdk/device/dpdk_priv.h>
 
+#ifndef CLIB_MULTIARCH_VARIANT
 static char *dpdk_error_strings[] = {
 #define _(n,s) s,
   foreach_dpdk_error
 #undef _
 };
+#endif
 
 always_inline int
 vlib_buffer_is_ip4 (vlib_buffer_t * b)
@@ -259,43 +261,6 @@ dpdk_prefetch_ethertype (struct rte_mbuf *mb)
                 CLIB_CACHE_LINE_BYTES, LOAD);
 }
 
-
-/*
-   This function should fill 1st cacheline of vlib_buffer_t metadata with data
-   from buffer template. Instead of filling field by field, we construct
-   template and then use 128/256 bit vector instruction to copy data.
-   This code first loads whole cacheline into 4 128-bit registers (xmm)
-   or two 256 bit registers (ymm) and then stores data into all 4 buffers
-   efectively saving on register load operations.
-*/
-
-static_always_inline void
-dpdk_buffer_init_from_template (void *d0, void *d1, void *d2, void *d3,
-                               void *s)
-{
-#if defined(CLIB_HAVE_VEC128)
-  int i;
-  for (i = 0; i < 2; i++)
-    {
-      *(u8x32 *) (((u8 *) d0) + i * 32) =
-       *(u8x32 *) (((u8 *) d1) + i * 32) =
-       *(u8x32 *) (((u8 *) d2) + i * 32) =
-       *(u8x32 *) (((u8 *) d3) + i * 32) = *(u8x32 *) (((u8 *) s) + i * 32);
-    }
-#elif defined(CLIB_HAVE_VEC64)
-  int i;
-  for (i = 0; i < 4; i++)
-    {
-      *(u8x16 *) (((u8 *) d0) + i * 16) =
-       *(u8x16 *) (((u8 *) d1) + i * 16) =
-       *(u8x16 *) (((u8 *) d2) + i * 16) =
-       *(u8x16 *) (((u8 *) d3) + i * 16) = *(u8x16 *) (((u8 *) s) + i * 16);
-    }
-#else
-#error "Either CLIB_HAVE_VEC128 or CLIB_HAVE_VEC64 has to be defined"
-#endif
-}
-
 /*
  * This function is used when there are no worker threads.
  * The main thread performs IO and forwards the packets.
@@ -401,7 +366,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
          b2 = vlib_buffer_from_rte_mbuf (mb2);
          b3 = vlib_buffer_from_rte_mbuf (mb3);
 
-         dpdk_buffer_init_from_template (b0, b1, b2, b3, bt);
+         clib_memcpy64_x4 (b0, b1, b2, b3, bt);
 
          dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 9]);
          dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 5]);
@@ -647,8 +612,9 @@ poll_rate_limit (dpdk_main_t * dm)
       <code>xd->per_interface_next_index</code>
 */
 
-static uword
-dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f)
+uword
+CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
+                               vlib_frame_t * f)
 {
   dpdk_main_t *dm = &dpdk_main;
   dpdk_device_t *xd;
@@ -678,6 +644,7 @@ dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f)
   return n_rx_packets;
 }
 
+#ifndef CLIB_MULTIARCH_VARIANT
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (dpdk_input_node) = {
   .function = dpdk_input,
@@ -694,10 +661,23 @@ VLIB_REGISTER_NODE (dpdk_input_node) = {
   .n_errors = DPDK_N_ERROR,
   .error_strings = dpdk_error_strings,
 };
-
-VLIB_NODE_FUNCTION_MULTIARCH (dpdk_input_node, dpdk_input);
 /* *INDENT-ON* */
 
+vlib_node_function_t __clib_weak dpdk_input_avx512;
+vlib_node_function_t __clib_weak dpdk_input_avx2;
+
+#if __x86_64__
+static void __clib_constructor
+dpdk_input_multiarch_select (void)
+{
+  if (dpdk_input_avx512 && clib_cpu_supports_avx512f ())
+    dpdk_input_node.function = dpdk_input_avx512;
+  else if (dpdk_input_avx2 && clib_cpu_supports_avx2 ())
+    dpdk_input_node.function = dpdk_input_avx2;
+}
+#endif
+#endif
+
 /*
  * fd.io coding-style-patch-verification: ON
  *