X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fcpu.h;h=c636cf8639c3c4645e5a6d24894ac8e8f250dfb2;hb=983cc7da8566fcbe2647ed2d3921e7683bd8d325;hp=9c149f3fa2aecd469c9c617604201b080896671a;hpb=7cd468a3d7dee7d6c92f69a0bb7061ae208ec727;p=vpp.git

diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h
index 9c149f3fa2a..c636cf8639c 100644
--- a/src/vppinfra/cpu.h
+++ b/src/vppinfra/cpu.h
@@ -34,7 +34,7 @@
 
 
 #if __GNUC__ > 4  && !__clang__
-#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("tree-vectorize")))
+#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
 #else
 #define CLIB_CPU_OPTIMIZED
 #endif
@@ -51,6 +51,15 @@
   return & fn;                                                         \
 }
 
+#ifdef CLIB_MARCH_VARIANT
+#define __CLIB_MULTIARCH_FN(a,b) a##_##b
+#define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
+#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
+#else
+#define CLIB_MULTIARCH_FN(fn) fn
+#endif
+
+#define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
 
 #define foreach_x86_64_flags \
 _ (sse3,     1, ecx, 0)   \
@@ -60,10 +69,36 @@ _ (sse42,    1, ecx, 20)  \
 _ (avx,      1, ecx, 28)  \
 _ (avx2,     7, ebx, 5)   \
 _ (avx512f,  7, ebx, 16)  \
-_ (aes,      1, ecx, 25)  \
+_ (x86_aes,  1, ecx, 25)  \
 _ (sha,      7, ebx, 29)  \
 _ (invariant_tsc, 0x80000007, edx, 8)
 
+
+#define foreach_aarch64_flags \
+_ (fp,          0) \
+_ (asimd,       1) \
+_ (evtstrm,     2) \
+_ (aarch64_aes, 3) \
+_ (pmull,       4) \
+_ (sha1,        5) \
+_ (sha2,        6) \
+_ (crc32,       7) \
+_ (atomics,     8) \
+_ (fphp,        9) \
+_ (asimdhp,    10) \
+_ (cpuid,      11) \
+_ (asimdrdm,   12) \
+_ (jscvt,      13) \
+_ (fcma,       14) \
+_ (lrcpc,      15) \
+_ (dcpop,      16) \
+_ (sha3,       17) \
+_ (sm3,        18) \
+_ (sm4,        19) \
+_ (asimddp,    20) \
+_ (sha512,     21) \
+_ (sve,        22)
+
 #if defined(__x86_64__)
 #include "cpuid.h"
 
@@ -91,15 +126,100 @@ clib_cpu_supports_ ## flag()						\
 }
 foreach_x86_64_flags
 #undef _
-#else
+#else /* __x86_64__ */
 
 #define _(flag, func, reg, bit) \
 static inline int clib_cpu_supports_ ## flag() { return 0; }
 foreach_x86_64_flags
 #undef _
+#endif /* __x86_64__ */
+#if defined(__aarch64__)
+#include <sys/auxv.h>
+#define _(flag, bit) \
+static inline int							\
+clib_cpu_supports_ ## flag()						\
+{									\
+  unsigned long hwcap = getauxval(AT_HWCAP);				\
+  return (hwcap & (1 << bit));						\
+}
+  foreach_aarch64_flags
+#undef _
+#else /* ! __x86_64__ && !__aarch64__ */
+#define _(flag, bit) \
+static inline int clib_cpu_supports_ ## flag() { return 0; }
+  foreach_aarch64_flags
+#undef _
+#endif /* __x86_64__, __aarch64__ */
+/*
+ * aes is the only feature with the same name in both flag lists
+ * handle this by prefixing it with the arch name, and handling it
+ * with the custom function below
+ */
+  static inline int
+clib_cpu_supports_aes ()
+{
+#if defined (__aarch64__)
+  return clib_cpu_supports_x86_aes ();
+#elif defined (__aarch64__)
+  return clib_cpu_supports_aarch64_aes ();
+#else
+  return 0;
 #endif
+}
+
+static inline int
+clib_cpu_march_priority_avx512 ()
+{
+  if (clib_cpu_supports_avx512f ())
+    return 20;
+  return -1;
+}
+
+static inline int
+clib_cpu_march_priority_avx2 ()
+{
+  if (clib_cpu_supports_avx2 ())
+    return 10;
+  return -1;
+}
+
+#ifdef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
+#else
+#define CLIB_MARCH_FN_PRIORITY() 0
+#endif
+#endif /* included_clib_cpu_h */
+
+#define CLIB_MARCH_FN_CONSTRUCTOR(fn)					\
+static void __clib_constructor 						\
+CLIB_MARCH_SFX(fn ## _march_constructor) (void)				\
+{									\
+  if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority)		\
+    {									\
+      fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma);			\
+      fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY();		\
+    }									\
+}									\
+
+#ifndef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_FN(fn, rtype, _args...)				\
+  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args);	\
+  rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma);	\
+  int fn ## _selected_priority = 0;					\
+  static inline rtype CLIB_CPU_OPTIMIZED 				\
+  CLIB_MARCH_SFX (fn ## _ma)(_args)
+#else
+#define CLIB_MARCH_FN(fn, rtype, _args...)				\
+  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args);	\
+  extern int (*fn ## _selected) (_args);				\
+  extern int fn ## _selected_priority;					\
+  CLIB_MARCH_FN_CONSTRUCTOR (fn)					\
+  static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
 #endif
-  format_function_t format_cpu_uarch;
+
+#define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
+
+format_function_t format_cpu_uarch;
 format_function_t format_cpu_model_name;
 format_function_t format_cpu_flags;