#ifndef included_clib_cpu_h
#define included_clib_cpu_h
+#include <sys/syscall.h>
#include <vppinfra/format.h>
/*
#endif
-#if __GNUC__ > 4 && !__clang__
-#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("tree-vectorize")))
+#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
+#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
#else
#define CLIB_CPU_OPTIMIZED
#endif
if (clib_cpu_supports_ ## arch()) \
return & fn ## _ ##arch;
-#define CLIB_MULTIARCH_SELECT_FN(fn,...) \
- __VA_ARGS__ void * fn ## _multiarch_select(void) \
-{ \
- foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \
- return & fn; \
+/* FIXME to be removed */
+#define CLIB_MULTIARCH_SELECT_FN(fn,...)
+
+#ifdef CLIB_MARCH_VARIANT
+#define __CLIB_MULTIARCH_FN(a,b) a##_##b
+#define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
+#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
+#else
+#define CLIB_MULTIARCH_FN(fn) fn
+#endif
+
+#define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
+
+typedef struct _clib_march_fn_registration
+{
+ void *function;
+ int priority;
+ struct _clib_march_fn_registration *next;
+ char *name;
+} clib_march_fn_registration;
+
+static_always_inline void *
+clib_march_select_fn_ptr (clib_march_fn_registration * r)
+{
+ void *rv = 0;
+ int last_prio = -1;
+
+ while (r)
+ {
+ if (last_prio < r->priority)
+ {
+ last_prio = r->priority;
+ rv = r->function;
+ }
+ r = r->next;
+ }
+ return rv;
}
+#define CLIB_MARCH_FN_POINTER(fn) \
+ clib_march_select_fn_ptr (fn##_march_fn_registrations);
+
+#define _CLIB_MARCH_FN_REGISTRATION(fn) \
+static clib_march_fn_registration \
+CLIB_MARCH_SFX(fn##_march_fn_registration) = \
+{ \
+ .name = CLIB_MARCH_VARIANT_STR \
+}; \
+\
+static void __clib_constructor \
+fn##_march_register () \
+{ \
+ clib_march_fn_registration *r; \
+ r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
+ r->priority = CLIB_MARCH_FN_PRIORITY(); \
+ r->next = fn##_march_fn_registrations; \
+ r->function = CLIB_MARCH_SFX (fn); \
+ fn##_march_fn_registrations = r; \
+}
+#ifdef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_FN_REGISTRATION(fn) \
+extern clib_march_fn_registration *fn##_march_fn_registrations; \
+_CLIB_MARCH_FN_REGISTRATION(fn)
+#else
+#define CLIB_MARCH_FN_REGISTRATION(fn) \
+clib_march_fn_registration *fn##_march_fn_registrations = 0; \
+_CLIB_MARCH_FN_REGISTRATION(fn)
+#endif
#define foreach_x86_64_flags \
_ (sse3, 1, ecx, 0) \
+_ (pclmulqdq, 1, ecx, 1) \
_ (ssse3, 1, ecx, 9) \
_ (sse41, 1, ecx, 19) \
_ (sse42, 1, ecx, 20) \
_ (avx, 1, ecx, 28) \
+_ (rdrand, 1, ecx, 30) \
_ (avx2, 7, ebx, 5) \
+_ (rtm, 7, ebx, 11) \
+_ (pqm, 7, ebx, 12) \
+_ (pqe, 7, ebx, 15) \
_ (avx512f, 7, ebx, 16) \
-_ (aes, 1, ecx, 25) \
+_ (rdseed, 7, ebx, 18) \
+_ (x86_aes, 1, ecx, 25) \
_ (sha, 7, ebx, 29) \
+_ (vaes, 7, ecx, 9) \
+_ (vpclmulqdq, 7, ecx, 10) \
_ (invariant_tsc, 0x80000007, edx, 8)
+
+#define foreach_aarch64_flags \
+_ (fp, 0) \
+_ (asimd, 1) \
+_ (evtstrm, 2) \
+_ (aarch64_aes, 3) \
+_ (pmull, 4) \
+_ (sha1, 5) \
+_ (sha2, 6) \
+_ (crc32, 7) \
+_ (atomics, 8) \
+_ (fphp, 9) \
+_ (asimdhp, 10) \
+_ (cpuid, 11) \
+_ (asimdrdm, 12) \
+_ (jscvt, 13) \
+_ (fcma, 14) \
+_ (lrcpc, 15) \
+_ (dcpop, 16) \
+_ (sha3, 17) \
+_ (sm3, 18) \
+_ (sm4, 19) \
+_ (asimddp, 20) \
+_ (sha512, 21) \
+_ (sve, 22)
+
+static inline u32
+clib_get_current_cpu_id ()
+{
+ unsigned cpu, node;
+ syscall (__NR_getcpu, &cpu, &node, 0);
+ return cpu;
+}
+
+static inline u32
+clib_get_current_numa_node ()
+{
+ unsigned cpu, node;
+ syscall (__NR_getcpu, &cpu, &node, 0);
+ return node;
+}
+
#if defined(__x86_64__)
#include "cpuid.h"
}
foreach_x86_64_flags
#undef _
-#else
+#else /* __x86_64__ */
#define _(flag, func, reg, bit) \
static inline int clib_cpu_supports_ ## flag() { return 0; }
foreach_x86_64_flags
#undef _
+#endif /* __x86_64__ */
+#if defined(__aarch64__)
+#include <sys/auxv.h>
+#define _(flag, bit) \
+static inline int \
+clib_cpu_supports_ ## flag() \
+{ \
+ unsigned long hwcap = getauxval(AT_HWCAP); \
+ return (hwcap & (1 << bit)); \
+}
+ foreach_aarch64_flags
+#undef _
+#else /* ! __x86_64__ && !__aarch64__ */
+#define _(flag, bit) \
+static inline int clib_cpu_supports_ ## flag() { return 0; }
+ foreach_aarch64_flags
+#undef _
+#endif /* __x86_64__, __aarch64__ */
+/*
+ * aes is the only feature with the same name in both flag lists
+ * handle this by prefixing it with the arch name, and handling it
+ * with the custom function below
+ */
+ static inline int
+clib_cpu_supports_aes ()
+{
+#if defined(__x86_64__)
+ return clib_cpu_supports_x86_aes ();
+#elif defined (__aarch64__)
+ return clib_cpu_supports_aarch64_aes ();
+#else
+ return 0;
#endif
+}
+
+static inline int
+clib_cpu_march_priority_avx512 ()
+{
+ if (clib_cpu_supports_avx512f ())
+ return 20;
+ return -1;
+}
+
+static inline int
+clib_cpu_march_priority_avx2 ()
+{
+ if (clib_cpu_supports_avx2 ())
+ return 50;
+ return -1;
+}
+
+static inline u32
+clib_cpu_implementer ()
+{
+ char buf[128];
+ static u32 implementer = -1;
+
+ if (-1 != implementer)
+ return implementer;
+
+ FILE *fp = fopen ("/proc/cpuinfo", "r");
+ if (!fp)
+ return implementer;
+
+ while (!feof (fp))
+ {
+ if (!fgets (buf, sizeof (buf), fp))
+ break;
+ buf[127] = '\0';
+ if (strstr (buf, "CPU implementer"))
+ implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
+ if (-1 != implementer)
+ break;
+ }
+ fclose (fp);
+
+ return implementer;
+}
+
+static inline u32
+clib_cpu_part ()
+{
+ char buf[128];
+ static u32 part = -1;
+
+ if (-1 != part)
+ return part;
+
+ FILE *fp = fopen ("/proc/cpuinfo", "r");
+ if (!fp)
+ return part;
+
+ while (!feof (fp))
+ {
+ if (!fgets (buf, sizeof (buf), fp))
+ break;
+ buf[127] = '\0';
+ if (strstr (buf, "CPU part"))
+ part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
+ if (-1 != part)
+ break;
+ }
+ fclose (fp);
+
+ return part;
+}
+
+#define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43
+#define AARCH64_CPU_PART_THUNERDERX2 0x0af
+#define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
+#define AARCH64_CPU_PART_QDF24XX 0xc00
+#define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
+#define AARCH64_CPU_PART_CORTEXA72 0xd08
+
+static inline int
+clib_cpu_march_priority_thunderx2t99 ()
+{
+ if ((AARCH64_CPU_IMPLEMENTER_THUNERDERX2 == clib_cpu_implementer ()) &&
+ (AARCH64_CPU_PART_THUNERDERX2 == clib_cpu_part ()))
+ return 20;
+ return -1;
+}
+
+static inline int
+clib_cpu_march_priority_qdf24xx ()
+{
+ if ((AARCH64_CPU_IMPLEMENTER_QDF24XX == clib_cpu_implementer ()) &&
+ (AARCH64_CPU_PART_QDF24XX == clib_cpu_part ()))
+ return 20;
+ return -1;
+}
+
+static inline int
+clib_cpu_march_priority_cortexa72 ()
+{
+ if ((AARCH64_CPU_IMPLEMENTER_CORTEXA72 == clib_cpu_implementer ()) &&
+ (AARCH64_CPU_PART_CORTEXA72 == clib_cpu_part ()))
+ return 10;
+ return -1;
+}
+
+#ifdef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
+#else
+#define CLIB_MARCH_FN_PRIORITY() 0
#endif
- format_function_t format_cpu_uarch;
+#endif /* included_clib_cpu_h */
+
+#define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
+static void __clib_constructor \
+CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
+{ \
+ if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
+ { \
+ fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
+ fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
+ } \
+} \
+
+#ifndef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_FN(fn, rtype, _args...) \
+ static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
+ rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
+ int fn ## _selected_priority = 0; \
+ static inline rtype CLIB_CPU_OPTIMIZED \
+ CLIB_MARCH_SFX (fn ## _ma)(_args)
+#else
+#define CLIB_MARCH_FN(fn, rtype, _args...) \
+ static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
+ extern rtype (*fn ## _selected) (_args); \
+ extern int fn ## _selected_priority; \
+ CLIB_MARCH_FN_CONSTRUCTOR (fn) \
+ static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
+#endif
+
+#define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
+
+format_function_t format_cpu_uarch;
format_function_t format_cpu_model_name;
format_function_t format_cpu_flags;
Code Review / vpp.git / blobdiff