X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fcpu.h;h=48c68a1169936641f779b10c59f77f68cd0c3923;hb=16e4a4a0ae39ebc1ded1b6dba2799b176aee1828;hp=75b01e606f3fda668736b907bb3b27a4a03702f1;hpb=04f3db3847d242857b9d9d858bcdca538a1be7d7;p=vpp.git diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index 75b01e606f3..48c68a11699 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -16,6 +16,7 @@ #ifndef included_clib_cpu_h #define included_clib_cpu_h +#include #include /* @@ -33,8 +34,8 @@ #endif -#if __GNUC__ > 4 && !__clang__ -#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("tree-vectorize"))) +#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 +#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3"))) #else #define CLIB_CPU_OPTIMIZED #endif @@ -44,33 +45,137 @@ if (clib_cpu_supports_ ## arch()) \ return & fn ## _ ##arch; -#define CLIB_MULTIARCH_SELECT_FN(fn,...) \ - __VA_ARGS__ void * fn ## _multiarch_select(void) \ -{ \ - foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \ - return & fn; \ -} +/* FIXME to be removed */ +#define CLIB_MULTIARCH_SELECT_FN(fn,...) -#ifdef CLIB_MULTIARCH_VARIANT +#ifdef CLIB_MARCH_VARIANT #define __CLIB_MULTIARCH_FN(a,b) a##_##b #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b) -#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MULTIARCH_VARIANT) +#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT) #else #define CLIB_MULTIARCH_FN(fn) fn #endif +#define CLIB_MARCH_SFX CLIB_MULTIARCH_FN + +typedef struct _clib_march_fn_registration +{ + void *function; + int priority; + struct _clib_march_fn_registration *next; + char *name; +} clib_march_fn_registration; + +static_always_inline void * +clib_march_select_fn_ptr (clib_march_fn_registration * r) +{ + void *rv = 0; + int last_prio = -1; + + while (r) + { + if (last_prio < r->priority) + { + last_prio = r->priority; + rv = r->function; + } + r = r->next; + } + return rv; +} + +#define CLIB_MARCH_FN_POINTER(fn) \ + clib_march_select_fn_ptr (fn##_march_fn_registrations); + +#define _CLIB_MARCH_FN_REGISTRATION(fn) \ +static clib_march_fn_registration \ +CLIB_MARCH_SFX(fn##_march_fn_registration) = \ +{ \ + .name = CLIB_MARCH_VARIANT_STR \ +}; \ +\ +static void __clib_constructor \ +fn##_march_register () \ +{ \ + clib_march_fn_registration *r; \ + r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \ + r->priority = CLIB_MARCH_FN_PRIORITY(); \ + r->next = fn##_march_fn_registrations; \ + r->function = CLIB_MARCH_SFX (fn); \ + fn##_march_fn_registrations = r; \ +} + +#ifdef CLIB_MARCH_VARIANT +#define CLIB_MARCH_FN_REGISTRATION(fn) \ +extern clib_march_fn_registration *fn##_march_fn_registrations; \ +_CLIB_MARCH_FN_REGISTRATION(fn) +#else +#define CLIB_MARCH_FN_REGISTRATION(fn) \ +clib_march_fn_registration *fn##_march_fn_registrations = 0; \ +_CLIB_MARCH_FN_REGISTRATION(fn) +#endif #define foreach_x86_64_flags \ _ (sse3, 1, ecx, 0) \ +_ (pclmulqdq, 1, ecx, 1) \ _ (ssse3, 1, ecx, 9) \ _ (sse41, 1, ecx, 19) \ _ (sse42, 1, ecx, 20) \ _ (avx, 1, ecx, 28) \ +_ (rdrand, 1, ecx, 30) \ _ (avx2, 7, ebx, 5) \ +_ (rtm, 7, ebx, 11) \ +_ (pqm, 7, ebx, 12) \ +_ (pqe, 7, ebx, 15) \ _ (avx512f, 7, ebx, 16) \ -_ (aes, 1, ecx, 25) \ +_ (rdseed, 7, ebx, 18) \ +_ (x86_aes, 1, ecx, 25) \ _ (sha, 7, ebx, 29) \ +_ (vaes, 7, ecx, 9) \ +_ (vpclmulqdq, 7, ecx, 10) \ _ (invariant_tsc, 0x80000007, edx, 8) + +#define foreach_aarch64_flags \ +_ (fp, 0) \ +_ (asimd, 1) \ +_ (evtstrm, 2) \ +_ (aarch64_aes, 3) \ +_ (pmull, 4) \ +_ (sha1, 5) \ +_ (sha2, 6) \ +_ (crc32, 7) \ +_ (atomics, 8) \ +_ (fphp, 9) \ +_ (asimdhp, 10) \ +_ (cpuid, 11) \ +_ (asimdrdm, 12) \ +_ (jscvt, 13) \ +_ (fcma, 14) \ +_ (lrcpc, 15) \ +_ (dcpop, 16) \ +_ (sha3, 17) \ +_ (sm3, 18) \ +_ (sm4, 19) \ +_ (asimddp, 20) \ +_ (sha512, 21) \ +_ (sve, 22) + +static inline u32 +clib_get_current_cpu_id () +{ + unsigned cpu, node; + syscall (__NR_getcpu, &cpu, &node, 0); + return cpu; +} + +static inline u32 +clib_get_current_numa_node () +{ + unsigned cpu, node; + syscall (__NR_getcpu, &cpu, &node, 0); + return node; +} + #if defined(__x86_64__) #include "cpuid.h" @@ -98,15 +203,190 @@ clib_cpu_supports_ ## flag() \ } foreach_x86_64_flags #undef _ -#else +#else /* __x86_64__ */ #define _(flag, func, reg, bit) \ static inline int clib_cpu_supports_ ## flag() { return 0; } foreach_x86_64_flags #undef _ +#endif /* __x86_64__ */ +#if defined(__aarch64__) +#include +#define _(flag, bit) \ +static inline int \ +clib_cpu_supports_ ## flag() \ +{ \ + unsigned long hwcap = getauxval(AT_HWCAP); \ + return (hwcap & (1 << bit)); \ +} + foreach_aarch64_flags +#undef _ +#else /* ! __x86_64__ && !__aarch64__ */ +#define _(flag, bit) \ +static inline int clib_cpu_supports_ ## flag() { return 0; } + foreach_aarch64_flags +#undef _ +#endif /* __x86_64__, __aarch64__ */ +/* + * aes is the only feature with the same name in both flag lists + * handle this by prefixing it with the arch name, and handling it + * with the custom function below + */ + static inline int +clib_cpu_supports_aes () +{ +#if defined(__x86_64__) + return clib_cpu_supports_x86_aes (); +#elif defined (__aarch64__) + return clib_cpu_supports_aarch64_aes (); +#else + return 0; #endif +} + +static inline int +clib_cpu_march_priority_avx512 () +{ + if (clib_cpu_supports_avx512f ()) + return 20; + return -1; +} + +static inline int +clib_cpu_march_priority_avx2 () +{ + if (clib_cpu_supports_avx2 ()) + return 50; + return -1; +} + +static inline u32 +clib_cpu_implementer () +{ + char buf[128]; + static u32 implementer = -1; + + if (-1 != implementer) + return implementer; + + FILE *fp = fopen ("/proc/cpuinfo", "r"); + if (!fp) + return implementer; + + while (!feof (fp)) + { + if (!fgets (buf, sizeof (buf), fp)) + break; + buf[127] = '\0'; + if (strstr (buf, "CPU implementer")) + implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0); + if (-1 != implementer) + break; + } + fclose (fp); + + return implementer; +} + +static inline u32 +clib_cpu_part () +{ + char buf[128]; + static u32 part = -1; + + if (-1 != part) + return part; + + FILE *fp = fopen ("/proc/cpuinfo", "r"); + if (!fp) + return part; + + while (!feof (fp)) + { + if (!fgets (buf, sizeof (buf), fp)) + break; + buf[127] = '\0'; + if (strstr (buf, "CPU part")) + part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0); + if (-1 != part) + break; + } + fclose (fp); + + return part; +} + +#define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43 +#define AARCH64_CPU_PART_THUNERDERX2 0x0af +#define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51 +#define AARCH64_CPU_PART_QDF24XX 0xc00 +#define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41 +#define AARCH64_CPU_PART_CORTEXA72 0xd08 + +static inline int +clib_cpu_march_priority_thunderx2t99 () +{ + if ((AARCH64_CPU_IMPLEMENTER_THUNERDERX2 == clib_cpu_implementer ()) && + (AARCH64_CPU_PART_THUNERDERX2 == clib_cpu_part ())) + return 20; + return -1; +} + +static inline int +clib_cpu_march_priority_qdf24xx () +{ + if ((AARCH64_CPU_IMPLEMENTER_QDF24XX == clib_cpu_implementer ()) && + (AARCH64_CPU_PART_QDF24XX == clib_cpu_part ())) + return 20; + return -1; +} + +static inline int +clib_cpu_march_priority_cortexa72 () +{ + if ((AARCH64_CPU_IMPLEMENTER_CORTEXA72 == clib_cpu_implementer ()) && + (AARCH64_CPU_PART_CORTEXA72 == clib_cpu_part ())) + return 10; + return -1; +} + +#ifdef CLIB_MARCH_VARIANT +#define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)() +#else +#define CLIB_MARCH_FN_PRIORITY() 0 #endif - format_function_t format_cpu_uarch; +#endif /* included_clib_cpu_h */ + +#define CLIB_MARCH_FN_CONSTRUCTOR(fn) \ +static void __clib_constructor \ +CLIB_MARCH_SFX(fn ## _march_constructor) (void) \ +{ \ + if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \ + { \ + fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \ + fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \ + } \ +} \ + +#ifndef CLIB_MARCH_VARIANT +#define CLIB_MARCH_FN(fn, rtype, _args...) \ + static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ + rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \ + int fn ## _selected_priority = 0; \ + static inline rtype CLIB_CPU_OPTIMIZED \ + CLIB_MARCH_SFX (fn ## _ma)(_args) +#else +#define CLIB_MARCH_FN(fn, rtype, _args...) \ + static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ + extern rtype (*fn ## _selected) (_args); \ + extern int fn ## _selected_priority; \ + CLIB_MARCH_FN_CONSTRUCTOR (fn) \ + static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) +#endif + +#define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected) + +format_function_t format_cpu_uarch; format_function_t format_cpu_model_name; format_function_t format_cpu_flags;