X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fcpu.h;h=bef2a029e947824560775001ddb73796320051af;hb=3d5f08a825a978cd6853989a5df8c9b9811e7fb8;hp=0ca9edb97fcc1ba3d82d70443a708106058e398c;hpb=2e2372117d35191a0e6c096c5f989930de6e12b1;p=vpp.git diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index 0ca9edb97fc..bef2a029e94 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -16,6 +16,7 @@ #ifndef included_clib_cpu_h #define included_clib_cpu_h +#include #include /* @@ -33,7 +34,7 @@ #endif -#if __GNUC__ > 4 && !__clang__ +#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3"))) #else #define CLIB_CPU_OPTIMIZED @@ -44,12 +45,8 @@ if (clib_cpu_supports_ ## arch()) \ return & fn ## _ ##arch; -#define CLIB_MULTIARCH_SELECT_FN(fn,...) \ - __VA_ARGS__ void * fn ## _multiarch_select(void) \ -{ \ - foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \ - return & fn; \ -} +/* FIXME to be removed */ +#define CLIB_MULTIARCH_SELECT_FN(fn,...) #ifdef CLIB_MARCH_VARIANT #define __CLIB_MULTIARCH_FN(a,b) a##_##b @@ -61,18 +58,106 @@ #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN -#define foreach_x86_64_flags \ -_ (sse3, 1, ecx, 0) \ -_ (ssse3, 1, ecx, 9) \ -_ (sse41, 1, ecx, 19) \ -_ (sse42, 1, ecx, 20) \ -_ (avx, 1, ecx, 28) \ -_ (avx2, 7, ebx, 5) \ -_ (avx512f, 7, ebx, 16) \ -_ (x86_aes, 1, ecx, 25) \ -_ (sha, 7, ebx, 29) \ -_ (invariant_tsc, 0x80000007, edx, 8) +typedef struct _clib_march_fn_registration +{ + void *function; + int priority; + struct _clib_march_fn_registration *next; + char *name; +} clib_march_fn_registration; + +static_always_inline void * +clib_march_select_fn_ptr (clib_march_fn_registration * r) +{ + void *rv = 0; + int last_prio = -1; + + while (r) + { + if (last_prio < r->priority) + { + last_prio = r->priority; + rv = r->function; + } + r = r->next; + } + return rv; +} + +static_always_inline void * +clib_march_select_fn_ptr_by_name (clib_march_fn_registration * r, char *name) +{ + void *rv = 0; + + while (r) + { + if (strncmp (name, r->name, vec_len (r->name) - 1) == 0) + { + rv = r->function; + break; + } + r = r->next; + } + return rv; +} + +#define CLIB_MARCH_FN_POINTER(fn) \ + clib_march_select_fn_ptr (fn##_march_fn_registrations); + +#define CLIB_MARCH_FN_POINTER_BY_NAME(fn, name) \ + clib_march_select_fn_ptr_by_name (fn##_march_fn_registrations, name); + +#define _CLIB_MARCH_FN_REGISTRATION(fn) \ +static clib_march_fn_registration \ +CLIB_MARCH_SFX(fn##_march_fn_registration) = \ +{ \ + .name = CLIB_MARCH_VARIANT_STR \ +}; \ +\ +static void __clib_constructor \ +fn##_march_register () \ +{ \ + clib_march_fn_registration *r; \ + r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \ + r->priority = CLIB_MARCH_FN_PRIORITY(); \ + r->next = fn##_march_fn_registrations; \ + r->function = CLIB_MARCH_SFX (fn); \ + fn##_march_fn_registrations = r; \ +} +#ifdef CLIB_MARCH_VARIANT +#define CLIB_MARCH_FN_REGISTRATION(fn) \ +extern clib_march_fn_registration *fn##_march_fn_registrations; \ +_CLIB_MARCH_FN_REGISTRATION(fn) +#else +#define CLIB_MARCH_FN_REGISTRATION(fn) \ +clib_march_fn_registration *fn##_march_fn_registrations = 0; \ +_CLIB_MARCH_FN_REGISTRATION(fn) +#endif +#define foreach_x86_64_flags \ + _ (sse3, 1, ecx, 0) \ + _ (pclmulqdq, 1, ecx, 1) \ + _ (ssse3, 1, ecx, 9) \ + _ (sse41, 1, ecx, 19) \ + _ (sse42, 1, ecx, 20) \ + _ (avx, 1, ecx, 28) \ + _ (rdrand, 1, ecx, 30) \ + _ (avx2, 7, ebx, 5) \ + _ (rtm, 7, ebx, 11) \ + _ (pqm, 7, ebx, 12) \ + _ (pqe, 7, ebx, 15) \ + _ (avx512f, 7, ebx, 16) \ + _ (rdseed, 7, ebx, 18) \ + _ (x86_aes, 1, ecx, 25) \ + _ (sha, 7, ebx, 29) \ + _ (vaes, 7, ecx, 9) \ + _ (vpclmulqdq, 7, ecx, 10) \ + _ (avx512_vnni, 7, ecx, 11) \ + _ (avx512_bitalg, 7, ecx, 12) \ + _ (avx512_vpopcntdq, 7, ecx, 14) \ + _ (movdiri, 7, ecx, 27) \ + _ (movdir64b, 7, ecx, 28) \ + _ (invariant_tsc, 0x80000007, edx, 8) #define foreach_aarch64_flags \ _ (fp, 0) \ @@ -99,6 +184,22 @@ _ (asimddp, 20) \ _ (sha512, 21) \ _ (sve, 22) +static inline u32 +clib_get_current_cpu_id () +{ + unsigned cpu, node; + syscall (__NR_getcpu, &cpu, &node, 0); + return cpu; +} + +static inline u32 +clib_get_current_numa_node () +{ + unsigned cpu, node; + syscall (__NR_getcpu, &cpu, &node, 0); + return node; +} + #if defined(__x86_64__) #include "cpuid.h" @@ -158,7 +259,7 @@ static inline int clib_cpu_supports_ ## flag() { return 0; } static inline int clib_cpu_supports_aes () { -#if defined (__aarch64__) +#if defined(__x86_64__) return clib_cpu_supports_x86_aes (); #elif defined (__aarch64__) return clib_cpu_supports_aarch64_aes (); @@ -168,18 +269,34 @@ clib_cpu_supports_aes () } static inline int -clib_cpu_march_priority_avx512 () +clib_cpu_march_priority_icl () +{ + if (clib_cpu_supports_avx512_bitalg ()) + return 200; + return -1; +} + +static inline int +clib_cpu_march_priority_skx () { if (clib_cpu_supports_avx512f ()) - return 20; + return 100; return -1; } static inline int -clib_cpu_march_priority_avx2 () +clib_cpu_march_priority_trm () +{ + if (clib_cpu_supports_movdiri ()) + return 60; + return -1; +} + +static inline int +clib_cpu_march_priority_hsw () { if (clib_cpu_supports_avx2 ()) - return 10; + return 50; return -1; } @@ -239,18 +356,32 @@ clib_cpu_part () return part; } -#define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43 -#define AARCH64_CPU_PART_THUNERDERX2 0x0af +#define AARCH64_CPU_IMPLEMENTER_CAVIUM 0x43 +#define AARCH64_CPU_PART_THUNDERX2 0x0af +#define AARCH64_CPU_PART_OCTEONTX2T96 0x0b2 +#define AARCH64_CPU_PART_OCTEONTX2T98 0x0b1 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51 #define AARCH64_CPU_PART_QDF24XX 0xc00 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41 #define AARCH64_CPU_PART_CORTEXA72 0xd08 +#define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 0x41 +#define AARCH64_CPU_PART_NEOVERSEN1 0xd0c + +static inline int +clib_cpu_march_priority_octeontx2 () +{ + if ((AARCH64_CPU_IMPLEMENTER_CAVIUM == clib_cpu_implementer ()) && + ((AARCH64_CPU_PART_OCTEONTX2T96 == clib_cpu_part ()) + || AARCH64_CPU_PART_OCTEONTX2T98 == clib_cpu_part ())) + return 20; + return -1; +} static inline int clib_cpu_march_priority_thunderx2t99 () { - if ((AARCH64_CPU_IMPLEMENTER_THUNERDERX2 == clib_cpu_implementer ()) && - (AARCH64_CPU_PART_THUNERDERX2 == clib_cpu_part ())) + if ((AARCH64_CPU_IMPLEMENTER_CAVIUM == clib_cpu_implementer ()) && + (AARCH64_CPU_PART_THUNDERX2 == clib_cpu_part ())) return 20; return -1; } @@ -273,6 +404,15 @@ clib_cpu_march_priority_cortexa72 () return -1; } +static inline int +clib_cpu_march_priority_neoversen1 () +{ + if ((AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 == clib_cpu_implementer ()) && + (AARCH64_CPU_PART_NEOVERSEN1 == clib_cpu_part ())) + return 10; + return -1; +} + #ifdef CLIB_MARCH_VARIANT #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)() #else @@ -301,7 +441,7 @@ CLIB_MARCH_SFX(fn ## _march_constructor) (void) \ #else #define CLIB_MARCH_FN(fn, rtype, _args...) \ static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ - extern int (*fn ## _selected) (_args); \ + extern rtype (*fn ## _selected) (_args); \ extern int fn ## _selected_priority; \ CLIB_MARCH_FN_CONSTRUCTOR (fn) \ static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)