2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
19 #include <sys/syscall.h>
20 #include <vppinfra/format.h>
22 #if defined(__x86_64__)
23 #define foreach_march_variant \
24 _ (hsw, "Intel Haswell") \
25 _ (trm, "Intel Tremont") \
26 _ (skx, "Intel Skylake (server) / Cascade Lake") \
27 _ (icl, "Intel Ice Lake")
28 #elif defined(__aarch64__)
29 #define foreach_march_variant \
30 _ (octeontx2, "Marvell Octeon TX2") \
31 _ (thunderx2t99, "Marvell ThunderX2 T99") \
32 _ (qdf24xx, "Qualcomm CentriqTM 2400") \
33 _ (cortexa72, "ARM Cortex-A72") \
34 _ (neoversen1, "ARM Neoverse N1")
36 #define foreach_march_variant
41 CLIB_MARCH_VARIANT_TYPE = 0,
42 #define _(s, n) CLIB_MARCH_VARIANT_TYPE_##s,
45 CLIB_MARCH_TYPE_N_VARIANTS
46 } clib_march_variant_type_t;
48 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
49 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
51 #define CLIB_CPU_OPTIMIZED
54 #ifdef CLIB_MARCH_VARIANT
55 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
56 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
57 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
59 #define CLIB_MULTIARCH_FN(fn) fn
62 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
64 typedef struct _clib_march_fn_registration
68 struct _clib_march_fn_registration *next;
70 } clib_march_fn_registration;
72 static_always_inline void *
73 clib_march_select_fn_ptr (clib_march_fn_registration * r)
80 if (last_prio < r->priority)
82 last_prio = r->priority;
90 #define CLIB_MARCH_FN_POINTER(fn) \
91 clib_march_select_fn_ptr (fn##_march_fn_registrations);
93 #define _CLIB_MARCH_FN_REGISTRATION(fn) \
94 static clib_march_fn_registration \
95 CLIB_MARCH_SFX(fn##_march_fn_registration) = \
97 .name = CLIB_MARCH_VARIANT_STR \
100 static void __clib_constructor \
101 fn##_march_register () \
103 clib_march_fn_registration *r; \
104 r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
105 r->priority = CLIB_MARCH_FN_PRIORITY(); \
106 r->next = fn##_march_fn_registrations; \
107 r->function = CLIB_MARCH_SFX (fn); \
108 fn##_march_fn_registrations = r; \
111 #ifdef CLIB_MARCH_VARIANT
112 #define CLIB_MARCH_FN_REGISTRATION(fn) \
113 extern clib_march_fn_registration *fn##_march_fn_registrations; \
114 _CLIB_MARCH_FN_REGISTRATION(fn)
116 #define CLIB_MARCH_FN_REGISTRATION(fn) \
117 clib_march_fn_registration *fn##_march_fn_registrations = 0; \
118 _CLIB_MARCH_FN_REGISTRATION(fn)
120 #define foreach_x86_64_flags \
121 _ (sse3, 1, ecx, 0) \
122 _ (pclmulqdq, 1, ecx, 1) \
123 _ (ssse3, 1, ecx, 9) \
124 _ (sse41, 1, ecx, 19) \
125 _ (sse42, 1, ecx, 20) \
126 _ (avx, 1, ecx, 28) \
127 _ (rdrand, 1, ecx, 30) \
128 _ (avx2, 7, ebx, 5) \
129 _ (rtm, 7, ebx, 11) \
130 _ (pqm, 7, ebx, 12) \
131 _ (pqe, 7, ebx, 15) \
132 _ (avx512f, 7, ebx, 16) \
133 _ (rdseed, 7, ebx, 18) \
134 _ (x86_aes, 1, ecx, 25) \
135 _ (sha, 7, ebx, 29) \
136 _ (vaes, 7, ecx, 9) \
137 _ (vpclmulqdq, 7, ecx, 10) \
138 _ (avx512_vnni, 7, ecx, 11) \
139 _ (avx512_bitalg, 7, ecx, 12) \
140 _ (avx512_vpopcntdq, 7, ecx, 14) \
141 _ (movdiri, 7, ecx, 27) \
142 _ (movdir64b, 7, ecx, 28) \
143 _ (invariant_tsc, 0x80000007, edx, 8)
145 #define foreach_aarch64_flags \
170 u32 clib_get_current_cpu_id ();
171 u32 clib_get_current_numa_node ();
173 #if defined(__x86_64__)
177 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
179 if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
182 __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
184 __cpuid (lev, *eax, *ebx, *ecx, *edx);
188 typedef int (*clib_cpu_supports_func_t) ();
190 #define _(flag, func, reg, bit) \
192 clib_cpu_supports_ ## flag() \
194 u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
195 clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
197 return ((reg & (1 << bit)) != 0); \
201 #else /* __x86_64__ */
203 #define _(flag, func, reg, bit) \
204 static inline int clib_cpu_supports_ ## flag() { return 0; }
207 #endif /* __x86_64__ */
208 #if defined(__aarch64__)
209 #include <sys/auxv.h>
210 #define _(flag, bit) \
212 clib_cpu_supports_ ## flag() \
214 unsigned long hwcap = getauxval(AT_HWCAP); \
215 return (hwcap & (1 << bit)); \
217 foreach_aarch64_flags
219 #else /* ! __x86_64__ && !__aarch64__ */
220 #define _(flag, bit) \
221 static inline int clib_cpu_supports_ ## flag() { return 0; }
222 foreach_aarch64_flags
224 #endif /* __x86_64__, __aarch64__ */
226 * aes is the only feature with the same name in both flag lists
227 * handle this by prefixing it with the arch name, and handling it
228 * with the custom function below
231 clib_cpu_supports_aes ()
233 #if defined(__x86_64__)
234 return clib_cpu_supports_x86_aes ();
235 #elif defined (__aarch64__)
236 return clib_cpu_supports_aarch64_aes ();
243 clib_cpu_march_priority_icl ()
245 if (clib_cpu_supports_avx512_bitalg ())
251 clib_cpu_march_priority_skx ()
253 if (clib_cpu_supports_avx512f ())
259 clib_cpu_march_priority_trm ()
261 if (clib_cpu_supports_movdiri ())
267 clib_cpu_march_priority_hsw ()
269 if (clib_cpu_supports_avx2 ())
275 clib_cpu_implementer ()
278 static u32 implementer = -1;
280 if (-1 != implementer)
283 FILE *fp = fopen ("/proc/cpuinfo", "r");
289 if (!fgets (buf, sizeof (buf), fp))
292 if (strstr (buf, "CPU implementer"))
293 implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
294 if (-1 != implementer)
306 static u32 part = -1;
311 FILE *fp = fopen ("/proc/cpuinfo", "r");
317 if (!fgets (buf, sizeof (buf), fp))
320 if (strstr (buf, "CPU part"))
321 part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
330 #define AARCH64_CPU_IMPLEMENTER_CAVIUM 0x43
331 #define AARCH64_CPU_PART_THUNDERX2 0x0af
332 #define AARCH64_CPU_PART_OCTEONTX2T96 0x0b2
333 #define AARCH64_CPU_PART_OCTEONTX2T98 0x0b1
334 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
335 #define AARCH64_CPU_PART_QDF24XX 0xc00
336 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
337 #define AARCH64_CPU_PART_CORTEXA72 0xd08
338 #define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 0x41
339 #define AARCH64_CPU_PART_NEOVERSEN1 0xd0c
342 clib_cpu_march_priority_octeontx2 ()
344 if ((AARCH64_CPU_IMPLEMENTER_CAVIUM == clib_cpu_implementer ()) &&
345 ((AARCH64_CPU_PART_OCTEONTX2T96 == clib_cpu_part ())
346 || AARCH64_CPU_PART_OCTEONTX2T98 == clib_cpu_part ()))
352 clib_cpu_march_priority_thunderx2t99 ()
354 if ((AARCH64_CPU_IMPLEMENTER_CAVIUM == clib_cpu_implementer ()) &&
355 (AARCH64_CPU_PART_THUNDERX2 == clib_cpu_part ()))
361 clib_cpu_march_priority_qdf24xx ()
363 if ((AARCH64_CPU_IMPLEMENTER_QDF24XX == clib_cpu_implementer ()) &&
364 (AARCH64_CPU_PART_QDF24XX == clib_cpu_part ()))
370 clib_cpu_march_priority_cortexa72 ()
372 if ((AARCH64_CPU_IMPLEMENTER_CORTEXA72 == clib_cpu_implementer ()) &&
373 (AARCH64_CPU_PART_CORTEXA72 == clib_cpu_part ()))
379 clib_cpu_march_priority_neoversen1 ()
381 if ((AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 == clib_cpu_implementer ()) &&
382 (AARCH64_CPU_PART_NEOVERSEN1 == clib_cpu_part ()))
387 #ifdef CLIB_MARCH_VARIANT
388 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
390 #define CLIB_MARCH_FN_PRIORITY() 0
392 #endif /* included_clib_cpu_h */
394 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
395 static void __clib_constructor \
396 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
398 if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
400 fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
401 fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
405 #ifndef CLIB_MARCH_VARIANT
406 #define CLIB_MARCH_FN(fn, rtype, _args...) \
407 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
408 rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
409 int fn ## _selected_priority = 0; \
410 static inline rtype CLIB_CPU_OPTIMIZED \
411 CLIB_MARCH_SFX (fn ## _ma)(_args)
413 #define CLIB_MARCH_FN(fn, rtype, _args...) \
414 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
415 extern rtype (*fn ## _selected) (_args); \
416 extern int fn ## _selected_priority; \
417 CLIB_MARCH_FN_CONSTRUCTOR (fn) \
418 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
421 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
423 format_function_t format_cpu_uarch;
424 format_function_t format_cpu_model_name;
425 format_function_t format_cpu_flags;
428 * fd.io coding-style-patch-verification: ON
431 * eval: (c-set-style "gnu")