2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
19 #include <sys/syscall.h>
20 #include <vppinfra/format.h>
23 * multiarchitecture support. Adding new entry will produce
24 * new graph node function variant optimized for specific cpu
26 * Order is important for runtime selection, as 1st match wins...
29 #if __x86_64__ && CLIB_DEBUG == 0
30 #define foreach_march_variant(macro, x) \
31 macro(avx2, x, "arch=core-avx2")
33 #define foreach_march_variant(macro, x)
37 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
38 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
40 #define CLIB_CPU_OPTIMIZED
44 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
45 if (clib_cpu_supports_ ## arch()) \
46 return & fn ## _ ##arch;
48 #define CLIB_MULTIARCH_SELECT_FN(fn,...) \
49 __VA_ARGS__ void * fn ## _multiarch_select(void) \
51 foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \
55 #ifdef CLIB_MARCH_VARIANT
56 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
57 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
58 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
60 #define CLIB_MULTIARCH_FN(fn) fn
63 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
65 #define foreach_x86_64_flags \
67 _ (ssse3, 1, ecx, 9) \
68 _ (sse41, 1, ecx, 19) \
69 _ (sse42, 1, ecx, 20) \
72 _ (avx512f, 7, ebx, 16) \
73 _ (x86_aes, 1, ecx, 25) \
75 _ (invariant_tsc, 0x80000007, edx, 8)
78 #define foreach_aarch64_flags \
104 clib_get_current_cpu_id ()
107 syscall (__NR_getcpu, &cpu, &node, 0);
112 clib_get_current_numa_node ()
115 syscall (__NR_getcpu, &cpu, &node, 0);
119 #if defined(__x86_64__)
123 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
125 if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
128 __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
130 __cpuid (lev, *eax, *ebx, *ecx, *edx);
135 #define _(flag, func, reg, bit) \
137 clib_cpu_supports_ ## flag() \
139 u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
140 clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
142 return ((reg & (1 << bit)) != 0); \
146 #else /* __x86_64__ */
148 #define _(flag, func, reg, bit) \
149 static inline int clib_cpu_supports_ ## flag() { return 0; }
152 #endif /* __x86_64__ */
153 #if defined(__aarch64__)
154 #include <sys/auxv.h>
155 #define _(flag, bit) \
157 clib_cpu_supports_ ## flag() \
159 unsigned long hwcap = getauxval(AT_HWCAP); \
160 return (hwcap & (1 << bit)); \
162 foreach_aarch64_flags
164 #else /* ! __x86_64__ && !__aarch64__ */
165 #define _(flag, bit) \
166 static inline int clib_cpu_supports_ ## flag() { return 0; }
167 foreach_aarch64_flags
169 #endif /* __x86_64__, __aarch64__ */
171 * aes is the only feature with the same name in both flag lists
172 * handle this by prefixing it with the arch name, and handling it
173 * with the custom function below
176 clib_cpu_supports_aes ()
178 #if defined (__aarch64__)
179 return clib_cpu_supports_x86_aes ();
180 #elif defined (__aarch64__)
181 return clib_cpu_supports_aarch64_aes ();
188 clib_cpu_march_priority_avx512 ()
190 if (clib_cpu_supports_avx512f ())
196 clib_cpu_march_priority_avx2 ()
198 if (clib_cpu_supports_avx2 ())
204 clib_cpu_implementer ()
207 static u32 implementer = -1;
209 if (-1 != implementer)
212 FILE *fp = fopen ("/proc/cpuinfo", "r");
218 if (!fgets (buf, sizeof (buf), fp))
221 if (strstr (buf, "CPU implementer"))
222 implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
223 if (-1 != implementer)
235 static u32 part = -1;
240 FILE *fp = fopen ("/proc/cpuinfo", "r");
246 if (!fgets (buf, sizeof (buf), fp))
249 if (strstr (buf, "CPU part"))
250 part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
259 #define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43
260 #define AARCH64_CPU_PART_THUNERDERX2 0x0af
261 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
262 #define AARCH64_CPU_PART_QDF24XX 0xc00
263 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
264 #define AARCH64_CPU_PART_CORTEXA72 0xd08
267 clib_cpu_march_priority_thunderx2t99 ()
269 if ((AARCH64_CPU_IMPLEMENTER_THUNERDERX2 == clib_cpu_implementer ()) &&
270 (AARCH64_CPU_PART_THUNERDERX2 == clib_cpu_part ()))
276 clib_cpu_march_priority_qdf24xx ()
278 if ((AARCH64_CPU_IMPLEMENTER_QDF24XX == clib_cpu_implementer ()) &&
279 (AARCH64_CPU_PART_QDF24XX == clib_cpu_part ()))
285 clib_cpu_march_priority_cortexa72 ()
287 if ((AARCH64_CPU_IMPLEMENTER_CORTEXA72 == clib_cpu_implementer ()) &&
288 (AARCH64_CPU_PART_CORTEXA72 == clib_cpu_part ()))
293 #ifdef CLIB_MARCH_VARIANT
294 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
296 #define CLIB_MARCH_FN_PRIORITY() 0
298 #endif /* included_clib_cpu_h */
300 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
301 static void __clib_constructor \
302 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
304 if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
306 fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
307 fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
311 #ifndef CLIB_MARCH_VARIANT
312 #define CLIB_MARCH_FN(fn, rtype, _args...) \
313 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
314 rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
315 int fn ## _selected_priority = 0; \
316 static inline rtype CLIB_CPU_OPTIMIZED \
317 CLIB_MARCH_SFX (fn ## _ma)(_args)
319 #define CLIB_MARCH_FN(fn, rtype, _args...) \
320 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
321 extern int (*fn ## _selected) (_args); \
322 extern int fn ## _selected_priority; \
323 CLIB_MARCH_FN_CONSTRUCTOR (fn) \
324 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
327 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
329 format_function_t format_cpu_uarch;
330 format_function_t format_cpu_model_name;
331 format_function_t format_cpu_flags;
334 * fd.io coding-style-patch-verification: ON
337 * eval: (c-set-style "gnu")