2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
19 #include <sys/syscall.h>
20 #include <vppinfra/format.h>
23 * multiarchitecture support. Adding new entry will produce
24 * new graph node function variant optimized for specific cpu
26 * Order is important for runtime selection, as 1st match wins...
29 #if __x86_64__ && CLIB_DEBUG == 0
30 #define foreach_march_variant(macro, x) \
31 macro(avx2, x, "arch=core-avx2")
33 #define foreach_march_variant(macro, x)
37 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
38 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
40 #define CLIB_CPU_OPTIMIZED
44 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
45 if (clib_cpu_supports_ ## arch()) \
46 return & fn ## _ ##arch;
48 #define CLIB_MULTIARCH_SELECT_FN(fn,...) \
49 __VA_ARGS__ void * fn ## _multiarch_select(void) \
51 foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \
55 #ifdef CLIB_MARCH_VARIANT
56 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
57 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
58 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
60 #define CLIB_MULTIARCH_FN(fn) fn
63 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
65 typedef struct _clib_march_fn_registration
69 struct _clib_march_fn_registration *next;
71 } clib_march_fn_registration;
73 static_always_inline void *
74 clib_march_select_fn_ptr (clib_march_fn_registration * r)
81 if (last_prio < r->priority)
83 last_prio = r->priority;
91 #define CLIB_MARCH_FN_POINTER(fn) \
92 clib_march_select_fn_ptr (fn##_march_fn_registrations);
94 #define _CLIB_MARCH_FN_REGISTRATION(fn) \
95 static clib_march_fn_registration \
96 CLIB_MARCH_SFX(fn##_march_fn_registration) = \
98 .name = CLIB_MARCH_VARIANT_STR \
101 static void __clib_constructor \
102 fn##_march_register () \
104 clib_march_fn_registration *r; \
105 r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
106 r->priority = CLIB_MARCH_FN_PRIORITY(); \
107 r->next = fn##_march_fn_registrations; \
108 r->function = CLIB_MARCH_SFX (fn); \
109 fn##_march_fn_registrations = r; \
112 #ifdef CLIB_MARCH_VARIANT
113 #define CLIB_MARCH_FN_REGISTRATION(fn) \
114 extern clib_march_fn_registration *fn##_march_fn_registrations; \
115 _CLIB_MARCH_FN_REGISTRATION(fn)
117 #define CLIB_MARCH_FN_REGISTRATION(fn) \
118 clib_march_fn_registration *fn##_march_fn_registrations = 0; \
119 _CLIB_MARCH_FN_REGISTRATION(fn)
121 #define foreach_x86_64_flags \
122 _ (sse3, 1, ecx, 0) \
123 _ (ssse3, 1, ecx, 9) \
124 _ (sse41, 1, ecx, 19) \
125 _ (sse42, 1, ecx, 20) \
126 _ (avx, 1, ecx, 28) \
127 _ (avx2, 7, ebx, 5) \
128 _ (avx512f, 7, ebx, 16) \
129 _ (x86_aes, 1, ecx, 25) \
130 _ (sha, 7, ebx, 29) \
131 _ (invariant_tsc, 0x80000007, edx, 8)
134 #define foreach_aarch64_flags \
160 clib_get_current_cpu_id ()
163 syscall (__NR_getcpu, &cpu, &node, 0);
168 clib_get_current_numa_node ()
171 syscall (__NR_getcpu, &cpu, &node, 0);
175 #if defined(__x86_64__)
179 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
181 if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
184 __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
186 __cpuid (lev, *eax, *ebx, *ecx, *edx);
191 #define _(flag, func, reg, bit) \
193 clib_cpu_supports_ ## flag() \
195 u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
196 clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
198 return ((reg & (1 << bit)) != 0); \
202 #else /* __x86_64__ */
204 #define _(flag, func, reg, bit) \
205 static inline int clib_cpu_supports_ ## flag() { return 0; }
208 #endif /* __x86_64__ */
209 #if defined(__aarch64__)
210 #include <sys/auxv.h>
211 #define _(flag, bit) \
213 clib_cpu_supports_ ## flag() \
215 unsigned long hwcap = getauxval(AT_HWCAP); \
216 return (hwcap & (1 << bit)); \
218 foreach_aarch64_flags
220 #else /* ! __x86_64__ && !__aarch64__ */
221 #define _(flag, bit) \
222 static inline int clib_cpu_supports_ ## flag() { return 0; }
223 foreach_aarch64_flags
225 #endif /* __x86_64__, __aarch64__ */
227 * aes is the only feature with the same name in both flag lists
228 * handle this by prefixing it with the arch name, and handling it
229 * with the custom function below
232 clib_cpu_supports_aes ()
234 #if defined (__aarch64__)
235 return clib_cpu_supports_x86_aes ();
236 #elif defined (__aarch64__)
237 return clib_cpu_supports_aarch64_aes ();
244 clib_cpu_march_priority_avx512 ()
246 if (clib_cpu_supports_avx512f ())
252 clib_cpu_march_priority_avx2 ()
254 if (clib_cpu_supports_avx2 ())
260 clib_cpu_implementer ()
263 static u32 implementer = -1;
265 if (-1 != implementer)
268 FILE *fp = fopen ("/proc/cpuinfo", "r");
274 if (!fgets (buf, sizeof (buf), fp))
277 if (strstr (buf, "CPU implementer"))
278 implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
279 if (-1 != implementer)
291 static u32 part = -1;
296 FILE *fp = fopen ("/proc/cpuinfo", "r");
302 if (!fgets (buf, sizeof (buf), fp))
305 if (strstr (buf, "CPU part"))
306 part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
315 #define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43
316 #define AARCH64_CPU_PART_THUNERDERX2 0x0af
317 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
318 #define AARCH64_CPU_PART_QDF24XX 0xc00
319 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
320 #define AARCH64_CPU_PART_CORTEXA72 0xd08
323 clib_cpu_march_priority_thunderx2t99 ()
325 if ((AARCH64_CPU_IMPLEMENTER_THUNERDERX2 == clib_cpu_implementer ()) &&
326 (AARCH64_CPU_PART_THUNERDERX2 == clib_cpu_part ()))
332 clib_cpu_march_priority_qdf24xx ()
334 if ((AARCH64_CPU_IMPLEMENTER_QDF24XX == clib_cpu_implementer ()) &&
335 (AARCH64_CPU_PART_QDF24XX == clib_cpu_part ()))
341 clib_cpu_march_priority_cortexa72 ()
343 if ((AARCH64_CPU_IMPLEMENTER_CORTEXA72 == clib_cpu_implementer ()) &&
344 (AARCH64_CPU_PART_CORTEXA72 == clib_cpu_part ()))
349 #ifdef CLIB_MARCH_VARIANT
350 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
352 #define CLIB_MARCH_FN_PRIORITY() 0
354 #endif /* included_clib_cpu_h */
356 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
357 static void __clib_constructor \
358 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
360 if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
362 fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
363 fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
367 #ifndef CLIB_MARCH_VARIANT
368 #define CLIB_MARCH_FN(fn, rtype, _args...) \
369 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
370 rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
371 int fn ## _selected_priority = 0; \
372 static inline rtype CLIB_CPU_OPTIMIZED \
373 CLIB_MARCH_SFX (fn ## _ma)(_args)
375 #define CLIB_MARCH_FN(fn, rtype, _args...) \
376 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
377 extern int (*fn ## _selected) (_args); \
378 extern int fn ## _selected_priority; \
379 CLIB_MARCH_FN_CONSTRUCTOR (fn) \
380 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
383 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
385 format_function_t format_cpu_uarch;
386 format_function_t format_cpu_model_name;
387 format_function_t format_cpu_flags;
390 * fd.io coding-style-patch-verification: ON
393 * eval: (c-set-style "gnu")