2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef included_clib_cpu_h
17 #define included_clib_cpu_h
19 #include <sys/syscall.h>
20 #include <vppinfra/format.h>
23 * multiarchitecture support. Adding new entry will produce
24 * new graph node function variant optimized for specific cpu
26 * Order is important for runtime selection, as 1st match wins...
29 #if __x86_64__ && CLIB_DEBUG == 0
30 #define foreach_march_variant(macro, x) \
31 macro(avx2, x, "arch=core-avx2")
33 #define foreach_march_variant(macro, x)
37 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
38 #define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
40 #define CLIB_CPU_OPTIMIZED
44 #define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
45 if (clib_cpu_supports_ ## arch()) \
46 return & fn ## _ ##arch;
48 /* FIXME to be removed */
49 #define CLIB_MULTIARCH_SELECT_FN(fn,...)
51 #ifdef CLIB_MARCH_VARIANT
52 #define __CLIB_MULTIARCH_FN(a,b) a##_##b
53 #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
54 #define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
56 #define CLIB_MULTIARCH_FN(fn) fn
59 #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
61 typedef struct _clib_march_fn_registration
65 struct _clib_march_fn_registration *next;
67 } clib_march_fn_registration;
69 static_always_inline void *
70 clib_march_select_fn_ptr (clib_march_fn_registration * r)
77 if (last_prio < r->priority)
79 last_prio = r->priority;
87 static_always_inline void *
88 clib_march_select_fn_ptr_by_name (clib_march_fn_registration * r, char *name)
94 if (strncmp (name, r->name, vec_len (r->name) - 1) == 0)
104 #define CLIB_MARCH_FN_POINTER(fn) \
105 clib_march_select_fn_ptr (fn##_march_fn_registrations);
107 #define CLIB_MARCH_FN_POINTER_BY_NAME(fn, name) \
108 clib_march_select_fn_ptr_by_name (fn##_march_fn_registrations, name);
110 #define _CLIB_MARCH_FN_REGISTRATION(fn) \
111 static clib_march_fn_registration \
112 CLIB_MARCH_SFX(fn##_march_fn_registration) = \
114 .name = CLIB_MARCH_VARIANT_STR \
117 static void __clib_constructor \
118 fn##_march_register () \
120 clib_march_fn_registration *r; \
121 r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
122 r->priority = CLIB_MARCH_FN_PRIORITY(); \
123 r->next = fn##_march_fn_registrations; \
124 r->function = CLIB_MARCH_SFX (fn); \
125 fn##_march_fn_registrations = r; \
128 #ifdef CLIB_MARCH_VARIANT
129 #define CLIB_MARCH_FN_REGISTRATION(fn) \
130 extern clib_march_fn_registration *fn##_march_fn_registrations; \
131 _CLIB_MARCH_FN_REGISTRATION(fn)
133 #define CLIB_MARCH_FN_REGISTRATION(fn) \
134 clib_march_fn_registration *fn##_march_fn_registrations = 0; \
135 _CLIB_MARCH_FN_REGISTRATION(fn)
137 #define foreach_x86_64_flags \
138 _ (sse3, 1, ecx, 0) \
139 _ (pclmulqdq, 1, ecx, 1) \
140 _ (ssse3, 1, ecx, 9) \
141 _ (sse41, 1, ecx, 19) \
142 _ (sse42, 1, ecx, 20) \
143 _ (avx, 1, ecx, 28) \
144 _ (rdrand, 1, ecx, 30) \
145 _ (avx2, 7, ebx, 5) \
146 _ (rtm, 7, ebx, 11) \
147 _ (pqm, 7, ebx, 12) \
148 _ (pqe, 7, ebx, 15) \
149 _ (avx512f, 7, ebx, 16) \
150 _ (rdseed, 7, ebx, 18) \
151 _ (x86_aes, 1, ecx, 25) \
152 _ (sha, 7, ebx, 29) \
153 _ (vaes, 7, ecx, 9) \
154 _ (vpclmulqdq, 7, ecx, 10) \
155 _ (avx512_vnni, 7, ecx, 11) \
156 _ (avx512_bitalg, 7, ecx, 12) \
157 _ (avx512_vpopcntdq, 7, ecx, 14) \
158 _ (movdiri, 7, ecx, 27) \
159 _ (movdir64b, 7, ecx, 28) \
160 _ (invariant_tsc, 0x80000007, edx, 8)
162 #define foreach_aarch64_flags \
188 clib_get_current_cpu_id ()
191 syscall (__NR_getcpu, &cpu, &node, 0);
196 clib_get_current_numa_node ()
199 syscall (__NR_getcpu, &cpu, &node, 0);
203 #if defined(__x86_64__)
207 clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
209 if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
212 __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
214 __cpuid (lev, *eax, *ebx, *ecx, *edx);
219 #define _(flag, func, reg, bit) \
221 clib_cpu_supports_ ## flag() \
223 u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
224 clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
226 return ((reg & (1 << bit)) != 0); \
230 #else /* __x86_64__ */
232 #define _(flag, func, reg, bit) \
233 static inline int clib_cpu_supports_ ## flag() { return 0; }
236 #endif /* __x86_64__ */
237 #if defined(__aarch64__)
238 #include <sys/auxv.h>
239 #define _(flag, bit) \
241 clib_cpu_supports_ ## flag() \
243 unsigned long hwcap = getauxval(AT_HWCAP); \
244 return (hwcap & (1 << bit)); \
246 foreach_aarch64_flags
248 #else /* ! __x86_64__ && !__aarch64__ */
249 #define _(flag, bit) \
250 static inline int clib_cpu_supports_ ## flag() { return 0; }
251 foreach_aarch64_flags
253 #endif /* __x86_64__, __aarch64__ */
255 * aes is the only feature with the same name in both flag lists
256 * handle this by prefixing it with the arch name, and handling it
257 * with the custom function below
260 clib_cpu_supports_aes ()
262 #if defined(__x86_64__)
263 return clib_cpu_supports_x86_aes ();
264 #elif defined (__aarch64__)
265 return clib_cpu_supports_aarch64_aes ();
272 clib_cpu_march_priority_icl ()
274 if (clib_cpu_supports_avx512_bitalg ())
280 clib_cpu_march_priority_skx ()
282 if (clib_cpu_supports_avx512f ())
288 clib_cpu_march_priority_trm ()
290 if (clib_cpu_supports_movdiri ())
296 clib_cpu_march_priority_hsw ()
298 if (clib_cpu_supports_avx2 ())
304 clib_cpu_implementer ()
307 static u32 implementer = -1;
309 if (-1 != implementer)
312 FILE *fp = fopen ("/proc/cpuinfo", "r");
318 if (!fgets (buf, sizeof (buf), fp))
321 if (strstr (buf, "CPU implementer"))
322 implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
323 if (-1 != implementer)
335 static u32 part = -1;
340 FILE *fp = fopen ("/proc/cpuinfo", "r");
346 if (!fgets (buf, sizeof (buf), fp))
349 if (strstr (buf, "CPU part"))
350 part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
359 #define AARCH64_CPU_IMPLEMENTER_CAVIUM 0x43
360 #define AARCH64_CPU_PART_THUNDERX2 0x0af
361 #define AARCH64_CPU_PART_OCTEONTX2T96 0x0b2
362 #define AARCH64_CPU_PART_OCTEONTX2T98 0x0b1
363 #define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
364 #define AARCH64_CPU_PART_QDF24XX 0xc00
365 #define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
366 #define AARCH64_CPU_PART_CORTEXA72 0xd08
367 #define AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 0x41
368 #define AARCH64_CPU_PART_NEOVERSEN1 0xd0c
371 clib_cpu_march_priority_octeontx2 ()
373 if ((AARCH64_CPU_IMPLEMENTER_CAVIUM == clib_cpu_implementer ()) &&
374 ((AARCH64_CPU_PART_OCTEONTX2T96 == clib_cpu_part ())
375 || AARCH64_CPU_PART_OCTEONTX2T98 == clib_cpu_part ()))
381 clib_cpu_march_priority_thunderx2t99 ()
383 if ((AARCH64_CPU_IMPLEMENTER_CAVIUM == clib_cpu_implementer ()) &&
384 (AARCH64_CPU_PART_THUNDERX2 == clib_cpu_part ()))
390 clib_cpu_march_priority_qdf24xx ()
392 if ((AARCH64_CPU_IMPLEMENTER_QDF24XX == clib_cpu_implementer ()) &&
393 (AARCH64_CPU_PART_QDF24XX == clib_cpu_part ()))
399 clib_cpu_march_priority_cortexa72 ()
401 if ((AARCH64_CPU_IMPLEMENTER_CORTEXA72 == clib_cpu_implementer ()) &&
402 (AARCH64_CPU_PART_CORTEXA72 == clib_cpu_part ()))
408 clib_cpu_march_priority_neoversen1 ()
410 if ((AARCH64_CPU_IMPLEMENTER_NEOVERSEN1 == clib_cpu_implementer ()) &&
411 (AARCH64_CPU_PART_NEOVERSEN1 == clib_cpu_part ()))
416 #ifdef CLIB_MARCH_VARIANT
417 #define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
419 #define CLIB_MARCH_FN_PRIORITY() 0
421 #endif /* included_clib_cpu_h */
423 #define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
424 static void __clib_constructor \
425 CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
427 if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
429 fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
430 fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
434 #ifndef CLIB_MARCH_VARIANT
435 #define CLIB_MARCH_FN(fn, rtype, _args...) \
436 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
437 rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
438 int fn ## _selected_priority = 0; \
439 static inline rtype CLIB_CPU_OPTIMIZED \
440 CLIB_MARCH_SFX (fn ## _ma)(_args)
442 #define CLIB_MARCH_FN(fn, rtype, _args...) \
443 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
444 extern rtype (*fn ## _selected) (_args); \
445 extern int fn ## _selected_priority; \
446 CLIB_MARCH_FN_CONSTRUCTOR (fn) \
447 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
450 #define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
452 format_function_t format_cpu_uarch;
453 format_function_t format_cpu_model_name;
454 format_function_t format_cpu_flags;
457 * fd.io coding-style-patch-verification: ON
460 * eval: (c-set-style "gnu")