From 983cc7da8566fcbe2647ed2d3921e7683bd8d325 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Tue, 18 Sep 2018 23:11:55 -0700 Subject: [PATCH] svm: march svm_fifo take 2 Change-Id: Ifa4fceef7edbe43d444790a624957db0817064de Signed-off-by: Florin Coras --- src/svm/svm_fifo.c | 88 +++++++++++------------------------------------------- src/vppinfra/cpu.h | 29 ++++++++++++++++++ 2 files changed, 46 insertions(+), 71 deletions(-) diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 6a9c0df676f..ac368dddb2c 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -16,41 +16,6 @@ #include #include -#if __x86_64__ && CLIB_DEBUG == 0 -#define foreach_march_variant_runtime(macro, _args...) \ - macro(avx512, avx512f, _args) \ - macro(avx2, avx2, _args) -#else -#define foreach_march_variant_runtime(macro, _args...) -#endif - -#define CLIB_MARCH_ARCH_CHECK(arch, archname, fn) \ - if (clib_cpu_supports_ ## archname ()) \ - return & fn ## _ ##arch; - -#define CLIB_MARCH_SELECT_FN(fn,...) \ - __VA_ARGS__ void * fn ## _multiarch_select(void) \ -{ \ - foreach_march_variant_runtime(CLIB_MARCH_ARCH_CHECK, fn) \ - return & fn; \ -} - -#define DEFINE_WEAK_FN(_arch, _fn, _args...) \ - int __attribute__((weak)) _fn##_ma_##_arch(_args); -#define DEFINE_FN_HELPER(arch, archname, macro, _args...) \ - macro(arch, _args) - -#ifndef CLIB_MARCH_VARIANT -#define MARCH_FN(fn, _args...) \ - static void * (*fn ## _selected) (void); \ - foreach_march_variant_runtime (DEFINE_FN_HELPER, DEFINE_WEAK_FN, fn, \ - _args); \ - static inline int CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) -#else -#define MARCH_FN(fn, _args...) \ - int CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) -#endif - static inline u8 position_lt (svm_fifo_t * f, u32 a, u32 b) { @@ -489,8 +454,8 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued) return bytes; } -MARCH_FN (svm_fifo_enqueue_nowait, svm_fifo_t * f, u32 max_bytes, - const u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes, + const u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -557,8 +522,8 @@ int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, const u8 * copy_from_here) { - return ((int (*)(svm_fifo_t * f, u32, const u8 *)) - (*svm_fifo_enqueue_nowait_selected)) (f, max_bytes, copy_from_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes, + copy_from_here); } #endif @@ -569,8 +534,8 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, * Returns 0 of the entire segment was copied * Returns -1 if none of the segment was copied due to lack of space */ -MARCH_FN (svm_fifo_enqueue_with_offset, svm_fifo_t * f, - u32 offset, u32 required_bytes, u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f, + u32 offset, u32 required_bytes, u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, normalized_offset; @@ -624,10 +589,9 @@ int svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes, u8 * copy_from_here) { - return ((int (*)(svm_fifo_t * f, u32, u32, u8 *)) - (*svm_fifo_enqueue_with_offset_selected)) (f, offset, - required_bytes, - copy_from_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset, + required_bytes, + copy_from_here); } void @@ -646,8 +610,8 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len) } #endif -MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes, - u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes, + u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -704,13 +668,13 @@ MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes, int svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) { - return ((int (*)(svm_fifo_t * f, u32, u8 *)) - (*svm_fifo_dequeue_nowait_selected)) (f, max_bytes, copy_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes, + copy_here); } #endif -MARCH_FN (svm_fifo_peek, svm_fifo_t * f, u32 relative_offset, u32 max_bytes, - u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset, + u32 max_bytes, u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, real_head; @@ -753,9 +717,8 @@ int svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, u8 * copy_here) { - return ((int (*)(svm_fifo_t * f, u32, u32, u8 *)) - (*svm_fifo_peek_selected)) (f, relative_offset, max_bytes, - copy_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes, + copy_here); } int @@ -873,23 +836,6 @@ svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer) f->head = f->tail = pointer % f->nitems; } -#define foreach_svm_fifo_march_fn \ - _(svm_fifo_enqueue_nowait) \ - _(svm_fifo_enqueue_with_offset) \ - _(svm_fifo_dequeue_nowait) \ - _(svm_fifo_peek) \ - -#define _(_fn, _args...) CLIB_MARCH_SELECT_FN(_fn ## _ma); -foreach_svm_fifo_march_fn -#undef _ -void __clib_constructor -svm_fifo_select_march_fns (void) -{ -#define _(_fn, _args...) _fn ## _selected = _fn ## _ma_multiarch_select (); - foreach_svm_fifo_march_fn -#undef _ -} - #endif /* * fd.io coding-style-patch-verification: ON diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index 110815c9f87..c636cf8639c 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -190,6 +190,35 @@ clib_cpu_march_priority_avx2 () #endif #endif /* included_clib_cpu_h */ +#define CLIB_MARCH_FN_CONSTRUCTOR(fn) \ +static void __clib_constructor \ +CLIB_MARCH_SFX(fn ## _march_constructor) (void) \ +{ \ + if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \ + { \ + fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \ + fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \ + } \ +} \ + +#ifndef CLIB_MARCH_VARIANT +#define CLIB_MARCH_FN(fn, rtype, _args...) \ + static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ + rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \ + int fn ## _selected_priority = 0; \ + static inline rtype CLIB_CPU_OPTIMIZED \ + CLIB_MARCH_SFX (fn ## _ma)(_args) +#else +#define CLIB_MARCH_FN(fn, rtype, _args...) \ + static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \ + extern int (*fn ## _selected) (_args); \ + extern int fn ## _selected_priority; \ + CLIB_MARCH_FN_CONSTRUCTOR (fn) \ + static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) +#endif + +#define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected) + format_function_t format_cpu_uarch; format_function_t format_cpu_model_name; format_function_t format_cpu_flags; -- 2.16.6