X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fvector.h;h=2b84cc24869b41a3a7b536d04d14f26ec4224b00;hb=bb5d22d;hp=491e7cfe5476beebf5de2e36f5f9776d554f9004;hpb=7cd468a3d7dee7d6c92f69a0bb7061ae208ec727;p=vpp.git diff --git a/src/vppinfra/vector.h b/src/vppinfra/vector.h index 491e7cfe547..2b84cc24869 100644 --- a/src/vppinfra/vector.h +++ b/src/vppinfra/vector.h @@ -42,11 +42,16 @@ /* Vector types. */ -#if defined (__MMX__) || defined (__IWMMXT__) || defined (__aarch64__) +#if defined (__MMX__) || defined (__IWMMXT__) || defined (__aarch64__) \ + || defined (__i686__) #define CLIB_HAVE_VEC64 #endif -#if defined (__SSE2__) && __GNUC__ >= 4 +#if defined (__aarch64__) && defined(__ARM_NEON) || defined (__i686__) +#define CLIB_HAVE_VEC128 +#endif + +#if defined (__SSE4_2__) && __GNUC__ >= 4 #define CLIB_HAVE_VEC128 #endif @@ -54,6 +59,17 @@ #define CLIB_HAVE_VEC128 #endif +#if defined (__AVX2__) +#define CLIB_HAVE_VEC256 +#if defined (__clang__) && __clang_major__ < 4 +#undef CLIB_HAVE_VEC256 +#endif +#endif + +#if defined (__AVX512F__) +#define CLIB_HAVE_VEC512 +#endif + /* 128 implies 64 */ #ifdef CLIB_HAVE_VEC128 #define CLIB_HAVE_VEC64 @@ -61,60 +77,50 @@ #define _vector_size(n) __attribute__ ((vector_size (n))) -#if defined (__aarch64__) || defined (__arm__) -typedef unsigned int u32x4 _vector_size (16); -typedef u8 u8x16 _vector_size (16); -typedef u16 u16x8 _vector_size (16); -typedef u32 u32x4 _vector_size (16); -typedef u64 u64x2 _vector_size (16); +#define foreach_vec64i _(i,8,8) _(i,16,4) _(i,32,2) +#define foreach_vec64u _(u,8,8) _(u,16,4) _(u,32,2) +#define foreach_vec64f _(f,32,2) +#define foreach_vec128i _(i,8,16) _(i,16,8) _(i,32,4) _(i,64,2) +#define foreach_vec128u _(u,8,16) _(u,16,8) _(u,32,4) _(u,64,2) +#define foreach_vec128f _(f,32,4) _(f,64,2) +#define foreach_vec256i _(i,8,32) _(i,16,16) _(i,32,8) _(i,64,4) +#define foreach_vec256u _(u,8,32) _(u,16,16) _(u,32,8) _(u,64,4) +#define foreach_vec256f _(f,32,8) _(f,64,4) +#define foreach_vec512i _(i,8,64) _(i,16,32) _(i,32,16) _(i,64,8) +#define foreach_vec512u _(u,8,64) _(u,16,32) _(u,32,16) _(u,64,8) +#define foreach_vec512f _(f,32,16) _(f,64,8) + +#if defined (CLIB_HAVE_VEC512) +#define foreach_int_vec foreach_vec64i foreach_vec128i foreach_vec256i foreach_vec512i +#define foreach_uint_vec foreach_vec64u foreach_vec128u foreach_vec256u foreach_vec512u +#define foreach_float_vec foreach_vec64f foreach_vec128f foreach_vec256f foreach_vec512f +#elif defined (CLIB_HAVE_VEC256) +#define foreach_int_vec foreach_vec64i foreach_vec128i foreach_vec256i +#define foreach_uint_vec foreach_vec64u foreach_vec128u foreach_vec256u +#define foreach_float_vec foreach_vec64f foreach_vec128f foreach_vec256f +#else +#define foreach_int_vec foreach_vec64i foreach_vec128i +#define foreach_uint_vec foreach_vec64u foreach_vec128u +#define foreach_float_vec foreach_vec64f foreach_vec128f #endif -#ifdef CLIB_HAVE_VEC64 -/* Signed 64 bit. */ -typedef char i8x8 _vector_size (8); -typedef short i16x4 _vector_size (8); -typedef int i32x2 _vector_size (8); +#define foreach_vec foreach_int_vec foreach_uint_vec foreach_float_vec -/* Unsigned 64 bit. */ -typedef unsigned char u8x8 _vector_size (8); -typedef unsigned short u16x4 _vector_size (8); -typedef unsigned int u32x2 _vector_size (8); +/* *INDENT-OFF* */ -/* Floating point 64 bit. */ -typedef float f32x2 _vector_size (8); -#endif /* CLIB_HAVE_VEC64 */ +/* Type Definitions */ +#define _(t,s,c) \ +typedef t##s t##s##x##c _vector_size (s/8*c); \ +typedef union { \ + t##s##x##c as_##t##s##x##c; \ + t##s as_##t##s[c]; \ +} t##s##x##c##_union_t; -#ifdef CLIB_HAVE_VEC128 -/* Signed 128 bit. */ -typedef i8 i8x16 _vector_size (16); -typedef i16 i16x8 _vector_size (16); -typedef i32 i32x4 _vector_size (16); -typedef long long i64x2 _vector_size (16); - -/* Unsigned 128 bit. */ -typedef u8 u8x16 _vector_size (16); -typedef u16 u16x8 _vector_size (16); -typedef u32 u32x4 _vector_size (16); -typedef u64 u64x2 _vector_size (16); - -typedef f32 f32x4 _vector_size (16); -typedef f64 f64x2 _vector_size (16); - -/* Signed 256 bit. */ -typedef i8 i8x32 _vector_size (32); -typedef i16 i16x16 _vector_size (32); -typedef i32 i32x8 _vector_size (32); -typedef long long i64x4 _vector_size (32); - -/* Unsigned 256 bit. */ -typedef u8 u8x32 _vector_size (32); -typedef u16 u16x16 _vector_size (32); -typedef u32 u32x8 _vector_size (32); -typedef u64 u64x4 _vector_size (32); - -typedef f32 f32x8 _vector_size (32); -typedef f64 f64x4 _vector_size (32); -#endif /* CLIB_HAVE_VEC128 */ + foreach_vec64i foreach_vec64u foreach_vec64f + foreach_vec128i foreach_vec128u foreach_vec128f + foreach_vec256i foreach_vec256u foreach_vec256f + foreach_vec512i foreach_vec512u foreach_vec512f +#undef _ /* Vector word sized types. */ #ifndef CLIB_VECTOR_WORD_BITS @@ -152,103 +158,22 @@ typedef u64 u64x _vector_size (8); #define VECTOR_WORD_TYPE(t) t##x #define VECTOR_WORD_TYPE_LEN(t) (sizeof (VECTOR_WORD_TYPE(t)) / sizeof (t)) -/* Union types. */ -#if (defined(CLIB_HAVE_VEC128) || defined(CLIB_HAVE_VEC64)) - -#define _(t) \ - typedef union { \ - t##x as_##t##x; \ - t as_##t[VECTOR_WORD_TYPE_LEN (t)]; \ - } t##x##_union_t; - -_(u8); -_(u16); -_(u32); -_(u64); -_(i8); -_(i16); -_(i32); -_(i64); - -#undef _ - -#endif - -#ifdef CLIB_HAVE_VEC64 - -#define _(t,n) \ - typedef union { \ - t##x##n as_##t##x##n; \ - t as_##t[n]; \ - } t##x##n##_union_t; \ - -_(u8, 8); -_(u16, 4); -_(u32, 2); -_(i8, 8); -_(i16, 4); -_(i32, 2); - -#undef _ - +#if defined (__SSE4_2__) && __GNUC__ >= 4 +#include #endif -#ifdef CLIB_HAVE_VEC128 - -#define _(t,n) \ - typedef union { \ - t##x##n as_##t##x##n; \ - t as_##t[n]; \ - } t##x##n##_union_t; \ - -_(u8, 16); -_(u16, 8); -_(u32, 4); -_(u64, 2); -_(i8, 16); -_(i16, 8); -_(i32, 4); -_(i64, 2); -_(f32, 4); -_(f64, 2); - -#undef _ - +#if defined (__AVX2__) +#include #endif -/* When we don't have vector types, still define e.g. u32x4_union_t but as an array. */ -#if !defined(CLIB_HAVE_VEC128) && !defined(CLIB_HAVE_VEC64) - -#define _(t,n) \ - typedef union { \ - t as_##t[n]; \ - } t##x##n##_union_t; \ - -_(u8, 16); -_(u16, 8); -_(u32, 4); -_(u64, 2); -_(i8, 16); -_(i16, 8); -_(i32, 4); -_(i64, 2); - -#undef _ - -#endif - -#if defined (__SSE2__) && __GNUC__ >= 4 -#include +#if defined (__AVX512F__) +#include #endif #if defined (__ALTIVEC__) #include #endif -#if defined (__IWMMXT__) -#include -#endif - #if defined (__aarch64__) #include #endif @@ -257,8 +182,27 @@ _(i64, 2); #include #endif -#endif /* included_clib_vector_h */ +/* this macro generate _splat inline functions for each scalar vector type */ +#ifndef CLIB_VEC128_SPLAT_DEFINED +#define _(t, s, c) \ + static_always_inline t##s##x##c \ +t##s##x##c##_splat (t##s x) \ +{ \ + t##s##x##c r; \ + int i; \ + \ + for (i = 0; i < c; i++) \ + r[i] = x; \ + \ + return r; \ +} + foreach_vec128i foreach_vec128u +#undef _ +#endif +/* *INDENT-ON* */ + +#endif /* included_clib_vector_h */ /* * fd.io coding-style-patch-verification: ON *