/* Vector types. */
-#if defined (__MMX__) || defined (__IWMMXT__) || defined (__aarch64__)
-#define CLIB_HAVE_VEC64
+#if defined (__aarch64__) && defined(__ARM_NEON) || defined (__i686__)
+#define CLIB_HAVE_VEC128
#endif
-#if defined (__SSE2__) && __GNUC__ >= 4
+#if defined (__SSE4_2__) && __GNUC__ >= 4
#define CLIB_HAVE_VEC128
#endif
#define CLIB_HAVE_VEC128
#endif
-/* 128 implies 64 */
-#ifdef CLIB_HAVE_VEC128
-#define CLIB_HAVE_VEC64
+#if defined (__AVX2__)
+#define CLIB_HAVE_VEC256
+#if defined (__clang__) && __clang_major__ < 4
+#undef CLIB_HAVE_VEC256
+#endif
#endif
-#define _vector_size(n) __attribute__ ((vector_size (n)))
-
-#if defined (__aarch64__) || defined (__arm__)
-typedef unsigned int u32x4 _vector_size (16);
-typedef u8 u8x16 _vector_size (16);
-typedef u16 u16x8 _vector_size (16);
-typedef u32 u32x4 _vector_size (16);
-typedef u64 u64x2 _vector_size (16);
+#if defined (__AVX512BITALG__)
+#define CLIB_HAVE_VEC512
#endif
-#ifdef CLIB_HAVE_VEC64
-/* Signed 64 bit. */
-typedef char i8x8 _vector_size (8);
-typedef short i16x4 _vector_size (8);
-typedef int i32x2 _vector_size (8);
-
-/* Unsigned 64 bit. */
-typedef unsigned char u8x8 _vector_size (8);
-typedef unsigned short u16x4 _vector_size (8);
-typedef unsigned int u32x2 _vector_size (8);
-
-/* Floating point 64 bit. */
-typedef float f32x2 _vector_size (8);
-#endif /* CLIB_HAVE_VEC64 */
-
-#ifdef CLIB_HAVE_VEC128
-/* Signed 128 bit. */
-typedef i8 i8x16 _vector_size (16);
-typedef i16 i16x8 _vector_size (16);
-typedef i32 i32x4 _vector_size (16);
-typedef long long i64x2 _vector_size (16);
-
-/* Unsigned 128 bit. */
-typedef u8 u8x16 _vector_size (16);
-typedef u16 u16x8 _vector_size (16);
-typedef u32 u32x4 _vector_size (16);
-typedef u64 u64x2 _vector_size (16);
-
-typedef f32 f32x4 _vector_size (16);
-typedef f64 f64x2 _vector_size (16);
-
-/* Signed 256 bit. */
-typedef i8 i8x32 _vector_size (32);
-typedef i16 i16x16 _vector_size (32);
-typedef i32 i32x8 _vector_size (32);
-typedef long long i64x4 _vector_size (32);
-
-/* Unsigned 256 bit. */
-typedef u8 u8x32 _vector_size (32);
-typedef u16 u16x16 _vector_size (32);
-typedef u32 u32x8 _vector_size (32);
-typedef u64 u64x4 _vector_size (32);
-
-typedef f32 f32x8 _vector_size (32);
-typedef f64 f64x4 _vector_size (32);
-#endif /* CLIB_HAVE_VEC128 */
-
-/* Vector word sized types. */
-#ifndef CLIB_VECTOR_WORD_BITS
-#ifdef CLIB_HAVE_VEC128
-#define CLIB_VECTOR_WORD_BITS 128
+#define _vector_size(n) __attribute__ ((vector_size (n), __may_alias__))
+#define _vector_size_unaligned(n) \
+ __attribute__ ((vector_size (n), __aligned__ (1), __may_alias__))
+
+#define foreach_vec64i _(i,8,8) _(i,16,4) _(i,32,2)
+#define foreach_vec64u _(u,8,8) _(u,16,4) _(u,32,2)
+#define foreach_vec64f _(f,32,2)
+#define foreach_vec128i _(i,8,16) _(i,16,8) _(i,32,4) _(i,64,2)
+#define foreach_vec128u _(u,8,16) _(u,16,8) _(u,32,4) _(u,64,2)
+#define foreach_vec128f _(f,32,4) _(f,64,2)
+#define foreach_vec256i _(i,8,32) _(i,16,16) _(i,32,8) _(i,64,4)
+#define foreach_vec256u _(u,8,32) _(u,16,16) _(u,32,8) _(u,64,4)
+#define foreach_vec256f _(f,32,8) _(f,64,4)
+#define foreach_vec512i _(i,8,64) _(i,16,32) _(i,32,16) _(i,64,8)
+#define foreach_vec512u _(u,8,64) _(u,16,32) _(u,32,16) _(u,64,8)
+#define foreach_vec512f _(f,32,16) _(f,64,8)
+
+#if defined (CLIB_HAVE_VEC512)
+#define foreach_int_vec foreach_vec64i foreach_vec128i foreach_vec256i foreach_vec512i
+#define foreach_uint_vec foreach_vec64u foreach_vec128u foreach_vec256u foreach_vec512u
+#define foreach_float_vec foreach_vec64f foreach_vec128f foreach_vec256f foreach_vec512f
+#elif defined (CLIB_HAVE_VEC256)
+#define foreach_int_vec foreach_vec64i foreach_vec128i foreach_vec256i
+#define foreach_uint_vec foreach_vec64u foreach_vec128u foreach_vec256u
+#define foreach_float_vec foreach_vec64f foreach_vec128f foreach_vec256f
#else
-#define CLIB_VECTOR_WORD_BITS 64
-#endif
-#endif /* CLIB_VECTOR_WORD_BITS */
-
-/* Vector word sized types. */
-#if CLIB_VECTOR_WORD_BITS == 128
-typedef i8 i8x _vector_size (16);
-typedef i16 i16x _vector_size (16);
-typedef i32 i32x _vector_size (16);
-typedef i64 i64x _vector_size (16);
-typedef u8 u8x _vector_size (16);
-typedef u16 u16x _vector_size (16);
-typedef u32 u32x _vector_size (16);
-typedef u64 u64x _vector_size (16);
-#endif
-#if CLIB_VECTOR_WORD_BITS == 64
-typedef i8 i8x _vector_size (8);
-typedef i16 i16x _vector_size (8);
-typedef i32 i32x _vector_size (8);
-typedef i64 i64x _vector_size (8);
-typedef u8 u8x _vector_size (8);
-typedef u16 u16x _vector_size (8);
-typedef u32 u32x _vector_size (8);
-typedef u64 u64x _vector_size (8);
+#define foreach_int_vec foreach_vec64i foreach_vec128i
+#define foreach_uint_vec foreach_vec64u foreach_vec128u
+#define foreach_float_vec foreach_vec64f foreach_vec128f
#endif
-#undef _vector_size
-
-#define VECTOR_WORD_TYPE(t) t##x
-#define VECTOR_WORD_TYPE_LEN(t) (sizeof (VECTOR_WORD_TYPE(t)) / sizeof (t))
-
-/* Union types. */
-#if (defined(CLIB_HAVE_VEC128) || defined(CLIB_HAVE_VEC64))
-
-#define _(t) \
- typedef union { \
- t##x as_##t##x; \
- t as_##t[VECTOR_WORD_TYPE_LEN (t)]; \
- } t##x##_union_t;
-
-_(u8);
-_(u16);
-_(u32);
-_(u64);
-_(i8);
-_(i16);
-_(i32);
-_(i64);
-
+#define foreach_vec foreach_int_vec foreach_uint_vec foreach_float_vec
+
+/* Type Definitions */
+#define _(t, s, c) \
+ typedef t##s t##s##x##c _vector_size (s / 8 * c); \
+ typedef t##s t##s##x##c##u _vector_size_unaligned (s / 8 * c); \
+ typedef union \
+ { \
+ t##s##x##c as_##t##s##x##c; \
+ t##s as_##t##s[c]; \
+ } t##s##x##c##_union_t;
+
+/* clang-format off */
+ foreach_vec64i foreach_vec64u foreach_vec64f
+ foreach_vec128i foreach_vec128u foreach_vec128f
+ foreach_vec256i foreach_vec256u foreach_vec256f
+ foreach_vec512i foreach_vec512u foreach_vec512f
+/* clang-format on */
#undef _
-#endif
-
-#ifdef CLIB_HAVE_VEC64
-
-#define _(t,n) \
- typedef union { \
- t##x##n as_##t##x##n; \
- t as_##t[n]; \
- } t##x##n##_union_t; \
-
-_(u8, 8);
-_(u16, 4);
-_(u32, 2);
-_(i8, 8);
-_(i16, 4);
-_(i32, 2);
+ typedef union
+{
+#define _(t, s, c) t##s##x##c as_##t##s##x##c;
+ foreach_vec128i foreach_vec128u foreach_vec128f
+#undef _
+} vec128_t;
+typedef union
+{
+#define _(t, s, c) t##s##x##c as_##t##s##x##c;
+ foreach_vec256i foreach_vec256u foreach_vec256f
+#undef _
+#define _(t, s, c) t##s##x##c as_##t##s##x##c[2];
+ foreach_vec128i foreach_vec128u foreach_vec128f
#undef _
+} vec256_t;
-#endif
+typedef union
+{
+#define _(t, s, c) t##s##x##c as_##t##s##x##c;
+ foreach_vec512i foreach_vec512u foreach_vec512f
+#undef _
+#define _(t, s, c) t##s##x##c as_##t##s##x##c[2];
+ foreach_vec256i foreach_vec256u foreach_vec256f
+#undef _
+#define _(t, s, c) t##s##x##c as_##t##s##x##c[4];
+ foreach_vec128i foreach_vec128u foreach_vec128f
+#undef _
+} vec512_t;
-#ifdef CLIB_HAVE_VEC128
-
-#define _(t,n) \
- typedef union { \
- t##x##n as_##t##x##n; \
- t as_##t[n]; \
- } t##x##n##_union_t; \
-
-_(u8, 16);
-_(u16, 8);
-_(u32, 4);
-_(u64, 2);
-_(i8, 16);
-_(i16, 8);
-_(i32, 4);
-_(i64, 2);
-_(f32, 4);
-_(f64, 2);
+/* universal inlines */
+#define _(t, s, c) \
+static_always_inline t##s##x##c \
+t##s##x##c##_zero () \
+{ return (t##s##x##c) {}; } \
+foreach_vec
#undef _
-#endif
+#undef _vector_size
-/* When we don't have vector types, still define e.g. u32x4_union_t but as an array. */
-#if !defined(CLIB_HAVE_VEC128) && !defined(CLIB_HAVE_VEC64)
+ /* _shuffle and _shuffle2 */
+#if defined(__GNUC__) && !defined(__clang__)
+#define __builtin_shufflevector(v1, v2, ...) \
+ __builtin_shuffle ((v1), (v2), (__typeof__ (v1)){ __VA_ARGS__ })
+#endif
-#define _(t,n) \
- typedef union { \
- t as_##t[n]; \
- } t##x##n##_union_t; \
+#define u8x16_shuffle(v1, ...) \
+ (u8x16) __builtin_shufflevector ((u8x16) (v1), (u8x16) (v1), __VA_ARGS__)
+#define u8x32_shuffle(v1, ...) \
+ (u8x32) __builtin_shufflevector ((u8x32) (v1), (u8x32) (v1), __VA_ARGS__)
+#define u8x64_shuffle(v1, ...) \
+ (u8x64) __builtin_shufflevector ((u8x64) (v1), (u8x64) (v1), __VA_ARGS__)
+
+#define u16x8_shuffle(v1, ...) \
+ (u16x8) __builtin_shufflevector ((u16x8) (v1), (u16x8) (v1), __VA_ARGS__)
+#define u16x16_shuffle(v1, ...) \
+ (u16x16) __builtin_shufflevector ((u16x16) (v1), (u16x16) (v1), __VA_ARGS__)
+#define u16x32_shuffle(v1, ...) \
+ (u16u32) __builtin_shufflevector ((u16x32) (v1), (u16x32) (v1), __VA_ARGS__);
+
+#define u32x4_shuffle(v1, ...) \
+ (u32x4) __builtin_shufflevector ((u32x4) (v1), (u32x4) (v1), __VA_ARGS__)
+#define u32x8_shuffle(v1, ...) \
+ (u32x8) __builtin_shufflevector ((u32x8) (v1), (u32x8) (v1), __VA_ARGS__)
+#define u32x16_shuffle(v1, ...) \
+ (u32x16) __builtin_shufflevector ((u32x16) (v1), (u32x16) (v1), __VA_ARGS__)
+
+#define u64x2_shuffle(v1, ...) \
+ (u64x2) __builtin_shufflevector ((u64x2) (v1), (u64x2) (v1), __VA_ARGS__)
+#define u64x4_shuffle(v1, ...) \
+ (u64x4) __builtin_shufflevector ((u64x4) (v1), (u64x4) (v1), __VA_ARGS__)
+#define u64x8_shuffle(v1, ...) \
+ (u64x8) __builtin_shufflevector ((u64x8) (v1), (u64x8) (v1), __VA_ARGS__)
+
+#define u8x16_shuffle2(v1, v2, ...) \
+ (u8x16) __builtin_shufflevector ((u8x16) (v1), (u8x16) (v2), __VA_ARGS__)
+#define u8x32_shuffle2(v1, v2, ...) \
+ (u8x32) __builtin_shufflevector ((u8x32) (v1), (u8x32) (v2), __VA_ARGS__)
+#define u8x64_shuffle2(v1, v2, ...) \
+ (u8x64) __builtin_shufflevector ((u8x64) (v1), (u8x64) (v2), __VA_ARGS__)
+
+#define u16x8_shuffle2(v1, v2, ...) \
+ (u16x8) __builtin_shufflevector ((u16x8) (v1), (u16x8) (v2), __VA_ARGS__)
+#define u16x16_shuffle2(v1, v2, ...) \
+ (u16x16) __builtin_shufflevector ((u16x16) (v1), (u16x16) (v2), __VA_ARGS__)
+#define u16x32_shuffle2(v1, v2, ...) \
+ (u16u32) __builtin_shufflevector ((u16x32) (v1), (u16x32) (v2), __VA_ARGS__);
+
+#define u32x4_shuffle2(v1, v2, ...) \
+ (u32x4) __builtin_shufflevector ((u32x4) (v1), (u32x4) (v2), __VA_ARGS__)
+#define u32x8_shuffle2(v1, v2, ...) \
+ (u32x8) __builtin_shufflevector ((u32x8) (v1), (u32x8) (v2), __VA_ARGS__)
+#define u32x16_shuffle2(v1, v2, ...) \
+ (u32x16) __builtin_shufflevector ((u32x16) (v1), (u32x16) (v2), __VA_ARGS__)
+
+#define u64x2_shuffle2(v1, v2, ...) \
+ (u64x2) __builtin_shufflevector ((u64x2) (v1), (u64x2) (v2), __VA_ARGS__)
+#define u64x4_shuffle2(v1, v2, ...) \
+ (u64x4) __builtin_shufflevector ((u64x4) (v1), (u64x4) (v2), __VA_ARGS__)
+#define u64x8_shuffle2(v1, v2, ...) \
+ (u64x8) __builtin_shufflevector ((u64x8) (v1), (u64x8) (v2), __VA_ARGS__)
-_(u8, 16);
-_(u16, 8);
-_(u32, 4);
-_(u64, 2);
-_(i8, 16);
-_(i16, 8);
-_(i32, 4);
-_(i64, 2);
+#define VECTOR_WORD_TYPE(t) t##x
+#define VECTOR_WORD_TYPE_LEN(t) (sizeof (VECTOR_WORD_TYPE(t)) / sizeof (t))
-#undef _
+#if defined (__SSE4_2__) && __GNUC__ >= 4
+#include <vppinfra/vector_sse42.h>
+#endif
+#if defined (__AVX2__)
+#include <vppinfra/vector_avx2.h>
#endif
-#if defined (__SSE2__) && __GNUC__ >= 4
-#include <vppinfra/vector_sse2.h>
+#if defined(__AVX512F__)
+#include <vppinfra/vector_avx512.h>
#endif
#if defined (__ALTIVEC__)
#include <vppinfra/vector_altivec.h>
#endif
-#if defined (__IWMMXT__)
-#include <vppinfra/vector_iwmmxt.h>
-#endif
-
#if defined (__aarch64__)
#include <vppinfra/vector_neon.h>
#endif
-#if (defined(CLIB_HAVE_VEC128) || defined(CLIB_HAVE_VEC64))
-#include <vppinfra/vector_funcs.h>
+/* this macro generate _splat inline functions for each scalar vector type */
+#ifndef CLIB_VEC128_SPLAT_DEFINED
+#define _(t, s, c) \
+ static_always_inline t##s##x##c \
+t##s##x##c##_splat (t##s x) \
+{ \
+ t##s##x##c r; \
+ int i; \
+ \
+ for (i = 0; i < c; i++) \
+ r[i] = x; \
+ \
+ return r; \
+}
+ foreach_vec128i foreach_vec128u
+#undef _
#endif
#endif /* included_clib_vector_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
Code Review / vpp.git / blobdiff