Optimize xxx_zero_byte_mask NEON function

Optimize zero byte mask NEON functions below with less intrinsics,
and get their outputs consistent with functions in vector_sse42.h

always_inline u32 u64x2_zero_byte_mask (u64x2 input)
always_inline u32 u32x4_zero_byte_mask (u32x4 input)
always_inline u32 u16x8_zero_byte_mask (u16x8 input)
always_inline u32 u8x16_zero_byte_mask (u8x16 input)
always_inline u32 i64x2_zero_byte_mask (i64x2 input)
always_inline u32 i32x4_zero_byte_mask (i32x4 input)
always_inline u32 i16x8_zero_byte_mask (i16x8 input)
always_inline u32 i8x16_zero_byte_mask (i8x16 input)

Change-Id: I7f485915baeb37fa2dd484699b8769e0136f6574
Signed-off-by: Lijian Zhang <Lijian.Zhang@arm.com>
Reviewed-by: Sirshak Das <Sirshak.Das@arm.com>

diff --git a/src/vppinfra/vector_neon.h b/src/vppinfra/vector_neon.h
index 3ac41bb..aef8365 100644 (file)
--- a/src/vppinfra/vector_neon.h
+++ b/src/vppinfra/vector_neon.h
@@ -20,6 +20,8 @@
 /* Arithmetic */
 #define u16x8_sub_saturate(a,b) vsubq_u16(a,b)
 #define i16x8_sub_saturate(a,b) vsubq_s16(a,b)
+/* Dummy. Aid making uniform macros */
+#define vreinterpretq_u8_u8(a)  a
 
 /* Converts all ones/zeros compare mask to bitmap. */
 always_inline u32
@@ -37,50 +39,6 @@ u8x16_compare_byte_mask (u8x16 v)
   return (u32) (vgetq_lane_u64 (x64, 0) + (vgetq_lane_u64 (x64, 1) << 8));
 }
 
-always_inline u32
-u16x8_zero_byte_mask (u16x8 input)
-{
-  u8x16 vall_one = vdupq_n_u8 (0x0);
-  u8x16 res_values = { 0x01, 0x02, 0x04, 0x08,
-    0x10, 0x20, 0x40, 0x80,
-    0x01, 0x02, 0x04, 0x08,
-    0x10, 0x20, 0x40, 0x80
-  };
-
-  /* input --> [0x80, 0x40, 0x01, 0xf0, ... ] */
-  u8x16 test_result =
-    vreinterpretq_u8_u16 (vceqq_u16 (input, vreinterpretq_u16_u8 (vall_one)));
-  u8x16 before_merge = vminq_u8 (test_result, res_values);
-  /*before_merge--> [0x80, 0x00, 0x00, 0x10, ... ] */
-  /* u8x16 --> [a,b,c,d, e,f,g,h, i,j,k,l, m,n,o,p] */
-  /* pair add until we have 2 uint64_t  */
-  u16x8 merge1 = vpaddlq_u8 (before_merge);
-  /* u16x8-->  [a+b,c+d, e+f,g+h, i+j,k+l, m+n,o+p] */
-  u32x4 merge2 = vpaddlq_u16 (merge1);
-  /* u32x4-->  [a+b+c+d, e+f+g+h, i+j+k+l, m+n+o+p] */
-  u64x2 merge3 = vpaddlq_u32 (merge2);
-  /* u64x2-->  [a+b+c+d+e+f+g+h,  i+j+k+l+m+n+o+p]  */
-  return (u32) (vgetq_lane_u64 (merge3, 1) << 8) + vgetq_lane_u64 (merge3, 0);
-}
-
-always_inline u32
-u8x16_zero_byte_mask (u8x16 input)
-{
-  return u16x8_zero_byte_mask ((u16x8) input);
-}
-
-always_inline u32
-u32x4_zero_byte_mask (u32x4 input)
-{
-  return u16x8_zero_byte_mask ((u16x8) input);
-}
-
-always_inline u32
-u64x2_zero_byte_mask (u64x2 input)
-{
-  return u16x8_zero_byte_mask ((u16x8) input);
-}
-
 /* *INDENT-OFF* */
 #define foreach_neon_vec128i \
   _(i,8,16,s8) _(i,16,8,s16) _(i,32,4,s32)  _(i,64,2,s64)
@@ -113,6 +71,11 @@ t##s##x##c##_is_equal (t##s##x##c a, t##s##x##c b)                  \
 static_always_inline int                                               \
 t##s##x##c##_is_all_equal (t##s##x##c v, t##s x)                       \
 { return t##s##x##c##_is_equal (v, t##s##x##c##_splat (x)); };         \
+\
+static_always_inline u32                                               \
+t##s##x##c##_zero_byte_mask (t##s##x##c x)                     \
+{ uint8x16_t v = vreinterpretq_u8_u##s (vceqq_##i (vdupq_n_##i(0), x));  \
+  return u8x16_compare_byte_mask (v); } \
 
 foreach_neon_vec128i foreach_neon_vec128u