1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright(c) 2021 Cisco Systems, Inc.
5 #ifndef included_vector_count_equal_h
6 #define included_vector_count_equal_h
7 #include <vppinfra/clib.h>
9 static_always_inline uword
10 clib_count_equal_u64 (u64 *data, uword max_count)
17 if (data[0] != data[1])
23 #if defined(CLIB_HAVE_VEC256)
24 u64x4 splat = u64x4_splat (first);
25 while (count + 3 < max_count)
28 bmp = u8x32_msb_mask ((u8x32) (u64x4_load_unaligned (data) == splat));
29 if (bmp != 0xffffffff)
31 count += count_trailing_zeros (~bmp) / 8;
41 while (count + 3 < max_count && ((data[0] ^ first) | (data[1] ^ first) |
42 (data[2] ^ first) | (data[3] ^ first)) == 0)
48 while (count < max_count && (data[0] == first))
56 static_always_inline uword
57 clib_count_equal_u32 (u32 *data, uword max_count)
64 if (data[0] != data[1])
70 #if defined(CLIB_HAVE_VEC256)
71 u32x8 splat = u32x8_splat (first);
72 while (count + 7 < max_count)
75 bmp = u8x32_msb_mask ((u8x32) (u32x8_load_unaligned (data) == splat));
76 if (bmp != 0xffffffff)
78 count += count_trailing_zeros (~bmp) / 4;
85 #elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK)
86 u32x4 splat = u32x4_splat (first);
87 while (count + 3 < max_count)
90 bmp = u8x16_msb_mask ((u8x16) (u32x4_load_unaligned (data) == splat));
93 count += count_trailing_zeros (~bmp) / 4;
103 while (count + 3 < max_count && ((data[0] ^ first) | (data[1] ^ first) |
104 (data[2] ^ first) | (data[3] ^ first)) == 0)
110 while (count < max_count && (data[0] == first))
118 static_always_inline uword
119 clib_count_equal_u16 (u16 *data, uword max_count)
126 if (data[0] != data[1])
132 #if defined(CLIB_HAVE_VEC256)
133 u16x16 splat = u16x16_splat (first);
134 while (count + 15 < max_count)
137 bmp = u8x32_msb_mask ((u8x32) (u16x16_load_unaligned (data) == splat));
138 if (bmp != 0xffffffff)
140 count += count_trailing_zeros (~bmp) / 2;
147 #elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK)
148 u16x8 splat = u16x8_splat (first);
149 while (count + 7 < max_count)
152 bmp = u8x16_msb_mask ((u8x16) (u16x8_load_unaligned (data) == splat));
155 count += count_trailing_zeros (~bmp) / 2;
165 while (count + 3 < max_count && ((data[0] ^ first) | (data[1] ^ first) |
166 (data[2] ^ first) | (data[3] ^ first)) == 0)
172 while (count < max_count && (data[0] == first))
180 static_always_inline uword
181 clib_count_equal_u8 (u8 *data, uword max_count)
188 if (data[0] != data[1])
194 #if defined(CLIB_HAVE_VEC256)
195 u8x32 splat = u8x32_splat (first);
196 while (count + 31 < max_count)
199 bmp = u8x32_msb_mask ((u8x32) (u8x32_load_unaligned (data) == splat));
200 if (bmp != 0xffffffff)
206 #elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK)
207 u8x16 splat = u8x16_splat (first);
208 while (count + 15 < max_count)
211 bmp = u8x16_msb_mask ((u8x16) (u8x16_load_unaligned (data) == splat));
214 count += count_trailing_zeros (~bmp);
224 while (count + 3 < max_count && ((data[0] ^ first) | (data[1] ^ first) |
225 (data[2] ^ first) | (data[3] ^ first)) == 0)
231 while (count < max_count && (data[0] == first))