1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright(c) 2021 Cisco Systems, Inc.
5 #ifndef included_vector_compress_h
6 #define included_vector_compress_h
7 #include <vppinfra/clib.h>
8 #include <vppinfra/memcpy.h>
10 static_always_inline u64 *
11 clib_compress_u64_x64 (u64 *dst, u64 *src, u64 mask)
13 #if defined(CLIB_HAVE_VEC512_COMPRESS)
14 u64x8u *sv = (u64x8u *) src;
15 for (int i = 0; i < 8; i++)
17 u64x8_compress_store (sv[i], mask, dst);
18 dst += _popcnt32 ((u8) mask);
21 #elif defined(CLIB_HAVE_VEC256_COMPRESS)
22 u64x4u *sv = (u64x4u *) src;
23 for (int i = 0; i < 16; i++)
25 u64x4_compress_store (sv[i], mask, dst);
26 dst += _popcnt32 (((u8) mask) & 0x0f);
31 foreach_set_bit_index (i, mask)
37 /** \brief Compress array of 64-bit elemments into destination array based on
40 @param dst destination array of u64 elements
41 @param src source array of u64 elements
42 @param mask array of u64 values representing compress mask
43 @param n_elts number of elements in the source array
44 @return number of elements stored in destionation array
47 static_always_inline u32
48 clib_compress_u64 (u64 *dst, u64 *src, u64 *mask, u32 n_elts)
55 clib_memcpy_fast (dst, src, 64 * sizeof (u64));
59 dst = clib_compress_u64_x64 (dst, src, mask[0]);
66 if (PREDICT_TRUE (n_elts == 0))
69 return clib_compress_u64_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0;
72 static_always_inline u32 *
73 clib_compress_u32_x64 (u32 *dst, u32 *src, u64 mask)
75 #if defined(CLIB_HAVE_VEC512_COMPRESS)
76 u32x16u *sv = (u32x16u *) src;
77 for (int i = 0; i < 4; i++)
79 u32x16_compress_store (sv[i], mask, dst);
80 dst += _popcnt32 ((u16) mask);
84 #elif defined(CLIB_HAVE_VEC256_COMPRESS)
85 u32x8u *sv = (u32x8u *) src;
86 for (int i = 0; i < 8; i++)
88 u32x8_compress_store (sv[i], mask, dst);
89 dst += _popcnt32 ((u8) mask);
94 foreach_set_bit_index (i, mask)
100 /** \brief Compress array of 32-bit elemments into destination array based on
103 @param dst destination array of u32 elements
104 @param src source array of u32 elements
105 @param mask array of u64 values representing compress mask
106 @param n_elts number of elements in the source array
107 @return number of elements stored in destionation array
110 static_always_inline u32
111 clib_compress_u32 (u32 *dst, u32 *src, u64 *mask, u32 n_elts)
116 if (mask[0] == ~0ULL)
118 clib_memcpy_u32 (dst, src, 64);
122 dst = clib_compress_u32_x64 (dst, src, mask[0]);
129 if (PREDICT_TRUE (n_elts == 0))
132 return clib_compress_u32_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0;
135 static_always_inline u16 *
136 clib_compress_u16_x64 (u16 *dst, u16 *src, u64 mask)
138 #if defined(CLIB_HAVE_VEC512_COMPRESS_U8_U16)
139 u16x32u *sv = (u16x32u *) src;
140 for (int i = 0; i < 2; i++)
142 u16x32_compress_store (sv[i], mask, dst);
143 dst += _popcnt32 ((u32) mask);
148 foreach_set_bit_index (i, mask)
154 /** \brief Compress array of 16-bit elemments into destination array based on
157 @param dst destination array of u16 elements
158 @param src source array of u16 elements
159 @param mask array of u64 values representing compress mask
160 @param n_elts number of elements in the source array
161 @return number of elements stored in destionation array
164 static_always_inline u32
165 clib_compress_u16 (u16 *dst, u16 *src, u64 *mask, u32 n_elts)
170 if (mask[0] == ~0ULL)
172 clib_memcpy_fast (dst, src, 64 * sizeof (u16));
176 dst = clib_compress_u16_x64 (dst, src, mask[0]);
183 if (PREDICT_TRUE (n_elts == 0))
186 return clib_compress_u16_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0;
189 static_always_inline u8 *
190 clib_compress_u8_x64 (u8 *dst, u8 *src, u64 mask)
192 #if defined(CLIB_HAVE_VEC512_COMPRESS_U8_U16)
193 u8x64u *sv = (u8x64u *) src;
194 u8x64_compress_store (sv[0], mask, dst);
195 dst += _popcnt64 (mask);
198 foreach_set_bit_index (i, mask)
204 /** \brief Compress array of 8-bit elemments into destination array based on
207 @param dst destination array of u8 elements
208 @param src source array of u8 elements
209 @param mask array of u64 values representing compress mask
210 @param n_elts number of elements in the source array
211 @return number of elements stored in destionation array
214 static_always_inline u32
215 clib_compress_u8 (u8 *dst, u8 *src, u64 *mask, u32 n_elts)
220 if (mask[0] == ~0ULL)
222 clib_memcpy_fast (dst, src, 64);
226 dst = clib_compress_u8_x64 (dst, src, mask[0]);
233 if (PREDICT_TRUE (n_elts == 0))
236 return clib_compress_u8_x64 (dst, src, mask[0] & pow2_mask (n_elts)) - dst0;