#include <vppinfra/clib.h> /* for CLIB_LINUX_KERNEL */
#include <vppinfra/vector.h>
+#include <vppinfra/error_bootstrap.h>
#ifdef CLIB_LINUX_KERNEL
#include <linux/string.h>
* so don't let it anywhere near them.
*/
#ifndef __COVERITY__
-#if __AVX512F__
+#if __AVX512BITALG__
#include <vppinfra/memcpy_avx512.h>
+#define clib_memcpy_fast_arch(a, b, c) clib_memcpy_fast_avx512 (a, b, c)
#elif __AVX2__
#include <vppinfra/memcpy_avx2.h>
+#define clib_memcpy_fast_arch(a, b, c) clib_memcpy_fast_avx2 (a, b, c)
#elif __SSSE3__
#include <vppinfra/memcpy_sse3.h>
-#else
-#define clib_memcpy_fast(a,b,c) memcpy(a,b,c)
-#endif
-#else /* __COVERITY__ */
-#define clib_memcpy_fast(a,b,c) memcpy(a,b,c)
-#endif
+#define clib_memcpy_fast_arch(a, b, c) clib_memcpy_fast_sse3 (a, b, c)
+#endif /* __AVX512BITALG__ */
+#endif /* __COVERITY__ */
+
+#ifndef clib_memcpy_fast_arch
+#define clib_memcpy_fast_arch(a, b, c) memcpy (a, b, c)
+#endif /* clib_memcpy_fast_arch */
+
+static_always_inline void *
+clib_memcpy_fast (void *restrict dst, const void *restrict src, size_t n)
+{
+ ASSERT (dst && src &&
+ "memcpy(src, dst, n) with src == NULL or dst == NULL is undefined "
+ "behaviour");
+ return clib_memcpy_fast_arch (dst, src, n);
+}
+
+#undef clib_memcpy_fast_arch
+
+#include <vppinfra/memcpy.h>
/* c-11 string manipulation variants */
* Optimize constant-number-of-bytes calls without asking
* "too many questions for someone from New Jersey"
*/
- if (__builtin_constant_p (n))
+ if (COMPILE_TIME_CONST (n))
{
clib_memcpy_fast (dest, src, n);
return EOK;
static_always_inline void
clib_memcpy_le (u8 * dst, u8 * src, u8 len, u8 max_len)
{
-#if defined (CLIB_HxAVE_VEC256)
+#if defined (CLIB_HAVE_VEC256)
u8x32 s0, s1, d0, d1;
u8x32 mask = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
uword count;
u64 first;
- if (max_count == 1)
- return 1;
+ if (max_count <= 1)
+ return max_count;
if (data[0] != data[1])
return 1;
#if defined(CLIB_HAVE_VEC256)
u64x4 splat = u64x4_splat (first);
- while (1)
+ while (count + 3 < max_count)
{
u64 bmp;
bmp = u8x32_msb_mask ((u8x32) (u64x4_load_unaligned (data) == splat));
if (bmp != 0xffffffff)
{
count += count_trailing_zeros (~bmp) / 8;
- return clib_min (count, max_count);
+ return count;
}
data += 4;
count += 4;
-
- if (count >= max_count)
- return max_count;
}
-#endif
+#else
count += 2;
data += 2;
while (count + 3 < max_count &&
data += 4;
count += 4;
}
+#endif
while (count < max_count && (data[0] == first))
{
data += 1;
uword count;
u32 first;
- if (max_count == 1)
- return 1;
+ if (max_count <= 1)
+ return max_count;
if (data[0] != data[1])
return 1;
#if defined(CLIB_HAVE_VEC256)
u32x8 splat = u32x8_splat (first);
- while (1)
+ while (count + 7 < max_count)
{
u64 bmp;
bmp = u8x32_msb_mask ((u8x32) (u32x8_load_unaligned (data) == splat));
if (bmp != 0xffffffff)
{
count += count_trailing_zeros (~bmp) / 4;
- return clib_min (count, max_count);
+ return count;
}
data += 8;
count += 8;
-
- if (count >= max_count)
- return max_count;
}
#elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK)
u32x4 splat = u32x4_splat (first);
- while (1)
+ while (count + 3 < max_count)
{
u64 bmp;
bmp = u8x16_msb_mask ((u8x16) (u32x4_load_unaligned (data) == splat));
if (bmp != 0xffff)
{
count += count_trailing_zeros (~bmp) / 4;
- return clib_min (count, max_count);
+ return count;
}
data += 4;
count += 4;
-
- if (count >= max_count)
- return max_count;
}
-#endif
+#else
count += 2;
data += 2;
while (count + 3 < max_count &&
data += 4;
count += 4;
}
+#endif
while (count < max_count && (data[0] == first))
{
data += 1;
uword count;
u16 first;
- if (max_count == 1)
- return 1;
+ if (max_count <= 1)
+ return max_count;
if (data[0] != data[1])
return 1;
#if defined(CLIB_HAVE_VEC256)
u16x16 splat = u16x16_splat (first);
- while (1)
+ while (count + 15 < max_count)
{
u64 bmp;
bmp = u8x32_msb_mask ((u8x32) (u16x16_load_unaligned (data) == splat));
if (bmp != 0xffffffff)
{
count += count_trailing_zeros (~bmp) / 2;
- return clib_min (count, max_count);
+ return count;
}
data += 16;
count += 16;
-
- if (count >= max_count)
- return max_count;
}
#elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK)
u16x8 splat = u16x8_splat (first);
- while (1)
+ while (count + 7 < max_count)
{
u64 bmp;
bmp = u8x16_msb_mask ((u8x16) (u16x8_load_unaligned (data) == splat));
if (bmp != 0xffff)
{
count += count_trailing_zeros (~bmp) / 2;
- return clib_min (count, max_count);
+ return count;
}
data += 8;
count += 8;
-
- if (count >= max_count)
- return max_count;
}
-#endif
+#else
count += 2;
data += 2;
while (count + 3 < max_count &&
data += 4;
count += 4;
}
+#endif
while (count < max_count && (data[0] == first))
{
data += 1;
uword count;
u8 first;
- if (max_count == 1)
- return 1;
+ if (max_count <= 1)
+ return max_count;
if (data[0] != data[1])
return 1;
#if defined(CLIB_HAVE_VEC256)
u8x32 splat = u8x32_splat (first);
- while (1)
+ while (count + 31 < max_count)
{
u64 bmp;
bmp = u8x32_msb_mask ((u8x32) (u8x32_load_unaligned (data) == splat));
if (bmp != 0xffffffff)
- {
- count += count_trailing_zeros (~bmp);
- return clib_min (count, max_count);
- }
+ return max_count;
data += 32;
count += 32;
-
- if (count >= max_count)
- return max_count;
}
#elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK)
u8x16 splat = u8x16_splat (first);
- while (1)
+ while (count + 15 < max_count)
{
u64 bmp;
bmp = u8x16_msb_mask ((u8x16) (u8x16_load_unaligned (data) == splat));
if (bmp != 0xffff)
{
count += count_trailing_zeros (~bmp);
- return clib_min (count, max_count);
+ return count;
}
data += 16;
count += 16;
-
- if (count >= max_count)
- return max_count;
}
-#endif
+#else
count += 2;
data += 2;
while (count + 3 < max_count &&
data += 4;
count += 4;
}
+#endif
while (count < max_count && (data[0] == first))
{
data += 1;
clib_c11_violation ("substring NULL");
if (s1 && s1max && (s1[clib_strnlen (s1, s1max)] != '\0'))
clib_c11_violation ("s1 unterminated");
- if (s2 && s2max && (s2[clib_strnlen (s2, s1max)] != '\0'))
+ if (s2 && s2max && (s2[clib_strnlen (s2, s2max)] != '\0'))
clib_c11_violation ("s2 unterminated");
return EINVAL;
}