1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright(c) 2021 Cisco Systems, Inc.
5 #include <vppinfra/clib.h>
6 #ifndef included_memcpy_h
7 #define included_memcpy_h
11 static_always_inline void
12 clib_memcpy_u32_x4 (u32 *dst, u32 *src)
14 #if defined(CLIB_HAVE_VEC128)
15 u32x4_store_unaligned (u32x4_load_unaligned (src), dst);
17 clib_memcpy_fast (dst, src, 4 * sizeof (u32));
20 static_always_inline void
21 clib_memcpy_u32_x8 (u32 *dst, u32 *src)
23 #if defined(CLIB_HAVE_VEC256)
24 u32x8_store_unaligned (u32x8_load_unaligned (src), dst);
26 clib_memcpy_u32_x4 (dst, src);
27 clib_memcpy_u32_x4 (dst + 4, src + 4);
31 static_always_inline void
32 clib_memcpy_u32_x16 (u32 *dst, u32 *src)
34 #if defined(CLIB_HAVE_VEC512)
35 u32x16_store_unaligned (u32x16_load_unaligned (src), dst);
37 clib_memcpy_u32_x8 (dst, src);
38 clib_memcpy_u32_x8 (dst + 8, src + 8);
42 static_always_inline void
43 clib_memcpy_u32 (u32 *dst, u32 *src, u32 n_left)
45 #if defined(CLIB_HAVE_VEC128)
46 if (COMPILE_TIME_CONST (n_left))
48 /* for n_left defined as compile-time constant we should prevent compiler
49 * to use more expensive mask load/store for common cases where smaller
50 * register load/store exists */
54 clib_memcpy_u32_x4 (dst, src);
57 clib_memcpy_u32_x8 (dst, src);
60 clib_memcpy_u32_x8 (dst, src);
61 clib_memcpy_u32_x4 (dst + 8, src + 8);
64 clib_memcpy_u32_x16 (dst, src);
67 clib_memcpy_u32_x16 (dst, src);
68 clib_memcpy_u32_x16 (dst + 16, src + 16);
71 clib_memcpy_u32_x16 (dst, src);
72 clib_memcpy_u32_x16 (dst + 16, src + 16);
73 clib_memcpy_u32_x16 (dst + 32, src + 32);
74 clib_memcpy_u32_x16 (dst + 48, src + 48);
81 #if defined(CLIB_HAVE_VEC512)
84 clib_memcpy_u32_x16 (dst, src);
85 clib_memcpy_u32_x16 (dst + 16, src + 16);
86 clib_memcpy_u32_x16 (dst + 32, src + 32);
87 clib_memcpy_u32_x16 (dst + 48, src + 48);
94 #if defined(CLIB_HAVE_VEC256)
97 clib_memcpy_u32_x16 (dst, src);
98 clib_memcpy_u32_x16 (dst + 16, src + 16);
107 clib_memcpy_u32_x16 (dst, src);
113 #if defined(CLIB_HAVE_VEC512_MASK_LOAD_STORE)
116 u16 mask = pow2_mask (n_left);
117 u32x16_mask_store (u32x16_mask_load_zero (src, mask), dst, mask);
124 clib_memcpy_u32_x8 (dst, src);
130 #if defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE)
133 u8 mask = pow2_mask (n_left);
134 u32x8_mask_store (u32x8_mask_load_zero (src, mask), dst, mask);
141 clib_memcpy_u32_x4 (dst, src);
157 #else /* __COVERITY__ */
158 static_always_inline void
159 clib_memcpy_u32 (u32 *dst, u32 *src, u32 n_left)
161 memcpy (dst, src, n_left * sizeof (u32));