1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright(c) 2021 Cisco Systems, Inc.
5 #include <vppinfra/clib.h>
6 #ifndef included_memcpy_h
7 #define included_memcpy_h
9 static_always_inline void
10 clib_memcpy_may_overrun (void *dst, void *src, u32 n_bytes)
12 word n_left = n_bytes;
13 #if defined(CLIB_HAVE_VEC512)
14 u8x64u *sv = (u8x64u *) src;
15 u8x64u *dv = (u8x64u *) dst;
16 #elif defined(CLIB_HAVE_VEC256)
17 u8x32u *sv = (u8x32u *) src;
18 u8x32u *dv = (u8x32u *) dst;
19 #elif defined(CLIB_HAVE_VEC128)
20 u8x16u *sv = (u8x16u *) src;
21 u8x16u *dv = (u8x16u *) dst;
23 u64u *sv = (u64u *) src;
24 u64u *dv = (u64u *) dst;
27 while (n_left >= 4 * sizeof (sv[0]))
29 __typeof__ (*sv) v0, v1, v2, v3;
35 n_left -= 4 * sizeof (sv[0]);
48 n_left -= sizeof (sv[0]);
54 static_always_inline void
55 clib_memcpy_u32_x4 (u32 *dst, u32 *src)
57 #if defined(CLIB_HAVE_VEC128)
58 u32x4_store_unaligned (u32x4_load_unaligned (src), dst);
60 clib_memcpy_fast (dst, src, 4 * sizeof (u32));
63 static_always_inline void
64 clib_memcpy_u32_x8 (u32 *dst, u32 *src)
66 #if defined(CLIB_HAVE_VEC256)
67 u32x8_store_unaligned (u32x8_load_unaligned (src), dst);
69 clib_memcpy_u32_x4 (dst, src);
70 clib_memcpy_u32_x4 (dst + 4, src + 4);
74 static_always_inline void
75 clib_memcpy_u32_x16 (u32 *dst, u32 *src)
77 #if defined(CLIB_HAVE_VEC512)
78 u32x16_store_unaligned (u32x16_load_unaligned (src), dst);
80 clib_memcpy_u32_x8 (dst, src);
81 clib_memcpy_u32_x8 (dst + 8, src + 8);
85 static_always_inline void
86 clib_memcpy_u32 (u32 *dst, u32 *src, u32 n_left)
88 #if defined(CLIB_HAVE_VEC128)
89 if (COMPILE_TIME_CONST (n_left))
91 /* for n_left defined as compile-time constant we should prevent compiler
92 * to use more expensive mask load/store for common cases where smaller
93 * register load/store exists */
97 clib_memcpy_u32_x4 (dst, src);
100 clib_memcpy_u32_x8 (dst, src);
103 clib_memcpy_u32_x8 (dst, src);
104 clib_memcpy_u32_x4 (dst + 8, src + 8);
107 clib_memcpy_u32_x16 (dst, src);
110 clib_memcpy_u32_x16 (dst, src);
111 clib_memcpy_u32_x16 (dst + 16, src + 16);
114 clib_memcpy_u32_x16 (dst, src);
115 clib_memcpy_u32_x16 (dst + 16, src + 16);
116 clib_memcpy_u32_x16 (dst + 32, src + 32);
117 clib_memcpy_u32_x16 (dst + 48, src + 48);
124 #if defined(CLIB_HAVE_VEC512)
127 clib_memcpy_u32_x16 (dst, src);
128 clib_memcpy_u32_x16 (dst + 16, src + 16);
129 clib_memcpy_u32_x16 (dst + 32, src + 32);
130 clib_memcpy_u32_x16 (dst + 48, src + 48);
137 #if defined(CLIB_HAVE_VEC256)
140 clib_memcpy_u32_x16 (dst, src);
141 clib_memcpy_u32_x16 (dst + 16, src + 16);
150 clib_memcpy_u32_x16 (dst, src);
156 #if defined(CLIB_HAVE_VEC512_MASK_LOAD_STORE)
159 u16 mask = pow2_mask (n_left);
160 u32x16_mask_store (u32x16_mask_load_zero (src, mask), dst, mask);
167 clib_memcpy_u32_x8 (dst, src);
173 #if defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE)
176 u8 mask = pow2_mask (n_left);
177 u32x8_mask_store (u32x8_mask_load_zero (src, mask), dst, mask);
184 clib_memcpy_u32_x4 (dst, src);
200 #else /* __COVERITY__ */
201 static_always_inline void
202 clib_memcpy_u32 (u32 *dst, u32 *src, u32 n_left)
204 memcpy (dst, src, n_left * sizeof (u32));