1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright(c) 2022 Cisco Systems, Inc.
5 #include <vppinfra/vec.h>
6 #include <vppinfra/mem.h>
8 #ifndef CLIB_VECTOR_GROW_BY_ONE
9 #define CLIB_VECTOR_GROW_BY_ONE 0
13 vec_mem_size (void *v)
15 return v ? clib_mem_size (v - vec_get_header_size (v)) : 0;
19 _vec_alloc_internal (uword n_elts, const vec_attr_t *const attr)
21 uword req_size, alloc_size, data_offset, align;
22 uword elt_sz = attr->elt_sz;
23 void *p, *v, *heap = attr->heap;
25 /* alignment must be power of 2 */
26 align = clib_max (attr->align, VEC_MIN_ALIGN);
27 ASSERT (count_set_bits (align) == 1);
29 /* calc offset where vector data starts */
30 data_offset = attr->hdr_sz + sizeof (vec_header_t);
31 data_offset += heap ? sizeof (void *) : 0;
32 data_offset = round_pow2 (data_offset, align);
34 req_size = data_offset + n_elts * elt_sz;
35 p = clib_mem_heap_alloc_aligned (heap, req_size, align);
37 /* zero out whole alocation */
38 alloc_size = clib_mem_size (p);
39 clib_mem_unpoison (p, alloc_size);
40 clib_memset_u8 (p, 0, alloc_size);
42 /* fill vector header */
44 _vec_find (v)->len = n_elts;
45 _vec_find (v)->hdr_size = data_offset / VEC_MIN_ALIGN;
46 _vec_find (v)->log2_align = min_log2 (align);
49 _vec_find (v)->default_heap = 0;
53 _vec_find (v)->default_heap = 1;
55 /* poison extra space given by allocator */
56 clib_mem_poison (p + req_size, alloc_size - req_size);
57 _vec_set_grow_elts (v, (alloc_size - req_size) / elt_sz);
62 _vec_update_len (void *v, uword n_elts, uword elt_sz, uword n_data_bytes,
65 _vec_find (v)->len = n_elts;
66 _vec_set_grow_elts (v, unused_bytes / elt_sz);
67 clib_mem_unpoison (v, n_data_bytes);
68 clib_mem_poison (v + n_data_bytes, unused_bytes);
72 _vec_realloc_internal (void *v, uword n_elts, const vec_attr_t *const attr)
74 uword old_alloc_sz, new_alloc_sz, new_data_size, n_data_bytes, data_offset;
77 if (PREDICT_FALSE (v == 0))
78 return _vec_alloc_internal (n_elts, attr);
80 elt_sz = attr->elt_sz;
81 n_data_bytes = n_elts * elt_sz;
82 data_offset = vec_get_header_size (v);
83 new_data_size = data_offset + n_data_bytes;
84 new_alloc_sz = old_alloc_sz = clib_mem_size (vec_header (v));
86 /* realloc if new size cannot fit into existing allocation */
87 if (old_alloc_sz < new_data_size)
89 uword n_bytes, req_size = new_data_size;
90 void *p = v - data_offset;
92 req_size += CLIB_VECTOR_GROW_BY_ONE ? 0 : n_data_bytes / 2;
94 p = clib_mem_heap_realloc_aligned (vec_get_heap (v), p, req_size,
96 new_alloc_sz = clib_mem_size (p);
99 /* zero out new allocation */
100 n_bytes = new_alloc_sz - old_alloc_sz;
101 clib_mem_unpoison (p + old_alloc_sz, n_bytes);
102 clib_memset_u8 (p + old_alloc_sz, 0, n_bytes);
105 _vec_update_len (v, n_elts, elt_sz, n_data_bytes,
106 new_alloc_sz - new_data_size);
111 _vec_resize_internal (void *v, uword n_elts, const vec_attr_t *const attr)
113 uword elt_sz = attr->elt_sz;
114 if (PREDICT_TRUE (v != 0))
116 uword hs = _vec_find (v)->hdr_size * VEC_MIN_ALIGN;
117 uword alloc_sz = clib_mem_size (v - hs);
118 uword n_data_bytes = elt_sz * n_elts;
119 word unused_bytes = alloc_sz - (n_data_bytes + hs);
121 if (PREDICT_TRUE (unused_bytes >= 0))
123 _vec_update_len (v, n_elts, elt_sz, n_data_bytes, unused_bytes);
128 /* this shouled emit tail jump and likely avoid stack usasge inside this
130 return _vec_realloc_internal (v, n_elts, attr);
134 vec_len_not_inline (void *v)
140 vec_free_not_inline (void *v)