return sm->shared_header->epoch;
}
-/*
- * Return the maximum element count of the vector based on its allocated
- * memory.
- */
-static int
-get_vec_mem_size (void *v, uword data_size)
-{
- vlib_stats_segment_t *sm = vlib_stats_get_segment ();
-
- if (v == 0)
- return 0;
-
- uword aligned_header_bytes = vec_header_bytes (0);
- void *p = v - aligned_header_bytes;
- void *oldheap = clib_mem_set_heap (sm->heap);
- int mem_size = (clib_mem_size (p) - aligned_header_bytes) / data_size;
- clib_mem_set_heap (oldheap);
-
- return mem_size;
-}
-
/* number of times to repeat the counter expand tests */
#define EXPAND_TEST_ROUNDS 3
// Check how many elements fit into the counter vector without expanding
// that. The next validate calls should not increase the stats segment
// epoch.
- int mem_size = get_vec_mem_size (counter.counters[0],
- sizeof ((counter.counters[0])[0]));
+ int mem_size = vec_max_len (counter.counters[0]);
for (index = 1; index <= mem_size - 1; index++)
{
vlib_validate_simple_counter (&counter, index);
// Check how many elements fit into the counter vector without expanding
// that. The next validate calls should not increase the stats segment
// epoch.
- int mem_size = get_vec_mem_size (counter.counters[0],
- sizeof ((counter.counters[0])[0]));
+ int mem_size = vec_max_len (counter.counters[0]);
for (index = 1; index <= mem_size - 1; index++)
{
vlib_validate_combined_counter (&counter, index);
return (rv);
}
-static inline void *
-svm_mem_alloc_aligned_at_offset (svm_region_t * rp,
- uword size, uword align, uword offset)
-{
- clib_mem_heap_t *oldheap;
- ASSERT (rp->flags & SVM_FLAGS_MHEAP);
- u8 *rv;
-
- pthread_mutex_lock (&rp->mutex);
- oldheap = clib_mem_set_heap (rp->data_heap);
- rv = clib_mem_alloc_aligned_at_offset (size, align, offset,
- 1 /* yes, call os_out_of_memory */ );
- clib_mem_set_heap (oldheap);
- pthread_mutex_unlock (&rp->mutex);
- return (rv);
-}
-
static inline void
svm_mem_free (svm_region_t * rp, void *ptr)
{
u32 runtime_index;
/* Runtime data for this node. */
- void *runtime_data;
+ u8 *runtime_data;
/* Node flags. */
u16 flags;
l = vec_len (data_vec);
- data_vec = _vec_resize (data_vec,
- /* length_increment */ n_data_elts,
- /* total size after increment */
- (l + n_data_elts) * n_data_elt_bytes,
- /* header_bytes */ 0, /* data_align */ 0);
+ data_vec =
+ _vec_realloc (data_vec, l + n_data_elts, n_data_elt_bytes,
+ /* header_bytes */ 0, /* data_align */ 0, /* heap */ 0);
p->pending_event_data_by_type_index[t] = data_vec;
data_to_be_written_by_caller = data_vec + l * n_data_elt_bytes;
if (!check_vpp_heap ())
return __libc_realloc (p, size);
- return clib_mem_realloc (p, size, clib_mem_size (p));
+ return clib_mem_realloc (p, size);
}
int
/* ------------ Internal support for realloc, memalign, etc -------------- */
/* Try to realloc; only in-place unless can_move true */
-static mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb,
+static CLIB_NOSANITIZE_ADDR mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb,
int can_move) {
mchunkptr newp = 0;
size_t oldsize = chunksize(p);
*sizep = this_seg->size;
}
-CLIB_NOSANITIZE_ADDR __clib_export
+CLIB_NOSANITIZE_ADDR
int mspace_is_heap_object (mspace msp, void *p)
{
msegment *this_seg;
return 0;
}
-CLIB_NOSANITIZE_ADDR __clib_export
+CLIB_NOSANITIZE_ADDR
void* mspace_get_aligned (mspace msp,
unsigned long n_user_data_bytes,
unsigned long align,
return (void *) searchp;
}
-CLIB_NOSANITIZE_ADDR __clib_export
+CLIB_NOSANITIZE_ADDR
void mspace_put (mspace msp, void *p_arg)
{
char *object_header;
mspace_free (msp, p_arg);
}
-CLIB_NOSANITIZE_ADDR __clib_export
+CLIB_NOSANITIZE_ADDR
size_t mspace_usable_size_with_delta (const void *p)
{
size_t usable_size;
return mem;
}
+CLIB_NOSANITIZE_ADDR
void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) {
void* mem = 0;
if (oldmem != 0) {
return mem;
}
+CLIB_NOSANITIZE_ADDR
void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) {
mstate ms = (mstate)msp;
if (!ok_magic(ms)) {
*/
DLMALLOC_EXPORT int mspace_mallopt(int, int);
+DLMALLOC_EXPORT void* mspace_realloc_in_place (mspace msp, void *oldmem, size_t bytes);
+
DLMALLOC_EXPORT void* mspace_get_aligned (mspace msp,
unsigned long n_user_data_bytes,
unsigned long align,
if (vec_len (s->contents) > 0)
{
/* Make vector copy of contents with given element size. */
- result = _vec_resize (result,
- vec_len (s->contents) / elt_size,
- vec_len (s->contents),
- /* header_bytes */ 0,
- /* align */ 0);
+ result =
+ _vec_realloc (result, vec_len (s->contents) / elt_size, elt_size,
+ /* header_bytes */ 0, /* align */ 0, 0);
clib_memcpy (result, s->contents, vec_len (s->contents));
}
new_len = len + 1;
if (new_len * hash_pair_bytes (h) > (1ULL << log2_bytes))
{
- pi->pairs = clib_mem_realloc (pi->pairs,
- 1ULL << (log2_bytes + 1),
- 1ULL << log2_bytes);
+ pi->pairs = clib_mem_realloc (pi->pairs, 1ULL << (log2_bytes + 1));
log2_bytes++;
}
if (h_user)
log2_pair_size = h_user->log2_pair_size;
- v = _vec_resize ((void *) 0,
- /* vec len: */ elts,
- /* data bytes: */
- (elts << log2_pair_size) * sizeof (hash_pair_t),
- /* header bytes: */
- sizeof (h[0]),
- /* alignment */ sizeof (hash_pair_t));
+ v = _vec_realloc (0, elts, (1 << log2_pair_size) * sizeof (hash_pair_t),
+ sizeof (h[0]), sizeof (hash_pair_t), 0);
h = hash_header (v);
if (h_user)
h = heap_header (v);
if (!v || !(h->flags & HEAP_IS_STATIC))
- v = _vec_resize (v,
- align_size,
- (offset + align_size) * elt_bytes,
- sizeof (h[0]), HEAP_DATA_ALIGN);
+ v = _vec_realloc (v, offset + align_size, elt_bytes, sizeof (h[0]),
+ HEAP_DATA_ALIGN, 0);
else
_vec_len (v) += align_size;
return vec_header (v);
}
-always_inline uword
-heap_header_bytes ()
-{
- return vec_header_bytes (sizeof (heap_header_t));
-}
-
always_inline void
heap_dup_header (heap_header_t * old, heap_header_t * new)
{
if (!v_old)
return v_old;
- v_new = 0;
- v_new =
- _vec_resize (v_new, _vec_len (v_old), v_bytes, sizeof (heap_header_t),
- HEAP_DATA_ALIGN);
+ v_new = _vec_realloc (0, _vec_len (v_old), 1, sizeof (heap_header_t),
+ HEAP_DATA_ALIGN, 0);
h_new = heap_header (v_new);
heap_dup_header (h_old, h_new);
clib_memcpy_fast (v_new, v_old, v_bytes);
always_inline void *
_heap_new (u32 len, u32 n_elt_bytes)
{
- void *v = _vec_resize ((void *) 0, len, (uword) len * n_elt_bytes,
- sizeof (heap_header_t),
- HEAP_DATA_ALIGN);
+ void *v = _vec_realloc ((void *) 0, len, n_elt_bytes, sizeof (heap_header_t),
+ HEAP_DATA_ALIGN, 0);
heap_header (v)->elt_bytes = n_elt_bytes;
return v;
}
return v ? heap_header (v)->max_len : 0;
}
-/* Create fixed size heap with given block of memory. */
-always_inline void *
-heap_create_from_memory (void *memory, uword max_len, uword elt_bytes)
-{
- heap_header_t *h;
- void *v;
-
- if (max_len * elt_bytes < sizeof (h[0]))
- return 0;
-
- h = memory;
- clib_memset (h, 0, sizeof (h[0]));
- h->max_len = max_len;
- h->elt_bytes = elt_bytes;
- h->flags = HEAP_IS_STATIC;
-
- v = (void *) (memory + heap_header_bytes ());
- _vec_len (v) = 0;
- return v;
-}
-
/* Execute BODY for each allocated heap element. */
#define heap_foreach(var,len,heap,body) \
do { \
#define CLIB_MAX_NUMAS 16
#define CLIB_MEM_VM_MAP_FAILED ((void *) ~0)
#define CLIB_MEM_ERROR (-1)
+#define CLIB_MEM_LOG2_MIN_ALIGN (3)
+#define CLIB_MEM_MIN_ALIGN (1 << CLIB_MEM_LOG2_MIN_ALIGN)
typedef enum
{
struct _clib_mem_vm_map_hdr *prev, *next;
} clib_mem_vm_map_hdr_t;
-#define foreach_clib_mem_heap_flag \
- _(0, LOCKED, "locked") \
- _(1, UNMAP_ON_DESTROY, "unmap-on-destroy")
+#define foreach_clib_mem_heap_flag \
+ _ (0, LOCKED, "locked") \
+ _ (1, UNMAP_ON_DESTROY, "unmap-on-destroy") \
+ _ (2, TRACED, "traced")
typedef enum
{
ASSERT (__os_thread_index > 0);
}
-always_inline uword
-clib_mem_size_nocheck (void *p)
-{
- size_t mspace_usable_size_with_delta (const void *p);
- return mspace_usable_size_with_delta (p);
-}
-
-/* Memory allocator which may call os_out_of_memory() if it fails */
-always_inline void *
-clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
- int os_out_of_memory_on_failure)
-{
- void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes,
- unsigned long align, unsigned long align_offset);
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
- void *p;
-
- if (align_offset > align)
- {
- if (align > 0)
- align_offset %= align;
- else
- align_offset = align;
- }
-
- p = mspace_get_aligned (h->mspace, size, align, align_offset);
-
- if (PREDICT_FALSE (0 == p))
- {
- if (os_out_of_memory_on_failure)
- os_out_of_memory ();
- return 0;
- }
-
- CLIB_MEM_UNPOISON (p, size);
- return p;
-}
-
-/* Memory allocator which calls os_out_of_memory() when it fails */
-always_inline void *
-clib_mem_alloc (uword size)
-{
- return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
- /* align_offset */ 0,
- /* os_out_of_memory */ 1);
-}
-
-always_inline void *
-clib_mem_alloc_aligned (uword size, uword align)
-{
- return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
- /* os_out_of_memory */ 1);
-}
-
/* Memory allocator which calls os_out_of_memory() when it fails */
-always_inline void *
-clib_mem_alloc_or_null (uword size)
-{
- return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
- /* align_offset */ 0,
- /* os_out_of_memory */ 0);
-}
-
-always_inline void *
-clib_mem_alloc_aligned_or_null (uword size, uword align)
-{
- return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
- /* os_out_of_memory */ 0);
-}
-
-
+void *clib_mem_alloc (uword size);
+void *clib_mem_alloc_aligned (uword size, uword align);
+void *clib_mem_alloc_or_null (uword size);
+void *clib_mem_alloc_aligned_or_null (uword size, uword align);
+void *clib_mem_realloc (void *p, uword new_size);
+void *clib_mem_realloc_aligned (void *p, uword new_size, uword align);
/* Memory allocator which panics when it fails.
Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
/* Alias to stack allocator for naming consistency. */
#define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
-always_inline uword
-clib_mem_is_heap_object (void *p)
-{
- int mspace_is_heap_object (void *msp, void *p);
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
- return mspace_is_heap_object (h->mspace, p);
-}
-
-always_inline void
-clib_mem_free (void *p)
-{
- void mspace_put (void *msp, void *p_arg);
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
-
- /* Make sure object is in the correct heap. */
- ASSERT (clib_mem_is_heap_object (p));
-
- CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
-
- mspace_put (h->mspace, p);
-}
-
-always_inline void *
-clib_mem_realloc (void *p, uword new_size, uword old_size)
-{
- /* By default use alloc, copy and free to emulate realloc. */
- void *q = clib_mem_alloc (new_size);
- if (q)
- {
- uword copy_size;
- if (old_size < new_size)
- copy_size = old_size;
- else
- copy_size = new_size;
- clib_memcpy_fast (q, p, copy_size);
- clib_mem_free (p);
- }
- return q;
-}
-
-always_inline uword
-clib_mem_size (void *p)
-{
- ASSERT (clib_mem_is_heap_object (p));
- return clib_mem_size_nocheck (p);
-}
-
-always_inline void
-clib_mem_free_s (void *p)
-{
- uword size = clib_mem_size (p);
- CLIB_MEM_UNPOISON (p, size);
- memset_s_inline (p, size, 0, size);
- clib_mem_free (p);
-}
+uword clib_mem_is_heap_object (void *p);
+void clib_mem_free (void *p);
+uword clib_mem_size (void *p);
+void clib_mem_free_s (void *p);
always_inline clib_mem_heap_t *
clib_mem_get_heap (void)
format_white_space, indent + 2, format_msize, mi.usmblks);
}
- if (mspace_is_traced (heap->mspace))
+ if (heap->flags & CLIB_MEM_HEAP_F_TRACED)
s = format (s, "\n%U", format_mheap_trace, tm, verbose);
return s;
}
__clib_export void
mheap_trace (clib_mem_heap_t * h, int enable)
{
- (void) mspace_enable_disable_trace (h->mspace, enable);
+ if (enable)
+ h->flags |= CLIB_MEM_HEAP_F_TRACED;
+ else
+ h->flags &= ~CLIB_MEM_HEAP_F_TRACED;
if (enable == 0)
mheap_trace_main_free (&mheap_trace_main);
clib_mem_is_traced (void)
{
clib_mem_heap_t *h = clib_mem_get_heap ();
- return mspace_is_traced (h->mspace);
+ return (h->flags &= CLIB_MEM_HEAP_F_TRACED) != 0;
}
__clib_export uword
return heap->size;
}
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
+/* Memory allocator which may call os_out_of_memory() if it fails */
+static void *
+clib_mem_alloc_inline (uword size, uword align,
+ int os_out_of_memory_on_failure)
+{
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ void *p;
+
+ align = clib_max (CLIB_MEM_MIN_ALIGN, align);
+
+ p = mspace_memalign (h->mspace, align, size);
+
+ if (PREDICT_FALSE (0 == p))
+ {
+ if (os_out_of_memory_on_failure)
+ os_out_of_memory ();
+ return 0;
+ }
+
+ if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED))
+ mheap_get_trace (pointer_to_uword (p), clib_mem_size (p));
+
+ CLIB_MEM_UNPOISON (p, size);
+ return p;
+}
+
+/* Memory allocator which calls os_out_of_memory() when it fails */
+__clib_export void *
+clib_mem_alloc (uword size)
+{
+ return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN,
+ /* os_out_of_memory */ 1);
+}
+
+__clib_export void *
+clib_mem_alloc_aligned (uword size, uword align)
+{
+ return clib_mem_alloc_inline (size, align,
+ /* os_out_of_memory */ 1);
+}
+
+/* Memory allocator which calls os_out_of_memory() when it fails */
+__clib_export void *
+clib_mem_alloc_or_null (uword size)
+{
+ return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN,
+ /* os_out_of_memory */ 0);
+}
+
+__clib_export void *
+clib_mem_alloc_aligned_or_null (uword size, uword align)
+{
+ return clib_mem_alloc_inline (size, align,
+ /* os_out_of_memory */ 0);
+}
+
+__clib_export void *
+clib_mem_realloc_aligned (void *p, uword new_size, uword align)
+{
+ uword old_alloc_size;
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ void *new;
+
+ ASSERT (count_set_bits (align) == 1);
+
+ old_alloc_size = p ? mspace_usable_size (p) : 0;
+
+ if (new_size == old_alloc_size)
+ return p;
+
+ if (p && pointer_is_aligned (p, align) &&
+ mspace_realloc_in_place (h->mspace, p, new_size))
+ {
+ CLIB_MEM_UNPOISON (p, new_size);
+ }
+ else
+ {
+ new = clib_mem_alloc_inline (new_size, align, 1);
+
+ CLIB_MEM_UNPOISON (new, new_size);
+ if (old_alloc_size)
+ {
+ CLIB_MEM_UNPOISON (p, old_alloc_size);
+ clib_memcpy_fast (new, p, clib_min (new_size, old_alloc_size));
+ clib_mem_free (p);
+ }
+ p = new;
+ }
+
+ return p;
+}
+
+__clib_export void *
+clib_mem_realloc (void *p, uword new_size)
+{
+ return clib_mem_realloc_aligned (p, new_size, CLIB_MEM_MIN_ALIGN);
+}
+
+__clib_export uword
+clib_mem_is_heap_object (void *p)
+{
+ int mspace_is_heap_object (void *msp, void *p);
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ return mspace_is_heap_object (h->mspace, p);
+}
+
+__clib_export void
+clib_mem_free (void *p)
+{
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ uword size = clib_mem_size (p);
+
+ /* Make sure object is in the correct heap. */
+ ASSERT (clib_mem_is_heap_object (p));
+
+ if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED))
+ mheap_put_trace (pointer_to_uword (p), size);
+ CLIB_MEM_POISON (p, clib_mem_size (p));
+
+ mspace_free (h->mspace, p);
+}
+
+__clib_export uword
+clib_mem_size (void *p)
+{
+ return mspace_usable_size (p);
+}
+
+__clib_export void
+clib_mem_free_s (void *p)
+{
+ uword size = clib_mem_size (p);
+ CLIB_MEM_UNPOISON (p, size);
+ memset_s_inline (p, size, 0, size);
+ clib_mem_free (p);
+}
#include <vppinfra/pool.h>
__clib_export void
-_pool_init_fixed (void **pool_ptr, u32 elt_size, u32 max_elts)
+_pool_init_fixed (void **pool_ptr, uword elt_size, uword max_elts, uword align)
{
uword *b;
pool_header_t *ph;
ASSERT (elt_size);
ASSERT (max_elts);
- v =
- vec_resize_allocate_memory (0, max_elts, elt_size * max_elts,
- sizeof (pool_header_t), CLIB_CACHE_LINE_BYTES);
+ v = _vec_realloc (0, max_elts, elt_size, sizeof (pool_header_t), align, 0);
ph = pool_header (v);
ph->max_elts = max_elts;
} pool_header_t;
-/** Align pool header so that pointers are naturally aligned. */
-#define pool_aligned_header_bytes \
- round_pow2 (sizeof (pool_header_t), sizeof (void *))
-
/** Get pool header from user pool pointer */
always_inline pool_header_t *
pool_header (void *v)
return vec_header (v);
}
-extern void _pool_init_fixed (void **, u32, u32);
-extern void fpool_free (void *);
+void _pool_init_fixed (void **pool_ptr, uword elt_sz, uword max_elts,
+ uword align);
/** initialize a fixed-size, preallocated pool */
-#define pool_init_fixed(pool,max_elts) \
-{ \
- _pool_init_fixed((void **)&(pool),sizeof(pool[0]),max_elts); \
-}
+#define pool_init_fixed(P, E) \
+ _pool_init_fixed ((void **) &(P), _vec_elt_sz (P), E, _vec_align (P, 0));
/** Validate a pool */
always_inline void
ASSERT (clib_bitmap_get (p->free_bitmap, p->free_indices[i]) == 1);
}
-always_inline void
-pool_header_validate_index (void *v, uword index)
-{
- pool_header_t *p = pool_header (v);
-
- if (v)
- vec_validate (p->free_bitmap, index / BITS (uword));
-}
-
-#define pool_validate_index(v,i) \
-do { \
- uword __pool_validate_index = (i); \
- vec_validate_ha ((v), __pool_validate_index, \
- pool_aligned_header_bytes, /* align */ 0); \
- pool_header_validate_index ((v), __pool_validate_index); \
-} while (0)
-
/** Number of active elements in a pool.
* @return Number of active elements in a pool
*/
#define pool_max_len(P) vec_max_len (P)
/** Number of free elements in pool */
-#define pool_free_elts(P) \
- ({ \
- pool_header_t *_pool_var (p) = pool_header (P); \
- uword n_free = 0; \
- if (P) \
- { \
- n_free += vec_len (_pool_var (p)->free_indices); \
- /* Fixed-size pools have max_elts set non-zero */ \
- if (_pool_var (p)->max_elts == 0) \
- n_free += pool_max_len (P) - vec_len (P); \
- } \
- n_free; \
- })
+static_always_inline uword
+_pool_free_elts (void *p, uword elt_sz)
+{
+ pool_header_t *ph;
+ uword n_free;
+
+ if (p == 0)
+ return 0;
+
+ ph = pool_header (p);
+
+ n_free = vec_len (ph->free_indices);
+
+ /* Fixed-size pools have max_elts set non-zero */
+ if (ph->max_elts == 0)
+ n_free += _vec_max_len (p, elt_sz) - vec_len (p);
+
+ return n_free;
+}
+
+#define pool_free_elts(P) _pool_free_elts ((void *) (P), _vec_elt_sz (P))
/** Allocate an object E from a pool P (general version).
First search free list. If nothing is free extend vector of objects.
*/
+
+static_always_inline void
+_pool_get (void **pp, void **ep, uword align, int zero, uword elt_sz)
+{
+ uword len = 0;
+ void *p = pp[0];
+ void *e;
+
+ if (p)
+ {
+ pool_header_t *ph = pool_header (p);
+ uword n_free = vec_len (ph->free_indices);
+
+ if (n_free)
+ {
+ uword index = ph->free_indices[n_free - 1];
+ e = p + index * elt_sz;
+ ph->free_bitmap =
+ clib_bitmap_andnoti_notrim (ph->free_bitmap, index);
+ _vec_len (ph->free_indices) = n_free - 1;
+ CLIB_MEM_UNPOISON (e, elt_sz);
+ goto done;
+ }
+
+ if (ph->max_elts)
+ {
+ clib_warning ("can't expand fixed-size pool");
+ os_out_of_memory ();
+ }
+ }
+
+ len = vec_len (p);
+
+ /* Nothing on free list, make a new element and return it. */
+ p =
+ _vec_realloc_inline (p, len + 1, elt_sz, sizeof (pool_header_t), align, 0);
+ e = p + len * elt_sz;
+
+ _vec_update_pointer (pp, p);
+
+done:
+ ep[0] = e;
+ if (zero)
+ clib_memset_u8 (e, 0, elt_sz);
+}
+
#define _pool_get_aligned_internal(P, E, A, Z) \
- do \
- { \
- pool_header_t *_pool_var (p) = pool_header (P); \
- uword _pool_var (l); \
- \
- STATIC_ASSERT (A == 0 || ((A % sizeof (P[0])) == 0) || \
- ((sizeof (P[0]) % A) == 0), \
- "Pool aligned alloc of incorrectly sized object"); \
- _pool_var (l) = 0; \
- if (P) \
- _pool_var (l) = vec_len (_pool_var (p)->free_indices); \
- \
- if (_pool_var (l) > 0) \
- { \
- /* Return free element from free list. */ \
- uword _pool_var (i) = \
- _pool_var (p)->free_indices[_pool_var (l) - 1]; \
- (E) = (P) + _pool_var (i); \
- _pool_var (p)->free_bitmap = clib_bitmap_andnoti_notrim ( \
- _pool_var (p)->free_bitmap, _pool_var (i)); \
- _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \
- CLIB_MEM_UNPOISON ((E), sizeof ((E)[0])); \
- } \
- else \
- { \
- /* fixed-size, preallocated pools cannot expand */ \
- if ((P) && _pool_var (p)->max_elts) \
- { \
- clib_warning ("can't expand fixed-size pool"); \
- os_out_of_memory (); \
- } \
- /* Nothing on free list, make a new element and return it. */ \
- P = _vec_resize (P, /* length_increment */ 1, \
- /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
- pool_aligned_header_bytes, /* align */ (A)); \
- E = vec_end (P) - 1; \
- } \
- if (Z) \
- memset (E, 0, sizeof (*E)); \
- } \
- while (0)
+ _pool_get ((void **) &(P), (void **) &(E), _vec_align (P, A), Z, \
+ _vec_elt_sz (P))
/** Allocate an object E from a pool P with alignment A */
#define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)
#define pool_get_zero(P,E) pool_get_aligned_zero(P,E,0)
always_inline int
-_pool_get_will_expand (void *p, uword elt_size)
+_pool_get_will_expand (void *p, uword elt_sz)
{
pool_header_t *ph;
uword len;
if (len > 0)
return 0;
- return _vec_resize_will_expand (p, 1, elt_size);
+ return _vec_resize_will_expand (p, 1, elt_sz);
}
#define pool_get_will_expand(P) _pool_get_will_expand (P, sizeof ((P)[0]))
always_inline int
-_pool_put_will_expand (void *p, uword index, uword elt_size)
+_pool_put_will_expand (void *p, uword index, uword elt_sz)
{
pool_header_t *ph = pool_header (p);
#define pool_put_will_expand(P, E) _pool_put_will_expand (P, (E) - (P), sizeof ((P)[0])
/** Use free bitmap to query whether given element is free. */
-#define pool_is_free(P,E) \
-({ \
- pool_header_t * _pool_var (p) = pool_header (P); \
- uword _pool_var (i) = (E) - (P); \
- (_pool_var (i) < vec_len (P)) ? clib_bitmap_get (_pool_var (p)->free_bitmap, _pool_i) : 1; \
-})
+static_always_inline int
+pool_is_free_index (void *p, uword index)
+{
+ pool_header_t *ph = pool_header (p);
+ return index < vec_len (p) ? clib_bitmap_get (ph->free_bitmap, index) : 1;
+}
-/** Use free bitmap to query whether given index is free */
-#define pool_is_free_index(P,I) pool_is_free((P),(P)+(I))
+#define pool_is_free(P, E) pool_is_free_index ((void *) (P), (E) - (P))
/** Free an object E in pool P. */
-#define pool_put(P, E) \
- do \
- { \
- typeof (P) _pool_var (p__) = (P); \
- typeof (E) _pool_var (e__) = (E); \
- pool_header_t *_pool_var (p) = pool_header (_pool_var (p__)); \
- uword _pool_var (l) = _pool_var (e__) - _pool_var (p__); \
- if (_pool_var (p)->max_elts == 0) \
- ASSERT (vec_is_member (_pool_var (p__), _pool_var (e__))); \
- ASSERT (!pool_is_free (_pool_var (p__), _pool_var (e__))); \
- \
- /* Add element to free bitmap and to free list. */ \
- _pool_var (p)->free_bitmap = \
- clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, _pool_var (l)); \
- \
- /* Preallocated pool? */ \
- if (_pool_var (p)->max_elts) \
- { \
- ASSERT (_pool_var (l) < _pool_var (p)->max_elts); \
- _pool_var (p) \
- ->free_indices[_vec_len (_pool_var (p)->free_indices)] = \
- _pool_var (l); \
- _vec_len (_pool_var (p)->free_indices) += 1; \
- } \
- else \
- vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \
- \
- CLIB_MEM_POISON (_pool_var (e__), sizeof (_pool_var (e__)[0])); \
- } \
- while (0)
-
-/** Free pool element with given index. */
-#define pool_put_index(p,i) \
-do { \
- typeof (p) _e = (p) + (i); \
- pool_put (p, _e); \
-} while (0)
+static_always_inline void
+_pool_put_index (void *p, uword index, uword elt_sz)
+{
+ pool_header_t *ph = pool_header (p);
+
+ ASSERT (index < ph->max_elts ? ph->max_elts : vec_len (p));
+ ASSERT (!pool_is_free_index (p, index));
+
+ /* Add element to free bitmap and to free list. */
+ ph->free_bitmap = clib_bitmap_ori_notrim (ph->free_bitmap, index);
+
+ /* Preallocated pool? */
+ if (ph->max_elts)
+ {
+ ph->free_indices[_vec_len (ph->free_indices)] = index;
+ _vec_len (ph->free_indices) += 1;
+ }
+ else
+ vec_add1 (ph->free_indices, index);
+
+ CLIB_MEM_POISON (p + index * elt_sz, elt_sz);
+}
+
+#define pool_put_index(P, I) _pool_put_index ((void *) (P), I, _vec_elt_sz (P))
+#define pool_put(P, E) pool_put_index (P, (E) - (P))
/** Allocate N more free elements to pool (general version). */
-#define pool_alloc_aligned(P,N,A) \
-do { \
- pool_header_t * _p; \
- \
- if ((P)) \
- { \
- _p = pool_header (P); \
- if (_p->max_elts) \
- { \
- clib_warning ("Can't expand fixed-size pool"); \
- os_out_of_memory(); \
- } \
- } \
- \
- (P) = _vec_resize ((P), 0, (vec_len (P) + (N)) * sizeof (P[0]), \
- pool_aligned_header_bytes, \
- (A)); \
- _p = pool_header (P); \
- vec_resize (_p->free_indices, (N)); \
- _vec_len (_p->free_indices) -= (N); \
-} while (0)
+
+static_always_inline void
+_pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz)
+{
+ pool_header_t *ph = pool_header (pp[0]);
+ uword len = vec_len (pp[0]);
+
+ if (ph && ph->max_elts)
+ {
+ clib_warning ("Can't expand fixed-size pool");
+ os_out_of_memory ();
+ }
+
+ pp[0] = _vec_realloc_inline (pp[0], len + n_elts, elt_sz,
+ sizeof (pool_header_t), align, 0);
+ _vec_len (pp[0]) = len;
+ CLIB_MEM_POISON (pp[0] + len * elt_sz, n_elts * elt_sz);
+
+ ph = pool_header (pp[0]);
+ vec_resize (ph->free_indices, n_elts);
+ _vec_len (ph->free_indices) -= n_elts;
+ clib_bitmap_vec_validate (ph->free_bitmap, len + n_elts - 1);
+}
+
+#define pool_alloc_aligned(P, N, A) \
+ _pool_alloc ((void **) &(P), N, _vec_align (P, A), _vec_elt_sz (P))
/** Allocate N more free elements to pool (unspecified alignment). */
#define pool_alloc(P,N) pool_alloc_aligned(P,N,0)
+static_always_inline void *
+_pool_dup (void *p, uword align, uword elt_sz)
+{
+ pool_header_t *nph, *ph = pool_header (p);
+ uword len = vec_len (p);
+ void *n;
+
+ if (ph && ph->max_elts)
+ {
+ clib_warning ("Can't expand fixed-size pool");
+ os_out_of_memory ();
+ }
+
+ n = _vec_realloc_inline (0, len, elt_sz, sizeof (pool_header_t), align, 0);
+ nph = pool_header (n);
+ clib_memset_u8 (nph, 0, sizeof (vec_header_t));
+
+ if (len)
+ {
+ u32 *fi;
+ vec_foreach (fi, ph->free_indices)
+ CLIB_MEM_UNPOISON (p + elt_sz * fi[0], elt_sz);
+
+ clib_memcpy_fast (n, p, len * elt_sz);
+
+ nph->free_bitmap = clib_bitmap_dup (ph->free_bitmap);
+ nph->free_indices = vec_dup (ph->free_indices);
+
+ vec_foreach (fi, ph->free_indices)
+ {
+ uword offset = elt_sz * fi[0];
+ CLIB_MEM_POISON (p + offset, elt_sz);
+ CLIB_MEM_POISON (n + offset, elt_sz);
+ }
+ }
+
+ return n;
+}
+
/**
* Return copy of pool with alignment
*
* @param A alignment (may be zero)
* @return copy of pool
*/
+
#define pool_dup_aligned(P, A) \
- ({ \
- typeof (P) _pool_var (new) = 0; \
- pool_header_t *_pool_var (ph), *_pool_var (new_ph); \
- u32 _pool_var (n) = pool_len (P); \
- if ((P)) \
- { \
- _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n), \
- _pool_var (n) * sizeof ((P)[0]), \
- pool_aligned_header_bytes, (A)); \
- CLIB_MEM_OVERFLOW_PUSH ((P), _pool_var (n) * sizeof ((P)[0])); \
- clib_memcpy_fast (_pool_var (new), (P), \
- _pool_var (n) * sizeof ((P)[0])); \
- CLIB_MEM_OVERFLOW_POP (); \
- _pool_var (ph) = pool_header (P); \
- _pool_var (new_ph) = pool_header (_pool_var (new)); \
- _pool_var (new_ph)->free_bitmap = \
- clib_bitmap_dup (_pool_var (ph)->free_bitmap); \
- _pool_var (new_ph)->free_indices = \
- vec_dup (_pool_var (ph)->free_indices); \
- _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts; \
- } \
- _pool_var (new); \
- })
+ _pool_dup (P, _vec_align (P, A), _vec_elt_sz (P))
/**
* Return copy of pool without alignment
#define pool_dup(P) pool_dup_aligned(P,0)
/** Low-level free pool operator (do not call directly). */
-always_inline void *
-_pool_free (void *v)
+always_inline void
+_pool_free (void **v)
{
- pool_header_t *p = pool_header (v);
- if (!v)
- return v;
+ pool_header_t *p = pool_header (v[0]);
+ if (!p)
+ return;
+
clib_bitmap_free (p->free_bitmap);
vec_free (p->free_indices);
- vec_free (v);
- return 0;
+ _vec_free (v);
}
+#define pool_free(p) _pool_free ((void **) &(p))
static_always_inline uword
pool_get_first_index (void *pool)
return clib_bitmap_next_clear (h->free_bitmap, last + 1);
}
-/** Free a pool. */
-#define pool_free(p) (p) = _pool_free(p)
-
/** Optimized iteration through pool.
@param LO pointer to first element in chunk
void *v;
clib_ring_header_t *h;
- v = _vec_resize ((void *) 0,
- /* length increment */ size,
- /* data bytes */ elt_bytes * size,
- /* header bytes */ sizeof (h[0]),
- /* data align */ align);
+ v = _vec_realloc (0,
+ /* length increment */ size,
+ /* data bytes */ elt_bytes,
+ /* header bytes */ sizeof (h[0]),
+ /* data align */ align, 0);
h = clib_ring_header (v);
h->next = 0;
if (l > max_length)
serialize_error (&m->header,
clib_error_create ("bad vector length %d", l));
- p = v = _vec_resize ((void *) 0, l, (uword) l * elt_bytes, header_bytes,
- /* align */ align);
+ p = v = _vec_realloc ((void *) 0, l, elt_bytes, header_bytes,
+ /* align */ align, 0);
while (l != 0)
{
return 0;
}
- v = _vec_resize ((void *) 0, l, (uword) l * elt_bytes, sizeof (p[0]),
- align);
+ v = _vec_realloc ((void *) 0, l, elt_bytes, sizeof (p[0]), align, 0);
p = pool_header (v);
vec_unserialize (m, &p->free_indices, unserialize_vec_32);
ASSERT (sparse_index_bits <= 16);
- v = _vec_resize ((void *) 0,
- /* length increment */ 8,
- /* data bytes */ 8 * elt_bytes,
- /* header bytes */ sizeof (h[0]),
- /* data align */ 0);
+ v = _vec_realloc (0, /* data bytes */ 8, elt_bytes,
+ /* header bytes */ sizeof (h[0]), /* data align */ 0,
+ /* heap */ 0);
/* Make space for invalid entry (entry 0). */
_vec_len (v) = 1;
#endif
}
+static_always_inline void *
+clib_memmove (void *dst, const void *src, size_t n)
+{
+ u8 *d = (u8 *) dst;
+ u8 *s = (u8 *) src;
+
+ if (s == d)
+ return d;
+
+ if (d > s)
+ for (uword i = n - 1; (i + 1) > 0; i--)
+ d[i] = s[i];
+ else
+ for (uword i = 0; i < n; i++)
+ d[i] = s[i];
+
+ return d;
+}
+
#include <vppinfra/memcpy.h>
/* c-11 string manipulation variants */
uword *objects = 0;
uword *handles = 0;
uword objects_used;
- uword align, fixed_size;
+ uword align;
clib_mem_init (0, 10 << 20);
n = 10;
seed = (u32) getpid ();
check_mask = 0;
- fixed_size = 0;
if (argc > 1)
{
objects_used = 0;
- if (fixed_size)
- {
- uword max_len = 1024 * 1024;
- void *memory = clib_mem_alloc (max_len * sizeof (h[0]));
- h = heap_create_from_memory (memory, max_len, sizeof (h[0]));
- }
-
for (i = 0; i < n; i++)
{
while (1)
vec_free (objects);
vec_free (handles);
- if (fixed_size)
- vec_free (h);
-
if (verbose)
fformat (stderr, "%U\n", format_clib_mem_usage, /* verbose */ 0);
if (_v (l) == ~0) \
_v (l) = bounded_random_u32 (&(seed), 0, MAX_VEC_LEN); \
\
- _v (v) = \
- _vec_resize (NULL, _v (l), _v (l) * sizeof (elt_type), _v (h), 0); \
+ _v (v) = _vec_realloc (NULL, _v (l), sizeof (elt_type), _v (h), 0, 0); \
fill_with_random_data (_v (v), vec_bytes (_v (v)), (seed)); \
\
/* Fill header with random data as well. */ \
return (uword) (clib_address_t) p;
}
+static inline __attribute__ ((always_inline)) uword
+pointer_is_aligned (void *p, uword align)
+{
+ if ((pointer_to_uword (p) & (align - 1)) == 0)
+ return 1;
+ return 0;
+}
+
#define uword_to_pointer(u,type) ((type) (clib_address_t) (u))
/* Any type: can be either word or pointer. */
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2022 Cisco Systems, Inc.
*/
-/*
- Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
#include <vppinfra/vec.h>
#include <vppinfra/mem.h>
#define CLIB_VECTOR_GROW_BY_ONE 0
#endif
-/* Vector resize operator. Called as needed by various macros such as
- vec_add1() when we need to allocate memory. */
-__clib_export void *
-vec_resize_allocate_memory (void *v, word length_increment, uword data_bytes,
- uword header_bytes, uword data_align)
+__clib_export uword
+vec_mem_size (void *v)
{
- vec_header_t *vh = _vec_find (v);
- uword old_alloc_bytes, new_alloc_bytes;
- void *old, *new;
-
- header_bytes = vec_header_bytes (header_bytes);
- data_align = data_align == 0 ? 1 : data_align;
+ return v ? clib_mem_size (v - vec_get_header_size (v)) : 0;
+}
- data_bytes += header_bytes;
+__clib_export void *
+_vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align,
+ void *heap)
+{
+ uword n_data_bytes, data_offset, new_data_size, alloc_size;
+ void *p;
/* alignment must be power of 2 */
- ASSERT (count_set_bits (data_align) == 1);
-
- if (!v)
- {
- new = clib_mem_alloc_aligned_at_offset (data_bytes, data_align, header_bytes, 1 /* yes, call os_out_of_memory */
- );
- new_alloc_bytes = clib_mem_size (new);
- CLIB_MEM_UNPOISON (new + data_bytes, new_alloc_bytes - data_bytes);
- clib_memset (new, 0, new_alloc_bytes);
- CLIB_MEM_POISON (new + data_bytes, new_alloc_bytes - data_bytes);
- v = new + header_bytes;
- _vec_len (v) = length_increment;
- ASSERT (header_bytes / VEC_HEADER_ROUND <= 255);
- _vec_find (v)->hdr_size = header_bytes / VEC_HEADER_ROUND;
- _vec_find (v)->log2_align = min_log2 (data_align);
- return v;
- }
-
- ASSERT (_vec_find (v)->hdr_size * VEC_HEADER_ROUND == header_bytes);
- header_bytes = _vec_find (v)->hdr_size * VEC_HEADER_ROUND;
+ align = clib_max (align, VEC_MIN_ALIGN);
+ ASSERT (count_set_bits (align) == 1);
- ASSERT (data_align == (1 << _vec_find (v)->log2_align));
- data_align = 1 << _vec_find (v)->log2_align;
+ /* number of bytes needed to store both vector header and optional user
+ * header */
+ data_offset = round_pow2 (hdr_sz + sizeof (vec_header_t), align);
- vh->len += length_increment;
- old = v - header_bytes;
+ /* mumber of bytes needed to store vector data */
+ n_data_bytes = n_elts * elt_sz;
- /* Vector header must start heap object. */
- ASSERT (clib_mem_is_heap_object (old));
+ /* minimal allocation needed to store data and headers */
+ new_data_size = data_offset + n_data_bytes;
- old_alloc_bytes = clib_mem_size (old);
-
- /* Need to resize? */
- if (data_bytes <= old_alloc_bytes)
+ if (v)
{
- CLIB_MEM_UNPOISON (v, data_bytes);
- return v;
+ uword old_data_size = data_offset + _vec_len (v) * elt_sz;
+ p = vec_header (v);
+ alloc_size = clib_mem_size (p);
+
+ /* check that we are still dealing with the same vector type */
+ ASSERT (_vec_find (v)->hdr_size * VEC_MIN_ALIGN == data_offset);
+ ASSERT (_vec_find (v)->log2_align == min_log2 (align));
+
+ /* realloc if new size cannot fit into existing allocation */
+ if (alloc_size < new_data_size)
+ {
+ if (CLIB_VECTOR_GROW_BY_ONE)
+ alloc_size = n_data_bytes + data_offset;
+ else
+ alloc_size = (n_data_bytes * 3) / 2 + data_offset;
+
+ p = clib_mem_realloc_aligned (p, alloc_size, align);
+ alloc_size = clib_mem_size (p);
+ v = p + data_offset;
+ }
+
+ CLIB_MEM_UNPOISON (p, alloc_size);
+ clib_memset_u8 (p + old_data_size, 0, alloc_size - old_data_size);
+ }
+ else
+ {
+ /* new allocation */
+ p = clib_mem_alloc_aligned (new_data_size, align);
+ alloc_size = clib_mem_size (p);
+ CLIB_MEM_UNPOISON (p, alloc_size);
+ clib_memset_u8 (p, 0, alloc_size);
+ v = p + data_offset;
+ _vec_find (v)->hdr_size = data_offset / VEC_MIN_ALIGN;
+ _vec_find (v)->log2_align = min_log2 (align);
}
-#if CLIB_VECTOR_GROW_BY_ONE > 0
- new_alloc_bytes = data_bytes;
-#else
- new_alloc_bytes = (old_alloc_bytes * 3) / 2;
- if (new_alloc_bytes < data_bytes)
- new_alloc_bytes = data_bytes;
-#endif
-
- new =
- clib_mem_alloc_aligned_at_offset (new_alloc_bytes, data_align,
- header_bytes,
- 1 /* yes, call os_out_of_memory */ );
-
- /* FIXME fail gracefully. */
- if (!new)
- clib_panic
- ("vec_resize fails, length increment %d, data bytes %d, alignment %d",
- length_increment, data_bytes, data_align);
-
- CLIB_MEM_UNPOISON (old, old_alloc_bytes);
- clib_memcpy_fast (new, old, old_alloc_bytes);
- clib_mem_free (old);
-
- /* Allocator may give a bit of extra room. */
- new_alloc_bytes = clib_mem_size (new);
- v = new;
-
- /* Zero new memory. */
- CLIB_MEM_UNPOISON (new + data_bytes, new_alloc_bytes - data_bytes);
- memset (v + old_alloc_bytes, 0, new_alloc_bytes - old_alloc_bytes);
- CLIB_MEM_POISON (new + data_bytes, new_alloc_bytes - data_bytes);
-
- return v + header_bytes;
+ CLIB_MEM_POISON (p + new_data_size, alloc_size - new_data_size);
+ _vec_len (v) = n_elts;
+ return v;
}
__clib_export u32
{
vec_free (v);
}
-
-/** \cond */
-
-#ifdef TEST
-
-#include <stdio.h>
-
-void
-main (int argc, char *argv[])
-{
- word n = atoi (argv[1]);
- word i, *x = 0;
-
- typedef struct
- {
- word x, y, z;
- } FOO;
-
- FOO *foos = vec_init (FOO, 10), *f;
-
- vec_validate (foos, 100);
- foos[100].x = 99;
-
- _vec_len (foos) = 0;
- for (i = 0; i < n; i++)
- {
- vec_add1 (x, i);
- vec_add2 (foos, f, 1);
- f->x = 2 * i;
- f->y = 3 * i;
- f->z = 4 * i;
- }
-
- {
- word n = 2;
- word m = 42;
- vec_delete (foos, n, m);
- }
-
- {
- word n = 2;
- word m = 42;
- vec_insert (foos, n, m);
- }
-
- vec_free (x);
- vec_free (foos);
- exit (0);
-}
-#endif
-/** \endcond */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
The memory layout looks like this:
~~~~~~~~
- user header (aligned to uword boundary)
- vector length: number of elements
+ user header (start of memory allocation)
+ padding
+ vector header: number of elements, header size
user's pointer-> vector element #0
- vector element #1
- ...
+ vector element #1
+ ...
~~~~~~~~
The user pointer contains the address of vector element # 0. Null
Typically, the header is not present. Headers allow for other
data structures to be built atop CLIB vectors.
- Users may specify the alignment for first data element of a vector
- via the vec_*_aligned macros.
+ While users may specify the alignment for first data element of a vector
+ via the vec_*_aligned macros that is typically not needed as alignment
+ is set based on native alignment of the data structure used.
Vector elements can be any C type e.g. (int, double, struct bar).
This is also true for data types built atop vectors (e.g. heap,
which are invariant.
*/
-/** \brief Low-level resize allocation function, usually not called directly
+/** \brief Low-level (re)allocation function, usually not called directly
@param v pointer to a vector
- @param length_increment length increment in elements
- @param data_bytes requested size in bytes
- @param header_bytes header size in bytes (may be zero)
- @param data_align alignment (may be zero)
+ @param n_elts requested number of elements
+ @param elt_sz requested size of one element
+ @param hdr_sz header size in bytes (may be zero)
+ @param align alignment (may be zero)
@return v_prime pointer to resized vector, may or may not equal v
*/
-void *vec_resize_allocate_memory (void *v, word length_increment,
- uword data_bytes, uword header_bytes,
- uword data_align);
+void *_vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz,
+ uword align, void *heap);
-/** \brief Low-level vector resize function, usually not called directly
+/* calculate minimum alignment out of data natural alignment and provided
+ * value, should not be < VEC_MIN_ALIGN */
+static_always_inline uword
+__vec_align (uword data_align, uword configuered_align)
+{
+ data_align = clib_max (data_align, configuered_align);
+ ASSERT (count_set_bits (data_align) == 1);
+ return clib_max (VEC_MIN_ALIGN, data_align);
+}
- @param v pointer to a vector
- @param length_increment length increment in elements
- @param data_bytes requested size in bytes
- @param header_bytes header size in bytes (may be zero)
- @param data_align alignment (may be zero)
- @return v_prime pointer to resized vector, may or may not equal v
-*/
+/* function used t o catch cases where vec_* macros on used on void * */
+static_always_inline uword
+__vec_elt_sz (uword elt_sz, int is_void)
+{
+ /* vector macro operations on void * are not allowed */
+ ASSERT (is_void == 0);
+ return elt_sz;
+}
-#define _vec_resize(V, L, DB, HB, A) \
- ({ \
- __typeof__ ((V)) _V; \
- _V = _vec_resize_inline ((void *) V, L, DB, HB, \
- clib_max ((__alignof__((V)[0])), (A))); \
- _V; \
- })
+static_always_inline void
+_vec_update_pointer (void **vp, void *v)
+{
+ /* avoid store if not needed */
+ if (v != vp[0])
+ vp[0] = v;
+}
always_inline void *
-_vec_resize_inline (void *v, word length_increment, uword data_bytes,
- uword header_bytes, uword data_align)
+_vec_realloc_inline (void *v, uword n_elts, uword elt_sz, uword hdr_sz,
+ uword align, void *heap)
{
- vec_header_t *vh = _vec_find (v);
- uword new_data_bytes, aligned_header_bytes;
-
- aligned_header_bytes = vec_header_bytes (header_bytes);
-
- new_data_bytes = data_bytes + aligned_header_bytes;
-
if (PREDICT_TRUE (v != 0))
{
- void *p = v - aligned_header_bytes;
-
/* Vector header must start heap object. */
- ASSERT (clib_mem_is_heap_object (p));
+ ASSERT (clib_mem_is_heap_object (vec_header (v)));
/* Typically we'll not need to resize. */
- if (new_data_bytes <= clib_mem_size (p))
+ if ((n_elts * elt_sz) <= vec_max_bytes (v))
{
- CLIB_MEM_UNPOISON (v, data_bytes);
- vh->len += length_increment;
+ _vec_set_len (v, n_elts, elt_sz);
return v;
}
}
/* Slow path: call helper function. */
- return vec_resize_allocate_memory (
- v, length_increment, data_bytes, header_bytes,
- clib_max (sizeof (vec_header_t), data_align));
+ return _vec_realloc (v, n_elts, elt_sz, hdr_sz, align, heap);
}
-/** \brief Determine if vector will resize with next allocation
-
- @param v pointer to a vector
- @param length_increment length increment in elements
- @param data_bytes requested size in bytes
- @param header_bytes header size in bytes (may be zero)
- @param data_align alignment (may be zero)
- @return 1 if vector will resize 0 otherwise
-*/
-
always_inline int
-_vec_resize_will_expand (void *v, uword n_elts, uword elt_size)
+_vec_resize_will_expand (void *v, uword n_elts, uword elt_sz)
{
- if (PREDICT_TRUE (v != 0))
- {
- /* Vector header must start heap object. */
- ASSERT (clib_mem_is_heap_object (vec_header (v)));
+ if (v == 0)
+ return 1;
+
+ /* Vector header must start heap object. */
+ ASSERT (clib_mem_is_heap_object (vec_header (v)));
+
+ n_elts += _vec_len (v);
+ if ((n_elts * elt_sz) <= vec_max_bytes (v))
+ return 0;
- if (vec_mem_size (v) >= ((_vec_len (v) + n_elts)) * elt_size)
- return 0;
- }
return 1;
}
*/
#define vec_resize_will_expand(V, N) \
- _vec_resize_will_expand (V, N, sizeof ((V)[0]))
+ _vec_resize_will_expand (V, N, _vec_elt_sz (V))
/* Local variable naming macro (prevents collisions with other macro naming). */
#define _v(var) _vec_##var
@return V (value-result macro parameter)
*/
+static_always_inline void
+_vec_resize (void **vp, uword n_add, uword hdr_sz, uword align, uword elt_sz)
+{
+ void *v = vp[0];
+ v = _vec_realloc_inline (v, vec_len (v) + n_add, elt_sz, hdr_sz, align, 0);
+ _vec_update_pointer (vp, v);
+}
+
#define vec_resize_ha(V, N, H, A) \
- do \
- { \
- word _v (n) = (N); \
- word _v (l) = vec_len (V); \
- V = _vec_resize ((V), _v (n), (_v (l) + _v (n)) * sizeof ((V)[0]), (H), \
- (A)); \
- } \
- while (0)
+ _vec_resize ((void **) &(V), N, H, _vec_align (V, A), _vec_elt_sz (V))
/** \brief Resize a vector (no header, unspecified alignment)
Add N elements to end of given vector V, return pointer to start of vector.
@return V (value-result macro parameter)
*/
-#define vec_alloc_ha(V,N,H,A) \
-do { \
- uword _v(l) = vec_len (V); \
- vec_resize_ha (V, N, H, A); \
- _vec_len (V) = _v(l); \
-} while (0)
+#define vec_alloc_ha(V, N, H, A) \
+ do \
+ { \
+ uword _v (l) = vec_len (V); \
+ vec_resize_ha (V, N, H, A); \
+ vec_set_len (V, _v (l)); \
+ } \
+ while (0)
/** \brief Allocate space for N more elements
(no header, unspecified alignment)
@param A alignment (may be zero)
@return V new vector
*/
-#define vec_new_ha(T,N,H,A) \
-({ \
- word _v(n) = (N); \
- (T *)_vec_resize ((T *) 0, _v(n), _v(n) * sizeof (T), (H), (A)); \
-})
+#define vec_new_ha(T, N, H, A) \
+ _vec_realloc (0, N, sizeof (T), H, _vec_align ((T *) 0, A), 0)
/** \brief Create new vector of given type and length
(unspecified alignment, no header).
@param V pointer to a vector
@return V (value-result parameter, V=0)
*/
-#define vec_free(V) \
- do \
- { \
- if (V) \
- { \
- clib_mem_free (vec_header ((V))); \
- V = 0; \
- } \
- } \
- while (0)
+
+static_always_inline void
+_vec_free (void **vp)
+{
+ if (vp[0] == 0)
+ return;
+ clib_mem_free (vec_header (vp[0]));
+ vp[0] = 0;
+}
+
+#define vec_free(V) _vec_free ((void **) &(V))
void vec_free_not_inline (void *v);
@return Vdup copy of vector
*/
+static_always_inline void *
+_vec_dup (void *v, uword hdr_size, uword align, uword elt_sz)
+{
+ uword len = vec_len (v);
+ void *n = 0;
+
+ if (len)
+ {
+ n = _vec_realloc (0, len, elt_sz, hdr_size, align, 0);
+ clib_memcpy_fast (n, v, len * elt_sz);
+ }
+ return n;
+}
+
#define vec_dup_ha(V, H, A) \
- ({ \
- __typeof__ ((V)[0]) *_v (v) = 0; \
- uword _v (l) = vec_len (V); \
- if (_v (l) > 0) \
- { \
- vec_resize_ha (_v (v), _v (l), (H), (A)); \
- clib_memcpy_fast (_v (v), (V), _v (l) * sizeof ((V)[0])); \
- } \
- _v (v); \
- })
+ _vec_dup ((void *) (V), H, _vec_align (V, A), _vec_elt_sz (V))
/** \brief Return copy of vector (no header, no alignment)
@param NEW_V pointer to new vector
@param OLD_V pointer to old vector
*/
-#define vec_clone(NEW_V,OLD_V) \
-do { \
- (NEW_V) = 0; \
- (NEW_V) = _vec_resize ((NEW_V), vec_len (OLD_V), \
- vec_len (OLD_V) * sizeof ((NEW_V)[0]), (0), (0)); \
-} while (0)
+
+static_always_inline void
+_vec_clone (void **v1p, void *v2, uword align, uword elt_sz)
+{
+ v1p[0] = _vec_realloc (0, vec_len (v2), elt_sz, 0, align, 0);
+}
+#define vec_clone(NEW_V, OLD_V) \
+ _vec_clone ((void **) &(NEW_V), OLD_V, _vec_align (NEW_V, 0), \
+ _vec_elt_sz (NEW_V))
/** \brief Make sure vector is long enough for given index (general version).
@return V (value-result macro parameter)
*/
+always_inline void
+_vec_zero_elts (void *v, uword first, uword count, uword elt_sz)
+{
+ clib_memset_u8 (v + (first * elt_sz), 0, count * elt_sz);
+}
+#define vec_zero_elts(V, F, C) _vec_zero_elts (V, F, C, sizeof ((V)[0]))
+
+static_always_inline void *
+_vec_validate_ha (void *v, uword index, uword header_size, uword align,
+ uword elt_sz)
+{
+ uword vl = vec_len (v);
+ if (index >= vl)
+ {
+ v = _vec_realloc_inline (v, index + 1, elt_sz, header_size, align, 0);
+ _vec_zero_elts (v, vl, index - vl + 1, elt_sz);
+ }
+ return v;
+}
+
#define vec_validate_ha(V, I, H, A) \
- do \
- { \
- STATIC_ASSERT (A == 0 || ((A % sizeof (V[0])) == 0) || \
- ((sizeof (V[0]) % A) == 0), \
- "vector validate aligned on incorrectly sized object"); \
- word _v (i) = (I); \
- word _v (l) = vec_len (V); \
- if (_v (i) >= _v (l)) \
- { \
- vec_resize_ha ((V), 1 + (_v (i) - _v (l)), (H), (A)); \
- /* Must zero new space since user may have previously \
- used e.g. _vec_len (v) -= 10 */ \
- clib_memset ((V) + _v (l), 0, \
- (1 + (_v (i) - _v (l))) * sizeof ((V)[0])); \
- } \
- } \
- while (0)
+ (V) = \
+ _vec_validate_ha ((void *) (V), I, H, _vec_align (V, A), sizeof ((V)[0]))
/** \brief Make sure vector is long enough for given index
(no header, unspecified alignment)
@param A alignment (may be zero)
@return V (value-result macro parameter)
*/
-#define vec_validate_init_empty_ha(V,I,INIT,H,A) \
-do { \
- word _v(i) = (I); \
- word _v(l) = vec_len (V); \
- if (_v(i) >= _v(l)) \
- { \
- vec_resize_ha ((V), 1 + (_v(i) - _v(l)), (H), (A)); \
- while (_v(l) <= _v(i)) \
- { \
- (V)[_v(l)] = (INIT); \
- _v(l)++; \
- } \
- } \
-} while (0)
+#define vec_validate_init_empty_ha(V, I, INIT, H, A) \
+ do \
+ { \
+ word _v (i) = (I); \
+ word _v (l) = vec_len (V); \
+ if (_v (i) >= _v (l)) \
+ { \
+ vec_resize_ha (V, 1 + (_v (i) - _v (l)), H, A); \
+ while (_v (l) <= _v (i)) \
+ { \
+ (V)[_v (l)] = (INIT); \
+ _v (l)++; \
+ } \
+ } \
+ } \
+ while (0)
/** \brief Make sure vector is long enough for given index
and initialize empty space (no header, unspecified alignment)
@param A alignment (may be zero)
@return V (value-result macro parameter)
*/
-#define vec_add1_ha(V,E,H,A) \
-do { \
- word _v(l) = vec_len (V); \
- V = _vec_resize ((V), 1, (_v(l) + 1) * sizeof ((V)[0]), (H), (A)); \
- (V)[_v(l)] = (E); \
-} while (0)
+
+static_always_inline void *
+_vec_add1 (void **vp, uword hdr_sz, uword align, uword elt_sz)
+{
+ void *v = vp[0];
+ uword len = vec_len (v);
+ v = _vec_realloc_inline (v, len + 1, elt_sz, hdr_sz, align, 0);
+
+ _vec_update_pointer (vp, v);
+
+ return v + len * elt_sz;
+}
+
+#define vec_add1_ha(V, E, H, A) \
+ ((__typeof__ ((V)[0]) *) _vec_add1 ((void **) &(V), H, _vec_align (V, A), \
+ _vec_elt_sz (V)))[0] = (E)
/** \brief Add 1 element to end of vector (unspecified alignment).
@param A alignment (may be zero)
@return V and P (value-result macro parameters)
*/
-#define vec_add2_ha(V,P,N,H,A) \
-do { \
- word _v(n) = (N); \
- word _v(l) = vec_len (V); \
- V = _vec_resize ((V), _v(n), (_v(l) + _v(n)) * sizeof ((V)[0]), (H), (A)); \
- P = (V) + _v(l); \
-} while (0)
+
+static_always_inline void
+_vec_add2 (void **vp, void **pp, uword n_add, uword hdr_sz, uword align,
+ uword elt_sz)
+{
+ void *v = vp[0];
+ uword len = vec_len (vp[0]);
+ v = _vec_realloc_inline (v, len + n_add, elt_sz, hdr_sz, align, 0);
+ _vec_update_pointer (vp, v);
+ pp[0] = v + len * elt_sz;
+}
+
+#define vec_add2_ha(V, P, N, H, A) \
+ _vec_add2 ((void **) &(V), (void **) &(P), N, H, _vec_align (V, A), \
+ _vec_elt_sz (V))
/** \brief Add N elements to end of vector V,
return pointer to new elements in P. (no header, unspecified alignment)
@param A alignment (may be zero)
@return V (value-result macro parameter)
*/
+static_always_inline void
+_vec_add (void **vp, void *e, word n_add, uword hdr_sz, uword align,
+ uword elt_sz)
+{
+ void *v = vp[0];
+ uword len = vec_len (v);
+
+ ASSERT (n_add >= 0);
+
+ if (n_add < 1)
+ return;
+
+ v = _vec_realloc_inline (v, len + n_add, elt_sz, hdr_sz, align, 0);
+ clib_memcpy_fast (v + len * elt_sz, e, n_add * elt_sz);
+ _vec_update_pointer (vp, v);
+}
+
#define vec_add_ha(V, E, N, H, A) \
- do \
- { \
- word _v (n) = (N); \
- if (PREDICT_TRUE (_v (n) > 0)) \
- { \
- word _v (l) = vec_len (V); \
- V = _vec_resize ((V), _v (n), (_v (l) + _v (n)) * sizeof ((V)[0]), \
- (H), (A)); \
- clib_memcpy_fast ((V) + _v (l), (E), _v (n) * sizeof ((V)[0])); \
- } \
- } \
- while (0)
+ _vec_add ((void **) &(V), (void *) (E), N, H, _vec_align (V, A), \
+ _vec_elt_sz (V))
/** \brief Add N elements to end of vector V (no header, unspecified alignment)
@param V pointer to a vector
@return E element removed from the end of the vector
*/
-#define vec_pop(V) \
-({ \
- uword _v(l) = vec_len (V); \
- ASSERT (_v(l) > 0); \
- _v(l) -= 1; \
- _vec_len (V) = _v (l); \
- (V)[_v(l)]; \
-})
+#define vec_pop(V) \
+ ({ \
+ uword _v (l) = vec_len (V); \
+ __typeof__ ((V)[0]) _v (rv); \
+ ASSERT (_v (l) > 0); \
+ _v (l) -= 1; \
+ _v (rv) = (V)[_v (l)]; \
+ vec_set_len (V, _v (l)); \
+ (_v (rv)); \
+ })
/** \brief Set E to the last element of a vector, decrement vector length
@param V pointer to a vector
@param A alignment (may be zero)
@return V (value-result macro parameter)
*/
-#define vec_insert_init_empty_ha(V,N,M,INIT,H,A) \
-do { \
- word _v(l) = vec_len (V); \
- word _v(n) = (N); \
- word _v(m) = (M); \
- V = _vec_resize ((V), \
- _v(n), \
- (_v(l) + _v(n))*sizeof((V)[0]), \
- (H), (A)); \
- ASSERT (_v(m) <= _v(l)); \
- memmove ((V) + _v(m) + _v(n), \
- (V) + _v(m), \
- (_v(l) - _v(m)) * sizeof ((V)[0])); \
- clib_memset ((V) + _v(m), INIT, _v(n) * sizeof ((V)[0])); \
-} while (0)
+
+static_always_inline void
+_vec_insert (void **vp, uword n_insert, uword ins_pt, u8 init, uword hdr_sz,
+ uword align, uword elt_sz)
+{
+ void *v = vp[0];
+ uword len = vec_len (v);
+
+ ASSERT (ins_pt <= len);
+
+ v = _vec_realloc_inline (v, len + n_insert, elt_sz, hdr_sz, align, 0);
+ clib_memmove (v + elt_sz * (ins_pt + n_insert), v + ins_pt * elt_sz,
+ (len - ins_pt) * elt_sz);
+ _vec_zero_elts (v, ins_pt, n_insert, elt_sz);
+ _vec_update_pointer (vp, v);
+}
+
+#define vec_insert_init_empty_ha(V, N, M, INIT, H, A) \
+ _vec_insert ((void **) &(V), N, M, INIT, H, _vec_align (V, A), \
+ _vec_elt_sz (V))
/** \brief Insert N vector elements starting at element M,
initialize new elements to zero (general version)
@return V (value-result macro parameter)
*/
+static_always_inline void
+_vec_insert_elts (void **vp, void *e, uword n_insert, uword ins_pt,
+ uword hdr_sz, uword align, uword elt_sz)
+{
+ void *v = vp[0];
+ uword len = vec_len (v);
+
+ ASSERT (ins_pt <= len);
+
+ v = _vec_realloc_inline (v, len + n_insert, elt_sz, hdr_sz, align, 0);
+ clib_memmove (v + elt_sz * (ins_pt + n_insert), v + ins_pt * elt_sz,
+ (len - ins_pt) * elt_sz);
+ _vec_zero_elts (v, ins_pt, n_insert, elt_sz);
+ clib_memcpy_fast (v + ins_pt * elt_sz, e, n_insert * elt_sz);
+ _vec_update_pointer (vp, v);
+}
+
#define vec_insert_elts_ha(V, E, N, M, H, A) \
- do \
- { \
- word _v (n) = (N); \
- if (PREDICT_TRUE (_v (n) > 0)) \
- { \
- word _v (l) = vec_len (V); \
- word _v (m) = (M); \
- V = _vec_resize ((V), _v (n), (_v (l) + _v (n)) * sizeof ((V)[0]), \
- (H), (A)); \
- ASSERT (_v (m) <= _v (l)); \
- memmove ((V) + _v (m) + _v (n), (V) + _v (m), \
- (_v (l) - _v (m)) * sizeof ((V)[0])); \
- clib_memcpy_fast ((V) + _v (m), (E), _v (n) * sizeof ((V)[0])); \
- } \
- } \
- while (0)
+ _vec_insert_elts ((void **) &(V), E, N, M, H, _vec_align (V, A), \
+ _vec_elt_sz (V))
/** \brief Insert N vector elements starting at element M,
insert given elements (no header, unspecified alignment)
@param M first element to delete
@return V (value-result macro parameter)
*/
-#define vec_delete(V,N,M) \
-do { \
- word _v(l) = vec_len (V); \
- word _v(n) = (N); \
- word _v(m) = (M); \
- /* Copy over deleted elements. */ \
- if (_v(l) - _v(n) - _v(m) > 0) \
- memmove ((V) + _v(m), (V) + _v(m) + _v(n), \
- (_v(l) - _v(n) - _v(m)) * sizeof ((V)[0])); \
- /* Zero empty space at end (for future re-allocation). */ \
- if (_v(n) > 0) \
- clib_memset ((V) + _v(l) - _v(n), 0, _v(n) * sizeof ((V)[0])); \
- _vec_len (V) -= _v(n); \
- CLIB_MEM_POISON(vec_end(V), _v(n) * sizeof ((V)[0])); \
-} while (0)
+
+static_always_inline void
+_vec_delete (void *v, uword n_del, uword first, uword elt_sz)
+{
+ word n_bytes_del, n_bytes_to_move, len = vec_len (v);
+ u8 *dst;
+
+ if (n_del == 0)
+ return;
+
+ ASSERT (first + n_del <= len);
+
+ n_bytes_del = n_del * elt_sz;
+ n_bytes_to_move = (len - first - n_del) * elt_sz;
+ dst = v + first * elt_sz;
+
+ if (n_bytes_to_move > 0)
+ clib_memmove (dst, dst + n_bytes_del, n_bytes_to_move);
+ clib_memset (dst + n_bytes_to_move, 0, n_bytes_del);
+
+ _vec_set_len (v, _vec_len (v) - n_del, elt_sz);
+}
+
+#define vec_delete(V, N, M) _vec_delete ((void *) (V), N, M, _vec_elt_sz (V))
/** \brief Delete the element at index I
@param V pointer to a vector
@param I index to delete
*/
-#define vec_del1(v,i) \
-do { \
- uword _vec_del_l = _vec_len (v) - 1; \
- uword _vec_del_i = (i); \
- if (_vec_del_i < _vec_del_l) \
- (v)[_vec_del_i] = (v)[_vec_del_l]; \
- _vec_len (v) = _vec_del_l; \
- CLIB_MEM_POISON(vec_end(v), sizeof ((v)[0])); \
-} while (0)
-/** \brief Append v2 after v1. Result in v1.
- @param V1 target vector
- @param V2 vector to append
-*/
+static_always_inline void
+_vec_del1 (void *v, uword index, uword elt_sz)
+{
+ uword len = _vec_len (v) - 1;
-#define vec_append(v1, v2) \
- do \
- { \
- uword _v (l1) = vec_len (v1); \
- uword _v (l2) = vec_len (v2); \
- \
- if (PREDICT_TRUE (_v (l2) > 0)) \
- { \
- v1 = _vec_resize ((v1), _v (l2), \
- (_v (l1) + _v (l2)) * sizeof ((v1)[0]), 0, 0); \
- clib_memcpy_fast ((v1) + _v (l1), (v2), \
- _v (l2) * sizeof ((v2)[0])); \
- } \
- } \
- while (0)
+ if (index < len)
+ clib_memcpy_fast (v + index * elt_sz, v + len * elt_sz, elt_sz);
+
+ _vec_set_len (v, len, elt_sz);
+}
+
+#define vec_del1(v, i) _vec_del1 ((void *) (v), i, _vec_elt_sz (v))
+
+static_always_inline void
+_vec_append (void **v1p, void *v2, uword v1_elt_sz, uword v2_elt_sz,
+ uword align)
+{
+ void *v1 = v1p[0];
+ uword len1 = vec_len (v1);
+ uword len2 = vec_len (v2);
+
+ if (PREDICT_TRUE (len2 > 0))
+ {
+ v1 = _vec_realloc_inline (v1, len1 + len2, v2_elt_sz, 0, align, 0);
+ clib_memcpy_fast (v1 + len1 * v1_elt_sz, v2, len2 * v2_elt_sz);
+ _vec_update_pointer (v1p, v1);
+ }
+}
/** \brief Append v2 after v1. Result in v1. Specified alignment.
@param V1 target vector
*/
#define vec_append_aligned(v1, v2, align) \
- do \
- { \
- uword _v (l1) = vec_len (v1); \
- uword _v (l2) = vec_len (v2); \
- \
- if (PREDICT_TRUE (_v (l2) > 0)) \
- { \
- v1 = _vec_resize ( \
- (v1), _v (l2), (_v (l1) + _v (l2)) * sizeof ((v1)[0]), 0, align); \
- clib_memcpy_fast ((v1) + _v (l1), (v2), \
- _v (l2) * sizeof ((v2)[0])); \
- } \
- } \
- while (0)
+ _vec_append ((void **) &(v1), (void *) (v2), _vec_elt_sz (v1), \
+ _vec_elt_sz (v2), _vec_align (v1, align))
-/** \brief Prepend v2 before v1. Result in v1.
+/** \brief Append v2 after v1. Result in v1.
@param V1 target vector
- @param V2 vector to prepend
+ @param V2 vector to append
*/
-#define vec_prepend(v1, v2) \
- do \
- { \
- uword _v (l1) = vec_len (v1); \
- uword _v (l2) = vec_len (v2); \
- \
- if (PREDICT_TRUE (_v (l2) > 0)) \
- { \
- v1 = _vec_resize ((v1), _v (l2), \
- (_v (l1) + _v (l2)) * sizeof ((v1)[0]), 0, 0); \
- memmove ((v1) + _v (l2), (v1), _v (l1) * sizeof ((v1)[0])); \
- clib_memcpy_fast ((v1), (v2), _v (l2) * sizeof ((v2)[0])); \
- } \
- } \
- while (0)
+#define vec_append(v1, v2) vec_append_aligned (v1, v2, 0)
+
+static_always_inline void
+_vec_prepend (void **v1p, void *v2, uword v1_elt_sz, uword v2_elt_sz,
+ uword align)
+{
+ void *v1 = v1p[0];
+ uword len1 = vec_len (v1);
+ uword len2 = vec_len (v2);
+
+ if (PREDICT_TRUE (len2 > 0))
+ {
+ v1 = _vec_realloc_inline (v1, len1 + len2, v2_elt_sz, 0, align, 0);
+ clib_memmove (v1 + len2 * v2_elt_sz, v1p[0], len1 * v1_elt_sz);
+ clib_memcpy_fast (v1, v2, len2 * v2_elt_sz);
+ _vec_update_pointer (v1p, v1);
+ }
+}
/** \brief Prepend v2 before v1. Result in v1. Specified alignment
@param V1 target vector
*/
#define vec_prepend_aligned(v1, v2, align) \
- do \
- { \
- uword _v (l1) = vec_len (v1); \
- uword _v (l2) = vec_len (v2); \
- \
- if (PREDICT_TRUE (_v (l2) > 0)) \
- { \
- v1 = _vec_resize ( \
- (v1), _v (l2), (_v (l1) + _v (l2)) * sizeof ((v1)[0]), 0, align); \
- memmove ((v1) + _v (l2), (v1), _v (l1) * sizeof ((v1)[0])); \
- clib_memcpy_fast ((v1), (v2), _v (l2) * sizeof ((v2)[0])); \
- } \
- } \
- while (0)
+ _vec_prepend ((void **) &(v1), (void *) (v2), _vec_elt_sz (v1), \
+ _vec_elt_sz (v2), _vec_align (v1, align))
+
+/** \brief Prepend v2 before v1. Result in v1.
+ @param V1 target vector
+ @param V2 vector to prepend
+*/
+
+#define vec_prepend(v1, v2) vec_prepend_aligned (v1, v2, 0)
/** \brief Zero all vector elements. Null-pointer tolerant.
@param var Vector to zero
*/
-#define vec_zero(var) \
-do { \
- if (var) \
- clib_memset ((var), 0, vec_len (var) * sizeof ((var)[0])); \
-} while (0)
+static_always_inline void
+_vec_zero (void *v, uword elt_sz)
+{
+ uword len = vec_len (v);
+
+ if (len)
+ clib_memset_u8 (v, 0, len * elt_sz);
+}
+
+#define vec_zero(var) _vec_zero ((void *) (var), _vec_elt_sz (var))
/** \brief Set all vector elements to given value. Null-pointer tolerant.
@param v vector to set
@param v2 Pointer to a vector
@return 1 if equal, 0 if unequal
*/
-#define vec_is_equal(v1,v2) \
- (vec_len (v1) == vec_len (v2) && ! memcmp ((v1), (v2), vec_len (v1) * sizeof ((v1)[0])))
+static_always_inline int
+_vec_is_equal (void *v1, void *v2, uword v1_elt_sz, uword v2_elt_sz)
+{
+ uword vec_len_v1 = vec_len (v1);
+
+ if ((vec_len_v1 != vec_len (v2)) || (v1_elt_sz != v2_elt_sz))
+ return 0;
+
+ if ((vec_len_v1 == 0) || (memcmp (v1, v2, vec_len_v1 * v1_elt_sz) == 0))
+ return 1;
+
+ return 0;
+}
+
+#define vec_is_equal(v1, v2) \
+ _vec_is_equal ((void *) (v1), (void *) (v2), _vec_elt_sz (v1), \
+ _vec_elt_sz (v2))
/** \brief Compare two vectors (only applicable to vectors of signed numbers).
Used in qsort compare functions.
@param S pointer to string buffer.
@param L string length (NOT including the terminating NULL; a la strlen())
*/
-#define vec_validate_init_c_string(V, S, L) \
- do { \
- vec_reset_length (V); \
- vec_validate ((V), (L)); \
- if ((S) && (L)) \
- clib_memcpy_fast ((V), (S), (L)); \
- (V)[(L)] = 0; \
- } while (0)
-
+#define vec_validate_init_c_string(V, S, L) \
+ do \
+ { \
+ vec_reset_length (V); \
+ vec_validate (V, (L)); \
+ if ((S) && (L)) \
+ clib_memcpy_fast (V, (S), (L)); \
+ (V)[(L)] = 0; \
+ } \
+ while (0)
/** \brief Test whether a vector is a NULL terminated c-string.
@param V (possibly NULL) pointer to a vector.
@return V (value-result macro parameter)
*/
-#define vec_terminate_c_string(V) \
- do { \
- u32 vl = vec_len ((V)); \
- if (!vec_c_string_is_terminated(V)) \
- { \
- vec_validate ((V), vl); \
- (V)[vl] = 0; \
- } \
- } while (0)
+#define vec_terminate_c_string(V) \
+ do \
+ { \
+ if (!vec_c_string_is_terminated (V)) \
+ vec_add1 (V, 0); \
+ } \
+ while (0)
#endif /* included_vec_h */
-
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
typedef struct
{
u32 len; /**< Number of elements in vector (NOT its allocated length). */
- u8 hdr_size; /**< header size divided by VEC_HEADER_ROUND */
+ u8 hdr_size; /**< header size divided by VEC_MIN_ALIGN */
u8 log2_align; /**< data alignment */
u8 vpad[2]; /**< pad to 8 bytes */
u8 vector_data[0]; /**< Vector data . */
} vec_header_t;
-#define VEC_HEADER_ROUND 8
+#define VEC_MIN_ALIGN 8
/** \brief Find the vector header
*/
#define _vec_find(v) ((vec_header_t *) (v) - 1)
+always_inline uword __vec_align (uword data_align, uword configuered_align);
+always_inline uword __vec_elt_sz (uword elt_sz, int is_void);
+
#define _vec_round_size(s) \
(((s) + sizeof (uword) - 1) &~ (sizeof (uword) - 1))
+#define _vec_is_void(P) \
+ __builtin_types_compatible_p (__typeof__ ((P)[0]), void)
+#define _vec_elt_sz(V) __vec_elt_sz (sizeof ((V)[0]), _vec_is_void (V))
+#define _vec_align(V, A) __vec_align (__alignof__((V)[0]), A)
-always_inline uword
-vec_header_bytes (uword header_bytes)
-{
- return round_pow2 (header_bytes + sizeof (vec_header_t), VEC_HEADER_ROUND);
-}
-
-always_inline uword
+always_inline CLIB_NOSANITIZE_ADDR uword
vec_get_header_size (void *v)
{
- uword header_size = _vec_find (v)->hdr_size * VEC_HEADER_ROUND;
+ uword header_size = _vec_find (v)->hdr_size * VEC_MIN_ALIGN;
return header_size;
}
* @return memory size allocated for the vector
*/
-always_inline uword
-vec_mem_size (void *v)
-{
- return v ? clib_mem_size (v - vec_get_header_size (v)) : 0;
-}
+uword vec_mem_size (void *v);
/**
* Number of elements that can fit into generic vector
*/
always_inline uword
-_vec_max_len (void *v, uword elt_size)
+vec_max_bytes (void *v)
{
- return v ? vec_mem_size (v) / elt_size : 0;
+ return v ? vec_mem_size (v) - vec_get_header_size (v) : 0;
}
-#define vec_max_len(v) _vec_max_len (v, sizeof ((v)[0]))
+always_inline uword
+_vec_max_len (void *v, uword elt_sz)
+{
+ return vec_max_bytes (v) / elt_sz;
+}
+
+#define vec_max_len(v) _vec_max_len (v, _vec_elt_sz (v))
always_inline void
-_vec_set_len (void *v, uword len, uword elt_size)
+_vec_set_len (void *v, uword len, uword elt_sz)
{
ASSERT (v);
- ASSERT (len <= vec_max_len (v));
+ ASSERT (len <= _vec_max_len (v, elt_sz));
+ uword old_len = _vec_len (v);
+
+ if (len > old_len)
+ CLIB_MEM_UNPOISON (v + old_len * elt_sz, (len - old_len) * elt_sz);
+ else if (len > old_len)
+ CLIB_MEM_POISON (v + len * elt_sz, (old_len - len) * elt_sz);
- CLIB_MEM_POISON_LEN (v, _vec_len (v) * elt_size, len * elt_size);
_vec_len (v) = len;
}
-#define vec_set_len(v, l) _vec_set_len ((void *) v, l, sizeof ((v)[0]))
+#define vec_set_len(v, l) _vec_set_len ((void *) v, l, _vec_elt_sz (v))
/** \brief Reset vector length to zero
NULL-pointer tolerant
Code Review / vpp.git / commitdiff