#define __clib_unused __attribute__ ((unused))
#define __clib_weak __attribute__ ((weak))
#define __clib_packed __attribute__ ((packed))
+#define __clib_flatten __attribute__ ((flatten))
#define __clib_constructor __attribute__ ((constructor))
#define __clib_noinline __attribute__ ((noinline))
#ifdef __clang__
void *clib_mem_alloc_aligned_or_null (uword size, uword align);
void *clib_mem_realloc (void *p, uword new_size);
void *clib_mem_realloc_aligned (void *p, uword new_size, uword align);
+uword clib_mem_is_heap_object (void *p);
+void clib_mem_free (void *p);
+
+void *clib_mem_heap_alloc (void *heap, uword size);
+void *clib_mem_heap_alloc_aligned (void *heap, uword size, uword align);
+void *clib_mem_heap_alloc_or_null (void *heap, uword size);
+void *clib_mem_heap_alloc_aligned_or_null (void *heap, uword size,
+ uword align);
+void *clib_mem_heap_realloc (void *heap, void *p, uword new_size);
+void *clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size,
+ uword align);
+uword clib_mem_heap_is_heap_object (void *heap, void *p);
+void clib_mem_heap_free (void *heap, void *p);
+
+uword clib_mem_size (void *p);
+void clib_mem_free_s (void *p);
/* Memory allocator which panics when it fails.
Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
/* Alias to stack allocator for naming consistency. */
#define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
-uword clib_mem_is_heap_object (void *p);
-void clib_mem_free (void *p);
-uword clib_mem_size (void *p);
-void clib_mem_free_s (void *p);
-
always_inline clib_mem_heap_t *
clib_mem_get_heap (void)
{
return s;
}
-__clib_export void
-clib_mem_get_heap_usage (clib_mem_heap_t * heap, clib_mem_usage_t * usage)
+__clib_export __clib_flatten void
+clib_mem_get_heap_usage (clib_mem_heap_t *heap, clib_mem_usage_t *usage)
{
struct dlmallinfo mi = mspace_mallinfo (heap->mspace);
clib_mem_vm_unmap (h->base);
}
-__clib_export uword
-clib_mem_get_heap_free_space (clib_mem_heap_t * h)
+__clib_export __clib_flatten uword
+clib_mem_get_heap_free_space (clib_mem_heap_t *h)
{
struct dlmallinfo dlminfo = mspace_mallinfo (h->mspace);
return dlminfo.fordblks;
}
-__clib_export void *
-clib_mem_get_heap_base (clib_mem_heap_t * h)
+__clib_export __clib_flatten void *
+clib_mem_get_heap_base (clib_mem_heap_t *h)
{
return h->base;
}
-__clib_export uword
-clib_mem_get_heap_size (clib_mem_heap_t * heap)
+__clib_export __clib_flatten uword
+clib_mem_get_heap_size (clib_mem_heap_t *heap)
{
return heap->size;
}
/* Memory allocator which may call os_out_of_memory() if it fails */
-static void *
-clib_mem_alloc_inline (uword size, uword align,
- int os_out_of_memory_on_failure)
+static inline void *
+clib_mem_heap_alloc_inline (void *heap, uword size, uword align,
+ int os_out_of_memory_on_failure)
{
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
void *p;
align = clib_max (CLIB_MEM_MIN_ALIGN, align);
}
/* Memory allocator which calls os_out_of_memory() when it fails */
-__clib_export void *
+__clib_export __clib_flatten void *
clib_mem_alloc (uword size)
{
- return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN,
- /* os_out_of_memory */ 1);
+ return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN,
+ /* os_out_of_memory */ 1);
}
-__clib_export void *
+__clib_export __clib_flatten void *
clib_mem_alloc_aligned (uword size, uword align)
{
- return clib_mem_alloc_inline (size, align,
- /* os_out_of_memory */ 1);
+ return clib_mem_heap_alloc_inline (0, size, align,
+ /* os_out_of_memory */ 1);
}
/* Memory allocator which calls os_out_of_memory() when it fails */
-__clib_export void *
+__clib_export __clib_flatten void *
clib_mem_alloc_or_null (uword size)
{
- return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN,
- /* os_out_of_memory */ 0);
+ return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN,
+ /* os_out_of_memory */ 0);
}
-__clib_export void *
+__clib_export __clib_flatten void *
clib_mem_alloc_aligned_or_null (uword size, uword align)
{
- return clib_mem_alloc_inline (size, align,
- /* os_out_of_memory */ 0);
+ return clib_mem_heap_alloc_inline (0, size, align,
+ /* os_out_of_memory */ 0);
}
-__clib_export void *
-clib_mem_realloc_aligned (void *p, uword new_size, uword align)
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc (void *heap, uword size)
+{
+ return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN,
+ /* os_out_of_memory */ 1);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc_aligned (void *heap, uword size, uword align)
+{
+ return clib_mem_heap_alloc_inline (heap, size, align,
+ /* os_out_of_memory */ 1);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc_or_null (void *heap, uword size)
+{
+ return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN,
+ /* os_out_of_memory */ 0);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc_aligned_or_null (void *heap, uword size, uword align)
+{
+ return clib_mem_heap_alloc_inline (heap, size, align,
+ /* os_out_of_memory */ 0);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size,
+ uword align)
{
uword old_alloc_size;
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
void *new;
ASSERT (count_set_bits (align) == 1);
}
else
{
- new = clib_mem_alloc_inline (new_size, align, 1);
+ new = clib_mem_heap_alloc_inline (h, new_size, align, 1);
CLIB_MEM_UNPOISON (new, new_size);
if (old_alloc_size)
{
CLIB_MEM_UNPOISON (p, old_alloc_size);
clib_memcpy_fast (new, p, clib_min (new_size, old_alloc_size));
- clib_mem_free (p);
+ clib_mem_heap_free (h, p);
}
p = new;
}
return p;
}
-__clib_export void *
+__clib_export __clib_flatten void *
+clib_mem_heap_realloc (void *heap, void *p, uword new_size)
+{
+ return clib_mem_heap_realloc_aligned (heap, p, new_size, CLIB_MEM_MIN_ALIGN);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_realloc_aligned (void *p, uword new_size, uword align)
+{
+ return clib_mem_heap_realloc_aligned (0, p, new_size, align);
+}
+
+__clib_export __clib_flatten void *
clib_mem_realloc (void *p, uword new_size)
{
- return clib_mem_realloc_aligned (p, new_size, CLIB_MEM_MIN_ALIGN);
+ return clib_mem_heap_realloc_aligned (0, p, new_size, CLIB_MEM_MIN_ALIGN);
}
-__clib_export uword
-clib_mem_is_heap_object (void *p)
+__clib_export __clib_flatten uword
+clib_mem_heap_is_heap_object (void *heap, void *p)
{
- int mspace_is_heap_object (void *msp, void *p);
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
return mspace_is_heap_object (h->mspace, p);
}
-__clib_export void
-clib_mem_free (void *p)
+__clib_export __clib_flatten uword
+clib_mem_is_heap_object (void *p)
+{
+ return clib_mem_heap_is_heap_object (0, p);
+}
+
+__clib_export __clib_flatten void
+clib_mem_heap_free (void *heap, void *p)
{
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
uword size = clib_mem_size (p);
/* Make sure object is in the correct heap. */
- ASSERT (clib_mem_is_heap_object (p));
+ ASSERT (clib_mem_heap_is_heap_object (h, p));
if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED))
mheap_put_trace (pointer_to_uword (p), size);
mspace_free (h->mspace, p);
}
-__clib_export uword
+__clib_export __clib_flatten void
+clib_mem_free (void *p)
+{
+ clib_mem_heap_free (0, p);
+}
+
+__clib_export __clib_flatten uword
clib_mem_size (void *p)
{
return mspace_usable_size (p);
/** Allocate N more free elements to pool (general version). */
static_always_inline void
-_pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz)
+_pool_alloc (void **pp, uword n_elts, uword align, void *heap, uword elt_sz)
{
pool_header_t *ph = pool_header (pp[0]);
uword len = vec_len (pp[0]);
}
pp[0] = _vec_realloc_inline (pp[0], len + n_elts, elt_sz,
- sizeof (pool_header_t), align, 0);
+ sizeof (pool_header_t), align, heap);
_vec_len (pp[0]) = len;
CLIB_MEM_POISON (pp[0] + len * elt_sz, n_elts * elt_sz);
clib_bitmap_vec_validate (ph->free_bitmap, len + n_elts - 1);
}
-#define pool_alloc_aligned(P, N, A) \
- _pool_alloc ((void **) &(P), N, _vec_align (P, A), _vec_elt_sz (P))
+#define pool_alloc_aligned_heap(P, N, A, H) \
+ _pool_alloc ((void **) &(P), N, _vec_align (P, A), H, _vec_elt_sz (P))
-/** Allocate N more free elements to pool (unspecified alignment). */
-#define pool_alloc(P,N) pool_alloc_aligned(P,N,0)
+#define pool_alloc_heap(P, N, H) pool_alloc_aligned_heap (P, N, 0, H)
+#define pool_alloc_aligned(P, N, A) pool_alloc_aligned_heap (P, N, A, 0)
+#define pool_alloc(P, N) pool_alloc_aligned_heap (P, N, 0, 0)
static_always_inline void *
_pool_dup (void *p, uword align, uword elt_sz)
_vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align,
void *heap)
{
- uword n_data_bytes, data_offset, new_data_size, alloc_size;
+ uword n_data_bytes, alloc_size, new_data_size;
void *p;
/* alignment must be power of 2 */
align = clib_max (align, VEC_MIN_ALIGN);
ASSERT (count_set_bits (align) == 1);
- /* number of bytes needed to store both vector header and optional user
- * header */
- data_offset = round_pow2 (hdr_sz + sizeof (vec_header_t), align);
-
/* mumber of bytes needed to store vector data */
n_data_bytes = n_elts * elt_sz;
- /* minimal allocation needed to store data and headers */
- new_data_size = data_offset + n_data_bytes;
-
if (v)
{
+ uword data_offset = vec_get_header_size (v);
uword old_data_size = data_offset + _vec_len (v) * elt_sz;
+ new_data_size = data_offset + n_data_bytes;
+ heap = _vec_find (v)->default_heap ? 0 : _vec_heap (v);
p = vec_header (v);
alloc_size = clib_mem_size (p);
else
alloc_size = (n_data_bytes * 3) / 2 + data_offset;
- p = clib_mem_realloc_aligned (p, alloc_size, align);
+ p = clib_mem_heap_realloc_aligned (heap, p, alloc_size, align);
alloc_size = clib_mem_size (p);
v = p + data_offset;
}
else
{
/* new allocation */
- p = clib_mem_alloc_aligned (new_data_size, align);
+ uword data_offset = hdr_sz + sizeof (vec_header_t);
+ data_offset += heap ? sizeof (void *) : 0;
+ data_offset = round_pow2 (data_offset, align);
+
+ new_data_size = data_offset + n_data_bytes;
+ p = clib_mem_heap_alloc_aligned (heap, new_data_size, align);
alloc_size = clib_mem_size (p);
CLIB_MEM_UNPOISON (p, alloc_size);
clib_memset_u8 (p, 0, alloc_size);
v = p + data_offset;
_vec_find (v)->hdr_size = data_offset / VEC_MIN_ALIGN;
_vec_find (v)->log2_align = min_log2 (align);
+ if (heap)
+ {
+ _vec_find (v)->default_heap = 0;
+ _vec_heap (v) = heap;
+ }
+ else
+ _vec_find (v)->default_heap = 1;
}
CLIB_MEM_POISON (p + new_data_size, alloc_size - new_data_size);
~~~~~~~~
user header (start of memory allocation)
padding
+ heap pointer (optional, only if default_heap == 0)
vector header: number of elements, header size
user's pointer-> vector element #0
vector element #1
vp[0] = v;
}
-always_inline void *
+static_always_inline void *
+vec_get_heap (void *v)
+{
+ if (v == 0 || _vec_find (v)->default_heap == 1)
+ return 0;
+ return _vec_heap (v);
+}
+
+static_always_inline void *
_vec_realloc_inline (void *v, uword n_elts, uword elt_sz, uword hdr_sz,
uword align, void *heap)
{
if (PREDICT_TRUE (v != 0))
{
/* Vector header must start heap object. */
- ASSERT (clib_mem_is_heap_object (vec_header (v)));
+ ASSERT (clib_mem_heap_is_heap_object (vec_get_heap (v), vec_header (v)));
/* Typically we'll not need to resize. */
if ((n_elts * elt_sz) <= vec_max_bytes (v))
return _vec_realloc (v, n_elts, elt_sz, hdr_sz, align, heap);
}
+static_always_inline void
+_vec_prealloc (void **vp, uword n_elts, uword hdr_sz, uword align, void *heap,
+ uword elt_sz)
+{
+ void *v;
+
+ ASSERT (vp[0] == 0);
+
+ v = _vec_realloc (0, n_elts, elt_sz, hdr_sz, align, heap);
+ _vec_set_len (v, 0, elt_sz);
+ _vec_update_pointer (vp, v);
+}
+
+/** \brief Pre-allocate a vector (generic version)
+
+ @param V pointer to a vector
+ @param N number of elements to pre-allocate
+ @param H header size in bytes (may be zero)
+ @param A alignment (zero means default alignment of the data structure)
+ @param P heap (zero means default heap)
+ @return V (value-result macro parameter)
+*/
+
+#define vec_prealloc_hap(V, N, H, A, P) \
+ _vec_prealloc ((void **) &(V), N, H, _vec_align (V, A), P, _vec_elt_sz (V))
+
+/** \brief Pre-allocate a vector (simple version)
+
+ @param V pointer to a vector
+ @param N number of elements to pre-allocate
+ @return V (value-result macro parameter)
+*/
+#define vec_prealloc(V, N) vec_prealloc_hap (V, N, 0, 0, 0)
+
+/** \brief Pre-allocate a vector (heap version)
+
+ @param V pointer to a vector
+ @param N number of elements to pre-allocate
+ @param P heap (zero means default heap)
+ @return V (value-result macro parameter)
+*/
+#define vec_prealloc_heap(V, N, P) vec_prealloc_hap (V, N, 0, 0, P)
+
always_inline int
_vec_resize_will_expand (void *v, uword n_elts, uword elt_sz)
{
return 1;
/* Vector header must start heap object. */
- ASSERT (clib_mem_is_heap_object (vec_header (v)));
+ ASSERT (clib_mem_heap_is_heap_object (vec_get_heap (v), vec_header (v)));
n_elts += _vec_len (v);
if ((n_elts * elt_sz) <= vec_max_bytes (v))
}
#define vec_zero_elts(V, F, C) _vec_zero_elts (V, F, C, sizeof ((V)[0]))
-static_always_inline void *
-_vec_validate_ha (void *v, uword index, uword header_size, uword align,
- uword elt_sz)
+static_always_inline void
+_vec_validate (void **vp, uword index, uword header_size, uword align,
+ void *heap, uword elt_sz)
{
+ void *v = vp[0];
uword vl = vec_len (v);
if (index >= vl)
{
- v = _vec_realloc_inline (v, index + 1, elt_sz, header_size, align, 0);
+ v = _vec_realloc_inline (v, index + 1, elt_sz, header_size, align, heap);
_vec_zero_elts (v, vl, index - vl + 1, elt_sz);
+ _vec_update_pointer (vp, v);
}
- return v;
}
-#define vec_validate_ha(V, I, H, A) \
- (V) = \
- _vec_validate_ha ((void *) (V), I, H, _vec_align (V, A), sizeof ((V)[0]))
+#define vec_validate_hap(V, I, H, A, P) \
+ _vec_validate ((void **) &(V), I, H, _vec_align (V, A), 0, sizeof ((V)[0]))
/** \brief Make sure vector is long enough for given index
(no header, unspecified alignment)
@param I vector index which will be valid upon return
@return V (value-result macro parameter)
*/
-#define vec_validate(V,I) vec_validate_ha(V,I,0,0)
+#define vec_validate(V, I) vec_validate_hap (V, I, 0, 0, 0)
/** \brief Make sure vector is long enough for given index
(no header, specified alignment)
@return V (value-result macro parameter)
*/
-#define vec_validate_aligned(V,I,A) vec_validate_ha(V,I,0,A)
+#define vec_validate_aligned(V, I, A) vec_validate_hap (V, I, 0, A, 0)
+
+/** \brief Make sure vector is long enough for given index
+ (no header, specified heap)
+
+ @param V (possibly NULL) pointer to a vector.
+ @param I vector index which will be valid upon return
+ @param H heap (may be zero)
+ @return V (value-result macro parameter)
+*/
+
+#define vec_validate_heap(V, I, P) vec_validate_hap (V, I, 0, 0, P)
/** \brief Make sure vector is long enough for given index
and initialize empty space (general version)
{
u32 len; /**< Number of elements in vector (NOT its allocated length). */
u8 hdr_size; /**< header size divided by VEC_MIN_ALIGN */
- u8 log2_align; /**< data alignment */
+ u8 log2_align : 7; /**< data alignment */
+ u8 default_heap : 1; /**< vector uses default heap */
u8 vpad[2]; /**< pad to 8 bytes */
u8 vector_data[0]; /**< Vector data . */
} vec_header_t;
@return pointer to the vector's vector_header_t
*/
#define _vec_find(v) ((vec_header_t *) (v) - 1)
+#define _vec_heap(v) (((void **) (_vec_find (v)))[-1])
always_inline uword __vec_align (uword data_align, uword configuered_align);
always_inline uword __vec_elt_sz (uword elt_sz, int is_void);
Code Review / vpp.git / commitdiff