/** Vector of free indices. One element for each set bit in bitmap. */
u32 *free_indices;
+
+ /* The following fields are set for fixed-size, preallocated pools */
+
+ /** Maximum size of the pool, in elements */
+ u32 max_elts;
+
+ /** mmap segment info: base + length */
+ u8 *mmap_base;
+ u64 mmap_size;
+
} pool_header_t;
/** Align pool header so that pointers are naturally aligned. */
return vec_aligned_header (v, sizeof (pool_header_t), sizeof (void *));
}
+extern void _pool_init_fixed (void **, u32, u32);
+extern void fpool_free (void *);
+
+/** initialize a fixed-size, preallocated pool */
+#define pool_init_fixed(pool,max_elts) \
+{ \
+ _pool_init_fixed((void **)&(pool),sizeof(pool[0]),max_elts); \
+}
+
/** Validate a pool */
always_inline void
pool_validate (void *v)
do { \
uword __pool_validate_index = (i); \
vec_validate_ha ((v), __pool_validate_index, \
- pool_aligned_header_bytes, /* align */ 0); \
+ pool_aligned_header_bytes, /* align */ 0); \
pool_header_validate_index ((v), __pool_validate_index); \
} while (0)
First search free list. If nothing is free extend vector of objects.
*/
-#define pool_get_aligned(P,E,A) \
-do { \
- pool_header_t * _pool_var (p) = pool_header (P); \
- uword _pool_var (l); \
- \
- _pool_var (l) = 0; \
- if (P) \
- _pool_var (l) = vec_len (_pool_var (p)->free_indices); \
- \
- if (_pool_var (l) > 0) \
- { \
- /* Return free element from free list. */ \
- uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \
- (E) = (P) + _pool_var (i); \
- _pool_var (p)->free_bitmap = \
- clib_bitmap_andnoti (_pool_var (p)->free_bitmap, _pool_var (i)); \
- _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \
- } \
- else \
- { \
- /* Nothing on free list, make a new element and return it. */ \
- P = _vec_resize (P, \
- /* length_increment */ 1, \
+#define _pool_get_aligned_internal_numa(P,E,A,Z,N) \
+do { \
+ pool_header_t * _pool_var (p) = pool_header (P); \
+ uword _pool_var (l); \
+ \
+ STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \
+ || ((sizeof(P[0]) % A) == 0), \
+ "Pool aligned alloc of incorrectly sized object"); \
+ _pool_var (l) = 0; \
+ if (P) \
+ _pool_var (l) = vec_len (_pool_var (p)->free_indices); \
+ \
+ if (_pool_var (l) > 0) \
+ { \
+ /* Return free element from free list. */ \
+ uword _pool_var (i) = \
+ _pool_var (p)->free_indices[_pool_var (l) - 1]; \
+ (E) = (P) + _pool_var (i); \
+ _pool_var (p)->free_bitmap = \
+ clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \
+ _pool_var (i)); \
+ _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \
+ CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \
+ } \
+ else \
+ { \
+ /* fixed-size, preallocated pools cannot expand */ \
+ if ((P) && _pool_var(p)->max_elts) \
+ { \
+ clib_warning ("can't expand fixed-size pool"); \
+ os_out_of_memory(); \
+ } \
+ /* Nothing on free list, make a new element and return it. */ \
+ P = _vec_resize_numa (P, \
+ /* length_increment */ 1, \
/* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
- pool_aligned_header_bytes, \
- /* align */ (A)); \
- E = vec_end (P) - 1; \
- } \
+ pool_aligned_header_bytes, \
+ /* align */ (A), \
+ /* numa */ (N)); \
+ E = vec_end (P) - 1; \
+ } \
+ if (Z) \
+ memset(E, 0, sizeof(*E)); \
} while (0)
+#define pool_get_aligned_zero_numa(P,E,A,Z,S) \
+ _pool_get_aligned_internal_numa(P,E,A,Z,S)
+
+#define pool_get_aligned_numa(P,E,A,S) \
+ _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S)
+
+#define pool_get_numa(P,E,S) \
+ _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S)
+
+#define _pool_get_aligned_internal(P,E,A,Z) \
+ _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED)
+
+/** Allocate an object E from a pool P with alignment A */
+#define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)
+
+/** Allocate an object E from a pool P with alignment A and zero it */
+#define pool_get_aligned_zero(P,E,A) _pool_get_aligned_internal(P,E,A,1)
+
/** Allocate an object E from a pool P (unspecified alignment). */
#define pool_get(P,E) pool_get_aligned(P,E,0)
+/** Allocate an object E from a pool P and zero it */
+#define pool_get_zero(P,E) pool_get_aligned_zero(P,E,0)
+
+/** See if pool_get will expand the pool or not */
+#define pool_get_aligned_will_expand(P,YESNO,A) \
+do { \
+ pool_header_t * _pool_var (p) = pool_header (P); \
+ uword _pool_var (l); \
+ \
+ _pool_var (l) = 0; \
+ if (P) \
+ { \
+ if (_pool_var (p)->max_elts) \
+ _pool_var (l) = _pool_var (p)->max_elts; \
+ else \
+ _pool_var (l) = vec_len (_pool_var (p)->free_indices); \
+ } \
+ \
+ /* Free elements, certainly won't expand */ \
+ if (_pool_var (l) > 0) \
+ YESNO=0; \
+ else \
+ { \
+ /* Nothing on free list, make a new element and return it. */ \
+ YESNO = _vec_resize_will_expand \
+ (P, \
+ /* length_increment */ 1, \
+ /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
+ pool_aligned_header_bytes, \
+ /* align */ (A)); \
+ } \
+} while (0)
+
+/** Tell the caller if pool get will expand the pool */
+#define pool_get_will_expand(P,YESNO) pool_get_aligned_will_expand(P,YESNO,0)
+
/** Use free bitmap to query whether given element is free. */
#define pool_is_free(P,E) \
({ \
/** Free an object E in pool P. */
#define pool_put(P,E) \
do { \
- pool_header_t * _pool_var (p) = pool_header (P); \
- uword _pool_var (l) = (E) - (P); \
- ASSERT (vec_is_member (P, E)); \
- ASSERT (! pool_is_free (P, E)); \
+ typeof (P) _pool_var(p__) = (P); \
+ typeof (E) _pool_var(e__) = (E); \
+ pool_header_t * _pool_var (p) = pool_header (_pool_var(p__)); \
+ uword _pool_var (l) = _pool_var(e__) - _pool_var(p__); \
+ ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__))); \
+ ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__))); \
\
/* Add element to free bitmap and to free list. */ \
_pool_var (p)->free_bitmap = \
- clib_bitmap_ori (_pool_var (p)->free_bitmap, _pool_var (l)); \
- vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \
+ clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, \
+ _pool_var (l)); \
+ \
+ /* Preallocated pool? */ \
+ if (_pool_var (p)->max_elts) \
+ { \
+ ASSERT(_pool_var(l) < _pool_var (p)->max_elts); \
+ _pool_var(p)->free_indices[_vec_len(_pool_var(p)->free_indices)] = \
+ _pool_var(l); \
+ _vec_len(_pool_var(p)->free_indices) += 1; \
+ } \
+ else \
+ vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \
+ \
+ CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0])); \
} while (0)
/** Free pool element with given index. */
#define pool_alloc_aligned(P,N,A) \
do { \
pool_header_t * _p; \
+ \
+ if ((P)) \
+ { \
+ _p = pool_header (P); \
+ if (_p->max_elts) \
+ { \
+ clib_warning ("Can't expand fixed-size pool"); \
+ os_out_of_memory(); \
+ } \
+ } \
+ \
(P) = _vec_resize ((P), 0, (vec_len (P) + (N)) * sizeof (P[0]), \
pool_aligned_header_bytes, \
(A)); \
/** Allocate N more free elements to pool (unspecified alignment). */
#define pool_alloc(P,N) pool_alloc_aligned(P,N,0)
+/**
+ * Return copy of pool with alignment
+ *
+ * @param P pool to copy
+ * @param A alignment (may be zero)
+ * @return copy of pool
+ */
+#define pool_dup_aligned(P,A) \
+({ \
+ typeof (P) _pool_var (new) = 0; \
+ pool_header_t * _pool_var (ph), * _pool_var (new_ph); \
+ u32 _pool_var (n) = pool_len (P); \
+ if ((P)) \
+ { \
+ _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n), \
+ _pool_var (n) * sizeof ((P)[0]), \
+ pool_aligned_header_bytes, (A)); \
+ clib_memcpy_fast (_pool_var (new), (P), \
+ _pool_var (n) * sizeof ((P)[0])); \
+ _pool_var (ph) = pool_header (P); \
+ _pool_var (new_ph) = pool_header (_pool_var (new)); \
+ _pool_var (new_ph)->free_bitmap = \
+ clib_bitmap_dup (_pool_var (ph)->free_bitmap); \
+ _pool_var (new_ph)->free_indices = \
+ vec_dup (_pool_var (ph)->free_indices); \
+ _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts; \
+ } \
+ _pool_var (new); \
+})
+
+/**
+ * Return copy of pool without alignment
+ *
+ * @param P pool to copy
+ * @return copy of pool
+ */
+#define pool_dup(P) pool_dup_aligned(P,0)
+
/** Low-level free pool operator (do not call directly). */
always_inline void *
_pool_free (void *v)
if (!v)
return v;
clib_bitmap_free (p->free_bitmap);
- vec_free (p->free_indices);
- vec_free_h (v, pool_aligned_header_bytes);
+
+ if (p->max_elts)
+ {
+ int rv;
+
+ rv = munmap (p->mmap_base, p->mmap_size);
+ if (rv)
+ clib_unix_warning ("munmap");
+ }
+ else
+ {
+ vec_free (p->free_indices);
+ vec_free_h (v, pool_aligned_header_bytes);
+ }
return 0;
}
It is a bad idea to allocate or free pool element from within
@c pool_foreach. Build a vector of indices and dispose of them later.
+ Or call pool_flush.
@par Example
(_pool_var (rv) < vec_len (P) ? \
clib_bitmap_next_clear (_pool_var (p)->free_bitmap, _pool_var(rv)) \
: ~0); \
+ _pool_var(rv) = \
+ (_pool_var (rv) < vec_len (P) ? \
+ _pool_var (rv) : ~0); \
_pool_var(rv); \
})
do { body; } while (0); \
}
+/**
+ * @brief Remove all elements from a pool in a safe way
+ *
+ * @param VAR each element in the pool
+ * @param POOL The pool to flush
+ * @param BODY The actions to perform on each element before it is returned to
+ * the pool. i.e. before it is 'freed'
+ */
+#define pool_flush(VAR, POOL, BODY) \
+{ \
+ uword *_pool_var(ii), *_pool_var(dv) = NULL; \
+ \
+ pool_foreach((VAR), (POOL), \
+ ({ \
+ vec_add1(_pool_var(dv), (VAR) - (POOL)); \
+ })); \
+ vec_foreach(_pool_var(ii), _pool_var(dv)) \
+ { \
+ (VAR) = pool_elt_at_index((POOL), *_pool_var(ii)); \
+ do { BODY; } while (0); \
+ pool_put((POOL), (VAR)); \
+ } \
+ vec_free(_pool_var(dv)); \
+}
+
#endif /* included_pool_h */
/*
Code Review / vpp.git / blobdiff