#include <vppinfra/bitmap.h>
#include <vppinfra/error.h>
-#include <vppinfra/mheap.h>
typedef struct
} pool_header_t;
/** Align pool header so that pointers are naturally aligned. */
-#define pool_aligned_header_bytes \
- vec_aligned_header_bytes (sizeof (pool_header_t), sizeof (void *))
+#define pool_aligned_header_bytes \
+ round_pow2 (sizeof (pool_header_t), sizeof (void *))
/** Get pool header from user pool pointer */
always_inline pool_header_t *
/** Local variable naming macro. */
#define _pool_var(v) _pool_##v
-/** Queries whether pool has at least N_FREE free elements. */
-always_inline uword
-pool_free_elts (void *v)
-{
- pool_header_t *p = pool_header (v);
- uword n_free = 0;
-
- if (v)
- {
- n_free += vec_len (p->free_indices);
-
- /* Space left at end of vector? */
- n_free += vec_capacity (v, sizeof (p[0])) - vec_len (v);
- }
-
- return n_free;
-}
+/** Number of bytes that can fit into pool with current allocation */
+#define pool_capacity(P) vec_capacity (P, pool_aligned_header_bytes)
+
+/** Number of elements that can fit into pool with current allocation */
+#define pool_max_len(P) (pool_capacity (P) / sizeof (P[0]))
+
+/** Number of free elements in pool */
+#define pool_free_elts(P) \
+ ({ \
+ pool_header_t *_pool_var (p) = pool_header (P); \
+ uword n_free = 0; \
+ if (P) \
+ { \
+ n_free += vec_len (_pool_var (p)->free_indices); \
+ /* Fixed-size pools have max_elts set non-zero */ \
+ if (_pool_var (p)->max_elts == 0) \
+ n_free += pool_max_len (P) - vec_len (P); \
+ } \
+ n_free; \
+ })
/** Allocate an object E from a pool P (general version).
First search free list. If nothing is free extend vector of objects.
*/
-#define _pool_get_aligned_internal(P,E,A,Z) \
+#define _pool_get_aligned_internal_numa(P,E,A,Z,N) \
do { \
pool_header_t * _pool_var (p) = pool_header (P); \
uword _pool_var (l); \
\
- STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \
+ STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \
+ || ((sizeof(P[0]) % A) == 0), \
"Pool aligned alloc of incorrectly sized object"); \
_pool_var (l) = 0; \
if (P) \
if (_pool_var (l) > 0) \
{ \
/* Return free element from free list. */ \
- uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \
+ uword _pool_var (i) = \
+ _pool_var (p)->free_indices[_pool_var (l) - 1]; \
(E) = (P) + _pool_var (i); \
- _pool_var (p)->free_bitmap = \
- clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \
- _pool_var (i)); \
+ _pool_var (p)->free_bitmap = \
+ clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \
+ _pool_var (i)); \
_vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \
+ CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \
} \
else \
{ \
os_out_of_memory(); \
} \
/* Nothing on free list, make a new element and return it. */ \
- P = _vec_resize (P, \
+ P = _vec_resize_numa (P, \
/* length_increment */ 1, \
/* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
pool_aligned_header_bytes, \
- /* align */ (A)); \
+ /* align */ (A), \
+ /* numa */ (N)); \
E = vec_end (P) - 1; \
- } \
+ } \
if (Z) \
- memset(E, 0, sizeof(*E)); \
+ memset(E, 0, sizeof(*E)); \
} while (0)
+#define pool_get_aligned_zero_numa(P,E,A,Z,S) \
+ _pool_get_aligned_internal_numa(P,E,A,Z,S)
+
+#define pool_get_aligned_numa(P,E,A,S) \
+ _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S)
+
+#define pool_get_numa(P,E,S) \
+ _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S)
+
+#define _pool_get_aligned_internal(P,E,A,Z) \
+ _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED)
+
/** Allocate an object E from a pool P with alignment A */
#define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)
} \
} while (0)
+/** See if pool_put will expand free_bitmap or free_indices or not */
+#define pool_put_will_expand(P, E, YESNO) \
+ do \
+ { \
+ pool_header_t *_pool_var (p) = pool_header (P); \
+ \
+ uword _pool_var (i) = (E) - (P); \
+ /* free_bitmap or free_indices may expand. */ \
+ YESNO = \
+ clib_bitmap_will_expand (_pool_var (p)->free_bitmap, _pool_var (i)); \
+ \
+ YESNO += _vec_resize_will_expand ( \
+ _pool_var (p)->free_indices, 1, \
+ (vec_len (_pool_var (p)->free_indices) + 1) * \
+ sizeof (_pool_var (p)->free_indices[0]), \
+ 0, 0); \
+ } \
+ while (0)
+
/** Tell the caller if pool get will expand the pool */
#define pool_get_will_expand(P,YESNO) pool_get_aligned_will_expand(P,YESNO,0)
#define pool_is_free_index(P,I) pool_is_free((P),(P)+(I))
/** Free an object E in pool P. */
-#define pool_put(P,E) \
-do { \
- pool_header_t * _pool_var (p) = pool_header (P); \
- uword _pool_var (l) = (E) - (P); \
- ASSERT (vec_is_member (P, E)); \
- ASSERT (! pool_is_free (P, E)); \
- \
- /* Add element to free bitmap and to free list. */ \
- _pool_var (p)->free_bitmap = \
- clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, \
- _pool_var (l)); \
- \
- /* Preallocated pool? */ \
- if (_pool_var (p)->max_elts) \
- { \
- ASSERT(_pool_var(l) < _pool_var (p)->max_elts); \
- _pool_var(p)->free_indices[_vec_len(_pool_var(p)->free_indices)] = \
- _pool_var(l); \
- _vec_len(_pool_var(p)->free_indices) += 1; \
- } \
- else \
- vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \
-} while (0)
+#define pool_put(P, E) \
+ do \
+ { \
+ typeof (P) _pool_var (p__) = (P); \
+ typeof (E) _pool_var (e__) = (E); \
+ pool_header_t *_pool_var (p) = pool_header (_pool_var (p__)); \
+ uword _pool_var (l) = _pool_var (e__) - _pool_var (p__); \
+ if (_pool_var (p)->max_elts == 0) \
+ ASSERT (vec_is_member (_pool_var (p__), _pool_var (e__))); \
+ ASSERT (!pool_is_free (_pool_var (p__), _pool_var (e__))); \
+ \
+ /* Add element to free bitmap and to free list. */ \
+ _pool_var (p)->free_bitmap = \
+ clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, _pool_var (l)); \
+ \
+ /* Preallocated pool? */ \
+ if (_pool_var (p)->max_elts) \
+ { \
+ ASSERT (_pool_var (l) < _pool_var (p)->max_elts); \
+ _pool_var (p) \
+ ->free_indices[_vec_len (_pool_var (p)->free_indices)] = \
+ _pool_var (l); \
+ _vec_len (_pool_var (p)->free_indices) += 1; \
+ } \
+ else \
+ vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \
+ \
+ CLIB_MEM_POISON (_pool_var (e__), sizeof (_pool_var (e__)[0])); \
+ } \
+ while (0)
/** Free pool element with given index. */
#define pool_put_index(p,i) \
/** Allocate N more free elements to pool (unspecified alignment). */
#define pool_alloc(P,N) pool_alloc_aligned(P,N,0)
+/**
+ * Return copy of pool with alignment
+ *
+ * @param P pool to copy
+ * @param A alignment (may be zero)
+ * @return copy of pool
+ */
+#define pool_dup_aligned(P, A) \
+ ({ \
+ typeof (P) _pool_var (new) = 0; \
+ pool_header_t *_pool_var (ph), *_pool_var (new_ph); \
+ u32 _pool_var (n) = pool_len (P); \
+ if ((P)) \
+ { \
+ _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n), \
+ _pool_var (n) * sizeof ((P)[0]), \
+ pool_aligned_header_bytes, (A)); \
+ CLIB_MEM_OVERFLOW_PUSH ((P), _pool_var (n) * sizeof ((P)[0])); \
+ clib_memcpy_fast (_pool_var (new), (P), \
+ _pool_var (n) * sizeof ((P)[0])); \
+ CLIB_MEM_OVERFLOW_POP (); \
+ _pool_var (ph) = pool_header (P); \
+ _pool_var (new_ph) = pool_header (_pool_var (new)); \
+ _pool_var (new_ph)->free_bitmap = \
+ clib_bitmap_dup (_pool_var (ph)->free_bitmap); \
+ _pool_var (new_ph)->free_indices = \
+ vec_dup (_pool_var (ph)->free_indices); \
+ _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts; \
+ } \
+ _pool_var (new); \
+ })
+
+/**
+ * Return copy of pool without alignment
+ *
+ * @param P pool to copy
+ * @return copy of pool
+ */
+#define pool_dup(P) pool_dup_aligned(P,0)
+
/** Low-level free pool operator (do not call directly). */
always_inline void *
_pool_free (void *v)
return 0;
}
+static_always_inline uword
+pool_get_first_index (void *pool)
+{
+ pool_header_t *h = pool_header (pool);
+ return clib_bitmap_first_clear (h->free_bitmap);
+}
+
+static_always_inline uword
+pool_get_next_index (void *pool, uword last)
+{
+ pool_header_t *h = pool_header (pool);
+ return clib_bitmap_next_clear (h->free_bitmap, last + 1);
+}
+
/** Free a pool. */
#define pool_free(p) (p) = _pool_free(p)
@c pool_foreach which builds a vector of active indices, and a
vec_foreach() (or plain for-loop) to walk the active index vector.
*/
-#define pool_foreach(VAR,POOL,BODY) \
-do { \
- uword _pool_foreach_lo, _pool_foreach_hi; \
- pool_foreach_region (_pool_foreach_lo, _pool_foreach_hi, (POOL), \
- ({ \
- for ((VAR) = (POOL) + _pool_foreach_lo; \
- (VAR) < (POOL) + _pool_foreach_hi; \
- (VAR)++) \
- do { BODY; } while (0); \
- })); \
-} while (0)
+
+#define pool_foreach(VAR,POOL) \
+ if (POOL) \
+ for (VAR = POOL + pool_get_first_index (POOL); \
+ VAR < vec_end (POOL); \
+ VAR = POOL + pool_get_next_index (POOL, VAR - POOL))
/** Returns pointer to element at given index.
_pool_var(rv); \
})
-/** Iterate pool by index. */
-#define pool_foreach_index(i,v,body) \
- for ((i) = 0; (i) < vec_len (v); (i)++) \
- { \
- if (! pool_is_free_index ((v), (i))) \
- do { body; } while (0); \
- }
+#define pool_foreach_index(i,v) \
+ if (v) \
+ for (i = pool_get_first_index (v); \
+ i < vec_len (v); \
+ i = pool_get_next_index (v, i)) \
/**
* @brief Remove all elements from a pool in a safe way
{ \
uword *_pool_var(ii), *_pool_var(dv) = NULL; \
\
- pool_foreach((VAR), (POOL), \
- ({ \
+ pool_foreach((VAR), (POOL)) \
+ { \
vec_add1(_pool_var(dv), (VAR) - (POOL)); \
- })); \
+ } \
vec_foreach(_pool_var(ii), _pool_var(dv)) \
{ \
(VAR) = pool_elt_at_index((POOL), *_pool_var(ii)); \
Code Review / vpp.git / blobdiff