X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fpool.h;h=116c1cd132176314a3f6f227121da00f7f6ca903;hb=refs%2Fchanges%2F83%2F32183%2F2;hp=0f64262abf1383c7fd09092450d8b16083ad41dd;hpb=7f358b30b04aabe9a35680997cc1eeefb080a869;p=vpp.git diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index 0f64262abf1..116c1cd1321 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -46,7 +46,6 @@ #include #include -#include typedef struct @@ -185,12 +184,13 @@ pool_free_elts (void *v) First search free list. If nothing is free extend vector of objects. */ -#define pool_get_aligned(P,E,A) \ +#define _pool_get_aligned_internal_numa(P,E,A,Z,N) \ do { \ pool_header_t * _pool_var (p) = pool_header (P); \ uword _pool_var (l); \ \ - STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \ + STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \ + || ((sizeof(P[0]) % A) == 0), \ "Pool aligned alloc of incorrectly sized object"); \ _pool_var (l) = 0; \ if (P) \ @@ -199,11 +199,14 @@ do { \ if (_pool_var (l) > 0) \ { \ /* Return free element from free list. */ \ - uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \ + uword _pool_var (i) = \ + _pool_var (p)->free_indices[_pool_var (l) - 1]; \ (E) = (P) + _pool_var (i); \ _pool_var (p)->free_bitmap = \ - clib_bitmap_andnoti (_pool_var (p)->free_bitmap, _pool_var (i)); \ + clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \ + _pool_var (i)); \ _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \ + CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \ } \ else \ { \ @@ -214,18 +217,42 @@ do { \ os_out_of_memory(); \ } \ /* Nothing on free list, make a new element and return it. */ \ - P = _vec_resize (P, \ + P = _vec_resize_numa (P, \ /* length_increment */ 1, \ /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \ pool_aligned_header_bytes, \ - /* align */ (A)); \ + /* align */ (A), \ + /* numa */ (N)); \ E = vec_end (P) - 1; \ } \ + if (Z) \ + memset(E, 0, sizeof(*E)); \ } while (0) +#define pool_get_aligned_zero_numa(P,E,A,Z,S) \ + _pool_get_aligned_internal_numa(P,E,A,Z,S) + +#define pool_get_aligned_numa(P,E,A,S) \ + _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S) + +#define pool_get_numa(P,E,S) \ + _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S) + +#define _pool_get_aligned_internal(P,E,A,Z) \ + _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED) + +/** Allocate an object E from a pool P with alignment A */ +#define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0) + +/** Allocate an object E from a pool P with alignment A and zero it */ +#define pool_get_aligned_zero(P,E,A) _pool_get_aligned_internal(P,E,A,1) + /** Allocate an object E from a pool P (unspecified alignment). */ #define pool_get(P,E) pool_get_aligned(P,E,0) +/** Allocate an object E from a pool P and zero it */ +#define pool_get_zero(P,E) pool_get_aligned_zero(P,E,0) + /** See if pool_get will expand the pool or not */ #define pool_get_aligned_will_expand(P,YESNO,A) \ do { \ @@ -256,6 +283,7 @@ do { \ } \ } while (0) +/** Tell the caller if pool get will expand the pool */ #define pool_get_will_expand(P,YESNO) pool_get_aligned_will_expand(P,YESNO,0) /** Use free bitmap to query whether given element is free. */ @@ -272,14 +300,18 @@ do { \ /** Free an object E in pool P. */ #define pool_put(P,E) \ do { \ - pool_header_t * _pool_var (p) = pool_header (P); \ - uword _pool_var (l) = (E) - (P); \ - ASSERT (vec_is_member (P, E)); \ - ASSERT (! pool_is_free (P, E)); \ + typeof (P) _pool_var(p__) = (P); \ + typeof (E) _pool_var(e__) = (E); \ + pool_header_t * _pool_var (p) = pool_header (_pool_var(p__)); \ + uword _pool_var (l) = _pool_var(e__) - _pool_var(p__); \ + ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__))); \ + ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__))); \ \ /* Add element to free bitmap and to free list. */ \ _pool_var (p)->free_bitmap = \ - clib_bitmap_ori (_pool_var (p)->free_bitmap, _pool_var (l)); \ + clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, \ + _pool_var (l)); \ + \ /* Preallocated pool? */ \ if (_pool_var (p)->max_elts) \ { \ @@ -290,6 +322,8 @@ do { \ } \ else \ vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \ + \ + CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0])); \ } while (0) /** Free pool element with given index. */ @@ -325,6 +359,44 @@ do { \ /** Allocate N more free elements to pool (unspecified alignment). */ #define pool_alloc(P,N) pool_alloc_aligned(P,N,0) +/** + * Return copy of pool with alignment + * + * @param P pool to copy + * @param A alignment (may be zero) + * @return copy of pool + */ +#define pool_dup_aligned(P,A) \ +({ \ + typeof (P) _pool_var (new) = 0; \ + pool_header_t * _pool_var (ph), * _pool_var (new_ph); \ + u32 _pool_var (n) = pool_len (P); \ + if ((P)) \ + { \ + _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n), \ + _pool_var (n) * sizeof ((P)[0]), \ + pool_aligned_header_bytes, (A)); \ + clib_memcpy_fast (_pool_var (new), (P), \ + _pool_var (n) * sizeof ((P)[0])); \ + _pool_var (ph) = pool_header (P); \ + _pool_var (new_ph) = pool_header (_pool_var (new)); \ + _pool_var (new_ph)->free_bitmap = \ + clib_bitmap_dup (_pool_var (ph)->free_bitmap); \ + _pool_var (new_ph)->free_indices = \ + vec_dup (_pool_var (ph)->free_indices); \ + _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts; \ + } \ + _pool_var (new); \ +}) + +/** + * Return copy of pool without alignment + * + * @param P pool to copy + * @return copy of pool + */ +#define pool_dup(P) pool_dup_aligned(P,0) + /** Low-level free pool operator (do not call directly). */ always_inline void * _pool_free (void *v) @@ -350,6 +422,20 @@ _pool_free (void *v) return 0; } +static_always_inline uword +pool_get_first_index (void *pool) +{ + pool_header_t *h = pool_header (pool); + return clib_bitmap_first_clear (h->free_bitmap); +} + +static_always_inline uword +pool_get_next_index (void *pool, uword last) +{ + pool_header_t *h = pool_header (pool); + return clib_bitmap_next_clear (h->free_bitmap, last + 1); +} + /** Free a pool. */ #define pool_free(p) (p) = _pool_free(p) @@ -437,17 +523,16 @@ do { \ @c pool_foreach which builds a vector of active indices, and a vec_foreach() (or plain for-loop) to walk the active index vector. */ -#define pool_foreach(VAR,POOL,BODY) \ -do { \ - uword _pool_foreach_lo, _pool_foreach_hi; \ - pool_foreach_region (_pool_foreach_lo, _pool_foreach_hi, (POOL), \ - ({ \ - for ((VAR) = (POOL) + _pool_foreach_lo; \ - (VAR) < (POOL) + _pool_foreach_hi; \ - (VAR)++) \ - do { BODY; } while (0); \ - })); \ -} while (0) + +#define pool_foreach(VAR,POOL) \ + if (POOL) \ + for (VAR = POOL + pool_get_first_index (POOL); \ + VAR < vec_end (POOL); \ + VAR = POOL + pool_get_next_index (POOL, VAR - POOL)) + +#define pool_foreach_old(VAR,POOL,BODY) \ + pool_foreach(VAR,POOL) \ + { BODY; } /** Returns pointer to element at given index. @@ -481,16 +566,19 @@ do { \ _pool_var(rv); \ }) +#define pool_foreach_index(i,v) \ + if (v) \ + for (i = pool_get_first_index (v); \ + i < vec_len (v); \ + i = pool_get_next_index (v, i)) \ + /** Iterate pool by index. */ -#define pool_foreach_index(i,v,body) \ - for ((i) = 0; (i) < vec_len (v); (i)++) \ - { \ - if (! pool_is_free_index ((v), (i))) \ - do { body; } while (0); \ - } +#define pool_foreach_index_old(i,v,body) \ + pool_foreach_index (i,v) \ + { body; } /** - * @brief Remove all elemenets from a pool in a safe way + * @brief Remove all elements from a pool in a safe way * * @param VAR each element in the pool * @param POOL The pool to flush @@ -501,10 +589,10 @@ do { \ { \ uword *_pool_var(ii), *_pool_var(dv) = NULL; \ \ - pool_foreach((VAR), (POOL), \ - ({ \ + pool_foreach((VAR), (POOL)) \ + { \ vec_add1(_pool_var(dv), (VAR) - (POOL)); \ - })); \ + } \ vec_foreach(_pool_var(ii), _pool_var(dv)) \ { \ (VAR) = pool_elt_at_index((POOL), *_pool_var(ii)); \