X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fpool.h;h=db950d27d1855df7c66a15eea355298338ca645a;hb=a690fdbfe179e0ea65818c03b52535bf9210efd0;hp=56536b779632904672e190be3d71ddcf76b9e09a;hpb=66b11318a1e5f24880e3ec77c95d70647732a4a8;p=vpp.git diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index 56536b77963..db950d27d18 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -56,6 +56,16 @@ typedef struct /** Vector of free indices. One element for each set bit in bitmap. */ u32 *free_indices; + + /* The following fields are set for fixed-size, preallocated pools */ + + /** Maximum size of the pool, in elements */ + u32 max_elts; + + /** mmap segment info: base + length */ + u8 *mmap_base; + u64 mmap_size; + } pool_header_t; /** Align pool header so that pointers are naturally aligned. */ @@ -69,6 +79,15 @@ pool_header (void *v) return vec_aligned_header (v, sizeof (pool_header_t), sizeof (void *)); } +extern void _pool_init_fixed (void **, u32, u32); +extern void fpool_free (void *); + +/** initialize a fixed-size, preallocated pool */ +#define pool_init_fixed(pool,max_elts) \ +{ \ + _pool_init_fixed((void **)&(pool),sizeof(pool[0]),max_elts); \ +} + /** Validate a pool */ always_inline void pool_validate (void *v) @@ -98,7 +117,7 @@ pool_header_validate_index (void *v, uword index) do { \ uword __pool_validate_index = (i); \ vec_validate_ha ((v), __pool_validate_index, \ - pool_aligned_header_bytes, /* align */ 0); \ + pool_aligned_header_bytes, /* align */ 0); \ pool_header_validate_index ((v), __pool_validate_index); \ } while (0) @@ -166,39 +185,75 @@ pool_free_elts (void *v) First search free list. If nothing is free extend vector of objects. */ -#define pool_get_aligned(P,E,A) \ -do { \ - pool_header_t * _pool_var (p) = pool_header (P); \ - uword _pool_var (l); \ - \ - _pool_var (l) = 0; \ - if (P) \ - _pool_var (l) = vec_len (_pool_var (p)->free_indices); \ - \ - if (_pool_var (l) > 0) \ - { \ - /* Return free element from free list. */ \ - uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \ - (E) = (P) + _pool_var (i); \ - _pool_var (p)->free_bitmap = \ - clib_bitmap_andnoti (_pool_var (p)->free_bitmap, _pool_var (i)); \ - _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \ - } \ - else \ - { \ - /* Nothing on free list, make a new element and return it. */ \ - P = _vec_resize (P, \ - /* length_increment */ 1, \ +#define _pool_get_aligned_internal_numa(P,E,A,Z,N) \ +do { \ + pool_header_t * _pool_var (p) = pool_header (P); \ + uword _pool_var (l); \ + \ + STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \ + || ((sizeof(P[0]) % A) == 0), \ + "Pool aligned alloc of incorrectly sized object"); \ + _pool_var (l) = 0; \ + if (P) \ + _pool_var (l) = vec_len (_pool_var (p)->free_indices); \ + \ + if (_pool_var (l) > 0) \ + { \ + /* Return free element from free list. */ \ + uword _pool_var (i) = \ + _pool_var (p)->free_indices[_pool_var (l) - 1]; \ + (E) = (P) + _pool_var (i); \ + _pool_var (p)->free_bitmap = \ + clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \ + _pool_var (i)); \ + _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \ + CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \ + } \ + else \ + { \ + /* fixed-size, preallocated pools cannot expand */ \ + if ((P) && _pool_var(p)->max_elts) \ + { \ + clib_warning ("can't expand fixed-size pool"); \ + os_out_of_memory(); \ + } \ + /* Nothing on free list, make a new element and return it. */ \ + P = _vec_resize_numa (P, \ + /* length_increment */ 1, \ /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \ - pool_aligned_header_bytes, \ - /* align */ (A)); \ - E = vec_end (P) - 1; \ - } \ + pool_aligned_header_bytes, \ + /* align */ (A), \ + /* numa */ (N)); \ + E = vec_end (P) - 1; \ + } \ + if (Z) \ + memset(E, 0, sizeof(*E)); \ } while (0) +#define pool_get_aligned_zero_numa(P,E,A,Z,S) \ + _pool_get_aligned_internal_numa(P,E,A,Z,S) + +#define pool_get_aligned_numa(P,E,A,S) \ + _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S) + +#define pool_get_numa(P,E,S) \ + _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S) + +#define _pool_get_aligned_internal(P,E,A,Z) \ + _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED) + +/** Allocate an object E from a pool P with alignment A */ +#define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0) + +/** Allocate an object E from a pool P with alignment A and zero it */ +#define pool_get_aligned_zero(P,E,A) _pool_get_aligned_internal(P,E,A,1) + /** Allocate an object E from a pool P (unspecified alignment). */ #define pool_get(P,E) pool_get_aligned(P,E,0) +/** Allocate an object E from a pool P and zero it */ +#define pool_get_zero(P,E) pool_get_aligned_zero(P,E,0) + /** See if pool_get will expand the pool or not */ #define pool_get_aligned_will_expand(P,YESNO,A) \ do { \ @@ -207,7 +262,12 @@ do { \ \ _pool_var (l) = 0; \ if (P) \ - _pool_var (l) = vec_len (_pool_var (p)->free_indices); \ + { \ + if (_pool_var (p)->max_elts) \ + _pool_var (l) = _pool_var (p)->max_elts; \ + else \ + _pool_var (l) = vec_len (_pool_var (p)->free_indices); \ + } \ \ /* Free elements, certainly won't expand */ \ if (_pool_var (l) > 0) \ @@ -224,6 +284,7 @@ do { \ } \ } while (0) +/** Tell the caller if pool get will expand the pool */ #define pool_get_will_expand(P,YESNO) pool_get_aligned_will_expand(P,YESNO,0) /** Use free bitmap to query whether given element is free. */ @@ -240,15 +301,30 @@ do { \ /** Free an object E in pool P. */ #define pool_put(P,E) \ do { \ - pool_header_t * _pool_var (p) = pool_header (P); \ - uword _pool_var (l) = (E) - (P); \ - ASSERT (vec_is_member (P, E)); \ - ASSERT (! pool_is_free (P, E)); \ + typeof (P) _pool_var(p__) = (P); \ + typeof (E) _pool_var(e__) = (E); \ + pool_header_t * _pool_var (p) = pool_header (_pool_var(p__)); \ + uword _pool_var (l) = _pool_var(e__) - _pool_var(p__); \ + ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__))); \ + ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__))); \ \ /* Add element to free bitmap and to free list. */ \ _pool_var (p)->free_bitmap = \ - clib_bitmap_ori (_pool_var (p)->free_bitmap, _pool_var (l)); \ - vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \ + clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, \ + _pool_var (l)); \ + \ + /* Preallocated pool? */ \ + if (_pool_var (p)->max_elts) \ + { \ + ASSERT(_pool_var(l) < _pool_var (p)->max_elts); \ + _pool_var(p)->free_indices[_vec_len(_pool_var(p)->free_indices)] = \ + _pool_var(l); \ + _vec_len(_pool_var(p)->free_indices) += 1; \ + } \ + else \ + vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \ + \ + CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0])); \ } while (0) /** Free pool element with given index. */ @@ -262,6 +338,17 @@ do { \ #define pool_alloc_aligned(P,N,A) \ do { \ pool_header_t * _p; \ + \ + if ((P)) \ + { \ + _p = pool_header (P); \ + if (_p->max_elts) \ + { \ + clib_warning ("Can't expand fixed-size pool"); \ + os_out_of_memory(); \ + } \ + } \ + \ (P) = _vec_resize ((P), 0, (vec_len (P) + (N)) * sizeof (P[0]), \ pool_aligned_header_bytes, \ (A)); \ @@ -273,6 +360,44 @@ do { \ /** Allocate N more free elements to pool (unspecified alignment). */ #define pool_alloc(P,N) pool_alloc_aligned(P,N,0) +/** + * Return copy of pool with alignment + * + * @param P pool to copy + * @param A alignment (may be zero) + * @return copy of pool + */ +#define pool_dup_aligned(P,A) \ +({ \ + typeof (P) _pool_var (new) = 0; \ + pool_header_t * _pool_var (ph), * _pool_var (new_ph); \ + u32 _pool_var (n) = pool_len (P); \ + if ((P)) \ + { \ + _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n), \ + _pool_var (n) * sizeof ((P)[0]), \ + pool_aligned_header_bytes, (A)); \ + clib_memcpy_fast (_pool_var (new), (P), \ + _pool_var (n) * sizeof ((P)[0])); \ + _pool_var (ph) = pool_header (P); \ + _pool_var (new_ph) = pool_header (_pool_var (new)); \ + _pool_var (new_ph)->free_bitmap = \ + clib_bitmap_dup (_pool_var (ph)->free_bitmap); \ + _pool_var (new_ph)->free_indices = \ + vec_dup (_pool_var (ph)->free_indices); \ + _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts; \ + } \ + _pool_var (new); \ +}) + +/** + * Return copy of pool without alignment + * + * @param P pool to copy + * @return copy of pool + */ +#define pool_dup(P) pool_dup_aligned(P,0) + /** Low-level free pool operator (do not call directly). */ always_inline void * _pool_free (void *v) @@ -281,8 +406,20 @@ _pool_free (void *v) if (!v) return v; clib_bitmap_free (p->free_bitmap); - vec_free (p->free_indices); - vec_free_h (v, pool_aligned_header_bytes); + + if (p->max_elts) + { + int rv; + + rv = munmap (p->mmap_base, p->mmap_size); + if (rv) + clib_unix_warning ("munmap"); + } + else + { + vec_free (p->free_indices); + vec_free_h (v, pool_aligned_header_bytes); + } return 0; } @@ -411,6 +548,9 @@ do { \ (_pool_var (rv) < vec_len (P) ? \ clib_bitmap_next_clear (_pool_var (p)->free_bitmap, _pool_var(rv)) \ : ~0); \ + _pool_var(rv) = \ + (_pool_var (rv) < vec_len (P) ? \ + _pool_var (rv) : ~0); \ _pool_var(rv); \ }) @@ -423,7 +563,7 @@ do { \ } /** - * @brief Remove all elemenets from a pool in a safe way + * @brief Remove all elements from a pool in a safe way * * @param VAR each element in the pool * @param POOL The pool to flush