X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fpool.h;h=db950d27d1855df7c66a15eea355298338ca645a;hb=a690fdbfe179e0ea65818c03b52535bf9210efd0;hp=0f64262abf1383c7fd09092450d8b16083ad41dd;hpb=7f358b30b04aabe9a35680997cc1eeefb080a869;p=vpp.git diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index 0f64262abf1..db950d27d18 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -185,12 +185,13 @@ pool_free_elts (void *v) First search free list. If nothing is free extend vector of objects. */ -#define pool_get_aligned(P,E,A) \ +#define _pool_get_aligned_internal_numa(P,E,A,Z,N) \ do { \ pool_header_t * _pool_var (p) = pool_header (P); \ uword _pool_var (l); \ \ - STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \ + STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \ + || ((sizeof(P[0]) % A) == 0), \ "Pool aligned alloc of incorrectly sized object"); \ _pool_var (l) = 0; \ if (P) \ @@ -199,11 +200,14 @@ do { \ if (_pool_var (l) > 0) \ { \ /* Return free element from free list. */ \ - uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \ + uword _pool_var (i) = \ + _pool_var (p)->free_indices[_pool_var (l) - 1]; \ (E) = (P) + _pool_var (i); \ _pool_var (p)->free_bitmap = \ - clib_bitmap_andnoti (_pool_var (p)->free_bitmap, _pool_var (i)); \ + clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \ + _pool_var (i)); \ _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \ + CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \ } \ else \ { \ @@ -214,18 +218,42 @@ do { \ os_out_of_memory(); \ } \ /* Nothing on free list, make a new element and return it. */ \ - P = _vec_resize (P, \ + P = _vec_resize_numa (P, \ /* length_increment */ 1, \ /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \ pool_aligned_header_bytes, \ - /* align */ (A)); \ + /* align */ (A), \ + /* numa */ (N)); \ E = vec_end (P) - 1; \ } \ + if (Z) \ + memset(E, 0, sizeof(*E)); \ } while (0) +#define pool_get_aligned_zero_numa(P,E,A,Z,S) \ + _pool_get_aligned_internal_numa(P,E,A,Z,S) + +#define pool_get_aligned_numa(P,E,A,S) \ + _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S) + +#define pool_get_numa(P,E,S) \ + _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S) + +#define _pool_get_aligned_internal(P,E,A,Z) \ + _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED) + +/** Allocate an object E from a pool P with alignment A */ +#define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0) + +/** Allocate an object E from a pool P with alignment A and zero it */ +#define pool_get_aligned_zero(P,E,A) _pool_get_aligned_internal(P,E,A,1) + /** Allocate an object E from a pool P (unspecified alignment). */ #define pool_get(P,E) pool_get_aligned(P,E,0) +/** Allocate an object E from a pool P and zero it */ +#define pool_get_zero(P,E) pool_get_aligned_zero(P,E,0) + /** See if pool_get will expand the pool or not */ #define pool_get_aligned_will_expand(P,YESNO,A) \ do { \ @@ -256,6 +284,7 @@ do { \ } \ } while (0) +/** Tell the caller if pool get will expand the pool */ #define pool_get_will_expand(P,YESNO) pool_get_aligned_will_expand(P,YESNO,0) /** Use free bitmap to query whether given element is free. */ @@ -272,14 +301,18 @@ do { \ /** Free an object E in pool P. */ #define pool_put(P,E) \ do { \ - pool_header_t * _pool_var (p) = pool_header (P); \ - uword _pool_var (l) = (E) - (P); \ - ASSERT (vec_is_member (P, E)); \ - ASSERT (! pool_is_free (P, E)); \ + typeof (P) _pool_var(p__) = (P); \ + typeof (E) _pool_var(e__) = (E); \ + pool_header_t * _pool_var (p) = pool_header (_pool_var(p__)); \ + uword _pool_var (l) = _pool_var(e__) - _pool_var(p__); \ + ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__))); \ + ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__))); \ \ /* Add element to free bitmap and to free list. */ \ _pool_var (p)->free_bitmap = \ - clib_bitmap_ori (_pool_var (p)->free_bitmap, _pool_var (l)); \ + clib_bitmap_ori_notrim (_pool_var (p)->free_bitmap, \ + _pool_var (l)); \ + \ /* Preallocated pool? */ \ if (_pool_var (p)->max_elts) \ { \ @@ -290,6 +323,8 @@ do { \ } \ else \ vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \ + \ + CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0])); \ } while (0) /** Free pool element with given index. */ @@ -325,6 +360,44 @@ do { \ /** Allocate N more free elements to pool (unspecified alignment). */ #define pool_alloc(P,N) pool_alloc_aligned(P,N,0) +/** + * Return copy of pool with alignment + * + * @param P pool to copy + * @param A alignment (may be zero) + * @return copy of pool + */ +#define pool_dup_aligned(P,A) \ +({ \ + typeof (P) _pool_var (new) = 0; \ + pool_header_t * _pool_var (ph), * _pool_var (new_ph); \ + u32 _pool_var (n) = pool_len (P); \ + if ((P)) \ + { \ + _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n), \ + _pool_var (n) * sizeof ((P)[0]), \ + pool_aligned_header_bytes, (A)); \ + clib_memcpy_fast (_pool_var (new), (P), \ + _pool_var (n) * sizeof ((P)[0])); \ + _pool_var (ph) = pool_header (P); \ + _pool_var (new_ph) = pool_header (_pool_var (new)); \ + _pool_var (new_ph)->free_bitmap = \ + clib_bitmap_dup (_pool_var (ph)->free_bitmap); \ + _pool_var (new_ph)->free_indices = \ + vec_dup (_pool_var (ph)->free_indices); \ + _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts; \ + } \ + _pool_var (new); \ +}) + +/** + * Return copy of pool without alignment + * + * @param P pool to copy + * @return copy of pool + */ +#define pool_dup(P) pool_dup_aligned(P,0) + /** Low-level free pool operator (do not call directly). */ always_inline void * _pool_free (void *v) @@ -490,7 +563,7 @@ do { \ } /** - * @brief Remove all elemenets from a pool in a safe way + * @brief Remove all elements from a pool in a safe way * * @param VAR each element in the pool * @param POOL The pool to flush