First search free list. If nothing is free extend vector of objects.
*/
-#define _pool_get_aligned_internal(P,E,A,Z) \
+#define _pool_get_aligned_internal_numa(P,E,A,Z,N) \
do { \
pool_header_t * _pool_var (p) = pool_header (P); \
uword _pool_var (l); \
\
- STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \
+ STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \
+ || ((sizeof(P[0]) % A) == 0), \
"Pool aligned alloc of incorrectly sized object"); \
_pool_var (l) = 0; \
if (P) \
if (_pool_var (l) > 0) \
{ \
/* Return free element from free list. */ \
- uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \
+ uword _pool_var (i) = \
+ _pool_var (p)->free_indices[_pool_var (l) - 1]; \
(E) = (P) + _pool_var (i); \
- _pool_var (p)->free_bitmap = \
- clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \
- _pool_var (i)); \
+ _pool_var (p)->free_bitmap = \
+ clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \
+ _pool_var (i)); \
_vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \
+ CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \
} \
else \
{ \
os_out_of_memory(); \
} \
/* Nothing on free list, make a new element and return it. */ \
- P = _vec_resize (P, \
+ P = _vec_resize_numa (P, \
/* length_increment */ 1, \
/* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
pool_aligned_header_bytes, \
- /* align */ (A)); \
+ /* align */ (A), \
+ /* numa */ (N)); \
E = vec_end (P) - 1; \
- } \
+ } \
if (Z) \
- memset(E, 0, sizeof(*E)); \
+ memset(E, 0, sizeof(*E)); \
} while (0)
+#define pool_get_aligned_zero_numa(P,E,A,Z,S) \
+ _pool_get_aligned_internal_numa(P,E,A,Z,S)
+
+#define pool_get_aligned_numa(P,E,A,S) \
+ _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S)
+
+#define pool_get_numa(P,E,S) \
+ _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S)
+
+#define _pool_get_aligned_internal(P,E,A,Z) \
+ _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED)
+
/** Allocate an object E from a pool P with alignment A */
#define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)
/** Free an object E in pool P. */
#define pool_put(P,E) \
do { \
- pool_header_t * _pool_var (p) = pool_header (P); \
- uword _pool_var (l) = (E) - (P); \
- ASSERT (vec_is_member (P, E)); \
- ASSERT (! pool_is_free (P, E)); \
+ typeof (P) _pool_var(p__) = (P); \
+ typeof (E) _pool_var(e__) = (E); \
+ pool_header_t * _pool_var (p) = pool_header (_pool_var(p__)); \
+ uword _pool_var (l) = _pool_var(e__) - _pool_var(p__); \
+ ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__))); \
+ ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__))); \
\
/* Add element to free bitmap and to free list. */ \
_pool_var (p)->free_bitmap = \
} \
else \
vec_add1 (_pool_var (p)->free_indices, _pool_var (l)); \
+ \
+ CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0])); \
} while (0)
/** Free pool element with given index. */