X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fpool.h;h=db950d27d1855df7c66a15eea355298338ca645a;hb=a690fdbfe179e0ea65818c03b52535bf9210efd0;hp=47bae07add03d7c2743464d609e62e574a25528f;hpb=8be6c40cf39f0d637a9dbfaa40ccec1d72bff34a;p=vpp.git

diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h
index 47bae07add0..db950d27d18 100644
--- a/src/vppinfra/pool.h
+++ b/src/vppinfra/pool.h
@@ -185,12 +185,13 @@ pool_free_elts (void *v)
 
    First search free list.  If nothing is free extend vector of objects.
 */
-#define _pool_get_aligned_internal(P,E,A,Z)                             \
+#define _pool_get_aligned_internal_numa(P,E,A,Z,N)                      \
 do {                                                                    \
   pool_header_t * _pool_var (p) = pool_header (P);                      \
   uword _pool_var (l);                                                  \
                                                                         \
-  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \
+  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0)                         \
+                || ((sizeof(P[0]) % A) == 0),                           \
                 "Pool aligned alloc of incorrectly sized object");      \
   _pool_var (l) = 0;                                                    \
   if (P)                                                                \
@@ -199,12 +200,14 @@ do {                                                                    \
   if (_pool_var (l) > 0)                                                \
     {                                                                   \
       /* Return free element from free list. */                         \
-      uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \
+      uword _pool_var (i) =                                             \
+        _pool_var (p)->free_indices[_pool_var (l) - 1];                 \
       (E) = (P) + _pool_var (i);                                        \
-      _pool_var (p)->free_bitmap =					\
-	clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,        \
-	                             _pool_var (i));	               	\
+      _pool_var (p)->free_bitmap =                                      \
+	clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,         \
+	                             _pool_var (i));                    \
       _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1;       \
+      CLIB_MEM_UNPOISON((E), sizeof((E)[0]));                           \
     }                                                                   \
   else                                                                  \
     {                                                                   \
@@ -215,17 +218,30 @@ do {                                                                    \
           os_out_of_memory();                                           \
         }                                                               \
       /* Nothing on free list, make a new element and return it. */     \
-      P = _vec_resize (P,                                               \
+      P = _vec_resize_numa (P,                                          \
 		       /* length_increment */ 1,                        \
 		       /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
 		       pool_aligned_header_bytes,                       \
-		       /* align */ (A));                                \
+                       /* align */ (A),                                 \
+                       /* numa */ (N));                                 \
       E = vec_end (P) - 1;                                              \
-    }									\
+    }                                                                   \
   if (Z)                                                                \
-    memset(E, 0, sizeof(*E));						\
+    memset(E, 0, sizeof(*E));                                           \
 } while (0)
 
+#define pool_get_aligned_zero_numa(P,E,A,Z,S) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,S)
+
+#define pool_get_aligned_numa(P,E,A,S) \
+  _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S)
+
+#define pool_get_numa(P,E,S) \
+  _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S)
+
+#define _pool_get_aligned_internal(P,E,A,Z) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED)
+
 /** Allocate an object E from a pool P with alignment A */
 #define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)
 
@@ -285,10 +301,12 @@ do {                                                                    \
 /** Free an object E in pool P. */
 #define pool_put(P,E)							\
 do {									\
-  pool_header_t * _pool_var (p) = pool_header (P);			\
-  uword _pool_var (l) = (E) - (P);					\
-  ASSERT (vec_is_member (P, E));					\
-  ASSERT (! pool_is_free (P, E));					\
+  typeof (P) _pool_var(p__) = (P);			                \
+  typeof (E) _pool_var(e__) = (E);			                \
+  pool_header_t * _pool_var (p) = pool_header (_pool_var(p__));		\
+  uword _pool_var (l) = _pool_var(e__) - _pool_var(p__);		\
+  ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__)));		\
+  ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__)));		\
 									\
   /* Add element to free bitmap and to free list. */			\
   _pool_var (p)->free_bitmap =						\
@@ -305,6 +323,8 @@ do {									\
     }                                                                   \
   else                                                                  \
     vec_add1 (_pool_var (p)->free_indices, _pool_var (l));		\
+                                                                        \
+  CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0]));                                 \
 } while (0)
 
 /** Free pool element with given index. */
@@ -352,18 +372,21 @@ do {									\
   typeof (P) _pool_var (new) = 0;					\
   pool_header_t * _pool_var (ph), * _pool_var (new_ph);			\
   u32 _pool_var (n) = pool_len (P);					\
-  _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n),	\
-				 _pool_var (n) * sizeof ((P)[0]), 	\
-				 pool_aligned_header_bytes, (A));	\
-  clib_memcpy_fast (_pool_var (new), (P), 				\
-		    _pool_var (n) * sizeof ((P)[0]));			\
-  _pool_var (ph) = pool_header (P);					\
-  _pool_var (new_ph) = pool_header (_pool_var (new));			\
-  _pool_var (new_ph)->free_bitmap = 					\
-    clib_bitmap_dup (_pool_var (ph)->free_bitmap);			\
-  _pool_var (new_ph)->free_indices = 					\
-    vec_dup (_pool_var (ph)->free_indices);				\
-  _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts;		\
+  if ((P))								\
+    {									\
+      _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n),	\
+                                     _pool_var (n) * sizeof ((P)[0]), 	\
+                                     pool_aligned_header_bytes, (A));	\
+      clib_memcpy_fast (_pool_var (new), (P), 				\
+                        _pool_var (n) * sizeof ((P)[0]));		\
+      _pool_var (ph) = pool_header (P);					\
+      _pool_var (new_ph) = pool_header (_pool_var (new));		\
+      _pool_var (new_ph)->free_bitmap = 				\
+        clib_bitmap_dup (_pool_var (ph)->free_bitmap);			\
+      _pool_var (new_ph)->free_indices = 				\
+        vec_dup (_pool_var (ph)->free_indices);				\
+      _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts;		\
+    }									\
   _pool_var (new);							\
 })
 
@@ -373,7 +396,7 @@ do {									\
  * @param P pool to copy
  * @return copy of pool
  */
-#define pool_dup(P) pool_dup_aligned(P, 0)
+#define pool_dup(P) pool_dup_aligned(P,0)
 
 /** Low-level free pool operator (do not call directly). */
 always_inline void *