X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fpool.h;h=db950d27d1855df7c66a15eea355298338ca645a;hb=b4e5e50fe;hp=75d4c95ce11cdc20d6e2f44ac8a606e314ae7e05;hpb=bf3443b0f852f5a4c551d12f926defbd047f2161;p=vpp.git

diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h
index 75d4c95ce11..db950d27d18 100644
--- a/src/vppinfra/pool.h
+++ b/src/vppinfra/pool.h
@@ -185,12 +185,13 @@ pool_free_elts (void *v)
 
    First search free list.  If nothing is free extend vector of objects.
 */
-#define _pool_get_aligned_internal(P,E,A,Z)                             \
+#define _pool_get_aligned_internal_numa(P,E,A,Z,N)                      \
 do {                                                                    \
   pool_header_t * _pool_var (p) = pool_header (P);                      \
   uword _pool_var (l);                                                  \
                                                                         \
-  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \
+  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0)                         \
+                || ((sizeof(P[0]) % A) == 0),                           \
                 "Pool aligned alloc of incorrectly sized object");      \
   _pool_var (l) = 0;                                                    \
   if (P)                                                                \
@@ -199,12 +200,14 @@ do {                                                                    \
   if (_pool_var (l) > 0)                                                \
     {                                                                   \
       /* Return free element from free list. */                         \
-      uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \
+      uword _pool_var (i) =                                             \
+        _pool_var (p)->free_indices[_pool_var (l) - 1];                 \
       (E) = (P) + _pool_var (i);                                        \
-      _pool_var (p)->free_bitmap =					\
-	clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,        \
-	                             _pool_var (i));	               	\
+      _pool_var (p)->free_bitmap =                                      \
+	clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,         \
+	                             _pool_var (i));                    \
       _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1;       \
+      CLIB_MEM_UNPOISON((E), sizeof((E)[0]));                           \
     }                                                                   \
   else                                                                  \
     {                                                                   \
@@ -215,17 +218,30 @@ do {                                                                    \
           os_out_of_memory();                                           \
         }                                                               \
       /* Nothing on free list, make a new element and return it. */     \
-      P = _vec_resize (P,                                               \
+      P = _vec_resize_numa (P,                                          \
 		       /* length_increment */ 1,                        \
 		       /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
 		       pool_aligned_header_bytes,                       \
-		       /* align */ (A));                                \
+                       /* align */ (A),                                 \
+                       /* numa */ (N));                                 \
       E = vec_end (P) - 1;                                              \
-    }									\
+    }                                                                   \
   if (Z)                                                                \
-    memset(E, 0, sizeof(*E));						\
+    memset(E, 0, sizeof(*E));                                           \
 } while (0)
 
+#define pool_get_aligned_zero_numa(P,E,A,Z,S) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,S)
+
+#define pool_get_aligned_numa(P,E,A,S) \
+  _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S)
+
+#define pool_get_numa(P,E,S) \
+  _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S)
+
+#define _pool_get_aligned_internal(P,E,A,Z) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED)
+
 /** Allocate an object E from a pool P with alignment A */
 #define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)
 
@@ -285,10 +301,12 @@ do {                                                                    \
 /** Free an object E in pool P. */
 #define pool_put(P,E)							\
 do {									\
-  pool_header_t * _pool_var (p) = pool_header (P);			\
-  uword _pool_var (l) = (E) - (P);					\
-  ASSERT (vec_is_member (P, E));					\
-  ASSERT (! pool_is_free (P, E));					\
+  typeof (P) _pool_var(p__) = (P);			                \
+  typeof (E) _pool_var(e__) = (E);			                \
+  pool_header_t * _pool_var (p) = pool_header (_pool_var(p__));		\
+  uword _pool_var (l) = _pool_var(e__) - _pool_var(p__);		\
+  ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__)));		\
+  ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__)));		\
 									\
   /* Add element to free bitmap and to free list. */			\
   _pool_var (p)->free_bitmap =						\
@@ -305,6 +323,8 @@ do {									\
     }                                                                   \
   else                                                                  \
     vec_add1 (_pool_var (p)->free_indices, _pool_var (l));		\
+                                                                        \
+  CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0]));                                 \
 } while (0)
 
 /** Free pool element with given index. */
@@ -340,6 +360,44 @@ do {									\
 /** Allocate N more free elements to pool (unspecified alignment). */
 #define pool_alloc(P,N) pool_alloc_aligned(P,N,0)
 
+/**
+ * Return copy of pool with alignment
+ *
+ * @param P pool to copy
+ * @param A alignment (may be zero)
+ * @return copy of pool
+ */
+#define pool_dup_aligned(P,A)						\
+({									\
+  typeof (P) _pool_var (new) = 0;					\
+  pool_header_t * _pool_var (ph), * _pool_var (new_ph);			\
+  u32 _pool_var (n) = pool_len (P);					\
+  if ((P))								\
+    {									\
+      _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n),	\
+                                     _pool_var (n) * sizeof ((P)[0]), 	\
+                                     pool_aligned_header_bytes, (A));	\
+      clib_memcpy_fast (_pool_var (new), (P), 				\
+                        _pool_var (n) * sizeof ((P)[0]));		\
+      _pool_var (ph) = pool_header (P);					\
+      _pool_var (new_ph) = pool_header (_pool_var (new));		\
+      _pool_var (new_ph)->free_bitmap = 				\
+        clib_bitmap_dup (_pool_var (ph)->free_bitmap);			\
+      _pool_var (new_ph)->free_indices = 				\
+        vec_dup (_pool_var (ph)->free_indices);				\
+      _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts;		\
+    }									\
+  _pool_var (new);							\
+})
+
+/**
+ * Return copy of pool without alignment
+ *
+ * @param P pool to copy
+ * @return copy of pool
+ */
+#define pool_dup(P) pool_dup_aligned(P,0)
+
 /** Low-level free pool operator (do not call directly). */
 always_inline void *
 _pool_free (void *v)