vppinfra: numa vector placement support

[vpp.git] / src / vppinfra / pool.h

diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h
index 47bae07..db950d2 100644 (file)
--- a/src/vppinfra/pool.h
+++ b/src/vppinfra/pool.h
@@ -185,12 +185,13 @@ pool_free_elts (void *v)
 
    First search free list.  If nothing is free extend vector of objects.
 */
-#define _pool_get_aligned_internal(P,E,A,Z)                             \
+#define _pool_get_aligned_internal_numa(P,E,A,Z,N)                      \
 do {                                                                    \
   pool_header_t * _pool_var (p) = pool_header (P);                      \
   uword _pool_var (l);                                                  \
                                                                         \
-  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \
+  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0)                         \
+                || ((sizeof(P[0]) % A) == 0),                           \
                 "Pool aligned alloc of incorrectly sized object");      \
   _pool_var (l) = 0;                                                    \
   if (P)                                                                \
@@ -199,12 +200,14 @@ do {                                                                    \
   if (_pool_var (l) > 0)                                                \
     {                                                                   \
       /* Return free element from free list. */                         \
-      uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \
+      uword _pool_var (i) =                                             \
+        _pool_var (p)->free_indices[_pool_var (l) - 1];                 \
       (E) = (P) + _pool_var (i);                                        \
-      _pool_var (p)->free_bitmap =                                     \
-       clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,        \
-                                    _pool_var (i));                    \
+      _pool_var (p)->free_bitmap =                                      \
+       clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,         \
+                                    _pool_var (i));                    \
       _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1;       \
+      CLIB_MEM_UNPOISON((E), sizeof((E)[0]));                           \
     }                                                                   \
   else                                                                  \
     {                                                                   \
@@ -215,17 +218,30 @@ do {                                                                    \
           os_out_of_memory();                                           \
         }                                                               \
       /* Nothing on free list, make a new element and return it. */     \
-      P = _vec_resize (P,                                               \
+      P = _vec_resize_numa (P,                                          \
                       /* length_increment */ 1,                        \
                       /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
                       pool_aligned_header_bytes,                       \
-                      /* align */ (A));                                \
+                       /* align */ (A),                                 \
+                       /* numa */ (N));                                 \
       E = vec_end (P) - 1;                                              \
-    }                                                                  \
+    }                                                                   \
   if (Z)                                                                \
-    memset(E, 0, sizeof(*E));                                          \
+    memset(E, 0, sizeof(*E));                                           \
 } while (0)
 
+#define pool_get_aligned_zero_numa(P,E,A,Z,S) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,S)
+
+#define pool_get_aligned_numa(P,E,A,S) \
+  _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S)
+
+#define pool_get_numa(P,E,S) \
+  _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S)
+
+#define _pool_get_aligned_internal(P,E,A,Z) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED)
+
 /** Allocate an object E from a pool P with alignment A */
 #define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)
 
@@ -285,10 +301,12 @@ do {                                                                    \
 /** Free an object E in pool P. */
 #define pool_put(P,E)                                                  \
 do {                                                                   \
-  pool_header_t * _pool_var (p) = pool_header (P);                     \
-  uword _pool_var (l) = (E) - (P);                                     \
-  ASSERT (vec_is_member (P, E));                                       \
-  ASSERT (! pool_is_free (P, E));                                      \
+  typeof (P) _pool_var(p__) = (P);                                     \
+  typeof (E) _pool_var(e__) = (E);                                     \
+  pool_header_t * _pool_var (p) = pool_header (_pool_var(p__));                \
+  uword _pool_var (l) = _pool_var(e__) - _pool_var(p__);               \
+  ASSERT (vec_is_member (_pool_var(p__), _pool_var(e__)));             \
+  ASSERT (! pool_is_free (_pool_var(p__), _pool_var(e__)));            \
                                                                        \
   /* Add element to free bitmap and to free list. */                   \
   _pool_var (p)->free_bitmap =                                         \
@@ -305,6 +323,8 @@ do {                                                                        \
     }                                                                   \
   else                                                                  \
     vec_add1 (_pool_var (p)->free_indices, _pool_var (l));             \
+                                                                        \
+  CLIB_MEM_POISON(_pool_var(e__), sizeof(_pool_var(e__)[0]));                                 \
 } while (0)
 
 /** Free pool element with given index. */
@@ -352,18 +372,21 @@ do {                                                                      \
   typeof (P) _pool_var (new) = 0;                                      \
   pool_header_t * _pool_var (ph), * _pool_var (new_ph);                        \
   u32 _pool_var (n) = pool_len (P);                                    \
-  _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n),       \
-                                _pool_var (n) * sizeof ((P)[0]),       \
-                                pool_aligned_header_bytes, (A));       \
-  clib_memcpy_fast (_pool_var (new), (P),                              \
-                   _pool_var (n) * sizeof ((P)[0]));                   \
-  _pool_var (ph) = pool_header (P);                                    \
-  _pool_var (new_ph) = pool_header (_pool_var (new));                  \
-  _pool_var (new_ph)->free_bitmap =                                    \
-    clib_bitmap_dup (_pool_var (ph)->free_bitmap);                     \
-  _pool_var (new_ph)->free_indices =                                   \
-    vec_dup (_pool_var (ph)->free_indices);                            \
-  _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts;             \
+  if ((P))                                                             \
+    {                                                                  \
+      _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n),   \
+                                     _pool_var (n) * sizeof ((P)[0]),  \
+                                     pool_aligned_header_bytes, (A));  \
+      clib_memcpy_fast (_pool_var (new), (P),                          \
+                        _pool_var (n) * sizeof ((P)[0]));              \
+      _pool_var (ph) = pool_header (P);                                        \
+      _pool_var (new_ph) = pool_header (_pool_var (new));              \
+      _pool_var (new_ph)->free_bitmap =                                \
+        clib_bitmap_dup (_pool_var (ph)->free_bitmap);                 \
+      _pool_var (new_ph)->free_indices =                               \
+        vec_dup (_pool_var (ph)->free_indices);                                \
+      _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts;         \
+    }                                                                  \
   _pool_var (new);                                                     \
 })
 
@@ -373,7 +396,7 @@ do {                                                                        \
  * @param P pool to copy
  * @return copy of pool
  */
-#define pool_dup(P) pool_dup_aligned(P, 0)
+#define pool_dup(P) pool_dup_aligned(P,0)
 
 /** Low-level free pool operator (do not call directly). */
 always_inline void *