vppinfra: numa vector placement support

[vpp.git] / src / vppinfra / pool.h

diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h
index e6ffe1e..db950d2 100644 (file)
--- a/src/vppinfra/pool.h
+++ b/src/vppinfra/pool.h
@@ -185,12 +185,13 @@ pool_free_elts (void *v)
 
    First search free list.  If nothing is free extend vector of objects.
 */
-#define _pool_get_aligned_internal(P,E,A,Z)                             \
+#define _pool_get_aligned_internal_numa(P,E,A,Z,N)                      \
 do {                                                                    \
   pool_header_t * _pool_var (p) = pool_header (P);                      \
   uword _pool_var (l);                                                  \
                                                                         \
-  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \
+  STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0)                         \
+                || ((sizeof(P[0]) % A) == 0),                           \
                 "Pool aligned alloc of incorrectly sized object");      \
   _pool_var (l) = 0;                                                    \
   if (P)                                                                \
@@ -199,11 +200,12 @@ do {                                                                    \
   if (_pool_var (l) > 0)                                                \
     {                                                                   \
       /* Return free element from free list. */                         \
-      uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \
+      uword _pool_var (i) =                                             \
+        _pool_var (p)->free_indices[_pool_var (l) - 1];                 \
       (E) = (P) + _pool_var (i);                                        \
-      _pool_var (p)->free_bitmap =                                     \
-       clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,        \
-                                    _pool_var (i));                    \
+      _pool_var (p)->free_bitmap =                                      \
+       clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap,         \
+                                    _pool_var (i));                    \
       _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1;       \
       CLIB_MEM_UNPOISON((E), sizeof((E)[0]));                           \
     }                                                                   \
@@ -216,17 +218,30 @@ do {                                                                    \
           os_out_of_memory();                                           \
         }                                                               \
       /* Nothing on free list, make a new element and return it. */     \
-      P = _vec_resize (P,                                               \
+      P = _vec_resize_numa (P,                                          \
                       /* length_increment */ 1,                        \
                       /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \
                       pool_aligned_header_bytes,                       \
-                      /* align */ (A));                                \
+                       /* align */ (A),                                 \
+                       /* numa */ (N));                                 \
       E = vec_end (P) - 1;                                              \
-    }                                                                  \
+    }                                                                   \
   if (Z)                                                                \
-    memset(E, 0, sizeof(*E));                                          \
+    memset(E, 0, sizeof(*E));                                           \
 } while (0)
 
+#define pool_get_aligned_zero_numa(P,E,A,Z,S) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,S)
+
+#define pool_get_aligned_numa(P,E,A,S) \
+  _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S)
+
+#define pool_get_numa(P,E,S) \
+  _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S)
+
+#define _pool_get_aligned_internal(P,E,A,Z) \
+  _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED)
+
 /** Allocate an object E from a pool P with alignment A */
 #define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0)