vppinfra: numa vector placement support

[vpp.git] / src / vppinfra / vec_bootstrap.h

diff --git a/src/vppinfra/vec_bootstrap.h b/src/vppinfra/vec_bootstrap.h
index 3b8c770..879703c 100644 (file)
--- a/src/vppinfra/vec_bootstrap.h
+++ b/src/vppinfra/vec_bootstrap.h
@@ -39,7 +39,7 @@
 #define included_clib_vec_bootstrap_h
 
 /** \file
-    Vector bootsrap header file
+    Vector bootstrap header file
 */
 
 /* Bootstrap include so that #include <vppinfra/mem.h> can include e.g.
@@ -47,7 +47,7 @@
 
 /** \brief vector header structure
 
-   Bookeeping header preceding vector elements in memory.
+   Bookkeeping header preceding vector elements in memory.
    User header information may preceed standard vec header.
    If you change u32 len -> u64 len, single vectors can
    exceed 2**32 elements. Clib heaps are vectors. */
@@ -58,10 +58,14 @@ typedef struct
   u64 len;
 #else
   u32 len; /**< Number of elements in vector (NOT its allocated length). */
+  u8 numa_id; /**< NUMA id */
+  u8 vpad[3]; /**< pad to 8 bytes */
 #endif
   u8 vector_data[0];  /**< Vector data . */
 } vec_header_t;
 
+#define VEC_NUMA_UNSPECIFIED (0xFF)
+
 /** \brief Find the vector header
 
     Given the user's pointer to a vector, find the corresponding
@@ -127,7 +131,7 @@ vec_aligned_header_end (void *v, uword header_bytes, uword align)
 
 /** \brief Number of elements in vector (lvalue-capable)
 
-   _vec_len (v) does not check for null, but can be used as a lvalue
+   _vec_len (v) does not check for null, but can be used as an lvalue
    (e.g. _vec_len (v) = 99).
 */
 
@@ -141,11 +145,19 @@ vec_aligned_header_end (void *v, uword header_bytes, uword align)
 
 #define vec_len(v)     ((v) ? _vec_len(v) : 0)
 
-/** \brief Reset vector length to zero
-    NULL-pointer tolerant
+/** \brief Vector's NUMA id (lvalue-capable)
+
+    _vec_numa(v) does not check for null, but can be used as an lvalue
+    (e.g. _vec_numa(v) = 1).
 */
 
-#define vec_reset_length(v) do { if (v) _vec_len (v) = 0; } while (0)
+#define _vec_numa(v) (_vec_find(v)->numa_id)
+
+/** \brief Return vector's NUMA ID (rvalue-only, NULL tolerant)
+    vec_numa(v) checks for NULL, but cannot be used as an lvalue.
+*/
+#define vec_numa(v) ((v) ? _vec_numa(v) : 0)
+
 
 /** \brief Number of data bytes in vector. */
 
@@ -164,6 +176,25 @@ vec_aligned_header_end (void *v, uword header_bytes, uword align)
 /** \brief Total number of elements that can fit into vector. */
 #define vec_max_len(v) (vec_capacity(v,0) / sizeof (v[0]))
 
+/** \brief Set vector length to a user-defined value */
+#ifndef __COVERITY__           /* Coverity gets confused by ASSERT() */
+#define vec_set_len(v, l) do {     \
+    ASSERT(v);                     \
+    ASSERT((l) <= vec_max_len(v)); \
+    CLIB_MEM_POISON_LEN((void *)(v), _vec_len(v) * sizeof((v)[0]), (l) * sizeof((v)[0])); \
+    _vec_len(v) = (l);             \
+} while (0)
+#else /* __COVERITY__ */
+#define vec_set_len(v, l) do {     \
+    _vec_len(v) = (l);             \
+} while (0)
+#endif /* __COVERITY__ */
+
+/** \brief Reset vector length to zero
+    NULL-pointer tolerant
+*/
+#define vec_reset_length(v) do { if (v) vec_set_len (v, 0); } while (0)
+
 /** \brief End (last data address) of vector. */
 #define vec_end(v)     ((v) + vec_len (v))
 
@@ -190,6 +221,21 @@ for (var = vec_end (vec) - 1; var >= (vec); var--)
 /** \brief Iterate over vector indices. */
 #define vec_foreach_index(var,v) for ((var) = 0; (var) < vec_len (v); (var)++)
 
+/** \brief Iterate over vector indices (reverse). */
+#define vec_foreach_index_backwards(var,v) \
+  for ((var) = vec_len((v)) - 1; (var) >= 0; (var)--)
+
+/** \brief return the NUMA index for a vector */
+always_inline uword
+vec_get_numa (void *v)
+{
+  vec_header_t *vh;
+  if (v == 0)
+    return 0;
+  vh = _vec_find (v);
+  return vh->numa_id;
+}
+
 #endif /* included_clib_vec_bootstrap_h */
 
 /*