#include <vppinfra/clib.h> /* uword, etc */
#include <vppinfra/clib_error.h>
-#if USE_DLMALLOC == 0
-#include <vppinfra/mheap_bootstrap.h>
-#else
#include <vppinfra/dlmalloc.h>
-#endif
#include <vppinfra/os.h>
#include <vppinfra/string.h> /* memcpy, clib_memset */
-#include <vppinfra/valgrind.h>
+#include <vppinfra/sanitizer.h>
#define CLIB_MAX_MHEAPS 256
+#define CLIB_MAX_NUMAS 8
+
+/* Unspecified NUMA socket */
+#define VEC_NUMA_UNSPECIFIED (0xFF)
/* Per CPU heaps. */
extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS];
+extern void *clib_per_numa_mheaps[CLIB_MAX_NUMAS];
always_inline void *
clib_mem_get_per_cpu_heap (void)
return old;
}
+always_inline void *
+clib_mem_get_per_numa_heap (u32 numa_id)
+{
+ ASSERT (numa_id < ARRAY_LEN (clib_per_numa_mheaps));
+ return clib_per_numa_mheaps[numa_id];
+}
+
+always_inline void *
+clib_mem_set_per_numa_heap (u8 * new_heap)
+{
+ int numa = os_get_numa_index ();
+ void *old = clib_per_numa_mheaps[numa];
+ clib_per_numa_mheaps[numa] = new_heap;
+ return old;
+}
+
+always_inline void
+clib_mem_set_thread_index (void)
+{
+ /*
+ * Find an unused slot in the per-cpu-mheaps array,
+ * and grab it for this thread. We need to be able to
+ * push/pop the thread heap without affecting other thread(s).
+ */
+ int i;
+ if (__os_thread_index != 0)
+ return;
+ for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++)
+ if (clib_atomic_bool_cmp_and_swap (&clib_per_cpu_mheaps[i],
+ 0, clib_per_cpu_mheaps[0]))
+ {
+ os_set_thread_index (i);
+ break;
+ }
+ ASSERT (__os_thread_index > 0);
+}
+
+always_inline uword
+clib_mem_size_nocheck (void *p)
+{
+ return mspace_usable_size_with_delta (p);
+}
+
/* Memory allocator which may call os_out_of_memory() if it fails */
always_inline void *
clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
cpu = os_get_thread_index ();
heap = clib_per_cpu_mheaps[cpu];
-#if USE_DLMALLOC == 0
- uword offset;
- heap = mheap_get_aligned (heap, size, align, align_offset, &offset);
- clib_per_cpu_mheaps[cpu] = heap;
-
- if (offset != ~0)
- {
- p = heap + offset;
-#if CLIB_DEBUG > 0
- VALGRIND_MALLOCLIKE_BLOCK (p, mheap_data_bytes (heap, offset), 0, 0);
-#endif
- return p;
- }
- else
- {
- if (os_out_of_memory_on_failure)
- os_out_of_memory ();
- return 0;
- }
-#else
p = mspace_get_aligned (heap, size, align, align_offset);
- if (PREDICT_FALSE (p == 0))
+
+ if (PREDICT_FALSE (0 == p))
{
if (os_out_of_memory_on_failure)
os_out_of_memory ();
return 0;
}
+ CLIB_MEM_UNPOISON (p, size);
return p;
-#endif /* USE_DLMALLOC */
}
/* Memory allocator which calls os_out_of_memory() when it fails */
always_inline uword
clib_mem_is_heap_object (void *p)
{
-#if USE_DLMALLOC == 0
- void *heap = clib_mem_get_per_cpu_heap ();
- uword offset = (uword) p - (uword) heap;
- mheap_elt_t *e, *n;
-
- if (offset >= vec_len (heap))
- return 0;
-
- e = mheap_elt_at_uoffset (heap, offset);
- n = mheap_next_elt (e);
-
- /* Check that heap forward and reverse pointers agree. */
- return e->n_user_data == n->prev_n_user_data;
-#else
void *heap = clib_mem_get_per_cpu_heap ();
return mspace_is_heap_object (heap, p);
-#endif /* USE_DLMALLOC */
}
always_inline void
/* Make sure object is in the correct heap. */
ASSERT (clib_mem_is_heap_object (p));
-#if USE_DLMALLOC == 0
- mheap_put (heap, (u8 *) p - heap);
-#else
- mspace_put (heap, p);
-#endif
+ CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
-#if CLIB_DEBUG > 0
- VALGRIND_FREELIKE_BLOCK (p, 0);
-#endif
+ mspace_put (heap, p);
}
always_inline void *
always_inline uword
clib_mem_size (void *p)
{
-#if USE_DLMALLOC == 0
- mheap_elt_t *e = mheap_user_pointer_to_elt (p);
- ASSERT (clib_mem_is_heap_object (p));
- return mheap_elt_data_bytes (e);
-#else
ASSERT (clib_mem_is_heap_object (p));
- return mspace_usable_size_with_delta (p);
-#endif
+ return clib_mem_size_nocheck (p);
+}
+
+always_inline void
+clib_mem_free_s (void *p)
+{
+ uword size = clib_mem_size (p);
+ CLIB_MEM_UNPOISON (p, size);
+ memset_s_inline (p, size, 0, size);
+ clib_mem_free (p);
}
always_inline void *
void *clib_mem_init (void *heap, uword size);
void *clib_mem_init_thread_safe (void *memory, uword memory_size);
+void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size,
+ u8 numa);
void clib_mem_exit (void);
void clib_mem_trace (int enable);
+int clib_mem_is_traced (void);
+
typedef struct
{
/* Total number of objects allocated. */
mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0);
if (mmap_addr == (void *) -1)
mmap_addr = 0;
+ else
+ CLIB_MEM_UNPOISON (mmap_addr, size);
return mmap_addr;
}
clib_mem_vm_free (void *addr, uword size)
{
munmap (addr, size);
+ CLIB_MEM_POISON (addr, size);
}
always_inline void *
mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0);
if (mmap_addr == (void *) -1)
mmap_addr = 0;
+ else
+ CLIB_MEM_UNPOISON (mmap_addr, size);
return mmap_addr;
}
mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0);
if (mmap_addr == (void *) -1)
mmap_addr = 0;
+ else
+ CLIB_MEM_UNPOISON (mmap_addr, size);
return mmap_addr;
}
int fd; /**< File descriptor to be mapped */
uword requested_va; /**< Request fixed position mapping */
void *addr; /**< Pointer to mapped memory, if successful */
+ u8 numa_node;
} clib_mem_vm_map_t;
clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a);
Code Review / vpp.git / blobdiff