#include <vppinfra/clib.h> /* uword, etc */
#include <vppinfra/clib_error.h>
+
+#if USE_DLMALLOC == 0
#include <vppinfra/mheap_bootstrap.h>
+#else
+#include <vppinfra/dlmalloc.h>
+#endif
+
#include <vppinfra/os.h>
-#include <vppinfra/string.h> /* memcpy, memset */
+#include <vppinfra/string.h> /* memcpy, clib_memset */
#include <vppinfra/valgrind.h>
#define CLIB_MAX_MHEAPS 256
/* Per CPU heaps. */
extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS];
+always_inline void
+clib_mem_set_thread_index (void)
+{
+ /*
+ * Find an unused slot in the per-cpu-mheaps array,
+ * and grab it for this thread. We need to be able to
+ * push/pop the thread heap without affecting other thread(s).
+ */
+ int i;
+ if (__os_thread_index != 0)
+ return;
+ for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++)
+ if (clib_atomic_bool_cmp_and_swap (&clib_per_cpu_mheaps[i],
+ 0, clib_per_cpu_mheaps[0]))
+ {
+ os_set_thread_index (i);
+ break;
+ }
+ ASSERT (__os_thread_index > 0);
+}
+
always_inline void *
clib_mem_get_per_cpu_heap (void)
{
int os_out_of_memory_on_failure)
{
void *heap, *p;
- uword offset, cpu;
+ uword cpu;
if (align_offset > align)
{
cpu = os_get_thread_index ();
heap = clib_per_cpu_mheaps[cpu];
+
+#if USE_DLMALLOC == 0
+ uword offset;
heap = mheap_get_aligned (heap, size, align, align_offset, &offset);
clib_per_cpu_mheaps[cpu] = heap;
os_out_of_memory ();
return 0;
}
+#else
+ p = mspace_get_aligned (heap, size, align, align_offset);
+ if (PREDICT_FALSE (p == 0))
+ {
+ if (os_out_of_memory_on_failure)
+ os_out_of_memory ();
+ return 0;
+ }
+
+ return p;
+#endif /* USE_DLMALLOC */
}
/* Memory allocator which calls os_out_of_memory() when it fails */
always_inline uword
clib_mem_is_heap_object (void *p)
{
+#if USE_DLMALLOC == 0
void *heap = clib_mem_get_per_cpu_heap ();
uword offset = (uword) p - (uword) heap;
mheap_elt_t *e, *n;
/* Check that heap forward and reverse pointers agree. */
return e->n_user_data == n->prev_n_user_data;
+#else
+ void *heap = clib_mem_get_per_cpu_heap ();
+
+ return mspace_is_heap_object (heap, p);
+#endif /* USE_DLMALLOC */
}
always_inline void
/* Make sure object is in the correct heap. */
ASSERT (clib_mem_is_heap_object (p));
+#if USE_DLMALLOC == 0
mheap_put (heap, (u8 *) p - heap);
+#else
+ mspace_put (heap, p);
+#endif
#if CLIB_DEBUG > 0
VALGRIND_FREELIKE_BLOCK (p, 0);
copy_size = old_size;
else
copy_size = new_size;
- clib_memcpy (q, p, copy_size);
+ clib_memcpy_fast (q, p, copy_size);
clib_mem_free (p);
}
return q;
always_inline uword
clib_mem_size (void *p)
{
- ASSERT (clib_mem_is_heap_object (p));
+#if USE_DLMALLOC == 0
mheap_elt_t *e = mheap_user_pointer_to_elt (p);
+ ASSERT (clib_mem_is_heap_object (p));
return mheap_elt_data_bytes (e);
+#else
+ ASSERT (clib_mem_is_heap_object (p));
+ return mspace_usable_size_with_delta (p);
+#endif
+}
+
+always_inline void
+clib_mem_free_s (void *p)
+{
+ uword size = clib_mem_size (p);
+ memset_s_inline (p, size, 0, size);
+ clib_mem_free (p);
}
always_inline void *
}
void *clib_mem_init (void *heap, uword size);
+void *clib_mem_init_thread_safe (void *memory, uword memory_size);
void clib_mem_exit (void);
void clib_mem_trace (int enable);
+int clib_mem_is_traced (void);
+
typedef struct
{
/* Total number of objects allocated. */
clib_mem_vm_map (void *addr, uword size)
{
void *mmap_addr;
- uword flags = MAP_PRIVATE | MAP_FIXED;
+ uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0);
if (mmap_addr == (void *) -1)
uword requested_va; /**< Request fixed position mapping */
} clib_mem_vm_alloc_t;
+clib_error_t *clib_mem_create_fd (char *name, int *fdp);
+clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp);
clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a);
-u64 clib_mem_vm_get_page_size (int fd);
-int clib_mem_vm_get_log2_page_size (int fd);
+void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a);
+u64 clib_mem_get_fd_page_size (int fd);
+uword clib_mem_get_default_hugepage_size (void);
+int clib_mem_get_fd_log2_page_size (int fd);
u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages);
typedef struct
clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a);
void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size);
+void mheap_trace (void *v, int enable);
+uword clib_mem_trace_enable_disable (uword enable);
+void clib_mem_trace (int enable);
#include <vppinfra/error.h> /* clib_panic */