X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fmem.h;h=1cab0ae7252b1c7149660273e9ba62626756015a;hb=dd9299c68a02b7c476cb60b8d9ac4e145503a44f;hp=e998eb0f7bc1ccde6dad301d1d6594dca017d574;hpb=459a0c4e3be1473c4c2b93811280c738e60d0524;p=vpp.git
diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index e998eb0f7bc..1cab0ae7252 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -93,6 +93,38 @@ typedef struct _clib_mem_vm_map_hdr
struct _clib_mem_vm_map_hdr *prev, *next;
} clib_mem_vm_map_hdr_t;
+#define foreach_clib_mem_heap_flag \
+ _(0, LOCKED, "locked") \
+ _(1, UNMAP_ON_DESTROY, "unmap-on-destroy")
+
+typedef enum
+{
+#define _(i, v, s) CLIB_MEM_HEAP_F_##v = (1 << i),
+ foreach_clib_mem_heap_flag
+#undef _
+} clib_mem_heap_flag_t;
+
+typedef struct
+{
+ /* base address */
+ void *base;
+
+ /* dlmalloc mspace */
+ void *mspace;
+
+ /* heap size */
+ uword size;
+
+ /* page size (log2) */
+ clib_mem_page_sz_t log2_page_sz:8;
+
+ /* flags */
+ clib_mem_heap_flag_t flags:8;
+
+ /* name - _MUST_ be last */
+ char name[0];
+} clib_mem_heap_t;
+
typedef struct
{
/* log2 system page size */
@@ -113,6 +145,9 @@ typedef struct
/* memory maps */
clib_mem_vm_map_hdr_t *first_map, *last_map;
+ /* map lock */
+ u8 map_lock;
+
/* last error */
clib_error_t *error;
} clib_mem_main_t;
@@ -122,7 +157,7 @@ extern clib_mem_main_t clib_mem_main;
/* Unspecified NUMA socket */
#define VEC_NUMA_UNSPECIFIED (0xFF)
-always_inline void *
+always_inline clib_mem_heap_t *
clib_mem_get_per_cpu_heap (void)
{
int cpu = os_get_thread_index ();
@@ -130,7 +165,7 @@ clib_mem_get_per_cpu_heap (void)
}
always_inline void *
-clib_mem_set_per_cpu_heap (u8 * new_heap)
+clib_mem_set_per_cpu_heap (void *new_heap)
{
int cpu = os_get_thread_index ();
void *old = clib_mem_main.per_cpu_mheaps[cpu];
@@ -146,7 +181,7 @@ clib_mem_get_per_numa_heap (u32 numa_id)
}
always_inline void *
-clib_mem_set_per_numa_heap (u8 * new_heap)
+clib_mem_set_per_numa_heap (void *new_heap)
{
int numa = os_get_numa_index ();
void *old = clib_mem_main.per_numa_mheaps[numa];
@@ -187,10 +222,10 @@ always_inline void *
clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
int os_out_of_memory_on_failure)
{
- void *heap, *p;
- uword cpu;
void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes,
unsigned long align, unsigned long align_offset);
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ void *p;
if (align_offset > align)
{
@@ -200,10 +235,7 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
align_offset = align;
}
- cpu = os_get_thread_index ();
- heap = clib_mem_main.per_cpu_mheaps[cpu];
-
- p = mspace_get_aligned (heap, size, align, align_offset);
+ p = mspace_get_aligned (h->mspace, size, align, align_offset);
if (PREDICT_FALSE (0 == p))
{
@@ -270,24 +302,23 @@ clib_mem_alloc_aligned_or_null (uword size, uword align)
always_inline uword
clib_mem_is_heap_object (void *p)
{
- void *heap = clib_mem_get_per_cpu_heap ();
int mspace_is_heap_object (void *msp, void *p);
-
- return mspace_is_heap_object (heap, p);
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ return mspace_is_heap_object (h->mspace, p);
}
always_inline void
clib_mem_free (void *p)
{
- u8 *heap = clib_mem_get_per_cpu_heap ();
-
void mspace_put (void *msp, void *p_arg);
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+
/* Make sure object is in the correct heap. */
ASSERT (clib_mem_is_heap_object (p));
CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
- mspace_put (heap, p);
+ mspace_put (h->mspace, p);
}
always_inline void *
@@ -324,29 +355,27 @@ clib_mem_free_s (void *p)
clib_mem_free (p);
}
-always_inline void *
+always_inline clib_mem_heap_t *
clib_mem_get_heap (void)
{
return clib_mem_get_per_cpu_heap ();
}
-always_inline void *
-clib_mem_set_heap (void *heap)
+always_inline clib_mem_heap_t *
+clib_mem_set_heap (clib_mem_heap_t * heap)
{
return clib_mem_set_per_cpu_heap (heap);
}
-void clib_mem_destroy_heap (void *heap);
-void *clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt,
- ...);
+void clib_mem_destroy_heap (clib_mem_heap_t * heap);
+clib_mem_heap_t *clib_mem_create_heap (void *base, uword size, int is_locked,
+ char *fmt, ...);
void clib_mem_main_init ();
-void *clib_mem_init (void *heap, uword size);
+void *clib_mem_init (void *base, uword size);
void *clib_mem_init_with_page_size (uword memory_size,
clib_mem_page_sz_t log2_page_sz);
void *clib_mem_init_thread_safe (void *memory, uword memory_size);
-void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size,
- u8 numa);
void clib_mem_exit (void);
@@ -379,14 +408,16 @@ typedef struct
uword bytes_max;
} clib_mem_usage_t;
-void clib_mem_get_heap_usage (void *heap, clib_mem_usage_t * usage);
+void clib_mem_get_heap_usage (clib_mem_heap_t * heap,
+ clib_mem_usage_t * usage);
-void *clib_mem_get_heap_base (void *heap);
-uword clib_mem_get_heap_size (void *heap);
-uword clib_mem_get_heap_free_space (void *heap);
+void *clib_mem_get_heap_base (clib_mem_heap_t * heap);
+uword clib_mem_get_heap_size (clib_mem_heap_t * heap);
+uword clib_mem_get_heap_free_space (clib_mem_heap_t * heap);
u8 *format_clib_mem_usage (u8 * s, va_list * args);
u8 *format_clib_mem_heap (u8 * s, va_list * va);
+u8 *format_clib_mem_page_stats (u8 * s, va_list * va);
/* Allocate virtual address space. */
always_inline void *
@@ -427,36 +458,6 @@ int clib_mem_vm_unmap (void *base);
clib_mem_vm_map_hdr_t *clib_mem_vm_get_next_map_hdr (clib_mem_vm_map_hdr_t *
hdr);
-typedef struct
-{
-#define CLIB_MEM_VM_F_SHARED (1 << 0)
-#define CLIB_MEM_VM_F_HUGETLB (1 << 1)
-#define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2)
-#define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3)
-#define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4)
-#define CLIB_MEM_VM_F_LOCKED (1 << 5)
- u32 flags; /**< vm allocation flags:
-
CLIB_MEM_VM_F_SHARED: request shared memory, file
- descriptor will be provided on successful allocation.
-
CLIB_MEM_VM_F_HUGETLB: request hugepages.
-
CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid
- numa node preference.
-
CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails.
-
CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if
- number of available pages is not sufficient.
-
CLIB_MEM_VM_F_LOCKED: request locked memory.
- */
- char *name; /**< Name for memory allocation, set by caller. */
- uword size; /**< Allocation size, set by caller. */
- int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */
- void *addr; /**< Pointer to allocated memory, set on successful allocation. */
- int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */
- int log2_page_size; /* Page size in log2 format, set on successful allocation. */
- int n_pages; /* Number of pages. */
- uword requested_va; /**< Request fixed position mapping */
-} clib_mem_vm_alloc_t;
-
-
static_always_inline clib_mem_page_sz_t
clib_mem_get_log2_page_size (void)
{
@@ -476,8 +477,6 @@ clib_mem_get_log2_default_hugepage_size ()
}
int clib_mem_vm_create_fd (clib_mem_page_sz_t log2_page_size, char *fmt, ...);
-clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a);
-void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a);
uword clib_mem_get_fd_page_size (int fd);
uword clib_mem_get_default_hugepage_size (void);
clib_mem_page_sz_t clib_mem_get_fd_log2_page_size (int fd);
@@ -488,19 +487,9 @@ u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size,
void clib_mem_destroy (void);
int clib_mem_set_numa_affinity (u8 numa_node, int force);
int clib_mem_set_default_numa_affinity ();
-
-typedef struct
-{
- uword size; /**< Map size */
- int fd; /**< File descriptor to be mapped */
- uword requested_va; /**< Request fixed position mapping */
- void *addr; /**< Pointer to mapped memory, if successful */
- u8 numa_node;
-} clib_mem_vm_map_t;
-
void clib_mem_vm_randomize_va (uword * requested_va,
clib_mem_page_sz_t log2_page_size);
-void mheap_trace (void *v, int enable);
+void mheap_trace (clib_mem_heap_t * v, int enable);
uword clib_mem_trace_enable_disable (uword enable);
void clib_mem_trace (int enable);
@@ -519,6 +508,8 @@ clib_mem_round_to_page_size (uword size, clib_mem_page_sz_t log2_page_size)
typedef struct
{
+ clib_mem_page_sz_t log2_page_sz;
+ uword total;
uword mapped;
uword not_mapped;
uword per_numa[CLIB_MAX_NUMAS];
@@ -555,7 +546,7 @@ clib_mem_log2_page_size_validate (clib_mem_page_sz_t log2_page_size)
static_always_inline uword
clib_mem_page_bytes (clib_mem_page_sz_t log2_page_size)
{
- return 1 << clib_mem_log2_page_size_validate (log2_page_size);
+ return 1ULL << clib_mem_log2_page_size_validate (log2_page_size);
}
static_always_inline clib_error_t *
@@ -564,6 +555,15 @@ clib_mem_get_last_error (void)
return clib_mem_main.error;
}
+/* bulk allocator */
+
+typedef void *clib_mem_bulk_handle_t;
+clib_mem_bulk_handle_t clib_mem_bulk_init (u32 elt_sz, u32 align,
+ u32 min_elts_per_chunk);
+void clib_mem_bulk_destroy (clib_mem_bulk_handle_t h);
+void *clib_mem_bulk_alloc (clib_mem_bulk_handle_t h);
+void clib_mem_bulk_free (clib_mem_bulk_handle_t h, void *p);
+u8 *format_clib_mem_bulk (u8 *s, va_list *args);
#include /* clib_panic */