X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fmem.h;h=1cab0ae7252b1c7149660273e9ba62626756015a;hb=041372b79b843b54dfad048dda237e64d1a5f127;hp=e6f019cd097258316964eaa18be5c5f6c29472dc;hpb=6bfd07670b991c30761ef74fb09f42181dbfd182;p=vpp.git diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index e6f019cd097..1cab0ae7252 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -45,8 +45,6 @@ #include /* uword, etc */ #include -#include - #include #include /* memcpy, clib_memset */ #include @@ -54,6 +52,7 @@ #define CLIB_MAX_MHEAPS 256 #define CLIB_MAX_NUMAS 16 #define CLIB_MEM_VM_MAP_FAILED ((void *) ~0) +#define CLIB_MEM_ERROR (-1) typedef enum { @@ -83,6 +82,9 @@ typedef struct _clib_mem_vm_map_hdr /* page size (log2) */ clib_mem_page_sz_t log2_page_sz; + /* file descriptor, -1 if memory is not shared */ + int fd; + /* allocation mame */ #define CLIB_VM_MAP_HDR_NAME_MAX_LEN 64 char name[CLIB_VM_MAP_HDR_NAME_MAX_LEN]; @@ -91,6 +93,38 @@ typedef struct _clib_mem_vm_map_hdr struct _clib_mem_vm_map_hdr *prev, *next; } clib_mem_vm_map_hdr_t; +#define foreach_clib_mem_heap_flag \ + _(0, LOCKED, "locked") \ + _(1, UNMAP_ON_DESTROY, "unmap-on-destroy") + +typedef enum +{ +#define _(i, v, s) CLIB_MEM_HEAP_F_##v = (1 << i), + foreach_clib_mem_heap_flag +#undef _ +} clib_mem_heap_flag_t; + +typedef struct +{ + /* base address */ + void *base; + + /* dlmalloc mspace */ + void *mspace; + + /* heap size */ + uword size; + + /* page size (log2) */ + clib_mem_page_sz_t log2_page_sz:8; + + /* flags */ + clib_mem_heap_flag_t flags:8; + + /* name - _MUST_ be last */ + char name[0]; +} clib_mem_heap_t; + typedef struct { /* log2 system page size */ @@ -110,6 +144,12 @@ typedef struct /* memory maps */ clib_mem_vm_map_hdr_t *first_map, *last_map; + + /* map lock */ + u8 map_lock; + + /* last error */ + clib_error_t *error; } clib_mem_main_t; extern clib_mem_main_t clib_mem_main; @@ -117,7 +157,7 @@ extern clib_mem_main_t clib_mem_main; /* Unspecified NUMA socket */ #define VEC_NUMA_UNSPECIFIED (0xFF) -always_inline void * +always_inline clib_mem_heap_t * clib_mem_get_per_cpu_heap (void) { int cpu = os_get_thread_index (); @@ -125,7 +165,7 @@ clib_mem_get_per_cpu_heap (void) } always_inline void * -clib_mem_set_per_cpu_heap (u8 * new_heap) +clib_mem_set_per_cpu_heap (void *new_heap) { int cpu = os_get_thread_index (); void *old = clib_mem_main.per_cpu_mheaps[cpu]; @@ -141,7 +181,7 @@ clib_mem_get_per_numa_heap (u32 numa_id) } always_inline void * -clib_mem_set_per_numa_heap (u8 * new_heap) +clib_mem_set_per_numa_heap (void *new_heap) { int numa = os_get_numa_index (); void *old = clib_mem_main.per_numa_mheaps[numa]; @@ -173,6 +213,7 @@ clib_mem_set_thread_index (void) always_inline uword clib_mem_size_nocheck (void *p) { + size_t mspace_usable_size_with_delta (const void *p); return mspace_usable_size_with_delta (p); } @@ -181,8 +222,10 @@ always_inline void * clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, int os_out_of_memory_on_failure) { - void *heap, *p; - uword cpu; + void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes, + unsigned long align, unsigned long align_offset); + clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + void *p; if (align_offset > align) { @@ -192,10 +235,7 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, align_offset = align; } - cpu = os_get_thread_index (); - heap = clib_mem_main.per_cpu_mheaps[cpu]; - - p = mspace_get_aligned (heap, size, align, align_offset); + p = mspace_get_aligned (h->mspace, size, align, align_offset); if (PREDICT_FALSE (0 == p)) { @@ -262,22 +302,23 @@ clib_mem_alloc_aligned_or_null (uword size, uword align) always_inline uword clib_mem_is_heap_object (void *p) { - void *heap = clib_mem_get_per_cpu_heap (); - - return mspace_is_heap_object (heap, p); + int mspace_is_heap_object (void *msp, void *p); + clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + return mspace_is_heap_object (h->mspace, p); } always_inline void clib_mem_free (void *p) { - u8 *heap = clib_mem_get_per_cpu_heap (); + void mspace_put (void *msp, void *p_arg); + clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); /* Make sure object is in the correct heap. */ ASSERT (clib_mem_is_heap_object (p)); CLIB_MEM_POISON (p, clib_mem_size_nocheck (p)); - mspace_put (heap, p); + mspace_put (h->mspace, p); } always_inline void * @@ -314,30 +355,30 @@ clib_mem_free_s (void *p) clib_mem_free (p); } -always_inline void * +always_inline clib_mem_heap_t * clib_mem_get_heap (void) { return clib_mem_get_per_cpu_heap (); } -always_inline void * -clib_mem_set_heap (void *heap) +always_inline clib_mem_heap_t * +clib_mem_set_heap (clib_mem_heap_t * heap) { return clib_mem_set_per_cpu_heap (heap); } +void clib_mem_destroy_heap (clib_mem_heap_t * heap); +clib_mem_heap_t *clib_mem_create_heap (void *base, uword size, int is_locked, + char *fmt, ...); + void clib_mem_main_init (); -void *clib_mem_init (void *heap, uword size); +void *clib_mem_init (void *base, uword size); void *clib_mem_init_with_page_size (uword memory_size, clib_mem_page_sz_t log2_page_sz); void *clib_mem_init_thread_safe (void *memory, uword memory_size); -void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size, - u8 numa); void clib_mem_exit (void); -void clib_mem_validate (void); - void clib_mem_trace (int enable); int clib_mem_is_traced (void); @@ -367,9 +408,16 @@ typedef struct uword bytes_max; } clib_mem_usage_t; -void clib_mem_usage (clib_mem_usage_t * usage); +void clib_mem_get_heap_usage (clib_mem_heap_t * heap, + clib_mem_usage_t * usage); + +void *clib_mem_get_heap_base (clib_mem_heap_t * heap); +uword clib_mem_get_heap_size (clib_mem_heap_t * heap); +uword clib_mem_get_heap_free_space (clib_mem_heap_t * heap); u8 *format_clib_mem_usage (u8 * s, va_list * args); +u8 *format_clib_mem_heap (u8 * s, va_list * va); +u8 *format_clib_mem_page_stats (u8 * s, va_list * va); /* Allocate virtual address space. */ always_inline void * @@ -410,36 +458,6 @@ int clib_mem_vm_unmap (void *base); clib_mem_vm_map_hdr_t *clib_mem_vm_get_next_map_hdr (clib_mem_vm_map_hdr_t * hdr); -typedef struct -{ -#define CLIB_MEM_VM_F_SHARED (1 << 0) -#define CLIB_MEM_VM_F_HUGETLB (1 << 1) -#define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2) -#define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3) -#define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4) -#define CLIB_MEM_VM_F_LOCKED (1 << 5) - u32 flags; /**< vm allocation flags: -
CLIB_MEM_VM_F_SHARED: request shared memory, file - descriptor will be provided on successful allocation. -
CLIB_MEM_VM_F_HUGETLB: request hugepages. -
CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid - numa node preference. -
CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails. -
CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if - number of available pages is not sufficient. -
CLIB_MEM_VM_F_LOCKED: request locked memory. - */ - char *name; /**< Name for memory allocation, set by caller. */ - uword size; /**< Allocation size, set by caller. */ - int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */ - void *addr; /**< Pointer to allocated memory, set on successful allocation. */ - int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */ - int log2_page_size; /* Page size in log2 format, set on successful allocation. */ - int n_pages; /* Number of pages. */ - uword requested_va; /**< Request fixed position mapping */ -} clib_mem_vm_alloc_t; - - static_always_inline clib_mem_page_sz_t clib_mem_get_log2_page_size (void) { @@ -458,10 +476,7 @@ clib_mem_get_log2_default_hugepage_size () return clib_mem_main.log2_default_hugepage_sz; } -clib_error_t *clib_mem_create_fd (char *name, int *fdp); -clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp); -clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a); -void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a); +int clib_mem_vm_create_fd (clib_mem_page_sz_t log2_page_size, char *fmt, ...); uword clib_mem_get_fd_page_size (int fd); uword clib_mem_get_default_hugepage_size (void); clib_mem_page_sz_t clib_mem_get_fd_log2_page_size (int fd); @@ -469,22 +484,12 @@ uword clib_mem_vm_reserve (uword start, uword size, clib_mem_page_sz_t log2_page_sz); u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size, int n_pages); -void clib_mem_destroy_mspace (void *mspace); void clib_mem_destroy (void); - -typedef struct -{ - uword size; /**< Map size */ - int fd; /**< File descriptor to be mapped */ - uword requested_va; /**< Request fixed position mapping */ - void *addr; /**< Pointer to mapped memory, if successful */ - u8 numa_node; -} clib_mem_vm_map_t; - -clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a); +int clib_mem_set_numa_affinity (u8 numa_node, int force); +int clib_mem_set_default_numa_affinity (); void clib_mem_vm_randomize_va (uword * requested_va, clib_mem_page_sz_t log2_page_size); -void mheap_trace (void *v, int enable); +void mheap_trace (clib_mem_heap_t * v, int enable); uword clib_mem_trace_enable_disable (uword enable); void clib_mem_trace (int enable); @@ -503,6 +508,8 @@ clib_mem_round_to_page_size (uword size, clib_mem_page_sz_t log2_page_size) typedef struct { + clib_mem_page_sz_t log2_page_sz; + uword total; uword mapped; uword not_mapped; uword per_numa[CLIB_MAX_NUMAS]; @@ -539,9 +546,24 @@ clib_mem_log2_page_size_validate (clib_mem_page_sz_t log2_page_size) static_always_inline uword clib_mem_page_bytes (clib_mem_page_sz_t log2_page_size) { - return 1 << clib_mem_log2_page_size_validate (log2_page_size); + return 1ULL << clib_mem_log2_page_size_validate (log2_page_size); } +static_always_inline clib_error_t * +clib_mem_get_last_error (void) +{ + return clib_mem_main.error; +} + +/* bulk allocator */ + +typedef void *clib_mem_bulk_handle_t; +clib_mem_bulk_handle_t clib_mem_bulk_init (u32 elt_sz, u32 align, + u32 min_elts_per_chunk); +void clib_mem_bulk_destroy (clib_mem_bulk_handle_t h); +void *clib_mem_bulk_alloc (clib_mem_bulk_handle_t h); +void clib_mem_bulk_free (clib_mem_bulk_handle_t h, void *p); +u8 *format_clib_mem_bulk (u8 *s, va_list *args); #include /* clib_panic */