X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fmem.h;h=06694ccadebb6ef7c811750d697fdd073ca4cf3f;hb=68e5fd5206e75cb367375b4fea2e531a3712fd06;hp=1260eab28c03da348d4037d659e80cb05deb0225;hpb=7cd468a3d7dee7d6c92f69a0bb7061ae208ec727;p=vpp.git diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 1260eab28c0..06694ccadeb 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -39,41 +39,94 @@ #define _included_clib_mem_h #include +#include +#include #include /* uword, etc */ -#include +#include + +#include + #include -#include /* memcpy, memset */ -#include +#include /* memcpy, clib_memset */ +#include #define CLIB_MAX_MHEAPS 256 +#define CLIB_MAX_NUMAS 8 + +/* Unspecified NUMA socket */ +#define VEC_NUMA_UNSPECIFIED (0xFF) /* Per CPU heaps. */ extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; +extern void *clib_per_numa_mheaps[CLIB_MAX_NUMAS]; always_inline void * clib_mem_get_per_cpu_heap (void) { - int cpu = os_get_cpu_number (); + int cpu = os_get_thread_index (); return clib_per_cpu_mheaps[cpu]; } always_inline void * clib_mem_set_per_cpu_heap (u8 * new_heap) { - int cpu = os_get_cpu_number (); + int cpu = os_get_thread_index (); void *old = clib_per_cpu_mheaps[cpu]; clib_per_cpu_mheaps[cpu] = new_heap; return old; } +always_inline void * +clib_mem_get_per_numa_heap (u32 numa_id) +{ + ASSERT (numa_id < ARRAY_LEN (clib_per_numa_mheaps)); + return clib_per_numa_mheaps[numa_id]; +} + +always_inline void * +clib_mem_set_per_numa_heap (u8 * new_heap) +{ + int numa = os_get_numa_index (); + void *old = clib_per_numa_mheaps[numa]; + clib_per_numa_mheaps[numa] = new_heap; + return old; +} + +always_inline void +clib_mem_set_thread_index (void) +{ + /* + * Find an unused slot in the per-cpu-mheaps array, + * and grab it for this thread. We need to be able to + * push/pop the thread heap without affecting other thread(s). + */ + int i; + if (__os_thread_index != 0) + return; + for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++) + if (clib_atomic_bool_cmp_and_swap (&clib_per_cpu_mheaps[i], + 0, clib_per_cpu_mheaps[0])) + { + os_set_thread_index (i); + break; + } + ASSERT (__os_thread_index > 0); +} + +always_inline uword +clib_mem_size_nocheck (void *p) +{ + return mspace_usable_size_with_delta (p); +} + /* Memory allocator which may call os_out_of_memory() if it fails */ always_inline void * clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, int os_out_of_memory_on_failure) { void *heap, *p; - uword offset, cpu; + uword cpu; if (align_offset > align) { @@ -83,25 +136,20 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, align_offset = align; } - cpu = os_get_cpu_number (); + cpu = os_get_thread_index (); heap = clib_per_cpu_mheaps[cpu]; - heap = mheap_get_aligned (heap, size, align, align_offset, &offset); - clib_per_cpu_mheaps[cpu] = heap; - if (offset != ~0) - { - p = heap + offset; -#if CLIB_DEBUG > 0 - VALGRIND_MALLOCLIKE_BLOCK (p, mheap_data_bytes (heap, offset), 0, 0); -#endif - return p; - } - else + p = mspace_get_aligned (heap, size, align, align_offset); + + if (PREDICT_FALSE (0 == p)) { if (os_out_of_memory_on_failure) os_out_of_memory (); return 0; } + + CLIB_MEM_UNPOISON (p, size); + return p; } /* Memory allocator which calls os_out_of_memory() when it fails */ @@ -159,17 +207,8 @@ always_inline uword clib_mem_is_heap_object (void *p) { void *heap = clib_mem_get_per_cpu_heap (); - uword offset = (uword) p - (uword) heap; - mheap_elt_t *e, *n; - - if (offset >= vec_len (heap)) - return 0; - e = mheap_elt_at_uoffset (heap, offset); - n = mheap_next_elt (e); - - /* Check that heap forward and reverse pointers agree. */ - return e->n_user_data == n->prev_n_user_data; + return mspace_is_heap_object (heap, p); } always_inline void @@ -180,11 +219,9 @@ clib_mem_free (void *p) /* Make sure object is in the correct heap. */ ASSERT (clib_mem_is_heap_object (p)); - mheap_put (heap, (u8 *) p - heap); + CLIB_MEM_POISON (p, clib_mem_size_nocheck (p)); -#if CLIB_DEBUG > 0 - VALGRIND_FREELIKE_BLOCK (p, 0); -#endif + mspace_put (heap, p); } always_inline void * @@ -199,7 +236,7 @@ clib_mem_realloc (void *p, uword new_size, uword old_size) copy_size = old_size; else copy_size = new_size; - clib_memcpy (q, p, copy_size); + clib_memcpy_fast (q, p, copy_size); clib_mem_free (p); } return q; @@ -209,8 +246,16 @@ always_inline uword clib_mem_size (void *p) { ASSERT (clib_mem_is_heap_object (p)); - mheap_elt_t *e = mheap_user_pointer_to_elt (p); - return mheap_elt_data_bytes (e); + return clib_mem_size_nocheck (p); +} + +always_inline void +clib_mem_free_s (void *p) +{ + uword size = clib_mem_size (p); + CLIB_MEM_UNPOISON (p, size); + memset_s_inline (p, size, 0, size); + clib_mem_free (p); } always_inline void * @@ -226,6 +271,9 @@ clib_mem_set_heap (void *heap) } void *clib_mem_init (void *heap, uword size); +void *clib_mem_init_thread_safe (void *memory, uword memory_size); +void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size, + u8 numa); void clib_mem_exit (void); @@ -235,6 +283,8 @@ void clib_mem_validate (void); void clib_mem_trace (int enable); +int clib_mem_is_traced (void); + typedef struct { /* Total number of objects allocated. */ @@ -264,19 +314,119 @@ void clib_mem_usage (clib_mem_usage_t * usage); u8 *format_clib_mem_usage (u8 * s, va_list * args); -/* Include appropriate VM functions depending on whether - we are compiling for linux kernel, for Unix or standalone. */ -#ifdef CLIB_LINUX_KERNEL -#include -#endif +/* Allocate virtual address space. */ +always_inline void * +clib_mem_vm_alloc (uword size) +{ + void *mmap_addr; + uword flags = MAP_PRIVATE; -#ifdef CLIB_UNIX -#include +#ifdef MAP_ANONYMOUS + flags |= MAP_ANONYMOUS; #endif -#ifdef CLIB_STANDALONE -#include -#endif + mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0); + if (mmap_addr == (void *) -1) + mmap_addr = 0; + else + CLIB_MEM_UNPOISON (mmap_addr, size); + + return mmap_addr; +} + +always_inline void +clib_mem_vm_free (void *addr, uword size) +{ + munmap (addr, size); +} + +always_inline void * +clib_mem_vm_unmap (void *addr, uword size) +{ + void *mmap_addr; + uword flags = MAP_PRIVATE | MAP_FIXED; + + /* To unmap we "map" with no protection. If we actually called + munmap then other callers could steal the address space. By + changing to PROT_NONE the kernel can free up the pages which is + really what we want "unmap" to mean. */ + mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0); + if (mmap_addr == (void *) -1) + mmap_addr = 0; + else + CLIB_MEM_UNPOISON (mmap_addr, size); + + return mmap_addr; +} + +always_inline void * +clib_mem_vm_map (void *addr, uword size) +{ + void *mmap_addr; + uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS; + + mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0); + if (mmap_addr == (void *) -1) + mmap_addr = 0; + else + CLIB_MEM_UNPOISON (mmap_addr, size); + + return mmap_addr; +} + +typedef struct +{ +#define CLIB_MEM_VM_F_SHARED (1 << 0) +#define CLIB_MEM_VM_F_HUGETLB (1 << 1) +#define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2) +#define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3) +#define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4) +#define CLIB_MEM_VM_F_LOCKED (1 << 5) + u32 flags; /**< vm allocation flags: +
CLIB_MEM_VM_F_SHARED: request shared memory, file + descriptor will be provided on successful allocation. +
CLIB_MEM_VM_F_HUGETLB: request hugepages. +
CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid + numa node preference. +
CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails. +
CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if + number of available pages is not sufficient. +
CLIB_MEM_VM_F_LOCKED: request locked memory. + */ + char *name; /**< Name for memory allocation, set by caller. */ + uword size; /**< Allocation size, set by caller. */ + int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */ + void *addr; /**< Pointer to allocated memory, set on successful allocation. */ + int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */ + int log2_page_size; /* Page size in log2 format, set on successful allocation. */ + int n_pages; /* Number of pages. */ + uword requested_va; /**< Request fixed position mapping */ +} clib_mem_vm_alloc_t; + +clib_error_t *clib_mem_create_fd (char *name, int *fdp); +clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp); +clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a); +void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a); +u64 clib_mem_get_fd_page_size (int fd); +uword clib_mem_get_default_hugepage_size (void); +int clib_mem_get_fd_log2_page_size (int fd); +uword clib_mem_vm_reserve (uword start, uword size, u32 log2_page_sz); +u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages); + +typedef struct +{ + uword size; /**< Map size */ + int fd; /**< File descriptor to be mapped */ + uword requested_va; /**< Request fixed position mapping */ + void *addr; /**< Pointer to mapped memory, if successful */ + u8 numa_node; +} clib_mem_vm_map_t; + +clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a); +void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size); +void mheap_trace (void *v, int enable); +uword clib_mem_trace_enable_disable (uword enable); +void clib_mem_trace (int enable); #include /* clib_panic */