X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fmem.h;h=9fc7f0efa761503c7c737e676479a127e0ee55e8;hb=561ae5d17b5565d0644a5249acc8fdb7a6f72b44;hp=f254601043ce2c89d6eb938d2d20c75072e3db6d;hpb=16e4a4a0ae39ebc1ded1b6dba2799b176aee1828;p=vpp.git diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index f254601043c..9fc7f0efa76 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -52,44 +52,107 @@ #include #define CLIB_MAX_MHEAPS 256 -#define CLIB_MAX_NUMAS 8 +#define CLIB_MAX_NUMAS 16 +#define CLIB_MEM_VM_MAP_FAILED ((void *) ~0) +#define CLIB_MEM_ERROR (-1) + +typedef enum +{ + CLIB_MEM_PAGE_SZ_UNKNOWN = 0, + CLIB_MEM_PAGE_SZ_DEFAULT = 1, + CLIB_MEM_PAGE_SZ_DEFAULT_HUGE = 2, + CLIB_MEM_PAGE_SZ_4K = 12, + CLIB_MEM_PAGE_SZ_16K = 14, + CLIB_MEM_PAGE_SZ_64K = 16, + CLIB_MEM_PAGE_SZ_1M = 20, + CLIB_MEM_PAGE_SZ_2M = 21, + CLIB_MEM_PAGE_SZ_16M = 24, + CLIB_MEM_PAGE_SZ_32M = 25, + CLIB_MEM_PAGE_SZ_512M = 29, + CLIB_MEM_PAGE_SZ_1G = 30, + CLIB_MEM_PAGE_SZ_16G = 34, +} clib_mem_page_sz_t; + +typedef struct _clib_mem_vm_map_hdr +{ + /* base address */ + uword base_addr; + + /* number of pages */ + uword num_pages; + + /* page size (log2) */ + clib_mem_page_sz_t log2_page_sz; + + /* file descriptor, -1 if memory is not shared */ + int fd; + + /* allocation mame */ +#define CLIB_VM_MAP_HDR_NAME_MAX_LEN 64 + char name[CLIB_VM_MAP_HDR_NAME_MAX_LEN]; + + /* linked list */ + struct _clib_mem_vm_map_hdr *prev, *next; +} clib_mem_vm_map_hdr_t; + +typedef struct +{ + /* log2 system page size */ + clib_mem_page_sz_t log2_page_sz; + + /* log2 system default hugepage size */ + clib_mem_page_sz_t log2_default_hugepage_sz; + + /* bitmap of available numa nodes */ + u32 numa_node_bitmap; + + /* per CPU heaps */ + void *per_cpu_mheaps[CLIB_MAX_MHEAPS]; + + /* per NUMA heaps */ + void *per_numa_mheaps[CLIB_MAX_NUMAS]; + + /* memory maps */ + clib_mem_vm_map_hdr_t *first_map, *last_map; + + /* last error */ + clib_error_t *error; +} clib_mem_main_t; + +extern clib_mem_main_t clib_mem_main; /* Unspecified NUMA socket */ #define VEC_NUMA_UNSPECIFIED (0xFF) -/* Per CPU heaps. */ -extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; -extern void *clib_per_numa_mheaps[CLIB_MAX_NUMAS]; - always_inline void * clib_mem_get_per_cpu_heap (void) { int cpu = os_get_thread_index (); - return clib_per_cpu_mheaps[cpu]; + return clib_mem_main.per_cpu_mheaps[cpu]; } always_inline void * clib_mem_set_per_cpu_heap (u8 * new_heap) { int cpu = os_get_thread_index (); - void *old = clib_per_cpu_mheaps[cpu]; - clib_per_cpu_mheaps[cpu] = new_heap; + void *old = clib_mem_main.per_cpu_mheaps[cpu]; + clib_mem_main.per_cpu_mheaps[cpu] = new_heap; return old; } always_inline void * clib_mem_get_per_numa_heap (u32 numa_id) { - ASSERT (numa_id < ARRAY_LEN (clib_per_numa_mheaps)); - return clib_per_numa_mheaps[numa_id]; + ASSERT (numa_id < ARRAY_LEN (clib_mem_main.per_numa_mheaps)); + return clib_mem_main.per_numa_mheaps[numa_id]; } always_inline void * clib_mem_set_per_numa_heap (u8 * new_heap) { int numa = os_get_numa_index (); - void *old = clib_per_numa_mheaps[numa]; - clib_per_numa_mheaps[numa] = new_heap; + void *old = clib_mem_main.per_numa_mheaps[numa]; + clib_mem_main.per_numa_mheaps[numa] = new_heap; return old; } @@ -104,9 +167,9 @@ clib_mem_set_thread_index (void) int i; if (__os_thread_index != 0) return; - for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++) - if (clib_atomic_bool_cmp_and_swap (&clib_per_cpu_mheaps[i], - 0, clib_per_cpu_mheaps[0])) + for (i = 0; i < ARRAY_LEN (clib_mem_main.per_cpu_mheaps); i++) + if (clib_atomic_bool_cmp_and_swap (&clib_mem_main.per_cpu_mheaps[i], + 0, clib_mem_main.per_cpu_mheaps[0])) { os_set_thread_index (i); break; @@ -137,7 +200,7 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, } cpu = os_get_thread_index (); - heap = clib_per_cpu_mheaps[cpu]; + heap = clib_mem_main.per_cpu_mheaps[cpu]; p = mspace_get_aligned (heap, size, align, align_offset); @@ -270,15 +333,16 @@ clib_mem_set_heap (void *heap) return clib_mem_set_per_cpu_heap (heap); } +void clib_mem_main_init (); void *clib_mem_init (void *heap, uword size); +void *clib_mem_init_with_page_size (uword memory_size, + clib_mem_page_sz_t log2_page_sz); void *clib_mem_init_thread_safe (void *memory, uword memory_size); void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size, u8 numa); void clib_mem_exit (void); -uword clib_mem_get_page_size (void); - void clib_mem_validate (void); void clib_mem_trace (int enable); @@ -338,42 +402,20 @@ always_inline void clib_mem_vm_free (void *addr, uword size) { munmap (addr, size); - CLIB_MEM_POISON (addr, size); } -always_inline void * -clib_mem_vm_unmap (void *addr, uword size) -{ - void *mmap_addr; - uword flags = MAP_PRIVATE | MAP_FIXED; - - /* To unmap we "map" with no protection. If we actually called - munmap then other callers could steal the address space. By - changing to PROT_NONE the kernel can free up the pages which is - really what we want "unmap" to mean. */ - mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0); - if (mmap_addr == (void *) -1) - mmap_addr = 0; - else - CLIB_MEM_UNPOISON (mmap_addr, size); +void *clib_mem_vm_map_internal (void *base, clib_mem_page_sz_t log2_page_sz, + uword size, int fd, uword offset, char *name); - return mmap_addr; -} - -always_inline void * -clib_mem_vm_map (void *addr, uword size) -{ - void *mmap_addr; - uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS; - - mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0); - if (mmap_addr == (void *) -1) - mmap_addr = 0; - else - CLIB_MEM_UNPOISON (mmap_addr, size); - - return mmap_addr; -} +void *clib_mem_vm_map (void *start, uword size, + clib_mem_page_sz_t log2_page_size, char *fmt, ...); +void *clib_mem_vm_map_stack (uword size, clib_mem_page_sz_t log2_page_size, + char *fmt, ...); +void *clib_mem_vm_map_shared (void *start, uword size, int fd, uword offset, + char *fmt, ...); +int clib_mem_vm_unmap (void *base); +clib_mem_vm_map_hdr_t *clib_mem_vm_get_next_map_hdr (clib_mem_vm_map_hdr_t * + hdr); typedef struct { @@ -404,15 +446,39 @@ typedef struct uword requested_va; /**< Request fixed position mapping */ } clib_mem_vm_alloc_t; -clib_error_t *clib_mem_create_fd (char *name, int *fdp); -clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp); + +static_always_inline clib_mem_page_sz_t +clib_mem_get_log2_page_size (void) +{ + return clib_mem_main.log2_page_sz; +} + +static_always_inline uword +clib_mem_get_page_size (void) +{ + return 1ULL << clib_mem_main.log2_page_sz; +} + +static_always_inline clib_mem_page_sz_t +clib_mem_get_log2_default_hugepage_size () +{ + return clib_mem_main.log2_default_hugepage_sz; +} + +int clib_mem_vm_create_fd (clib_mem_page_sz_t log2_page_size, char *fmt, ...); clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a); void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a); -u64 clib_mem_get_fd_page_size (int fd); +uword clib_mem_get_fd_page_size (int fd); uword clib_mem_get_default_hugepage_size (void); -int clib_mem_get_fd_log2_page_size (int fd); -uword clib_mem_vm_reserve (uword start, uword size, u32 log2_page_sz); -u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages); +clib_mem_page_sz_t clib_mem_get_fd_log2_page_size (int fd); +uword clib_mem_vm_reserve (uword start, uword size, + clib_mem_page_sz_t log2_page_sz); +u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size, + int n_pages); +void clib_mem_destroy_mspace (void *mspace); +void clib_mem_destroy (void); +int clib_mem_set_numa_affinity (u8 numa_node, int force); +int clib_mem_set_default_numa_affinity (); typedef struct { @@ -424,11 +490,73 @@ typedef struct } clib_mem_vm_map_t; clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a); -void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size); +void clib_mem_vm_randomize_va (uword * requested_va, + clib_mem_page_sz_t log2_page_size); void mheap_trace (void *v, int enable); uword clib_mem_trace_enable_disable (uword enable); void clib_mem_trace (int enable); +always_inline uword +clib_mem_round_to_page_size (uword size, clib_mem_page_sz_t log2_page_size) +{ + ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN); + + if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT) + log2_page_size = clib_mem_get_log2_page_size (); + else if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE) + log2_page_size = clib_mem_get_log2_default_hugepage_size (); + + return round_pow2 (size, 1ULL << log2_page_size); +} + +typedef struct +{ + uword mapped; + uword not_mapped; + uword per_numa[CLIB_MAX_NUMAS]; + uword unknown; +} clib_mem_page_stats_t; + +void clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size, + uword n_pages, clib_mem_page_stats_t * stats); + +static_always_inline int +vlib_mem_get_next_numa_node (int numa) +{ + clib_mem_main_t *mm = &clib_mem_main; + u32 bitmap = mm->numa_node_bitmap; + + if (numa >= 0) + bitmap &= ~pow2_mask (numa + 1); + if (bitmap == 0) + return -1; + + return count_trailing_zeros (bitmap); +} + +static_always_inline clib_mem_page_sz_t +clib_mem_log2_page_size_validate (clib_mem_page_sz_t log2_page_size) +{ + if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT) + return clib_mem_get_log2_page_size (); + if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE) + return clib_mem_get_log2_default_hugepage_size (); + return log2_page_size; +} + +static_always_inline uword +clib_mem_page_bytes (clib_mem_page_sz_t log2_page_size) +{ + return 1 << clib_mem_log2_page_size_validate (log2_page_size); +} + +static_always_inline clib_error_t * +clib_mem_get_last_error (void) +{ + return clib_mem_main.error; +} + + #include /* clib_panic */ #endif /* _included_clib_mem_h */