/* check if pagemap is accessible - if we get zero result
dpdk will not be able to get physical memory address and game is over
unless we have IOMMU */
- pt = clib_mem_vm_get_paddr (&pt, min_log2 (sysconf (_SC_PAGESIZE)), 1);
+ pt = clib_mem_vm_get_paddr (&pt, CLIB_MEM_PAGE_SZ_DEFAULT, 1);
if (pt && pt[0])
goto check_hugetlb;
clib_mem_main_t *mm = &clib_mem_main;
int verbose __attribute__ ((unused)) = 0;
int api_segment = 0, stats_segment = 0, main_heap = 0, numa_heaps = 0;
+ int map = 0;
clib_error_t *error;
u32 index = 0;
int i;
main_heap = 1;
else if (unformat (input, "numa-heaps"))
numa_heaps = 1;
+ else if (unformat (input, "map"))
+ map = 1;
else
{
error = clib_error_return (0, "unknown input `%U'",
}
}
- if ((api_segment + stats_segment + main_heap + numa_heaps) == 0)
+ if ((api_segment + stats_segment + main_heap + numa_heaps + map) == 0)
return clib_error_return
- (0, "Need one of api-segment, stats-segment, main-heap or numa-heaps");
+ (0, "Need one of api-segment, stats-segment, main-heap, numa-heaps "
+ "or map");
if (api_segment)
{
mm->per_numa_mheaps[index], verbose);
}
}
+ if (map)
+ {
+ clib_mem_page_stats_t stats = { };
+ clib_mem_vm_map_hdr_t *hdr = 0;
+ u8 *s = 0;
+ int numa = -1;
+
+ s = format (s, "\n%-16s%7s%7s%7s",
+ "StartAddr", "size", "PageSz", "Pages");
+ while ((numa = vlib_mem_get_next_numa_node (numa)) != -1)
+ s = format (s, " Numa%u", numa);
+ s = format (s, " NotMap");
+ s = format (s, " Name");
+ vlib_cli_output (vm, "%v", s);
+ vec_reset_length (s);
+
+ while ((hdr = clib_mem_vm_get_next_map_hdr (hdr)))
+ {
+ clib_mem_get_page_stats ((void *) hdr->base_addr,
+ hdr->log2_page_sz, hdr->num_pages,
+ &stats);
+ s = format (s, "%016lx%7U%7U%7lu",
+ hdr->base_addr, format_memory_size,
+ hdr->num_pages << hdr->log2_page_sz,
+ format_log2_page_size, hdr->log2_page_sz,
+ hdr->num_pages);
+ while ((numa = vlib_mem_get_next_numa_node (numa)) != -1)
+ s = format (s, "%6lu", stats.per_numa[numa]);
+ s = format (s, "%7lu", stats.not_mapped);
+ s = format (s, " %s", hdr->name);
+ vlib_cli_output (vm, "%v", s);
+ vec_reset_length (s);
+ }
+ vec_free (s);
+ }
}
return 0;
}
VLIB_CLI_COMMAND (show_memory_usage_command, static) = {
.path = "show memory",
.short_help = "show memory [api-segment][stats-segment][verbose]\n"
- " [numa-heaps]",
+ " [numa-heaps][map]",
.function = show_memory_usage,
};
/* *INDENT-ON* */
if (p->type == LINUX_PCI_DEVICE_TYPE_UIO && addr != 0)
flags |= MAP_FIXED;
- *result = mmap (addr, size, PROT_READ | PROT_WRITE, flags, fd, offset);
- if (*result == (void *) -1)
+ *result = clib_mem_vm_map_shared (addr, size, fd, offset,
+ "PCIe %U region %u", format_vlib_pci_addr,
+ vlib_pci_get_addr (vm, h), bar);
+ if (*result == CLIB_MEM_VM_MAP_FAILED)
{
error = clib_error_return_unix (0, "mmap `BAR%u'", bar);
if (p->type == LINUX_PCI_DEVICE_TYPE_UIO && (fd != -1))
{
if (res->size == 0)
continue;
- munmap (res->addr, res->size);
+ clib_mem_vm_unmap (res->addr);
if (res->fd != -1)
close (res->fd);
}
#define MFD_HUGETLB 0x0004U
#endif
-uword
-clib_mem_get_page_size (void)
-{
- return getpagesize ();
-}
+#ifndef MAP_HUGE_SHIFT
+#define MAP_HUGE_SHIFT 26
+#endif
+
+#ifndef MAP_FIXED_NOREPLACE
+#define MAP_FIXED_NOREPLACE 0x100000
+#endif
uword
clib_mem_get_default_hugepage_size (void)
return st.st_blksize;
}
-int
+clib_mem_page_sz_t
clib_mem_get_fd_log2_page_size (int fd)
{
- return min_log2 (clib_mem_get_fd_page_size (fd));
+ uword page_size = clib_mem_get_fd_page_size (fd);
+ return page_size ? min_log2 (page_size) : CLIB_MEM_PAGE_SZ_UNKNOWN;
}
void
uword
clib_mem_vm_reserve (uword start, uword size, clib_mem_page_sz_t log2_page_sz)
{
- uword off, pagesize = 1ULL << log2_page_sz;
- int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
- u8 *p;
-
- if (start)
- mmap_flags |= MAP_FIXED;
+ clib_mem_main_t *mm = &clib_mem_main;
+ uword pagesize = 1ULL << log2_page_sz;
+ uword sys_page_sz = 1ULL << mm->log2_page_sz;
+ uword n_bytes;
+ void *base = 0, *p;
size = round_pow2 (size, pagesize);
- p = uword_to_pointer (start, void *);
- p = mmap (p, size + pagesize, PROT_NONE, mmap_flags, -1, 0);
+ /* in adition of requested reservation, we also rserve one system page
+ * (typically 4K) adjacent to the start off reservation */
+
+ if (start)
+ {
+ /* start address is provided, so we just need to make sure we are not
+ * replacing existing map */
+ if (start & pow2_mask (log2_page_sz))
+ return ~0;
+
+ base = (void *) start - sys_page_sz;
+ base = mmap (base, size + sys_page_sz, PROT_NONE,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0);
+ return (base == MAP_FAILED) ? ~0 : start;
+ }
+
+ /* to make sure that we get reservation aligned to page_size we need to
+ * request one additional page as mmap will return us address which is
+ * aligned only to system page size */
+ base = mmap (0, size + pagesize, PROT_NONE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (p == MAP_FAILED)
+ if (base == MAP_FAILED)
return ~0;
- off = round_pow2 ((uword) p, pagesize) - (uword) p;
+ /* return additional space at the end of allocation */
+ p = base + size + pagesize;
+ n_bytes = (uword) p & pow2_mask (log2_page_sz);
+ if (n_bytes)
+ {
+ p -= n_bytes;
+ munmap (p, n_bytes);
+ }
- /* trim start and end of reservation to be page aligned */
- if (off)
+ /* return additional space at the start of allocation */
+ n_bytes = pagesize - sys_page_sz - n_bytes;
+ if (n_bytes)
{
- munmap (p, off);
- p += off;
+ munmap (base, n_bytes);
+ base += n_bytes;
}
- munmap (p + size, pagesize - off);
+ return (uword) base + sys_page_sz;
+}
- return (uword) p;
+clib_mem_vm_map_hdr_t *
+clib_mem_vm_get_next_map_hdr (clib_mem_vm_map_hdr_t * hdr)
+{
+ clib_mem_main_t *mm = &clib_mem_main;
+ uword sys_page_sz = 1 << mm->log2_page_sz;
+ clib_mem_vm_map_hdr_t *next;
+ if (hdr == 0)
+ {
+ hdr = mm->first_map;
+ if (hdr)
+ mprotect (hdr, sys_page_sz, PROT_READ);
+ return hdr;
+ }
+ next = hdr->next;
+ mprotect (hdr, sys_page_sz, PROT_NONE);
+ if (next)
+ mprotect (next, sys_page_sz, PROT_READ);
+ return next;
}
+void *
+clib_mem_vm_map_internal (void *base, clib_mem_page_sz_t log2_page_sz,
+ uword size, int fd, uword offset, char *name)
+{
+ clib_mem_main_t *mm = &clib_mem_main;
+ clib_mem_vm_map_hdr_t *hdr;
+ uword sys_page_sz = 1 << mm->log2_page_sz;
+ int mmap_flags = MAP_FIXED, is_huge = 0;
+
+ if (fd != -1)
+ {
+ mmap_flags |= MAP_SHARED;
+ log2_page_sz = clib_mem_get_fd_log2_page_size (fd);
+ if (log2_page_sz > mm->log2_page_sz)
+ is_huge = 1;
+ }
+ else
+ {
+ mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
+
+ if (log2_page_sz == mm->log2_page_sz)
+ log2_page_sz = CLIB_MEM_PAGE_SZ_DEFAULT;
+
+ switch (log2_page_sz)
+ {
+ case CLIB_MEM_PAGE_SZ_UNKNOWN:
+ /* will fail later */
+ break;
+ case CLIB_MEM_PAGE_SZ_DEFAULT:
+ log2_page_sz = mm->log2_page_sz;
+ break;
+ case CLIB_MEM_PAGE_SZ_DEFAULT_HUGE:
+ mmap_flags |= MAP_HUGETLB;
+ log2_page_sz = mm->log2_default_hugepage_sz;
+ is_huge = 1;
+ break;
+ default:
+ mmap_flags |= MAP_HUGETLB;
+ mmap_flags |= log2_page_sz << MAP_HUGE_SHIFT;
+ is_huge = 1;
+ }
+ }
+
+ if (log2_page_sz == CLIB_MEM_PAGE_SZ_UNKNOWN)
+ return CLIB_MEM_VM_MAP_FAILED;
+
+ size = round_pow2 (size, 1 << log2_page_sz);
+
+ base = (void *) clib_mem_vm_reserve ((uword) base, size, log2_page_sz);
+
+ if (base == (void *) ~0)
+ return CLIB_MEM_VM_MAP_FAILED;
+
+ base = mmap (base, size, PROT_READ | PROT_WRITE, mmap_flags, fd, offset);
+
+ if (base == MAP_FAILED)
+ return CLIB_MEM_VM_MAP_FAILED;
+
+ if (is_huge && (mlock (base, size) != 0))
+ {
+ munmap (base, size);
+ return CLIB_MEM_VM_MAP_FAILED;
+ }
+
+ hdr = mmap (base - sys_page_sz, sys_page_sz, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
+
+ if (hdr != base - sys_page_sz)
+ {
+ munmap (base, size);
+ return CLIB_MEM_VM_MAP_FAILED;
+ }
+
+ if (mm->last_map)
+ {
+ mprotect (mm->last_map, sys_page_sz, PROT_READ | PROT_WRITE);
+ mm->last_map->next = hdr;
+ mprotect (mm->last_map, sys_page_sz, PROT_NONE);
+ }
+ else
+ mm->first_map = hdr;
+
+ hdr->next = 0;
+ hdr->prev = mm->last_map;
+ mm->last_map = hdr;
+
+ hdr->base_addr = (uword) base;
+ hdr->log2_page_sz = log2_page_sz;
+ hdr->num_pages = size >> log2_page_sz;
+ snprintf (hdr->name, CLIB_VM_MAP_HDR_NAME_MAX_LEN - 1, "%s", (char *) name);
+ hdr->name[CLIB_VM_MAP_HDR_NAME_MAX_LEN - 1] = 0;
+ mprotect (hdr, sys_page_sz, PROT_NONE);
+
+ CLIB_MEM_UNPOISON (base, size);
+ return base;
+}
+
+int
+clib_mem_vm_unmap (void *base)
+{
+ clib_mem_main_t *mm = &clib_mem_main;
+ uword size, sys_page_sz = 1 << mm->log2_page_sz;
+ clib_mem_vm_map_hdr_t *hdr = base - sys_page_sz;;
+
+ if (mprotect (hdr, sys_page_sz, PROT_READ | PROT_WRITE) != 0)
+ return -1;
+
+ size = hdr->num_pages << hdr->log2_page_sz;
+ if (munmap ((void *) hdr->base_addr, size) != 0)
+ return -1;
+
+ if (hdr->next)
+ {
+ mprotect (hdr->next, sys_page_sz, PROT_READ | PROT_WRITE);
+ hdr->next->prev = hdr->prev;
+ mprotect (hdr->next, sys_page_sz, PROT_NONE);
+ }
+ else
+ mm->last_map = hdr->prev;
+
+ if (hdr->prev)
+ {
+ mprotect (hdr->prev, sys_page_sz, PROT_READ | PROT_WRITE);
+ hdr->prev->next = hdr->next;
+ mprotect (hdr->prev, sys_page_sz, PROT_NONE);
+ }
+ else
+ mm->first_map = hdr->next;
+
+ if (munmap (hdr, sys_page_sz) != 0)
+ return -1;
+
+ return 0;
+}
+
+void
+clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
+ uword n_pages, clib_mem_page_stats_t * stats)
+{
+ int i, *status = 0;
+ void **ptr = 0;
+
+ log2_page_size = clib_mem_log2_page_size_validate (log2_page_size);
+
+ vec_validate (status, n_pages - 1);
+ vec_validate (ptr, n_pages - 1);
+
+ for (i = 0; i < n_pages; i++)
+ ptr[i] = start + (i << log2_page_size);
+
+ clib_memset (stats, 0, sizeof (clib_mem_page_stats_t));
+
+ if (move_pages (0, n_pages, ptr, 0, status, 0) != 0)
+ {
+ stats->unknown = n_pages;
+ return;
+ }
+
+ for (i = 0; i < n_pages; i++)
+ {
+ if (status[i] >= 0 && status[i] < CLIB_MAX_NUMAS)
+ {
+ stats->mapped++;
+ stats->per_numa[status[i]]++;
+ }
+ else if (status[i] == -EFAULT)
+ stats->not_mapped++;
+ else
+ stats->unknown++;
+ }
+}
+
+
u64 *
-clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages)
+clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size,
+ int n_pages)
{
int pagesize = sysconf (_SC_PAGESIZE);
int fd;
int i;
u64 *r = 0;
+ log2_page_size = clib_mem_log2_page_size_validate (log2_page_size);
+
if ((fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1)
return 0;
clib_mem_main_t clib_mem_main;
+void *
+clib_mem_vm_map (void *base, uword size, clib_mem_page_sz_t log2_page_sz,
+ char *fmt, ...)
+{
+ va_list va;
+ void *rv;
+ u8 *s;
+
+ va_start (va, fmt);
+ s = va_format (0, fmt, &va);
+ vec_add1 (s, 0);
+ rv = clib_mem_vm_map_internal (base, log2_page_sz, size, -1, 0, (char *) s);
+ va_end (va);
+ vec_free (s);
+ return rv;
+}
+
+void *
+clib_mem_vm_map_stack (uword size, clib_mem_page_sz_t log2_page_sz,
+ char *fmt, ...)
+{
+ va_list va;
+ void *rv;
+ u8 *s;
+
+ va_start (va, fmt);
+ s = va_format (0, fmt, &va);
+ vec_add1 (s, 0);
+ rv = clib_mem_vm_map_internal (0, log2_page_sz, size, -1, 0, (char *) s);
+ va_end (va);
+ vec_free (s);
+ return rv;
+}
+
+void *
+clib_mem_vm_map_shared (void *base, uword size, int fd, uword offset,
+ char *fmt, ...)
+{
+ va_list va;
+ void *rv;
+ u8 *s;
+ va_start (va, fmt);
+ s = va_format (0, fmt, &va);
+ vec_add1 (s, 0);
+ rv = clib_mem_vm_map_internal (base, 0, size, fd, offset, (char *) s);
+ va_end (va);
+ vec_free (s);
+ return rv;
+}
+
/*
* fd.io coding-style-patch-verification: ON
*
#include <vppinfra/sanitizer.h>
#define CLIB_MAX_MHEAPS 256
-#define CLIB_MAX_NUMAS 8
+#define CLIB_MAX_NUMAS 16
+#define CLIB_MEM_VM_MAP_FAILED ((void *) ~0)
typedef enum
{
CLIB_MEM_PAGE_SZ_16G = 34,
} clib_mem_page_sz_t;
+typedef struct _clib_mem_vm_map_hdr
+{
+ /* base address */
+ uword base_addr;
+
+ /* number of pages */
+ uword num_pages;
+
+ /* page size (log2) */
+ clib_mem_page_sz_t log2_page_sz;
+
+ /* allocation mame */
+#define CLIB_VM_MAP_HDR_NAME_MAX_LEN 64
+ char name[CLIB_VM_MAP_HDR_NAME_MAX_LEN];
+
+ /* linked list */
+ struct _clib_mem_vm_map_hdr *prev, *next;
+} clib_mem_vm_map_hdr_t;
+
typedef struct
{
/* log2 system page size */
/* per NUMA heaps */
void *per_numa_mheaps[CLIB_MAX_NUMAS];
+
+ /* memory maps */
+ clib_mem_vm_map_hdr_t *first_map, *last_map;
} clib_mem_main_t;
extern clib_mem_main_t clib_mem_main;
void clib_mem_main_init ();
void *clib_mem_init (void *heap, uword size);
+void *clib_mem_init_with_page_size (uword memory_size,
+ clib_mem_page_sz_t log2_page_sz);
void *clib_mem_init_thread_safe (void *memory, uword memory_size);
void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size,
u8 numa);
void clib_mem_exit (void);
-uword clib_mem_get_page_size (void);
-
void clib_mem_validate (void);
void clib_mem_trace (int enable);
munmap (addr, size);
}
-always_inline void *
-clib_mem_vm_unmap (void *addr, uword size)
-{
- void *mmap_addr;
- uword flags = MAP_PRIVATE | MAP_FIXED;
-
- /* To unmap we "map" with no protection. If we actually called
- munmap then other callers could steal the address space. By
- changing to PROT_NONE the kernel can free up the pages which is
- really what we want "unmap" to mean. */
- mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0);
- if (mmap_addr == (void *) -1)
- mmap_addr = 0;
- else
- CLIB_MEM_UNPOISON (mmap_addr, size);
-
- return mmap_addr;
-}
-
-always_inline void *
-clib_mem_vm_map (void *addr, uword size)
-{
- void *mmap_addr;
- uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
-
- mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0);
- if (mmap_addr == (void *) -1)
- mmap_addr = 0;
- else
- CLIB_MEM_UNPOISON (mmap_addr, size);
+void *clib_mem_vm_map_internal (void *base, clib_mem_page_sz_t log2_page_sz,
+ uword size, int fd, uword offset, char *name);
- return mmap_addr;
-}
+void *clib_mem_vm_map (void *start, uword size,
+ clib_mem_page_sz_t log2_page_size, char *fmt, ...);
+void *clib_mem_vm_map_stack (uword size, clib_mem_page_sz_t log2_page_size,
+ char *fmt, ...);
+void *clib_mem_vm_map_shared (void *start, uword size, int fd, uword offset,
+ char *fmt, ...);
+int clib_mem_vm_unmap (void *base);
+clib_mem_vm_map_hdr_t *clib_mem_vm_get_next_map_hdr (clib_mem_vm_map_hdr_t *
+ hdr);
typedef struct
{
uword requested_va; /**< Request fixed position mapping */
} clib_mem_vm_alloc_t;
+
+static_always_inline clib_mem_page_sz_t
+clib_mem_get_log2_page_size (void)
+{
+ return clib_mem_main.log2_page_sz;
+}
+
+static_always_inline uword
+clib_mem_get_page_size (void)
+{
+ return 1ULL << clib_mem_main.log2_page_sz;
+}
+
+static_always_inline clib_mem_page_sz_t
+clib_mem_get_log2_default_hugepage_size ()
+{
+ return clib_mem_main.log2_default_hugepage_sz;
+}
+
clib_error_t *clib_mem_create_fd (char *name, int *fdp);
clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp);
clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a);
void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a);
-u64 clib_mem_get_fd_page_size (int fd);
+uword clib_mem_get_fd_page_size (int fd);
uword clib_mem_get_default_hugepage_size (void);
-int clib_mem_get_fd_log2_page_size (int fd);
+clib_mem_page_sz_t clib_mem_get_fd_log2_page_size (int fd);
uword clib_mem_vm_reserve (uword start, uword size,
clib_mem_page_sz_t log2_page_sz);
-u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages);
+u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size,
+ int n_pages);
void clib_mem_destroy_mspace (void *mspace);
void clib_mem_destroy (void);
uword clib_mem_trace_enable_disable (uword enable);
void clib_mem_trace (int enable);
+always_inline uword
+clib_mem_round_to_page_size (uword size, clib_mem_page_sz_t log2_page_size)
+{
+ ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN);
+
+ if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT)
+ log2_page_size = clib_mem_get_log2_page_size ();
+ else if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE)
+ log2_page_size = clib_mem_get_log2_default_hugepage_size ();
+
+ return round_pow2 (size, 1ULL << log2_page_size);
+}
+
+typedef struct
+{
+ uword mapped;
+ uword not_mapped;
+ uword per_numa[CLIB_MAX_NUMAS];
+ uword unknown;
+} clib_mem_page_stats_t;
+
+void clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
+ uword n_pages, clib_mem_page_stats_t * stats);
+
+static_always_inline int
+vlib_mem_get_next_numa_node (int numa)
+{
+ clib_mem_main_t *mm = &clib_mem_main;
+ u32 bitmap = mm->numa_node_bitmap;
+
+ if (numa >= 0)
+ bitmap &= ~pow2_mask (numa + 1);
+ if (bitmap == 0)
+ return -1;
+
+ return count_trailing_zeros (bitmap);
+}
+
+static_always_inline clib_mem_page_sz_t
+clib_mem_log2_page_size_validate (clib_mem_page_sz_t log2_page_size)
+{
+ if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT)
+ return clib_mem_get_log2_page_size ();
+ if (log2_page_size == CLIB_MEM_PAGE_SZ_DEFAULT_HUGE)
+ return clib_mem_get_log2_default_hugepage_size ();
+ return log2_page_size;
+}
+
+static_always_inline uword
+clib_mem_page_bytes (clib_mem_page_sz_t log2_page_size)
+{
+ return 1 << clib_mem_log2_page_size_validate (log2_page_size);
+}
+
+
#include <vppinfra/error.h> /* clib_panic */
#endif /* _included_clib_mem_h */
/* Initialize CLIB heap based on memory/size given by user.
Set memory to 0 and CLIB will try to allocate its own heap. */
static void *
-clib_mem_init_internal (void *memory, uword memory_size, int set_heap)
+clib_mem_init_internal (void *memory, uword memory_size,
+ clib_mem_page_sz_t log2_page_sz, int set_heap)
{
u8 *heap;
mspace_disable_expand (heap);
}
else
- heap = create_mspace (memory_size, 1 /* locked */ );
+ {
+ memory_size = round_pow2 (memory_size,
+ clib_mem_page_bytes (log2_page_sz));
+ memory = clib_mem_vm_map_internal (0, log2_page_sz, memory_size, -1, 0,
+ "main heap");
+
+ if (memory == CLIB_MEM_VM_MAP_FAILED)
+ return 0;
+
+ heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ );
+ mspace_disable_expand (heap);
+ }
CLIB_MEM_POISON (mspace_least_addr (heap), mspace_footprint (heap));
clib_mem_init (void *memory, uword memory_size)
{
return clib_mem_init_internal (memory, memory_size,
+ CLIB_MEM_PAGE_SZ_DEFAULT,
+ 1 /* do clib_mem_set_heap */ );
+}
+
+void *
+clib_mem_init_with_page_size (uword memory_size,
+ clib_mem_page_sz_t log2_page_sz)
+{
+ return clib_mem_init_internal (0, memory_size, log2_page_sz,
1 /* do clib_mem_set_heap */ );
}
clib_mem_init_thread_safe (void *memory, uword memory_size)
{
return clib_mem_init_internal (memory, memory_size,
+ CLIB_MEM_PAGE_SZ_DEFAULT,
1 /* do clib_mem_set_heap */ );
}
void
clib_mem_destroy (void)
{
+ void *heap = clib_mem_get_heap ();
+ void *base = mspace_least_addr (heap);
clib_mem_destroy_mspace (clib_mem_get_heap ());
+ clib_mem_vm_unmap (base);
}
void *
}
heap = clib_mem_init_internal (memory, memory_size,
+ CLIB_MEM_PAGE_SZ_DEFAULT,
0 /* do NOT clib_mem_set_heap */ );
ASSERT (heap);
pagesize = clib_mem_get_default_hugepage_size ();
pm->def_log2_page_sz = min_log2 (pagesize);
- pm->sys_log2_page_sz = min_log2 (sysconf (_SC_PAGESIZE));
pm->lookup_log2_page_sz = pm->def_log2_page_sz;
/* check if pagemap is accessible */
- pt = clib_mem_vm_get_paddr (&pt, pm->sys_log2_page_sz, 1);
+ pt = clib_mem_vm_get_paddr (&pt, CLIB_MEM_PAGE_SZ_DEFAULT, 1);
if (pt == 0 || pt[0] == 0)
pm->flags |= CLIB_PMALLOC_F_NO_PAGEMAP;
{
va = pointer_to_uword (pm->base) + (p << pm->lookup_log2_page_sz);
pa = 0;
- seek = (va >> pm->sys_log2_page_sz) * sizeof (pa);
+ seek = (va >> clib_mem_get_log2_page_size ()) * sizeof (pa);
if (fd != -1 && lseek (fd, seek, SEEK_SET) == seek &&
read (fd, &pa, sizeof (pa)) == (sizeof (pa)) &&
pa & (1ULL << 63) /* page present bit */ )
{
- pa = (pa & pow2_mask (55)) << pm->sys_log2_page_sz;
+ pa = (pa & pow2_mask (55)) << clib_mem_get_log2_page_size ();
}
pm->lookup_table[p] = va - pa;
p++;
return 0;
}
- if (a->log2_subpage_sz != pm->sys_log2_page_sz)
+ if (a->log2_subpage_sz != clib_mem_get_log2_page_size ())
{
pm->error = clib_sysfs_prealloc_hugepages (numa_node,
a->log2_subpage_sz, n_pages);
if (a->flags & CLIB_PMALLOC_ARENA_F_SHARED_MEM)
{
mmap_flags |= MAP_SHARED;
- if (a->log2_subpage_sz != pm->sys_log2_page_sz)
+ if (a->log2_subpage_sz != clib_mem_get_log2_page_size ())
pm->error = clib_mem_create_hugetlb_fd ((char *) a->name, &a->fd);
else
pm->error = clib_mem_create_fd ((char *) a->name, &a->fd);
}
else
{
- if (a->log2_subpage_sz != pm->sys_log2_page_sz)
+ if (a->log2_subpage_sz != clib_mem_get_log2_page_size ())
mmap_flags |= MAP_HUGETLB;
mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
goto error;
}
- if (a->log2_subpage_sz != pm->sys_log2_page_sz && mlock (va, size) != 0)
+ if (a->log2_subpage_sz != clib_mem_get_log2_page_size () &&
+ mlock (va, size) != 0)
{
pm->error = clib_error_return_unix (0, "Unable to lock pages");
goto error;
if (log2_page_sz == 0)
log2_page_sz = pm->def_log2_page_sz;
else if (log2_page_sz != pm->def_log2_page_sz &&
- log2_page_sz != pm->sys_log2_page_sz)
+ log2_page_sz != clib_mem_get_log2_page_size ())
{
pm->error = clib_error_create ("unsupported page size (%uKB)",
1 << (log2_page_sz - 10));
u8 *base;
/* default page size - typically 2M */
- u32 def_log2_page_sz;
-
- /* system page size - typically 4K */
- u32 sys_log2_page_sz;
+ clib_mem_page_sz_t def_log2_page_sz;
/* maximum number of pages, limited by VA preallocation size */
u32 max_pages;
Code Review / vpp.git / commitdiff