{
uword page_size, log2_page_size, rnd_size = 0;
ssvm_shared_header_t *sh;
- void *oldheap;
- u8 *heap;
+ clib_mem_heap_t *heap, *oldheap;
log2_page_size = clib_mem_get_log2_page_size ();
if (log2_page_size == 0)
static inline void *
ssvm_push_heap (ssvm_shared_header_t * sh)
{
- u8 *oldheap;
+ clib_mem_heap_t *oldheap;
oldheap = clib_mem_set_heap (sh->heap);
return ((void *) oldheap);
}
static inline void *
ssvm_mem_alloc (ssvm_private_t * ssvm, uword size)
{
- u8 *oldheap;
+ clib_mem_heap_t *oldheap;
void *rv;
oldheap = clib_mem_set_heap (ssvm->sh->heap);
static inline void *
svm_mem_alloc (svm_region_t * rp, uword size)
{
- u8 *oldheap;
+ clib_mem_heap_t *oldheap;
ASSERT (rp->flags & SVM_FLAGS_MHEAP);
u8 *rv;
svm_mem_alloc_aligned_at_offset (svm_region_t * rp,
uword size, uword align, uword offset)
{
- u8 *oldheap;
+ clib_mem_heap_t *oldheap;
ASSERT (rp->flags & SVM_FLAGS_MHEAP);
u8 *rv;
static inline void
svm_mem_free (svm_region_t * rp, void *ptr)
{
- u8 *oldheap;
+ clib_mem_heap_t *oldheap;
ASSERT (rp->flags & SVM_FLAGS_MHEAP);
pthread_mutex_lock (&rp->mutex);
static inline void *
svm_push_pvt_heap (svm_region_t * rp)
{
- u8 *oldheap;
+ clib_mem_heap_t *oldheap;
oldheap = clib_mem_set_heap (rp->region_heap);
return ((void *) oldheap);
}
static inline void *
svm_push_data_heap (svm_region_t * rp)
{
- u8 *oldheap;
+ clib_mem_heap_t *oldheap;
oldheap = clib_mem_set_heap (rp->data_heap);
return ((void *) oldheap);
}
/* *INDENT-OFF* */
foreach_vlib_main (
({
- void *heap = mm->per_cpu_mheaps[index];
-
vlib_cli_output (vm, "%sThread %d %s\n", index ? "\n":"", index,
vlib_worker_threads[index].name);
- vlib_cli_output (vm, " %U\n", format_page_map,
- pointer_to_uword (clib_mem_get_heap_base(heap)),
- clib_mem_get_heap_size (heap));
vlib_cli_output (vm, " %U\n", format_clib_mem_heap,
mm->per_cpu_mheaps[index],
verbose);
}
if (numa_heaps)
{
- void *heap;
-
for (i = 0; i < ARRAY_LEN (mm->per_numa_mheaps); i++)
{
if (mm->per_numa_mheaps[i] == 0)
continue;
}
was_enabled = clib_mem_trace_enable_disable (0);
- heap = mm->per_numa_mheaps[i];
vlib_cli_output (vm, "Numa %d:", i);
- vlib_cli_output (vm, " %U\n", format_page_map,
- pointer_to_uword (clib_mem_get_heap_base (heap)),
- clib_mem_get_heap_size (heap));
vlib_cli_output (vm, " %U\n", format_clib_mem_heap,
mm->per_numa_mheaps[index], verbose);
}
vlib_node_runtime_t *rt;
u32 n_vlib_mains = tm->n_vlib_mains;
u32 worker_thread_index;
- u8 *main_heap = clib_mem_get_per_cpu_heap ();
+ clib_mem_heap_t *main_heap = clib_mem_get_per_cpu_heap ();
vec_reset_length (vlib_worker_threads);
/* Vector of all bihashes */
void **clib_all_bihashes;
-static void **clib_all_bihash_heap;
+static clib_mem_heap_t *clib_all_bihash_heap;
-void *
+clib_mem_heap_t *
clib_all_bihash_set_heap (void)
{
if (PREDICT_FALSE (clib_all_bihash_heap == 0))
_(format_time_float);
_(format_signal);
_(format_ucontext_pc);
-_(format_page_map);
#endif
#undef _
ptr[i] = start + (i << log2_page_size);
clib_memset (stats, 0, sizeof (clib_mem_page_stats_t));
+ stats->total = n_pages;
+ stats->log2_page_sz = log2_page_size;
if (move_pages (0, n_pages, ptr, 0, status, 0) != 0)
{
return rv;
}
+u8 *
+format_clib_mem_page_stats (u8 * s, va_list * va)
+{
+ clib_mem_page_stats_t *stats = va_arg (*va, clib_mem_page_stats_t *);
+ u32 indent = format_get_indent (s) + 2;
+
+ s = format (s, "page stats: page-size %U, total %lu, mapped %lu, "
+ "not-mapped %lu", format_log2_page_size, stats->log2_page_sz,
+ stats->total, stats->mapped, stats->not_mapped);
+
+ if (stats->unknown)
+ s = format (s, ", unknown %lu", stats->unknown);
+
+ for (int i = 0; i < CLIB_MAX_NUMAS; i++)
+ if (stats->per_numa[i])
+ s = format (s, "\n%Unuma %u: %lu pages, %U bytes",
+ format_white_space, indent, i,
+ stats->per_numa[i],
+ format_memory_size,
+ stats->per_numa[i] << stats->log2_page_sz);
+
+ return s;
+}
+
/*
* fd.io coding-style-patch-verification: ON
*
struct _clib_mem_vm_map_hdr *prev, *next;
} clib_mem_vm_map_hdr_t;
+#define foreach_clib_mem_heap_flag \
+ _(0, LOCKED, "locked") \
+ _(1, UNMAP_ON_DESTROY, "unmap-on-destroy")
+
+typedef enum
+{
+#define _(i, v, s) CLIB_MEM_HEAP_F_##v = (1 << i),
+ foreach_clib_mem_heap_flag
+#undef _
+} clib_mem_heap_flag_t;
+
+typedef struct
+{
+ /* base address */
+ void *base;
+
+ /* dlmalloc mspace */
+ void *mspace;
+
+ /* heap size */
+ uword size;
+
+ /* page size (log2) */
+ clib_mem_page_sz_t log2_page_sz:8;
+
+ /* flags */
+ clib_mem_heap_flag_t flags:8;
+
+ /* name - _MUST_ be last */
+ char name[0];
+} clib_mem_heap_t;
+
typedef struct
{
/* log2 system page size */
/* Unspecified NUMA socket */
#define VEC_NUMA_UNSPECIFIED (0xFF)
-always_inline void *
+always_inline clib_mem_heap_t *
clib_mem_get_per_cpu_heap (void)
{
int cpu = os_get_thread_index ();
}
always_inline void *
-clib_mem_set_per_cpu_heap (u8 * new_heap)
+clib_mem_set_per_cpu_heap (void *new_heap)
{
int cpu = os_get_thread_index ();
void *old = clib_mem_main.per_cpu_mheaps[cpu];
}
always_inline void *
-clib_mem_set_per_numa_heap (u8 * new_heap)
+clib_mem_set_per_numa_heap (void *new_heap)
{
int numa = os_get_numa_index ();
void *old = clib_mem_main.per_numa_mheaps[numa];
clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
int os_out_of_memory_on_failure)
{
- void *heap, *p;
- uword cpu;
void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes,
unsigned long align, unsigned long align_offset);
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ void *p;
if (align_offset > align)
{
align_offset = align;
}
- cpu = os_get_thread_index ();
- heap = clib_mem_main.per_cpu_mheaps[cpu];
-
- p = mspace_get_aligned (heap, size, align, align_offset);
+ p = mspace_get_aligned (h->mspace, size, align, align_offset);
if (PREDICT_FALSE (0 == p))
{
always_inline uword
clib_mem_is_heap_object (void *p)
{
- void *heap = clib_mem_get_per_cpu_heap ();
int mspace_is_heap_object (void *msp, void *p);
-
- return mspace_is_heap_object (heap, p);
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+ return mspace_is_heap_object (h->mspace, p);
}
always_inline void
clib_mem_free (void *p)
{
- u8 *heap = clib_mem_get_per_cpu_heap ();
-
void mspace_put (void *msp, void *p_arg);
+ clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+
/* Make sure object is in the correct heap. */
ASSERT (clib_mem_is_heap_object (p));
CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
- mspace_put (heap, p);
+ mspace_put (h->mspace, p);
}
always_inline void *
clib_mem_free (p);
}
-always_inline void *
+always_inline clib_mem_heap_t *
clib_mem_get_heap (void)
{
return clib_mem_get_per_cpu_heap ();
}
-always_inline void *
-clib_mem_set_heap (void *heap)
+always_inline clib_mem_heap_t *
+clib_mem_set_heap (clib_mem_heap_t * heap)
{
return clib_mem_set_per_cpu_heap (heap);
}
-void clib_mem_destroy_heap (void *heap);
-void *clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt,
- ...);
+void clib_mem_destroy_heap (clib_mem_heap_t * heap);
+clib_mem_heap_t *clib_mem_create_heap (void *base, uword size, int is_locked,
+ char *fmt, ...);
void clib_mem_main_init ();
-void *clib_mem_init (void *heap, uword size);
+void *clib_mem_init (void *base, uword size);
void *clib_mem_init_with_page_size (uword memory_size,
clib_mem_page_sz_t log2_page_sz);
void *clib_mem_init_thread_safe (void *memory, uword memory_size);
uword bytes_max;
} clib_mem_usage_t;
-void clib_mem_get_heap_usage (void *heap, clib_mem_usage_t * usage);
+void clib_mem_get_heap_usage (clib_mem_heap_t * heap,
+ clib_mem_usage_t * usage);
-void *clib_mem_get_heap_base (void *heap);
-uword clib_mem_get_heap_size (void *heap);
-uword clib_mem_get_heap_free_space (void *heap);
+void *clib_mem_get_heap_base (clib_mem_heap_t * heap);
+uword clib_mem_get_heap_size (clib_mem_heap_t * heap);
+uword clib_mem_get_heap_free_space (clib_mem_heap_t * heap);
u8 *format_clib_mem_usage (u8 * s, va_list * args);
u8 *format_clib_mem_heap (u8 * s, va_list * va);
+u8 *format_clib_mem_page_stats (u8 * s, va_list * va);
/* Allocate virtual address space. */
always_inline void *
void clib_mem_destroy (void);
int clib_mem_set_numa_affinity (u8 numa_node, int force);
int clib_mem_set_default_numa_affinity ();
-
-typedef struct
-{
- uword size; /**< Map size */
- int fd; /**< File descriptor to be mapped */
- uword requested_va; /**< Request fixed position mapping */
- void *addr; /**< Pointer to mapped memory, if successful */
- u8 numa_node;
-} clib_mem_vm_map_t;
-
void clib_mem_vm_randomize_va (uword * requested_va,
clib_mem_page_sz_t log2_page_size);
-void mheap_trace (void *v, int enable);
+void mheap_trace (clib_mem_heap_t * v, int enable);
uword clib_mem_trace_enable_disable (uword enable);
void clib_mem_trace (int enable);
typedef struct
{
+ clib_mem_page_sz_t log2_page_sz;
+ uword total;
uword mapped;
uword not_mapped;
uword per_numa[CLIB_MAX_NUMAS];
hash_free (tm->trace_index_by_offset);
}
-/* Initialize CLIB heap based on memory/size given by user.
- Set memory to 0 and CLIB will try to allocate its own heap. */
-static void *
-clib_mem_init_internal (void *memory, uword memory_size,
- clib_mem_page_sz_t log2_page_sz, int set_heap)
+static clib_mem_heap_t *
+clib_mem_create_heap_internal (void *base, uword size,
+ clib_mem_page_sz_t log2_page_sz, int is_locked,
+ char *name)
{
- u8 *heap;
-
- clib_mem_main_init ();
+ clib_mem_heap_t *h;
+ u8 flags = 0;
+ int sz = sizeof (clib_mem_heap_t);
- if (memory)
- {
- heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ );
- mspace_disable_expand (heap);
- }
- else
+ if (base == 0)
{
- memory_size = round_pow2 (memory_size,
- clib_mem_page_bytes (log2_page_sz));
- memory = clib_mem_vm_map_internal (0, log2_page_sz, memory_size, -1, 0,
- "main heap");
+ log2_page_sz = clib_mem_log2_page_size_validate (log2_page_sz);
+ size = round_pow2 (size, clib_mem_page_bytes (log2_page_sz));
+ base = clib_mem_vm_map_internal (0, log2_page_sz, size, -1, 0,
+ "main heap");
- if (memory == CLIB_MEM_VM_MAP_FAILED)
+ if (base == CLIB_MEM_VM_MAP_FAILED)
return 0;
- heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ );
- mspace_disable_expand (heap);
+ flags = CLIB_MEM_HEAP_F_UNMAP_ON_DESTROY;
}
+ else
+ log2_page_sz = CLIB_MEM_PAGE_SZ_UNKNOWN;
+
+ if (is_locked)
+ flags |= CLIB_MEM_HEAP_F_LOCKED;
+
+ h = base;
+ h->base = base;
+ h->size = size;
+ h->log2_page_sz = log2_page_sz;
+ h->flags = flags;
+ sz = strlen (name);
+ strcpy (h->name, name);
+ sz = round_pow2 (sz + sizeof (clib_mem_heap_t), 16);
+ h->mspace = create_mspace_with_base (base + sz, size - sz, is_locked);
- CLIB_MEM_POISON (mspace_least_addr (heap), mspace_footprint (heap));
+ mspace_disable_expand (h->mspace);
- if (set_heap)
- clib_mem_set_heap (heap);
+ CLIB_MEM_POISON (mspace_least_addr (h->mspace),
+ mspace_footprint (h->mspace));
+
+ return h;
+}
+
+/* Initialize CLIB heap based on memory/size given by user.
+ Set memory to 0 and CLIB will try to allocate its own heap. */
+static void *
+clib_mem_init_internal (void *base, uword size,
+ clib_mem_page_sz_t log2_page_sz)
+{
+ clib_mem_heap_t *h;
+
+ clib_mem_main_init ();
+
+ h = clib_mem_create_heap_internal (base, size, log2_page_sz,
+ 1 /*is_locked */ , "main heap");
+
+ clib_mem_set_heap (h);
if (mheap_trace_main.lock == 0)
clib_spinlock_init (&mheap_trace_main.lock);
- return heap;
+ return h;
}
void *
clib_mem_init (void *memory, uword memory_size)
{
return clib_mem_init_internal (memory, memory_size,
- CLIB_MEM_PAGE_SZ_DEFAULT,
- 1 /* do clib_mem_set_heap */ );
+ CLIB_MEM_PAGE_SZ_DEFAULT);
}
void *
clib_mem_init_with_page_size (uword memory_size,
clib_mem_page_sz_t log2_page_sz)
{
- return clib_mem_init_internal (0, memory_size, log2_page_sz,
- 1 /* do clib_mem_set_heap */ );
+ return clib_mem_init_internal (0, memory_size, log2_page_sz);
}
void *
clib_mem_init_thread_safe (void *memory, uword memory_size)
{
return clib_mem_init_internal (memory, memory_size,
- CLIB_MEM_PAGE_SZ_DEFAULT,
- 1 /* do clib_mem_set_heap */ );
+ CLIB_MEM_PAGE_SZ_DEFAULT);
}
void
-clib_mem_destroy_mspace (void *mspace)
+clib_mem_destroy (void)
{
mheap_trace_main_t *tm = &mheap_trace_main;
+ clib_mem_heap_t *heap = clib_mem_get_heap ();
+ void *base = mspace_least_addr (heap->mspace);
- if (tm->enabled && mspace == tm->current_traced_mheap)
+ if (tm->enabled && heap->mspace == tm->current_traced_mheap)
tm->enabled = 0;
- destroy_mspace (mspace);
-}
-
-void
-clib_mem_destroy (void)
-{
- void *heap = clib_mem_get_heap ();
- void *base = mspace_least_addr (heap);
- clib_mem_destroy_mspace (clib_mem_get_heap ());
+ destroy_mspace (heap->mspace);
clib_mem_vm_unmap (base);
}
return s;
}
-
u8 *
format_clib_mem_heap (u8 * s, va_list * va)
{
- void *heap = va_arg (*va, u8 *);
+ clib_mem_heap_t *heap = va_arg (*va, clib_mem_heap_t *);
int verbose = va_arg (*va, int);
struct dlmallinfo mi;
mheap_trace_main_t *tm = &mheap_trace_main;
+ u32 indent = format_get_indent (s) + 2;
if (heap == 0)
heap = clib_mem_get_heap ();
- mi = mspace_mallinfo (heap);
+ mi = mspace_mallinfo (heap->mspace);
+
+ s = format (s, "base %p, size %U",
+ heap->base, format_memory_size, heap->size);
+
+#define _(i,v,str) \
+ if (heap->flags & CLIB_MEM_HEAP_F_##v) s = format (s, ", %s", str);
+ foreach_clib_mem_heap_flag;
+#undef _
+
+ s = format (s, ", name '%s'", heap->name);
+
+ if (heap->log2_page_sz != CLIB_MEM_PAGE_SZ_UNKNOWN)
+ {
+ clib_mem_page_stats_t stats;
+ clib_mem_get_page_stats (heap->base, heap->log2_page_sz,
+ heap->size >> heap->log2_page_sz, &stats);
+ s = format (s, "\n%U%U", format_white_space, indent,
+ format_clib_mem_page_stats, &stats);
+ }
- s = format (s, "total: %U, used: %U, free: %U, trimmable: %U",
+ s = format (s, "\n%Utotal: %U, used: %U, free: %U, trimmable: %U",
+ format_white_space, indent,
format_msize, mi.arena,
format_msize, mi.uordblks,
format_msize, mi.fordblks, format_msize, mi.keepcost);
if (verbose > 0)
{
- s = format (s, "\n free chunks %llu free fastbin blks %llu",
- mi.ordblks, mi.smblks);
- s =
- format (s, "\n max total allocated %U", format_msize, mi.usmblks);
+ s = format (s, "\n%Ufree chunks %llu free fastbin blks %llu",
+ format_white_space, indent + 2, mi.ordblks, mi.smblks);
+ s = format (s, "\n%Umax total allocated %U",
+ format_white_space, indent + 2, format_msize, mi.usmblks);
}
- if (mspace_is_traced (heap))
+ if (mspace_is_traced (heap->mspace))
s = format (s, "\n%U", format_mheap_trace, tm, verbose);
return s;
}
void
-clib_mem_usage (clib_mem_usage_t * u)
+clib_mem_get_heap_usage (clib_mem_heap_t * heap, clib_mem_usage_t * usage)
{
- clib_warning ("unimp");
-}
-
-void
-clib_mem_get_heap_usage (void *heap, clib_mem_usage_t * usage)
-{
- struct dlmallinfo mi = mspace_mallinfo (heap);
+ struct dlmallinfo mi = mspace_mallinfo (heap->mspace);
/* TODO: Fill in some more values */
usage->object_count = 0;
uword clib_mem_validate_serial = 0;
void
-clib_mem_validate (void)
-{
- clib_warning ("unimp");
-}
-
-void
-mheap_trace (void *v, int enable)
+mheap_trace (clib_mem_heap_t * h, int enable)
{
- (void) mspace_enable_disable_trace (v, enable);
+ (void) mspace_enable_disable_trace (h->mspace, enable);
if (enable == 0)
mheap_trace_main_free (&mheap_trace_main);
int
clib_mem_is_traced (void)
{
- return mspace_is_traced (clib_mem_get_heap ());
+ clib_mem_heap_t *h = clib_mem_get_heap ();
+ return mspace_is_traced (h->mspace);
}
uword
return rv;
}
-void *
+clib_mem_heap_t *
clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt, ...)
{
- void *rv;
- if (base == 0)
- rv = create_mspace (size, is_locked);
+ clib_mem_page_sz_t log2_page_sz = clib_mem_get_log2_page_size ();
+ clib_mem_heap_t *h;
+ char *name;
+ u8 *s = 0;
+
+ if (fmt == 0)
+ {
+ name = "";
+ }
+ else if (strchr (fmt, '%'))
+ {
+ va_list va;
+ va_start (va, fmt);
+ s = va_format (0, fmt, &va);
+ vec_add1 (s, 0);
+ va_end (va);
+ name = (char *) s;
+ }
else
- rv = create_mspace_with_base (base, size, is_locked);
+ name = fmt;
- if (rv)
- mspace_disable_expand (rv);
- return rv;
+ h = clib_mem_create_heap_internal (base, size, log2_page_sz, is_locked,
+ name);
+ vec_free (s);
+ return h;
}
void
-clib_mem_destroy_heap (void *heap)
+clib_mem_destroy_heap (clib_mem_heap_t * h)
{
- destroy_mspace (heap);
+ mheap_trace_main_t *tm = &mheap_trace_main;
+
+ if (tm->enabled && h->mspace == tm->current_traced_mheap)
+ tm->enabled = 0;
+
+ destroy_mspace (h->mspace);
+ if (h->flags & CLIB_MEM_HEAP_F_UNMAP_ON_DESTROY)
+ clib_mem_vm_unmap (h->base);
}
uword
-clib_mem_get_heap_free_space (void *heap)
+clib_mem_get_heap_free_space (clib_mem_heap_t * h)
{
- struct dlmallinfo dlminfo = mspace_mallinfo (heap);
+ struct dlmallinfo dlminfo = mspace_mallinfo (h->mspace);
return dlminfo.fordblks;
}
void *
-clib_mem_get_heap_base (void *heap)
+clib_mem_get_heap_base (clib_mem_heap_t * h)
{
- return mspace_least_addr (heap);
+ return h->base;
}
uword
-clib_mem_get_heap_size (void *heap)
+clib_mem_get_heap_size (clib_mem_heap_t * heap)
{
- struct dlmallinfo mi;
- mi = mspace_mallinfo (heap);
- return mi.arena;
+ return heap->size;
}
/*
return 0;
}
-#define MAX_NUMNODES 16
-u8 *
-format_page_map (u8 * s, va_list * args)
-{
- uword va = va_arg (*args, uword);
- uword size = va_arg (*args, uword);
- uword page_size = clib_mem_get_page_size ();
- u32 indent = format_get_indent (s);
- uword n_pages = size / page_size;
- uword pages_per_numa[MAX_NUMNODES] = { 0 };
- uword pages_not_mapped = 0;
- uword pages_unknown = 0;
- int *status = 0;
- void **ptr = 0;
- int i;
-
- s = format (s, "virtual memory start 0x%llx, size %lluk, %u pages, "
- "page size %uk", va, size / 1024, n_pages, page_size / 1024);
-
- vec_validate (status, n_pages - 1);
- vec_validate (ptr, n_pages - 1);
-
- for (i = 0; i < n_pages; i++)
- ptr[i] = uword_to_pointer (va + i * page_size, void *);
-
- if (move_pages (0, n_pages, ptr, 0, status, 0) != 0)
- {
- s = format (s, "\n%Upage information not available (errno %u)",
- format_white_space, indent + 2, errno);
- goto done;
- }
-
- for (i = 0; i < n_pages; i++)
- {
- if (status[i] >= 0 && status[i] < MAX_NUMNODES)
- pages_per_numa[status[i]]++;
- else if (status[i] == -EFAULT)
- pages_not_mapped++;
- else
- pages_unknown++;
- }
-
- for (i = 0; i < MAX_NUMNODES; i++)
- if (pages_per_numa[i])
- s = format (s, "\n%Unuma %u: %d pages, %luk", format_white_space,
- indent + 2, i, pages_per_numa[i], pages_per_numa[i] *
- page_size / 1024);
-
- s = format (s, "\n%Unot mapped: %u pages, %luk", format_white_space,
- indent + 2, pages_not_mapped, pages_not_mapped *
- page_size / 1024);
-
- if (pages_unknown)
- s = format (s, "\n%Uunknown: %u pages, %luk", format_white_space,
- indent + 2, pages_unknown, pages_unknown * page_size / 1024);
-
-done:
- vec_free (status);
- vec_free (ptr);
- return s;
-}
-
#endif /* __KERNEL__ */
Code Review / vpp.git / commitdiff