2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vppinfra/format.h>
17 #include <vppinfra/dlmalloc.h>
18 #include <vppinfra/os.h>
19 #include <vppinfra/lock.h>
20 #include <vppinfra/hash.h>
21 #include <vppinfra/elf_clib.h>
25 /* Address of callers: outer first, inner last. */
28 /* Count of allocations with this traceback. */
31 /* Count of bytes allocated with this traceback. */
34 /* Offset of this item */
42 mheap_trace_t *traces;
44 /* Indices of free traces. */
47 /* Hash table mapping callers to trace index. */
48 uword *trace_by_callers;
50 /* Hash table mapping mheap offset to trace index. */
51 uword *trace_index_by_offset;
53 /* So we can easily shut off current segment trace, if any */
54 const clib_mem_heap_t *current_traced_mheap;
58 mheap_trace_main_t mheap_trace_main;
60 static __thread int mheap_trace_thread_disable;
63 mheap_get_trace_internal (const clib_mem_heap_t *heap, uword offset,
66 mheap_trace_main_t *tm = &mheap_trace_main;
68 uword i, n_callers, trace_index, *p;
71 if (heap != tm->current_traced_mheap || mheap_trace_thread_disable)
74 /* Spurious Coverity warnings be gone. */
75 clib_memset (&trace, 0, sizeof (trace));
77 clib_spinlock_lock (&tm->lock);
79 /* heap could have changed while we were waiting on the lock */
80 if (heap != tm->current_traced_mheap)
83 /* Turn off tracing for this thread to avoid embarrassment... */
84 mheap_trace_thread_disable = 1;
86 /* Skip our frame and mspace_get_aligned's frame */
87 n_callers = clib_backtrace (trace.callers, ARRAY_LEN (trace.callers), 2);
91 if (!tm->trace_by_callers)
92 tm->trace_by_callers =
93 hash_create_shmem (0, sizeof (trace.callers), sizeof (uword));
95 p = hash_get_mem (tm->trace_by_callers, &trace.callers);
99 t = tm->traces + trace_index;
103 i = vec_len (tm->trace_free_list);
106 trace_index = tm->trace_free_list[i - 1];
107 vec_set_len (tm->trace_free_list, i - 1);
111 mheap_trace_t *old_start = tm->traces;
112 mheap_trace_t *old_end = vec_end (tm->traces);
114 vec_add2 (tm->traces, t, 1);
116 if (tm->traces != old_start)
121 hash_foreach_pair (p, tm->trace_by_callers,
123 q = uword_to_pointer (p->key, mheap_trace_t *);
124 ASSERT (q >= old_start && q < old_end);
125 p->key = pointer_to_uword (tm->traces + (q - old_start));
129 trace_index = t - tm->traces;
132 t = tm->traces + trace_index;
134 t->n_allocations = 0;
136 hash_set_mem (tm->trace_by_callers, t->callers, trace_index);
139 t->n_allocations += 1;
141 t->offset = offset; /* keep a sample to autopsy */
142 hash_set (tm->trace_index_by_offset, offset, t - tm->traces);
145 mheap_trace_thread_disable = 0;
146 clib_spinlock_unlock (&tm->lock);
150 mheap_put_trace_internal (const clib_mem_heap_t *heap, uword offset,
154 uword trace_index, *p;
155 mheap_trace_main_t *tm = &mheap_trace_main;
157 if (heap != tm->current_traced_mheap || mheap_trace_thread_disable)
160 clib_spinlock_lock (&tm->lock);
162 /* heap could have changed while we were waiting on the lock */
163 if (heap != tm->current_traced_mheap)
166 /* Turn off tracing for this thread for a moment */
167 mheap_trace_thread_disable = 1;
169 p = hash_get (tm->trace_index_by_offset, offset);
174 hash_unset (tm->trace_index_by_offset, offset);
175 ASSERT (trace_index < vec_len (tm->traces));
177 t = tm->traces + trace_index;
178 ASSERT (t->n_allocations > 0);
179 ASSERT (t->n_bytes >= size);
180 t->n_allocations -= 1;
182 if (t->n_allocations == 0)
184 hash_unset_mem (tm->trace_by_callers, t->callers);
185 vec_add1 (tm->trace_free_list, trace_index);
186 clib_memset (t, 0, sizeof (t[0]));
190 mheap_trace_thread_disable = 0;
191 clib_spinlock_unlock (&tm->lock);
195 mheap_get_trace (uword offset, uword size)
197 mheap_get_trace_internal (clib_mem_get_heap (), offset, size);
201 mheap_put_trace (uword offset, uword size)
203 mheap_put_trace_internal (clib_mem_get_heap (), offset, size);
207 mheap_trace_main_free (mheap_trace_main_t * tm)
209 CLIB_SPINLOCK_ASSERT_LOCKED (&tm->lock);
210 tm->current_traced_mheap = 0;
211 vec_free (tm->traces);
212 vec_free (tm->trace_free_list);
213 hash_free (tm->trace_by_callers);
214 hash_free (tm->trace_index_by_offset);
215 mheap_trace_thread_disable = 0;
218 static clib_mem_heap_t *
219 clib_mem_create_heap_internal (void *base, uword size,
220 clib_mem_page_sz_t log2_page_sz, int is_locked,
225 int sz = sizeof (clib_mem_heap_t);
229 log2_page_sz = clib_mem_log2_page_size_validate (log2_page_sz);
230 size = round_pow2 (size, clib_mem_page_bytes (log2_page_sz));
231 base = clib_mem_vm_map_internal (0, log2_page_sz, size, -1, 0,
234 if (base == CLIB_MEM_VM_MAP_FAILED)
237 flags = CLIB_MEM_HEAP_F_UNMAP_ON_DESTROY;
240 log2_page_sz = CLIB_MEM_PAGE_SZ_UNKNOWN;
243 flags |= CLIB_MEM_HEAP_F_LOCKED;
248 h->log2_page_sz = log2_page_sz;
251 strcpy (h->name, name);
252 sz = round_pow2 (sz + sizeof (clib_mem_heap_t), 16);
253 h->mspace = create_mspace_with_base (base + sz, size - sz, is_locked);
255 mspace_disable_expand (h->mspace);
257 clib_mem_poison (mspace_least_addr (h->mspace),
258 mspace_footprint (h->mspace));
263 /* Initialize CLIB heap based on memory/size given by user.
264 Set memory to 0 and CLIB will try to allocate its own heap. */
266 clib_mem_init_internal (void *base, uword size,
267 clib_mem_page_sz_t log2_page_sz)
271 clib_mem_main_init ();
273 h = clib_mem_create_heap_internal (base, size, log2_page_sz,
274 1 /*is_locked */ , "main heap");
276 clib_mem_set_heap (h);
278 if (mheap_trace_main.lock == 0)
280 /* clib_spinlock_init() dynamically allocates the spinlock in the current
281 * per-cpu heap, but it is used for all traces accross all heaps and
282 * hence we can't really allocate it in the current per-cpu heap as it
283 * could be destroyed later */
284 static struct clib_spinlock_s mheap_trace_main_lock = {};
285 mheap_trace_main.lock = &mheap_trace_main_lock;
292 clib_mem_init (void *memory, uword memory_size)
294 return clib_mem_init_internal (memory, memory_size,
295 CLIB_MEM_PAGE_SZ_DEFAULT);
299 clib_mem_init_with_page_size (uword memory_size,
300 clib_mem_page_sz_t log2_page_sz)
302 return clib_mem_init_internal (0, memory_size, log2_page_sz);
306 clib_mem_init_thread_safe (void *memory, uword memory_size)
308 return clib_mem_init_internal (memory, memory_size,
309 CLIB_MEM_PAGE_SZ_DEFAULT);
313 clib_mem_destroy (void)
315 mheap_trace_main_t *tm = &mheap_trace_main;
316 clib_mem_heap_t *heap = clib_mem_get_heap ();
318 if (heap->mspace == tm->current_traced_mheap)
319 mheap_trace (heap, 0);
321 destroy_mspace (heap->mspace);
322 clib_mem_vm_unmap (heap);
326 format_clib_mem_usage (u8 *s, va_list *va)
328 int verbose = va_arg (*va, int);
329 return format (s, "$$$$ heap at %llx verbose %d", clib_mem_get_heap (),
334 * Magic decoder ring for mallinfo stats (ala dlmalloc):
336 * size_t arena; / * Non-mmapped space allocated (bytes) * /
337 * size_t ordblks; / * Number of free chunks * /
338 * size_t smblks; / * Number of free fastbin blocks * /
339 * size_t hblks; / * Number of mmapped regions * /
340 * size_t hblkhd; / * Space allocated in mmapped regions (bytes) * /
341 * size_t usmblks; / * Maximum total allocated space (bytes) * /
342 * size_t fsmblks; / * Space in freed fastbin blocks (bytes) * /
343 * size_t uordblks; / * Total allocated space (bytes) * /
344 * size_t fordblks; / * Total free space (bytes) * /
345 * size_t keepcost; / * Top-most, releasable space (bytes) * /
350 format_msize (u8 * s, va_list * va)
352 uword a = va_arg (*va, uword);
355 s = format (s, "%.2fG", (((f64) a) / ((f64) (1ULL << 30))));
356 else if (a >= 1ULL << 20)
357 s = format (s, "%.2fM", (((f64) a) / ((f64) (1ULL << 20))));
358 else if (a >= 1ULL << 10)
359 s = format (s, "%.2fK", (((f64) a) / ((f64) (1ULL << 10))));
361 s = format (s, "%lld", a);
366 mheap_trace_sort (const void *_t1, const void *_t2)
368 const mheap_trace_t *t1 = _t1;
369 const mheap_trace_t *t2 = _t2;
372 cmp = (word) t2->n_bytes - (word) t1->n_bytes;
374 cmp = (word) t2->n_allocations - (word) t1->n_allocations;
379 format_mheap_trace (u8 * s, va_list * va)
381 mheap_trace_main_t *tm = va_arg (*va, mheap_trace_main_t *);
382 int verbose = va_arg (*va, int);
387 clib_spinlock_lock (&tm->lock);
388 if (vec_len (tm->traces) > 0 &&
389 clib_mem_get_heap () == tm->current_traced_mheap)
393 /* Make a copy of traces since we'll be sorting them. */
394 mheap_trace_t *t, *traces_copy;
395 u32 indent, total_objects_traced;
397 traces_copy = vec_dup (tm->traces);
399 qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]),
402 total_objects_traced = 0;
403 s = format (s, "\n");
404 vec_foreach (t, traces_copy)
406 /* Skip over free elements. */
407 if (t->n_allocations == 0)
410 total_objects_traced += t->n_allocations;
412 /* When not verbose only report the 50 biggest allocations */
413 if (!verbose && n >= 50)
417 if (t == traces_copy)
418 s = format (s, "%=9s%=9s %=10s Traceback\n", "Bytes", "Count",
420 s = format (s, "%9d%9d %p", t->n_bytes, t->n_allocations, t->offset);
421 indent = format_get_indent (s);
422 for (i = 0; i < ARRAY_LEN (t->callers) && t->callers[i]; i++)
425 s = format (s, "%U", format_white_space, indent);
426 #if defined(CLIB_UNIX) && !defined(__APPLE__)
427 /* $$$$ does this actually work? */
429 format (s, " %U\n", format_clib_elf_symbol_with_address,
432 s = format (s, " %p\n", t->callers[i]);
437 s = format (s, "%d total traced objects\n", total_objects_traced);
439 vec_free (traces_copy);
441 clib_spinlock_unlock (&tm->lock);
442 if (have_traces == 0)
443 s = format (s, "no traced allocations\n");
449 format_clib_mem_heap (u8 * s, va_list * va)
451 clib_mem_heap_t *heap = va_arg (*va, clib_mem_heap_t *);
452 int verbose = va_arg (*va, int);
453 struct dlmallinfo mi;
454 mheap_trace_main_t *tm = &mheap_trace_main;
455 u32 indent = format_get_indent (s) + 2;
458 heap = clib_mem_get_heap ();
460 mi = mspace_mallinfo (heap->mspace);
462 s = format (s, "base %p, size %U",
463 heap->base, format_memory_size, heap->size);
466 if (heap->flags & CLIB_MEM_HEAP_F_##v) s = format (s, ", %s", str);
467 foreach_clib_mem_heap_flag;
470 s = format (s, ", name '%s'", heap->name);
472 if (heap->log2_page_sz != CLIB_MEM_PAGE_SZ_UNKNOWN)
474 clib_mem_page_stats_t stats;
475 clib_mem_get_page_stats (heap->base, heap->log2_page_sz,
476 heap->size >> heap->log2_page_sz, &stats);
477 s = format (s, "\n%U%U", format_white_space, indent,
478 format_clib_mem_page_stats, &stats);
481 s = format (s, "\n%Utotal: %U, used: %U, free: %U, trimmable: %U",
482 format_white_space, indent,
483 format_msize, mi.arena,
484 format_msize, mi.uordblks,
485 format_msize, mi.fordblks, format_msize, mi.keepcost);
488 s = format (s, "\n%Ufree chunks %llu free fastbin blks %llu",
489 format_white_space, indent + 2, mi.ordblks, mi.smblks);
490 s = format (s, "\n%Umax total allocated %U",
491 format_white_space, indent + 2, format_msize, mi.usmblks);
494 if (heap->flags & CLIB_MEM_HEAP_F_TRACED)
495 s = format (s, "\n%U", format_mheap_trace, tm, verbose);
499 __clib_export __clib_flatten void
500 clib_mem_get_heap_usage (clib_mem_heap_t *heap, clib_mem_usage_t *usage)
502 struct dlmallinfo mi = mspace_mallinfo (heap->mspace);
504 usage->bytes_total = mi.arena; /* non-mmapped space allocated from system */
505 usage->bytes_used = mi.uordblks; /* total allocated space */
506 usage->bytes_free = mi.fordblks; /* total free space */
507 usage->bytes_used_mmap = mi.hblkhd; /* space in mmapped regions */
508 usage->bytes_max = mi.usmblks; /* maximum total allocated space */
509 usage->bytes_free_reclaimed = mi.ordblks; /* number of free chunks */
510 usage->bytes_overhead = mi.keepcost; /* releasable (via malloc_trim) space */
513 usage->bytes_used_sbrk = 0;
514 usage->object_count = 0;
517 /* Call serial number for debugger breakpoints. */
518 uword clib_mem_validate_serial = 0;
521 mheap_trace (clib_mem_heap_t * h, int enable)
523 mheap_trace_main_t *tm = &mheap_trace_main;
525 clib_spinlock_lock (&tm->lock);
527 if (tm->current_traced_mheap != 0 && tm->current_traced_mheap != h)
529 clib_warning ("tracing already enabled for another heap, ignoring");
535 h->flags |= CLIB_MEM_HEAP_F_TRACED;
536 tm->current_traced_mheap = h;
540 h->flags &= ~CLIB_MEM_HEAP_F_TRACED;
541 mheap_trace_main_free (&mheap_trace_main);
545 clib_spinlock_unlock (&tm->lock);
549 clib_mem_trace (int enable)
551 void *current_heap = clib_mem_get_heap ();
552 mheap_trace (current_heap, enable);
556 clib_mem_is_traced (void)
558 clib_mem_heap_t *h = clib_mem_get_heap ();
559 return (h->flags &= CLIB_MEM_HEAP_F_TRACED) != 0;
563 clib_mem_trace_enable_disable (uword enable)
565 uword rv = !mheap_trace_thread_disable;
566 mheap_trace_thread_disable = !enable;
570 __clib_export clib_mem_heap_t *
571 clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt, ...)
573 clib_mem_page_sz_t log2_page_sz = clib_mem_get_log2_page_size ();
582 else if (strchr (fmt, '%'))
586 s = va_format (0, fmt, &va);
594 h = clib_mem_create_heap_internal (base, size, log2_page_sz, is_locked,
601 clib_mem_destroy_heap (clib_mem_heap_t * h)
603 mheap_trace_main_t *tm = &mheap_trace_main;
605 if (h->mspace == tm->current_traced_mheap)
608 destroy_mspace (h->mspace);
609 if (h->flags & CLIB_MEM_HEAP_F_UNMAP_ON_DESTROY)
610 clib_mem_vm_unmap (h->base);
613 __clib_export __clib_flatten uword
614 clib_mem_get_heap_free_space (clib_mem_heap_t *h)
616 struct dlmallinfo dlminfo = mspace_mallinfo (h->mspace);
617 return dlminfo.fordblks;
620 __clib_export __clib_flatten void *
621 clib_mem_get_heap_base (clib_mem_heap_t *h)
626 __clib_export __clib_flatten uword
627 clib_mem_get_heap_size (clib_mem_heap_t *heap)
632 /* Memory allocator which may call os_out_of_memory() if it fails */
634 clib_mem_heap_alloc_inline (void *heap, uword size, uword align,
635 int os_out_of_memory_on_failure)
637 clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
640 align = clib_max (CLIB_MEM_MIN_ALIGN, align);
642 p = mspace_memalign (h->mspace, align, size);
644 if (PREDICT_FALSE (0 == p))
646 if (os_out_of_memory_on_failure)
651 if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED))
652 mheap_get_trace_internal (h, pointer_to_uword (p), clib_mem_size (p));
654 clib_mem_unpoison (p, size);
658 /* Memory allocator which calls os_out_of_memory() when it fails */
659 __clib_export __clib_flatten void *
660 clib_mem_alloc (uword size)
662 return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN,
663 /* os_out_of_memory */ 1);
666 __clib_export __clib_flatten void *
667 clib_mem_alloc_aligned (uword size, uword align)
669 return clib_mem_heap_alloc_inline (0, size, align,
670 /* os_out_of_memory */ 1);
673 /* Memory allocator which calls os_out_of_memory() when it fails */
674 __clib_export __clib_flatten void *
675 clib_mem_alloc_or_null (uword size)
677 return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN,
678 /* os_out_of_memory */ 0);
681 __clib_export __clib_flatten void *
682 clib_mem_alloc_aligned_or_null (uword size, uword align)
684 return clib_mem_heap_alloc_inline (0, size, align,
685 /* os_out_of_memory */ 0);
688 __clib_export __clib_flatten void *
689 clib_mem_heap_alloc (void *heap, uword size)
691 return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN,
692 /* os_out_of_memory */ 1);
695 __clib_export __clib_flatten void *
696 clib_mem_heap_alloc_aligned (void *heap, uword size, uword align)
698 return clib_mem_heap_alloc_inline (heap, size, align,
699 /* os_out_of_memory */ 1);
702 __clib_export __clib_flatten void *
703 clib_mem_heap_alloc_or_null (void *heap, uword size)
705 return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN,
706 /* os_out_of_memory */ 0);
709 __clib_export __clib_flatten void *
710 clib_mem_heap_alloc_aligned_or_null (void *heap, uword size, uword align)
712 return clib_mem_heap_alloc_inline (heap, size, align,
713 /* os_out_of_memory */ 0);
716 __clib_export __clib_flatten void *
717 clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size,
720 uword old_alloc_size;
721 clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
724 ASSERT (count_set_bits (align) == 1);
726 old_alloc_size = p ? mspace_usable_size (p) : 0;
728 if (new_size == old_alloc_size)
731 if (p && pointer_is_aligned (p, align) &&
732 mspace_realloc_in_place (h->mspace, p, new_size))
734 clib_mem_unpoison (p, new_size);
735 if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED))
737 mheap_put_trace_internal (h, pointer_to_uword (p), old_alloc_size);
738 mheap_get_trace_internal (h, pointer_to_uword (p),
744 new = clib_mem_heap_alloc_inline (h, new_size, align, 1);
746 clib_mem_unpoison (new, new_size);
749 clib_mem_unpoison (p, old_alloc_size);
750 clib_memcpy_fast (new, p, clib_min (new_size, old_alloc_size));
751 clib_mem_heap_free (h, p);
759 __clib_export __clib_flatten void *
760 clib_mem_heap_realloc (void *heap, void *p, uword new_size)
762 return clib_mem_heap_realloc_aligned (heap, p, new_size, CLIB_MEM_MIN_ALIGN);
765 __clib_export __clib_flatten void *
766 clib_mem_realloc_aligned (void *p, uword new_size, uword align)
768 return clib_mem_heap_realloc_aligned (0, p, new_size, align);
771 __clib_export __clib_flatten void *
772 clib_mem_realloc (void *p, uword new_size)
774 return clib_mem_heap_realloc_aligned (0, p, new_size, CLIB_MEM_MIN_ALIGN);
777 __clib_export __clib_flatten uword
778 clib_mem_heap_is_heap_object (void *heap, void *p)
780 clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
781 return mspace_is_heap_object (h->mspace, p);
784 __clib_export __clib_flatten uword
785 clib_mem_is_heap_object (void *p)
787 return clib_mem_heap_is_heap_object (0, p);
790 __clib_export __clib_flatten void
791 clib_mem_heap_free (void *heap, void *p)
793 clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
794 uword size = clib_mem_size (p);
796 /* Make sure object is in the correct heap. */
797 ASSERT (clib_mem_heap_is_heap_object (h, p));
799 if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED))
800 mheap_put_trace_internal (h, pointer_to_uword (p), size);
801 clib_mem_poison (p, clib_mem_size (p));
803 mspace_free (h->mspace, p);
806 __clib_export __clib_flatten void
807 clib_mem_free (void *p)
809 clib_mem_heap_free (0, p);
812 __clib_export __clib_flatten uword
813 clib_mem_size (void *p)
815 return mspace_usable_size (p);
819 clib_mem_free_s (void *p)
821 uword size = clib_mem_size (p);
822 clib_mem_unpoison (p, size);
823 memset_s_inline (p, size, 0, size);