2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vppinfra/format.h>
17 #include <vppinfra/dlmalloc.h>
18 #include <vppinfra/os.h>
19 #include <vppinfra/lock.h>
20 #include <vppinfra/hash.h>
21 #include <vppinfra/elf_clib.h>
23 void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS];
27 /* Address of callers: outer first, inner last. */
30 /* Count of allocations with this traceback. */
37 /* Count of bytes allocated with this traceback. */
40 /* Offset of this item */
49 mheap_trace_t *traces;
51 /* Indices of free traces. */
54 /* Hash table mapping callers to trace index. */
55 uword *trace_by_callers;
57 /* Hash table mapping mheap offset to trace index. */
58 uword *trace_index_by_offset;
61 mheap_trace_main_t mheap_trace_main;
64 mheap_get_trace (uword offset, uword size)
66 mheap_trace_main_t *tm = &mheap_trace_main;
68 uword i, n_callers, trace_index, *p;
75 /* Spurious Coverity warnings be gone. */
76 clib_memset (&trace, 0, sizeof (trace));
78 /* Skip our frame and mspace_get_aligned's frame */
79 n_callers = clib_backtrace (trace.callers, ARRAY_LEN (trace.callers), 2);
83 /* $$$ This looks like dreck to remove... */
85 for (i = n_callers; i < ARRAY_LEN (trace.callers); i++)
88 clib_spinlock_lock (&tm->lock);
90 /* Turn off tracing to avoid embarrassment... */
91 save_enabled = tm->enabled;
94 if (!tm->trace_by_callers)
95 tm->trace_by_callers =
96 hash_create_shmem (0, sizeof (trace.callers), sizeof (uword));
98 p = hash_get_mem (tm->trace_by_callers, &trace.callers);
102 t = tm->traces + trace_index;
106 i = vec_len (tm->trace_free_list);
109 trace_index = tm->trace_free_list[i - 1];
110 _vec_len (tm->trace_free_list) = i - 1;
114 mheap_trace_t *old_start = tm->traces;
115 mheap_trace_t *old_end = vec_end (tm->traces);
117 vec_add2 (tm->traces, t, 1);
119 if (tm->traces != old_start)
124 hash_foreach_pair (p, tm->trace_by_callers,
126 q = uword_to_pointer (p->key, mheap_trace_t *);
127 ASSERT (q >= old_start && q < old_end);
128 p->key = pointer_to_uword (tm->traces + (q - old_start));
132 trace_index = t - tm->traces;
135 t = tm->traces + trace_index;
137 t->n_allocations = 0;
139 hash_set_mem (tm->trace_by_callers, t->callers, trace_index);
142 t->n_allocations += 1;
144 t->offset = offset; /* keep a sample to autopsy */
145 hash_set (tm->trace_index_by_offset, offset, t - tm->traces);
146 tm->enabled = save_enabled;
147 clib_spinlock_unlock (&tm->lock);
151 mheap_put_trace (uword offset, uword size)
154 uword trace_index, *p;
155 mheap_trace_main_t *tm = &mheap_trace_main;
158 if (tm->enabled == 0)
161 clib_spinlock_lock (&tm->lock);
163 /* Turn off tracing for a moment */
164 save_enabled = tm->enabled;
167 p = hash_get (tm->trace_index_by_offset, offset);
170 tm->enabled = save_enabled;
171 clib_spinlock_unlock (&tm->lock);
176 hash_unset (tm->trace_index_by_offset, offset);
177 ASSERT (trace_index < vec_len (tm->traces));
179 t = tm->traces + trace_index;
180 ASSERT (t->n_allocations > 0);
181 ASSERT (t->n_bytes >= size);
182 t->n_allocations -= 1;
184 if (t->n_allocations == 0)
186 hash_unset_mem (tm->trace_by_callers, t->callers);
187 vec_add1 (tm->trace_free_list, trace_index);
188 clib_memset (t, 0, sizeof (t[0]));
190 tm->enabled = save_enabled;
191 clib_spinlock_unlock (&tm->lock);
195 mheap_trace_main_free (mheap_trace_main_t * tm)
197 vec_free (tm->traces);
198 vec_free (tm->trace_free_list);
199 hash_free (tm->trace_by_callers);
200 hash_free (tm->trace_index_by_offset);
203 /* Initialize CLIB heap based on memory/size given by user.
204 Set memory to 0 and CLIB will try to allocate its own heap. */
206 clib_mem_init (void *memory, uword memory_size)
212 heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ );
213 mspace_disable_expand (heap);
216 heap = create_mspace (memory_size, 1 /* locked */ );
218 clib_mem_set_heap (heap);
220 if (mheap_trace_main.lock == 0)
221 clib_spinlock_init (&mheap_trace_main.lock);
227 clib_mem_init_thread_safe (void *memory, uword memory_size)
229 return clib_mem_init (memory, memory_size);
233 format_clib_mem_usage (u8 * s, va_list * va)
235 int verbose = va_arg (*va, int);
236 return format (s, "$$$$ heap at %llx verbose %d", clib_mem_get_heap (),
241 * Magic decoder ring for mallinfo stats (ala dlmalloc):
243 * size_t arena; / * Non-mmapped space allocated (bytes) * /
244 * size_t ordblks; / * Number of free chunks * /
245 * size_t smblks; / * Number of free fastbin blocks * /
246 * size_t hblks; / * Number of mmapped regions * /
247 * size_t hblkhd; / * Space allocated in mmapped regions (bytes) * /
248 * size_t usmblks; / * Maximum total allocated space (bytes) * /
249 * size_t fsmblks; / * Space in freed fastbin blocks (bytes) * /
250 * size_t uordblks; / * Total allocated space (bytes) * /
251 * size_t fordblks; / * Total free space (bytes) * /
252 * size_t keepcost; / * Top-most, releasable space (bytes) * /
257 format_msize (u8 * s, va_list * va)
259 uword a = va_arg (*va, uword);
262 s = format (s, "%.2fG", (((f64) a) / ((f64) (1ULL << 30))));
263 else if (a >= 1ULL << 20)
264 s = format (s, "%.2fM", (((f64) a) / ((f64) (1ULL << 20))));
265 else if (a >= 1ULL << 10)
266 s = format (s, "%.2fK", (((f64) a) / ((f64) (1ULL << 10))));
268 s = format (s, "%lld", a);
273 mheap_trace_sort (const void *_t1, const void *_t2)
275 const mheap_trace_t *t1 = _t1;
276 const mheap_trace_t *t2 = _t2;
279 cmp = (word) t2->n_bytes - (word) t1->n_bytes;
281 cmp = (word) t2->n_allocations - (word) t1->n_allocations;
286 format_mheap_trace (u8 * s, va_list * va)
288 mheap_trace_main_t *tm = va_arg (*va, mheap_trace_main_t *);
289 int verbose = va_arg (*va, int);
293 clib_spinlock_lock (&tm->lock);
294 if (vec_len (tm->traces) > 0)
298 /* Make a copy of traces since we'll be sorting them. */
299 mheap_trace_t *t, *traces_copy;
300 u32 indent, total_objects_traced;
302 traces_copy = vec_dup (tm->traces);
304 qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]),
307 total_objects_traced = 0;
308 s = format (s, "\n");
309 vec_foreach (t, traces_copy)
311 /* Skip over free elements. */
312 if (t->n_allocations == 0)
315 total_objects_traced += t->n_allocations;
317 /* When not verbose only report allocations of more than 1k. */
318 if (!verbose && t->n_bytes < 1024)
321 if (t == traces_copy)
322 s = format (s, "%=9s%=9s %=10s Traceback\n", "Bytes", "Count",
324 s = format (s, "%9d%9d %p", t->n_bytes, t->n_allocations, t->offset);
325 indent = format_get_indent (s);
326 for (i = 0; i < ARRAY_LEN (t->callers) && t->callers[i]; i++)
329 s = format (s, "%U", format_white_space, indent);
330 #if defined(CLIB_UNIX) && !defined(__APPLE__)
331 /* $$$$ does this actually work? */
333 format (s, " %U\n", format_clib_elf_symbol_with_address,
336 s = format (s, " %p\n", t->callers[i]);
341 s = format (s, "%d total traced objects\n", total_objects_traced);
343 vec_free (traces_copy);
345 clib_spinlock_unlock (&tm->lock);
346 if (have_traces == 0)
347 s = format (s, "no traced allocations\n");
354 format_mheap (u8 * s, va_list * va)
356 void *heap = va_arg (*va, u8 *);
357 int verbose = va_arg (*va, int);
358 struct dlmallinfo mi;
359 mheap_trace_main_t *tm = &mheap_trace_main;
361 mi = mspace_mallinfo (heap);
363 s = format (s, "total: %U, used: %U, free: %U, trimmable: %U",
364 format_msize, mi.arena,
365 format_msize, mi.uordblks,
366 format_msize, mi.fordblks, format_msize, mi.keepcost);
369 s = format (s, "\n free chunks %llu free fastbin blks %llu",
370 mi.ordblks, mi.smblks);
372 format (s, "\n max total allocated %U", format_msize, mi.usmblks);
375 s = format (s, "\n%U", format_mheap_trace, tm, verbose);
380 clib_mem_usage (clib_mem_usage_t * u)
382 clib_warning ("unimp");
385 /* Call serial number for debugger breakpoints. */
386 uword clib_mem_validate_serial = 0;
389 clib_mem_validate (void)
391 clib_warning ("unimp");
395 mheap_trace (void *v, int enable)
397 (void) mspace_enable_disable_trace (v, enable);
400 mheap_trace_main_free (&mheap_trace_main);
404 clib_mem_trace (int enable)
406 mheap_trace_main_t *tm = &mheap_trace_main;
408 tm->enabled = enable;
409 mheap_trace (clib_mem_get_heap (), enable);
413 clib_mem_trace_enable_disable (uword enable)
416 mheap_trace_main_t *tm = &mheap_trace_main;
419 tm->enabled = enable;
424 * These API functions seem like layering violations, but
425 * by introducing them we greatly reduce the number
426 * of code changes required to use dlmalloc spaces
429 mheap_alloc_with_lock (void *memory, uword size, int locked)
433 return create_mspace (size, locked);
436 rv = create_mspace_with_base (memory, size, locked);
438 mspace_disable_expand (rv);
444 * fd.io coding-style-patch-verification: ON
447 * eval: (c-set-style "gnu")