X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fmem.h;h=4fedd1077544d75f9c3926cfd6673889c7fecac9;hb=2c8e0023f91882e53f06eb99c901b97fe013f981;hp=14b2761c881ca7fe0eaf78830e1c70f608d44f78;hpb=30cca512ce3818a9b2c56140f7859bd544bc825d;p=vpp.git diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 14b2761c881..4fedd107754 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -45,19 +45,53 @@ #include /* uword, etc */ #include -#if USE_DLMALLOC == 0 -#include -#else #include -#endif #include #include /* memcpy, clib_memset */ +#include #define CLIB_MAX_MHEAPS 256 +#define CLIB_MAX_NUMAS 8 + +/* Unspecified NUMA socket */ +#define VEC_NUMA_UNSPECIFIED (0xFF) /* Per CPU heaps. */ extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; +extern void *clib_per_numa_mheaps[CLIB_MAX_NUMAS]; + +always_inline void * +clib_mem_get_per_cpu_heap (void) +{ + int cpu = os_get_thread_index (); + return clib_per_cpu_mheaps[cpu]; +} + +always_inline void * +clib_mem_set_per_cpu_heap (u8 * new_heap) +{ + int cpu = os_get_thread_index (); + void *old = clib_per_cpu_mheaps[cpu]; + clib_per_cpu_mheaps[cpu] = new_heap; + return old; +} + +always_inline void * +clib_mem_get_per_numa_heap (u32 numa_id) +{ + ASSERT (numa_id < ARRAY_LEN (clib_per_numa_mheaps)); + return clib_per_numa_mheaps[numa_id]; +} + +always_inline void * +clib_mem_set_per_numa_heap (u8 * new_heap) +{ + int numa = os_get_numa_index (); + void *old = clib_per_numa_mheaps[numa]; + clib_per_numa_mheaps[numa] = new_heap; + return old; +} always_inline void clib_mem_set_thread_index (void) @@ -80,20 +114,10 @@ clib_mem_set_thread_index (void) ASSERT (__os_thread_index > 0); } -always_inline void * -clib_mem_get_per_cpu_heap (void) -{ - int cpu = os_get_thread_index (); - return clib_per_cpu_mheaps[cpu]; -} - -always_inline void * -clib_mem_set_per_cpu_heap (u8 * new_heap) +always_inline uword +clib_mem_size_nocheck (void *p) { - int cpu = os_get_thread_index (); - void *old = clib_per_cpu_mheaps[cpu]; - clib_per_cpu_mheaps[cpu] = new_heap; - return old; + return mspace_usable_size_with_delta (p); } /* Memory allocator which may call os_out_of_memory() if it fails */ @@ -115,33 +139,17 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, cpu = os_get_thread_index (); heap = clib_per_cpu_mheaps[cpu]; -#if USE_DLMALLOC == 0 - uword offset; - heap = mheap_get_aligned (heap, size, align, align_offset, &offset); - clib_per_cpu_mheaps[cpu] = heap; - - if (offset != ~0) - { - p = heap + offset; - return p; - } - else - { - if (os_out_of_memory_on_failure) - os_out_of_memory (); - return 0; - } -#else p = mspace_get_aligned (heap, size, align, align_offset); - if (PREDICT_FALSE (p == 0)) + + if (PREDICT_FALSE (0 == p)) { if (os_out_of_memory_on_failure) os_out_of_memory (); return 0; } + CLIB_MEM_UNPOISON (p, size); return p; -#endif /* USE_DLMALLOC */ } /* Memory allocator which calls os_out_of_memory() when it fails */ @@ -198,24 +206,9 @@ clib_mem_alloc_aligned_or_null (uword size, uword align) always_inline uword clib_mem_is_heap_object (void *p) { -#if USE_DLMALLOC == 0 - void *heap = clib_mem_get_per_cpu_heap (); - uword offset = (uword) p - (uword) heap; - mheap_elt_t *e, *n; - - if (offset >= vec_len (heap)) - return 0; - - e = mheap_elt_at_uoffset (heap, offset); - n = mheap_next_elt (e); - - /* Check that heap forward and reverse pointers agree. */ - return e->n_user_data == n->prev_n_user_data; -#else void *heap = clib_mem_get_per_cpu_heap (); return mspace_is_heap_object (heap, p); -#endif /* USE_DLMALLOC */ } always_inline void @@ -226,11 +219,9 @@ clib_mem_free (void *p) /* Make sure object is in the correct heap. */ ASSERT (clib_mem_is_heap_object (p)); -#if USE_DLMALLOC == 0 - mheap_put (heap, (u8 *) p - heap); -#else + CLIB_MEM_POISON (p, clib_mem_size_nocheck (p)); + mspace_put (heap, p); -#endif } always_inline void * @@ -254,20 +245,15 @@ clib_mem_realloc (void *p, uword new_size, uword old_size) always_inline uword clib_mem_size (void *p) { -#if USE_DLMALLOC == 0 - mheap_elt_t *e = mheap_user_pointer_to_elt (p); ASSERT (clib_mem_is_heap_object (p)); - return mheap_elt_data_bytes (e); -#else - ASSERT (clib_mem_is_heap_object (p)); - return mspace_usable_size_with_delta (p); -#endif + return clib_mem_size_nocheck (p); } always_inline void clib_mem_free_s (void *p) { uword size = clib_mem_size (p); + CLIB_MEM_UNPOISON (p, size); memset_s_inline (p, size, 0, size); clib_mem_free (p); } @@ -286,6 +272,8 @@ clib_mem_set_heap (void *heap) void *clib_mem_init (void *heap, uword size); void *clib_mem_init_thread_safe (void *memory, uword memory_size); +void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size, + u8 numa); void clib_mem_exit (void); @@ -340,6 +328,8 @@ clib_mem_vm_alloc (uword size) mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0); if (mmap_addr == (void *) -1) mmap_addr = 0; + else + CLIB_MEM_UNPOISON (mmap_addr, size); return mmap_addr; } @@ -348,6 +338,7 @@ always_inline void clib_mem_vm_free (void *addr, uword size) { munmap (addr, size); + CLIB_MEM_POISON (addr, size); } always_inline void * @@ -363,6 +354,8 @@ clib_mem_vm_unmap (void *addr, uword size) mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0); if (mmap_addr == (void *) -1) mmap_addr = 0; + else + CLIB_MEM_UNPOISON (mmap_addr, size); return mmap_addr; } @@ -376,6 +369,8 @@ clib_mem_vm_map (void *addr, uword size) mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0); if (mmap_addr == (void *) -1) mmap_addr = 0; + else + CLIB_MEM_UNPOISON (mmap_addr, size); return mmap_addr; } @@ -424,6 +419,7 @@ typedef struct int fd; /**< File descriptor to be mapped */ uword requested_va; /**< Request fixed position mapping */ void *addr; /**< Pointer to mapped memory, if successful */ + u8 numa_node; } clib_mem_vm_map_t; clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a);