X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fmem.h;h=5492e106d916da86c8de6f2d82ee711059e9b090;hb=a690fdbfe179e0ea65818c03b52535bf9210efd0;hp=63c5ac16a026dbf222be303c4d3887341a22fc6f;hpb=f55f9b851f59264d737d92c6277a87588c565d24;p=vpp.git

diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index 63c5ac16a02..5492e106d91 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -39,17 +39,31 @@
 #define _included_clib_mem_h
 
 #include <stdarg.h>
+#include <unistd.h>
+#include <sys/mman.h>
 
 #include <vppinfra/clib.h>	/* uword, etc */
+#include <vppinfra/clib_error.h>
+
+#if USE_DLMALLOC == 0
 #include <vppinfra/mheap_bootstrap.h>
+#else
+#include <vppinfra/dlmalloc.h>
+#endif
+
 #include <vppinfra/os.h>
-#include <vppinfra/string.h>	/* memcpy, memset */
-#include <vppinfra/valgrind.h>
+#include <vppinfra/string.h>	/* memcpy, clib_memset */
+#include <vppinfra/sanitizer.h>
 
 #define CLIB_MAX_MHEAPS 256
+#define CLIB_MAX_NUMAS 8
+
+/* Unspecified NUMA socket */
+#define VEC_NUMA_UNSPECIFIED (0xFF)
 
 /* Per CPU heaps. */
 extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS];
+extern void *clib_per_numa_mheaps[CLIB_MAX_NUMAS];
 
 always_inline void *
 clib_mem_get_per_cpu_heap (void)
@@ -67,13 +81,61 @@ clib_mem_set_per_cpu_heap (u8 * new_heap)
   return old;
 }
 
+always_inline void *
+clib_mem_get_per_numa_heap (u32 numa_id)
+{
+  ASSERT (numa_id >= 0 && numa_id < ARRAY_LEN (clib_per_numa_mheaps));
+  return clib_per_numa_mheaps[numa_id];
+}
+
+always_inline void *
+clib_mem_set_per_numa_heap (u8 * new_heap)
+{
+  int numa = os_get_numa_index ();
+  void *old = clib_per_numa_mheaps[numa];
+  clib_per_numa_mheaps[numa] = new_heap;
+  return old;
+}
+
+always_inline void
+clib_mem_set_thread_index (void)
+{
+  /*
+   * Find an unused slot in the per-cpu-mheaps array,
+   * and grab it for this thread. We need to be able to
+   * push/pop the thread heap without affecting other thread(s).
+   */
+  int i;
+  if (__os_thread_index != 0)
+    return;
+  for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++)
+    if (clib_atomic_bool_cmp_and_swap (&clib_per_cpu_mheaps[i],
+				       0, clib_per_cpu_mheaps[0]))
+      {
+	os_set_thread_index (i);
+	break;
+      }
+  ASSERT (__os_thread_index > 0);
+}
+
+always_inline uword
+clib_mem_size_nocheck (void *p)
+{
+#if USE_DLMALLOC == 0
+  mheap_elt_t *e = mheap_user_pointer_to_elt (p);
+  return mheap_elt_data_bytes (e);
+#else
+  return mspace_usable_size_with_delta (p);
+#endif
+}
+
 /* Memory allocator which may call os_out_of_memory() if it fails */
 always_inline void *
 clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
 				  int os_out_of_memory_on_failure)
 {
   void *heap, *p;
-  uword offset, cpu;
+  uword cpu;
 
   if (align_offset > align)
     {
@@ -85,23 +147,26 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
 
   cpu = os_get_thread_index ();
   heap = clib_per_cpu_mheaps[cpu];
+
+#if USE_DLMALLOC == 0
+  uword offset;
   heap = mheap_get_aligned (heap, size, align, align_offset, &offset);
   clib_per_cpu_mheaps[cpu] = heap;
+  if (PREDICT_TRUE (offset != ~0))
+    p = heap + offset;
+#else
+  p = mspace_get_aligned (heap, size, align, align_offset);
+#endif /* USE_DLMALLOC */
 
-  if (offset != ~0)
-    {
-      p = heap + offset;
-#if CLIB_DEBUG > 0
-      VALGRIND_MALLOCLIKE_BLOCK (p, mheap_data_bytes (heap, offset), 0, 0);
-#endif
-      return p;
-    }
-  else
+  if (PREDICT_FALSE (0 == p))
     {
       if (os_out_of_memory_on_failure)
 	os_out_of_memory ();
       return 0;
     }
+
+  CLIB_MEM_UNPOISON (p, size);
+  return p;
 }
 
 /* Memory allocator which calls os_out_of_memory() when it fails */
@@ -158,6 +223,7 @@ clib_mem_alloc_aligned_or_null (uword size, uword align)
 always_inline uword
 clib_mem_is_heap_object (void *p)
 {
+#if USE_DLMALLOC == 0
   void *heap = clib_mem_get_per_cpu_heap ();
   uword offset = (uword) p - (uword) heap;
   mheap_elt_t *e, *n;
@@ -170,6 +236,11 @@ clib_mem_is_heap_object (void *p)
 
   /* Check that heap forward and reverse pointers agree. */
   return e->n_user_data == n->prev_n_user_data;
+#else
+  void *heap = clib_mem_get_per_cpu_heap ();
+
+  return mspace_is_heap_object (heap, p);
+#endif /* USE_DLMALLOC */
 }
 
 always_inline void
@@ -180,10 +251,12 @@ clib_mem_free (void *p)
   /* Make sure object is in the correct heap. */
   ASSERT (clib_mem_is_heap_object (p));
 
-  mheap_put (heap, (u8 *) p - heap);
+  CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
 
-#if CLIB_DEBUG > 0
-  VALGRIND_FREELIKE_BLOCK (p, 0);
+#if USE_DLMALLOC == 0
+  mheap_put (heap, (u8 *) p - heap);
+#else
+  mspace_put (heap, p);
 #endif
 }
 
@@ -199,7 +272,7 @@ clib_mem_realloc (void *p, uword new_size, uword old_size)
 	copy_size = old_size;
       else
 	copy_size = new_size;
-      clib_memcpy (q, p, copy_size);
+      clib_memcpy_fast (q, p, copy_size);
       clib_mem_free (p);
     }
   return q;
@@ -209,8 +282,16 @@ always_inline uword
 clib_mem_size (void *p)
 {
   ASSERT (clib_mem_is_heap_object (p));
-  mheap_elt_t *e = mheap_user_pointer_to_elt (p);
-  return mheap_elt_data_bytes (e);
+  return clib_mem_size_nocheck (p);
+}
+
+always_inline void
+clib_mem_free_s (void *p)
+{
+  uword size = clib_mem_size (p);
+  CLIB_MEM_UNPOISON (p, size);
+  memset_s_inline (p, size, 0, size);
+  clib_mem_free (p);
 }
 
 always_inline void *
@@ -226,6 +307,8 @@ clib_mem_set_heap (void *heap)
 }
 
 void *clib_mem_init (void *heap, uword size);
+void *clib_mem_init_thread_safe (void *memory, uword memory_size);
+void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size);
 
 void clib_mem_exit (void);
 
@@ -235,6 +318,8 @@ void clib_mem_validate (void);
 
 void clib_mem_trace (int enable);
 
+int clib_mem_is_traced (void);
+
 typedef struct
 {
   /* Total number of objects allocated. */
@@ -264,19 +349,111 @@ void clib_mem_usage (clib_mem_usage_t * usage);
 
 u8 *format_clib_mem_usage (u8 * s, va_list * args);
 
-/* Include appropriate VM functions depending on whether
-   we are compiling for linux kernel, for Unix or standalone. */
-#ifdef CLIB_LINUX_KERNEL
-#include <vppinfra/vm_linux_kernel.h>
-#endif
+/* Allocate virtual address space. */
+always_inline void *
+clib_mem_vm_alloc (uword size)
+{
+  void *mmap_addr;
+  uword flags = MAP_PRIVATE;
 
-#ifdef CLIB_UNIX
-#include <vppinfra/vm_unix.h>
+#ifdef MAP_ANONYMOUS
+  flags |= MAP_ANONYMOUS;
 #endif
 
-#ifdef CLIB_STANDALONE
-#include <vppinfra/vm_standalone.h>
-#endif
+  mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0);
+  if (mmap_addr == (void *) -1)
+    mmap_addr = 0;
+
+  return mmap_addr;
+}
+
+always_inline void
+clib_mem_vm_free (void *addr, uword size)
+{
+  munmap (addr, size);
+}
+
+always_inline void *
+clib_mem_vm_unmap (void *addr, uword size)
+{
+  void *mmap_addr;
+  uword flags = MAP_PRIVATE | MAP_FIXED;
+
+  /* To unmap we "map" with no protection.  If we actually called
+     munmap then other callers could steal the address space.  By
+     changing to PROT_NONE the kernel can free up the pages which is
+     really what we want "unmap" to mean. */
+  mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0);
+  if (mmap_addr == (void *) -1)
+    mmap_addr = 0;
+
+  return mmap_addr;
+}
+
+always_inline void *
+clib_mem_vm_map (void *addr, uword size)
+{
+  void *mmap_addr;
+  uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
+
+  mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0);
+  if (mmap_addr == (void *) -1)
+    mmap_addr = 0;
+
+  return mmap_addr;
+}
+
+typedef struct
+{
+#define CLIB_MEM_VM_F_SHARED (1 << 0)
+#define CLIB_MEM_VM_F_HUGETLB (1 << 1)
+#define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2)
+#define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3)
+#define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4)
+#define CLIB_MEM_VM_F_LOCKED (1 << 5)
+  u32 flags; /**< vm allocation flags:
+                <br> CLIB_MEM_VM_F_SHARED: request shared memory, file
+		descriptor will be provided on successful allocation.
+                <br> CLIB_MEM_VM_F_HUGETLB: request hugepages.
+		<br> CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid
+		numa node preference.
+		<br> CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails.
+		<br> CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if
+		number of available pages is not sufficient.
+		<br> CLIB_MEM_VM_F_LOCKED: request locked memory.
+             */
+  char *name; /**< Name for memory allocation, set by caller. */
+  uword size; /**< Allocation size, set by caller. */
+  int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */
+  void *addr; /**< Pointer to allocated memory, set on successful allocation. */
+  int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */
+  int log2_page_size;		/* Page size in log2 format, set on successful allocation. */
+  int n_pages;			/* Number of pages. */
+  uword requested_va;		/**< Request fixed position mapping */
+} clib_mem_vm_alloc_t;
+
+clib_error_t *clib_mem_create_fd (char *name, int *fdp);
+clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp);
+clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a);
+void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a);
+u64 clib_mem_get_fd_page_size (int fd);
+uword clib_mem_get_default_hugepage_size (void);
+int clib_mem_get_fd_log2_page_size (int fd);
+u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages);
+
+typedef struct
+{
+  uword size;		/**< Map size */
+  int fd;		/**< File descriptor to be mapped */
+  uword requested_va;	/**< Request fixed position mapping */
+  void *addr;		/**< Pointer to mapped memory, if successful */
+} clib_mem_vm_map_t;
+
+clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a);
+void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size);
+void mheap_trace (void *v, int enable);
+uword clib_mem_trace_enable_disable (uword enable);
+void clib_mem_trace (int enable);
 
 #include <vppinfra/error.h>	/* clib_panic */