X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Flinux%2Fphysmem.c;h=4d16486dea1f27151afb24305ce6d95fe601d594;hb=13637632b87938a055618f17ed21b2a54b02459d;hp=b5f84ecc8a4e86173eb5db97c75869eb6a77bc6b;hpb=2ca8ced6445d627f6768d1a69b6b871e3dcb5281;p=vpp.git diff --git a/src/vlib/linux/physmem.c b/src/vlib/linux/physmem.c old mode 100644 new mode 100755 index b5f84ecc8a4..4d16486dea1 --- a/src/vlib/linux/physmem.c +++ b/src/vlib/linux/physmem.c @@ -43,7 +43,6 @@ #include #include #include -#include #include #include @@ -51,8 +50,8 @@ #include #include #include - -static int vfio_container_fd = -1; +#include +#include static void * unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx, @@ -70,23 +69,28 @@ unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx, while (1) { +#if USE_DLMALLOC == 0 + mheap_get_aligned (pr->heap, n_bytes, /* align */ alignment, /* align offset */ 0, &lo_offset); +#else + lo_offset = (uword) mspace_get_aligned (pr->heap, n_bytes, + alignment, ~0ULL /* offset */ ); + if (lo_offset == 0) + lo_offset = ~0ULL; +#endif /* Allocation failed? */ if (lo_offset == ~0) break; - if (pr->flags & VLIB_PHYSMEM_F_FAKE) - break; - /* Make sure allocation does not span DMA physical chunk boundary. */ hi_offset = lo_offset + n_bytes - 1; - if ((lo_offset >> pr->log2_page_size) == - (hi_offset >> pr->log2_page_size)) + if (((pointer_to_uword (pr->heap) + lo_offset) >> pr->log2_page_size) == + ((pointer_to_uword (pr->heap) + hi_offset) >> pr->log2_page_size)) break; /* Allocation would span chunk boundary, queue it to be freed as soon as @@ -98,11 +102,21 @@ unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx, { uword i; for (i = 0; i < vec_len (to_free); i++) - mheap_put (pr->heap, to_free[i]); + { +#if USE_DLMALLOC == 0 + mheap_put (pr->heap, to_free[i]); +#else + mspace_put_no_offset (pr->heap, (void *) to_free[i]); +#endif + } vec_free (to_free); } - return lo_offset != ~0 ? pr->heap + lo_offset : 0; +#if USE_DLMALLOC == 0 + return lo_offset != ~0 ? (void *) (pr->heap + lo_offset) : 0; +#else + return lo_offset != ~0 ? (void *) lo_offset : 0; +#endif } static void @@ -110,57 +124,11 @@ unix_physmem_free (vlib_main_t * vm, vlib_physmem_region_index_t idx, void *x) { vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); /* Return object to region's heap. */ +#if USE_DLMALLOC == 0 mheap_put (pr->heap, x - pr->heap); -} - -static clib_error_t * -scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name) -{ - const char fn[] = "/dev/vfio/vfio"; - char buff[sizeof (fn)] = { 0 }; - - if (readlink ((char *) path_name, buff, sizeof (fn)) + 1 != sizeof (fn)) - return 0; - - if (strncmp (fn, buff, sizeof (fn))) - return 0; - - vfio_container_fd = atoi ((char *) file_name); - return 0; -} - -static clib_error_t * -unix_physmem_region_iommu_register (vlib_physmem_region_t * pr) -{ - struct vfio_iommu_type1_dma_map dma_map = { 0 }; - int i, fd; - - if (vfio_container_fd == -1) - foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0); - - fd = vfio_container_fd; - - if (fd < 0) - return 0; - - if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION) - return 0; - - if (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 0) - return 0; - - dma_map.argsz = sizeof (struct vfio_iommu_type1_dma_map); - dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; - - vec_foreach_index (i, pr->page_table) - { - dma_map.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size); - dma_map.size = 1 << pr->log2_page_size; - dma_map.iova = pr->page_table[i]; - if (ioctl (fd, VFIO_IOMMU_MAP_DMA, &dma_map) != 0) - return clib_error_return_unix (0, "ioctl (VFIO_IOMMU_MAP_DMA)"); - } - return 0; +#else + mspace_put_no_offset (pr->heap, x); +#endif } static clib_error_t * @@ -168,14 +136,11 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t * idx) { - vlib_physmem_main_t *vpm = &vm->physmem_main; + vlib_physmem_main_t *vpm = &physmem_main; vlib_physmem_region_t *pr; clib_error_t *error = 0; clib_mem_vm_alloc_t alloc = { 0 }; - - - if (geteuid () != 0 && (flags & VLIB_PHYSMEM_F_FAKE) == 0) - return clib_error_return (0, "not allowed"); + int i; pool_get (vpm->regions, pr); @@ -188,9 +153,11 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, alloc.name = name; alloc.size = size; alloc.numa_node = numa_node; - alloc.flags = CLIB_MEM_VM_F_SHARED; - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) + alloc.flags = (flags & VLIB_PHYSMEM_F_SHARED) ? + CLIB_MEM_VM_F_SHARED : CLIB_MEM_VM_F_LOCKED; + + if ((flags & VLIB_PHYSMEM_F_HUGETLB)) { alloc.flags |= CLIB_MEM_VM_F_HUGETLB; alloc.flags |= CLIB_MEM_VM_F_HUGETLB_PREALLOC; @@ -214,37 +181,37 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, pr->size = (u64) pr->n_pages << (u64) pr->log2_page_size; pr->page_mask = (1 << pr->log2_page_size) - 1; pr->numa_node = numa_node; - pr->name = format (0, "%s", name); + pr->name = format (0, "%s%c", name, 0); - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) + for (i = 0; i < pr->n_pages; i++) { - int i; - for (i = 0; i < pr->n_pages; i++) + void *ptr = pr->mem + ((u64) i << pr->log2_page_size); + int node; + if ((move_pages (0, 1, &ptr, 0, &node, 0) == 0) && (numa_node != node)) { - void *ptr = pr->mem + (i << pr->log2_page_size); - int node; - move_pages (0, 1, &ptr, 0, &node, 0); - if (numa_node != node) - { - clib_warning ("physmem page for region \'%s\' allocated on the" - " wrong numa node (requested %u actual %u)", - pr->name, pr->numa_node, node, i); - break; - } + clib_warning ("physmem page for region \'%s\' allocated on the" + " wrong numa node (requested %u actual %u)", + pr->name, pr->numa_node, node, i); + break; } - pr->page_table = clib_mem_vm_get_paddr (pr->mem, pr->log2_page_size, - pr->n_pages); - error = unix_physmem_region_iommu_register (pr); - if (error) - clib_error_report (error); } + pr->page_table = clib_mem_vm_get_paddr (pr->mem, pr->log2_page_size, + pr->n_pages); + + linux_vfio_dma_map_regions (vm); + if (flags & VLIB_PHYSMEM_F_INIT_MHEAP) { +#if USE_DLMALLOC == 0 pr->heap = mheap_alloc_with_flags (pr->mem, pr->size, /* Don't want mheap mmap/munmap with IO memory. */ MHEAP_FLAG_DISABLE_VM | MHEAP_FLAG_THREAD_SAFE); +#else + pr->heap = create_mspace_with_base (pr->mem, pr->size, 1 /* locked */ ); + mspace_disable_expand (pr->heap); +#endif } *idx = pr->index; @@ -262,7 +229,7 @@ done: static void unix_physmem_region_free (vlib_main_t * vm, vlib_physmem_region_index_t idx) { - vlib_physmem_main_t *vpm = &vm->physmem_main; + vlib_physmem_main_t *vpm = &physmem_main; vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); if (pr->fd > 0) @@ -275,12 +242,23 @@ unix_physmem_region_free (vlib_main_t * vm, vlib_physmem_region_index_t idx) clib_error_t * unix_physmem_init (vlib_main_t * vm) { + vlib_physmem_main_t *vpm = &physmem_main; clib_error_t *error = 0; + u64 *pt = 0; /* Avoid multiple calls. */ if (vm->os_physmem_alloc_aligned) return error; + /* check if pagemap is accessible */ + pt = clib_mem_vm_get_paddr (&pt, min_log2 (sysconf (_SC_PAGESIZE)), 1); + if (pt[0]) + vpm->flags |= VLIB_PHYSMEM_MAIN_F_HAVE_PAGEMAP; + vec_free (pt); + + if ((error = linux_vfio_init (vm))) + return error; + vm->os_physmem_alloc_aligned = unix_physmem_alloc_aligned; vm->os_physmem_free = unix_physmem_free; vm->os_physmem_region_alloc = unix_physmem_region_alloc; @@ -293,7 +271,7 @@ static clib_error_t * show_physmem (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - vlib_physmem_main_t *vpm = &vm->physmem_main; + vlib_physmem_main_t *vpm = &physmem_main; vlib_physmem_region_t *pr; /* *INDENT-OFF* */