2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * physmem.c: Unix physical memory
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
41 #include <sys/types.h>
42 #include <sys/mount.h>
44 #include <sys/fcntl.h>
46 #include <linux/vfio.h>
49 #include <vppinfra/linux/syscall.h>
50 #include <vppinfra/linux/sysfs.h>
51 #include <vlib/vlib.h>
52 #include <vlib/physmem.h>
53 #include <vlib/unix/unix.h>
55 static int vfio_container_fd = -1;
58 unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx,
59 uword n_bytes, uword alignment)
61 vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx);
62 uword lo_offset, hi_offset;
68 /* IO memory is always at least cache aligned. */
69 alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
73 mheap_get_aligned (pr->heap, n_bytes,
74 /* align */ alignment,
78 /* Allocation failed? */
82 if (pr->flags & VLIB_PHYSMEM_F_FAKE)
85 /* Make sure allocation does not span DMA physical chunk boundary. */
86 hi_offset = lo_offset + n_bytes - 1;
88 if ((lo_offset >> pr->log2_page_size) ==
89 (hi_offset >> pr->log2_page_size))
92 /* Allocation would span chunk boundary, queue it to be freed as soon as
93 we find suitable chunk. */
94 vec_add1 (to_free, lo_offset);
100 for (i = 0; i < vec_len (to_free); i++)
101 mheap_put (pr->heap, to_free[i]);
105 return lo_offset != ~0 ? pr->heap + lo_offset : 0;
109 unix_physmem_free (vlib_main_t * vm, vlib_physmem_region_index_t idx, void *x)
111 vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx);
112 /* Return object to region's heap. */
113 mheap_put (pr->heap, x - pr->heap);
116 static clib_error_t *
117 scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name)
119 const char fn[] = "/dev/vfio/vfio";
120 char buff[sizeof (fn)] = { 0 };
122 if (readlink ((char *) path_name, buff, sizeof (fn)) + 1 != sizeof (fn))
125 if (strncmp (fn, buff, sizeof (fn)))
128 vfio_container_fd = atoi ((char *) file_name);
132 static clib_error_t *
133 unix_physmem_region_iommu_register (vlib_physmem_region_t * pr)
135 struct vfio_iommu_type1_dma_map dma_map = { 0 };
138 if (vfio_container_fd == -1)
139 foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0);
141 fd = vfio_container_fd;
146 if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION)
149 if (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 0)
152 dma_map.argsz = sizeof (struct vfio_iommu_type1_dma_map);
153 dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
155 vec_foreach_index (i, pr->page_table)
157 dma_map.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size);
158 dma_map.size = 1 << pr->log2_page_size;
159 dma_map.iova = pr->page_table[i];
160 if (ioctl (fd, VFIO_IOMMU_MAP_DMA, &dma_map) != 0)
161 return clib_error_return_unix (0, "ioctl (VFIO_IOMMU_MAP_DMA)");
166 static clib_error_t *
167 unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size,
168 u8 numa_node, u32 flags,
169 vlib_physmem_region_index_t * idx)
171 vlib_physmem_main_t *vpm = &vm->physmem_main;
172 vlib_physmem_region_t *pr;
173 clib_error_t *error = 0;
174 clib_mem_vm_alloc_t alloc = { 0 };
177 if (geteuid () != 0 && (flags & VLIB_PHYSMEM_F_FAKE) == 0)
178 return clib_error_return (0, "not allowed");
180 pool_get (vpm->regions, pr);
182 if ((pr - vpm->regions) >= 256)
184 error = clib_error_return (0, "maximum number of regions reached");
190 alloc.numa_node = numa_node;
191 alloc.flags = CLIB_MEM_VM_F_SHARED;
193 if ((flags & VLIB_PHYSMEM_F_FAKE) == 0)
195 alloc.flags |= CLIB_MEM_VM_F_HUGETLB;
196 alloc.flags |= CLIB_MEM_VM_F_HUGETLB_PREALLOC;
197 alloc.flags |= CLIB_MEM_VM_F_NUMA_FORCE;
201 alloc.flags |= CLIB_MEM_VM_F_NUMA_PREFER;
204 error = clib_mem_vm_ext_alloc (&alloc);
208 pr->index = pr - vpm->regions;
211 pr->mem = alloc.addr;
212 pr->log2_page_size = alloc.log2_page_size;
213 pr->n_pages = alloc.n_pages;
214 pr->size = (u64) pr->n_pages << (u64) pr->log2_page_size;
215 pr->page_mask = (1 << pr->log2_page_size) - 1;
216 pr->numa_node = numa_node;
217 pr->name = format (0, "%s", name);
219 if ((flags & VLIB_PHYSMEM_F_FAKE) == 0)
222 for (i = 0; i < pr->n_pages; i++)
224 void *ptr = pr->mem + (i << pr->log2_page_size);
226 move_pages (0, 1, &ptr, 0, &node, 0);
227 if (numa_node != node)
229 clib_warning ("physmem page for region \'%s\' allocated on the"
230 " wrong numa node (requested %u actual %u)",
231 pr->name, pr->numa_node, node, i);
235 pr->page_table = clib_mem_vm_get_paddr (pr->mem, pr->log2_page_size,
237 error = unix_physmem_region_iommu_register (pr);
239 clib_error_report (error);
242 if (flags & VLIB_PHYSMEM_F_INIT_MHEAP)
244 pr->heap = mheap_alloc_with_flags (pr->mem, pr->size,
245 /* Don't want mheap mmap/munmap with IO memory. */
246 MHEAP_FLAG_DISABLE_VM |
247 MHEAP_FLAG_THREAD_SAFE);
255 memset (pr, 0, sizeof (*pr));
256 pool_put (vpm->regions, pr);
263 unix_physmem_region_free (vlib_main_t * vm, vlib_physmem_region_index_t idx)
265 vlib_physmem_main_t *vpm = &vm->physmem_main;
266 vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx);
270 munmap (pr->mem, pr->size);
272 pool_put (vpm->regions, pr);
276 unix_physmem_init (vlib_main_t * vm)
278 clib_error_t *error = 0;
280 /* Avoid multiple calls. */
281 if (vm->os_physmem_alloc_aligned)
284 vm->os_physmem_alloc_aligned = unix_physmem_alloc_aligned;
285 vm->os_physmem_free = unix_physmem_free;
286 vm->os_physmem_region_alloc = unix_physmem_region_alloc;
287 vm->os_physmem_region_free = unix_physmem_region_free;
292 static clib_error_t *
293 show_physmem (vlib_main_t * vm,
294 unformat_input_t * input, vlib_cli_command_t * cmd)
296 vlib_physmem_main_t *vpm = &vm->physmem_main;
297 vlib_physmem_region_t *pr;
300 pool_foreach (pr, vpm->regions, (
302 vlib_cli_output (vm, "index %u name '%s' page-size %uKB num-pages %d "
303 "numa-node %u fd %d\n",
304 pr->index, pr->name, (1 << (pr->log2_page_size -10)),
305 pr->n_pages, pr->numa_node, pr->fd);
307 vlib_cli_output (vm, " %U", format_mheap, pr->heap, /* verbose */ 1);
309 vlib_cli_output (vm, " no heap\n");
316 VLIB_CLI_COMMAND (show_physmem_command, static) = {
317 .path = "show physmem",
318 .short_help = "Show physical memory allocation",
319 .function = show_physmem,
324 * fd.io coding-style-patch-verification: ON
327 * eval: (c-set-style "gnu")