2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * physmem.c: Unix physical memory
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
41 #include <sys/types.h>
42 #include <sys/mount.h>
44 #include <sys/fcntl.h>
48 #include <vppinfra/linux/syscall.h>
49 #include <vppinfra/linux/sysfs.h>
50 #include <vlib/vlib.h>
51 #include <vlib/physmem.h>
52 #include <vlib/unix/unix.h>
53 #include <vlib/pci/pci.h>
54 #include <vlib/linux/vfio.h>
57 unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx,
58 uword n_bytes, uword alignment)
60 vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx);
61 uword lo_offset, hi_offset;
67 /* IO memory is always at least cache aligned. */
68 alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
72 mheap_get_aligned (pr->heap, n_bytes,
73 /* align */ alignment,
77 /* Allocation failed? */
81 /* Make sure allocation does not span DMA physical chunk boundary. */
82 hi_offset = lo_offset + n_bytes - 1;
84 if (((pointer_to_uword (pr->heap) + lo_offset) >> pr->log2_page_size) ==
85 ((pointer_to_uword (pr->heap) + hi_offset) >> pr->log2_page_size))
88 /* Allocation would span chunk boundary, queue it to be freed as soon as
89 we find suitable chunk. */
90 vec_add1 (to_free, lo_offset);
96 for (i = 0; i < vec_len (to_free); i++)
97 mheap_put (pr->heap, to_free[i]);
101 return lo_offset != ~0 ? pr->heap + lo_offset : 0;
105 unix_physmem_free (vlib_main_t * vm, vlib_physmem_region_index_t idx, void *x)
107 vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx);
108 /* Return object to region's heap. */
109 mheap_put (pr->heap, x - pr->heap);
112 static clib_error_t *
113 unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size,
114 u8 numa_node, u32 flags,
115 vlib_physmem_region_index_t * idx)
117 vlib_physmem_main_t *vpm = &physmem_main;
118 vlib_physmem_region_t *pr;
119 clib_error_t *error = 0;
120 clib_mem_vm_alloc_t alloc = { 0 };
123 pool_get (vpm->regions, pr);
125 if ((pr - vpm->regions) >= 256)
127 error = clib_error_return (0, "maximum number of regions reached");
133 alloc.numa_node = numa_node;
135 alloc.flags = (flags & VLIB_PHYSMEM_F_SHARED) ?
136 CLIB_MEM_VM_F_SHARED : CLIB_MEM_VM_F_LOCKED;
138 if ((flags & VLIB_PHYSMEM_F_HUGETLB))
140 alloc.flags |= CLIB_MEM_VM_F_HUGETLB;
141 alloc.flags |= CLIB_MEM_VM_F_HUGETLB_PREALLOC;
142 alloc.flags |= CLIB_MEM_VM_F_NUMA_FORCE;
146 alloc.flags |= CLIB_MEM_VM_F_NUMA_PREFER;
149 error = clib_mem_vm_ext_alloc (&alloc);
153 pr->index = pr - vpm->regions;
156 pr->mem = alloc.addr;
157 pr->log2_page_size = alloc.log2_page_size;
158 pr->n_pages = alloc.n_pages;
159 pr->size = (u64) pr->n_pages << (u64) pr->log2_page_size;
160 pr->page_mask = (1 << pr->log2_page_size) - 1;
161 pr->numa_node = numa_node;
162 pr->name = format (0, "%s", name);
164 for (i = 0; i < pr->n_pages; i++)
166 void *ptr = pr->mem + ((u64) i << pr->log2_page_size);
168 if ((move_pages (0, 1, &ptr, 0, &node, 0) == 0) && (numa_node != node))
170 clib_warning ("physmem page for region \'%s\' allocated on the"
171 " wrong numa node (requested %u actual %u)",
172 pr->name, pr->numa_node, node, i);
177 pr->page_table = clib_mem_vm_get_paddr (pr->mem, pr->log2_page_size,
180 linux_vfio_dma_map_regions (vm);
182 if (flags & VLIB_PHYSMEM_F_INIT_MHEAP)
184 pr->heap = mheap_alloc_with_flags (pr->mem, pr->size,
185 /* Don't want mheap mmap/munmap with IO memory. */
186 MHEAP_FLAG_DISABLE_VM |
187 MHEAP_FLAG_THREAD_SAFE);
195 memset (pr, 0, sizeof (*pr));
196 pool_put (vpm->regions, pr);
203 unix_physmem_region_free (vlib_main_t * vm, vlib_physmem_region_index_t idx)
205 vlib_physmem_main_t *vpm = &physmem_main;
206 vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx);
210 munmap (pr->mem, pr->size);
212 pool_put (vpm->regions, pr);
216 unix_physmem_init (vlib_main_t * vm)
218 vlib_physmem_main_t *vpm = &physmem_main;
219 clib_error_t *error = 0;
222 /* Avoid multiple calls. */
223 if (vm->os_physmem_alloc_aligned)
226 /* check if pagemap is accessible */
227 pt = clib_mem_vm_get_paddr (&pt, min_log2 (sysconf (_SC_PAGESIZE)), 1);
229 vpm->flags |= VLIB_PHYSMEM_MAIN_F_HAVE_PAGEMAP;
232 if ((error = linux_vfio_init (vm)))
235 vm->os_physmem_alloc_aligned = unix_physmem_alloc_aligned;
236 vm->os_physmem_free = unix_physmem_free;
237 vm->os_physmem_region_alloc = unix_physmem_region_alloc;
238 vm->os_physmem_region_free = unix_physmem_region_free;
243 static clib_error_t *
244 show_physmem (vlib_main_t * vm,
245 unformat_input_t * input, vlib_cli_command_t * cmd)
247 vlib_physmem_main_t *vpm = &physmem_main;
248 vlib_physmem_region_t *pr;
251 pool_foreach (pr, vpm->regions, (
253 vlib_cli_output (vm, "index %u name '%s' page-size %uKB num-pages %d "
254 "numa-node %u fd %d\n",
255 pr->index, pr->name, (1 << (pr->log2_page_size -10)),
256 pr->n_pages, pr->numa_node, pr->fd);
258 vlib_cli_output (vm, " %U", format_mheap, pr->heap, /* verbose */ 1);
260 vlib_cli_output (vm, " no heap\n");
267 VLIB_CLI_COMMAND (show_physmem_command, static) = {
268 .path = "show physmem",
269 .short_help = "Show physical memory allocation",
270 .function = show_physmem,
275 * fd.io coding-style-patch-verification: ON
278 * eval: (c-set-style "gnu")