2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 #include <sys/types.h>
20 #include <linux/vfio.h>
21 #include <sys/ioctl.h>
23 #include <vppinfra/linux/sysfs.h>
25 #include <vlib/vlib.h>
26 #include <vlib/unix/unix.h>
27 #include <vlib/pci/pci.h>
28 #include <vlib/linux/vfio.h>
29 #include <vlib/physmem.h>
31 #ifndef VFIO_NOIOMMU_IOMMU
32 #define VFIO_NOIOMMU_IOMMU 8
35 linux_vfio_main_t vfio_main;
38 vfio_map_physmem_page (vlib_main_t * vm, void *addr)
40 vlib_physmem_main_t *vpm = &vm->physmem_main;
41 linux_vfio_main_t *lvm = &vfio_main;
42 struct vfio_iommu_type1_dma_map dm = { 0 };
43 uword log2_page_size = vpm->pmalloc_main->def_log2_page_sz;
44 uword physmem_start = pointer_to_uword (vpm->pmalloc_main->base);
46 if (lvm->container_fd == -1)
47 return clib_error_return (0, "No cointainer fd");
49 u32 page_index = vlib_physmem_get_page_index (vm, addr);
51 if (clib_bitmap_get (lvm->physmem_pages_mapped, page_index))
53 vlib_log_debug (lvm->log_default, "map DMA va:%p page:%u already "
54 "mapped", addr, page_index);
58 dm.argsz = sizeof (struct vfio_iommu_type1_dma_map);
59 dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
60 dm.vaddr = physmem_start + (page_index << log2_page_size);
61 dm.size = 1ULL << log2_page_size;
63 vlib_log_debug (lvm->log_default, "map DMA page:%u va:0x%lx iova:%lx "
64 "size:0x%lx", page_index, dm.vaddr, dm.iova, dm.size);
66 if (ioctl (lvm->container_fd, VFIO_IOMMU_MAP_DMA, &dm) == -1)
68 vlib_log_err (lvm->log_default, "map DMA page:%u va:0x%lx iova:%lx "
69 "size:0x%lx failed, error %s (errno %d)", page_index,
70 dm.vaddr, dm.iova, dm.size, strerror (errno), errno);
71 return clib_error_return_unix (0, "physmem DMA map failed");
74 lvm->physmem_pages_mapped = clib_bitmap_set (lvm->physmem_pages_mapped,
79 static linux_pci_vfio_iommu_group_t *
80 get_vfio_iommu_group (int group)
82 linux_vfio_main_t *lvm = &vfio_main;
85 p = hash_get (lvm->iommu_pool_index_by_group, group);
87 return p ? pool_elt_at_index (lvm->iommu_groups, p[0]) : 0;
91 open_vfio_iommu_group (int group, int is_noiommu)
93 linux_vfio_main_t *lvm = &vfio_main;
94 linux_pci_vfio_iommu_group_t *g;
95 clib_error_t *err = 0;
96 struct vfio_group_status group_status;
100 if (lvm->container_fd == -1)
102 if ((fd = open ("/dev/vfio/vfio", O_RDWR)) == -1)
103 return clib_error_return_unix (0, "failed to open VFIO container");
105 if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION)
108 return clib_error_return_unix (0, "incompatible VFIO version");
111 lvm->iommu_pool_index_by_group = hash_create (0, sizeof (uword));
112 lvm->container_fd = fd;
115 g = get_vfio_iommu_group (group);
121 s = format (s, "/dev/vfio/%s%u%c", is_noiommu ? "noiommu-" : "", group, 0);
122 fd = open ((char *) s, O_RDWR);
124 return clib_error_return_unix (0, "open '%s'", s);
126 group_status.argsz = sizeof (group_status);
127 if (ioctl (fd, VFIO_GROUP_GET_STATUS, &group_status) < 0)
129 err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_STATUS) '%s'",
134 if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE))
136 err = clib_error_return (0, "iommu group %d is not viable (not all "
137 "devices in this group bound to vfio-pci)",
142 if (ioctl (fd, VFIO_GROUP_SET_CONTAINER, &lvm->container_fd) < 0)
144 err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_SET_CONTAINER) '%s'",
149 if (lvm->iommu_mode == 0)
152 lvm->iommu_mode = VFIO_NOIOMMU_IOMMU;
154 lvm->iommu_mode = VFIO_TYPE1_IOMMU;
156 if (ioctl (lvm->container_fd, VFIO_SET_IOMMU, lvm->iommu_mode) < 0)
158 err = clib_error_return_unix (0, "ioctl(VFIO_SET_IOMMU) "
165 pool_get (lvm->iommu_groups, g);
168 hash_set (lvm->iommu_pool_index_by_group, group, g - lvm->iommu_groups);
177 linux_vfio_group_get_device_fd (vlib_pci_addr_t * addr, int *fdp,
180 clib_error_t *err = 0;
181 linux_pci_vfio_iommu_group_t *g;
188 s = format (s, "/sys/bus/pci/devices/%U/iommu_group", format_vlib_pci_addr,
190 tmpstr = clib_sysfs_link_to_name ((char *) s);
193 iommu_group = atoi ((char *) tmpstr);
198 err = clib_error_return (0, "Cannot find IOMMU group for PCI device ",
199 "'%U'", format_vlib_pci_addr, addr);
202 vec_reset_length (s);
204 s = format (s, "/sys/bus/pci/devices/%U/iommu_group/name",
205 format_vlib_pci_addr, addr);
206 err = clib_sysfs_read ((char *) s, "%s", &tmpstr);
209 if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0)
215 clib_error_free (err);
216 vec_reset_length (s);
217 if ((err = open_vfio_iommu_group (iommu_group, *is_noiommu)))
220 g = get_vfio_iommu_group (iommu_group);
222 s = format (s, "%U%c", format_vlib_pci_addr, addr, 0);
223 if ((fd = ioctl (g->fd, VFIO_GROUP_GET_DEVICE_FD, (char *) s)) < 0)
225 err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_DEVICE_FD) '%U'",
226 format_vlib_pci_addr, addr);
229 vec_reset_length (s);
239 linux_vfio_init (vlib_main_t * vm)
241 linux_vfio_main_t *lvm = &vfio_main;
243 lvm->log_default = vlib_log_register_class ("vfio", 0);
244 lvm->container_fd = -1;
250 * fd.io coding-style-patch-verification: ON
253 * eval: (c-set-style "gnu")