1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
7 #include <linux/pci_regs.h>
8 #include <sys/eventfd.h>
9 #include <sys/socket.h>
10 #include <sys/ioctl.h>
16 #include <rte_bus_pci.h>
17 #include <rte_eal_memconfig.h>
18 #include <rte_malloc.h>
23 #include "eal_filesystem.h"
30 * PCI probing under linux (VFIO version)
32 * This code tries to determine if the PCI device is bound to VFIO driver,
33 * and initialize it (map BARs, set up interrupts) if that's the case.
35 * This file is only compiled if CONFIG_RTE_EAL_VFIO is set to "y".
41 #define PAGE_SIZE (sysconf(_SC_PAGESIZE))
43 #define PAGE_MASK (~(PAGE_SIZE - 1))
45 static struct rte_tailq_elem rte_vfio_tailq = {
46 .name = "VFIO_RESOURCE_LIST",
48 EAL_REGISTER_TAILQ(rte_vfio_tailq)
51 pci_vfio_read_config(const struct rte_intr_handle *intr_handle,
52 void *buf, size_t len, off_t offs)
54 return pread64(intr_handle->vfio_dev_fd, buf, len,
55 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs);
59 pci_vfio_write_config(const struct rte_intr_handle *intr_handle,
60 const void *buf, size_t len, off_t offs)
62 return pwrite64(intr_handle->vfio_dev_fd, buf, len,
63 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs);
66 /* get PCI BAR number where MSI-X interrupts are */
68 pci_vfio_get_msix_bar(int fd, struct pci_msix_table *msix_table)
73 uint8_t cap_id, cap_offset;
75 /* read PCI capability pointer from config space */
76 ret = pread64(fd, ®, sizeof(reg),
77 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) +
79 if (ret != sizeof(reg)) {
80 RTE_LOG(ERR, EAL, "Cannot read capability pointer from PCI "
85 /* we need first byte */
86 cap_offset = reg & 0xFF;
90 /* read PCI capability ID */
91 ret = pread64(fd, ®, sizeof(reg),
92 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) +
94 if (ret != sizeof(reg)) {
95 RTE_LOG(ERR, EAL, "Cannot read capability ID from PCI "
100 /* we need first byte */
103 /* if we haven't reached MSI-X, check next capability */
104 if (cap_id != PCI_CAP_ID_MSIX) {
105 ret = pread64(fd, ®, sizeof(reg),
106 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) +
108 if (ret != sizeof(reg)) {
109 RTE_LOG(ERR, EAL, "Cannot read capability pointer from PCI "
114 /* we need second byte */
115 cap_offset = (reg & 0xFF00) >> 8;
119 /* else, read table offset */
121 /* table offset resides in the next 4 bytes */
122 ret = pread64(fd, ®, sizeof(reg),
123 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) +
125 if (ret != sizeof(reg)) {
126 RTE_LOG(ERR, EAL, "Cannot read table offset from PCI config "
131 ret = pread64(fd, &flags, sizeof(flags),
132 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) +
134 if (ret != sizeof(flags)) {
135 RTE_LOG(ERR, EAL, "Cannot read table flags from PCI config "
140 msix_table->bar_index = reg & RTE_PCI_MSIX_TABLE_BIR;
141 msix_table->offset = reg & RTE_PCI_MSIX_TABLE_OFFSET;
143 16 * (1 + (flags & RTE_PCI_MSIX_FLAGS_QSIZE));
151 /* set PCI bus mastering */
153 pci_vfio_set_bus_master(int dev_fd, bool op)
158 ret = pread64(dev_fd, ®, sizeof(reg),
159 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) +
161 if (ret != sizeof(reg)) {
162 RTE_LOG(ERR, EAL, "Cannot read command from PCI config space!\n");
167 /* set the master bit */
168 reg |= PCI_COMMAND_MASTER;
170 reg &= ~(PCI_COMMAND_MASTER);
172 ret = pwrite64(dev_fd, ®, sizeof(reg),
173 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) +
176 if (ret != sizeof(reg)) {
177 RTE_LOG(ERR, EAL, "Cannot write command to PCI config space!\n");
184 /* set up interrupt support (but not enable interrupts) */
186 pci_vfio_setup_interrupts(struct rte_pci_device *dev, int vfio_dev_fd)
188 int i, ret, intr_idx;
189 enum rte_intr_mode intr_mode;
191 /* default to invalid index */
192 intr_idx = VFIO_PCI_NUM_IRQS;
194 /* Get default / configured intr_mode */
195 intr_mode = rte_eal_vfio_intr_mode();
197 /* get interrupt type from internal config (MSI-X by default, can be
198 * overridden from the command line
201 case RTE_INTR_MODE_MSIX:
202 intr_idx = VFIO_PCI_MSIX_IRQ_INDEX;
204 case RTE_INTR_MODE_MSI:
205 intr_idx = VFIO_PCI_MSI_IRQ_INDEX;
207 case RTE_INTR_MODE_LEGACY:
208 intr_idx = VFIO_PCI_INTX_IRQ_INDEX;
210 /* don't do anything if we want to automatically determine interrupt type */
211 case RTE_INTR_MODE_NONE:
214 RTE_LOG(ERR, EAL, " unknown default interrupt type!\n");
218 /* start from MSI-X interrupt type */
219 for (i = VFIO_PCI_MSIX_IRQ_INDEX; i >= 0; i--) {
220 struct vfio_irq_info irq = { .argsz = sizeof(irq) };
223 /* skip interrupt modes we don't want */
224 if (intr_mode != RTE_INTR_MODE_NONE &&
230 ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq);
232 RTE_LOG(ERR, EAL, " cannot get IRQ info, "
233 "error %i (%s)\n", errno, strerror(errno));
237 /* if this vector cannot be used with eventfd, fail if we explicitly
238 * specified interrupt type, otherwise continue */
239 if ((irq.flags & VFIO_IRQ_INFO_EVENTFD) == 0) {
240 if (intr_mode != RTE_INTR_MODE_NONE) {
242 " interrupt vector does not support eventfd!\n");
248 /* set up an eventfd for interrupts */
249 fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
251 RTE_LOG(ERR, EAL, " cannot set up eventfd, "
252 "error %i (%s)\n", errno, strerror(errno));
256 dev->intr_handle.fd = fd;
257 dev->intr_handle.vfio_dev_fd = vfio_dev_fd;
260 case VFIO_PCI_MSIX_IRQ_INDEX:
261 intr_mode = RTE_INTR_MODE_MSIX;
262 dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
264 case VFIO_PCI_MSI_IRQ_INDEX:
265 intr_mode = RTE_INTR_MODE_MSI;
266 dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_MSI;
268 case VFIO_PCI_INTX_IRQ_INDEX:
269 intr_mode = RTE_INTR_MODE_LEGACY;
270 dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_LEGACY;
273 RTE_LOG(ERR, EAL, " unknown interrupt type!\n");
280 /* if we're here, we haven't found a suitable interrupt vector */
284 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
286 pci_vfio_req_handler(void *param)
290 struct rte_device *device = (struct rte_device *)param;
292 bus = rte_bus_find_by_device(device);
294 RTE_LOG(ERR, EAL, "Cannot find bus for device (%s)\n",
300 * vfio kernel module request user space to release allocated
301 * resources before device be deleted in kernel, so it can directly
302 * call the vfio bus hot-unplug handler to process it.
304 ret = bus->hot_unplug_handler(device);
307 "Can not handle hot-unplug for device (%s)\n",
311 /* enable notifier (only enable req now) */
313 pci_vfio_enable_notifier(struct rte_pci_device *dev, int vfio_dev_fd)
318 /* set up an eventfd for req notifier */
319 fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
321 RTE_LOG(ERR, EAL, "Cannot set up eventfd, error %i (%s)\n",
322 errno, strerror(errno));
326 dev->vfio_req_intr_handle.fd = fd;
327 dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_VFIO_REQ;
328 dev->vfio_req_intr_handle.vfio_dev_fd = vfio_dev_fd;
330 ret = rte_intr_callback_register(&dev->vfio_req_intr_handle,
331 pci_vfio_req_handler,
332 (void *)&dev->device);
334 RTE_LOG(ERR, EAL, "Fail to register req notifier handler.\n");
338 ret = rte_intr_enable(&dev->vfio_req_intr_handle);
340 RTE_LOG(ERR, EAL, "Fail to enable req notifier.\n");
341 ret = rte_intr_callback_unregister(&dev->vfio_req_intr_handle,
342 pci_vfio_req_handler,
343 (void *)&dev->device);
346 "Fail to unregister req notifier handler.\n");
354 dev->vfio_req_intr_handle.fd = -1;
355 dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
356 dev->vfio_req_intr_handle.vfio_dev_fd = -1;
361 /* disable notifier (only disable req now) */
363 pci_vfio_disable_notifier(struct rte_pci_device *dev)
367 ret = rte_intr_disable(&dev->vfio_req_intr_handle);
369 RTE_LOG(ERR, EAL, "fail to disable req notifier.\n");
373 ret = rte_intr_callback_unregister(&dev->vfio_req_intr_handle,
374 pci_vfio_req_handler,
375 (void *)&dev->device);
378 "fail to unregister req notifier handler.\n");
382 close(dev->vfio_req_intr_handle.fd);
384 dev->vfio_req_intr_handle.fd = -1;
385 dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
386 dev->vfio_req_intr_handle.vfio_dev_fd = -1;
393 pci_vfio_is_ioport_bar(int vfio_dev_fd, int bar_index)
398 ret = pread64(vfio_dev_fd, &ioport_bar, sizeof(ioport_bar),
399 VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX)
400 + PCI_BASE_ADDRESS_0 + bar_index*4);
401 if (ret != sizeof(ioport_bar)) {
402 RTE_LOG(ERR, EAL, "Cannot read command (%x) from config space!\n",
403 PCI_BASE_ADDRESS_0 + bar_index*4);
407 return (ioport_bar & PCI_BASE_ADDRESS_SPACE_IO) != 0;
411 pci_rte_vfio_setup_device(struct rte_pci_device *dev, int vfio_dev_fd)
413 if (pci_vfio_setup_interrupts(dev, vfio_dev_fd) != 0) {
414 RTE_LOG(ERR, EAL, "Error setting up interrupts!\n");
418 /* set bus mastering for the device */
419 if (pci_vfio_set_bus_master(vfio_dev_fd, true)) {
420 RTE_LOG(ERR, EAL, "Cannot set up bus mastering!\n");
425 * Reset the device. If the device is not capable of resetting,
426 * then it updates errno as EINVAL.
428 if (ioctl(vfio_dev_fd, VFIO_DEVICE_RESET) && errno != EINVAL) {
429 RTE_LOG(ERR, EAL, "Unable to reset device! Error: %d (%s)\n",
430 errno, strerror(errno));
438 pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res,
439 int bar_index, int additional_flags)
442 unsigned long offset, size;
445 struct pci_msix_table *msix_table = &vfio_res->msix_table;
446 struct pci_map *bar = &vfio_res->maps[bar_index];
452 if (msix_table->bar_index == bar_index) {
454 * VFIO will not let us map the MSI-X table,
455 * but we can map around it.
457 uint32_t table_start = msix_table->offset;
458 uint32_t table_end = table_start + msix_table->size;
459 table_end = (table_end + ~PAGE_MASK) & PAGE_MASK;
460 table_start &= PAGE_MASK;
462 if (table_start == 0 && table_end >= bar->size) {
463 /* Cannot map this BAR */
464 RTE_LOG(DEBUG, EAL, "Skipping BAR%d\n", bar_index);
470 memreg[0].offset = bar->offset;
471 memreg[0].size = table_start;
472 memreg[1].offset = bar->offset + table_end;
473 memreg[1].size = bar->size - table_end;
476 "Trying to map BAR%d that contains the MSI-X "
477 "table. Trying offsets: "
478 "0x%04lx:0x%04lx, 0x%04lx:0x%04lx\n", bar_index,
479 memreg[0].offset, memreg[0].size,
480 memreg[1].offset, memreg[1].size);
482 memreg[0].offset = bar->offset;
483 memreg[0].size = bar->size;
486 /* reserve the address using an inaccessible mapping */
487 bar_addr = mmap(bar->addr, bar->size, 0, MAP_PRIVATE |
488 MAP_ANONYMOUS | additional_flags, -1, 0);
489 if (bar_addr != MAP_FAILED) {
490 void *map_addr = NULL;
491 if (memreg[0].size) {
492 /* actual map of first part */
493 map_addr = pci_map_resource(bar_addr, vfio_dev_fd,
499 /* if there's a second part, try to map it */
500 if (map_addr != MAP_FAILED
501 && memreg[1].offset && memreg[1].size) {
502 void *second_addr = RTE_PTR_ADD(bar_addr,
504 (uintptr_t)bar->offset);
505 map_addr = pci_map_resource(second_addr,
512 if (map_addr == MAP_FAILED || !map_addr) {
513 munmap(bar_addr, bar->size);
514 bar_addr = MAP_FAILED;
515 RTE_LOG(ERR, EAL, "Failed to map pci BAR%d\n",
521 "Failed to create inaccessible mapping for BAR%d\n",
526 bar->addr = bar_addr;
531 * region info may contain capability headers, so we need to keep reallocating
532 * the memory until we match allocated memory size with argsz.
535 pci_vfio_get_region_info(int vfio_dev_fd, struct vfio_region_info **info,
538 struct vfio_region_info *ri;
539 size_t argsz = sizeof(*ri);
542 ri = malloc(sizeof(*ri));
544 RTE_LOG(ERR, EAL, "Cannot allocate memory for region info\n");
548 memset(ri, 0, argsz);
552 ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ri);
557 if (ri->argsz != argsz) {
558 struct vfio_region_info *tmp;
561 tmp = realloc(ri, argsz);
564 /* realloc failed but the ri is still there */
566 RTE_LOG(ERR, EAL, "Cannot reallocate memory for region info\n");
577 static struct vfio_info_cap_header *
578 pci_vfio_info_cap(struct vfio_region_info *info, int cap)
580 struct vfio_info_cap_header *h;
583 if ((info->flags & RTE_VFIO_INFO_FLAG_CAPS) == 0) {
584 /* VFIO info does not advertise capabilities */
588 offset = VFIO_CAP_OFFSET(info);
589 while (offset != 0) {
590 h = RTE_PTR_ADD(info, offset);
599 pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
601 struct vfio_region_info *info;
604 ret = pci_vfio_get_region_info(vfio_dev_fd, &info, msix_region);
608 ret = pci_vfio_info_cap(info, RTE_VFIO_CAP_MSIX_MAPPABLE) != NULL;
618 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
620 struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
621 char pci_addr[PATH_MAX] = {0};
623 struct rte_pci_addr *loc = &dev->addr;
625 struct mapped_pci_resource *vfio_res = NULL;
626 struct mapped_pci_res_list *vfio_res_list =
627 RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
629 struct pci_map *maps;
631 dev->intr_handle.fd = -1;
632 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
633 dev->vfio_req_intr_handle.fd = -1;
636 /* store PCI address string */
637 snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
638 loc->domain, loc->bus, loc->devid, loc->function);
640 ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
641 &vfio_dev_fd, &device_info);
645 /* allocate vfio_res and get region info */
646 vfio_res = rte_zmalloc("VFIO_RES", sizeof(*vfio_res), 0);
647 if (vfio_res == NULL) {
649 "%s(): cannot store uio mmap details\n", __func__);
650 goto err_vfio_dev_fd;
652 memcpy(&vfio_res->pci_addr, &dev->addr, sizeof(vfio_res->pci_addr));
654 /* get number of registers (up to BAR5) */
655 vfio_res->nb_maps = RTE_MIN((int) device_info.num_regions,
656 VFIO_PCI_BAR5_REGION_INDEX + 1);
659 maps = vfio_res->maps;
661 vfio_res->msix_table.bar_index = -1;
662 /* get MSI-X BAR, if any (we have to know where it is because we can't
663 * easily mmap it when using VFIO)
665 ret = pci_vfio_get_msix_bar(vfio_dev_fd, &vfio_res->msix_table);
667 RTE_LOG(ERR, EAL, " %s cannot get MSI-X BAR number!\n",
671 /* if we found our MSI-X BAR region, check if we can mmap it */
672 if (vfio_res->msix_table.bar_index != -1) {
673 int ret = pci_vfio_msix_is_mappable(vfio_dev_fd,
674 vfio_res->msix_table.bar_index);
676 RTE_LOG(ERR, EAL, "Couldn't check if MSI-X BAR is mappable\n");
678 } else if (ret != 0) {
679 /* we can map it, so we don't care where it is */
680 RTE_LOG(DEBUG, EAL, "VFIO reports MSI-X BAR as mappable\n");
681 vfio_res->msix_table.bar_index = -1;
685 for (i = 0; i < (int) vfio_res->nb_maps; i++) {
686 struct vfio_region_info *reg = NULL;
689 ret = pci_vfio_get_region_info(vfio_dev_fd, ®, i);
691 RTE_LOG(ERR, EAL, " %s cannot get device region info "
692 "error %i (%s)\n", pci_addr, errno,
697 /* chk for io port region */
698 ret = pci_vfio_is_ioport_bar(vfio_dev_fd, i);
703 RTE_LOG(INFO, EAL, "Ignore mapping IO port bar(%d)\n",
709 /* skip non-mmapable BARs */
710 if ((reg->flags & VFIO_REGION_INFO_FLAG_MMAP) == 0) {
715 /* try mapping somewhere close to the end of hugepages */
716 if (pci_map_addr == NULL)
717 pci_map_addr = pci_find_max_end_va();
719 bar_addr = pci_map_addr;
720 pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg->size);
722 maps[i].addr = bar_addr;
723 maps[i].offset = reg->offset;
724 maps[i].size = reg->size;
725 maps[i].path = NULL; /* vfio doesn't have per-resource paths */
727 ret = pci_vfio_mmap_bar(vfio_dev_fd, vfio_res, i, 0);
729 RTE_LOG(ERR, EAL, " %s mapping BAR%i failed: %s\n",
730 pci_addr, i, strerror(errno));
735 dev->mem_resource[i].addr = maps[i].addr;
740 if (pci_rte_vfio_setup_device(dev, vfio_dev_fd) < 0) {
741 RTE_LOG(ERR, EAL, " %s setup device failed\n", pci_addr);
745 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
746 if (pci_vfio_enable_notifier(dev, vfio_dev_fd) != 0) {
747 RTE_LOG(ERR, EAL, "Error setting up notifier!\n");
752 TAILQ_INSERT_TAIL(vfio_res_list, vfio_res, next);
763 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
765 struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
766 char pci_addr[PATH_MAX] = {0};
768 struct rte_pci_addr *loc = &dev->addr;
770 struct mapped_pci_resource *vfio_res = NULL;
771 struct mapped_pci_res_list *vfio_res_list =
772 RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
774 struct pci_map *maps;
776 dev->intr_handle.fd = -1;
777 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
778 dev->vfio_req_intr_handle.fd = -1;
781 /* store PCI address string */
782 snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
783 loc->domain, loc->bus, loc->devid, loc->function);
785 ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
786 &vfio_dev_fd, &device_info);
790 /* if we're in a secondary process, just find our tailq entry */
791 TAILQ_FOREACH(vfio_res, vfio_res_list, next) {
792 if (rte_pci_addr_cmp(&vfio_res->pci_addr,
797 /* if we haven't found our tailq entry, something's wrong */
798 if (vfio_res == NULL) {
799 RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n",
801 goto err_vfio_dev_fd;
805 maps = vfio_res->maps;
807 for (i = 0; i < (int) vfio_res->nb_maps; i++) {
808 ret = pci_vfio_mmap_bar(vfio_dev_fd, vfio_res, i, MAP_FIXED);
810 RTE_LOG(ERR, EAL, " %s mapping BAR%i failed: %s\n",
811 pci_addr, i, strerror(errno));
812 goto err_vfio_dev_fd;
815 dev->mem_resource[i].addr = maps[i].addr;
818 /* we need save vfio_dev_fd, so it can be used during release */
819 dev->intr_handle.vfio_dev_fd = vfio_dev_fd;
820 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
821 dev->vfio_req_intr_handle.vfio_dev_fd = vfio_dev_fd;
831 * map the PCI resources of a PCI device in virtual memory (VFIO version).
832 * primary and secondary processes follow almost exactly the same path
835 pci_vfio_map_resource(struct rte_pci_device *dev)
837 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
838 return pci_vfio_map_resource_primary(dev);
840 return pci_vfio_map_resource_secondary(dev);
843 static struct mapped_pci_resource *
844 find_and_unmap_vfio_resource(struct mapped_pci_res_list *vfio_res_list,
845 struct rte_pci_device *dev,
846 const char *pci_addr)
848 struct mapped_pci_resource *vfio_res = NULL;
849 struct pci_map *maps;
853 TAILQ_FOREACH(vfio_res, vfio_res_list, next) {
854 if (rte_pci_addr_cmp(&vfio_res->pci_addr, &dev->addr))
859 if (vfio_res == NULL)
862 RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n",
865 maps = vfio_res->maps;
866 for (i = 0; i < (int) vfio_res->nb_maps; i++) {
869 * We do not need to be aware of MSI-X table BAR mappings as
870 * when mapping. Just using current maps array is enough
873 RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n",
874 pci_addr, maps[i].addr);
875 pci_unmap_resource(maps[i].addr, maps[i].size);
883 pci_vfio_unmap_resource_primary(struct rte_pci_device *dev)
885 char pci_addr[PATH_MAX] = {0};
886 struct rte_pci_addr *loc = &dev->addr;
887 struct mapped_pci_resource *vfio_res = NULL;
888 struct mapped_pci_res_list *vfio_res_list;
891 /* store PCI address string */
892 snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
893 loc->domain, loc->bus, loc->devid, loc->function);
895 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
896 ret = pci_vfio_disable_notifier(dev);
898 RTE_LOG(ERR, EAL, "fail to disable req notifier.\n");
903 if (close(dev->intr_handle.fd) < 0) {
904 RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n",
909 if (pci_vfio_set_bus_master(dev->intr_handle.vfio_dev_fd, false)) {
910 RTE_LOG(ERR, EAL, " %s cannot unset bus mastering for PCI device!\n",
915 ret = rte_vfio_release_device(rte_pci_get_sysfs_path(), pci_addr,
916 dev->intr_handle.vfio_dev_fd);
919 "%s(): cannot release device\n", __func__);
924 RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
925 vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr);
927 /* if we haven't found our tailq entry, something's wrong */
928 if (vfio_res == NULL) {
929 RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n",
934 TAILQ_REMOVE(vfio_res_list, vfio_res, next);
940 pci_vfio_unmap_resource_secondary(struct rte_pci_device *dev)
942 char pci_addr[PATH_MAX] = {0};
943 struct rte_pci_addr *loc = &dev->addr;
944 struct mapped_pci_resource *vfio_res = NULL;
945 struct mapped_pci_res_list *vfio_res_list;
948 /* store PCI address string */
949 snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
950 loc->domain, loc->bus, loc->devid, loc->function);
952 ret = rte_vfio_release_device(rte_pci_get_sysfs_path(), pci_addr,
953 dev->intr_handle.vfio_dev_fd);
956 "%s(): cannot release device\n", __func__);
961 RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
962 vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr);
964 /* if we haven't found our tailq entry, something's wrong */
965 if (vfio_res == NULL) {
966 RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n",
975 pci_vfio_unmap_resource(struct rte_pci_device *dev)
977 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
978 return pci_vfio_unmap_resource_primary(dev);
980 return pci_vfio_unmap_resource_secondary(dev);
984 pci_vfio_ioport_map(struct rte_pci_device *dev, int bar,
985 struct rte_pci_ioport *p)
987 if (bar < VFIO_PCI_BAR0_REGION_INDEX ||
988 bar > VFIO_PCI_BAR5_REGION_INDEX) {
989 RTE_LOG(ERR, EAL, "invalid bar (%d)!\n", bar);
994 p->base = VFIO_GET_REGION_ADDR(bar);
999 pci_vfio_ioport_read(struct rte_pci_ioport *p,
1000 void *data, size_t len, off_t offset)
1002 const struct rte_intr_handle *intr_handle = &p->dev->intr_handle;
1004 if (pread64(intr_handle->vfio_dev_fd, data,
1005 len, p->base + offset) <= 0)
1007 "Can't read from PCI bar (%" PRIu64 ") : offset (%x)\n",
1008 VFIO_GET_REGION_IDX(p->base), (int)offset);
1012 pci_vfio_ioport_write(struct rte_pci_ioport *p,
1013 const void *data, size_t len, off_t offset)
1015 const struct rte_intr_handle *intr_handle = &p->dev->intr_handle;
1017 if (pwrite64(intr_handle->vfio_dev_fd, data,
1018 len, p->base + offset) <= 0)
1020 "Can't write to PCI bar (%" PRIu64 ") : offset (%x)\n",
1021 VFIO_GET_REGION_IDX(p->base), (int)offset);
1025 pci_vfio_ioport_unmap(struct rte_pci_ioport *p)
1032 pci_vfio_is_enabled(void)
1034 return rte_vfio_is_enabled("vfio_pci");