#include <vlib/vlib.h>
#include <vlib/pci/pci.h>
#include <vlib/unix/unix.h>
+#include <vlib/linux/vfio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <net/if.h>
#include <linux/ethtool.h>
#include <linux/sockios.h>
+#include <linux/vfio.h>
+#include <sys/eventfd.h>
+
+static const char *sysfs_pci_dev_path = "/sys/bus/pci/devices";
+static const char *sysfs_pci_drv_path = "/sys/bus/pci/drivers";
+static char *sysfs_mod_vfio_noiommu =
+ "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode";
+
+#define pci_log_debug(vm, dev, f, ...) \
+ vlib_log(VLIB_LOG_LEVEL_DEBUG, pci_main.log_default, "%U: " f, \
+ format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__)
+#define pci_log_err(vm, dev, f, ...) \
+ vlib_log(VLIB_LOG_LEVEL_ERR, pci_main.log_default, "%U: " f, \
+ format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__)
+
+typedef struct
+{
+ int fd;
+ void *addr;
+ size_t size;
+} linux_pci_region_t;
+
+typedef struct
+{
+ int fd;
+ u32 clib_file_index;
+ union
+ {
+ pci_intx_handler_function_t *intx_handler;
+ pci_msix_handler_function_t *msix_handler;
+ };
+} linux_pci_irq_t;
+
+typedef enum
+{
+ LINUX_PCI_DEVICE_TYPE_UNKNOWN,
+ LINUX_PCI_DEVICE_TYPE_UIO,
+ LINUX_PCI_DEVICE_TYPE_VFIO,
+} linux_pci_device_type_t;
typedef struct
{
- /* /sys/bus/pci/devices/... directory name for this device. */
- u8 *dev_dir_name;
+ linux_pci_device_type_t type;
+ vlib_pci_dev_handle_t handle;
+ vlib_pci_addr_t addr;
+ u32 numa_node;
/* Resource file descriptors. */
- int *resource_fds;
+ linux_pci_region_t *regions;
/* File descriptor for config space read/write. */
int config_fd;
+ u64 config_offset;
+
+ /* Device File descriptor */
+ int fd;
- /* File descriptor for /dev/uio%d */
- int uio_fd;
+ /* read/write file descriptor for io bar */
+ int io_fd;
+ u64 io_offset;
/* Minor device for uio device. */
u32 uio_minor;
- /* Index given by clib_file_add. */
- u32 clib_file_index;
+ /* Interrupt handlers */
+ linux_pci_irq_t intx_irq;
+ linux_pci_irq_t *msix_irqs;
+
+ /* private data */
+ uword private_data;
+
+ u8 supports_va_dma;
} linux_pci_device_t;
{
vlib_main_t *vlib_main;
linux_pci_device_t *linux_pci_devices;
+
} linux_pci_main_t;
extern linux_pci_main_t linux_pci_main;
+static linux_pci_device_t *
+linux_pci_get_device (vlib_pci_dev_handle_t h)
+{
+ linux_pci_main_t *lpm = &linux_pci_main;
+ return pool_elt_at_index (lpm->linux_pci_devices, h);
+}
+
+uword
+vlib_pci_get_private_data (vlib_main_t * vm, vlib_pci_dev_handle_t h)
+{
+ linux_pci_device_t *d = linux_pci_get_device (h);
+ return d->private_data;
+}
+
+void
+vlib_pci_set_private_data (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ uword private_data)
+{
+ linux_pci_device_t *d = linux_pci_get_device (h);
+ d->private_data = private_data;
+}
+
+vlib_pci_addr_t *
+vlib_pci_get_addr (vlib_main_t * vm, vlib_pci_dev_handle_t h)
+{
+ linux_pci_device_t *d = linux_pci_get_device (h);
+ return &d->addr;
+}
+
+u32
+vlib_pci_get_numa_node (vlib_main_t * vm, vlib_pci_dev_handle_t h)
+{
+ linux_pci_device_t *d = linux_pci_get_device (h);
+ return d->numa_node;
+}
+
+u32
+vlib_pci_get_num_msix_interrupts (vlib_main_t * vm, vlib_pci_dev_handle_t h)
+{
+ linux_pci_device_t *d = linux_pci_get_device (h);
+
+ if (d->type == LINUX_PCI_DEVICE_TYPE_VFIO)
+ {
+ struct vfio_irq_info ii = { 0 };
+
+ ii.argsz = sizeof (struct vfio_irq_info);
+ ii.index = VFIO_PCI_MSIX_IRQ_INDEX;
+ if (ioctl (d->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
+ return 0;
+ return ii.count;
+ }
+ return 0;
+}
+
/* Call to allocate/initialize the pci subsystem.
This is not an init function so that users can explicitly enable
pci only when it's needed. */
clib_error_t *pci_bus_init (vlib_main_t * vm);
-clib_error_t *vlib_pci_bind_to_uio (vlib_pci_device_t * d,
- char *uio_driver_name);
-
linux_pci_main_t linux_pci_main;
+vlib_pci_device_info_t *
+vlib_pci_get_device_info (vlib_main_t * vm, vlib_pci_addr_t * addr,
+ clib_error_t ** error)
+{
+ clib_error_t *err;
+ vlib_pci_device_info_t *di;
+ u8 *f = 0;
+ u32 tmp;
+ int fd;
+ u8 *tmpstr;
+ clib_bitmap_t *bmp = 0;
+
+ di = clib_mem_alloc (sizeof (vlib_pci_device_info_t));
+ clib_memset (di, 0, sizeof (vlib_pci_device_info_t));
+ di->addr.as_u32 = addr->as_u32;
+
+ u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path,
+ format_vlib_pci_addr, addr);
+
+ f = format (0, "%v/config%c", dev_dir_name, 0);
+ fd = open ((char *) f, O_RDWR);
+
+ /* Try read-only access if write fails. */
+ if (fd < 0)
+ fd = open ((char *) f, O_RDONLY);
+
+ if (fd < 0)
+ {
+ err = clib_error_return_unix (0, "open `%s'", f);
+ goto error;
+ }
+
+ /* You can only read more that 64 bytes of config space as root; so we try to
+ read the full space but fall back to just the first 64 bytes. */
+ if (read (fd, &di->config_data, sizeof (di->config_data)) <
+ sizeof (di->config0))
+ {
+ err = clib_error_return_unix (0, "read `%s'", f);
+ close (fd);
+ goto error;
+ }
+
+ {
+ static pci_config_header_t all_ones;
+ if (all_ones.vendor_id == 0)
+ clib_memset (&all_ones, ~0, sizeof (all_ones));
+
+ if (!memcmp (&di->config0.header, &all_ones, sizeof (all_ones)))
+ {
+ err = clib_error_return (0, "invalid PCI config for `%s'", f);
+ close (fd);
+ goto error;
+ }
+ }
+
+ if (di->config0.header.header_type == 0)
+ pci_config_type0_little_to_host (&di->config0);
+ else
+ pci_config_type1_little_to_host (&di->config1);
+
+ di->numa_node = -1;
+ vec_reset_length (f);
+ f = format (f, "%v/numa_node%c", dev_dir_name, 0);
+ err = clib_sysfs_read ((char *) f, "%d", &di->numa_node);
+ if (err)
+ {
+ di->numa_node = -1;
+ clib_error_free (err);
+ }
+ if (di->numa_node == -1)
+ {
+ /* if '/sys/bus/pci/devices/<device id>/numa_node' returns -1 and
+ it is a SMP system, set numa_node to 0. */
+ if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U",
+ unformat_bitmap_list, &bmp)))
+ clib_error_free (err);
+ if (clib_bitmap_count_set_bits (bmp) == 1)
+ di->numa_node = 0;
+ }
+
+ vec_reset_length (f);
+ f = format (f, "%v/class%c", dev_dir_name, 0);
+ err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
+ if (err)
+ goto error;
+ di->device_class = tmp >> 8;
+
+ vec_reset_length (f);
+ f = format (f, "%v/vendor%c", dev_dir_name, 0);
+ err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
+ if (err)
+ goto error;
+ di->vendor_id = tmp;
+
+ vec_reset_length (f);
+ f = format (f, "%v/device%c", dev_dir_name, 0);
+ err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
+ if (err)
+ goto error;
+ di->device_id = tmp;
+
+ vec_reset_length (f);
+ f = format (f, "%v/driver%c", dev_dir_name, 0);
+ di->driver_name = clib_sysfs_link_to_name ((char *) f);
+ if (!di->driver_name)
+ di->driver_name = format (0, "<NONE>%c", 0);
+
+ di->iommu_group = -1;
+ vec_reset_length (f);
+ f = format (f, "%v/iommu_group%c", dev_dir_name, 0);
+ tmpstr = clib_sysfs_link_to_name ((char *) f);
+ if (tmpstr)
+ {
+ di->iommu_group = atoi ((char *) tmpstr);
+ vec_free (tmpstr);
+ }
+
+ vec_reset_length (f);
+ f = format (f, "%v/iommu_group/name%c", dev_dir_name, 0);
+ err = clib_sysfs_read ((char *) f, "%s", &tmpstr);
+ if (err == 0)
+ {
+ if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0)
+ di->flags |= VLIB_PCI_DEVICE_INFO_F_NOIOMMU;
+ vec_free (tmpstr);
+ }
+ else
+ clib_error_free (err);
+
+ close (fd);
+
+ vec_reset_length (f);
+ f = format (f, "%v/vpd%c", dev_dir_name, 0);
+ fd = open ((char *) f, O_RDONLY);
+ if (fd >= 0)
+ {
+ while (1)
+ {
+ u8 tag[3];
+ u8 *data = 0;
+ uword len;
+
+ if (read (fd, &tag, 3) != 3)
+ break;
+
+ if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91)
+ break;
+
+ len = (tag[2] << 8) | tag[1];
+ vec_validate (data, len);
+
+ if (read (fd, data, len) != len)
+ {
+ vec_free (data);
+ break;
+ }
+ if (tag[0] == 0x82)
+ di->product_name = data;
+ else if (tag[0] == 0x90)
+ di->vpd_r = data;
+ else if (tag[0] == 0x91)
+ di->vpd_w = data;
+
+ data = 0;
+ }
+ close (fd);
+ }
+
+ goto done;
+
+error:
+ vlib_pci_free_device_info (di);
+ di = 0;
+
+done:
+ vec_free (bmp);
+ vec_free (f);
+ vec_free (dev_dir_name);
+ if (error)
+ *error = err;
+ else
+ clib_error_free (err);
+ return di;
+}
+
+static int
+directory_exists (char *path)
+{
+ struct stat s = { 0 };
+ if (stat (path, &s) == -1)
+ return 0;
+
+ return S_ISDIR (s.st_mode);
+}
+
clib_error_t *
-vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name)
+vlib_pci_bind_to_uio (vlib_main_t * vm, vlib_pci_addr_t * addr,
+ char *uio_drv_name)
{
clib_error_t *error = 0;
u8 *s = 0, *driver_name = 0;
DIR *dir = 0;
struct dirent *e;
+ vlib_pci_device_info_t *di;
int fd, clear_driver_override = 0;
- u8 *dev_dir_name = format (0, "/sys/bus/pci/devices/%U",
- format_vlib_pci_addr, &d->bus_address);
+ u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path,
+ format_vlib_pci_addr, addr);
+
+ di = vlib_pci_get_device_info (vm, addr, &error);
+
+ if (error)
+ return error;
+
+ if (strncmp ("auto", uio_drv_name, 5) == 0)
+ {
+ int vfio_pci_loaded = 0;
+
+ if (directory_exists ("/sys/module/vfio_pci"))
+ vfio_pci_loaded = 1;
+
+ if (di->iommu_group != -1)
+ {
+ /* device is bound to IOMMU group */
+ if (!vfio_pci_loaded)
+ {
+ error = clib_error_return (0, "Skipping PCI device %U: device "
+ "is bound to IOMMU group and "
+ "vfio-pci driver is not loaded",
+ format_vlib_pci_addr, addr);
+ goto done;
+ }
+ else
+ uio_drv_name = "vfio-pci";
+ }
+ else
+ {
+ /* device is not bound to IOMMU group so we have multiple options */
+ if (vfio_pci_loaded &&
+ (error = clib_sysfs_write (sysfs_mod_vfio_noiommu, "Y")) == 0)
+ uio_drv_name = "vfio-pci";
+ else if (directory_exists ("/sys/module/uio_pci_generic"))
+ uio_drv_name = "uio_pci_generic";
+ else if (directory_exists ("/sys/module/igb_uio"))
+ uio_drv_name = "igb_uio";
+ else
+ {
+ clib_error_free (error);
+ error = clib_error_return (0, "Skipping PCI device %U: missing "
+ "kernel VFIO or UIO driver",
+ format_vlib_pci_addr, addr);
+ goto done;
+ }
+ clib_error_free (error);
+ }
+ }
s = format (s, "%v/driver%c", dev_dir_name, 0);
driver_name = clib_sysfs_link_to_name ((char *) s);
/* walk trough all linux interfaces and if interface belonging to
this device is founf check if interface is admin up */
dir = opendir ("/sys/class/net");
- s = format (s, "%U%c", format_vlib_pci_addr, &d->bus_address, 0);
+ s = format (s, "%U%c", format_vlib_pci_addr, addr, 0);
if (!dir)
{
error = clib_error_return (0, "Skipping PCI device %U: failed to "
"read /sys/class/net",
- format_vlib_pci_addr, &d->bus_address);
+ format_vlib_pci_addr, addr);
goto done;
}
if (e->d_name[0] == '.') /* skip . and .. */
continue;
- memset (&ifr, 0, sizeof ifr);
- memset (&drvinfo, 0, sizeof drvinfo);
+ clib_memset (&ifr, 0, sizeof ifr);
+ clib_memset (&drvinfo, 0, sizeof drvinfo);
ifr.ifr_data = (char *) &drvinfo;
- strncpy (ifr.ifr_name, e->d_name, IFNAMSIZ - 1);
+ clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
+
drvinfo.cmd = ETHTOOL_GDRVINFO;
if (ioctl (fd, SIOCETHTOOL, &ifr) < 0)
{
if (strcmp ((char *) s, drvinfo.bus_info))
continue;
- memset (&ifr, 0, sizeof (ifr));
- strncpy (ifr.ifr_name, e->d_name, IFNAMSIZ - 1);
+ clib_memset (&ifr, 0, sizeof (ifr));
+ clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
+
if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
{
error = clib_error_return_unix (0, "ioctl fetch intf %s flags",
{
error = clib_error_return (0, "Skipping PCI device %U as host "
"interface %s is up",
- format_vlib_pci_addr, &d->bus_address,
- e->d_name);
+ format_vlib_pci_addr, addr, e->d_name);
close (fd);
goto done;
}
vec_reset_length (s);
s = format (s, "%v/driver/unbind%c", dev_dir_name, 0);
- clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address);
+ clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr);
vec_reset_length (s);
s = format (s, "%v/driver_override%c", dev_dir_name, 0);
if (access ((char *) s, F_OK) == 0)
{
- clib_sysfs_write ((char *) s, "%s", uio_driver_name);
+ clib_sysfs_write ((char *) s, "%s", uio_drv_name);
clear_driver_override = 1;
}
else
{
vec_reset_length (s);
- s = format (s, "/sys/bus/pci/drivers/%s/new_id%c", uio_driver_name, 0);
- clib_sysfs_write ((char *) s, "0x%04x 0x%04x", d->vendor_id,
- d->device_id);
+ s = format (s, "%s/%s/new_id%c", sysfs_pci_drv_path, uio_drv_name, 0);
+ clib_sysfs_write ((char *) s, "0x%04x 0x%04x", di->vendor_id,
+ di->device_id);
}
vec_reset_length (s);
- s = format (s, "/sys/bus/pci/drivers/%s/bind%c", uio_driver_name, 0);
- clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address);
+ s = format (s, "%s/%s/bind%c", sysfs_pci_drv_path, uio_drv_name, 0);
+ clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr);
vec_reset_length (s);
if (clear_driver_override)
return 0;
}
+static clib_error_t *
+vfio_set_irqs (vlib_main_t * vm, linux_pci_device_t * p, u32 index, u32 start,
+ u32 count, u32 flags, int *efds)
+{
+ int data_len = efds ? count * sizeof (int) : 0;
+ u8 buf[sizeof (struct vfio_irq_set) + data_len];
+ struct vfio_irq_info ii = { 0 };
+ struct vfio_irq_set *irq_set = (struct vfio_irq_set *) buf;
+
+
+ ii.argsz = sizeof (struct vfio_irq_info);
+ ii.index = index;
+
+ if (ioctl (p->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
+ return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) "
+ "'%U'", format_vlib_pci_addr, &p->addr);
+
+ pci_log_debug (vm, p, "%s index:%u count:%u flags: %s%s%s%s(0x%x)",
+ __func__, ii.index, ii.count,
+ ii.flags & VFIO_IRQ_INFO_EVENTFD ? "eventfd " : "",
+ ii.flags & VFIO_IRQ_INFO_MASKABLE ? "maskable " : "",
+ ii.flags & VFIO_IRQ_INFO_AUTOMASKED ? "automasked " : "",
+ ii.flags & VFIO_IRQ_INFO_NORESIZE ? "noresize " : "",
+ ii.flags);
+
+ if (ii.count < start + count)
+ return clib_error_return_unix (0, "vfio_set_irq: unexistng interrupt on "
+ "'%U'", format_vlib_pci_addr, &p->addr);
+
+
+ if (efds)
+ {
+ flags |= VFIO_IRQ_SET_DATA_EVENTFD;
+ clib_memcpy_fast (&irq_set->data, efds, data_len);
+ }
+ else
+ flags |= VFIO_IRQ_SET_DATA_NONE;
+
+ ASSERT ((flags & (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD)) !=
+ (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD));
+
+ irq_set->argsz = sizeof (struct vfio_irq_set) + data_len;
+ irq_set->index = index;
+ irq_set->start = start;
+ irq_set->count = count;
+ irq_set->flags = flags;
+
+ if (ioctl (p->fd, VFIO_DEVICE_SET_IRQS, irq_set) < 0)
+ return clib_error_return_unix (0, "%U:ioctl(VFIO_DEVICE_SET_IRQS) "
+ "[index = %u, start = %u, count = %u, "
+ "flags = 0x%x]",
+ format_vlib_pci_addr, &p->addr,
+ index, start, count, flags);
+ return 0;
+}
+
static clib_error_t *
linux_pci_uio_read_ready (clib_file_t * uf)
{
- vlib_pci_main_t *pm = &pci_main;
- vlib_pci_device_t *d;
+ vlib_main_t *vm = vlib_get_main ();
int __attribute__ ((unused)) rv;
+ vlib_pci_dev_handle_t h = uf->private_data;
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ linux_pci_irq_t *irq = &p->intx_irq;
u32 icount;
rv = read (uf->file_descriptor, &icount, 4);
- d = pool_elt_at_index (pm->pci_devs, uf->private_data);
-
- if (d->interrupt_handler)
- d->interrupt_handler (d);
+ if (irq->intx_handler)
+ irq->intx_handler (vm, h);
- vlib_pci_intr_enable (d);
+ vlib_pci_intr_enable (vm, h);
return /* no error */ 0;
}
+static clib_error_t *
+linux_pci_vfio_unmask_intx (vlib_main_t * vm, linux_pci_device_t * d)
+{
+ return vfio_set_irqs (vm, d, VFIO_PCI_INTX_IRQ_INDEX, 0, 1,
+ VFIO_IRQ_SET_ACTION_UNMASK, 0);
+}
+
static clib_error_t *
linux_pci_uio_error_ready (clib_file_t * uf)
{
return clib_error_return (0, "pci device %d: error", error_index);
}
-static void
-add_device (vlib_pci_device_t * dev, linux_pci_device_t * pdev)
+static clib_error_t *
+linux_pci_vfio_msix_read_ready (clib_file_t * uf)
{
- vlib_pci_main_t *pm = &pci_main;
- linux_pci_main_t *lpm = &linux_pci_main;
- linux_pci_device_t *l;
+ vlib_main_t *vm = vlib_get_main ();
+ int __attribute__ ((unused)) rv;
+ vlib_pci_dev_handle_t h = uf->private_data >> 16;
+ u16 line = uf->private_data & 0xffff;
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, line);
- pool_get (lpm->linux_pci_devices, l);
- l[0] = pdev[0];
+ u64 icount;
+ rv = read (uf->file_descriptor, &icount, sizeof (icount));
- l->dev_dir_name = vec_dup (l->dev_dir_name);
+ if (irq->msix_handler)
+ irq->msix_handler (vm, h, line);
- dev->os_handle = l - lpm->linux_pci_devices;
+ return /* no error */ 0;
+}
- {
- u8 *uio_dir = format (0, "%s/uio", l->dev_dir_name);
- foreach_directory_file ((char *) uio_dir, scan_uio_dir, l, /* scan_dirs */
- 1);
- vec_free (uio_dir);
- }
+static clib_error_t *
+linux_pci_vfio_intx_read_ready (clib_file_t * uf)
+{
+ vlib_main_t *vm = vlib_get_main ();
+ int __attribute__ ((unused)) rv;
+ vlib_pci_dev_handle_t h = uf->private_data;
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ linux_pci_irq_t *irq = &p->intx_irq;
- {
- char *uio_name = (char *) format (0, "/dev/uio%d%c", l->uio_minor, 0);
- l->uio_fd = open (uio_name, O_RDWR);
- if (l->uio_fd < 0)
- clib_unix_error ("open `%s'", uio_name);
- vec_free (uio_name);
- }
+ u64 icount;
+ rv = read (uf->file_descriptor, &icount, sizeof (icount));
- {
- clib_file_t template = { 0 };
+ if (irq->intx_handler)
+ irq->intx_handler (vm, h);
- template.read_function = linux_pci_uio_read_ready;
- template.file_descriptor = l->uio_fd;
- template.error_function = linux_pci_uio_error_ready;
- template.private_data = dev - pm->pci_devs;
+ linux_pci_vfio_unmask_intx (vm, p);
- l->clib_file_index = clib_file_add (&file_main, &template);
- }
+ return /* no error */ 0;
}
-static void
-linux_pci_device_free (linux_pci_device_t * l)
+static clib_error_t *
+linux_pci_vfio_error_ready (clib_file_t * uf)
{
- int i;
- for (i = 0; i < vec_len (l->resource_fds); i++)
- if (l->resource_fds[i] > 0)
- close (l->resource_fds[i]);
- if (l->config_fd > 0)
- close (l->config_fd);
- if (l->uio_fd > 0)
- close (l->uio_fd);
- vec_free (l->resource_fds);
- vec_free (l->dev_dir_name);
+ u32 error_index = (u32) uf->private_data;
+
+ return clib_error_return (0, "pci device %d: error", error_index);
}
-/* Configuration space read/write. */
-clib_error_t *
-vlib_pci_read_write_config (vlib_pci_device_t * dev,
- vlib_read_or_write_t read_or_write,
- uword address, void *data, u32 n_bytes)
+static clib_error_t *
+add_device_uio (vlib_main_t * vm, linux_pci_device_t * p,
+ vlib_pci_device_info_t * di, pci_device_registration_t * r)
{
linux_pci_main_t *lpm = &linux_pci_main;
- linux_pci_device_t *p;
- int n;
+ clib_error_t *err = 0;
+ u8 *s = 0;
- p = pool_elt_at_index (lpm->linux_pci_devices, dev->os_handle);
+ p->fd = -1;
+ p->type = LINUX_PCI_DEVICE_TYPE_UIO;
- if (read_or_write == VLIB_READ)
- n = pread (p->config_fd, data, n_bytes, address);
- else
- n = pwrite (p->config_fd, data, n_bytes, address);
-
- if (n != n_bytes)
- return clib_error_return_unix (0, "%s",
- read_or_write == VLIB_READ
- ? "read" : "write");
+ s = format (s, "%s/%U/config%c", sysfs_pci_dev_path,
+ format_vlib_pci_addr, &di->addr, 0);
- return 0;
-}
+ p->config_fd = open ((char *) s, O_RDWR);
+ p->config_offset = 0;
+ vec_reset_length (s);
-static clib_error_t *
-os_map_pci_resource_internal (uword os_handle,
- u32 resource, u8 * addr, void **result)
-{
- linux_pci_main_t *pm = &linux_pci_main;
- linux_pci_device_t *p;
- struct stat stat_buf;
- u8 *file_name;
- int fd;
- clib_error_t *error;
- int flags = MAP_SHARED;
+ if (p->config_fd == -1)
+ {
+ err = clib_error_return_unix (0, "open '%s'", s);
+ goto error;
+ }
- error = 0;
- p = pool_elt_at_index (pm->linux_pci_devices, os_handle);
+ s = format (0, "%s/%U/uio%c", sysfs_pci_dev_path,
+ format_vlib_pci_addr, &di->addr, 0);
+ foreach_directory_file ((char *) s, scan_uio_dir, p, /* scan_dirs */
+ 1);
+ vec_reset_length (s);
- file_name = format (0, "%v/resource%d%c", p->dev_dir_name, resource, 0);
- fd = open ((char *) file_name, O_RDWR);
- if (fd < 0)
+ s = format (s, "/dev/uio%d%c", p->uio_minor, 0);
+ p->fd = open ((char *) s, O_RDWR);
+ if (p->fd < 0)
{
- error = clib_error_return_unix (0, "open `%s'", file_name);
- goto done;
+ err = clib_error_return_unix (0, "open '%s'", s);
+ goto error;
}
- if (fstat (fd, &stat_buf) < 0)
+ if (r && r->interrupt_handler)
+ vlib_pci_register_intx_handler (vm, p->handle, r->interrupt_handler);
+
+ if (r && r->init_function)
+ err = r->init_function (lpm->vlib_main, p->handle);
+
+error:
+ vec_free (s);
+ if (err)
{
- error = clib_error_return_unix (0, "fstat `%s'", file_name);
- goto done;
+ if (p->config_fd != -1)
+ close (p->config_fd);
+ if (p->fd != -1)
+ close (p->fd);
}
+ return err;
+}
- vec_validate (p->resource_fds, resource);
- p->resource_fds[resource] = fd;
- if (addr != 0)
- flags |= MAP_FIXED;
+clib_error_t *
+vlib_pci_register_intx_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ pci_intx_handler_function_t * intx_handler)
+{
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ clib_file_t t = { 0 };
+ linux_pci_irq_t *irq = &p->intx_irq;
+ ASSERT (irq->fd == -1);
- *result = mmap (addr,
- /* size */ stat_buf.st_size,
- PROT_READ | PROT_WRITE, flags,
- /* file */ fd,
- /* offset */ 0);
- if (*result == (void *) -1)
+ if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
{
- error = clib_error_return_unix (0, "mmap `%s'", file_name);
- goto done;
+ struct vfio_irq_info ii = { 0 };
+ ii.argsz = sizeof (struct vfio_irq_info);
+ ii.index = VFIO_PCI_INTX_IRQ_INDEX;
+ if (ioctl (p->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
+ return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) '"
+ "%U'", format_vlib_pci_addr, &p->addr);
+ pci_log_debug (vm, p, "%s index:%u count:%u flags: %s%s%s%s(0x%x)",
+ __func__, ii.index, ii.count,
+ ii.flags & VFIO_IRQ_INFO_EVENTFD ? "eventfd " : "",
+ ii.flags & VFIO_IRQ_INFO_MASKABLE ? "maskable " : "",
+ ii.flags & VFIO_IRQ_INFO_AUTOMASKED ? "automasked " : "",
+ ii.flags & VFIO_IRQ_INFO_NORESIZE ? "noresize " : "",
+ ii.flags);
+ if (ii.count != 1)
+ return clib_error_return (0, "INTx interrupt does not exist on device"
+ "'%U'", format_vlib_pci_addr, &p->addr);
+
+ irq->fd = eventfd (0, EFD_NONBLOCK);
+ if (irq->fd == -1)
+ return clib_error_return_unix (0, "eventfd");
+
+ t.file_descriptor = irq->fd;
+ t.read_function = linux_pci_vfio_intx_read_ready;
}
-
-done:
- if (error)
+ else if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
{
- if (fd >= 0)
- close (fd);
+ t.file_descriptor = p->fd;
+ t.read_function = linux_pci_uio_read_ready;
}
- vec_free (file_name);
- return error;
+ else
+ return 0;
+
+ t.error_function = linux_pci_uio_error_ready;
+ t.private_data = p->handle;
+ t.description = format (0, "PCI %U INTx", format_vlib_pci_addr, &p->addr);
+ irq->clib_file_index = clib_file_add (&file_main, &t);
+ irq->intx_handler = intx_handler;
+ return 0;
}
clib_error_t *
-vlib_pci_map_resource (vlib_pci_device_t * dev, u32 resource, void **result)
+vlib_pci_register_msix_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ u32 start, u32 count,
+ pci_msix_handler_function_t * msix_handler)
{
- return (os_map_pci_resource_internal
- (dev->os_handle, resource, 0 /* addr */ ,
- result));
+ clib_error_t *err = 0;
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ u32 i;
+
+ if (p->type != LINUX_PCI_DEVICE_TYPE_VFIO)
+ return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
+ "support");
+
+ /* *INDENT-OFF* */
+ vec_validate_init_empty (p->msix_irqs, start + count - 1, (linux_pci_irq_t)
+ { .fd = -1});
+ /* *INDENT-ON* */
+
+ for (i = start; i < start + count; i++)
+ {
+ clib_file_t t = { 0 };
+ linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, i);
+ ASSERT (irq->fd == -1);
+
+ irq->fd = eventfd (0, EFD_NONBLOCK);
+ if (irq->fd == -1)
+ {
+ err = clib_error_return_unix (0, "eventfd");
+ goto error;
+ }
+
+ t.read_function = linux_pci_vfio_msix_read_ready;
+ t.file_descriptor = irq->fd;
+ t.error_function = linux_pci_vfio_error_ready;
+ t.private_data = p->handle << 16 | i;
+ t.description = format (0, "PCI %U MSI-X #%u", format_vlib_pci_addr,
+ &p->addr, i);
+ irq->clib_file_index = clib_file_add (&file_main, &t);
+ irq->msix_handler = msix_handler;
+ }
+
+ return 0;
+
+error:
+ while (i-- > start)
+ {
+ linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, i);
+ if (irq->fd != -1)
+ {
+ clib_file_del_by_index (&file_main, irq->clib_file_index);
+ close (irq->fd);
+ irq->fd = -1;
+ }
+ }
+ return err;
}
clib_error_t *
-vlib_pci_map_resource_fixed (vlib_pci_device_t * dev,
- u32 resource, u8 * addr, void **result)
+vlib_pci_enable_msix_irq (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ u16 start, u16 count)
{
- return (os_map_pci_resource_internal
- (dev->os_handle, resource, addr, result));
-}
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ int fds[count];
+ int i;
-void
-vlib_pci_free_device (vlib_pci_device_t * dev)
-{
- linux_pci_main_t *pm = &linux_pci_main;
- linux_pci_device_t *l;
+ if (p->type != LINUX_PCI_DEVICE_TYPE_VFIO)
+ return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
+ "support");
+
+ for (i = start; i < start + count; i++)
+ {
+ linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, i);
+ fds[i] = irq->fd;
+ }
- l = pool_elt_at_index (pm->linux_pci_devices, dev->os_handle);
- linux_pci_device_free (l);
- pool_put (pm->linux_pci_devices, l);
+ return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
+ VFIO_IRQ_SET_ACTION_TRIGGER, fds);
}
-pci_device_registration_t * __attribute__ ((unused))
-pci_device_next_registered (pci_device_registration_t * r)
+clib_error_t *
+vlib_pci_disable_msix_irq (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ u16 start, u16 count)
{
- uword i;
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ int i, fds[count];
+
+ if (p->type != LINUX_PCI_DEVICE_TYPE_VFIO)
+ return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
+ "support");
- /* Null vendor id marks end of initialized list. */
- for (i = 0; r->supported_devices[i].vendor_id != 0; i++)
- ;
+ for (i = start; i < start + count; i++)
+ fds[i] = -1;
- return clib_elf_section_data_next (r, i * sizeof (r->supported_devices[0]));
+ return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
+ VFIO_IRQ_SET_ACTION_TRIGGER, fds);
}
static clib_error_t *
-init_device_from_registered (vlib_main_t * vm,
- vlib_pci_device_t * dev,
- linux_pci_device_t * pdev)
+add_device_vfio (vlib_main_t * vm, linux_pci_device_t * p,
+ vlib_pci_device_info_t * di, pci_device_registration_t * r)
{
- vlib_pci_main_t *pm = &pci_main;
- pci_device_registration_t *r;
- pci_device_id_t *i;
- clib_error_t *error;
+ linux_pci_main_t *lpm = &linux_pci_main;
+ struct vfio_device_info device_info = { 0 };
+ struct vfio_region_info reg = { 0 };
+ clib_error_t *err = 0;
+ u8 *s = 0;
+ int is_noiommu;
- r = pm->pci_device_registrations;
+ p->type = LINUX_PCI_DEVICE_TYPE_VFIO;
- while (r)
+ if ((err = linux_vfio_group_get_device_fd (&p->addr, &p->fd, &is_noiommu)))
+ return err;
+
+ if (is_noiommu == 0)
+ p->supports_va_dma = 1;
+
+ device_info.argsz = sizeof (device_info);
+ if (ioctl (p->fd, VFIO_DEVICE_GET_INFO, &device_info) < 0)
{
- for (i = r->supported_devices; i->vendor_id != 0; i++)
- if (i->vendor_id == dev->vendor_id && i->device_id == dev->device_id)
- {
- error = vlib_pci_bind_to_uio (dev, "uio_pci_generic");
- if (error)
- {
- clib_error_report (error);
- continue;
- }
+ err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'",
+ format_vlib_pci_addr, &di->addr);
+ goto error;
+ }
- add_device (dev, pdev);
- dev->interrupt_handler = r->interrupt_handler;
- return r->init_function (vm, dev);
- }
- r = r->next_registration;
+ reg.argsz = sizeof (struct vfio_region_info);
+ reg.index = VFIO_PCI_CONFIG_REGION_INDEX;
+ if (ioctl (p->fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
+ {
+ err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'",
+ format_vlib_pci_addr, &di->addr);
+ goto error;
}
- /* No driver, close the PCI config-space FD */
- close (pdev->config_fd);
- return 0;
+
+ pci_log_debug (vm, p, "%s region_info index:%u size:0x%lx offset:0x%lx "
+ "flags: %s%s%s(0x%x)", __func__,
+ reg.index, reg.size, reg.offset,
+ reg.flags & VFIO_REGION_INFO_FLAG_READ ? "rd " : "",
+ reg.flags & VFIO_REGION_INFO_FLAG_WRITE ? "wr " : "",
+ reg.flags & VFIO_REGION_INFO_FLAG_MMAP ? "mmap " : "",
+ reg.flags);
+
+ p->config_offset = reg.offset;
+ p->config_fd = p->fd;
+
+ /* reset if device supports it */
+ if (device_info.flags & VFIO_DEVICE_FLAGS_RESET)
+ if (ioctl (p->fd, VFIO_DEVICE_RESET) < 0)
+ {
+ err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_RESET) '%U'",
+ format_vlib_pci_addr, &di->addr);
+ goto error;
+ }
+
+ if (r && r->interrupt_handler)
+ {
+ vlib_pci_register_intx_handler (vm, p->handle, r->interrupt_handler);
+ linux_pci_vfio_unmask_intx (vm, p);
+ }
+
+ if (p->supports_va_dma)
+ {
+ vlib_buffer_pool_t *bp;
+ /* *INDENT-OFF* */
+ vec_foreach (bp, vm->buffer_main->buffer_pools)
+ {
+ u32 i;
+ vlib_physmem_map_t *pm;
+ pm = vlib_physmem_get_map (vm, bp->physmem_map_index);
+ for (i = 0; i < pm->n_pages; i++)
+ vfio_map_physmem_page (vm, pm->base + (i << pm->log2_page_size));
+ }
+ /* *INDENT-ON* */
+ }
+
+ if (r && r->init_function)
+ err = r->init_function (lpm->vlib_main, p->handle);
+
+error:
+ vec_free (s);
+ if (err)
+ {
+ if (p->fd != -1)
+ close (p->fd);
+ if (p->config_fd != -1 && p->config_fd != p->fd)
+ close (p->config_fd);
+ p->config_fd = p->fd = -1;
+ }
+ return err;
}
-static clib_error_t *
-init_device (vlib_main_t * vm,
- vlib_pci_device_t * dev, linux_pci_device_t * pdev)
+/* Configuration space read/write. */
+clib_error_t *
+vlib_pci_read_write_config (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ vlib_read_or_write_t read_or_write,
+ uword address, void *data, u32 n_bytes)
{
- return init_device_from_registered (vm, dev, pdev);
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ int n;
+
+ if (read_or_write == VLIB_READ)
+ n = pread (p->config_fd, data, n_bytes, p->config_offset + address);
+ else
+ n = pwrite (p->config_fd, data, n_bytes, p->config_offset + address);
+
+ if (n != n_bytes)
+ return clib_error_return_unix (0, "%s",
+ read_or_write == VLIB_READ
+ ? "read" : "write");
+
+ return 0;
}
static clib_error_t *
-scan_device (void *arg, u8 * dev_dir_name, u8 * ignored)
+vlib_pci_region (vlib_main_t * vm, vlib_pci_dev_handle_t h, u32 bar, int *fd,
+ u64 * size, u64 * offset)
{
- vlib_main_t *vm = arg;
- vlib_pci_main_t *pm = &pci_main;
- int fd;
- u8 *f;
+ linux_pci_device_t *p = linux_pci_get_device (h);
clib_error_t *error = 0;
- vlib_pci_device_t *dev;
- linux_pci_device_t pdev = { 0 };
- u32 tmp;
+ int _fd = -1;
+ u64 _size = 0, _offset = 0;
- f = format (0, "%v/config%c", dev_dir_name, 0);
- fd = open ((char *) f, O_RDWR);
+ ASSERT (bar <= 5);
- /* Try read-only access if write fails. */
- if (fd < 0)
- fd = open ((char *) f, O_RDONLY);
+ error = 0;
- if (fd < 0)
+ if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
{
- error = clib_error_return_unix (0, "open `%s'", f);
- goto done;
- }
+ u8 *file_name;
+ struct stat stat_buf;
+ file_name = format (0, "%s/%U/resource%d%c", sysfs_pci_dev_path,
+ format_vlib_pci_addr, &p->addr, bar, 0);
- pool_get (pm->pci_devs, dev);
+ _fd = open ((char *) file_name, O_RDWR);
+ if (_fd < 0)
+ {
+ error = clib_error_return_unix (0, "open `%s'", file_name);
+ vec_free (file_name);
+ return error;
+ }
- /* You can only read more that 64 bytes of config space as root; so we try to
- read the full space but fall back to just the first 64 bytes. */
- if (read (fd, &dev->config_data, sizeof (dev->config_data)) !=
- sizeof (dev->config_data)
- && read (fd, &dev->config0,
- sizeof (dev->config0)) != sizeof (dev->config0))
+ if (fstat (_fd, &stat_buf) < 0)
+ {
+ error = clib_error_return_unix (0, "fstat `%s'", file_name);
+ vec_free (file_name);
+ close (_fd);
+ return error;
+ }
+
+ vec_free (file_name);
+ _size = stat_buf.st_size;
+ _offset = 0;
+ }
+ else if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
{
- pool_put (pm->pci_devs, dev);
- error = clib_error_return_unix (0, "read `%s'", f);
- close (fd);
- goto done;
+ struct vfio_region_info reg = { 0 };
+ reg.argsz = sizeof (struct vfio_region_info);
+ reg.index = bar;
+ if (ioctl (p->fd, VFIO_DEVICE_GET_REGION_INFO, ®) < 0)
+ return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) "
+ "'%U'", format_vlib_pci_addr,
+ &p->addr);
+ _fd = p->fd;
+ _size = reg.size;
+ _offset = reg.offset;
+ pci_log_debug (vm, p, "%s region_info index:%u size:0x%lx offset:0x%lx "
+ "flags: %s%s%s(0x%x)", __func__,
+ reg.index, reg.size, reg.offset,
+ reg.flags & VFIO_REGION_INFO_FLAG_READ ? "rd " : "",
+ reg.flags & VFIO_REGION_INFO_FLAG_WRITE ? "wr " : "",
+ reg.flags & VFIO_REGION_INFO_FLAG_MMAP ? "mmap " : "",
+ reg.flags);
}
+ else
+ ASSERT (0);
- {
- static pci_config_header_t all_ones;
- if (all_ones.vendor_id == 0)
- memset (&all_ones, ~0, sizeof (all_ones));
+ *fd = _fd;
+ *size = _size;
+ *offset = _offset;
- if (!memcmp (&dev->config0.header, &all_ones, sizeof (all_ones)))
- {
- pool_put (pm->pci_devs, dev);
- error = clib_error_return (0, "invalid PCI config for `%s'", f);
+ return error;
+}
+
+static clib_error_t *
+vlib_pci_map_region_int (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ u32 bar, u8 * addr, void **result)
+{
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ int fd = -1;
+ clib_error_t *error;
+ int flags = MAP_SHARED;
+ u64 size = 0, offset = 0;
+
+ pci_log_debug (vm, p, "map region %u to va %p", bar, addr);
+
+ if ((error = vlib_pci_region (vm, h, bar, &fd, &size, &offset)))
+ return error;
+
+ if (p->type == LINUX_PCI_DEVICE_TYPE_UIO && addr != 0)
+ flags |= MAP_FIXED;
+
+ *result = mmap (addr, size, PROT_READ | PROT_WRITE, flags, fd, offset);
+ if (*result == (void *) -1)
+ {
+ error = clib_error_return_unix (0, "mmap `BAR%u'", bar);
+ if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
close (fd);
- goto done;
- }
- }
+ return error;
+ }
+
+ /* *INDENT-OFF* */
+ vec_validate_init_empty (p->regions, bar,
+ (linux_pci_region_t) { .fd = -1});
+ /* *INDENT-ON* */
+ if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
+ p->regions[bar].fd = fd;
+ p->regions[bar].addr = *result;
+ p->regions[bar].size = size;
+ return 0;
+}
+
+clib_error_t *
+vlib_pci_map_region (vlib_main_t * vm, vlib_pci_dev_handle_t h, u32 resource,
+ void **result)
+{
+ return (vlib_pci_map_region_int (vm, h, resource, 0 /* addr */ , result));
+}
+
+clib_error_t *
+vlib_pci_map_region_fixed (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ u32 resource, u8 * addr, void **result)
+{
+ return (vlib_pci_map_region_int (vm, h, resource, addr, result));
+}
+
+clib_error_t *
+vlib_pci_io_region (vlib_main_t * vm, vlib_pci_dev_handle_t h, u32 resource)
+{
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ clib_error_t *error = 0;
+ int fd = -1;
+ u64 size = 0, offset = 0;
+
+ if ((error = vlib_pci_region (vm, h, resource, &fd, &size, &offset)))
+ return error;
- if (dev->config0.header.header_type == 0)
- pci_config_type0_little_to_host (&dev->config0);
+ p->io_fd = fd;
+ p->io_offset = offset;
+ return error;
+}
+
+clib_error_t *
+vlib_pci_read_write_io (vlib_main_t * vm, vlib_pci_dev_handle_t h,
+ vlib_read_or_write_t read_or_write,
+ uword offset, void *data, u32 length)
+{
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ int n = 0;
+
+ if (read_or_write == VLIB_READ)
+ n = pread (p->io_fd, data, length, p->io_offset + offset);
else
- pci_config_type1_little_to_host (&dev->config1);
+ n = pwrite (p->io_fd, data, length, p->io_offset + offset);
- /* Parse bus, dev, function from directory name. */
- {
- unformat_input_t input;
+ if (n != length)
+ return clib_error_return_unix (0, "%s",
+ read_or_write == VLIB_READ
+ ? "read" : "write");
+ return 0;
+}
- unformat_init_string (&input, (char *) dev_dir_name,
- vec_len (dev_dir_name));
+clib_error_t *
+vlib_pci_map_dma (vlib_main_t * vm, vlib_pci_dev_handle_t h, void *ptr)
+{
+ linux_pci_device_t *p = linux_pci_get_device (h);
- if (!unformat (&input, "/sys/bus/pci/devices/%U",
- unformat_vlib_pci_addr, &dev->bus_address))
- abort ();
+ if (!p->supports_va_dma)
+ return 0;
- unformat_free (&input);
+ return vfio_map_physmem_page (vm, ptr);
+}
- }
+int
+vlib_pci_supports_virtual_addr_dma (vlib_main_t * vm, vlib_pci_dev_handle_t h)
+{
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ return p->supports_va_dma != 0;
+}
- pdev.config_fd = fd;
- pdev.dev_dir_name = dev_dir_name;
+clib_error_t *
+vlib_pci_device_open (vlib_main_t * vm, vlib_pci_addr_t * addr,
+ pci_device_id_t ids[], vlib_pci_dev_handle_t * handle)
+{
+ linux_pci_main_t *lpm = &linux_pci_main;
+ vlib_pci_device_info_t *di;
+ linux_pci_device_t *p;
+ clib_error_t *err = 0;
+ pci_device_id_t *i;
- hash_set (pm->pci_dev_index_by_pci_addr, dev->bus_address.as_u32,
- dev - pm->pci_devs);
+ di = vlib_pci_get_device_info (vm, addr, &err);
+
+ if (err)
+ return err;
+ for (i = ids; i->vendor_id != 0; i++)
+ if (i->vendor_id == di->vendor_id && i->device_id == di->device_id)
+ break;
+
+ if (i->vendor_id == 0)
+ return clib_error_return (0, "Wrong vendor or device id");
+
+ pool_get (lpm->linux_pci_devices, p);
+ p->handle = p - lpm->linux_pci_devices;
+ p->addr.as_u32 = di->addr.as_u32;
+ p->intx_irq.fd = -1;
+ p->intx_irq.clib_file_index = -1;
+ p->numa_node = di->numa_node;
+ /*
+ * pci io bar read/write fd
+ */
+ p->io_fd = -1;
+
+ pci_log_debug (vm, p, "open vid:0x%04x did:0x%04x driver:%s iommu_group:%d",
+ di->vendor_id, di->device_id, di->driver_name,
+ di->iommu_group);
+
+ if (clib_strncmp ("vfio-pci", (char *) di->driver_name, 8) == 0)
+ err = add_device_vfio (vm, p, di, 0);
+ else if (clib_strncmp ("uio_pci_generic", (char *) di->driver_name, 8) == 0)
+ err = add_device_uio (vm, p, di, 0);
+ else
+ err = clib_error_create ("device not bound to 'vfio-pci' or "
+ "'uio_pci_generic' kernel module");
+ if (err)
+ goto error;
- vec_reset_length (f);
- f = format (f, "%v/vpd%c", dev_dir_name, 0);
- fd = open ((char *) f, O_RDONLY);
- if (fd >= 0)
+ *handle = p->handle;
+
+error:
+ vlib_pci_free_device_info (di);
+ if (err)
{
- while (1)
- {
- u8 tag[3];
- u8 *data = 0;
- int len;
+ pci_log_err (vm, p, "%U", format_clib_error, err);
+ clib_memset (p, 0, sizeof (linux_pci_device_t));
+ pool_put (lpm->linux_pci_devices, p);
+ }
- if (read (fd, &tag, 3) != 3)
- break;
+ return err;
+}
- if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91)
- break;
+void
+vlib_pci_device_close (vlib_main_t * vm, vlib_pci_dev_handle_t h)
+{
+ linux_pci_main_t *lpm = &linux_pci_main;
+ linux_pci_device_t *p = linux_pci_get_device (h);
+ linux_pci_irq_t *irq;
+ linux_pci_region_t *res;
+ clib_error_t *err = 0;
- len = (tag[2] << 8) | tag[1];
- vec_validate (data, len);
+ if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
+ {
+ irq = &p->intx_irq;
+ if (irq->clib_file_index != -1)
+ clib_file_del_by_index (&file_main, irq->clib_file_index);
+ close (p->config_fd);
+ if (p->io_fd != -1)
+ close (p->io_fd);
+ }
+ else if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
+ {
+ irq = &p->intx_irq;
+ /* close INTx irqs */
+ if (irq->fd != -1)
+ {
+ err = vfio_set_irqs (vm, p, VFIO_PCI_INTX_IRQ_INDEX, 0, 0,
+ VFIO_IRQ_SET_ACTION_TRIGGER, 0);
+ clib_error_free (err);
+ if (irq->clib_file_index != -1)
+ clib_file_del_by_index (&file_main, irq->clib_file_index);
+ close (irq->fd);
+ }
- if (read (fd, data, len) != len)
+ /* close MSI-X irqs */
+ if (vec_len (p->msix_irqs))
+ {
+ err = vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, 0, 0,
+ VFIO_IRQ_SET_ACTION_TRIGGER, 0);
+ clib_error_free (err);
+ /* *INDENT-OFF* */
+ vec_foreach (irq, p->msix_irqs)
{
- vec_free (data);
- break;
+ if (irq->fd == -1)
+ continue;
+ clib_file_del_by_index (&file_main, irq->clib_file_index);
+ close (irq->fd);
}
- if (tag[0] == 0x82)
- dev->product_name = data;
- else if (tag[0] == 0x90)
- dev->vpd_r = data;
- else if (tag[0] == 0x91)
- dev->vpd_w = data;
-
- data = 0;
+ /* *INDENT-ON* */
+ vec_free (p->msix_irqs);
}
- close (fd);
}
- dev->numa_node = -1;
- vec_reset_length (f);
- f = format (f, "%v/numa_node%c", dev_dir_name, 0);
- clib_sysfs_read ((char *) f, "%u", &dev->numa_node);
+ /* *INDENT-OFF* */
+ vec_foreach (res, p->regions)
+ {
+ if (res->size == 0)
+ continue;
+ munmap (res->addr, res->size);
+ if (res->fd != -1)
+ close (res->fd);
+ }
+ /* *INDENT-ON* */
+ vec_free (p->regions);
- vec_reset_length (f);
- f = format (f, "%v/class%c", dev_dir_name, 0);
- clib_sysfs_read ((char *) f, "0x%x", &tmp);
- dev->device_class = tmp >> 8;
+ close (p->fd);
+ clib_memset (p, 0, sizeof (linux_pci_device_t));
+ pool_put (lpm->linux_pci_devices, p);
+}
- vec_reset_length (f);
- f = format (f, "%v/vendor%c", dev_dir_name, 0);
- clib_sysfs_read ((char *) f, "0x%x", &tmp);
- dev->vendor_id = tmp;
+void
+init_device_from_registered (vlib_main_t * vm, vlib_pci_device_info_t * di)
+{
+ vlib_pci_main_t *pm = &pci_main;
+ linux_pci_main_t *lpm = &linux_pci_main;
+ pci_device_registration_t *r;
+ pci_device_id_t *i;
+ clib_error_t *err = 0;
+ linux_pci_device_t *p;
- vec_reset_length (f);
- f = format (f, "%v/device%c", dev_dir_name, 0);
- clib_sysfs_read ((char *) f, "0x%x", &tmp);
- dev->device_id = tmp;
+ pool_get (lpm->linux_pci_devices, p);
+ p->handle = p - lpm->linux_pci_devices;
+ p->intx_irq.fd = -1;
- error = init_device (vm, dev, &pdev);
+ r = pm->pci_device_registrations;
- vec_reset_length (f);
- f = format (f, "%v/driver%c", dev_dir_name, 0);
- dev->driver_name = clib_sysfs_link_to_name ((char *) f);
+ while (r)
+ {
+ for (i = r->supported_devices; i->vendor_id != 0; i++)
+ if (i->vendor_id == di->vendor_id && i->device_id == di->device_id)
+ {
+ if (di->iommu_group != -1)
+ err = add_device_vfio (vm, p, di, r);
+ else
+ err = add_device_uio (vm, p, di, r);
+
+ if (err)
+ clib_error_report (err);
+ else
+ return;
+ }
+ r = r->next_registration;
+ }
-done:
- vec_free (f);
- return error;
+ /* No driver, close the PCI config-space FD */
+ clib_memset (p, 0, sizeof (linux_pci_device_t));
+ pool_put (lpm->linux_pci_devices, p);
+}
+
+static clib_error_t *
+scan_pci_addr (void *arg, u8 * dev_dir_name, u8 * ignored)
+{
+ vlib_pci_addr_t addr, **addrv = arg;
+ unformat_input_t input;
+ clib_error_t *err = 0;
+
+ unformat_init_string (&input, (char *) dev_dir_name,
+ vec_len (dev_dir_name));
+
+ if (!unformat (&input, "/sys/bus/pci/devices/%U",
+ unformat_vlib_pci_addr, &addr))
+ err = clib_error_return (0, "unformat error `%v`", dev_dir_name);
+
+ unformat_free (&input);
+
+ if (err)
+ return err;
+
+ vec_add1 (*addrv, addr);
+ return 0;
+}
+
+static int
+pci_addr_cmp (void *v1, void *v2)
+{
+ vlib_pci_addr_t *a1 = v1;
+ vlib_pci_addr_t *a2 = v2;
+
+ if (a1->domain > a2->domain)
+ return 1;
+ if (a1->domain < a2->domain)
+ return -1;
+ if (a1->bus > a2->bus)
+ return 1;
+ if (a1->bus < a2->bus)
+ return -1;
+ if (a1->slot > a2->slot)
+ return 1;
+ if (a1->slot < a2->slot)
+ return -1;
+ if (a1->function > a2->function)
+ return 1;
+ if (a1->function < a2->function)
+ return -1;
+ return 0;
+}
+
+vlib_pci_addr_t *
+vlib_pci_get_all_dev_addrs ()
+{
+ vlib_pci_addr_t *addrs = 0;
+ clib_error_t *err;
+ err = foreach_directory_file ((char *) sysfs_pci_dev_path, scan_pci_addr,
+ &addrs, /* scan_dirs */ 0);
+ if (err)
+ {
+ vec_free (addrs);
+ return 0;
+ }
+
+ vec_sort_with_function (addrs, pci_addr_cmp);
+
+ return addrs;
}
clib_error_t *
linux_pci_init (vlib_main_t * vm)
{
vlib_pci_main_t *pm = &pci_main;
- clib_error_t *error;
+ vlib_pci_addr_t *addr = 0, *addrs;
pm->vlib_main = vm;
- if ((error = vlib_call_init_function (vm, unix_input_init)))
- return error;
-
ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32));
- pm->pci_dev_index_by_pci_addr = hash_create (0, sizeof (uword));
-
- error = foreach_directory_file ("/sys/bus/pci/devices", scan_device, vm,
- /* scan_dirs */ 0);
- /* Complain and continue. might not be root, etc. */
- if (error)
- clib_error_report (error);
+ addrs = vlib_pci_get_all_dev_addrs ();
+ /* *INDENT-OFF* */
+ vec_foreach (addr, addrs)
+ {
+ vlib_pci_device_info_t *d;
+ if ((d = vlib_pci_get_device_info (vm, addr, 0)))
+ {
+ init_device_from_registered (vm, d);
+ vlib_pci_free_device_info (d);
+ }
+ }
+ /* *INDENT-ON* */
- return error;
+ return 0;
}
-VLIB_INIT_FUNCTION (linux_pci_init);
+/* *INDENT-OFF* */
+VLIB_INIT_FUNCTION (linux_pci_init) =
+{
+ .runs_after = VLIB_INITS("unix_input_init"),
+};
+/* *INDENT-ON* */
/*
* fd.io coding-style-patch-verification: ON
Code Review / vpp.git / blobdiff