blob: 3ff2c0b075df1349ecf2f86c1036877a1ac583c0 [file] [log] [blame]
#include "kvm/kvm.h"
#include "kvm/vfio.h"
#include "kvm/ioport.h"
#include <linux/list.h>
#define VFIO_DEV_DIR "/dev/vfio"
#define VFIO_DEV_NODE VFIO_DEV_DIR "/vfio"
#define IOMMU_GROUP_DIR "/sys/kernel/iommu_groups"
static int vfio_container;
static LIST_HEAD(vfio_groups);
static struct vfio_device *vfio_devices;
static int vfio_device_pci_parser(const struct option *opt, char *arg,
struct vfio_device_params *dev)
{
unsigned int domain, bus, devnr, fn;
int nr = sscanf(arg, "%4x:%2x:%2x.%1x", &domain, &bus, &devnr, &fn);
if (nr < 4) {
domain = 0;
nr = sscanf(arg, "%2x:%2x.%1x", &bus, &devnr, &fn);
if (nr < 3) {
pr_err("Invalid device identifier %s", arg);
return -EINVAL;
}
}
dev->type = VFIO_DEVICE_PCI;
dev->bus = "pci";
dev->name = malloc(13);
if (!dev->name)
return -ENOMEM;
snprintf(dev->name, 13, "%04x:%02x:%02x.%x", domain, bus, devnr, fn);
return 0;
}
int vfio_device_parser(const struct option *opt, const char *arg, int unset)
{
int ret = -EINVAL;
static int idx = 0;
struct kvm *kvm = opt->ptr;
struct vfio_device_params *dev, *devs;
char *cur, *buf = strdup(arg);
if (!buf)
return -ENOMEM;
if (idx >= MAX_VFIO_DEVICES) {
pr_warning("Too many VFIO devices");
goto out_free_buf;
}
devs = realloc(kvm->cfg.vfio_devices, sizeof(*dev) * (idx + 1));
if (!devs) {
ret = -ENOMEM;
goto out_free_buf;
}
kvm->cfg.vfio_devices = devs;
dev = &devs[idx];
cur = strtok(buf, ",");
if (!cur)
goto out_free_buf;
if (!strcmp(opt->long_name, "vfio-pci"))
ret = vfio_device_pci_parser(opt, cur, dev);
else
ret = -EINVAL;
if (!ret)
kvm->cfg.num_vfio_devices = ++idx;
out_free_buf:
free(buf);
return ret;
}
static bool vfio_ioport_in(struct vfio_region *region, u32 offset,
void *data, int len)
{
struct vfio_device *vdev = region->vdev;
ssize_t nr;
u32 val;
if (!(region->info.flags & VFIO_REGION_INFO_FLAG_READ))
return false;
nr = pread(vdev->fd, &val, len, region->info.offset + offset);
if (nr != len) {
vfio_dev_err(vdev, "could not read %d bytes from I/O port 0x%x\n",
len, offset + region->port_base);
return false;
}
switch (len) {
case 1:
ioport__write8(data, val);
break;
case 2:
ioport__write16(data, val);
break;
case 4:
ioport__write32(data, val);
break;
default:
return false;
}
return true;
}
static bool vfio_ioport_out(struct vfio_region *region, u32 offset,
void *data, int len)
{
struct vfio_device *vdev = region->vdev;
ssize_t nr;
u32 val;
if (!(region->info.flags & VFIO_REGION_INFO_FLAG_WRITE))
return false;
switch (len) {
case 1:
val = ioport__read8(data);
break;
case 2:
val = ioport__read16(data);
break;
case 4:
val = ioport__read32(data);
break;
default:
return false;
}
nr = pwrite(vdev->fd, &val, len, region->info.offset + offset);
if (nr != len)
vfio_dev_err(vdev, "could not write %d bytes to I/O port 0x%x",
len, offset + region->port_base);
return nr == len;
}
static void vfio_ioport_mmio(struct kvm_cpu *vcpu, u64 addr, u8 *data, u32 len,
u8 is_write, void *ptr)
{
struct vfio_region *region = ptr;
u32 offset = addr - region->port_base;
if (is_write)
vfio_ioport_out(region, offset, data, len);
else
vfio_ioport_in(region, offset, data, len);
}
static void vfio_mmio_access(struct kvm_cpu *vcpu, u64 addr, u8 *data, u32 len,
u8 is_write, void *ptr)
{
u64 val;
ssize_t nr;
struct vfio_region *region = ptr;
struct vfio_device *vdev = region->vdev;
u32 offset = addr - region->guest_phys_addr;
if (len < 1 || len > 8)
goto err_report;
if (is_write) {
if (!(region->info.flags & VFIO_REGION_INFO_FLAG_WRITE))
goto err_report;
memcpy(&val, data, len);
nr = pwrite(vdev->fd, &val, len, region->info.offset + offset);
if ((u32)nr != len)
goto err_report;
} else {
if (!(region->info.flags & VFIO_REGION_INFO_FLAG_READ))
goto err_report;
nr = pread(vdev->fd, &val, len, region->info.offset + offset);
if ((u32)nr != len)
goto err_report;
memcpy(data, &val, len);
}
return;
err_report:
vfio_dev_err(vdev, "could not %s %u bytes at 0x%x (0x%llx)", is_write ?
"write" : "read", len, offset, addr);
}
static int vfio_setup_trap_region(struct kvm *kvm, struct vfio_device *vdev,
struct vfio_region *region)
{
if (region->is_ioport) {
int port;
port = kvm__register_pio(kvm, region->port_base,
region->info.size, vfio_ioport_mmio,
region);
if (port < 0)
return port;
return 0;
}
return kvm__register_mmio(kvm, region->guest_phys_addr,
region->info.size, false, vfio_mmio_access,
region);
}
int vfio_map_region(struct kvm *kvm, struct vfio_device *vdev,
struct vfio_region *region)
{
void *base;
int ret, prot = 0;
/* KVM needs page-aligned regions */
u64 map_size = ALIGN(region->info.size, PAGE_SIZE);
if (!(region->info.flags & VFIO_REGION_INFO_FLAG_MMAP))
return vfio_setup_trap_region(kvm, vdev, region);
/*
* KVM_SET_USER_MEMORY_REGION will fail because the guest physical
* address isn't page aligned, let's emulate the region ourselves.
*/
if (region->guest_phys_addr & (PAGE_SIZE - 1))
return kvm__register_mmio(kvm, region->guest_phys_addr,
region->info.size, false,
vfio_mmio_access, region);
if (region->info.flags & VFIO_REGION_INFO_FLAG_READ)
prot |= PROT_READ;
if (region->info.flags & VFIO_REGION_INFO_FLAG_WRITE)
prot |= PROT_WRITE;
base = mmap(NULL, region->info.size, prot, MAP_SHARED, vdev->fd,
region->info.offset);
if (base == MAP_FAILED) {
/* TODO: support sparse mmap */
vfio_dev_warn(vdev, "failed to mmap region %u (0x%llx bytes), falling back to trapping",
region->info.index, region->info.size);
return vfio_setup_trap_region(kvm, vdev, region);
}
region->host_addr = base;
ret = kvm__register_dev_mem(kvm, region->guest_phys_addr, map_size,
region->host_addr);
if (ret) {
vfio_dev_err(vdev, "failed to register region with KVM");
return ret;
}
return 0;
}
void vfio_unmap_region(struct kvm *kvm, struct vfio_region *region)
{
u64 map_size;
if (region->host_addr) {
map_size = ALIGN(region->info.size, PAGE_SIZE);
kvm__destroy_mem(kvm, region->guest_phys_addr, map_size,
region->host_addr);
munmap(region->host_addr, region->info.size);
region->host_addr = NULL;
} else if (region->is_ioport) {
kvm__deregister_pio(kvm, region->port_base);
} else {
kvm__deregister_mmio(kvm, region->guest_phys_addr);
}
}
static int vfio_configure_device(struct kvm *kvm, struct vfio_device *vdev)
{
int ret;
struct vfio_group *group = vdev->group;
vdev->fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD,
vdev->params->name);
if (vdev->fd < 0) {
vfio_dev_warn(vdev, "failed to get fd");
/* The device might be a bridge without an fd */
return 0;
}
vdev->info.argsz = sizeof(vdev->info);
if (ioctl(vdev->fd, VFIO_DEVICE_GET_INFO, &vdev->info)) {
ret = -errno;
vfio_dev_err(vdev, "failed to get info");
goto err_close_device;
}
if (vdev->info.flags & VFIO_DEVICE_FLAGS_RESET &&
ioctl(vdev->fd, VFIO_DEVICE_RESET) < 0)
vfio_dev_warn(vdev, "failed to reset device");
vdev->regions = calloc(vdev->info.num_regions, sizeof(*vdev->regions));
if (!vdev->regions) {
ret = -ENOMEM;
goto err_close_device;
}
/* Now for the bus-specific initialization... */
switch (vdev->params->type) {
case VFIO_DEVICE_PCI:
BUG_ON(!(vdev->info.flags & VFIO_DEVICE_FLAGS_PCI));
ret = vfio_pci_setup_device(kvm, vdev);
break;
default:
BUG_ON(1);
ret = -EINVAL;
}
if (ret)
goto err_free_regions;
vfio_dev_info(vdev, "assigned to device number 0x%x in group %lu",
vdev->dev_hdr.dev_num, group->id);
return 0;
err_free_regions:
free(vdev->regions);
err_close_device:
close(vdev->fd);
return ret;
}
static int vfio_configure_devices(struct kvm *kvm)
{
int i, ret;
for (i = 0; i < kvm->cfg.num_vfio_devices; ++i) {
ret = vfio_configure_device(kvm, &vfio_devices[i]);
if (ret)
return ret;
}
return 0;
}
static int vfio_get_iommu_type(void)
{
if (ioctl(vfio_container, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU))
return VFIO_TYPE1v2_IOMMU;
if (ioctl(vfio_container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU))
return VFIO_TYPE1_IOMMU;
return -ENODEV;
}
static int vfio_map_mem_bank(struct kvm *kvm, struct kvm_mem_bank *bank, void *data)
{
int ret = 0;
struct vfio_iommu_type1_dma_map dma_map = {
.argsz = sizeof(dma_map),
.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
.vaddr = (unsigned long)bank->host_addr,
.iova = (u64)bank->guest_phys_addr,
.size = bank->size,
};
/* Map the guest memory for DMA (i.e. provide isolation) */
if (ioctl(vfio_container, VFIO_IOMMU_MAP_DMA, &dma_map)) {
ret = -errno;
pr_err("Failed to map 0x%llx -> 0x%llx (%llu) for DMA",
dma_map.iova, dma_map.vaddr, dma_map.size);
}
return ret;
}
static int vfio_unmap_mem_bank(struct kvm *kvm, struct kvm_mem_bank *bank, void *data)
{
struct vfio_iommu_type1_dma_unmap dma_unmap = {
.argsz = sizeof(dma_unmap),
.size = bank->size,
.iova = bank->guest_phys_addr,
};
ioctl(vfio_container, VFIO_IOMMU_UNMAP_DMA, &dma_unmap);
return 0;
}
static int vfio_configure_reserved_regions(struct kvm *kvm,
struct vfio_group *group)
{
FILE *file;
int ret = 0;
char type[9];
char filename[PATH_MAX];
unsigned long long start, end;
snprintf(filename, PATH_MAX, IOMMU_GROUP_DIR "/%lu/reserved_regions",
group->id);
/* reserved_regions might not be present on older systems */
if (access(filename, F_OK))
return 0;
file = fopen(filename, "r");
if (!file)
return -errno;
while (fscanf(file, "0x%llx 0x%llx %8s\n", &start, &end, type) == 3) {
ret = kvm__reserve_mem(kvm, start, end - start + 1);
if (ret)
break;
}
fclose(file);
return ret;
}
static int vfio_configure_groups(struct kvm *kvm)
{
int ret;
struct vfio_group *group;
list_for_each_entry(group, &vfio_groups, list) {
ret = vfio_configure_reserved_regions(kvm, group);
if (ret)
return ret;
}
return 0;
}
static struct vfio_group *vfio_group_create(struct kvm *kvm, unsigned long id)
{
int ret;
struct vfio_group *group;
char group_node[PATH_MAX];
struct vfio_group_status group_status = {
.argsz = sizeof(group_status),
};
group = calloc(1, sizeof(*group));
if (!group)
return NULL;
group->id = id;
group->refs = 1;
ret = snprintf(group_node, PATH_MAX, VFIO_DEV_DIR "/%lu", id);
if (ret < 0 || ret == PATH_MAX)
return NULL;
group->fd = open(group_node, O_RDWR);
if (group->fd < 0) {
pr_err("Failed to open IOMMU group %s", group_node);
goto err_free_group;
}
if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &group_status)) {
pr_err("Failed to determine status of IOMMU group %lu", id);
goto err_close_group;
}
if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
pr_err("IOMMU group %lu is not viable", id);
goto err_close_group;
}
if (ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &vfio_container)) {
pr_err("Failed to add IOMMU group %lu to VFIO container", id);
goto err_close_group;
}
list_add(&group->list, &vfio_groups);
return group;
err_close_group:
close(group->fd);
err_free_group:
free(group);
return NULL;
}
static void vfio_group_exit(struct kvm *kvm, struct vfio_group *group)
{
if (--group->refs != 0)
return;
ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER);
list_del(&group->list);
close(group->fd);
free(group);
}
static struct vfio_group *
vfio_group_get_for_dev(struct kvm *kvm, struct vfio_device *vdev)
{
int dirfd;
ssize_t ret;
char *group_name;
unsigned long group_id;
char group_path[PATH_MAX];
struct vfio_group *group = NULL;
/* Find IOMMU group for this device */
dirfd = open(vdev->sysfs_path, O_DIRECTORY | O_PATH | O_RDONLY);
if (dirfd < 0) {
vfio_dev_err(vdev, "failed to open '%s'", vdev->sysfs_path);
return NULL;
}
ret = readlinkat(dirfd, "iommu_group", group_path, PATH_MAX);
if (ret < 0) {
vfio_dev_err(vdev, "no iommu_group");
goto out_close;
}
if (ret == PATH_MAX)
goto out_close;
group_path[ret] = '\0';
group_name = basename(group_path);
errno = 0;
group_id = strtoul(group_name, NULL, 10);
if (errno)
goto out_close;
list_for_each_entry(group, &vfio_groups, list) {
if (group->id == group_id) {
group->refs++;
return group;
}
}
group = vfio_group_create(kvm, group_id);
out_close:
close(dirfd);
return group;
}
static int vfio_device_init(struct kvm *kvm, struct vfio_device *vdev)
{
int ret;
char dev_path[PATH_MAX];
struct vfio_group *group;
ret = snprintf(dev_path, PATH_MAX, "/sys/bus/%s/devices/%s",
vdev->params->bus, vdev->params->name);
if (ret < 0 || ret == PATH_MAX)
return -EINVAL;
vdev->sysfs_path = strndup(dev_path, PATH_MAX);
if (!vdev->sysfs_path)
return -errno;
group = vfio_group_get_for_dev(kvm, vdev);
if (!group) {
free(vdev->sysfs_path);
return -EINVAL;
}
vdev->group = group;
return 0;
}
static void vfio_device_exit(struct kvm *kvm, struct vfio_device *vdev)
{
vfio_group_exit(kvm, vdev->group);
switch (vdev->params->type) {
case VFIO_DEVICE_PCI:
vfio_pci_teardown_device(kvm, vdev);
break;
default:
vfio_dev_warn(vdev, "no teardown function for device");
}
close(vdev->fd);
free(vdev->regions);
free(vdev->sysfs_path);
}
static int vfio_container_init(struct kvm *kvm)
{
int api, i, ret, iommu_type;;
/* Create a container for our IOMMU groups */
vfio_container = open(VFIO_DEV_NODE, O_RDWR);
if (vfio_container == -1) {
ret = errno;
pr_err("Failed to open %s", VFIO_DEV_NODE);
return ret;
}
api = ioctl(vfio_container, VFIO_GET_API_VERSION);
if (api != VFIO_API_VERSION) {
pr_err("Unknown VFIO API version %d", api);
return -ENODEV;
}
iommu_type = vfio_get_iommu_type();
if (iommu_type < 0) {
pr_err("VFIO type-1 IOMMU not supported on this platform");
return iommu_type;
}
/* Create groups for our devices and add them to the container */
for (i = 0; i < kvm->cfg.num_vfio_devices; ++i) {
vfio_devices[i].params = &kvm->cfg.vfio_devices[i];
ret = vfio_device_init(kvm, &vfio_devices[i]);
if (ret)
return ret;
}
/* Finalise the container */
if (ioctl(vfio_container, VFIO_SET_IOMMU, iommu_type)) {
ret = -errno;
pr_err("Failed to set IOMMU type %d for VFIO container",
iommu_type);
return ret;
} else {
pr_info("Using IOMMU type %d for VFIO container", iommu_type);
}
return kvm__for_each_mem_bank(kvm, KVM_MEM_TYPE_RAM, vfio_map_mem_bank,
NULL);
}
static int vfio__init(struct kvm *kvm)
{
int ret;
if (!kvm->cfg.num_vfio_devices)
return 0;
vfio_devices = calloc(kvm->cfg.num_vfio_devices, sizeof(*vfio_devices));
if (!vfio_devices)
return -ENOMEM;
ret = vfio_container_init(kvm);
if (ret)
return ret;
ret = vfio_configure_groups(kvm);
if (ret)
return ret;
ret = vfio_configure_devices(kvm);
if (ret)
return ret;
return 0;
}
dev_base_init(vfio__init);
static int vfio__exit(struct kvm *kvm)
{
int i;
if (!kvm->cfg.num_vfio_devices)
return 0;
for (i = 0; i < kvm->cfg.num_vfio_devices; i++)
vfio_device_exit(kvm, &vfio_devices[i]);
free(vfio_devices);
kvm__for_each_mem_bank(kvm, KVM_MEM_TYPE_RAM, vfio_unmap_mem_bank, NULL);
close(vfio_container);
free(kvm->cfg.vfio_devices);
return 0;
}
dev_base_exit(vfio__exit);