| // SPDX-License-Identifier: GPL-2.0-only | |
| /* | |
| * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved | |
| */ | |
| #include <linux/device.h> | |
| #include <linux/module.h> | |
| #include <linux/mutex.h> | |
| #include <linux/pci.h> | |
| #include <linux/pm_runtime.h> | |
| #include <linux/types.h> | |
| #include <linux/uaccess.h> | |
| #include <linux/vfio.h> | |
| #include <linux/vfio_pci_core.h> | |
| #include <linux/virtio_pci.h> | |
| #include <linux/virtio_net.h> | |
| #include <linux/virtio_pci_admin.h> | |
| #include "common.h" | |
| VFIO_PCI_CORE_DEFINE_CALLBACKS(virtiovf, struct virtiovf_pci_core_device, | |
| core_device) | |
| static int virtiovf_pci_open_device(struct vfio_device *core_vdev) | |
| { | |
| struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev, | |
| struct virtiovf_pci_core_device, core_device.vdev); | |
| struct vfio_pci_core_device *vdev = &virtvdev->core_device; | |
| int ret; | |
| ret = vfio_pci_core_enable(vdev); | |
| if (ret) | |
| return ret; | |
| #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY | |
| ret = virtiovf_open_legacy_io(virtvdev); | |
| if (ret) { | |
| vfio_pci_core_disable(vdev); | |
| return ret; | |
| } | |
| #endif | |
| virtiovf_open_migration(virtvdev); | |
| vfio_pci_core_finish_enable(vdev); | |
| return 0; | |
| } | |
| static void virtiovf_pci_close_device(struct vfio_device *core_vdev) | |
| { | |
| struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev, | |
| struct virtiovf_pci_core_device, core_device.vdev); | |
| virtiovf_close_migration(virtvdev); | |
| vfio_pci_core_close_device(core_vdev); | |
| } | |
| #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY | |
| static int virtiovf_pci_init_device(struct vfio_device *core_vdev) | |
| { | |
| struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev, | |
| struct virtiovf_pci_core_device, core_device.vdev); | |
| int ret; | |
| ret = vfio_pci_core_init_dev(core_vdev); | |
| if (ret) | |
| return ret; | |
| /* | |
| * The vfio_device_ops.init() callback is set to virtiovf_pci_init_device() | |
| * only when legacy I/O is supported. Now, let's initialize it. | |
| */ | |
| return virtiovf_init_legacy_io(virtvdev); | |
| } | |
| #endif | |
| static void virtiovf_pci_core_release_dev(struct vfio_device *core_vdev) | |
| { | |
| #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY | |
| struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev, | |
| struct virtiovf_pci_core_device, core_device.vdev); | |
| virtiovf_release_legacy_io(virtvdev); | |
| #endif | |
| vfio_pci_core_release_dev(core_vdev); | |
| } | |
| static const struct vfio_device_ops virtiovf_vfio_pci_lm_ops = { | |
| .name = "virtio-vfio-pci-lm", | |
| .init = vfio_pci_core_init_dev, | |
| .release = virtiovf_pci_core_release_dev, | |
| .open_device = virtiovf_pci_open_device, | |
| .close_device = virtiovf_pci_close_device, | |
| .ioctl = vfio_pci_core_ioctl, | |
| .get_region_info_caps = vfio_pci_ioctl_get_region_info, | |
| .device_feature = vfio_pci_core_ioctl_feature, | |
| .read = vfio_pci_core_read, | |
| .write = vfio_pci_core_write, | |
| .mmap = vfio_pci_core_mmap, | |
| .request = vfio_pci_core_request, | |
| .match = vfio_pci_core_match, | |
| .match_token_uuid = vfio_pci_core_match_token_uuid, | |
| .bind_iommufd = vfio_iommufd_physical_bind, | |
| .unbind_iommufd = vfio_iommufd_physical_unbind, | |
| .attach_ioas = vfio_iommufd_physical_attach_ioas, | |
| .detach_ioas = vfio_iommufd_physical_detach_ioas, | |
| }; | |
| #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY | |
| static const struct vfio_device_ops virtiovf_vfio_pci_tran_lm_ops = { | |
| .name = "virtio-vfio-pci-trans-lm", | |
| .init = virtiovf_pci_init_device, | |
| .release = virtiovf_pci_core_release_dev, | |
| .open_device = virtiovf_pci_open_device, | |
| .close_device = virtiovf_pci_close_device, | |
| .ioctl = vfio_pci_core_ioctl, | |
| .get_region_info_caps = virtiovf_pci_ioctl_get_region_info, | |
| .device_feature = vfio_pci_core_ioctl_feature, | |
| .read = virtiovf_pci_core_read, | |
| .write = virtiovf_pci_core_write, | |
| .mmap = vfio_pci_core_mmap, | |
| .request = vfio_pci_core_request, | |
| .match = vfio_pci_core_match, | |
| .match_token_uuid = vfio_pci_core_match_token_uuid, | |
| .bind_iommufd = vfio_iommufd_physical_bind, | |
| .unbind_iommufd = vfio_iommufd_physical_unbind, | |
| .attach_ioas = vfio_iommufd_physical_attach_ioas, | |
| .detach_ioas = vfio_iommufd_physical_detach_ioas, | |
| }; | |
| #endif | |
| static const struct vfio_device_ops virtiovf_vfio_pci_ops = { | |
| .name = "virtio-vfio-pci", | |
| .init = vfio_pci_core_init_dev, | |
| .release = vfio_pci_core_release_dev, | |
| .open_device = virtiovf_pci_open_device, | |
| .close_device = vfio_pci_core_close_device, | |
| .ioctl = vfio_pci_core_ioctl, | |
| .get_region_info_caps = vfio_pci_ioctl_get_region_info, | |
| .device_feature = vfio_pci_core_ioctl_feature, | |
| .read = vfio_pci_core_read, | |
| .write = vfio_pci_core_write, | |
| .mmap = vfio_pci_core_mmap, | |
| .request = vfio_pci_core_request, | |
| .match = vfio_pci_core_match, | |
| .match_token_uuid = vfio_pci_core_match_token_uuid, | |
| .bind_iommufd = vfio_iommufd_physical_bind, | |
| .unbind_iommufd = vfio_iommufd_physical_unbind, | |
| .attach_ioas = vfio_iommufd_physical_attach_ioas, | |
| .detach_ioas = vfio_iommufd_physical_detach_ioas, | |
| }; | |
| static int virtiovf_pci_probe(struct pci_dev *pdev, | |
| const struct pci_device_id *id) | |
| { | |
| const struct vfio_device_ops *ops = &virtiovf_vfio_pci_ops; | |
| struct virtiovf_pci_core_device *virtvdev; | |
| bool sup_legacy_io = false; | |
| bool sup_lm = false; | |
| int ret; | |
| if (pdev->is_virtfn) { | |
| #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY | |
| sup_legacy_io = virtiovf_support_legacy_io(pdev); | |
| if (sup_legacy_io) | |
| ops = &virtiovf_vfio_pci_tran_lm_ops; | |
| #endif | |
| sup_lm = virtio_pci_admin_has_dev_parts(pdev); | |
| if (sup_lm && !sup_legacy_io) | |
| ops = &virtiovf_vfio_pci_lm_ops; | |
| } | |
| virtvdev = vfio_alloc_device(virtiovf_pci_core_device, core_device.vdev, | |
| &pdev->dev, ops); | |
| if (IS_ERR(virtvdev)) | |
| return PTR_ERR(virtvdev); | |
| if (sup_lm) | |
| virtiovf_set_migratable(virtvdev); | |
| dev_set_drvdata(&pdev->dev, virtvdev); | |
| ret = vfio_pci_core_register_device(&virtvdev->core_device); | |
| if (ret) | |
| goto out; | |
| return 0; | |
| out: | |
| vfio_put_device(&virtvdev->core_device.vdev); | |
| return ret; | |
| } | |
| static void virtiovf_pci_remove(struct pci_dev *pdev) | |
| { | |
| struct virtiovf_pci_core_device *virtvdev = dev_get_drvdata(&pdev->dev); | |
| vfio_pci_core_unregister_device(&virtvdev->core_device); | |
| vfio_put_device(&virtvdev->core_device.vdev); | |
| } | |
| static const struct pci_device_id virtiovf_pci_table[] = { | |
| /* Only virtio-net and virtio-block are supported/tested so far */ | |
| { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_REDHAT_QUMRANET, 0x1041) }, | |
| { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_REDHAT_QUMRANET, 0x1042) }, | |
| {} | |
| }; | |
| MODULE_DEVICE_TABLE(pci, virtiovf_pci_table); | |
| static void virtiovf_pci_aer_reset_done(struct pci_dev *pdev) | |
| { | |
| #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY | |
| virtiovf_legacy_io_reset_done(pdev); | |
| #endif | |
| virtiovf_migration_reset_done(pdev); | |
| } | |
| static const struct pci_error_handlers virtiovf_err_handlers = { | |
| .reset_done = virtiovf_pci_aer_reset_done, | |
| .error_detected = virtiovf_aer_err_detected, | |
| }; | |
| static struct pci_driver virtiovf_pci_driver = { | |
| .name = KBUILD_MODNAME, | |
| .id_table = virtiovf_pci_table, | |
| .probe = virtiovf_pci_probe, | |
| .remove = virtiovf_pci_remove, | |
| .driver = { .pm = pm_ptr(&virtiovf_pm_ops) }, | |
| .err_handler = &virtiovf_err_handlers, | |
| .driver_managed_dma = true, | |
| }; | |
| module_pci_driver(virtiovf_pci_driver); | |
| MODULE_LICENSE("GPL"); | |
| MODULE_AUTHOR("Yishai Hadas <yishaih@nvidia.com>"); | |
| MODULE_DESCRIPTION( | |
| "VIRTIO VFIO PCI - User Level meta-driver for VIRTIO NET and BLOCK devices"); |