| // SPDX-License-Identifier: GPL-2.0-only | |
| /* | |
| * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved | |
| */ | |
| #include <linux/bitfield.h> | |
| #include <linux/sizes.h> | |
| #include <linux/time64.h> | |
| #include <linux/vfio_pci_core.h> | |
| #include <linux/delay.h> | |
| #include <linux/jiffies.h> | |
| #include <linux/sched.h> | |
| #include <linux/pci-p2pdma.h> | |
| #include <linux/pagemap.h> | |
| #include <linux/pm_runtime.h> | |
| #include <linux/memory-failure.h> | |
| /* | |
| * The device memory usable to the workloads running in the VM is cached | |
| * and showcased as a 64b device BAR (comprising of BAR4 and BAR5 region) | |
| * to the VM and is represented as usemem. | |
| * Moreover, the VM GPU device driver needs a non-cacheable region to | |
| * support the MIG feature. This region is also exposed as a 64b BAR | |
| * (comprising of BAR2 and BAR3 region) and represented as resmem. | |
| */ | |
| #define RESMEM_REGION_INDEX VFIO_PCI_BAR2_REGION_INDEX | |
| #define USEMEM_REGION_INDEX VFIO_PCI_BAR4_REGION_INDEX | |
| /* A hardwired and constant ABI value between the GPU FW and VFIO driver. */ | |
| #define MEMBLK_SIZE SZ_512M | |
| #define DVSEC_BITMAP_OFFSET 0xA | |
| #define MIG_SUPPORTED_WITH_CACHED_RESMEM BIT(0) | |
| #define GPU_CAP_DVSEC_REGISTER 3 | |
| #define C2C_LINK_BAR0_OFFSET 0x1498 | |
| #define HBM_TRAINING_BAR0_OFFSET 0x200BC | |
| #define STATUS_READY 0xFF | |
| #define POLL_QUANTUM_MS 1000 | |
| #define POLL_TIMEOUT_MS (30 * 1000) | |
| /* | |
| * The state of the two device memory region - resmem and usemem - is | |
| * saved as struct mem_region. | |
| */ | |
| struct mem_region { | |
| phys_addr_t memphys; /* Base physical address of the region */ | |
| size_t memlength; /* Region size */ | |
| size_t bar_size; /* Reported region BAR size */ | |
| __le64 bar_val; /* Emulated BAR offset registers */ | |
| union { | |
| void *memaddr; | |
| void __iomem *ioaddr; | |
| }; /* Base virtual address of the region */ | |
| struct pfn_address_space pfn_address_space; | |
| }; | |
| struct nvgrace_gpu_pci_core_device { | |
| struct vfio_pci_core_device core_device; | |
| /* Cached and usable memory for the VM. */ | |
| struct mem_region usemem; | |
| /* Non cached memory carved out from the end of device memory */ | |
| struct mem_region resmem; | |
| /* Lock to control device memory kernel mapping */ | |
| struct mutex remap_lock; | |
| void __iomem *bar0_base; | |
| bool has_mig_hw_bug; | |
| /* GPU has just been reset */ | |
| bool reset_done; | |
| /* CXL Device DVSEC offset; 0 if not present (legacy GB path) */ | |
| int cxl_dvsec; | |
| }; | |
| VFIO_PCI_CORE_DEFINE_CALLBACKS(nvgrace_gpu, struct nvgrace_gpu_pci_core_device, | |
| core_device) | |
| static void nvgrace_gpu_init_fake_bar_emu_regs(struct vfio_device *core_vdev) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| nvdev->resmem.bar_val = 0; | |
| nvdev->usemem.bar_val = 0; | |
| } | |
| /* Choose the structure corresponding to the fake BAR with a given index. */ | |
| static struct mem_region * | |
| nvgrace_gpu_memregion(int index, | |
| struct nvgrace_gpu_pci_core_device *nvdev) | |
| { | |
| if (index == USEMEM_REGION_INDEX) | |
| return &nvdev->usemem; | |
| if (nvdev->resmem.memlength && index == RESMEM_REGION_INDEX) | |
| return &nvdev->resmem; | |
| return NULL; | |
| } | |
| static int pfn_memregion_offset(struct nvgrace_gpu_pci_core_device *nvdev, | |
| unsigned int index, | |
| unsigned long pfn, | |
| pgoff_t *pfn_offset_in_region) | |
| { | |
| struct mem_region *region; | |
| unsigned long start_pfn, num_pages; | |
| region = nvgrace_gpu_memregion(index, nvdev); | |
| if (!region) | |
| return -EINVAL; | |
| start_pfn = PHYS_PFN(region->memphys); | |
| num_pages = region->memlength >> PAGE_SHIFT; | |
| if (pfn < start_pfn || pfn >= start_pfn + num_pages) | |
| return -EFAULT; | |
| *pfn_offset_in_region = pfn - start_pfn; | |
| return 0; | |
| } | |
| static inline | |
| struct nvgrace_gpu_pci_core_device *vma_to_nvdev(struct vm_area_struct *vma); | |
| static int nvgrace_gpu_pfn_to_vma_pgoff(struct vm_area_struct *vma, | |
| unsigned long pfn, | |
| pgoff_t *pgoff) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev; | |
| unsigned int index = | |
| vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); | |
| pgoff_t vma_offset_in_region = vma->vm_pgoff & | |
| ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); | |
| pgoff_t pfn_offset_in_region; | |
| int ret; | |
| nvdev = vma_to_nvdev(vma); | |
| if (!nvdev) | |
| return -ENOENT; | |
| ret = pfn_memregion_offset(nvdev, index, pfn, &pfn_offset_in_region); | |
| if (ret) | |
| return ret; | |
| /* Ensure PFN is not before VMA's start within the region */ | |
| if (pfn_offset_in_region < vma_offset_in_region) | |
| return -EFAULT; | |
| /* Calculate offset from VMA start */ | |
| *pgoff = vma->vm_pgoff + | |
| (pfn_offset_in_region - vma_offset_in_region); | |
| return 0; | |
| } | |
| static int | |
| nvgrace_gpu_vfio_pci_register_pfn_range(struct vfio_device *core_vdev, | |
| struct mem_region *region) | |
| { | |
| unsigned long pfn, nr_pages; | |
| pfn = PHYS_PFN(region->memphys); | |
| nr_pages = region->memlength >> PAGE_SHIFT; | |
| region->pfn_address_space.node.start = pfn; | |
| region->pfn_address_space.node.last = pfn + nr_pages - 1; | |
| region->pfn_address_space.mapping = core_vdev->inode->i_mapping; | |
| region->pfn_address_space.pfn_to_vma_pgoff = nvgrace_gpu_pfn_to_vma_pgoff; | |
| return register_pfn_address_space(®ion->pfn_address_space); | |
| } | |
| static int nvgrace_gpu_open_device(struct vfio_device *core_vdev) | |
| { | |
| struct vfio_pci_core_device *vdev = | |
| container_of(core_vdev, struct vfio_pci_core_device, vdev); | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| void __iomem *io; | |
| int ret; | |
| ret = vfio_pci_core_enable(vdev); | |
| if (ret) | |
| return ret; | |
| nvgrace_gpu_init_fake_bar_emu_regs(core_vdev); | |
| mutex_init(&nvdev->remap_lock); | |
| /* | |
| * GPU readiness is checked by reading the BAR0 registers. | |
| * The BAR map was just set up by vfio_pci_core_enable(), so | |
| * bail early if that wasn't successful: | |
| */ | |
| io = vfio_pci_core_get_iomap(vdev, 0); | |
| if (IS_ERR(io)) { | |
| ret = PTR_ERR(io); | |
| goto error_exit; | |
| } | |
| if (nvdev->resmem.memlength) { | |
| ret = nvgrace_gpu_vfio_pci_register_pfn_range(core_vdev, &nvdev->resmem); | |
| if (ret && ret != -EOPNOTSUPP) | |
| goto error_exit; | |
| } | |
| ret = nvgrace_gpu_vfio_pci_register_pfn_range(core_vdev, &nvdev->usemem); | |
| if (ret && ret != -EOPNOTSUPP) | |
| goto register_mem_failed; | |
| vfio_pci_core_finish_enable(vdev); | |
| nvdev->bar0_base = io; | |
| return 0; | |
| register_mem_failed: | |
| if (nvdev->resmem.memlength) | |
| unregister_pfn_address_space(&nvdev->resmem.pfn_address_space); | |
| error_exit: | |
| vfio_pci_core_disable(vdev); | |
| return ret; | |
| } | |
| static void nvgrace_gpu_close_device(struct vfio_device *core_vdev) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| nvdev->bar0_base = NULL; | |
| if (nvdev->resmem.memlength) | |
| unregister_pfn_address_space(&nvdev->resmem.pfn_address_space); | |
| unregister_pfn_address_space(&nvdev->usemem.pfn_address_space); | |
| /* Unmap the mapping to the device memory cached region */ | |
| if (nvdev->usemem.memaddr) { | |
| memunmap(nvdev->usemem.memaddr); | |
| nvdev->usemem.memaddr = NULL; | |
| } | |
| /* Unmap the mapping to the device memory non-cached region */ | |
| if (nvdev->resmem.ioaddr) { | |
| iounmap(nvdev->resmem.ioaddr); | |
| nvdev->resmem.ioaddr = NULL; | |
| } | |
| mutex_destroy(&nvdev->remap_lock); | |
| vfio_pci_core_close_device(core_vdev); | |
| } | |
| static int nvgrace_gpu_wait_device_ready_legacy(void __iomem *io) | |
| { | |
| unsigned long timeout = jiffies + msecs_to_jiffies(POLL_TIMEOUT_MS); | |
| do { | |
| if ((ioread32(io + C2C_LINK_BAR0_OFFSET) == STATUS_READY) && | |
| (ioread32(io + HBM_TRAINING_BAR0_OFFSET) == STATUS_READY)) | |
| return 0; | |
| if (schedule_timeout_killable(msecs_to_jiffies(POLL_QUANTUM_MS))) | |
| return -EINTR; | |
| } while (!time_after(jiffies, timeout)); | |
| return -ETIME; | |
| } | |
| /* | |
| * Decode the 3-bit Memory_Active_Timeout field from CXL DVSEC Range 1 Low | |
| * (bits 15:13) into milliseconds. Encoding per CXL spec r4.0 sec 8.1.3.8.2: | |
| * 000b = 1s, 001b = 4s, 010b = 16s, 011b = 64s, 100b = 256s, | |
| * 101b-111b = reserved (clamped to 256s). | |
| */ | |
| static inline unsigned long cxl_mem_active_timeout_ms(u8 timeout) | |
| { | |
| return MSEC_PER_SEC << (2 * min_t(u8, timeout, 4)); | |
| } | |
| /* | |
| * Check if CXL DVSEC reports memory as valid and active. | |
| */ | |
| static inline bool cxl_dvsec_mem_is_active(u32 status) | |
| { | |
| return (status & PCI_DVSEC_CXL_MEM_INFO_VALID) && | |
| (status & PCI_DVSEC_CXL_MEM_ACTIVE); | |
| } | |
| static int nvgrace_gpu_test_device_ready_cxl(struct nvgrace_gpu_pci_core_device *nvdev, | |
| u32 *status) | |
| { | |
| struct pci_dev *pdev = nvdev->core_device.pdev; | |
| int cxl_dvsec = nvdev->cxl_dvsec; | |
| u32 val; | |
| pci_read_config_dword(pdev, | |
| cxl_dvsec + PCI_DVSEC_CXL_RANGE_SIZE_LOW(0), | |
| &val); | |
| if (val == ~0U) | |
| return -ENODEV; | |
| if (status) | |
| *status = val; | |
| if (cxl_dvsec_mem_is_active(val)) | |
| return 0; | |
| return -EAGAIN; | |
| } | |
| /* | |
| * As per CXL spec r4.0 sec 8.1.3.8.2, MEM_INFO_VALID needs to be set | |
| * within 1s and MEM_ACTIVE within Memory_Active_Timeout (up to ~256s) | |
| * after reset and bootup. | |
| */ | |
| static int nvgrace_gpu_wait_device_ready_cxl(struct nvgrace_gpu_pci_core_device *nvdev) | |
| { | |
| unsigned long deadline = jiffies + msecs_to_jiffies(POLL_QUANTUM_MS); | |
| bool active_phase = false; | |
| u32 status; | |
| int ret; | |
| for (;;) { | |
| ret = nvgrace_gpu_test_device_ready_cxl(nvdev, &status); | |
| if (ret != -EAGAIN) | |
| return ret; | |
| if (!active_phase && (status & PCI_DVSEC_CXL_MEM_INFO_VALID)) { | |
| u8 t = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE_TIMEOUT, status); | |
| deadline = jiffies + | |
| msecs_to_jiffies(cxl_mem_active_timeout_ms(t)); | |
| active_phase = true; | |
| } | |
| if (time_after(jiffies, deadline)) | |
| return -ETIME; | |
| if (schedule_timeout_killable(msecs_to_jiffies(POLL_QUANTUM_MS))) | |
| return -EINTR; | |
| } | |
| } | |
| /* | |
| * If the GPU memory is accessed by the CPU while the GPU is not ready | |
| * after reset, it can cause harmless corrected RAS events to be logged. | |
| * Make sure the GPU is ready before establishing the mappings. | |
| * | |
| * Since the CXL polling wait could take 256s, it happens outside | |
| * memory_lock. Only do quick readiness check under the lock. Legacy | |
| * keeps the in-lock poll. | |
| */ | |
| static int | |
| nvgrace_gpu_check_device_ready(struct nvgrace_gpu_pci_core_device *nvdev) | |
| { | |
| struct vfio_pci_core_device *vdev = &nvdev->core_device; | |
| int ret; | |
| lockdep_assert_held_read(&vdev->memory_lock); | |
| if (!nvdev->reset_done) | |
| return 0; | |
| if (!__vfio_pci_memory_enabled(vdev)) | |
| return -EIO; | |
| if (nvdev->cxl_dvsec) | |
| ret = nvgrace_gpu_test_device_ready_cxl(nvdev, NULL); | |
| else | |
| ret = nvgrace_gpu_wait_device_ready_legacy(nvdev->bar0_base); | |
| if (ret) | |
| return ret; | |
| nvdev->reset_done = false; | |
| return 0; | |
| } | |
| static unsigned long addr_to_pgoff(struct vm_area_struct *vma, | |
| unsigned long addr) | |
| { | |
| u64 pgoff = vma->vm_pgoff & | |
| ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); | |
| return linear_page_delta(vma, addr) + pgoff; | |
| } | |
| static vm_fault_t nvgrace_gpu_vfio_pci_huge_fault(struct vm_fault *vmf, | |
| unsigned int order) | |
| { | |
| struct vm_area_struct *vma = vmf->vma; | |
| struct nvgrace_gpu_pci_core_device *nvdev = vma->vm_private_data; | |
| struct vfio_pci_core_device *vdev = &nvdev->core_device; | |
| unsigned int index = | |
| vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); | |
| vm_fault_t ret = VM_FAULT_FALLBACK; | |
| struct mem_region *memregion; | |
| unsigned long pfn, addr; | |
| memregion = nvgrace_gpu_memregion(index, nvdev); | |
| if (!memregion) | |
| return VM_FAULT_SIGBUS; | |
| addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order); | |
| pfn = PHYS_PFN(memregion->memphys) + addr_to_pgoff(vma, addr); | |
| if (is_aligned_for_order(vma, addr, pfn, order)) { | |
| /* | |
| * Exit early under memory_lock to avoid a potentially lengthy | |
| * device readiness wait on a runtime-suspended device. Any | |
| * race after the lock is dropped is benign as the re-check | |
| * inside the scoped guard below catches it. | |
| */ | |
| scoped_guard(rwsem_read, &vdev->memory_lock) { | |
| if (vdev->pm_runtime_engaged) | |
| return VM_FAULT_SIGBUS; | |
| } | |
| retry: | |
| if (nvdev->cxl_dvsec && READ_ONCE(nvdev->reset_done) && | |
| nvgrace_gpu_wait_device_ready_cxl(nvdev)) | |
| return VM_FAULT_SIGBUS; | |
| scoped_guard(rwsem_read, &vdev->memory_lock) { | |
| int rc; | |
| if (vdev->pm_runtime_engaged) | |
| return VM_FAULT_SIGBUS; | |
| /* Re-run the wait if a reset raced us, not SIGBUS. */ | |
| rc = nvgrace_gpu_check_device_ready(nvdev); | |
| if (rc == -EAGAIN) | |
| goto retry; | |
| if (rc) | |
| return VM_FAULT_SIGBUS; | |
| ret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order); | |
| } | |
| } | |
| dev_dbg_ratelimited(&vdev->pdev->dev, | |
| "%s order = %d pfn 0x%lx: 0x%x\n", | |
| __func__, order, pfn, | |
| (unsigned int)ret); | |
| return ret; | |
| } | |
| static vm_fault_t nvgrace_gpu_vfio_pci_fault(struct vm_fault *vmf) | |
| { | |
| return nvgrace_gpu_vfio_pci_huge_fault(vmf, 0); | |
| } | |
| static const struct vm_operations_struct nvgrace_gpu_vfio_pci_mmap_ops = { | |
| .fault = nvgrace_gpu_vfio_pci_fault, | |
| #ifdef CONFIG_ARCH_SUPPORTS_HUGE_PFNMAP | |
| .huge_fault = nvgrace_gpu_vfio_pci_huge_fault, | |
| #endif | |
| }; | |
| static inline | |
| struct nvgrace_gpu_pci_core_device *vma_to_nvdev(struct vm_area_struct *vma) | |
| { | |
| /* Check if this VMA belongs to us */ | |
| if (vma->vm_ops != &nvgrace_gpu_vfio_pci_mmap_ops) | |
| return NULL; | |
| return vma->vm_private_data; | |
| } | |
| static int nvgrace_gpu_mmap(struct vfio_device *core_vdev, | |
| struct vm_area_struct *vma) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| struct mem_region *memregion; | |
| u64 req_len, pgoff, end; | |
| unsigned int index; | |
| index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); | |
| memregion = nvgrace_gpu_memregion(index, nvdev); | |
| if (!memregion) | |
| return vfio_pci_core_mmap(core_vdev, vma); | |
| /* | |
| * Request to mmap the BAR. Map to the CPU accessible memory on the | |
| * GPU using the memory information gathered from the system ACPI | |
| * tables. | |
| */ | |
| pgoff = vma->vm_pgoff & | |
| ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); | |
| if (check_sub_overflow(vma->vm_end, vma->vm_start, &req_len) || | |
| check_add_overflow(PFN_PHYS(pgoff), req_len, &end)) | |
| return -EOVERFLOW; | |
| /* | |
| * Check that the mapping request does not go beyond the exposed | |
| * device memory size. | |
| */ | |
| if (end > memregion->memlength) | |
| return -EINVAL; | |
| vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); | |
| /* | |
| * The carved out region of the device memory needs the NORMAL_NC | |
| * property. Communicate as such to the hypervisor. | |
| */ | |
| if (index == RESMEM_REGION_INDEX) { | |
| /* | |
| * The nvgrace-gpu module has no issues with uncontained | |
| * failures on NORMAL_NC accesses. VM_ALLOW_ANY_UNCACHED is | |
| * set to communicate to the KVM to S2 map as NORMAL_NC. | |
| * This opens up guest usage of NORMAL_NC for this mapping. | |
| */ | |
| vm_flags_set(vma, VM_ALLOW_ANY_UNCACHED); | |
| vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); | |
| } | |
| vma->vm_ops = &nvgrace_gpu_vfio_pci_mmap_ops; | |
| vma->vm_private_data = nvdev; | |
| return 0; | |
| } | |
| static int nvgrace_gpu_ioctl_get_region_info(struct vfio_device *core_vdev, | |
| struct vfio_region_info *info, | |
| struct vfio_info_cap *caps) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| struct vfio_region_info_cap_sparse_mmap *sparse; | |
| struct mem_region *memregion; | |
| u32 size; | |
| int ret; | |
| /* | |
| * Request to determine the BAR region information. Send the | |
| * GPU memory information. | |
| */ | |
| memregion = nvgrace_gpu_memregion(info->index, nvdev); | |
| if (!memregion) | |
| return vfio_pci_ioctl_get_region_info(core_vdev, info, caps); | |
| size = struct_size(sparse, areas, 1); | |
| /* | |
| * Setup for sparse mapping for the device memory. Only the | |
| * available device memory on the hardware is shown as a | |
| * mappable region. | |
| */ | |
| sparse = kzalloc(size, GFP_KERNEL); | |
| if (!sparse) | |
| return -ENOMEM; | |
| sparse->nr_areas = 1; | |
| sparse->areas[0].offset = 0; | |
| sparse->areas[0].size = memregion->memlength; | |
| sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; | |
| sparse->header.version = 1; | |
| ret = vfio_info_add_capability(caps, &sparse->header, size); | |
| kfree(sparse); | |
| if (ret) | |
| return ret; | |
| info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index); | |
| /* | |
| * The region memory size may not be power-of-2 aligned. | |
| * Given that the memory is a BAR and may not be | |
| * aligned, roundup to the next power-of-2. | |
| */ | |
| info->size = memregion->bar_size; | |
| info->flags = VFIO_REGION_INFO_FLAG_READ | | |
| VFIO_REGION_INFO_FLAG_WRITE | | |
| VFIO_REGION_INFO_FLAG_MMAP; | |
| return 0; | |
| } | |
| static long nvgrace_gpu_ioctl(struct vfio_device *core_vdev, | |
| unsigned int cmd, unsigned long arg) | |
| { | |
| switch (cmd) { | |
| case VFIO_DEVICE_IOEVENTFD: | |
| return -ENOTTY; | |
| case VFIO_DEVICE_RESET: | |
| nvgrace_gpu_init_fake_bar_emu_regs(core_vdev); | |
| fallthrough; | |
| default: | |
| return vfio_pci_core_ioctl(core_vdev, cmd, arg); | |
| } | |
| } | |
| static __le64 | |
| nvgrace_gpu_get_read_value(size_t bar_size, u64 flags, __le64 val64) | |
| { | |
| u64 tmp_val; | |
| tmp_val = le64_to_cpu(val64); | |
| tmp_val &= ~(bar_size - 1); | |
| tmp_val |= flags; | |
| return cpu_to_le64(tmp_val); | |
| } | |
| /* | |
| * Both the usable (usemem) and the reserved (resmem) device memory region | |
| * are exposed as a 64b fake device BARs in the VM. These fake BARs must | |
| * respond to the accesses on their respective PCI config space offsets. | |
| * | |
| * resmem BAR owns PCI_BASE_ADDRESS_2 & PCI_BASE_ADDRESS_3. | |
| * usemem BAR owns PCI_BASE_ADDRESS_4 & PCI_BASE_ADDRESS_5. | |
| */ | |
| static ssize_t | |
| nvgrace_gpu_read_config_emu(struct vfio_device *core_vdev, | |
| char __user *buf, size_t count, loff_t *ppos) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| u64 pos = *ppos & VFIO_PCI_OFFSET_MASK; | |
| struct mem_region *memregion = NULL; | |
| __le64 val64; | |
| size_t register_offset; | |
| loff_t copy_offset; | |
| size_t copy_count; | |
| int ret; | |
| ret = vfio_pci_core_read(core_vdev, buf, count, ppos); | |
| if (ret < 0) | |
| return ret; | |
| if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_2, | |
| sizeof(val64), | |
| ©_offset, ©_count, | |
| ®ister_offset)) | |
| memregion = nvgrace_gpu_memregion(RESMEM_REGION_INDEX, nvdev); | |
| else if (vfio_pci_core_range_intersect_range(pos, count, | |
| PCI_BASE_ADDRESS_4, | |
| sizeof(val64), | |
| ©_offset, ©_count, | |
| ®ister_offset)) | |
| memregion = nvgrace_gpu_memregion(USEMEM_REGION_INDEX, nvdev); | |
| if (memregion) { | |
| val64 = nvgrace_gpu_get_read_value(memregion->bar_size, | |
| PCI_BASE_ADDRESS_MEM_TYPE_64 | | |
| PCI_BASE_ADDRESS_MEM_PREFETCH, | |
| memregion->bar_val); | |
| if (copy_to_user(buf + copy_offset, | |
| (void *)&val64 + register_offset, copy_count)) { | |
| /* | |
| * The position has been incremented in | |
| * vfio_pci_core_read. Reset the offset back to the | |
| * starting position. | |
| */ | |
| *ppos -= count; | |
| return -EFAULT; | |
| } | |
| } | |
| return count; | |
| } | |
| static ssize_t | |
| nvgrace_gpu_write_config_emu(struct vfio_device *core_vdev, | |
| const char __user *buf, size_t count, loff_t *ppos) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| u64 pos = *ppos & VFIO_PCI_OFFSET_MASK; | |
| struct mem_region *memregion = NULL; | |
| size_t register_offset; | |
| loff_t copy_offset; | |
| size_t copy_count; | |
| if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_2, | |
| sizeof(u64), ©_offset, | |
| ©_count, ®ister_offset)) | |
| memregion = nvgrace_gpu_memregion(RESMEM_REGION_INDEX, nvdev); | |
| else if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_4, | |
| sizeof(u64), ©_offset, | |
| ©_count, ®ister_offset)) | |
| memregion = nvgrace_gpu_memregion(USEMEM_REGION_INDEX, nvdev); | |
| if (memregion) { | |
| if (copy_from_user((void *)&memregion->bar_val + register_offset, | |
| buf + copy_offset, copy_count)) | |
| return -EFAULT; | |
| *ppos += copy_count; | |
| return copy_count; | |
| } | |
| return vfio_pci_core_write(core_vdev, buf, count, ppos); | |
| } | |
| /* | |
| * Ad hoc map the device memory in the module kernel VA space. Primarily needed | |
| * as vfio does not require the userspace driver to only perform accesses through | |
| * mmaps of the vfio-pci BAR regions and such accesses should be supported using | |
| * vfio_device_ops read/write implementations. | |
| * | |
| * The usemem region is cacheable memory and hence is memremaped. | |
| * The resmem region is non-cached and is mapped using ioremap_wc (NORMAL_NC). | |
| */ | |
| static int | |
| nvgrace_gpu_map_device_mem(int index, | |
| struct nvgrace_gpu_pci_core_device *nvdev) | |
| { | |
| struct mem_region *memregion; | |
| int ret = 0; | |
| memregion = nvgrace_gpu_memregion(index, nvdev); | |
| if (!memregion) | |
| return -EINVAL; | |
| mutex_lock(&nvdev->remap_lock); | |
| if (memregion->memaddr) | |
| goto unlock; | |
| if (index == USEMEM_REGION_INDEX) | |
| memregion->memaddr = memremap(memregion->memphys, | |
| memregion->memlength, | |
| MEMREMAP_WB); | |
| else | |
| memregion->ioaddr = ioremap_wc(memregion->memphys, | |
| memregion->memlength); | |
| if (!memregion->memaddr) | |
| ret = -ENOMEM; | |
| unlock: | |
| mutex_unlock(&nvdev->remap_lock); | |
| return ret; | |
| } | |
| /* | |
| * Read the data from the device memory (mapped either through ioremap | |
| * or memremap) into the user buffer. | |
| */ | |
| static int | |
| nvgrace_gpu_map_and_read(struct nvgrace_gpu_pci_core_device *nvdev, | |
| char __user *buf, size_t mem_count, loff_t *ppos) | |
| { | |
| unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); | |
| u64 offset = *ppos & VFIO_PCI_OFFSET_MASK; | |
| int ret; | |
| if (!mem_count) | |
| return 0; | |
| /* | |
| * Handle read on the BAR regions. Map to the target device memory | |
| * physical address and copy to the request read buffer. | |
| */ | |
| ret = nvgrace_gpu_map_device_mem(index, nvdev); | |
| if (ret) | |
| return ret; | |
| if (index == USEMEM_REGION_INDEX) { | |
| if (copy_to_user(buf, | |
| (u8 *)nvdev->usemem.memaddr + offset, | |
| mem_count)) | |
| ret = -EFAULT; | |
| } else { | |
| /* | |
| * The hardware ensures that the system does not crash when | |
| * the device memory is accessed with the memory enable | |
| * turned off. It synthesizes ~0 on such read. So there is | |
| * no need to check or support the disablement/enablement of | |
| * BAR through PCI_COMMAND config space register. Pass | |
| * test_mem flag as false. | |
| */ | |
| ret = vfio_pci_core_do_io_rw(&nvdev->core_device, false, | |
| nvdev->resmem.ioaddr, | |
| buf, offset, mem_count, | |
| 0, 0, false, VFIO_PCI_IO_WIDTH_8); | |
| } | |
| return ret; | |
| } | |
| /* | |
| * Read count bytes from the device memory at an offset. The actual device | |
| * memory size (available) may not be a power-of-2. So the driver fakes | |
| * the size to a power-of-2 (reported) when exposing to a user space driver. | |
| * | |
| * Reads starting beyond the reported size generate -EINVAL; reads extending | |
| * beyond the actual device size is filled with ~0; reads extending beyond | |
| * the reported size are truncated. | |
| */ | |
| static ssize_t | |
| nvgrace_gpu_read_mem(struct nvgrace_gpu_pci_core_device *nvdev, | |
| char __user *buf, size_t count, loff_t *ppos) | |
| { | |
| struct vfio_pci_core_device *vdev = &nvdev->core_device; | |
| u64 offset = *ppos & VFIO_PCI_OFFSET_MASK; | |
| unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); | |
| struct mem_region *memregion; | |
| size_t mem_count, i; | |
| u8 val = 0xFF; | |
| int ret; | |
| /* No need to do NULL check as caller does. */ | |
| memregion = nvgrace_gpu_memregion(index, nvdev); | |
| if (offset >= memregion->bar_size) | |
| return -EINVAL; | |
| /* Clip short the read request beyond reported BAR size */ | |
| count = min(count, memregion->bar_size - (size_t)offset); | |
| /* | |
| * Determine how many bytes to be actually read from the device memory. | |
| * Read request beyond the actual device memory size is filled with ~0, | |
| * while those beyond the actual reported size is skipped. | |
| */ | |
| if (offset >= memregion->memlength) | |
| mem_count = 0; | |
| else | |
| mem_count = min(count, memregion->memlength - (size_t)offset); | |
| if (nvdev->cxl_dvsec && READ_ONCE(nvdev->reset_done)) { | |
| ret = nvgrace_gpu_wait_device_ready_cxl(nvdev); | |
| if (ret) | |
| return ret; | |
| } | |
| scoped_guard(rwsem_read, &vdev->memory_lock) { | |
| ret = nvgrace_gpu_check_device_ready(nvdev); | |
| if (ret) | |
| return ret; | |
| ret = nvgrace_gpu_map_and_read(nvdev, buf, mem_count, ppos); | |
| if (ret) | |
| return ret; | |
| } | |
| /* | |
| * Only the device memory present on the hardware is mapped, which may | |
| * not be power-of-2 aligned. A read to an offset beyond the device memory | |
| * size is filled with ~0. | |
| */ | |
| for (i = mem_count; i < count; i++) { | |
| ret = put_user(val, (unsigned char __user *)(buf + i)); | |
| if (ret) | |
| return ret; | |
| } | |
| *ppos += count; | |
| return count; | |
| } | |
| static ssize_t | |
| nvgrace_gpu_read(struct vfio_device *core_vdev, | |
| char __user *buf, size_t count, loff_t *ppos) | |
| { | |
| unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| struct vfio_pci_core_device *vdev = &nvdev->core_device; | |
| int ret; | |
| if (nvgrace_gpu_memregion(index, nvdev)) { | |
| if (pm_runtime_resume_and_get(&vdev->pdev->dev)) | |
| return -EIO; | |
| ret = nvgrace_gpu_read_mem(nvdev, buf, count, ppos); | |
| pm_runtime_put(&vdev->pdev->dev); | |
| return ret; | |
| } | |
| if (index == VFIO_PCI_CONFIG_REGION_INDEX) | |
| return nvgrace_gpu_read_config_emu(core_vdev, buf, count, ppos); | |
| return vfio_pci_core_read(core_vdev, buf, count, ppos); | |
| } | |
| /* | |
| * Write the data to the device memory (mapped either through ioremap | |
| * or memremap) from the user buffer. | |
| */ | |
| static int | |
| nvgrace_gpu_map_and_write(struct nvgrace_gpu_pci_core_device *nvdev, | |
| const char __user *buf, size_t mem_count, | |
| loff_t *ppos) | |
| { | |
| unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); | |
| loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; | |
| int ret; | |
| if (!mem_count) | |
| return 0; | |
| ret = nvgrace_gpu_map_device_mem(index, nvdev); | |
| if (ret) | |
| return ret; | |
| if (index == USEMEM_REGION_INDEX) { | |
| if (copy_from_user((u8 *)nvdev->usemem.memaddr + pos, | |
| buf, mem_count)) | |
| return -EFAULT; | |
| } else { | |
| /* | |
| * The hardware ensures that the system does not crash when | |
| * the device memory is accessed with the memory enable | |
| * turned off. It drops such writes. So there is no need to | |
| * check or support the disablement/enablement of BAR | |
| * through PCI_COMMAND config space register. Pass test_mem | |
| * flag as false. | |
| */ | |
| ret = vfio_pci_core_do_io_rw(&nvdev->core_device, false, | |
| nvdev->resmem.ioaddr, | |
| (char __user *)buf, pos, mem_count, | |
| 0, 0, true, VFIO_PCI_IO_WIDTH_8); | |
| } | |
| return ret; | |
| } | |
| /* | |
| * Write count bytes to the device memory at a given offset. The actual device | |
| * memory size (available) may not be a power-of-2. So the driver fakes the | |
| * size to a power-of-2 (reported) when exposing to a user space driver. | |
| * | |
| * Writes extending beyond the reported size are truncated; writes starting | |
| * beyond the reported size generate -EINVAL. | |
| */ | |
| static ssize_t | |
| nvgrace_gpu_write_mem(struct nvgrace_gpu_pci_core_device *nvdev, | |
| size_t count, loff_t *ppos, const char __user *buf) | |
| { | |
| struct vfio_pci_core_device *vdev = &nvdev->core_device; | |
| unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); | |
| u64 offset = *ppos & VFIO_PCI_OFFSET_MASK; | |
| struct mem_region *memregion; | |
| size_t mem_count; | |
| int ret = 0; | |
| /* No need to do NULL check as caller does. */ | |
| memregion = nvgrace_gpu_memregion(index, nvdev); | |
| if (offset >= memregion->bar_size) | |
| return -EINVAL; | |
| /* Clip short the write request beyond reported BAR size */ | |
| count = min(count, memregion->bar_size - (size_t)offset); | |
| /* | |
| * Determine how many bytes to be actually written to the device memory. | |
| * Do not write to the offset beyond available size. | |
| */ | |
| if (offset >= memregion->memlength) | |
| goto exitfn; | |
| /* | |
| * Only the device memory present on the hardware is mapped, which may | |
| * not be power-of-2 aligned. Drop access outside the available device | |
| * memory on the hardware. | |
| */ | |
| mem_count = min(count, memregion->memlength - (size_t)offset); | |
| if (nvdev->cxl_dvsec && READ_ONCE(nvdev->reset_done)) { | |
| ret = nvgrace_gpu_wait_device_ready_cxl(nvdev); | |
| if (ret) | |
| return ret; | |
| } | |
| scoped_guard(rwsem_read, &vdev->memory_lock) { | |
| ret = nvgrace_gpu_check_device_ready(nvdev); | |
| if (ret) | |
| return ret; | |
| ret = nvgrace_gpu_map_and_write(nvdev, buf, mem_count, ppos); | |
| if (ret) | |
| return ret; | |
| } | |
| exitfn: | |
| *ppos += count; | |
| return count; | |
| } | |
| static ssize_t | |
| nvgrace_gpu_write(struct vfio_device *core_vdev, | |
| const char __user *buf, size_t count, loff_t *ppos) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = | |
| container_of(core_vdev, struct nvgrace_gpu_pci_core_device, | |
| core_device.vdev); | |
| struct vfio_pci_core_device *vdev = &nvdev->core_device; | |
| unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); | |
| int ret; | |
| if (nvgrace_gpu_memregion(index, nvdev)) { | |
| if (pm_runtime_resume_and_get(&vdev->pdev->dev)) | |
| return -EIO; | |
| ret = nvgrace_gpu_write_mem(nvdev, count, ppos, buf); | |
| pm_runtime_put(&vdev->pdev->dev); | |
| return ret; | |
| } | |
| if (index == VFIO_PCI_CONFIG_REGION_INDEX) | |
| return nvgrace_gpu_write_config_emu(core_vdev, buf, count, ppos); | |
| return vfio_pci_core_write(core_vdev, buf, count, ppos); | |
| } | |
| static int nvgrace_get_dmabuf_phys(struct vfio_pci_core_device *core_vdev, | |
| struct p2pdma_provider **provider, | |
| unsigned int region_index, | |
| struct phys_vec *phys_vec, | |
| struct vfio_region_dma_range *dma_ranges, | |
| size_t nr_ranges) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = container_of( | |
| core_vdev, struct nvgrace_gpu_pci_core_device, core_device); | |
| struct pci_dev *pdev = core_vdev->pdev; | |
| struct mem_region *mem_region; | |
| /* | |
| * if (nvdev->resmem.memlength && region_index == RESMEM_REGION_INDEX) { | |
| * The P2P properties of the non-BAR memory is the same as the | |
| * BAR memory, so just use the provider for index 0. Someday | |
| * when CXL gets P2P support we could create CXLish providers | |
| * for the non-BAR memory. | |
| * } else if (region_index == USEMEM_REGION_INDEX) { | |
| * This is actually cachable memory and isn't treated as P2P in | |
| * the chip. For now we have no way to push cachable memory | |
| * through everything and the Grace HW doesn't care what caching | |
| * attribute is programmed into the SMMU. So use BAR 0. | |
| * } | |
| */ | |
| mem_region = nvgrace_gpu_memregion(region_index, nvdev); | |
| if (mem_region) { | |
| *provider = pcim_p2pdma_provider(pdev, 0); | |
| if (!*provider) | |
| return -EINVAL; | |
| return vfio_pci_core_fill_phys_vec(phys_vec, dma_ranges, | |
| nr_ranges, | |
| mem_region->memphys, | |
| mem_region->memlength); | |
| } | |
| return vfio_pci_core_get_dmabuf_phys(core_vdev, provider, region_index, | |
| phys_vec, dma_ranges, nr_ranges); | |
| } | |
| static const struct vfio_pci_device_ops nvgrace_gpu_pci_dev_ops = { | |
| .get_dmabuf_phys = nvgrace_get_dmabuf_phys, | |
| }; | |
| static const struct vfio_device_ops nvgrace_gpu_pci_ops = { | |
| .name = "nvgrace-gpu-vfio-pci", | |
| .init = vfio_pci_core_init_dev, | |
| .release = vfio_pci_core_release_dev, | |
| .open_device = nvgrace_gpu_open_device, | |
| .close_device = nvgrace_gpu_close_device, | |
| .ioctl = nvgrace_gpu_ioctl, | |
| .get_region_info_caps = nvgrace_gpu_ioctl_get_region_info, | |
| .device_feature = vfio_pci_core_ioctl_feature, | |
| .read = nvgrace_gpu_read, | |
| .write = nvgrace_gpu_write, | |
| .mmap = nvgrace_gpu_mmap, | |
| .request = vfio_pci_core_request, | |
| .match = vfio_pci_core_match, | |
| .match_token_uuid = vfio_pci_core_match_token_uuid, | |
| .bind_iommufd = vfio_iommufd_physical_bind, | |
| .unbind_iommufd = vfio_iommufd_physical_unbind, | |
| .attach_ioas = vfio_iommufd_physical_attach_ioas, | |
| .detach_ioas = vfio_iommufd_physical_detach_ioas, | |
| }; | |
| static int nvgrace_gpu_core_open_device(struct vfio_device *core_vdev) | |
| { | |
| struct vfio_pci_core_device *vdev = | |
| container_of(core_vdev, struct vfio_pci_core_device, vdev); | |
| int ret; | |
| ret = vfio_pci_core_enable(vdev); | |
| if (ret) | |
| return ret; | |
| vfio_pci_core_finish_enable(vdev); | |
| return 0; | |
| } | |
| static const struct vfio_pci_device_ops nvgrace_gpu_pci_dev_core_ops = { | |
| .get_dmabuf_phys = vfio_pci_core_get_dmabuf_phys, | |
| }; | |
| static const struct vfio_device_ops nvgrace_gpu_pci_core_ops = { | |
| .name = "nvgrace-gpu-vfio-pci-core", | |
| .init = vfio_pci_core_init_dev, | |
| .release = vfio_pci_core_release_dev, | |
| .open_device = nvgrace_gpu_core_open_device, | |
| .close_device = vfio_pci_core_close_device, | |
| .ioctl = vfio_pci_core_ioctl, | |
| .get_region_info_caps = vfio_pci_ioctl_get_region_info, | |
| .device_feature = vfio_pci_core_ioctl_feature, | |
| .read = vfio_pci_core_read, | |
| .write = vfio_pci_core_write, | |
| .mmap = vfio_pci_core_mmap, | |
| .request = vfio_pci_core_request, | |
| .match = vfio_pci_core_match, | |
| .match_token_uuid = vfio_pci_core_match_token_uuid, | |
| .bind_iommufd = vfio_iommufd_physical_bind, | |
| .unbind_iommufd = vfio_iommufd_physical_unbind, | |
| .attach_ioas = vfio_iommufd_physical_attach_ioas, | |
| .detach_ioas = vfio_iommufd_physical_detach_ioas, | |
| }; | |
| static int | |
| nvgrace_gpu_fetch_memory_property(struct pci_dev *pdev, | |
| u64 *pmemphys, u64 *pmemlength) | |
| { | |
| int ret; | |
| /* | |
| * The memory information is present in the system ACPI tables as DSD | |
| * properties nvidia,gpu-mem-base-pa and nvidia,gpu-mem-size. | |
| */ | |
| ret = device_property_read_u64(&pdev->dev, "nvidia,gpu-mem-base-pa", | |
| pmemphys); | |
| if (ret) | |
| return ret; | |
| if (*pmemphys > type_max(phys_addr_t)) | |
| return -EOVERFLOW; | |
| ret = device_property_read_u64(&pdev->dev, "nvidia,gpu-mem-size", | |
| pmemlength); | |
| if (ret) | |
| return ret; | |
| if (*pmemlength > type_max(size_t)) | |
| return -EOVERFLOW; | |
| /* | |
| * If the C2C link is not up due to an error, the coherent device | |
| * memory size is returned as 0. Fail in such case. | |
| */ | |
| if (*pmemlength == 0) | |
| return -ENOMEM; | |
| return ret; | |
| } | |
| static int | |
| nvgrace_gpu_init_nvdev_struct(struct pci_dev *pdev, | |
| struct nvgrace_gpu_pci_core_device *nvdev, | |
| u64 memphys, u64 memlength) | |
| { | |
| int ret = 0; | |
| u64 resmem_size = 0; | |
| /* | |
| * On Grace Hopper systems, the VM GPU device driver needs a non-cacheable | |
| * region to support the MIG feature owing to a hardware bug. Since the | |
| * device memory is mapped as NORMAL cached, carve out a region from the end | |
| * with a different NORMAL_NC property (called as reserved memory and | |
| * represented as resmem). This region then is exposed as a 64b BAR | |
| * (region 2 and 3) to the VM, while exposing the rest (termed as usable | |
| * memory and represented using usemem) as cacheable 64b BAR (region 4 and 5). | |
| * | |
| * devmem (memlength) | |
| * |-------------------------------------------------| | |
| * | | | |
| * usemem.memphys resmem.memphys | |
| * | |
| * This hardware bug is fixed on the Grace Blackwell platforms and the | |
| * presence of the bug can be determined through nvdev->has_mig_hw_bug. | |
| * Thus on systems with the hardware fix, there is no need to partition | |
| * the GPU device memory and the entire memory is usable and mapped as | |
| * NORMAL cached (i.e. resmem size is 0). | |
| */ | |
| if (nvdev->has_mig_hw_bug) | |
| resmem_size = SZ_1G; | |
| nvdev->usemem.memphys = memphys; | |
| /* | |
| * The device memory exposed to the VM is added to the kernel by the | |
| * VM driver module in chunks of memory block size. Note that only the | |
| * usable memory (usemem) is added to the kernel for usage by the VM | |
| * workloads. | |
| */ | |
| if (check_sub_overflow(memlength, resmem_size, | |
| &nvdev->usemem.memlength)) { | |
| ret = -EOVERFLOW; | |
| goto done; | |
| } | |
| /* | |
| * The usemem region is exposed as a 64B Bar composed of region 4 and 5. | |
| * Calculate and save the BAR size for the region. | |
| */ | |
| nvdev->usemem.bar_size = roundup_pow_of_two(nvdev->usemem.memlength); | |
| /* | |
| * If the hardware has the fix for MIG, there is no requirement | |
| * for splitting the device memory to create RESMEM. The entire | |
| * device memory is usable and will be USEMEM. Return here for | |
| * such case. | |
| */ | |
| if (!nvdev->has_mig_hw_bug) | |
| goto done; | |
| /* | |
| * When the device memory is split to workaround the MIG bug on | |
| * Grace Hopper, the USEMEM part of the device memory has to be | |
| * MEMBLK_SIZE aligned. This is a hardwired ABI value between the | |
| * GPU FW and VFIO driver. The VM device driver is also aware of it | |
| * and make use of the value for its calculation to determine USEMEM | |
| * size. Note that the device memory may not be 512M aligned. | |
| */ | |
| nvdev->usemem.memlength = round_down(nvdev->usemem.memlength, | |
| MEMBLK_SIZE); | |
| if (nvdev->usemem.memlength == 0) { | |
| ret = -EINVAL; | |
| goto done; | |
| } | |
| if ((check_add_overflow(nvdev->usemem.memphys, | |
| nvdev->usemem.memlength, | |
| &nvdev->resmem.memphys)) || | |
| (check_sub_overflow(memlength, nvdev->usemem.memlength, | |
| &nvdev->resmem.memlength))) { | |
| ret = -EOVERFLOW; | |
| goto done; | |
| } | |
| /* | |
| * The resmem region is exposed as a 64b BAR composed of region 2 and 3 | |
| * for Grace Hopper. Calculate and save the BAR size for the region. | |
| */ | |
| nvdev->resmem.bar_size = roundup_pow_of_two(nvdev->resmem.memlength); | |
| done: | |
| return ret; | |
| } | |
| static bool nvgrace_gpu_has_mig_hw_bug(struct pci_dev *pdev) | |
| { | |
| int pcie_dvsec; | |
| u16 dvsec_ctrl16; | |
| pcie_dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_NVIDIA, | |
| GPU_CAP_DVSEC_REGISTER); | |
| if (pcie_dvsec) { | |
| pci_read_config_word(pdev, | |
| pcie_dvsec + DVSEC_BITMAP_OFFSET, | |
| &dvsec_ctrl16); | |
| if (dvsec_ctrl16 & MIG_SUPPORTED_WITH_CACHED_RESMEM) | |
| return false; | |
| } | |
| return true; | |
| } | |
| /* | |
| * To reduce the system bootup time, the HBM training has | |
| * been moved out of the UEFI on the Grace-Blackwell systems. | |
| * | |
| * The onus of checking whether the HBM training has completed | |
| * thus falls on the module. The HBM training status can be | |
| * determined from a BAR0 register. | |
| * | |
| * Similarly, another BAR0 register exposes the status of the | |
| * CPU-GPU chip-to-chip (C2C) cache coherent interconnect. | |
| * | |
| * Poll these register and check for 30s. If the HBM training is | |
| * not complete or if the C2C link is not ready, fail the probe. | |
| * | |
| * While the wait is not required on Grace Hopper systems, it | |
| * is beneficial to make the check to ensure the device is in an | |
| * expected state. | |
| * | |
| * On Blackwell-Next systems, memory readiness is determined via the | |
| * CXL Device DVSEC in PCI config space and does not require BAR0. | |
| * For the legacy path, ensure BAR0 is enabled before accessing the | |
| * registers. | |
| */ | |
| static int nvgrace_gpu_probe_check_device_ready(struct nvgrace_gpu_pci_core_device *nvdev) | |
| { | |
| struct pci_dev *pdev = nvdev->core_device.pdev; | |
| void __iomem *io; | |
| int ret; | |
| /* | |
| * Note that the worst-case wait here is ~256s (vs ~30s on the | |
| * legacy path) and may block device unbind/sysfs for the duration. | |
| */ | |
| if (nvdev->cxl_dvsec) | |
| return nvgrace_gpu_wait_device_ready_cxl(nvdev); | |
| ret = pci_enable_device(pdev); | |
| if (ret) | |
| return ret; | |
| ret = pci_request_selected_regions(pdev, 1 << 0, KBUILD_MODNAME); | |
| if (ret) | |
| goto request_region_exit; | |
| io = pci_iomap(pdev, 0, 0); | |
| if (!io) { | |
| ret = -ENOMEM; | |
| goto iomap_exit; | |
| } | |
| ret = nvgrace_gpu_wait_device_ready_legacy(io); | |
| pci_iounmap(pdev, io); | |
| iomap_exit: | |
| pci_release_selected_regions(pdev, 1 << 0); | |
| request_region_exit: | |
| pci_disable_device(pdev); | |
| return ret; | |
| } | |
| static int nvgrace_gpu_probe(struct pci_dev *pdev, | |
| const struct pci_device_id *id) | |
| { | |
| const struct vfio_device_ops *ops = &nvgrace_gpu_pci_core_ops; | |
| struct nvgrace_gpu_pci_core_device *nvdev; | |
| u64 memphys, memlength; | |
| int ret; | |
| ret = nvgrace_gpu_fetch_memory_property(pdev, &memphys, &memlength); | |
| if (!ret) | |
| ops = &nvgrace_gpu_pci_ops; | |
| nvdev = vfio_alloc_device(nvgrace_gpu_pci_core_device, core_device.vdev, | |
| &pdev->dev, ops); | |
| if (IS_ERR(nvdev)) | |
| return PTR_ERR(nvdev); | |
| nvdev->cxl_dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, | |
| PCI_DVSEC_CXL_DEVICE); | |
| ret = nvgrace_gpu_probe_check_device_ready(nvdev); | |
| if (ret) | |
| goto out_put_vdev; | |
| dev_set_drvdata(&pdev->dev, nvdev); | |
| if (ops == &nvgrace_gpu_pci_ops) { | |
| nvdev->has_mig_hw_bug = nvgrace_gpu_has_mig_hw_bug(pdev); | |
| /* | |
| * Device memory properties are identified in the host ACPI | |
| * table. Set the nvgrace_gpu_pci_core_device structure. | |
| */ | |
| ret = nvgrace_gpu_init_nvdev_struct(pdev, nvdev, | |
| memphys, memlength); | |
| if (ret) | |
| goto out_put_vdev; | |
| nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_ops; | |
| } else { | |
| nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_core_ops; | |
| } | |
| ret = vfio_pci_core_register_device(&nvdev->core_device); | |
| if (ret) | |
| goto out_put_vdev; | |
| return ret; | |
| out_put_vdev: | |
| vfio_put_device(&nvdev->core_device.vdev); | |
| return ret; | |
| } | |
| static void nvgrace_gpu_remove(struct pci_dev *pdev) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = pci_get_drvdata(pdev); | |
| vfio_pci_core_unregister_device(&nvdev->core_device); | |
| vfio_put_device(&nvdev->core_device.vdev); | |
| } | |
| static const struct pci_device_id nvgrace_gpu_vfio_pci_table[] = { | |
| /* GH200 120GB */ | |
| { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_NVIDIA, 0x2342) }, | |
| /* GH200 480GB */ | |
| { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_NVIDIA, 0x2345) }, | |
| /* GH200 SKU */ | |
| { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_NVIDIA, 0x2348) }, | |
| /* GB200 SKU */ | |
| { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_NVIDIA, 0x2941) }, | |
| /* GB300 SKU */ | |
| { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_NVIDIA, 0x31C2) }, | |
| {} | |
| }; | |
| MODULE_DEVICE_TABLE(pci, nvgrace_gpu_vfio_pci_table); | |
| /* | |
| * The GPU reset is required to be serialized against the *first* mapping | |
| * faults and read/writes accesses to prevent potential RAS events logging. | |
| * | |
| * First fault or access after a reset needs to poll device readiness, | |
| * flag that a reset has occurred. The readiness test is done by holding | |
| * the memory_lock read lock and we expect all vfio-pci initiated resets to | |
| * hold the memory_lock write lock to avoid races. However, .reset_done | |
| * extends beyond the scope of vfio-pci initiated resets therefore we | |
| * cannot assert this behavior and use lockdep_assert_held_write. | |
| */ | |
| static void nvgrace_gpu_vfio_pci_reset_done(struct pci_dev *pdev) | |
| { | |
| struct nvgrace_gpu_pci_core_device *nvdev = pci_get_drvdata(pdev); | |
| nvdev->reset_done = true; | |
| } | |
| static const struct pci_error_handlers nvgrace_gpu_vfio_pci_err_handlers = { | |
| .reset_done = nvgrace_gpu_vfio_pci_reset_done, | |
| .error_detected = nvgrace_gpu_aer_err_detected, | |
| }; | |
| static struct pci_driver nvgrace_gpu_vfio_pci_driver = { | |
| .name = KBUILD_MODNAME, | |
| .id_table = nvgrace_gpu_vfio_pci_table, | |
| .probe = nvgrace_gpu_probe, | |
| .remove = nvgrace_gpu_remove, | |
| .driver = { .pm = pm_ptr(&nvgrace_gpu_pm_ops) }, | |
| .err_handler = &nvgrace_gpu_vfio_pci_err_handlers, | |
| .driver_managed_dma = true, | |
| }; | |
| module_pci_driver(nvgrace_gpu_vfio_pci_driver); | |
| MODULE_LICENSE("GPL"); | |
| MODULE_AUTHOR("Ankit Agrawal <ankita@nvidia.com>"); | |
| MODULE_AUTHOR("Aniket Agashe <aniketa@nvidia.com>"); | |
| MODULE_DESCRIPTION("VFIO NVGRACE GPU PF - User Level driver for NVIDIA devices with CPU coherently accessible device memory"); |