| /* |
| * Copyright 2014 Advanced Micro Devices, Inc. |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in |
| * all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| * OTHER DEALINGS IN THE SOFTWARE. |
| * |
| */ |
| |
| #include <linux/device.h> |
| #include <linux/export.h> |
| #include <linux/err.h> |
| #include <linux/fs.h> |
| #include <linux/sched.h> |
| #include <linux/slab.h> |
| #include <linux/uaccess.h> |
| #include <linux/compat.h> |
| #include <uapi/linux/kfd_ioctl.h> |
| #include <linux/time.h> |
| #include "kfd_priv.h" |
| #include <linux/mm.h> |
| #include <linux/mman.h> |
| #include <asm/processor.h> |
| |
| /* |
| * The primary memory I/O features being added for revisions of gfxip |
| * beyond 7.0 (Kaveri) are: |
| * |
| * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b |
| * |
| * “Flat” shader memory access – These are new shader vector memory |
| * operations that do not reference a T#/V# so a “pointer” is what is |
| * sourced from the vector gprs for direct access to memory. |
| * This pointer space has the Shared(LDS) and Private(Scratch) memory |
| * mapped into this pointer space as apertures. |
| * The hardware then determines how to direct the memory request |
| * based on what apertures the request falls in. |
| * |
| * Unaligned support and alignment check |
| * |
| * |
| * System Unified Address - SUA |
| * |
| * The standard usage for GPU virtual addresses are that they are mapped by |
| * a set of page tables we call GPUVM and these page tables are managed by |
| * a combination of vidMM/driver software components. The current virtual |
| * address (VA) range for GPUVM is 40b. |
| * |
| * As of gfxip7.1 and beyond we’re adding the ability for compute memory |
| * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access |
| * the same page tables used by host x86 processors and that are managed by |
| * the operating system. This is via a technique and hardware called ATC/IOMMU. |
| * The GPU has the capability of accessing both the GPUVM and ATC address |
| * spaces for a given VMID (process) simultaneously and we call this feature |
| * system unified address (SUA). |
| * |
| * There are three fundamental address modes of operation for a given VMID |
| * (process) on the GPU: |
| * |
| * HSA64 – 64b pointers and the default address space is ATC |
| * HSA32 – 32b pointers and the default address space is ATC |
| * GPUVM – 64b pointers and the default address space is GPUVM (driver |
| * model mode) |
| * |
| * |
| * HSA64 - ATC/IOMMU 64b |
| * |
| * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized |
| * by the CPU so an AMD CPU can only access the high area |
| * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space |
| * so the actual VA carried to translation is 48b. There is a “hole” in |
| * the middle of the 64b VA space. |
| * |
| * The GPU not only has access to all of the CPU accessible address space via |
| * ATC/IOMMU, but it also has access to the GPUVM address space. The “system |
| * unified address” feature (SUA) is the mapping of GPUVM and ATC address |
| * spaces into a unified pointer space. The method we take for 64b mode is |
| * to map the full 40b GPUVM address space into the hole of the 64b address |
| * space. |
| |
| * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we |
| * direct requests to be translated via GPUVM page tables instead of the |
| * IOMMU path. |
| * |
| * |
| * 64b to 49b Address conversion |
| * |
| * Note that there are still significant portions of unused regions (holes) |
| * in the 64b address space even for the GPU. There are several places in |
| * the pipeline (sw and hw), we wish to compress the 64b virtual address |
| * to a 49b address. This 49b address is constituted of an “ATC” bit |
| * plus a 48b virtual address. This 49b address is what is passed to the |
| * translation hardware. ATC==0 means the 48b address is a GPUVM address |
| * (max of 2^40 – 1) intended to be translated via GPUVM page tables. |
| * ATC==1 means the 48b address is intended to be translated via IOMMU |
| * page tables. |
| * |
| * A 64b pointer is compared to the apertures that are defined (Base/Limit), in |
| * this case the GPUVM aperture (red) is defined and if a pointer falls in this |
| * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero |
| * as part of the 64b to 49b conversion. |
| * |
| * Where this 64b to 49b conversion is done is a function of the usage. |
| * Most GPU memory access is via memory objects where the driver builds |
| * a descriptor which consists of a base address and a memory access by |
| * the GPU usually consists of some kind of an offset or Cartesian coordinate |
| * that references this memory descriptor. This is the case for shader |
| * instructions that reference the T# or V# constants, or for specified |
| * locations of assets (ex. the shader program location). In these cases |
| * the driver is what handles the 64b to 49b conversion and the base |
| * address in the descriptor (ex. V# or T# or shader program location) |
| * is defined as a 48b address w/ an ATC bit. For this usage a given |
| * memory object cannot straddle multiple apertures in the 64b address |
| * space. For example a shader program cannot jump in/out between ATC |
| * and GPUVM space. |
| * |
| * In some cases we wish to pass a 64b pointer to the GPU hardware and |
| * the GPU hw does the 64b to 49b conversion before passing memory |
| * requests to the cache/memory system. This is the case for the |
| * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers |
| * in scalar and vector GPRs respectively. |
| * |
| * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip |
| * hardware sends a 48b address along w/ an ATC bit, to the memory controller |
| * on the memory request interfaces. |
| * |
| * <client>_MC_rdreq_atc // read request ATC bit |
| * |
| * 0 : <client>_MC_rdreq_addr is a GPUVM VA |
| * |
| * 1 : <client>_MC_rdreq_addr is a ATC VA |
| * |
| * |
| * “Spare” aperture (APE1) |
| * |
| * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use |
| * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the |
| * config tables for setting cache policies. The “spare” (APE1) aperture is |
| * motivated by getting a different Mtype from the default. |
| * The default aperture isn’t an actual base/limit aperture; it is just the |
| * address space that doesn’t hit any defined base/limit apertures. |
| * The following diagram is a complete picture of the gfxip7.x SUA apertures. |
| * The APE1 can be placed either below or above |
| * the hole (cannot be in the hole). |
| * |
| * |
| * General Aperture definitions and rules |
| * |
| * An aperture register definition consists of a Base, Limit, Mtype, and |
| * usually an ATC bit indicating which translation tables that aperture uses. |
| * In all cases (for SUA and DUA apertures discussed later), aperture base |
| * and limit definitions are 64KB aligned. |
| * |
| * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 } |
| * |
| * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF } |
| * |
| * The base and limit are considered inclusive to an aperture so being |
| * inside an aperture means (address >= Base) AND (address <= Limit). |
| * |
| * In no case is a payload that straddles multiple apertures expected to work. |
| * For example a load_dword_x4 that starts in one aperture and ends in another, |
| * does not work. For the vector FLAT_* ops we have detection capability in |
| * the shader for reporting a “memory violation” back to the |
| * SQ block for use in traps. |
| * A memory violation results when an op falls into the hole, |
| * or a payload straddles multiple apertures. The S_LOAD instruction |
| * does not have this detection. |
| * |
| * Apertures cannot overlap. |
| * |
| * |
| * |
| * HSA32 - ATC/IOMMU 32b |
| * |
| * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR |
| * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b |
| * will not fit so there is only partial visibility to the GPUVM |
| * space (defined by the aperture) for S_LOAD and FLAT_* ops. |
| * There is no spare (APE1) aperture for HSA32 mode. |
| * |
| * |
| * GPUVM 64b mode (driver model) |
| * |
| * This mode is related to HSA64 in that the difference really is that |
| * the default aperture is GPUVM (ATC==0) and not ATC space. |
| * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for |
| * SUA GPUVM mode, but does not support HSA32/HSA64. |
| * |
| * |
| * Device Unified Address - DUA |
| * |
| * Device unified address (DUA) is the name of the feature that maps the |
| * Shared(LDS) memory and Private(Scratch) memory into the overall address |
| * space for use by the new FLAT_* vector memory ops. The Shared and |
| * Private memories are mapped as apertures into the address space, |
| * and the hardware detects when a FLAT_* memory request is to be redirected |
| * to the LDS or Scratch memory when it falls into one of these apertures. |
| * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and |
| * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes, |
| * the Shared/Private apertures are always placed in a limited selection of |
| * options in the hole of the 64b address space. For HSA32 mode, the |
| * Shared/Private apertures can be placed anywhere in the 32b space |
| * except at 0. |
| * |
| * |
| * HSA64 Apertures for FLAT_* vector ops |
| * |
| * For HSA64 SUA mode, the Shared and Private apertures are always placed |
| * in the hole w/ a limited selection of possible locations. The requests |
| * that fall in the private aperture are expanded as a function of the |
| * work-item id (tid) and redirected to the location of the |
| * “hidden private memory”. The hidden private can be placed in either GPUVM |
| * or ATC space. The addresses that fall in the shared aperture are |
| * re-directed to the on-chip LDS memory hardware. |
| * |
| * |
| * HSA32 Apertures for FLAT_* vector ops |
| * |
| * In HSA32 mode, the Private and Shared apertures can be placed anywhere |
| * in the 32b space except at 0 (Private or Shared Base at zero disables |
| * the apertures). If the base address of the apertures are non-zero |
| * (ie apertures exists), the size is always 64KB. |
| * |
| * |
| * GPUVM Apertures for FLAT_* vector ops |
| * |
| * In GPUVM mode, the Shared/Private apertures are specified identically |
| * to HSA64 mode where they are always in the hole at a limited selection |
| * of locations. |
| * |
| * |
| * Aperture Definitions for SUA and DUA |
| * |
| * The interpretation of the aperture register definitions for a given |
| * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or |
| * GPUVM64 discussed in previous sections. The mode is first decoded, and |
| * then the remaining register decode is a function of the mode. |
| * |
| * |
| * SUA Mode Decode |
| * |
| * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from |
| * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and |
| * the SH_MEM_CONFIG:PTR32 bits. |
| * |
| * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode |
| * |
| * 1 0 HSA64 |
| * |
| * 1 1 HSA32 |
| * |
| * 0 X GPUVM64 |
| * |
| * In general the hardware will ignore the PTR32 bit and treat |
| * as “0” whenever DATA_ATC = “0”, but sw should set PTR32=0 |
| * when DATA_ATC=0. |
| * |
| * The DATA_ATC bit is only set for compute dispatches. |
| * All “Draw” dispatches are hardcoded to GPUVM64 mode |
| * for FLAT_* / S_LOAD operations. |
| */ |
| |
| #define MAKE_GPUVM_APP_BASE_VI(gpu_num) \ |
| (((uint64_t)(gpu_num) << 61) + 0x1000000000000L) |
| |
| #define MAKE_GPUVM_APP_LIMIT(base, size) \ |
| (((uint64_t)(base) & 0xFFFFFF0000000000UL) + (size) - 1) |
| |
| #define MAKE_SCRATCH_APP_BASE_VI() \ |
| (((uint64_t)(0x1UL) << 61) + 0x100000000L) |
| |
| #define MAKE_SCRATCH_APP_LIMIT(base) \ |
| (((uint64_t)base & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF) |
| |
| #define MAKE_LDS_APP_BASE_VI() \ |
| (((uint64_t)(0x1UL) << 61) + 0x0) |
| #define MAKE_LDS_APP_LIMIT(base) \ |
| (((uint64_t)(base) & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF) |
| |
| /* On GFXv9 the LDS and scratch apertures are programmed independently |
| * using the high 16 bits of the 64-bit virtual address. They must be |
| * in the hole, which will be the case as long as the high 16 bits are |
| * not 0. |
| * |
| * The aperture sizes are still 4GB implicitly. |
| * |
| * A GPUVM aperture is not applicable on GFXv9. |
| */ |
| #define MAKE_LDS_APP_BASE_V9() ((uint64_t)(0x1UL) << 48) |
| #define MAKE_SCRATCH_APP_BASE_V9() ((uint64_t)(0x2UL) << 48) |
| |
| /* User mode manages most of the SVM aperture address space. The low |
| * 16MB are reserved for kernel use (CWSR trap handler and kernel IB |
| * for now). |
| */ |
| #define SVM_USER_BASE (u64)(KFD_CWSR_TBA_TMA_SIZE + 2*PAGE_SIZE) |
| #define SVM_CWSR_BASE (SVM_USER_BASE - KFD_CWSR_TBA_TMA_SIZE) |
| #define SVM_IB_BASE (SVM_CWSR_BASE - PAGE_SIZE) |
| |
| static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id) |
| { |
| /* |
| * node id couldn't be 0 - the three MSB bits of |
| * aperture shouldn't be 0 |
| */ |
| pdd->lds_base = MAKE_LDS_APP_BASE_VI(); |
| pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); |
| |
| if (!pdd->dev->use_iommu_v2) { |
| /* dGPUs: SVM aperture starting at 0 |
| * with small reserved space for kernel. |
| * Set them to CANONICAL addresses. |
| */ |
| pdd->gpuvm_base = SVM_USER_BASE; |
| pdd->gpuvm_limit = |
| pdd->dev->shared_resources.gpuvm_size - 1; |
| } else { |
| /* set them to non CANONICAL addresses, and no SVM is |
| * allocated. |
| */ |
| pdd->gpuvm_base = MAKE_GPUVM_APP_BASE_VI(id + 1); |
| pdd->gpuvm_limit = MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base, |
| pdd->dev->shared_resources.gpuvm_size); |
| } |
| |
| pdd->scratch_base = MAKE_SCRATCH_APP_BASE_VI(); |
| pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); |
| } |
| |
| static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id) |
| { |
| pdd->lds_base = MAKE_LDS_APP_BASE_V9(); |
| pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); |
| |
| /* Raven needs SVM to support graphic handle, etc. Leave the small |
| * reserved space before SVM on Raven as well, even though we don't |
| * have to. |
| * Set gpuvm_base and gpuvm_limit to CANONICAL addresses so that they |
| * are used in Thunk to reserve SVM. |
| */ |
| pdd->gpuvm_base = SVM_USER_BASE; |
| pdd->gpuvm_limit = |
| pdd->dev->shared_resources.gpuvm_size - 1; |
| |
| pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9(); |
| pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); |
| } |
| |
| int kfd_init_apertures(struct kfd_process *process) |
| { |
| uint8_t id = 0; |
| struct kfd_dev *dev; |
| struct kfd_process_device *pdd; |
| |
| /*Iterating over all devices*/ |
| while (kfd_topology_enum_kfd_devices(id, &dev) == 0) { |
| if (!dev || kfd_devcgroup_check_permission(dev)) { |
| /* Skip non GPU devices and devices to which the |
| * current process have no access to. Access can be |
| * limited by placing the process in a specific |
| * cgroup hierarchy |
| */ |
| id++; |
| continue; |
| } |
| |
| pdd = kfd_create_process_device_data(dev, process); |
| if (!pdd) { |
| pr_err("Failed to create process device data\n"); |
| return -ENOMEM; |
| } |
| /* |
| * For 64 bit process apertures will be statically reserved in |
| * the x86_64 non canonical process address space |
| * amdkfd doesn't currently support apertures for 32 bit process |
| */ |
| if (process->is_32bit_user_mode) { |
| pdd->lds_base = pdd->lds_limit = 0; |
| pdd->gpuvm_base = pdd->gpuvm_limit = 0; |
| pdd->scratch_base = pdd->scratch_limit = 0; |
| } else { |
| switch (dev->device_info->asic_family) { |
| case CHIP_KAVERI: |
| case CHIP_HAWAII: |
| case CHIP_CARRIZO: |
| case CHIP_TONGA: |
| case CHIP_FIJI: |
| case CHIP_POLARIS10: |
| case CHIP_POLARIS11: |
| case CHIP_POLARIS12: |
| case CHIP_VEGAM: |
| kfd_init_apertures_vi(pdd, id); |
| break; |
| case CHIP_VEGA10: |
| case CHIP_VEGA12: |
| case CHIP_VEGA20: |
| case CHIP_RAVEN: |
| case CHIP_RENOIR: |
| case CHIP_ARCTURUS: |
| case CHIP_ALDEBARAN: |
| case CHIP_NAVI10: |
| case CHIP_NAVI12: |
| case CHIP_NAVI14: |
| case CHIP_SIENNA_CICHLID: |
| case CHIP_NAVY_FLOUNDER: |
| case CHIP_VANGOGH: |
| case CHIP_DIMGREY_CAVEFISH: |
| case CHIP_BEIGE_GOBY: |
| case CHIP_YELLOW_CARP: |
| case CHIP_CYAN_SKILLFISH: |
| kfd_init_apertures_v9(pdd, id); |
| break; |
| default: |
| WARN(1, "Unexpected ASIC family %u", |
| dev->device_info->asic_family); |
| return -EINVAL; |
| } |
| |
| if (!dev->use_iommu_v2) { |
| /* dGPUs: the reserved space for kernel |
| * before SVM |
| */ |
| pdd->qpd.cwsr_base = SVM_CWSR_BASE; |
| pdd->qpd.ib_base = SVM_IB_BASE; |
| } |
| } |
| |
| dev_dbg(kfd_device, "node id %u\n", id); |
| dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id); |
| dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base); |
| dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit); |
| dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base); |
| dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit); |
| dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base); |
| dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit); |
| |
| id++; |
| } |
| |
| return 0; |
| } |