blob: ab96edb058d6cee703ac8b2d9359087346aa56a2 [file] [log] [blame]
// SPDX-License-Identifier: MIT
/*
* Copyright © 2021 Intel Corporation
*/
#include "xe_ggtt.h"
#include <linux/sizes.h>
#include <drm/drm_managed.h>
#include <drm/i915_drm.h>
#include "regs/xe_gt_regs.h"
#include "regs/xe_regs.h"
#include "xe_assert.h"
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_gt.h"
#include "xe_gt_printk.h"
#include "xe_gt_tlb_invalidation.h"
#include "xe_map.h"
#include "xe_mmio.h"
#include "xe_sriov.h"
#include "xe_wopcm.h"
#define XELPG_GGTT_PTE_PAT0 BIT_ULL(52)
#define XELPG_GGTT_PTE_PAT1 BIT_ULL(53)
/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
#define GUC_GGTT_TOP 0xFEE00000
static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
u16 pat_index)
{
u64 pte;
pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
pte |= XE_PAGE_PRESENT;
if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
pte |= XE_GGTT_PTE_DM;
return pte;
}
static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
u16 pat_index)
{
struct xe_device *xe = xe_bo_device(bo);
u64 pte;
pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index);
xe_assert(xe, pat_index <= 3);
if (pat_index & BIT(0))
pte |= XELPG_GGTT_PTE_PAT0;
if (pat_index & BIT(1))
pte |= XELPG_GGTT_PTE_PAT1;
return pte;
}
static unsigned int probe_gsm_size(struct pci_dev *pdev)
{
u16 gmch_ctl, ggms;
pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
return ggms ? SZ_1M << ggms : 0;
}
void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
{
xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
xe_tile_assert(ggtt->tile, addr < ggtt->size);
writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
}
static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
{
u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
u64 end = start + size - 1;
u64 scratch_pte;
xe_tile_assert(ggtt->tile, start < end);
if (ggtt->scratch)
scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0,
pat_index);
else
scratch_pte = 0;
while (start < end) {
xe_ggtt_set_pte(ggtt, start, scratch_pte);
start += XE_PAGE_SIZE;
}
}
static void ggtt_fini_early(struct drm_device *drm, void *arg)
{
struct xe_ggtt *ggtt = arg;
mutex_destroy(&ggtt->lock);
drm_mm_takedown(&ggtt->mm);
}
static void ggtt_fini(struct drm_device *drm, void *arg)
{
struct xe_ggtt *ggtt = arg;
ggtt->scratch = NULL;
}
static void primelockdep(struct xe_ggtt *ggtt)
{
if (!IS_ENABLED(CONFIG_LOCKDEP))
return;
fs_reclaim_acquire(GFP_KERNEL);
might_lock(&ggtt->lock);
fs_reclaim_release(GFP_KERNEL);
}
static const struct xe_ggtt_pt_ops xelp_pt_ops = {
.pte_encode_bo = xelp_ggtt_pte_encode_bo,
};
static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
};
/*
* Early GGTT initialization, which allows to create new mappings usable by the
* GuC.
* Mappings are not usable by the HW engines, as it doesn't have scratch /
* initial clear done to it yet. That will happen in the regular, non-early
* GGTT init.
*/
int xe_ggtt_init_early(struct xe_ggtt *ggtt)
{
struct xe_device *xe = tile_to_xe(ggtt->tile);
struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
unsigned int gsm_size;
if (IS_SRIOV_VF(xe))
gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
else
gsm_size = probe_gsm_size(pdev);
if (gsm_size == 0) {
drm_err(&xe->drm, "Hardware reported no preallocated GSM\n");
return -ENOMEM;
}
ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
ggtt->flags |= XE_GGTT_FLAGS_64K;
/*
* 8B per entry, each points to a 4KB page.
*
* The GuC address space is limited on both ends of the GGTT, because
* the GuC shim HW redirects accesses to those addresses to other HW
* areas instead of going through the GGTT. On the bottom end, the GuC
* can't access offsets below the WOPCM size, while on the top side the
* limit is fixed at GUC_GGTT_TOP. To keep things simple, instead of
* checking each object to see if they are accessed by GuC or not, we
* just exclude those areas from the allocator. Additionally, to
* simplify the driver load, we use the maximum WOPCM size in this logic
* instead of the programmed one, so we don't need to wait until the
* actual size to be programmed is determined (which requires FW fetch)
* before initializing the GGTT. These simplifications might waste space
* in the GGTT (about 20-25 MBs depending on the platform) but we can
* live with this.
*
* Another benifit of this is the GuC bootrom can't access anything
* below the WOPCM max size so anything the bootom needs to access (e.g.
* a RSA key) needs to be placed in the GGTT above the WOPCM max size.
* Starting the GGTT allocations above the WOPCM max give us the correct
* placement for free.
*/
if (ggtt->size > GUC_GGTT_TOP)
ggtt->size = GUC_GGTT_TOP;
if (GRAPHICS_VERx100(xe) >= 1270)
ggtt->pt_ops = &xelpg_pt_ops;
else
ggtt->pt_ops = &xelp_pt_ops;
drm_mm_init(&ggtt->mm, xe_wopcm_size(xe),
ggtt->size - xe_wopcm_size(xe));
mutex_init(&ggtt->lock);
primelockdep(ggtt);
return drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
}
static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
{
struct drm_mm_node *hole;
u64 start, end;
/* Display may have allocated inside ggtt, so be careful with clearing here */
xe_device_mem_access_get(tile_to_xe(ggtt->tile));
mutex_lock(&ggtt->lock);
drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
xe_ggtt_clear(ggtt, start, end - start);
xe_ggtt_invalidate(ggtt);
mutex_unlock(&ggtt->lock);
xe_device_mem_access_put(tile_to_xe(ggtt->tile));
}
int xe_ggtt_init(struct xe_ggtt *ggtt)
{
struct xe_device *xe = tile_to_xe(ggtt->tile);
unsigned int flags;
int err;
/*
* So we don't need to worry about 64K GGTT layout when dealing with
* scratch entires, rather keep the scratch page in system memory on
* platforms where 64K pages are needed for VRAM.
*/
flags = XE_BO_CREATE_PINNED_BIT;
if (ggtt->flags & XE_GGTT_FLAGS_64K)
flags |= XE_BO_CREATE_SYSTEM_BIT;
else
flags |= XE_BO_CREATE_VRAM_IF_DGFX(ggtt->tile);
ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
if (IS_ERR(ggtt->scratch)) {
err = PTR_ERR(ggtt->scratch);
goto err;
}
xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size);
xe_ggtt_initial_clear(ggtt);
return drmm_add_action_or_reset(&xe->drm, ggtt_fini, ggtt);
err:
ggtt->scratch = NULL;
return err;
}
#define GUC_TLB_INV_CR XE_REG(0xcee8)
#define GUC_TLB_INV_CR_INVALIDATE REG_BIT(0)
#define PVC_GUC_TLB_INV_DESC0 XE_REG(0xcf7c)
#define PVC_GUC_TLB_INV_DESC0_VALID REG_BIT(0)
#define PVC_GUC_TLB_INV_DESC1 XE_REG(0xcf80)
#define PVC_GUC_TLB_INV_DESC1_INVALIDATE REG_BIT(6)
static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
{
if (!gt)
return;
/*
* Invalidation can happen when there's no in-flight work keeping the
* GT awake. We need to explicitly grab forcewake to ensure the GT
* and GuC are accessible.
*/
xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
/* TODO: vfunc for GuC vs. non-GuC */
if (gt->uc.guc.submission_state.enabled) {
int seqno;
seqno = xe_gt_tlb_invalidation_guc(gt);
xe_gt_assert(gt, seqno > 0);
if (seqno > 0)
xe_gt_tlb_invalidation_wait(gt, seqno);
} else if (xe_device_uc_enabled(gt_to_xe(gt))) {
struct xe_device *xe = gt_to_xe(gt);
if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1,
PVC_GUC_TLB_INV_DESC1_INVALIDATE);
xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0,
PVC_GUC_TLB_INV_DESC0_VALID);
} else
xe_mmio_write32(gt, GUC_TLB_INV_CR,
GUC_TLB_INV_CR_INVALIDATE);
}
xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
}
void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
{
/* Each GT in a tile has its own TLB to cache GGTT lookups */
ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
}
void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix)
{
u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
u64 addr, scratch_pte;
scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0, pat_index);
printk("%sGlobal GTT:", prefix);
for (addr = 0; addr < ggtt->size; addr += XE_PAGE_SIZE) {
unsigned int i = addr / XE_PAGE_SIZE;
xe_tile_assert(ggtt->tile, addr <= U32_MAX);
if (ggtt->gsm[i] == scratch_pte)
continue;
printk("%s ggtt[0x%08x] = 0x%016llx",
prefix, (u32)addr, ggtt->gsm[i]);
}
}
static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
const struct drm_mm_node *node, const char *description)
{
char buf[10];
if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
xe_gt_dbg(ggtt->tile->primary_gt, "GGTT %#llx-%#llx (%s) %s\n",
node->start, node->start + node->size, buf, description);
}
}
/**
* xe_ggtt_balloon - prevent allocation of specified GGTT addresses
* @ggtt: the &xe_ggtt where we want to make reservation
* @start: the starting GGTT address of the reserved region
* @end: then end GGTT address of the reserved region
* @node: the &drm_mm_node to hold reserved GGTT node
*
* Use xe_ggtt_deballoon() to release a reserved GGTT node.
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_ggtt_balloon(struct xe_ggtt *ggtt, u64 start, u64 end, struct drm_mm_node *node)
{
int err;
xe_tile_assert(ggtt->tile, start < end);
xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(node));
node->color = 0;
node->start = start;
node->size = end - start;
mutex_lock(&ggtt->lock);
err = drm_mm_reserve_node(&ggtt->mm, node);
mutex_unlock(&ggtt->lock);
if (xe_gt_WARN(ggtt->tile->primary_gt, err,
"Failed to balloon GGTT %#llx-%#llx (%pe)\n",
node->start, node->start + node->size, ERR_PTR(err)))
return err;
xe_ggtt_dump_node(ggtt, node, "balloon");
return 0;
}
/**
* xe_ggtt_deballoon - release a reserved GGTT region
* @ggtt: the &xe_ggtt where reserved node belongs
* @node: the &drm_mm_node with reserved GGTT region
*
* See xe_ggtt_balloon() for details.
*/
void xe_ggtt_deballoon(struct xe_ggtt *ggtt, struct drm_mm_node *node)
{
if (!drm_mm_node_allocated(node))
return;
xe_ggtt_dump_node(ggtt, node, "deballoon");
mutex_lock(&ggtt->lock);
drm_mm_remove_node(node);
mutex_unlock(&ggtt->lock);
}
int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt, struct drm_mm_node *node,
u32 size, u32 align, u32 mm_flags)
{
return drm_mm_insert_node_generic(&ggtt->mm, node, size, align, 0,
mm_flags);
}
int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
u32 size, u32 align)
{
int ret;
mutex_lock(&ggtt->lock);
ret = xe_ggtt_insert_special_node_locked(ggtt, node, size,
align, DRM_MM_INSERT_HIGH);
mutex_unlock(&ggtt->lock);
return ret;
}
void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
u16 cache_mode = bo->flags & XE_BO_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
u64 start = bo->ggtt_node.start;
u64 offset, pte;
for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) {
pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
xe_ggtt_set_pte(ggtt, start + offset, pte);
}
xe_ggtt_invalidate(ggtt);
}
static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
u64 start, u64 end)
{
int err;
u64 alignment = XE_PAGE_SIZE;
if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
alignment = SZ_64K;
if (XE_WARN_ON(bo->ggtt_node.size)) {
/* Someone's already inserted this BO in the GGTT */
xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
return 0;
}
err = xe_bo_validate(bo, NULL, false);
if (err)
return err;
xe_device_mem_access_get(tile_to_xe(ggtt->tile));
mutex_lock(&ggtt->lock);
err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node, bo->size,
alignment, 0, start, end, 0);
if (!err)
xe_ggtt_map_bo(ggtt, bo);
mutex_unlock(&ggtt->lock);
xe_device_mem_access_put(tile_to_xe(ggtt->tile));
return err;
}
int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
u64 start, u64 end)
{
return __xe_ggtt_insert_bo_at(ggtt, bo, start, end);
}
int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
}
void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node)
{
xe_device_mem_access_get(tile_to_xe(ggtt->tile));
mutex_lock(&ggtt->lock);
xe_ggtt_clear(ggtt, node->start, node->size);
drm_mm_remove_node(node);
node->size = 0;
xe_ggtt_invalidate(ggtt);
mutex_unlock(&ggtt->lock);
xe_device_mem_access_put(tile_to_xe(ggtt->tile));
}
void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
if (XE_WARN_ON(!bo->ggtt_node.size))
return;
/* This BO is not currently in the GGTT */
xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
xe_ggtt_remove_node(ggtt, &bo->ggtt_node);
}
int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
{
int err;
err = mutex_lock_interruptible(&ggtt->lock);
if (err)
return err;
drm_mm_print(&ggtt->mm, p);
mutex_unlock(&ggtt->lock);
return err;
}