| // SPDX-License-Identifier: MIT | |
| /* | |
| * Copyright © 2023 Intel Corporation | |
| */ | |
| #include "xe_pat.h" | |
| #include <uapi/drm/xe_drm.h> | |
| #include <generated/xe_wa_oob.h> | |
| #include "regs/xe_reg_defs.h" | |
| #include "xe_assert.h" | |
| #include "xe_device.h" | |
| #include "xe_force_wake.h" | |
| #include "xe_gt.h" | |
| #include "xe_gt_mcr.h" | |
| #include "xe_mmio.h" | |
| #include "xe_sriov.h" | |
| #include "xe_wa.h" | |
| #define _PAT_ATS 0x47fc | |
| #define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ | |
| 0x4800, 0x4804, \ | |
| 0x4848, 0x484c) | |
| #define _PAT_PTA 0x4820 | |
| #define XE2_NO_PROMOTE REG_BIT(10) | |
| #define XE2_COMP_EN REG_BIT(9) | |
| #define XE2_L3_CLOS REG_GENMASK(7, 6) | |
| #define XE2_L3_POLICY REG_GENMASK(5, 4) | |
| #define XE2_L4_POLICY REG_GENMASK(3, 2) | |
| #define XE2_COH_MODE REG_GENMASK(1, 0) | |
| #define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2) | |
| #define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3) | |
| #define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1) | |
| #define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0) | |
| #define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) | |
| #define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3) | |
| #define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2) | |
| #define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0) | |
| #define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2) | |
| #define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x) | |
| #define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0) | |
| #define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3) | |
| #define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2) | |
| #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) | |
| #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) | |
| static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; | |
| struct xe_pat_ops { | |
| void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], | |
| int n_entries); | |
| void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[], | |
| int n_entries); | |
| void (*dump)(struct xe_gt *gt, struct drm_printer *p); | |
| }; | |
| static const struct xe_pat_table_entry xelp_pat_table[] = { | |
| [0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, | |
| [1] = { XELP_PAT_WC, XE_COH_NONE }, | |
| [2] = { XELP_PAT_WT, XE_COH_NONE }, | |
| [3] = { XELP_PAT_UC, XE_COH_NONE }, | |
| }; | |
| static const struct xe_pat_table_entry xehpc_pat_table[] = { | |
| [0] = { XELP_PAT_UC, XE_COH_NONE }, | |
| [1] = { XELP_PAT_WC, XE_COH_NONE }, | |
| [2] = { XELP_PAT_WT, XE_COH_NONE }, | |
| [3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, | |
| [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, | |
| [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, | |
| [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, | |
| [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, | |
| }; | |
| static const struct xe_pat_table_entry xelpg_pat_table[] = { | |
| [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, | |
| [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, | |
| [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, | |
| [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY }, | |
| [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY }, | |
| }; | |
| /* | |
| * The Xe2 table is getting large/complicated so it's easier to review if | |
| * provided in a form that exactly matches the bspec's formatting. The meaning | |
| * of the fields here are: | |
| * - no_promote: 0=promotable, 1=no promote | |
| * - comp_en: 0=disable, 1=enable | |
| * - l3clos: L3 class of service (0-3) | |
| * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC | |
| * - l4_policy: 0=WB, 1=WT, 3=UC | |
| * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent | |
| * | |
| * Reserved entries should be programmed with the maximum caching, minimum | |
| * coherency (which matches an all-0's encoding), so we can just omit them | |
| * in the table. | |
| */ | |
| #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \ | |
| { \ | |
| .value = (no_promote ? XE2_NO_PROMOTE : 0) | \ | |
| (comp_en ? XE2_COMP_EN : 0) | \ | |
| REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \ | |
| REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \ | |
| REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \ | |
| REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \ | |
| .coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE \ | |
| } | |
| static const struct xe_pat_table_entry xe2_pat_table[] = { | |
| [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), | |
| [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), | |
| [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), | |
| [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), | |
| [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), | |
| [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), | |
| [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), | |
| [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), | |
| [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), | |
| [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), | |
| [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), | |
| [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), | |
| [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), | |
| [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), | |
| [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), | |
| [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), | |
| /* 16..19 are reserved; leave set to all 0's */ | |
| [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), | |
| [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), | |
| [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), | |
| [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), | |
| [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), | |
| [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), | |
| [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), | |
| [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), | |
| [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), | |
| [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), | |
| [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), | |
| [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), | |
| }; | |
| /* Special PAT values programmed outside the main table */ | |
| static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 ); | |
| static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 ); | |
| u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) | |
| { | |
| WARN_ON(pat_index >= xe->pat.n_entries); | |
| return xe->pat.table[pat_index].coh_mode; | |
| } | |
| static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], | |
| int n_entries) | |
| { | |
| for (int i = 0; i < n_entries; i++) { | |
| struct xe_reg reg = XE_REG(_PAT_INDEX(i)); | |
| xe_mmio_write32(gt, reg, table[i].value); | |
| } | |
| } | |
| static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[], | |
| int n_entries) | |
| { | |
| for (int i = 0; i < n_entries; i++) { | |
| struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); | |
| xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); | |
| } | |
| } | |
| static void xelp_dump(struct xe_gt *gt, struct drm_printer *p) | |
| { | |
| struct xe_device *xe = gt_to_xe(gt); | |
| int i, err; | |
| err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
| if (err) | |
| goto err_fw; | |
| drm_printf(p, "PAT table:\n"); | |
| for (i = 0; i < xe->pat.n_entries; i++) { | |
| u32 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i))); | |
| u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); | |
| drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i, | |
| XELP_MEM_TYPE_STR_MAP[mem_type], pat); | |
| } | |
| err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); | |
| err_fw: | |
| xe_assert(xe, !err); | |
| } | |
| static const struct xe_pat_ops xelp_pat_ops = { | |
| .program_graphics = program_pat, | |
| .dump = xelp_dump, | |
| }; | |
| static void xehp_dump(struct xe_gt *gt, struct drm_printer *p) | |
| { | |
| struct xe_device *xe = gt_to_xe(gt); | |
| int i, err; | |
| err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
| if (err) | |
| goto err_fw; | |
| drm_printf(p, "PAT table:\n"); | |
| for (i = 0; i < xe->pat.n_entries; i++) { | |
| u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); | |
| u8 mem_type; | |
| mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); | |
| drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i, | |
| XELP_MEM_TYPE_STR_MAP[mem_type], pat); | |
| } | |
| err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); | |
| err_fw: | |
| xe_assert(xe, !err); | |
| } | |
| static const struct xe_pat_ops xehp_pat_ops = { | |
| .program_graphics = program_pat_mcr, | |
| .dump = xehp_dump, | |
| }; | |
| static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p) | |
| { | |
| struct xe_device *xe = gt_to_xe(gt); | |
| int i, err; | |
| err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
| if (err) | |
| goto err_fw; | |
| drm_printf(p, "PAT table:\n"); | |
| for (i = 0; i < xe->pat.n_entries; i++) { | |
| u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); | |
| drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i, | |
| REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), | |
| REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); | |
| } | |
| err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); | |
| err_fw: | |
| xe_assert(xe, !err); | |
| } | |
| static const struct xe_pat_ops xehpc_pat_ops = { | |
| .program_graphics = program_pat_mcr, | |
| .dump = xehpc_dump, | |
| }; | |
| static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p) | |
| { | |
| struct xe_device *xe = gt_to_xe(gt); | |
| int i, err; | |
| err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
| if (err) | |
| goto err_fw; | |
| drm_printf(p, "PAT table:\n"); | |
| for (i = 0; i < xe->pat.n_entries; i++) { | |
| u32 pat; | |
| if (xe_gt_is_media_type(gt)) | |
| pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i))); | |
| else | |
| pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); | |
| drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i, | |
| REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), | |
| REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); | |
| } | |
| err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); | |
| err_fw: | |
| xe_assert(xe, !err); | |
| } | |
| /* | |
| * SAMedia register offsets are adjusted by the write methods and they target | |
| * registers that are not MCR, while for normal GT they are MCR | |
| */ | |
| static const struct xe_pat_ops xelpg_pat_ops = { | |
| .program_graphics = program_pat, | |
| .program_media = program_pat_mcr, | |
| .dump = xelpg_dump, | |
| }; | |
| static void xe2lpg_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], | |
| int n_entries) | |
| { | |
| program_pat_mcr(gt, table, n_entries); | |
| xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe2_pat_ats.value); | |
| if (IS_DGFX(gt_to_xe(gt))) | |
| xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe2_pat_pta.value); | |
| } | |
| static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], | |
| int n_entries) | |
| { | |
| program_pat(gt, table, n_entries); | |
| xe_mmio_write32(gt, XE_REG(_PAT_ATS), xe2_pat_ats.value); | |
| if (IS_DGFX(gt_to_xe(gt))) | |
| xe_mmio_write32(gt, XE_REG(_PAT_PTA), xe2_pat_pta.value); | |
| } | |
| static void xe2_dump(struct xe_gt *gt, struct drm_printer *p) | |
| { | |
| struct xe_device *xe = gt_to_xe(gt); | |
| int i, err; | |
| u32 pat; | |
| err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
| if (err) | |
| goto err_fw; | |
| drm_printf(p, "PAT table:\n"); | |
| for (i = 0; i < xe->pat.n_entries; i++) { | |
| if (xe_gt_is_media_type(gt)) | |
| pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i))); | |
| else | |
| pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); | |
| drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", i, | |
| !!(pat & XE2_NO_PROMOTE), | |
| !!(pat & XE2_COMP_EN), | |
| REG_FIELD_GET(XE2_L3_CLOS, pat), | |
| REG_FIELD_GET(XE2_L3_POLICY, pat), | |
| REG_FIELD_GET(XE2_L4_POLICY, pat), | |
| REG_FIELD_GET(XE2_COH_MODE, pat), | |
| pat); | |
| } | |
| /* | |
| * Also print PTA_MODE, which describes how the hardware accesses | |
| * PPGTT entries. | |
| */ | |
| if (xe_gt_is_media_type(gt)) | |
| pat = xe_mmio_read32(gt, XE_REG(_PAT_PTA)); | |
| else | |
| pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); | |
| drm_printf(p, "Page Table Access:\n"); | |
| drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", | |
| !!(pat & XE2_NO_PROMOTE), | |
| !!(pat & XE2_COMP_EN), | |
| REG_FIELD_GET(XE2_L3_CLOS, pat), | |
| REG_FIELD_GET(XE2_L3_POLICY, pat), | |
| REG_FIELD_GET(XE2_L4_POLICY, pat), | |
| REG_FIELD_GET(XE2_COH_MODE, pat), | |
| pat); | |
| err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); | |
| err_fw: | |
| xe_assert(xe, !err); | |
| } | |
| static const struct xe_pat_ops xe2_pat_ops = { | |
| .program_graphics = xe2lpg_program_pat, | |
| .program_media = xe2lpm_program_pat, | |
| .dump = xe2_dump, | |
| }; | |
| void xe_pat_init_early(struct xe_device *xe) | |
| { | |
| if (GRAPHICS_VER(xe) == 20) { | |
| xe->pat.ops = &xe2_pat_ops; | |
| xe->pat.table = xe2_pat_table; | |
| /* Wa_16023588340. XXX: Should use XE_WA */ | |
| if (GRAPHICS_VERx100(xe) == 2001) | |
| xe->pat.n_entries = 28; /* Disable CLOS3 */ | |
| else | |
| xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table); | |
| xe->pat.idx[XE_CACHE_NONE] = 3; | |
| xe->pat.idx[XE_CACHE_WT] = 15; | |
| xe->pat.idx[XE_CACHE_WB] = 2; | |
| xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */ | |
| } else if (xe->info.platform == XE_METEORLAKE) { | |
| xe->pat.ops = &xelpg_pat_ops; | |
| xe->pat.table = xelpg_pat_table; | |
| xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table); | |
| xe->pat.idx[XE_CACHE_NONE] = 2; | |
| xe->pat.idx[XE_CACHE_WT] = 1; | |
| xe->pat.idx[XE_CACHE_WB] = 3; | |
| } else if (xe->info.platform == XE_PVC) { | |
| xe->pat.ops = &xehpc_pat_ops; | |
| xe->pat.table = xehpc_pat_table; | |
| xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table); | |
| xe->pat.idx[XE_CACHE_NONE] = 0; | |
| xe->pat.idx[XE_CACHE_WT] = 2; | |
| xe->pat.idx[XE_CACHE_WB] = 3; | |
| } else if (xe->info.platform == XE_DG2) { | |
| /* | |
| * Table is the same as previous platforms, but programming | |
| * method has changed. | |
| */ | |
| xe->pat.ops = &xehp_pat_ops; | |
| xe->pat.table = xelp_pat_table; | |
| xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); | |
| xe->pat.idx[XE_CACHE_NONE] = 3; | |
| xe->pat.idx[XE_CACHE_WT] = 2; | |
| xe->pat.idx[XE_CACHE_WB] = 0; | |
| } else if (GRAPHICS_VERx100(xe) <= 1210) { | |
| WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc); | |
| xe->pat.ops = &xelp_pat_ops; | |
| xe->pat.table = xelp_pat_table; | |
| xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); | |
| xe->pat.idx[XE_CACHE_NONE] = 3; | |
| xe->pat.idx[XE_CACHE_WT] = 2; | |
| xe->pat.idx[XE_CACHE_WB] = 0; | |
| } else { | |
| /* | |
| * Going forward we expect to need new PAT settings for most | |
| * new platforms; failure to provide a new table can easily | |
| * lead to subtle, hard-to-debug problems. If none of the | |
| * conditions above match the platform we're running on we'll | |
| * raise an error rather than trying to silently inherit the | |
| * most recent platform's behavior. | |
| */ | |
| drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", | |
| GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); | |
| } | |
| /* VFs can't program nor dump PAT settings */ | |
| if (IS_SRIOV_VF(xe)) | |
| xe->pat.ops = NULL; | |
| xe_assert(xe, !xe->pat.ops || xe->pat.ops->dump); | |
| xe_assert(xe, !xe->pat.ops || xe->pat.ops->program_graphics); | |
| xe_assert(xe, !xe->pat.ops || MEDIA_VER(xe) < 13 || xe->pat.ops->program_media); | |
| } | |
| void xe_pat_init(struct xe_gt *gt) | |
| { | |
| struct xe_device *xe = gt_to_xe(gt); | |
| if (!xe->pat.ops) | |
| return; | |
| if (xe_gt_is_media_type(gt)) | |
| xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries); | |
| else | |
| xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries); | |
| } | |
| void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) | |
| { | |
| struct xe_device *xe = gt_to_xe(gt); | |
| if (!xe->pat.ops) | |
| return; | |
| xe->pat.ops->dump(gt, p); | |
| } |