drivers/edac/imh_base.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Driver for Intel(R) servers with Integrated Memory/IO Hub-based memory controller.
 * Copyright (c) 2025, Intel Corporation.
 */

#include <linux/kernel.h>
#include <linux/io.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>
#include <asm/mce.h>
#include <asm/cpu.h>
#include "edac_module.h"
#include "skx_common.h"

#define IMH_REVISION	"v0.0.1"
#define EDAC_MOD_STR	"imh_edac"

/* Debug macros */
#define imh_printk(level, fmt, arg...)	\
	edac_printk(level, "imh", fmt, ##arg)

/* Configuration Agent(Ubox) */
#define MMIO_BASE_H(reg)		(((u64)GET_BITFIELD(reg, 0, 29)) << 23)
#define SOCKET_ID(reg)			GET_BITFIELD(reg, 0, 3)

/* PUNIT */
#define DDR_IMC_BITMAP(reg)		GET_BITFIELD(reg, 23, 30)

/* Memory Controller */
#define ECC_ENABLED(reg)		GET_BITFIELD(reg, 2, 2)
#define DIMM_POPULATED(reg)		GET_BITFIELD(reg, 15, 15)

/* System Cache Agent(SCA) */
#define TOLM(reg)			(((u64)GET_BITFIELD(reg, 16, 31)) << 16)
#define TOHM(reg)			(((u64)GET_BITFIELD(reg, 16, 51)) << 16)

/* Home Agent (HA) */
#define NMCACHING(reg)			GET_BITFIELD(reg, 8, 8)

/**
 * struct local_reg - A register as described in the local package view.
 *
 * @pkg: (input)	The package where the register is located.
 * @pbase: (input)	The IP MMIO base physical address in the local package view.
 * @size: (input)	The IP MMIO size.
 * @offset: (input)	The register offset from the IP MMIO base @pbase.
 * @width: (input)	The register width in byte.
 * @vbase: (internal)	The IP MMIO base virtual address.
 * @val: (output)	The register value.
 */
struct local_reg {
	int pkg;
	u64 pbase;
	u32 size;
	u32 offset;
	u8  width;
	void __iomem *vbase;
	u64 val;
};

#define DEFINE_LOCAL_REG(name, cfg, package, north, ip_name, ip_idx, reg_name)	\
	struct local_reg name = {						\
		.pkg	= package,						\
		.pbase	= (north ? (cfg)->mmio_base_l_north :			\
			  (cfg)->mmio_base_l_south) +				\
			  (cfg)->ip_name##_base +				\
			  (cfg)->ip_name##_size * (ip_idx),			\
		.size	= (cfg)->ip_name##_size,				\
		.offset	= (cfg)->ip_name##_reg_##reg_name##_offset,		\
		.width	= (cfg)->ip_name##_reg_##reg_name##_width,		\
	}

static struct res_config *res_cfg;
static int retry_rd_err_log;

#define REG_RRL_DEFINE(a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3)	\
	{								\
		.set_num = 4,						\
		.reg_num = 7,						\
		.sources = {RRL_SRC_FRE_SCRUB, RRL_SRC_FRE_DEMAND, RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND},	\
		.offsets = {									\
			{a0,      a1,      a2,      a3,      a4,      a5,      a6},		\
			{a0 + 4,  a1 + 4,  a2 + 8,  a3 + 4,  a4 + 4,  a5 + 8,  a6 + 8},		\
			{a0 + 8,  a1 + 8,  a2 + 16, a3 + 8,  a4 + 8,  a5 + 16, a6 + 16},	\
			{a0 + 12, a1 + 12, a2 + 24, a3 + 12, a4 + 12, a5 + 24, a6 + 24},	\
		},										\
		.widths		= {4, 4, 8, 4, 4, 8, 8},		\
		.v_mask		= BIT(0),				\
		.uc_mask	= BIT(1),				\
		.over_mask	= BIT(2),				\
		.en_mask	= BIT(12),				\
		.en_patspr_mask	= BIT(14),				\
		.noover_mask	= BIT(15),				\
		.cecnt_num	= 4,					\
		.cecnt_offsets	= {b0, b1, b2, b3},			\
		.cecnt_widths	= {8, 8, 8, 8},				\
}

static struct reg_rrl dmr_reg_rrl_ddr_subch0 = REG_RRL_DEFINE(
	0x2dc0, 0x2dd0, 0x2de0, 0x2e00, 0x2e10, 0x2f70, 0x0200,
	0x2c10, 0x2c18, 0x2c20, 0x2c28);
static struct reg_rrl dmr_reg_rrl_ddr_subch1 = REG_RRL_DEFINE(
	0x6dc0, 0x6dd0, 0x6de0, 0x6e00, 0x6e10, 0x6f70, 0x4200,
	0x6c10, 0x6c18, 0x6c20, 0x6c28);

static void __read_local_reg(void *reg)
{
	struct local_reg *r = (struct local_reg *)reg;

	r->val = skx_readx(r->vbase + r->offset, r->width);
}

/* Read a local-view register. */
static bool read_local_reg(struct local_reg *reg)
{
	int cpu;

	/* Get the target CPU in the package @reg->pkg. */
	for_each_online_cpu(cpu) {
		if (reg->pkg == topology_physical_package_id(cpu))
			break;
	}

	if (cpu >= nr_cpu_ids)
		return false;

	reg->vbase = ioremap(reg->pbase, reg->size);
	if (!reg->vbase) {
		imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", reg->pbase);
		return false;
	}

	/* Get the target CPU to read the register. */
	smp_call_function_single(cpu, __read_local_reg, reg, 1);
	iounmap(reg->vbase);

	return true;
}

/* Get the bitmap of memory controller instances in package @pkg. */
static u32 get_imc_bitmap(struct res_config *cfg, int pkg, bool north)
{
	DEFINE_LOCAL_REG(reg, cfg, pkg, north, pcu, 0, capid3);

	if (!read_local_reg(&reg))
		return 0;

	edac_dbg(2, "Pkg%d %s mc instances bitmap 0x%llx (reg 0x%llx)\n",
		 pkg, north ? "north" : "south",
		 DDR_IMC_BITMAP(reg.val), reg.val);

	return DDR_IMC_BITMAP(reg.val);
}

static void imc_release(struct device *dev)
{
	edac_dbg(2, "imc device %s released\n", dev_name(dev));
	kfree(dev);
}

static int __get_ddr_munits(struct res_config *cfg, struct skx_dev *d,
			    bool north, int lmc)
{
	unsigned long size = cfg->ddr_chan_mmio_sz * cfg->ddr_chan_num;
	unsigned long bitmap = get_imc_bitmap(cfg, d->pkg, north);
	void __iomem *mbase;
	struct device *dev;
	int i, rc, pmc;
	u64 base;

	for_each_set_bit(i, &bitmap, sizeof(bitmap) * 8) {
		base  = north ? d->mmio_base_h_north : d->mmio_base_h_south;
		base += cfg->ddr_imc_base + size * i;

		edac_dbg(2, "Pkg%d mc%d mmio base 0x%llx size 0x%lx\n",
			 d->pkg, lmc, base, size);

		/* Set up the imc MMIO. */
		mbase = ioremap(base, size);
		if (!mbase) {
			imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", base);
			return -ENOMEM;
		}

		d->imc[lmc].mbase = mbase;
		d->imc[lmc].lmc = lmc;

		/* Create the imc device instance. */
		dev = kzalloc_obj(*dev);
		if (!dev)
			return -ENOMEM;

		dev->release = imc_release;
		device_initialize(dev);
		rc = dev_set_name(dev, "0x%llx", base);
		if (rc) {
			imh_printk(KERN_ERR, "Failed to set dev name\n");
			put_device(dev);
			return rc;
		}

		d->imc[lmc].dev = dev;

		/* Set up the imc index mapping. */
		pmc = north ? i : 8 + i;
		skx_set_mc_mapping(d, pmc, lmc);

		lmc++;
	}

	return lmc;
}

static bool get_ddr_munits(struct res_config *cfg, struct skx_dev *d)
{
	int lmc = __get_ddr_munits(cfg, d, true, 0);

	if (lmc < 0)
		return false;

	lmc = __get_ddr_munits(cfg, d, false, lmc);
	if (lmc <= 0)
		return false;

	return true;
}

static bool get_socket_id(struct res_config *cfg, struct skx_dev *d)
{
	DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ubox, 0, socket_id);
	u8 src_id;
	int i;

	if (!read_local_reg(&reg))
		return false;

	src_id = SOCKET_ID(reg.val);
	edac_dbg(2, "socket id 0x%x (reg 0x%llx)\n", src_id, reg.val);

	for (i = 0; i < cfg->ddr_imc_num; i++)
		d->imc[i].src_id   = src_id;

	return true;
}

/* Get TOLM (Top Of Low Memory) and TOHM (Top Of High Memory) parameters. */
static bool imh_get_tolm_tohm(struct res_config *cfg, u64 *tolm, u64 *tohm)
{
	DEFINE_LOCAL_REG(reg, cfg, 0, true, sca, 0, tolm);

	if (!read_local_reg(&reg))
		return false;

	*tolm = TOLM(reg.val);
	edac_dbg(2, "tolm 0x%llx (reg 0x%llx)\n", *tolm, reg.val);

	DEFINE_LOCAL_REG(reg2, cfg, 0, true, sca, 0, tohm);

	if (!read_local_reg(&reg2))
		return false;

	*tohm = TOHM(reg2.val);
	edac_dbg(2, "tohm 0x%llx (reg 0x%llx)\n", *tohm, reg2.val);

	return true;
}

/* Get the system-view MMIO_BASE_H for {north,south}-IMH. */
static int imh_get_all_mmio_base_h(struct res_config *cfg, struct list_head *edac_list)
{
	int i, n = topology_max_packages(), imc_num = cfg->ddr_imc_num + cfg->hbm_imc_num;
	struct skx_dev *d;

	for (i = 0; i < n; i++) {
		d = kzalloc_flex(*d, imc, imc_num);
		if (!d)
			return -ENOMEM;

		DEFINE_LOCAL_REG(reg, cfg, i, true, ubox, 0, mmio_base);

		/* Get MMIO_BASE_H for the north-IMH. */
		if (!read_local_reg(&reg) || !reg.val) {
			kfree(d);
			imh_printk(KERN_ERR, "Pkg%d has no north mmio_base_h\n", i);
			return -ENODEV;
		}

		d->mmio_base_h_north = MMIO_BASE_H(reg.val);
		edac_dbg(2, "Pkg%d north mmio_base_h 0x%llx (reg 0x%llx)\n",
			 i, d->mmio_base_h_north, reg.val);

		/* Get MMIO_BASE_H for the south-IMH (optional). */
		DEFINE_LOCAL_REG(reg2, cfg, i, false, ubox, 0, mmio_base);

		if (read_local_reg(&reg2)) {
			d->mmio_base_h_south = MMIO_BASE_H(reg2.val);
			edac_dbg(2, "Pkg%d south mmio_base_h 0x%llx (reg 0x%llx)\n",
				 i, d->mmio_base_h_south, reg2.val);
		}

		d->pkg = i;
		d->num_imc = imc_num;
		skx_init_mc_mapping(d);
		list_add_tail(&d->list, edac_list);
	}

	return 0;
}

/* Get the number of per-package memory controllers. */
static int imh_get_imc_num(struct res_config *cfg)
{
	int imc_num = hweight32(get_imc_bitmap(cfg, 0, true)) +
		      hweight32(get_imc_bitmap(cfg, 0, false));

	if (!imc_num) {
		imh_printk(KERN_ERR, "Invalid mc number\n");
		return -ENODEV;
	}

	if (cfg->ddr_imc_num != imc_num) {
		/*
		 * Update the configuration data to reflect the number of
		 * present DDR memory controllers.
		 */
		cfg->ddr_imc_num = imc_num;
		edac_dbg(2, "Set ddr mc number %d\n", imc_num);
	}

	return 0;
}

/* Get all memory controllers' parameters. */
static int imh_get_munits(struct res_config *cfg, struct list_head *edac_list)
{
	struct skx_imc *imc;
	struct skx_dev *d;
	u8 mc = 0;
	int i;

	list_for_each_entry(d, edac_list, list) {
		if (!get_ddr_munits(cfg, d)) {
			imh_printk(KERN_ERR, "No mc found\n");
			return -ENODEV;
		}

		if (!get_socket_id(cfg, d)) {
			imh_printk(KERN_ERR, "Failed to get socket id\n");
			return -ENODEV;
		}

		for (i = 0; i < cfg->ddr_imc_num; i++) {
			imc = &d->imc[i];
			if (!imc->mbase)
				continue;

			imc->chan_mmio_sz = cfg->ddr_chan_mmio_sz;
			imc->num_channels = cfg->ddr_chan_num;
			imc->num_dimms    = cfg->ddr_dimm_num;
			imc->mc		  = mc++;
		}
	}

	return 0;
}

static bool check_2lm_enabled(struct res_config *cfg, struct skx_dev *d, int ha_idx)
{
	DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ha, ha_idx, mode);

	if (!read_local_reg(&reg))
		return false;

	if (!NMCACHING(reg.val))
		return false;

	edac_dbg(2, "2-level memory configuration (reg 0x%llx, ha idx %d)\n", reg.val, ha_idx);
	return true;
}

/* Check whether the system has a 2-level memory configuration. */
static bool imh_2lm_enabled(struct res_config *cfg, struct list_head *head)
{
	struct skx_dev *d;
	int i;

	list_for_each_entry(d, head, list) {
		for (i = 0; i < cfg->ddr_imc_num; i++)
			if (check_2lm_enabled(cfg, d, i))
				return true;
	}

	return false;
}

static u32 read_imc_mcmtr(struct res_config *cfg, struct skx_imc *imc, int chan)
{
	return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_mcmtr_offset, cfg->ddr_reg_mcmtr_width);
}

static u32 read_imc_dimmmtr(struct res_config *cfg, struct skx_imc *imc, int chan, int dimm)
{
	return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_dimmmtr_offset +
				     cfg->ddr_reg_dimmmtr_width * dimm,
				     cfg->ddr_reg_dimmmtr_width);
}

static bool ecc_enabled(u32 mcmtr)
{
	return (bool)ECC_ENABLED(mcmtr);
}

static bool dimm_populated(u32 dimmmtr)
{
	return (bool)DIMM_POPULATED(dimmmtr);
}

/* Get each DIMM's configurations of the memory controller @mci. */
static int imh_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg)
{
	struct skx_pvt *pvt = mci->pvt_info;
	struct skx_imc *imc = pvt->imc;
	struct dimm_info *dimm;
	u32 mcmtr, dimmmtr;
	int i, j, ndimms;

	for (i = 0; i < imc->num_channels; i++) {
		if (!imc->mbase)
			continue;

		mcmtr = read_imc_mcmtr(cfg, imc, i);

		for (ndimms = 0, j = 0; j < imc->num_dimms; j++) {
			dimmmtr = read_imc_dimmmtr(cfg, imc, i, j);
			edac_dbg(1, "mcmtr 0x%x dimmmtr 0x%x (mc%d ch%d dimm%d)\n",
				 mcmtr, dimmmtr, imc->mc, i, j);

			if (!dimm_populated(dimmmtr))
				continue;

			dimm = edac_get_dimm(mci, i, j, 0);
			ndimms += skx_get_dimm_info(dimmmtr, 0, 0, dimm,
						    imc, i, j, cfg);
		}

		if (ndimms && !ecc_enabled(mcmtr)) {
			imh_printk(KERN_ERR, "ECC is disabled on mc%d ch%d\n",
				   imc->mc, i);
			return -ENODEV;
		}
	}

	return 0;
}

/* Register all memory controllers to the EDAC core. */
static int imh_register_mci(struct res_config *cfg, struct list_head *edac_list)
{
	struct skx_imc *imc;
	struct skx_dev *d;
	int i, rc;

	list_for_each_entry(d, edac_list, list) {
		for (i = 0; i < cfg->ddr_imc_num; i++) {
			imc = &d->imc[i];
			if (!imc->mbase)
				continue;

			rc = skx_register_mci(imc, imc->dev,
					      dev_name(imc->dev),
					      "Intel IMH-based Socket",
					      EDAC_MOD_STR,
					      imh_get_dimm_config, cfg);
			if (rc)
				return rc;
		}
	}

	return 0;
}

static struct res_config dmr_cfg = {
	.type				= DMR,
	.support_ddr5			= true,
	.mmio_base_l_north		= 0xf6800000,
	.mmio_base_l_south		= 0xf6000000,
	.ddr_chan_num			= 1,
	.ddr_dimm_num			= 2,
	.ddr_imc_base			= 0x39b000,
	.ddr_chan_mmio_sz		= 0x8000,
	.ddr_reg_mcmtr_offset		= 0x360,
	.ddr_reg_mcmtr_width		= 4,
	.ddr_reg_dimmmtr_offset		= 0x370,
	.ddr_reg_dimmmtr_width		= 4,
	.ubox_base			= 0x0,
	.ubox_size			= 0x2000,
	.ubox_reg_mmio_base_offset	= 0x580,
	.ubox_reg_mmio_base_width	= 4,
	.ubox_reg_socket_id_offset	= 0x1080,
	.ubox_reg_socket_id_width	= 4,
	.pcu_base			= 0x3000,
	.pcu_size			= 0x10000,
	.pcu_reg_capid3_offset		= 0x290,
	.pcu_reg_capid3_width		= 4,
	.sca_base			= 0x24c000,
	.sca_size			= 0x2500,
	.sca_reg_tolm_offset		= 0x2100,
	.sca_reg_tolm_width		= 8,
	.sca_reg_tohm_offset		= 0x2108,
	.sca_reg_tohm_width		= 8,
	.ha_base			= 0x3eb000,
	.ha_size			= 0x1000,
	.ha_reg_mode_offset		= 0x4a0,
	.ha_reg_mode_width		= 4,
	.reg_rrl_ddr[0]			= &dmr_reg_rrl_ddr_subch0,
	.reg_rrl_ddr[1]			= &dmr_reg_rrl_ddr_subch1,
};

static const struct x86_cpu_id imh_cpuids[] = {
	X86_MATCH_VFM(INTEL_DIAMONDRAPIDS_X, &dmr_cfg),
	{}
};
MODULE_DEVICE_TABLE(x86cpu, imh_cpuids);

static struct notifier_block imh_mce_dec = {
	.notifier_call	= skx_mce_check_error,
	.priority	= MCE_PRIO_EDAC,
};

static int __init imh_init(void)
{
	const struct x86_cpu_id *id;
	struct list_head *edac_list;
	struct res_config *cfg;
	const char *owner;
	u64 tolm, tohm;
	int rc;

	edac_dbg(2, "\n");

	if (ghes_get_devices())
		return -EBUSY;

	owner = edac_get_owner();
	if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
		return -EBUSY;

	if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
		return -ENODEV;

	id = x86_match_cpu(imh_cpuids);
	if (!id)
		return -ENODEV;
	cfg = (struct res_config *)id->driver_data;
	skx_set_res_cfg(cfg);
	res_cfg = cfg;

	if (!imh_get_tolm_tohm(cfg, &tolm, &tohm))
		return -ENODEV;

	skx_set_hi_lo(tolm, tohm);

	rc = imh_get_imc_num(cfg);
	if (rc < 0)
		goto fail;

	edac_list = skx_get_edac_list();

	rc = imh_get_all_mmio_base_h(cfg, edac_list);
	if (rc)
		goto fail;

	rc = imh_get_munits(cfg, edac_list);
	if (rc)
		goto fail;

	skx_set_mem_cfg(imh_2lm_enabled(cfg, edac_list));

	rc = imh_register_mci(cfg, edac_list);
	if (rc)
		goto fail;

	rc = skx_adxl_get();
	if (rc)
		goto fail;

	opstate_init();
	mce_register_decode_chain(&imh_mce_dec);
	skx_setup_debug("imh_test");

	cfg->rrl_ctrl_mode = retry_rd_err_log;
	if (retry_rd_err_log && cfg->reg_rrl_ddr[0]) {
		skx_set_show_rrl(skx_show_rrl);
		if (retry_rd_err_log == RRL_CTRL_LINUX)
			skx_enable_rrl(true);
	}

	imh_printk(KERN_INFO, "%s\n", IMH_REVISION);

	return 0;
fail:
	skx_remove();
	return rc;
}

static void __exit imh_exit(void)
{
	edac_dbg(2, "\n");

	if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) {
		if (retry_rd_err_log == RRL_CTRL_LINUX)
			skx_enable_rrl(false);
		skx_set_show_rrl(NULL);
	}

	skx_teardown_debug();
	mce_unregister_decode_chain(&imh_mce_dec);
	skx_adxl_put();
	skx_remove();
}

module_init(imh_init);
module_exit(imh_exit);

module_param(retry_rd_err_log, int, 0444);
MODULE_PARM_DESC(retry_rd_err_log, "retry_rd_err_log: 0=off(default), 1=bios(Linux doesn't reset any control bits, but just reports values.), 2=linux(Linux tries to take control and resets mode bits, clear valid/UC bits after reading.)");

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Qiuxu Zhuo");
MODULE_DESCRIPTION("MC Driver for Intel servers using IMH-based memory controller");