blob: 4fce75013674f26d1084a790cdcf956e9d12aa03 [file] [log] [blame]
Thomas Gleixner09c434b2019-05-19 13:08:20 +01001// SPDX-License-Identifier: GPL-2.0-only
Doug Thompson2bc65412009-05-04 20:11:14 +02002#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02003#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02004
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01005static struct edac_pci_ctl_info *pci_ctl;
Doug Thompson2bc65412009-05-04 20:11:14 +02006
Doug Thompson2bc65412009-05-04 20:11:14 +02007/*
8 * Set by command line parameter. If BIOS has enabled the ECC, this override is
9 * cleared to prevent re-enabling the hardware by this driver.
10 */
11static int ecc_enable_override;
12module_param(ecc_enable_override, int, 0644);
13
Tejun Heoa29d8b82010-02-02 14:39:15 +090014static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010015
Yazen Ghannam38ddd4d2019-10-22 20:35:09 +000016static struct amd64_family_type *fam_type;
17
Borislav Petkov2ec591a2015-02-17 10:58:34 +010018/* Per-node stuff */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +020019static struct ecc_settings **ecc_stngs;
Doug Thompson2bc65412009-05-04 20:11:14 +020020
Borislav Petkov706657b12020-11-22 15:57:21 +010021/* Device for the PCI component */
22static struct device *pci_ctl_dev;
23
Doug Thompson2bc65412009-05-04 20:11:14 +020024/*
Borislav Petkovb70ef012009-06-25 19:32:38 +020025 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
26 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
27 * or higher value'.
28 *
29 *FIXME: Produce a better mapping/linearisation.
30 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +080031static const struct scrubrate {
Borislav Petkov39094442010-11-24 19:52:09 +010032 u32 scrubval; /* bit pattern for scrub rate */
33 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
34} scrubrates[] = {
Borislav Petkovb70ef012009-06-25 19:32:38 +020035 { 0x01, 1600000000UL},
36 { 0x02, 800000000UL},
37 { 0x03, 400000000UL},
38 { 0x04, 200000000UL},
39 { 0x05, 100000000UL},
40 { 0x06, 50000000UL},
41 { 0x07, 25000000UL},
42 { 0x08, 12284069UL},
43 { 0x09, 6274509UL},
44 { 0x0A, 3121951UL},
45 { 0x0B, 1560975UL},
46 { 0x0C, 781440UL},
47 { 0x0D, 390720UL},
48 { 0x0E, 195300UL},
49 { 0x0F, 97650UL},
50 { 0x10, 48854UL},
51 { 0x11, 24427UL},
52 { 0x12, 12213UL},
53 { 0x13, 6101UL},
54 { 0x14, 3051UL},
55 { 0x15, 1523UL},
56 { 0x16, 761UL},
57 { 0x00, 0UL}, /* scrubbing off */
58};
59
Borislav Petkov66fed2d2012-08-09 18:41:07 +020060int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
61 u32 *val, const char *func)
Borislav Petkovb2b0c602010-10-08 18:32:29 +020062{
63 int err = 0;
64
65 err = pci_read_config_dword(pdev, offset, val);
66 if (err)
67 amd64_warn("%s: error reading F%dx%03x.\n",
68 func, PCI_FUNC(pdev->devfn), offset);
69
70 return err;
71}
72
73int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
74 u32 val, const char *func)
75{
76 int err = 0;
77
78 err = pci_write_config_dword(pdev, offset, val);
79 if (err)
80 amd64_warn("%s: error writing to F%dx%03x.\n",
81 func, PCI_FUNC(pdev->devfn), offset);
82
83 return err;
84}
85
86/*
Borislav Petkov73ba8592011-09-19 17:34:45 +020087 * Select DCT to which PCI cfg accesses are routed
88 */
89static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
90{
91 u32 reg = 0;
92
93 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -050094 reg &= (pvt->model == 0x30) ? ~3 : ~1;
Borislav Petkov73ba8592011-09-19 17:34:45 +020095 reg |= dct;
96 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
97}
98
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -050099/*
100 *
101 * Depending on the family, F2 DCT reads need special handling:
102 *
103 * K8: has a single DCT only and no address offsets >= 0x100
104 *
105 * F10h: each DCT has its own set of regs
106 * DCT0 -> F2x040..
107 * DCT1 -> F2x140..
108 *
109 * F16h: has only 1 DCT
110 *
111 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
112 */
113static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
114 int offset, u32 *val)
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200115{
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500116 switch (pvt->fam) {
117 case 0xf:
118 if (dct || offset >= 0x100)
119 return -EINVAL;
120 break;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200121
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500122 case 0x10:
123 if (dct) {
124 /*
125 * Note: If ganging is enabled, barring the regs
126 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
127 * return 0. (cf. Section 2.8.1 F10h BKDG)
128 */
129 if (dct_ganging_enabled(pvt))
130 return 0;
131
132 offset += 0x100;
133 }
134 break;
135
136 case 0x15:
137 /*
138 * F15h: F2x1xx addresses do not map explicitly to DCT1.
139 * We should select which DCT we access using F1x10C[DctCfgSel]
140 */
141 dct = (dct && pvt->model == 0x30) ? 3 : dct;
142 f15h_select_dct(pvt, dct);
143 break;
144
145 case 0x16:
146 if (dct)
147 return -EINVAL;
148 break;
149
150 default:
151 break;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200152 }
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500153 return amd64_read_pci_cfg(pvt->F2, offset, val);
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200154}
155
Borislav Petkovb70ef012009-06-25 19:32:38 +0200156/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200157 * Memory scrubber control interface. For K8, memory scrubbing is handled by
158 * hardware and can involve L2 cache, dcache as well as the main memory. With
159 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
160 * functionality.
161 *
162 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
163 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
164 * bytes/sec for the setting.
165 *
166 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
167 * other archs, we might not have access to the caches directly.
168 */
169
Yazen Ghannam8051c0a2016-11-17 17:57:42 -0500170static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
171{
172 /*
173 * Fam17h supports scrub values between 0x5 and 0x14. Also, the values
174 * are shifted down by 0x5, so scrubval 0x5 is written to the register
175 * as 0x0, scrubval 0x6 as 0x1, etc.
176 */
177 if (scrubval >= 0x5 && scrubval <= 0x14) {
178 scrubval -= 0x5;
179 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
180 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
181 } else {
182 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
183 }
184}
Doug Thompson2bc65412009-05-04 20:11:14 +0200185/*
Yazen Ghannam8051c0a2016-11-17 17:57:42 -0500186 * Scan the scrub rate mapping table for a close or matching bandwidth value to
Doug Thompson2bc65412009-05-04 20:11:14 +0200187 * issue. If requested is too big, then use last maximum value found.
188 */
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500189static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200190{
191 u32 scrubval;
192 int i;
193
194 /*
195 * map the configured rate (new_bw) to a value specific to the AMD64
196 * memory controller and apply to register. Search for the first
197 * bandwidth entry that is greater or equal than the setting requested
198 * and program that. If at last entry, turn off DRAM scrubbing.
Andrew Morton168bfee2012-10-23 14:09:39 -0700199 *
200 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
201 * by falling back to the last element in scrubrates[].
Doug Thompson2bc65412009-05-04 20:11:14 +0200202 */
Andrew Morton168bfee2012-10-23 14:09:39 -0700203 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200204 /*
205 * skip scrub rates which aren't recommended
206 * (see F10 BKDG, F3x58)
207 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200208 if (scrubrates[i].scrubval < min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200209 continue;
210
211 if (scrubrates[i].bandwidth <= new_bw)
212 break;
Doug Thompson2bc65412009-05-04 20:11:14 +0200213 }
214
215 scrubval = scrubrates[i].scrubval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200216
Yazen Ghannamdcd01392020-01-10 01:56:51 +0000217 if (pvt->umc) {
Yazen Ghannam8051c0a2016-11-17 17:57:42 -0500218 __f17h_set_scrubval(pvt, scrubval);
219 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500220 f15h_select_dct(pvt, 0);
221 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
222 f15h_select_dct(pvt, 1);
223 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
224 } else {
225 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
226 }
Doug Thompson2bc65412009-05-04 20:11:14 +0200227
Borislav Petkov39094442010-11-24 19:52:09 +0100228 if (scrubval)
229 return scrubrates[i].bandwidth;
230
Doug Thompson2bc65412009-05-04 20:11:14 +0200231 return 0;
232}
233
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100234static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
Doug Thompson2bc65412009-05-04 20:11:14 +0200235{
236 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100237 u32 min_scrubrate = 0x5;
Doug Thompson2bc65412009-05-04 20:11:14 +0200238
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200239 if (pvt->fam == 0xf)
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100240 min_scrubrate = 0x0;
241
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500242 if (pvt->fam == 0x15) {
243 /* Erratum #505 */
244 if (pvt->model < 0x10)
245 f15h_select_dct(pvt, 0);
Borislav Petkov73ba8592011-09-19 17:34:45 +0200246
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500247 if (pvt->model == 0x60)
248 min_scrubrate = 0x6;
249 }
250 return __set_scrub_rate(pvt, bw, min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200251}
252
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100253static int get_scrub_rate(struct mem_ctl_info *mci)
Doug Thompson2bc65412009-05-04 20:11:14 +0200254{
255 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov39094442010-11-24 19:52:09 +0100256 int i, retval = -EINVAL;
Yazen Ghannam8051c0a2016-11-17 17:57:42 -0500257 u32 scrubval = 0;
Doug Thompson2bc65412009-05-04 20:11:14 +0200258
Yazen Ghannamdcd01392020-01-10 01:56:51 +0000259 if (pvt->umc) {
Yazen Ghannam8051c0a2016-11-17 17:57:42 -0500260 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
261 if (scrubval & BIT(0)) {
262 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
263 scrubval &= 0xF;
264 scrubval += 0x5;
265 } else {
266 scrubval = 0;
267 }
Yazen Ghannamdcd01392020-01-10 01:56:51 +0000268 } else if (pvt->fam == 0x15) {
269 /* Erratum #505 */
270 if (pvt->model < 0x10)
271 f15h_select_dct(pvt, 0);
Yazen Ghannam8051c0a2016-11-17 17:57:42 -0500272
Yazen Ghannamdcd01392020-01-10 01:56:51 +0000273 if (pvt->model == 0x60)
274 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
Borislav Petkovee470bb2020-06-18 20:25:25 +0200275 else
276 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
Yazen Ghannamdcd01392020-01-10 01:56:51 +0000277 } else {
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500278 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
Yazen Ghannam8051c0a2016-11-17 17:57:42 -0500279 }
Doug Thompson2bc65412009-05-04 20:11:14 +0200280
281 scrubval = scrubval & 0x001F;
282
Roel Kluin926311f2010-01-11 20:58:21 +0100283 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200284 if (scrubrates[i].scrubval == scrubval) {
Borislav Petkov39094442010-11-24 19:52:09 +0100285 retval = scrubrates[i].bandwidth;
Doug Thompson2bc65412009-05-04 20:11:14 +0200286 break;
287 }
288 }
Borislav Petkov39094442010-11-24 19:52:09 +0100289 return retval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200290}
291
Doug Thompson67757632009-04-27 15:53:22 +0200292/*
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200293 * returns true if the SysAddr given by sys_addr matches the
294 * DRAM base/limit associated with node_id
Doug Thompson67757632009-04-27 15:53:22 +0200295 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100296static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
Doug Thompson67757632009-04-27 15:53:22 +0200297{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200298 u64 addr;
Doug Thompson67757632009-04-27 15:53:22 +0200299
300 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
301 * all ones if the most significant implemented address bit is 1.
302 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
303 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
304 * Application Programming.
305 */
306 addr = sys_addr & 0x000000ffffffffffull;
307
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200308 return ((addr >= get_dram_base(pvt, nid)) &&
309 (addr <= get_dram_limit(pvt, nid)));
Doug Thompson67757632009-04-27 15:53:22 +0200310}
311
312/*
313 * Attempt to map a SysAddr to a node. On success, return a pointer to the
314 * mem_ctl_info structure for the node that the SysAddr maps to.
315 *
316 * On failure, return NULL.
317 */
318static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
319 u64 sys_addr)
320{
321 struct amd64_pvt *pvt;
Daniel J Bluemanc7e53012012-11-30 16:44:20 +0800322 u8 node_id;
Doug Thompson67757632009-04-27 15:53:22 +0200323 u32 intlv_en, bits;
324
325 /*
326 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
327 * 3.4.4.2) registers to map the SysAddr to a node ID.
328 */
329 pvt = mci->pvt_info;
330
331 /*
332 * The value of this field should be the same for all DRAM Base
333 * registers. Therefore we arbitrarily choose to read it from the
334 * register for node 0.
335 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200336 intlv_en = dram_intlv_en(pvt, 0);
Doug Thompson67757632009-04-27 15:53:22 +0200337
338 if (intlv_en == 0) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200339 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100340 if (base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200341 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200342 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200343 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200344 }
345
Borislav Petkov72f158f2009-09-18 12:27:27 +0200346 if (unlikely((intlv_en != 0x01) &&
347 (intlv_en != 0x03) &&
348 (intlv_en != 0x07))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200349 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200350 return NULL;
351 }
352
353 bits = (((u32) sys_addr) >> 12) & intlv_en;
354
355 for (node_id = 0; ; ) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200356 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200357 break; /* intlv_sel field matches */
358
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200359 if (++node_id >= DRAM_RANGES)
Doug Thompson67757632009-04-27 15:53:22 +0200360 goto err_no_match;
361 }
362
363 /* sanity test for sys_addr */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100364 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200365 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
366 "range for node %d with node interleaving enabled.\n",
367 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200368 return NULL;
369 }
370
371found:
Borislav Petkovb487c332011-02-21 18:55:00 +0100372 return edac_mc_find((int)node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200373
374err_no_match:
Joe Perches956b9ba12012-04-29 17:08:39 -0300375 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
376 (unsigned long)sys_addr);
Doug Thompson67757632009-04-27 15:53:22 +0200377
378 return NULL;
379}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200380
381/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100382 * compute the CS base address of the @csrow on the DRAM controller @dct.
383 * For details see F2x[5C:40] in the processor's BKDG
Doug Thompsone2ce7252009-04-27 15:57:12 +0200384 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100385static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
386 u64 *base, u64 *mask)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200387{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100388 u64 csbase, csmask, base_bits, mask_bits;
389 u8 addr_shift;
390
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -0500391 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100392 csbase = pvt->csels[dct].csbases[csrow];
393 csmask = pvt->csels[dct].csmasks[csrow];
Chen, Gong10ef6b02013-10-18 14:29:07 -0700394 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
395 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100396 addr_shift = 4;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500397
398 /*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -0500399 * F16h and F15h, models 30h and later need two addr_shift values:
400 * 8 for high and 6 for low (cf. F16h BKDG).
401 */
402 } else if (pvt->fam == 0x16 ||
403 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500404 csbase = pvt->csels[dct].csbases[csrow];
405 csmask = pvt->csels[dct].csmasks[csrow >> 1];
406
Chen, Gong10ef6b02013-10-18 14:29:07 -0700407 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
408 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500409
410 *mask = ~0ULL;
411 /* poke holes for the csmask */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700412 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
413 (GENMASK_ULL(30, 19) << 8));
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500414
Chen, Gong10ef6b02013-10-18 14:29:07 -0700415 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
416 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500417
418 return;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100419 } else {
420 csbase = pvt->csels[dct].csbases[csrow];
421 csmask = pvt->csels[dct].csmasks[csrow >> 1];
422 addr_shift = 8;
423
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200424 if (pvt->fam == 0x15)
Chen, Gong10ef6b02013-10-18 14:29:07 -0700425 base_bits = mask_bits =
426 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100427 else
Chen, Gong10ef6b02013-10-18 14:29:07 -0700428 base_bits = mask_bits =
429 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100430 }
431
432 *base = (csbase & base_bits) << addr_shift;
433
434 *mask = ~0ULL;
435 /* poke holes for the csmask */
436 *mask &= ~(mask_bits << addr_shift);
437 /* OR them in */
438 *mask |= (csmask & mask_bits) << addr_shift;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200439}
440
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100441#define for_each_chip_select(i, dct, pvt) \
442 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200443
Borislav Petkov614ec9d2011-01-13 18:02:22 +0100444#define chip_select_base(i, dct, pvt) \
445 pvt->csels[dct].csbases[i]
446
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100447#define for_each_chip_select_mask(i, dct, pvt) \
448 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200449
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +0000450#define for_each_umc(i) \
Yazen Ghannam5e4c5522019-10-22 20:35:11 +0000451 for (i = 0; i < fam_type->max_mcs; i++)
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +0000452
Doug Thompsone2ce7252009-04-27 15:57:12 +0200453/*
454 * @input_addr is an InputAddr associated with the node given by mci. Return the
455 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
456 */
457static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
458{
459 struct amd64_pvt *pvt;
460 int csrow;
461 u64 base, mask;
462
463 pvt = mci->pvt_info;
464
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100465 for_each_chip_select(csrow, 0, pvt) {
466 if (!csrow_enabled(csrow, 0, pvt))
Doug Thompsone2ce7252009-04-27 15:57:12 +0200467 continue;
468
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100469 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
470
471 mask = ~mask;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200472
473 if ((input_addr & mask) == (base & mask)) {
Joe Perches956b9ba12012-04-29 17:08:39 -0300474 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
475 (unsigned long)input_addr, csrow,
476 pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200477
478 return csrow;
479 }
480 }
Joe Perches956b9ba12012-04-29 17:08:39 -0300481 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
482 (unsigned long)input_addr, pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200483
484 return -1;
485}
486
487/*
Doug Thompsone2ce7252009-04-27 15:57:12 +0200488 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
489 * for the node represented by mci. Info is passed back in *hole_base,
490 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
491 * info is invalid. Info may be invalid for either of the following reasons:
492 *
493 * - The revision of the node is not E or greater. In this case, the DRAM Hole
494 * Address Register does not exist.
495 *
496 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
497 * indicating that its contents are not valid.
498 *
499 * The values passed back in *hole_base, *hole_offset, and *hole_size are
500 * complete 32-bit values despite the fact that the bitfields in the DHAR
501 * only represent bits 31-24 of the base and offset values.
502 */
Borislav Petkov2a28ceef2020-12-14 20:47:11 +0100503static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
504 u64 *hole_offset, u64 *hole_size)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200505{
506 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200507
508 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200509 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
Joe Perches956b9ba12012-04-29 17:08:39 -0300510 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
511 pvt->ext_model, pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200512 return 1;
513 }
514
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100515 /* valid for Fam10h and above */
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200516 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
Joe Perches956b9ba12012-04-29 17:08:39 -0300517 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
Doug Thompsone2ce7252009-04-27 15:57:12 +0200518 return 1;
519 }
520
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100521 if (!dhar_valid(pvt)) {
Joe Perches956b9ba12012-04-29 17:08:39 -0300522 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
523 pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200524 return 1;
525 }
526
527 /* This node has Memory Hoisting */
528
529 /* +------------------+--------------------+--------------------+-----
530 * | memory | DRAM hole | relocated |
531 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
532 * | | | DRAM hole |
533 * | | | [0x100000000, |
534 * | | | (0x100000000+ |
535 * | | | (0xffffffff-x))] |
536 * +------------------+--------------------+--------------------+-----
537 *
538 * Above is a diagram of physical memory showing the DRAM hole and the
539 * relocated addresses from the DRAM hole. As shown, the DRAM hole
540 * starts at address x (the base address) and extends through address
541 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
542 * addresses in the hole so that they start at 0x100000000.
543 */
544
Borislav Petkov1f316772012-08-10 12:50:50 +0200545 *hole_base = dhar_base(pvt);
546 *hole_size = (1ULL << 32) - *hole_base;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200547
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200548 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
549 : k8_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200550
Joe Perches956b9ba12012-04-29 17:08:39 -0300551 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
552 pvt->mc_node_id, (unsigned long)*hole_base,
553 (unsigned long)*hole_offset, (unsigned long)*hole_size);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200554
555 return 0;
556}
Borislav Petkov2a28ceef2020-12-14 20:47:11 +0100557
558#ifdef CONFIG_EDAC_DEBUG
559#define EDAC_DCT_ATTR_SHOW(reg) \
560static ssize_t reg##_show(struct device *dev, \
561 struct device_attribute *mattr, char *data) \
562{ \
563 struct mem_ctl_info *mci = to_mci(dev); \
564 struct amd64_pvt *pvt = mci->pvt_info; \
565 \
566 return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \
567}
568
569EDAC_DCT_ATTR_SHOW(dhar);
570EDAC_DCT_ATTR_SHOW(dbam0);
571EDAC_DCT_ATTR_SHOW(top_mem);
572EDAC_DCT_ATTR_SHOW(top_mem2);
573
Dwaipayan Rayd19faf02021-07-13 12:21:30 +0530574static ssize_t dram_hole_show(struct device *dev, struct device_attribute *mattr,
575 char *data)
Borislav Petkov2a28ceef2020-12-14 20:47:11 +0100576{
577 struct mem_ctl_info *mci = to_mci(dev);
578
579 u64 hole_base = 0;
580 u64 hole_offset = 0;
581 u64 hole_size = 0;
582
583 get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
584
585 return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset,
586 hole_size);
587}
588
589/*
590 * update NUM_DBG_ATTRS in case you add new members
591 */
592static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL);
593static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL);
594static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL);
595static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL);
Dwaipayan Rayd19faf02021-07-13 12:21:30 +0530596static DEVICE_ATTR_RO(dram_hole);
Borislav Petkov2a28ceef2020-12-14 20:47:11 +0100597
598static struct attribute *dbg_attrs[] = {
599 &dev_attr_dhar.attr,
600 &dev_attr_dbam.attr,
601 &dev_attr_topmem.attr,
602 &dev_attr_topmem2.attr,
603 &dev_attr_dram_hole.attr,
604 NULL
605};
606
607static const struct attribute_group dbg_group = {
608 .attrs = dbg_attrs,
609};
Borislav Petkov2a28ceef2020-12-14 20:47:11 +0100610
Borislav Petkov61810092020-12-15 09:18:44 +0100611static ssize_t inject_section_show(struct device *dev,
612 struct device_attribute *mattr, char *buf)
613{
614 struct mem_ctl_info *mci = to_mci(dev);
615 struct amd64_pvt *pvt = mci->pvt_info;
616 return sprintf(buf, "0x%x\n", pvt->injection.section);
617}
618
619/*
620 * store error injection section value which refers to one of 4 16-byte sections
621 * within a 64-byte cacheline
622 *
623 * range: 0..3
624 */
625static ssize_t inject_section_store(struct device *dev,
626 struct device_attribute *mattr,
627 const char *data, size_t count)
628{
629 struct mem_ctl_info *mci = to_mci(dev);
630 struct amd64_pvt *pvt = mci->pvt_info;
631 unsigned long value;
632 int ret;
633
634 ret = kstrtoul(data, 10, &value);
635 if (ret < 0)
636 return ret;
637
638 if (value > 3) {
639 amd64_warn("%s: invalid section 0x%lx\n", __func__, value);
640 return -EINVAL;
641 }
642
643 pvt->injection.section = (u32) value;
644 return count;
645}
646
647static ssize_t inject_word_show(struct device *dev,
648 struct device_attribute *mattr, char *buf)
649{
650 struct mem_ctl_info *mci = to_mci(dev);
651 struct amd64_pvt *pvt = mci->pvt_info;
652 return sprintf(buf, "0x%x\n", pvt->injection.word);
653}
654
655/*
656 * store error injection word value which refers to one of 9 16-bit word of the
657 * 16-byte (128-bit + ECC bits) section
658 *
659 * range: 0..8
660 */
661static ssize_t inject_word_store(struct device *dev,
662 struct device_attribute *mattr,
663 const char *data, size_t count)
664{
665 struct mem_ctl_info *mci = to_mci(dev);
666 struct amd64_pvt *pvt = mci->pvt_info;
667 unsigned long value;
668 int ret;
669
670 ret = kstrtoul(data, 10, &value);
671 if (ret < 0)
672 return ret;
673
674 if (value > 8) {
675 amd64_warn("%s: invalid word 0x%lx\n", __func__, value);
676 return -EINVAL;
677 }
678
679 pvt->injection.word = (u32) value;
680 return count;
681}
682
683static ssize_t inject_ecc_vector_show(struct device *dev,
684 struct device_attribute *mattr,
685 char *buf)
686{
687 struct mem_ctl_info *mci = to_mci(dev);
688 struct amd64_pvt *pvt = mci->pvt_info;
689 return sprintf(buf, "0x%x\n", pvt->injection.bit_map);
690}
691
692/*
693 * store 16 bit error injection vector which enables injecting errors to the
694 * corresponding bit within the error injection word above. When used during a
695 * DRAM ECC read, it holds the contents of the of the DRAM ECC bits.
696 */
697static ssize_t inject_ecc_vector_store(struct device *dev,
698 struct device_attribute *mattr,
699 const char *data, size_t count)
700{
701 struct mem_ctl_info *mci = to_mci(dev);
702 struct amd64_pvt *pvt = mci->pvt_info;
703 unsigned long value;
704 int ret;
705
706 ret = kstrtoul(data, 16, &value);
707 if (ret < 0)
708 return ret;
709
710 if (value & 0xFFFF0000) {
711 amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value);
712 return -EINVAL;
713 }
714
715 pvt->injection.bit_map = (u32) value;
716 return count;
717}
718
719/*
720 * Do a DRAM ECC read. Assemble staged values in the pvt area, format into
721 * fields needed by the injection registers and read the NB Array Data Port.
722 */
723static ssize_t inject_read_store(struct device *dev,
724 struct device_attribute *mattr,
725 const char *data, size_t count)
726{
727 struct mem_ctl_info *mci = to_mci(dev);
728 struct amd64_pvt *pvt = mci->pvt_info;
729 unsigned long value;
730 u32 section, word_bits;
731 int ret;
732
733 ret = kstrtoul(data, 10, &value);
734 if (ret < 0)
735 return ret;
736
737 /* Form value to choose 16-byte section of cacheline */
738 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
739
740 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
741
742 word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection);
743
744 /* Issue 'word' and 'bit' along with the READ request */
745 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
746
747 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
748
749 return count;
750}
751
752/*
753 * Do a DRAM ECC write. Assemble staged values in the pvt area and format into
754 * fields needed by the injection registers.
755 */
756static ssize_t inject_write_store(struct device *dev,
757 struct device_attribute *mattr,
758 const char *data, size_t count)
759{
760 struct mem_ctl_info *mci = to_mci(dev);
761 struct amd64_pvt *pvt = mci->pvt_info;
762 u32 section, word_bits, tmp;
763 unsigned long value;
764 int ret;
765
766 ret = kstrtoul(data, 10, &value);
767 if (ret < 0)
768 return ret;
769
770 /* Form value to choose 16-byte section of cacheline */
771 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
772
773 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
774
775 word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection);
776
777 pr_notice_once("Don't forget to decrease MCE polling interval in\n"
778 "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
779 "so that you can get the error report faster.\n");
780
781 on_each_cpu(disable_caches, NULL, 1);
782
783 /* Issue 'word' and 'bit' along with the READ request */
784 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
785
786 retry:
787 /* wait until injection happens */
788 amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp);
789 if (tmp & F10_NB_ARR_ECC_WR_REQ) {
790 cpu_relax();
791 goto retry;
792 }
793
794 on_each_cpu(enable_caches, NULL, 1);
795
796 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
797
798 return count;
799}
800
801/*
802 * update NUM_INJ_ATTRS in case you add new members
803 */
804
Dwaipayan Rayd19faf02021-07-13 12:21:30 +0530805static DEVICE_ATTR_RW(inject_section);
806static DEVICE_ATTR_RW(inject_word);
807static DEVICE_ATTR_RW(inject_ecc_vector);
808static DEVICE_ATTR_WO(inject_write);
809static DEVICE_ATTR_WO(inject_read);
Borislav Petkov61810092020-12-15 09:18:44 +0100810
811static struct attribute *inj_attrs[] = {
812 &dev_attr_inject_section.attr,
813 &dev_attr_inject_word.attr,
814 &dev_attr_inject_ecc_vector.attr,
815 &dev_attr_inject_write.attr,
816 &dev_attr_inject_read.attr,
817 NULL
818};
819
820static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
821{
822 struct device *dev = kobj_to_dev(kobj);
823 struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
824 struct amd64_pvt *pvt = mci->pvt_info;
825
Borislav Petkov1865bc72020-12-22 18:55:06 +0100826 /* Families which have that injection hw */
827 if (pvt->fam >= 0x10 && pvt->fam <= 0x16)
828 return attr->mode;
829
830 return 0;
Borislav Petkov61810092020-12-15 09:18:44 +0100831}
832
833static const struct attribute_group inj_group = {
834 .attrs = inj_attrs,
835 .is_visible = inj_is_visible,
836};
837#endif /* CONFIG_EDAC_DEBUG */
Doug Thompsone2ce7252009-04-27 15:57:12 +0200838
Doug Thompson93c2df52009-05-04 20:46:50 +0200839/*
840 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
841 * assumed that sys_addr maps to the node given by mci.
842 *
843 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
844 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
845 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
846 * then it is also involved in translating a SysAddr to a DramAddr. Sections
847 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
848 * These parts of the documentation are unclear. I interpret them as follows:
849 *
850 * When node n receives a SysAddr, it processes the SysAddr as follows:
851 *
852 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
853 * Limit registers for node n. If the SysAddr is not within the range
854 * specified by the base and limit values, then node n ignores the Sysaddr
855 * (since it does not map to node n). Otherwise continue to step 2 below.
856 *
857 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
858 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
859 * the range of relocated addresses (starting at 0x100000000) from the DRAM
860 * hole. If not, skip to step 3 below. Else get the value of the
861 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
862 * offset defined by this value from the SysAddr.
863 *
864 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
865 * Base register for node n. To obtain the DramAddr, subtract the base
866 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
867 */
868static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
869{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200870 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson93c2df52009-05-04 20:46:50 +0200871 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
Borislav Petkov1f316772012-08-10 12:50:50 +0200872 int ret;
Doug Thompson93c2df52009-05-04 20:46:50 +0200873
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200874 dram_base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200875
Borislav Petkov2a28ceef2020-12-14 20:47:11 +0100876 ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
Doug Thompson93c2df52009-05-04 20:46:50 +0200877 if (!ret) {
Borislav Petkov1f316772012-08-10 12:50:50 +0200878 if ((sys_addr >= (1ULL << 32)) &&
879 (sys_addr < ((1ULL << 32) + hole_size))) {
Doug Thompson93c2df52009-05-04 20:46:50 +0200880 /* use DHAR to translate SysAddr to DramAddr */
881 dram_addr = sys_addr - hole_offset;
882
Joe Perches956b9ba12012-04-29 17:08:39 -0300883 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
884 (unsigned long)sys_addr,
885 (unsigned long)dram_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200886
887 return dram_addr;
888 }
889 }
890
891 /*
892 * Translate the SysAddr to a DramAddr as shown near the start of
893 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
894 * only deals with 40-bit values. Therefore we discard bits 63-40 of
895 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
896 * discard are all 1s. Otherwise the bits we discard are all 0s. See
897 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
898 * Programmer's Manual Volume 1 Application Programming.
899 */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700900 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
Doug Thompson93c2df52009-05-04 20:46:50 +0200901
Joe Perches956b9ba12012-04-29 17:08:39 -0300902 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
903 (unsigned long)sys_addr, (unsigned long)dram_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200904 return dram_addr;
905}
906
907/*
908 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
909 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
910 * for node interleaving.
911 */
912static int num_node_interleave_bits(unsigned intlv_en)
913{
914 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
915 int n;
916
917 BUG_ON(intlv_en > 7);
918 n = intlv_shift_table[intlv_en];
919 return n;
920}
921
922/* Translate the DramAddr given by @dram_addr to an InputAddr. */
923static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
924{
925 struct amd64_pvt *pvt;
926 int intlv_shift;
927 u64 input_addr;
928
929 pvt = mci->pvt_info;
930
931 /*
932 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
933 * concerning translating a DramAddr to an InputAddr.
934 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200935 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Chen, Gong10ef6b02013-10-18 14:29:07 -0700936 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100937 (dram_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200938
Joe Perches956b9ba12012-04-29 17:08:39 -0300939 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
940 intlv_shift, (unsigned long)dram_addr,
941 (unsigned long)input_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200942
943 return input_addr;
944}
945
946/*
947 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
948 * assumed that @sys_addr maps to the node given by mci.
949 */
950static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
951{
952 u64 input_addr;
953
954 input_addr =
955 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
956
Masanari Iidac19ca6c2016-02-08 20:53:12 +0900957 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
Joe Perches956b9ba12012-04-29 17:08:39 -0300958 (unsigned long)sys_addr, (unsigned long)input_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200959
960 return input_addr;
961}
962
Doug Thompson93c2df52009-05-04 20:46:50 +0200963/* Map the Error address to a PAGE and PAGE OFFSET. */
964static inline void error_address_to_page_and_offset(u64 error_address,
Borislav Petkov33ca0642012-08-30 18:01:36 +0200965 struct err_info *err)
Doug Thompson93c2df52009-05-04 20:46:50 +0200966{
Borislav Petkov33ca0642012-08-30 18:01:36 +0200967 err->page = (u32) (error_address >> PAGE_SHIFT);
968 err->offset = ((u32) error_address) & ~PAGE_MASK;
Doug Thompson93c2df52009-05-04 20:46:50 +0200969}
970
971/*
972 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
973 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
974 * of a node that detected an ECC memory error. mci represents the node that
975 * the error address maps to (possibly different from the node that detected
976 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
977 * error.
978 */
979static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
980{
981 int csrow;
982
983 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
984
985 if (csrow == -1)
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200986 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
987 "address 0x%lx\n", (unsigned long)sys_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200988 return csrow;
989}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200990
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100991static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200992
Doug Thompson2da11652009-04-27 16:09:09 +0200993/*
994 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
995 * are ECC capable.
996 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100997static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200998{
Dan Carpenter1f6189e2011-10-06 02:30:25 -0400999 unsigned long edac_cap = EDAC_FLAG_NONE;
Yazen Ghannamd27f3a32016-11-17 17:57:40 -05001000 u8 bit;
Doug Thompson2da11652009-04-27 16:09:09 +02001001
Yazen Ghannamd27f3a32016-11-17 17:57:40 -05001002 if (pvt->umc) {
1003 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
Doug Thompson2da11652009-04-27 16:09:09 +02001004
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +00001005 for_each_umc(i) {
Yazen Ghannamd27f3a32016-11-17 17:57:40 -05001006 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
1007 continue;
1008
1009 umc_en_mask |= BIT(i);
1010
1011 /* UMC Configuration bit 12 (DimmEccEn) */
1012 if (pvt->umc[i].umc_cfg & BIT(12))
1013 dimm_ecc_en_mask |= BIT(i);
1014 }
1015
1016 if (umc_en_mask == dimm_ecc_en_mask)
1017 edac_cap = EDAC_FLAG_SECDED;
1018 } else {
1019 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
1020 ? 19
1021 : 17;
1022
1023 if (pvt->dclr0 & BIT(bit))
1024 edac_cap = EDAC_FLAG_SECDED;
1025 }
Doug Thompson2da11652009-04-27 16:09:09 +02001026
1027 return edac_cap;
1028}
1029
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001030static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
Doug Thompson2da11652009-04-27 16:09:09 +02001031
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001032static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
Borislav Petkov68798e12009-11-03 16:18:33 +01001033{
Joe Perches956b9ba12012-04-29 17:08:39 -03001034 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
Borislav Petkov68798e12009-11-03 16:18:33 +01001035
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001036 if (pvt->dram_type == MEM_LRDDR3) {
1037 u32 dcsm = pvt->csels[chan].csmasks[0];
1038 /*
1039 * It's assumed all LRDIMMs in a DCT are going to be of
1040 * same 'type' until proven otherwise. So, use a cs
1041 * value of '0' here to get dcsm value.
1042 */
1043 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
1044 }
1045
1046 edac_dbg(1, "All DIMMs support ECC:%s\n",
1047 (dclr & BIT(19)) ? "yes" : "no");
1048
Borislav Petkov68798e12009-11-03 16:18:33 +01001049
Joe Perches956b9ba12012-04-29 17:08:39 -03001050 edac_dbg(1, " PAR/ERR parity: %s\n",
1051 (dclr & BIT(8)) ? "enabled" : "disabled");
Borislav Petkov68798e12009-11-03 16:18:33 +01001052
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001053 if (pvt->fam == 0x10)
Joe Perches956b9ba12012-04-29 17:08:39 -03001054 edac_dbg(1, " DCT 128bit mode width: %s\n",
1055 (dclr & BIT(11)) ? "128b" : "64b");
Borislav Petkov68798e12009-11-03 16:18:33 +01001056
Joe Perches956b9ba12012-04-29 17:08:39 -03001057 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
1058 (dclr & BIT(12)) ? "yes" : "no",
1059 (dclr & BIT(13)) ? "yes" : "no",
1060 (dclr & BIT(14)) ? "yes" : "no",
1061 (dclr & BIT(15)) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +01001062}
1063
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001064#define CS_EVEN_PRIMARY BIT(0)
1065#define CS_ODD_PRIMARY BIT(1)
Yazen Ghannam81f50902019-08-22 00:00:02 +00001066#define CS_EVEN_SECONDARY BIT(2)
1067#define CS_ODD_SECONDARY BIT(3)
Yazen Ghannam9f4873f2021-10-05 15:44:19 +00001068#define CS_3R_INTERLEAVE BIT(4)
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001069
Yazen Ghannam81f50902019-08-22 00:00:02 +00001070#define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
1071#define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001072
1073static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
Yazen Ghannamfc00c6a2019-02-28 15:36:12 +00001074{
Yazen Ghannam9f4873f2021-10-05 15:44:19 +00001075 u8 base, count = 0;
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001076 int cs_mode = 0;
Yazen Ghannamfc00c6a2019-02-28 15:36:12 +00001077
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001078 if (csrow_enabled(2 * dimm, ctrl, pvt))
1079 cs_mode |= CS_EVEN_PRIMARY;
Yazen Ghannamfc00c6a2019-02-28 15:36:12 +00001080
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001081 if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
1082 cs_mode |= CS_ODD_PRIMARY;
1083
Yazen Ghannam81f50902019-08-22 00:00:02 +00001084 /* Asymmetric dual-rank DIMM support. */
1085 if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
1086 cs_mode |= CS_ODD_SECONDARY;
1087
Yazen Ghannam9f4873f2021-10-05 15:44:19 +00001088 /*
1089 * 3 Rank inteleaving support.
1090 * There should be only three bases enabled and their two masks should
1091 * be equal.
1092 */
1093 for_each_chip_select(base, ctrl, pvt)
1094 count += csrow_enabled(base, ctrl, pvt);
1095
1096 if (count == 3 &&
1097 pvt->csels[ctrl].csmasks[0] == pvt->csels[ctrl].csmasks[1]) {
1098 edac_dbg(1, "3R interleaving in use.\n");
1099 cs_mode |= CS_3R_INTERLEAVE;
1100 }
1101
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001102 return cs_mode;
Yazen Ghannamfc00c6a2019-02-28 15:36:12 +00001103}
1104
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001105static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
1106{
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001107 int dimm, size0, size1, cs0, cs1, cs_mode;
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001108
1109 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
1110
Yazen Ghannamd971e282019-08-21 23:59:55 +00001111 for (dimm = 0; dimm < 2; dimm++) {
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05001112 cs0 = dimm * 2;
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05001113 cs1 = dimm * 2 + 1;
1114
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001115 cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
1116
1117 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
1118 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001119
1120 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05001121 cs0, size0,
1122 cs1, size1);
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001123 }
1124}
1125
1126static void __dump_misc_regs_df(struct amd64_pvt *pvt)
1127{
1128 struct amd64_umc *umc;
1129 u32 i, tmp, umc_base;
1130
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +00001131 for_each_umc(i) {
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001132 umc_base = get_umc_base(i);
1133 umc = &pvt->umc[i];
1134
1135 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
1136 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
1137 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
1138 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
1139
1140 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
1141 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
1142
1143 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
1144 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
1145 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
1146
1147 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
1148 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
1149 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
1150 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
1151 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
1152 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
1153 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
1154 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
1155 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
1156
1157 if (pvt->dram_type == MEM_LRDDR4) {
1158 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
1159 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
1160 i, 1 << ((tmp >> 4) & 0x3));
1161 }
1162
1163 debug_display_dimm_sizes_df(pvt, i);
1164 }
1165
1166 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
1167 pvt->dhar, dhar_base(pvt));
1168}
1169
Doug Thompson2da11652009-04-27 16:09:09 +02001170/* Display and decode various NB registers for debug purposes. */
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001171static void __dump_misc_regs(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +02001172{
Joe Perches956b9ba12012-04-29 17:08:39 -03001173 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +02001174
Joe Perches956b9ba12012-04-29 17:08:39 -03001175 edac_dbg(1, " NB two channel DRAM capable: %s\n",
1176 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +01001177
Joe Perches956b9ba12012-04-29 17:08:39 -03001178 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
1179 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
1180 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +01001181
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001182 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +02001183
Joe Perches956b9ba12012-04-29 17:08:39 -03001184 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +02001185
Joe Perches956b9ba12012-04-29 17:08:39 -03001186 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1187 pvt->dhar, dhar_base(pvt),
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001188 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
1189 : f10_dhar_offset(pvt));
Doug Thompson2da11652009-04-27 16:09:09 +02001190
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001191 debug_display_dimm_sizes(pvt, 0);
Borislav Petkov4d796362011-02-03 15:59:57 +01001192
Borislav Petkov8de1d912009-10-16 13:39:30 +02001193 /* everything below this point is Fam10h and above */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001194 if (pvt->fam == 0xf)
Doug Thompson2da11652009-04-27 16:09:09 +02001195 return;
Borislav Petkov4d796362011-02-03 15:59:57 +01001196
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001197 debug_display_dimm_sizes(pvt, 1);
Doug Thompson2da11652009-04-27 16:09:09 +02001198
Borislav Petkov8de1d912009-10-16 13:39:30 +02001199 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +01001200 if (!dct_ganging_enabled(pvt))
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001201 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +02001202}
1203
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001204/* Display and decode various NB registers for debug purposes. */
1205static void dump_misc_regs(struct amd64_pvt *pvt)
1206{
1207 if (pvt->umc)
1208 __dump_misc_regs_df(pvt);
1209 else
1210 __dump_misc_regs(pvt);
1211
1212 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
1213
Yazen Ghannam78359612019-02-28 15:36:11 +00001214 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06001215}
1216
Doug Thompson94be4bf2009-04-27 16:12:00 +02001217/*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001218 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
Doug Thompson94be4bf2009-04-27 16:12:00 +02001219 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001220static void prep_chip_selects(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +02001221{
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001222 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001223 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1224 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001225 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001226 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
1227 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
Yazen Ghannamd971e282019-08-21 23:59:55 +00001228 } else if (pvt->fam >= 0x17) {
1229 int umc;
1230
1231 for_each_umc(umc) {
1232 pvt->csels[umc].b_cnt = 4;
1233 pvt->csels[umc].m_cnt = 2;
1234 }
1235
Borislav Petkov9d858bb2009-09-21 14:35:51 +02001236 } else {
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001237 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1238 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +02001239 }
1240}
1241
Yazen Ghannamd971e282019-08-21 23:59:55 +00001242static void read_umc_base_mask(struct amd64_pvt *pvt)
1243{
Yazen Ghannam75747292019-08-22 00:00:01 +00001244 u32 umc_base_reg, umc_base_reg_sec;
1245 u32 umc_mask_reg, umc_mask_reg_sec;
1246 u32 base_reg, base_reg_sec;
1247 u32 mask_reg, mask_reg_sec;
1248 u32 *base, *base_sec;
1249 u32 *mask, *mask_sec;
Yazen Ghannamd971e282019-08-21 23:59:55 +00001250 int cs, umc;
1251
1252 for_each_umc(umc) {
1253 umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
Yazen Ghannam75747292019-08-22 00:00:01 +00001254 umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
Yazen Ghannamd971e282019-08-21 23:59:55 +00001255
1256 for_each_chip_select(cs, umc, pvt) {
1257 base = &pvt->csels[umc].csbases[cs];
Yazen Ghannam75747292019-08-22 00:00:01 +00001258 base_sec = &pvt->csels[umc].csbases_sec[cs];
Yazen Ghannamd971e282019-08-21 23:59:55 +00001259
1260 base_reg = umc_base_reg + (cs * 4);
Yazen Ghannam75747292019-08-22 00:00:01 +00001261 base_reg_sec = umc_base_reg_sec + (cs * 4);
Yazen Ghannamd971e282019-08-21 23:59:55 +00001262
1263 if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
1264 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
1265 umc, cs, *base, base_reg);
Yazen Ghannam75747292019-08-22 00:00:01 +00001266
1267 if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
1268 edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
1269 umc, cs, *base_sec, base_reg_sec);
Yazen Ghannamd971e282019-08-21 23:59:55 +00001270 }
1271
1272 umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
Yazen Ghannam75747292019-08-22 00:00:01 +00001273 umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC;
Yazen Ghannamd971e282019-08-21 23:59:55 +00001274
1275 for_each_chip_select_mask(cs, umc, pvt) {
1276 mask = &pvt->csels[umc].csmasks[cs];
Yazen Ghannam75747292019-08-22 00:00:01 +00001277 mask_sec = &pvt->csels[umc].csmasks_sec[cs];
Yazen Ghannamd971e282019-08-21 23:59:55 +00001278
1279 mask_reg = umc_mask_reg + (cs * 4);
Yazen Ghannam75747292019-08-22 00:00:01 +00001280 mask_reg_sec = umc_mask_reg_sec + (cs * 4);
Yazen Ghannamd971e282019-08-21 23:59:55 +00001281
1282 if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
1283 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
1284 umc, cs, *mask, mask_reg);
Yazen Ghannam75747292019-08-22 00:00:01 +00001285
1286 if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
1287 edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
1288 umc, cs, *mask_sec, mask_reg_sec);
Yazen Ghannamd971e282019-08-21 23:59:55 +00001289 }
1290 }
1291}
1292
Doug Thompson94be4bf2009-04-27 16:12:00 +02001293/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001294 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
Doug Thompson94be4bf2009-04-27 16:12:00 +02001295 */
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001296static void read_dct_base_mask(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +02001297{
Yazen Ghannamd971e282019-08-21 23:59:55 +00001298 int cs;
Doug Thompson94be4bf2009-04-27 16:12:00 +02001299
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001300 prep_chip_selects(pvt);
Doug Thompson94be4bf2009-04-27 16:12:00 +02001301
Yazen Ghannamd971e282019-08-21 23:59:55 +00001302 if (pvt->umc)
1303 return read_umc_base_mask(pvt);
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05001304
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001305 for_each_chip_select(cs, 0, pvt) {
Yazen Ghannamd971e282019-08-21 23:59:55 +00001306 int reg0 = DCSB0 + (cs * 4);
1307 int reg1 = DCSB1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001308 u32 *base0 = &pvt->csels[0].csbases[cs];
1309 u32 *base1 = &pvt->csels[1].csbases[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001310
Yazen Ghannamd971e282019-08-21 23:59:55 +00001311 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
1312 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
1313 cs, *base0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +02001314
Yazen Ghannamd971e282019-08-21 23:59:55 +00001315 if (pvt->fam == 0xf)
1316 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001317
Yazen Ghannamd971e282019-08-21 23:59:55 +00001318 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
1319 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
1320 cs, *base1, (pvt->fam == 0x10) ? reg1
1321 : reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +02001322 }
1323
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001324 for_each_chip_select_mask(cs, 0, pvt) {
Yazen Ghannamd971e282019-08-21 23:59:55 +00001325 int reg0 = DCSM0 + (cs * 4);
1326 int reg1 = DCSM1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001327 u32 *mask0 = &pvt->csels[0].csmasks[cs];
1328 u32 *mask1 = &pvt->csels[1].csmasks[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001329
Yazen Ghannamd971e282019-08-21 23:59:55 +00001330 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1331 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1332 cs, *mask0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +02001333
Yazen Ghannamd971e282019-08-21 23:59:55 +00001334 if (pvt->fam == 0xf)
1335 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001336
Yazen Ghannamd971e282019-08-21 23:59:55 +00001337 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1338 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1339 cs, *mask1, (pvt->fam == 0x10) ? reg1
1340 : reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +02001341 }
1342}
1343
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001344static void determine_memory_type(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +02001345{
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001346 u32 dram_ctrl, dcsm;
Doug Thompson94be4bf2009-04-27 16:12:00 +02001347
Yazen Ghannamdcd01392020-01-10 01:56:51 +00001348 if (pvt->umc) {
1349 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1350 pvt->dram_type = MEM_LRDDR4;
1351 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1352 pvt->dram_type = MEM_RDDR4;
1353 else
1354 pvt->dram_type = MEM_DDR4;
1355 return;
1356 }
1357
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001358 switch (pvt->fam) {
1359 case 0xf:
1360 if (pvt->ext_model >= K8_REV_F)
1361 goto ddr3;
1362
1363 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1364 return;
1365
1366 case 0x10:
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +01001367 if (pvt->dchr0 & DDR3_MODE)
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001368 goto ddr3;
1369
1370 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1371 return;
1372
1373 case 0x15:
1374 if (pvt->model < 0x60)
1375 goto ddr3;
1376
1377 /*
1378 * Model 0x60h needs special handling:
1379 *
1380 * We use a Chip Select value of '0' to obtain dcsm.
1381 * Theoretically, it is possible to populate LRDIMMs of different
1382 * 'Rank' value on a DCT. But this is not the common case. So,
1383 * it's reasonable to assume all DIMMs are going to be of same
1384 * 'type' until proven otherwise.
1385 */
1386 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1387 dcsm = pvt->csels[0].csmasks[0];
1388
1389 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1390 pvt->dram_type = MEM_DDR4;
1391 else if (pvt->dclr0 & BIT(16))
1392 pvt->dram_type = MEM_DDR3;
1393 else if (dcsm & 0x3)
1394 pvt->dram_type = MEM_LRDDR3;
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +01001395 else
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001396 pvt->dram_type = MEM_RDDR3;
1397
1398 return;
1399
1400 case 0x16:
1401 goto ddr3;
1402
1403 default:
1404 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1405 pvt->dram_type = MEM_EMPTY;
Doug Thompson94be4bf2009-04-27 16:12:00 +02001406 }
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001407 return;
Doug Thompson94be4bf2009-04-27 16:12:00 +02001408
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001409ddr3:
1410 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
Doug Thompson94be4bf2009-04-27 16:12:00 +02001411}
1412
Borislav Petkovcb328502010-12-22 14:28:24 +01001413/* Get the number of DCT channels the memory controller is using. */
Doug Thompsonddff8762009-04-27 16:14:52 +02001414static int k8_early_channel_count(struct amd64_pvt *pvt)
1415{
Borislav Petkovcb328502010-12-22 14:28:24 +01001416 int flag;
Doug Thompsonddff8762009-04-27 16:14:52 +02001417
Borislav Petkov9f56da02010-10-01 19:44:53 +02001418 if (pvt->ext_model >= K8_REV_F)
Doug Thompsonddff8762009-04-27 16:14:52 +02001419 /* RevF (NPT) and later */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001420 flag = pvt->dclr0 & WIDTH_128;
Borislav Petkov9f56da02010-10-01 19:44:53 +02001421 else
Doug Thompsonddff8762009-04-27 16:14:52 +02001422 /* RevE and earlier */
1423 flag = pvt->dclr0 & REVE_WIDTH_128;
Doug Thompsonddff8762009-04-27 16:14:52 +02001424
1425 /* not used */
1426 pvt->dclr1 = 0;
1427
1428 return (flag) ? 2 : 1;
1429}
1430
Borislav Petkov70046622011-01-10 14:37:27 +01001431/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001432static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
Doug Thompsonddff8762009-04-27 16:14:52 +02001433{
Yazen Ghannamdb970bd22020-11-09 21:06:57 +00001434 u16 mce_nid = topology_die_id(m->extcpu);
Borislav Petkov2ec591a2015-02-17 10:58:34 +01001435 struct mem_ctl_info *mci;
Borislav Petkov70046622011-01-10 14:37:27 +01001436 u8 start_bit = 1;
1437 u8 end_bit = 47;
Borislav Petkov2ec591a2015-02-17 10:58:34 +01001438 u64 addr;
1439
1440 mci = edac_mc_find(mce_nid);
1441 if (!mci)
1442 return 0;
1443
1444 pvt = mci->pvt_info;
Borislav Petkov70046622011-01-10 14:37:27 +01001445
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001446 if (pvt->fam == 0xf) {
Borislav Petkov70046622011-01-10 14:37:27 +01001447 start_bit = 3;
1448 end_bit = 39;
1449 }
1450
Chen, Gong10ef6b02013-10-18 14:29:07 -07001451 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001452
1453 /*
1454 * Erratum 637 workaround
1455 */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001456 if (pvt->fam == 0x15) {
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001457 u64 cc6_base, tmp_addr;
1458 u32 tmp;
Daniel J Blueman8b84c8d2012-11-27 14:32:10 +08001459 u8 intlv_en;
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001460
Chen, Gong10ef6b02013-10-18 14:29:07 -07001461 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001462 return addr;
1463
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001464
1465 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1466 intlv_en = tmp >> 21 & 0x7;
1467
1468 /* add [47:27] + 3 trailing bits */
Chen, Gong10ef6b02013-10-18 14:29:07 -07001469 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001470
1471 /* reverse and add DramIntlvEn */
1472 cc6_base |= intlv_en ^ 0x7;
1473
1474 /* pin at [47:24] */
1475 cc6_base <<= 24;
1476
1477 if (!intlv_en)
Chen, Gong10ef6b02013-10-18 14:29:07 -07001478 return cc6_base | (addr & GENMASK_ULL(23, 0));
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001479
1480 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1481
1482 /* faster log2 */
Chen, Gong10ef6b02013-10-18 14:29:07 -07001483 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001484
1485 /* OR DramIntlvSel into bits [14:12] */
Chen, Gong10ef6b02013-10-18 14:29:07 -07001486 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001487
1488 /* add remaining [11:0] bits from original MC4_ADDR */
Chen, Gong10ef6b02013-10-18 14:29:07 -07001489 tmp_addr |= addr & GENMASK_ULL(11, 0);
Borislav Petkovc1ae6832011-03-30 15:42:10 +02001490
1491 return cc6_base | tmp_addr;
1492 }
1493
1494 return addr;
Doug Thompsonddff8762009-04-27 16:14:52 +02001495}
1496
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001497static struct pci_dev *pci_get_related_function(unsigned int vendor,
1498 unsigned int device,
1499 struct pci_dev *related)
1500{
1501 struct pci_dev *dev = NULL;
1502
1503 while ((dev = pci_get_device(vendor, device, dev))) {
1504 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1505 (dev->bus->number == related->bus->number) &&
1506 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1507 break;
1508 }
1509
1510 return dev;
1511}
1512
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001513static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
Doug Thompsonddff8762009-04-27 16:14:52 +02001514{
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001515 struct amd_northbridge *nb;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001516 struct pci_dev *f1 = NULL;
1517 unsigned int pci_func;
Borislav Petkov71d2a322011-02-21 19:37:24 +01001518 int off = range << 3;
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001519 u32 llim;
Doug Thompsonddff8762009-04-27 16:14:52 +02001520
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001521 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1522 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
Doug Thompsonddff8762009-04-27 16:14:52 +02001523
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001524 if (pvt->fam == 0xf)
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001525 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001526
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001527 if (!dram_rw(pvt, range))
1528 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001529
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001530 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1531 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001532
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001533 /* F15h: factor in CC6 save area by reading dst node's limit reg */
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001534 if (pvt->fam != 0x15)
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001535 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001536
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001537 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1538 if (WARN_ON(!nb))
1539 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001540
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001541 if (pvt->model == 0x60)
1542 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1543 else if (pvt->model == 0x30)
1544 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1545 else
1546 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001547
1548 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001549 if (WARN_ON(!f1))
1550 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001551
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001552 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001553
Chen, Gong10ef6b02013-10-18 14:29:07 -07001554 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001555
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001556 /* {[39:27],111b} */
1557 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001558
Chen, Gong10ef6b02013-10-18 14:29:07 -07001559 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001560
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001561 /* [47:40] */
1562 pvt->ranges[range].lim.hi |= llim >> 13;
1563
1564 pci_dev_put(f1);
Doug Thompsonddff8762009-04-27 16:14:52 +02001565}
1566
Borislav Petkovf192c7b12011-01-10 14:24:32 +01001567static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001568 struct err_info *err)
Doug Thompsonddff8762009-04-27 16:14:52 +02001569{
Borislav Petkovf192c7b12011-01-10 14:24:32 +01001570 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonddff8762009-04-27 16:14:52 +02001571
Borislav Petkov33ca0642012-08-30 18:01:36 +02001572 error_address_to_page_and_offset(sys_addr, err);
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001573
1574 /*
1575 * Find out which node the error address belongs to. This may be
1576 * different from the node that detected the error.
1577 */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001578 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1579 if (!err->src_mci) {
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001580 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1581 (unsigned long)sys_addr);
Borislav Petkov33ca0642012-08-30 18:01:36 +02001582 err->err_code = ERR_NODE;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001583 return;
1584 }
1585
1586 /* Now map the sys_addr to a CSROW */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001587 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1588 if (err->csrow < 0) {
1589 err->err_code = ERR_CSROW;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001590 return;
1591 }
1592
Doug Thompsonddff8762009-04-27 16:14:52 +02001593 /* CHIPKILL enabled */
Borislav Petkovf192c7b12011-01-10 14:24:32 +01001594 if (pvt->nbcfg & NBCFG_CHIPKILL) {
Borislav Petkov33ca0642012-08-30 18:01:36 +02001595 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1596 if (err->channel < 0) {
Doug Thompsonddff8762009-04-27 16:14:52 +02001597 /*
1598 * Syndrome didn't map, so we don't know which of the
1599 * 2 DIMMs is in error. So we need to ID 'both' of them
1600 * as suspect.
1601 */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001602 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001603 "possible error reporting race\n",
Borislav Petkov33ca0642012-08-30 18:01:36 +02001604 err->syndrome);
1605 err->err_code = ERR_CHANNEL;
Doug Thompsonddff8762009-04-27 16:14:52 +02001606 return;
1607 }
1608 } else {
1609 /*
1610 * non-chipkill ecc mode
1611 *
1612 * The k8 documentation is unclear about how to determine the
1613 * channel number when using non-chipkill memory. This method
1614 * was obtained from email communication with someone at AMD.
1615 * (Wish the email was placed in this comment - norsk)
1616 */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001617 err->channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +02001618 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001619}
1620
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001621static int ddr2_cs_size(unsigned i, bool dct_width)
Doug Thompsonddff8762009-04-27 16:14:52 +02001622{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001623 unsigned shift = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001624
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001625 if (i <= 2)
1626 shift = i;
1627 else if (!(i & 0x1))
1628 shift = i >> 1;
Borislav Petkov1433eb92009-10-21 13:44:36 +02001629 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001630 shift = (i + 1) >> 1;
Doug Thompsonddff8762009-04-27 16:14:52 +02001631
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001632 return 128 << (shift + !!dct_width);
1633}
1634
1635static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001636 unsigned cs_mode, int cs_mask_nr)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001637{
1638 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1639
1640 if (pvt->ext_model >= K8_REV_F) {
1641 WARN_ON(cs_mode > 11);
1642 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1643 }
1644 else if (pvt->ext_model >= K8_REV_D) {
Borislav Petkov11b0a3142011-11-09 21:28:43 +01001645 unsigned diff;
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001646 WARN_ON(cs_mode > 10);
1647
Borislav Petkov11b0a3142011-11-09 21:28:43 +01001648 /*
1649 * the below calculation, besides trying to win an obfuscated C
1650 * contest, maps cs_mode values to DIMM chip select sizes. The
1651 * mappings are:
1652 *
1653 * cs_mode CS size (mb)
1654 * ======= ============
1655 * 0 32
1656 * 1 64
1657 * 2 128
1658 * 3 128
1659 * 4 256
1660 * 5 512
1661 * 6 256
1662 * 7 512
1663 * 8 1024
1664 * 9 1024
1665 * 10 2048
1666 *
1667 * Basically, it calculates a value with which to shift the
1668 * smallest CS size of 32MB.
1669 *
1670 * ddr[23]_cs_size have a similar purpose.
1671 */
1672 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1673
1674 return 32 << (cs_mode - diff);
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001675 }
1676 else {
1677 WARN_ON(cs_mode > 6);
1678 return 32 << cs_mode;
1679 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001680}
1681
Doug Thompson1afd3c92009-04-27 16:16:50 +02001682/*
1683 * Get the number of DCT channels in use.
1684 *
1685 * Return:
1686 * number of Memory Channels in operation
1687 * Pass back:
1688 * contents of the DCL0_LOW register
1689 */
Borislav Petkov7d20d142011-01-07 17:58:04 +01001690static int f1x_early_channel_count(struct amd64_pvt *pvt)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001691{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001692 int i, j, channels = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001693
Borislav Petkov7d20d142011-01-07 17:58:04 +01001694 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001695 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
Borislav Petkov7d20d142011-01-07 17:58:04 +01001696 return 2;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001697
1698 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001699 * Need to check if in unganged mode: In such, there are 2 channels,
1700 * but they are not in 128 bit mode and thus the above 'dclr0' status
1701 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001702 *
1703 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1704 * their CSEnable bit on. If so, then SINGLE DIMM case.
1705 */
Joe Perches956b9ba12012-04-29 17:08:39 -03001706 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001707
1708 /*
1709 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1710 * is more than just one DIMM present in unganged mode. Need to check
1711 * both controllers since DIMMs can be placed in either one.
1712 */
Borislav Petkov525a1b22010-12-21 15:53:27 +01001713 for (i = 0; i < 2; i++) {
1714 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001715
Wan Wei57a30852009-08-07 17:04:49 +02001716 for (j = 0; j < 4; j++) {
1717 if (DBAM_DIMM(j, dbam) > 0) {
1718 channels++;
1719 break;
1720 }
1721 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001722 }
1723
Borislav Petkovd16149e2009-10-16 19:55:49 +02001724 if (channels > 2)
1725 channels = 2;
1726
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001727 amd64_info("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001728
1729 return channels;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001730}
1731
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05001732static int f17_early_channel_count(struct amd64_pvt *pvt)
1733{
1734 int i, channels = 0;
1735
1736 /* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +00001737 for_each_umc(i)
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05001738 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1739
1740 amd64_info("MCT channel count: %d\n", channels);
1741
1742 return channels;
1743}
1744
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001745static int ddr3_cs_size(unsigned i, bool dct_width)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001746{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001747 unsigned shift = 0;
1748 int cs_size = 0;
1749
1750 if (i == 0 || i == 3 || i == 4)
1751 cs_size = -1;
1752 else if (i <= 2)
1753 shift = i;
1754 else if (i == 12)
1755 shift = 7;
1756 else if (!(i & 0x1))
1757 shift = i >> 1;
1758 else
1759 shift = (i + 1) >> 1;
1760
1761 if (cs_size != -1)
1762 cs_size = (128 * (1 << !!dct_width)) << shift;
1763
1764 return cs_size;
1765}
1766
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001767static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1768{
1769 unsigned shift = 0;
1770 int cs_size = 0;
1771
1772 if (i < 4 || i == 6)
1773 cs_size = -1;
1774 else if (i == 12)
1775 shift = 7;
1776 else if (!(i & 0x1))
1777 shift = i >> 1;
1778 else
1779 shift = (i + 1) >> 1;
1780
1781 if (cs_size != -1)
1782 cs_size = rank_multiply * (128 << shift);
1783
1784 return cs_size;
1785}
1786
1787static int ddr4_cs_size(unsigned i)
1788{
1789 int cs_size = 0;
1790
1791 if (i == 0)
1792 cs_size = -1;
1793 else if (i == 1)
1794 cs_size = 1024;
1795 else
1796 /* Min cs_size = 1G */
1797 cs_size = 1024 * (1 << (i >> 1));
1798
1799 return cs_size;
1800}
1801
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001802static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001803 unsigned cs_mode, int cs_mask_nr)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001804{
1805 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1806
1807 WARN_ON(cs_mode > 11);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001808
1809 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001810 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001811 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001812 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1813}
Borislav Petkov1433eb92009-10-21 13:44:36 +02001814
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001815/*
1816 * F15h supports only 64bit DCT interfaces
1817 */
1818static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001819 unsigned cs_mode, int cs_mask_nr)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001820{
1821 WARN_ON(cs_mode > 12);
1822
1823 return ddr3_cs_size(cs_mode, false);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001824}
1825
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001826/* F15h M60h supports DDR4 mapping as well.. */
1827static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1828 unsigned cs_mode, int cs_mask_nr)
1829{
1830 int cs_size;
1831 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1832
1833 WARN_ON(cs_mode > 12);
1834
1835 if (pvt->dram_type == MEM_DDR4) {
1836 if (cs_mode > 9)
1837 return -1;
1838
1839 cs_size = ddr4_cs_size(cs_mode);
1840 } else if (pvt->dram_type == MEM_LRDDR3) {
1841 unsigned rank_multiply = dcsm & 0xf;
1842
1843 if (rank_multiply == 3)
1844 rank_multiply = 4;
1845 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1846 } else {
1847 /* Minimum cs size is 512mb for F15hM60h*/
1848 if (cs_mode == 0x1)
1849 return -1;
1850
1851 cs_size = ddr3_cs_size(cs_mode, false);
1852 }
1853
1854 return cs_size;
1855}
1856
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001857/*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001858 * F16h and F15h model 30h have only limited cs_modes.
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001859 */
1860static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001861 unsigned cs_mode, int cs_mask_nr)
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001862{
1863 WARN_ON(cs_mode > 12);
1864
1865 if (cs_mode == 6 || cs_mode == 8 ||
1866 cs_mode == 9 || cs_mode == 12)
1867 return -1;
1868 else
1869 return ddr3_cs_size(cs_mode, false);
1870}
1871
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001872static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05001873 unsigned int cs_mode, int csrow_nr)
1874{
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001875 u32 addr_mask_orig, addr_mask_deinterleaved;
1876 u32 msb, weight, num_zero_bits;
1877 int dimm, size = 0;
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05001878
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001879 /* No Chip Selects are enabled. */
1880 if (!cs_mode)
1881 return size;
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05001882
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001883 /* Requested size of an even CS but none are enabled. */
1884 if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
1885 return size;
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05001886
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001887 /* Requested size of an odd CS but none are enabled. */
1888 if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
1889 return size;
1890
1891 /*
1892 * There is one mask per DIMM, and two Chip Selects per DIMM.
1893 * CS0 and CS1 -> DIMM0
1894 * CS2 and CS3 -> DIMM1
1895 */
1896 dimm = csrow_nr >> 1;
1897
Yazen Ghannam81f50902019-08-22 00:00:02 +00001898 /* Asymmetric dual-rank DIMM support. */
1899 if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
1900 addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm];
1901 else
1902 addr_mask_orig = pvt->csels[umc].csmasks[dimm];
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001903
1904 /*
1905 * The number of zero bits in the mask is equal to the number of bits
1906 * in a full mask minus the number of bits in the current mask.
1907 *
1908 * The MSB is the number of bits in the full mask because BIT[0] is
1909 * always 0.
Yazen Ghannam9f4873f2021-10-05 15:44:19 +00001910 *
1911 * In the special 3 Rank interleaving case, a single bit is flipped
1912 * without swapping with the most significant bit. This can be handled
1913 * by keeping the MSB where it is and ignoring the single zero bit.
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001914 */
1915 msb = fls(addr_mask_orig) - 1;
1916 weight = hweight_long(addr_mask_orig);
Yazen Ghannam9f4873f2021-10-05 15:44:19 +00001917 num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE);
Yazen Ghanname53a3b22019-08-21 23:59:59 +00001918
1919 /* Take the number of zero bits off from the top of the mask. */
1920 addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
1921
1922 edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
1923 edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig);
1924 edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
1925
1926 /* Register [31:1] = Address [39:9]. Size is in kBs here. */
1927 size = (addr_mask_deinterleaved >> 2) + 1;
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05001928
1929 /* Return size in MBs. */
1930 return size >> 10;
1931}
1932
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001933static void read_dram_ctl_register(struct amd64_pvt *pvt)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001934{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001935
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001936 if (pvt->fam == 0xf)
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001937 return;
1938
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05001939 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
Joe Perches956b9ba12012-04-29 17:08:39 -03001940 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1941 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001942
Joe Perches956b9ba12012-04-29 17:08:39 -03001943 edac_dbg(0, " DCTs operate in %s mode\n",
1944 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001945
Borislav Petkov72381bd2009-10-09 19:14:43 +02001946 if (!dct_ganging_enabled(pvt))
Joe Perches956b9ba12012-04-29 17:08:39 -03001947 edac_dbg(0, " Address range split per DCT: %s\n",
1948 (dct_high_range_enabled(pvt) ? "yes" : "no"));
Borislav Petkov72381bd2009-10-09 19:14:43 +02001949
Joe Perches956b9ba12012-04-29 17:08:39 -03001950 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1951 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1952 (dct_memory_cleared(pvt) ? "yes" : "no"));
Borislav Petkov72381bd2009-10-09 19:14:43 +02001953
Joe Perches956b9ba12012-04-29 17:08:39 -03001954 edac_dbg(0, " channel interleave: %s, "
1955 "interleave bits selector: 0x%x\n",
1956 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1957 dct_sel_interleave_addr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001958 }
1959
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05001960 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001961}
1962
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001963/*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001964 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1965 * 2.10.12 Memory Interleaving Modes).
1966 */
1967static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1968 u8 intlv_en, int num_dcts_intlv,
1969 u32 dct_sel)
1970{
1971 u8 channel = 0;
1972 u8 select;
1973
1974 if (!(intlv_en))
1975 return (u8)(dct_sel);
1976
1977 if (num_dcts_intlv == 2) {
1978 select = (sys_addr >> 8) & 0x3;
1979 channel = select ? 0x3 : 0;
Aravind Gopalakrishnan9d0e8d82014-01-21 15:03:36 -06001980 } else if (num_dcts_intlv == 4) {
1981 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1982 switch (intlv_addr) {
1983 case 0x4:
1984 channel = (sys_addr >> 8) & 0x3;
1985 break;
1986 case 0x5:
1987 channel = (sys_addr >> 9) & 0x3;
1988 break;
1989 }
1990 }
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001991 return channel;
1992}
1993
1994/*
Borislav Petkov229a7a12010-12-09 18:57:54 +01001995 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001996 * Interleaving Modes.
1997 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001998static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov229a7a12010-12-09 18:57:54 +01001999 bool hi_range_sel, u8 intlv_en)
Doug Thompson6163b5d2009-04-27 16:20:17 +02002000{
Borislav Petkov151fa712011-02-21 19:33:10 +01002001 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002002
2003 if (dct_ganging_enabled(pvt))
Borislav Petkov229a7a12010-12-09 18:57:54 +01002004 return 0;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002005
Borislav Petkov229a7a12010-12-09 18:57:54 +01002006 if (hi_range_sel)
2007 return dct_sel_high;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002008
Borislav Petkov229a7a12010-12-09 18:57:54 +01002009 /*
2010 * see F2x110[DctSelIntLvAddr] - channel interleave mode
2011 */
2012 if (dct_interleave_enabled(pvt)) {
2013 u8 intlv_addr = dct_sel_interleave_addr(pvt);
Doug Thompson6163b5d2009-04-27 16:20:17 +02002014
Borislav Petkov229a7a12010-12-09 18:57:54 +01002015 /* return DCT select function: 0=DCT0, 1=DCT1 */
2016 if (!intlv_addr)
2017 return sys_addr >> 6 & 1;
2018
2019 if (intlv_addr & 0x2) {
2020 u8 shift = intlv_addr & 0x1 ? 9 : 6;
Yazen Ghannamdc0a50a82016-08-03 10:59:15 -04002021 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
Borislav Petkov229a7a12010-12-09 18:57:54 +01002022
2023 return ((sys_addr >> shift) & 1) ^ temp;
2024 }
2025
Yazen Ghannamdc0a50a82016-08-03 10:59:15 -04002026 if (intlv_addr & 0x4) {
2027 u8 shift = intlv_addr & 0x1 ? 9 : 8;
2028
2029 return (sys_addr >> shift) & 1;
2030 }
2031
Borislav Petkov229a7a12010-12-09 18:57:54 +01002032 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
2033 }
2034
2035 if (dct_high_range_enabled(pvt))
2036 return ~dct_sel_high & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002037
2038 return 0;
2039}
2040
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002041/* Convert the sys_addr to the normalized DCT address */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002042static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002043 u64 sys_addr, bool hi_rng,
2044 u32 dct_sel_base_addr)
Doug Thompson6163b5d2009-04-27 16:20:17 +02002045{
2046 u64 chan_off;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002047 u64 dram_base = get_dram_base(pvt, range);
2048 u64 hole_off = f10_dhar_offset(pvt);
Dan Carpenter6f3508f2016-01-20 12:54:51 +03002049 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002050
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002051 if (hi_rng) {
2052 /*
2053 * if
2054 * base address of high range is below 4Gb
2055 * (bits [47:27] at [31:11])
2056 * DRAM address space on this DCT is hoisted above 4Gb &&
2057 * sys_addr > 4Gb
2058 *
2059 * remove hole offset from sys_addr
2060 * else
2061 * remove high range offset from sys_addr
2062 */
2063 if ((!(dct_sel_base_addr >> 16) ||
2064 dct_sel_base_addr < dhar_base(pvt)) &&
Borislav Petkov972ea172011-02-21 19:43:02 +01002065 dhar_valid(pvt) &&
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002066 (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01002067 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002068 else
2069 chan_off = dct_sel_base_off;
2070 } else {
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002071 /*
2072 * if
2073 * we have a valid hole &&
2074 * sys_addr > 4Gb
2075 *
2076 * remove hole
2077 * else
2078 * remove dram base to normalize to DCT address
2079 */
Borislav Petkov972ea172011-02-21 19:43:02 +01002080 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01002081 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002082 else
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002083 chan_off = dram_base;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002084 }
2085
Chen, Gong10ef6b02013-10-18 14:29:07 -07002086 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
Doug Thompson6163b5d2009-04-27 16:20:17 +02002087}
2088
Doug Thompson6163b5d2009-04-27 16:20:17 +02002089/*
2090 * checks if the csrow passed in is marked as SPARED, if so returns the new
2091 * spare row
2092 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002093static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
Doug Thompson6163b5d2009-04-27 16:20:17 +02002094{
Borislav Petkov614ec9d2011-01-13 18:02:22 +01002095 int tmp_cs;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002096
Borislav Petkov614ec9d2011-01-13 18:02:22 +01002097 if (online_spare_swap_done(pvt, dct) &&
2098 csrow == online_spare_bad_dramcs(pvt, dct)) {
2099
2100 for_each_chip_select(tmp_cs, dct, pvt) {
2101 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
2102 csrow = tmp_cs;
2103 break;
2104 }
2105 }
Doug Thompson6163b5d2009-04-27 16:20:17 +02002106 }
2107 return csrow;
2108}
2109
2110/*
2111 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
2112 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
2113 *
2114 * Return:
2115 * -EINVAL: NOT FOUND
2116 * 0..csrow = Chip-Select Row
2117 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002118static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
Doug Thompson6163b5d2009-04-27 16:20:17 +02002119{
2120 struct mem_ctl_info *mci;
2121 struct amd64_pvt *pvt;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002122 u64 cs_base, cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002123 int cs_found = -EINVAL;
2124 int csrow;
2125
Borislav Petkov2ec591a2015-02-17 10:58:34 +01002126 mci = edac_mc_find(nid);
Doug Thompson6163b5d2009-04-27 16:20:17 +02002127 if (!mci)
2128 return cs_found;
2129
2130 pvt = mci->pvt_info;
2131
Joe Perches956b9ba12012-04-29 17:08:39 -03002132 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
Doug Thompson6163b5d2009-04-27 16:20:17 +02002133
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002134 for_each_chip_select(csrow, dct, pvt) {
2135 if (!csrow_enabled(csrow, dct, pvt))
Doug Thompson6163b5d2009-04-27 16:20:17 +02002136 continue;
2137
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002138 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02002139
Joe Perches956b9ba12012-04-29 17:08:39 -03002140 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
2141 csrow, cs_base, cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02002142
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002143 cs_mask = ~cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02002144
Joe Perches956b9ba12012-04-29 17:08:39 -03002145 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
2146 (in_addr & cs_mask), (cs_base & cs_mask));
Doug Thompson6163b5d2009-04-27 16:20:17 +02002147
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002148 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002149 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
2150 cs_found = csrow;
2151 break;
2152 }
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002153 cs_found = f10_process_possible_spare(pvt, dct, csrow);
Doug Thompson6163b5d2009-04-27 16:20:17 +02002154
Joe Perches956b9ba12012-04-29 17:08:39 -03002155 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
Doug Thompson6163b5d2009-04-27 16:20:17 +02002156 break;
2157 }
2158 }
2159 return cs_found;
2160}
2161
Borislav Petkov95b0ef52011-01-11 22:08:07 +01002162/*
2163 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
2164 * swapped with a region located at the bottom of memory so that the GPU can use
2165 * the interleaved region and thus two channels.
2166 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002167static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
Borislav Petkov95b0ef52011-01-11 22:08:07 +01002168{
2169 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
2170
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002171 if (pvt->fam == 0x10) {
Borislav Petkov95b0ef52011-01-11 22:08:07 +01002172 /* only revC3 and revE have that feature */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002173 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
Borislav Petkov95b0ef52011-01-11 22:08:07 +01002174 return sys_addr;
2175 }
2176
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05002177 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01002178
2179 if (!(swap_reg & 0x1))
2180 return sys_addr;
2181
2182 swap_base = (swap_reg >> 3) & 0x7f;
2183 swap_limit = (swap_reg >> 11) & 0x7f;
2184 rgn_size = (swap_reg >> 20) & 0x7f;
2185 tmp_addr = sys_addr >> 27;
2186
2187 if (!(sys_addr >> 34) &&
2188 (((tmp_addr >= swap_base) &&
2189 (tmp_addr <= swap_limit)) ||
2190 (tmp_addr < rgn_size)))
2191 return sys_addr ^ (u64)swap_base << 27;
2192
2193 return sys_addr;
2194}
2195
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002196/* For a given @dram_range, check if @sys_addr falls within it. */
Borislav Petkove761359a2011-02-21 19:49:01 +01002197static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
Borislav Petkov33ca0642012-08-30 18:01:36 +02002198 u64 sys_addr, int *chan_sel)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002199{
Borislav Petkov229a7a12010-12-09 18:57:54 +01002200 int cs_found = -EINVAL;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002201 u64 chan_addr;
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01002202 u32 dct_sel_base;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002203 u8 channel;
Borislav Petkov229a7a12010-12-09 18:57:54 +01002204 bool high_range = false;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002205
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002206 u8 node_id = dram_dst_node(pvt, range);
Borislav Petkov229a7a12010-12-09 18:57:54 +01002207 u8 intlv_en = dram_intlv_en(pvt, range);
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002208 u32 intlv_sel = dram_intlv_sel(pvt, range);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002209
Joe Perches956b9ba12012-04-29 17:08:39 -03002210 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2211 range, sys_addr, get_dram_limit(pvt, range));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002212
Borislav Petkov355fba62011-01-17 13:03:26 +01002213 if (dhar_valid(pvt) &&
2214 dhar_base(pvt) <= sys_addr &&
2215 sys_addr < BIT_64(32)) {
2216 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2217 sys_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002218 return -EINVAL;
Borislav Petkov355fba62011-01-17 13:03:26 +01002219 }
2220
Borislav Petkovf030ddf2011-04-08 15:05:21 +02002221 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
Borislav Petkov355fba62011-01-17 13:03:26 +01002222 return -EINVAL;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002223
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002224 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01002225
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002226 dct_sel_base = dct_sel_baseaddr(pvt);
2227
2228 /*
2229 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
2230 * select between DCT0 and DCT1.
2231 */
2232 if (dct_high_range_enabled(pvt) &&
2233 !dct_ganging_enabled(pvt) &&
2234 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
Borislav Petkov229a7a12010-12-09 18:57:54 +01002235 high_range = true;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002236
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002237 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002238
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002239 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01002240 high_range, dct_sel_base);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002241
Borislav Petkove2f79db2011-01-13 14:57:34 +01002242 /* Remove node interleaving, see F1x120 */
2243 if (intlv_en)
2244 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
2245 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002246
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01002247 /* remove channel interleave */
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002248 if (dct_interleave_enabled(pvt) &&
2249 !dct_high_range_enabled(pvt) &&
2250 !dct_ganging_enabled(pvt)) {
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01002251
2252 if (dct_sel_interleave_addr(pvt) != 1) {
2253 if (dct_sel_interleave_addr(pvt) == 0x3)
2254 /* hash 9 */
2255 chan_addr = ((chan_addr >> 10) << 9) |
2256 (chan_addr & 0x1ff);
2257 else
2258 /* A[6] or hash 6 */
2259 chan_addr = ((chan_addr >> 7) << 6) |
2260 (chan_addr & 0x3f);
2261 } else
2262 /* A[12] */
2263 chan_addr = ((chan_addr >> 13) << 12) |
2264 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002265 }
2266
Joe Perches956b9ba12012-04-29 17:08:39 -03002267 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002268
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002269 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002270
Borislav Petkov33ca0642012-08-30 18:01:36 +02002271 if (cs_found >= 0)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002272 *chan_sel = channel;
Borislav Petkov33ca0642012-08-30 18:01:36 +02002273
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002274 return cs_found;
2275}
2276
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002277static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2278 u64 sys_addr, int *chan_sel)
2279{
2280 int cs_found = -EINVAL;
2281 int num_dcts_intlv = 0;
2282 u64 chan_addr, chan_offset;
2283 u64 dct_base, dct_limit;
2284 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
2285 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
2286
2287 u64 dhar_offset = f10_dhar_offset(pvt);
2288 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2289 u8 node_id = dram_dst_node(pvt, range);
2290 u8 intlv_en = dram_intlv_en(pvt, range);
2291
2292 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
2293 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
2294
2295 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
2296 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
2297
2298 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2299 range, sys_addr, get_dram_limit(pvt, range));
2300
2301 if (!(get_dram_base(pvt, range) <= sys_addr) &&
2302 !(get_dram_limit(pvt, range) >= sys_addr))
2303 return -EINVAL;
2304
2305 if (dhar_valid(pvt) &&
2306 dhar_base(pvt) <= sys_addr &&
2307 sys_addr < BIT_64(32)) {
2308 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2309 sys_addr);
2310 return -EINVAL;
2311 }
2312
2313 /* Verify sys_addr is within DCT Range. */
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05002314 dct_base = (u64) dct_sel_baseaddr(pvt);
2315 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002316
2317 if (!(dct_cont_base_reg & BIT(0)) &&
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05002318 !(dct_base <= (sys_addr >> 27) &&
2319 dct_limit >= (sys_addr >> 27)))
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002320 return -EINVAL;
2321
2322 /* Verify number of dct's that participate in channel interleaving. */
2323 num_dcts_intlv = (int) hweight8(intlv_en);
2324
2325 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
2326 return -EINVAL;
2327
Yazen Ghannamdc0a50a82016-08-03 10:59:15 -04002328 if (pvt->model >= 0x60)
2329 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
2330 else
2331 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
2332 num_dcts_intlv, dct_sel);
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002333
2334 /* Verify we stay within the MAX number of channels allowed */
Aravind Gopalakrishnan7f3f5242013-12-04 11:40:11 -06002335 if (channel > 3)
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002336 return -EINVAL;
2337
2338 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
2339
2340 /* Get normalized DCT addr */
2341 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
2342 chan_offset = dhar_offset;
2343 else
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05002344 chan_offset = dct_base << 27;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002345
2346 chan_addr = sys_addr - chan_offset;
2347
2348 /* remove channel interleave */
2349 if (num_dcts_intlv == 2) {
2350 if (intlv_addr == 0x4)
2351 chan_addr = ((chan_addr >> 9) << 8) |
2352 (chan_addr & 0xff);
2353 else if (intlv_addr == 0x5)
2354 chan_addr = ((chan_addr >> 10) << 9) |
2355 (chan_addr & 0x1ff);
2356 else
2357 return -EINVAL;
2358
2359 } else if (num_dcts_intlv == 4) {
2360 if (intlv_addr == 0x4)
2361 chan_addr = ((chan_addr >> 10) << 8) |
2362 (chan_addr & 0xff);
2363 else if (intlv_addr == 0x5)
2364 chan_addr = ((chan_addr >> 11) << 9) |
2365 (chan_addr & 0x1ff);
2366 else
2367 return -EINVAL;
2368 }
2369
2370 if (dct_offset_en) {
2371 amd64_read_pci_cfg(pvt->F1,
2372 DRAM_CONT_HIGH_OFF + (int) channel * 4,
2373 &tmp);
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05002374 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002375 }
2376
2377 f15h_select_dct(pvt, channel);
2378
2379 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2380
2381 /*
2382 * Find Chip select:
2383 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
2384 * there is support for 4 DCT's, but only 2 are currently functional.
2385 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
2386 * pvt->csels[1]. So we need to use '1' here to get correct info.
2387 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
2388 */
2389 alias_channel = (channel == 3) ? 1 : channel;
2390
2391 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2392
2393 if (cs_found >= 0)
2394 *chan_sel = alias_channel;
2395
2396 return cs_found;
2397}
2398
2399static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2400 u64 sys_addr,
2401 int *chan_sel)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002402{
Borislav Petkove761359a2011-02-21 19:49:01 +01002403 int cs_found = -EINVAL;
2404 unsigned range;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002405
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002406 for (range = 0; range < DRAM_RANGES; range++) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002407 if (!dram_rw(pvt, range))
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002408 continue;
2409
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002410 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2411 cs_found = f15_m30h_match_to_this_node(pvt, range,
2412 sys_addr,
2413 chan_sel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002414
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002415 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2416 (get_dram_limit(pvt, range) >= sys_addr)) {
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002417 cs_found = f1x_match_to_this_node(pvt, range,
Borislav Petkov33ca0642012-08-30 18:01:36 +02002418 sys_addr, chan_sel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002419 if (cs_found >= 0)
2420 break;
2421 }
2422 }
2423 return cs_found;
2424}
2425
2426/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01002427 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2428 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002429 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01002430 * The @sys_addr is usually an error address received from the hardware
2431 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002432 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002433static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkov33ca0642012-08-30 18:01:36 +02002434 struct err_info *err)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002435{
2436 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002437
Borislav Petkov33ca0642012-08-30 18:01:36 +02002438 error_address_to_page_and_offset(sys_addr, err);
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03002439
Borislav Petkov33ca0642012-08-30 18:01:36 +02002440 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2441 if (err->csrow < 0) {
2442 err->err_code = ERR_CSROW;
Borislav Petkovbdc30a02009-11-13 15:10:43 +01002443 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002444 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01002445
Borislav Petkovbdc30a02009-11-13 15:10:43 +01002446 /*
2447 * We need the syndromes for channel detection only when we're
2448 * ganged. Otherwise @chan should already contain the channel at
2449 * this point.
2450 */
Borislav Petkova97fa682010-12-23 14:07:18 +01002451 if (dct_ganging_enabled(pvt))
Borislav Petkov33ca0642012-08-30 18:01:36 +02002452 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002453}
2454
2455/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02002456 * debug routine to display the memory sizes of all logical DIMMs and its
Borislav Petkovcb328502010-12-22 14:28:24 +01002457 * CSROWs
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002458 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002459static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002460{
Borislav Petkovbb89f5a2012-09-12 18:06:00 +02002461 int dimm, size0, size1;
Borislav Petkov525a1b22010-12-21 15:53:27 +01002462 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2463 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002464
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002465 if (pvt->fam == 0xf) {
Borislav Petkov8566c4d2009-10-16 13:48:28 +02002466 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02002467 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02002468 return;
2469 else
2470 WARN_ON(ctrl != 0);
2471 }
2472
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05002473 if (pvt->fam == 0x10) {
2474 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2475 : pvt->dbam0;
2476 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2477 pvt->csels[1].csbases :
2478 pvt->csels[0].csbases;
2479 } else if (ctrl) {
2480 dbam = pvt->dbam0;
2481 dcsb = pvt->csels[1].csbases;
2482 }
Joe Perches956b9ba12012-04-29 17:08:39 -03002483 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2484 ctrl, dbam);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002485
Borislav Petkov8566c4d2009-10-16 13:48:28 +02002486 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2487
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002488 /* Dump memory sizes for DIMM and its CSROWs */
2489 for (dimm = 0; dimm < 4; dimm++) {
2490
2491 size0 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002492 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06002493 /*
2494 * For F15m60h, we need multiplier for LRDIMM cs_size
2495 * calculation. We pass dimm value to the dbam_to_cs
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002496 * mapper so we can find the multiplier from the
2497 * corresponding DCSM.
2498 */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002499 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002500 DBAM_DIMM(dimm, dbam),
2501 dimm);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002502
2503 size1 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002504 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002505 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002506 DBAM_DIMM(dimm, dbam),
2507 dimm);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002508
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002509 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
Borislav Petkovbb89f5a2012-09-12 18:06:00 +02002510 dimm * 2, size0,
2511 dimm * 2 + 1, size1);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02002512 }
2513}
2514
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002515static struct amd64_family_type family_types[] = {
Doug Thompson4d376072009-04-27 16:25:05 +02002516 [K8_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02002517 .ctl_name = "K8",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002518 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
Borislav Petkov3f37a362016-05-06 19:44:27 +02002519 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002520 .max_mcs = 2,
Doug Thompson4d376072009-04-27 16:25:05 +02002521 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02002522 .early_channel_count = k8_early_channel_count,
Borislav Petkov1433eb92009-10-21 13:44:36 +02002523 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2524 .dbam_to_cs = k8_dbam_to_chip_select,
Doug Thompson4d376072009-04-27 16:25:05 +02002525 }
2526 },
2527 [F10_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02002528 .ctl_name = "F10h",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002529 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
Borislav Petkov3f37a362016-05-06 19:44:27 +02002530 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002531 .max_mcs = 2,
Doug Thompson4d376072009-04-27 16:25:05 +02002532 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01002533 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002534 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov1433eb92009-10-21 13:44:36 +02002535 .dbam_to_cs = f10_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002536 }
2537 },
2538 [F15_CPUS] = {
2539 .ctl_name = "F15h",
Borislav Petkovdf71a052011-01-19 18:15:10 +01002540 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
Borislav Petkov3f37a362016-05-06 19:44:27 +02002541 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002542 .max_mcs = 2,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002543 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01002544 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01002545 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002546 .dbam_to_cs = f15_dbam_to_chip_select,
Doug Thompson4d376072009-04-27 16:25:05 +02002547 }
2548 },
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002549 [F15_M30H_CPUS] = {
2550 .ctl_name = "F15h_M30h",
2551 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
Borislav Petkov3f37a362016-05-06 19:44:27 +02002552 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002553 .max_mcs = 2,
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002554 .ops = {
2555 .early_channel_count = f1x_early_channel_count,
2556 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2557 .dbam_to_cs = f16_dbam_to_chip_select,
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002558 }
2559 },
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002560 [F15_M60H_CPUS] = {
2561 .ctl_name = "F15h_M60h",
2562 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
Borislav Petkov3f37a362016-05-06 19:44:27 +02002563 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002564 .max_mcs = 2,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002565 .ops = {
2566 .early_channel_count = f1x_early_channel_count,
2567 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2568 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2569 }
2570 },
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05002571 [F16_CPUS] = {
2572 .ctl_name = "F16h",
2573 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
Borislav Petkov3f37a362016-05-06 19:44:27 +02002574 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002575 .max_mcs = 2,
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05002576 .ops = {
2577 .early_channel_count = f1x_early_channel_count,
2578 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2579 .dbam_to_cs = f16_dbam_to_chip_select,
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05002580 }
2581 },
Aravind Gopalakrishnan85a88852014-02-20 10:28:46 -06002582 [F16_M30H_CPUS] = {
2583 .ctl_name = "F16h_M30h",
2584 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
Borislav Petkov3f37a362016-05-06 19:44:27 +02002585 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002586 .max_mcs = 2,
Aravind Gopalakrishnan85a88852014-02-20 10:28:46 -06002587 .ops = {
2588 .early_channel_count = f1x_early_channel_count,
2589 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2590 .dbam_to_cs = f16_dbam_to_chip_select,
Aravind Gopalakrishnan85a88852014-02-20 10:28:46 -06002591 }
2592 },
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05002593 [F17_CPUS] = {
2594 .ctl_name = "F17h",
2595 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2596 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002597 .max_mcs = 2,
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05002598 .ops = {
2599 .early_channel_count = f17_early_channel_count,
Yazen Ghanname53a3b22019-08-21 23:59:59 +00002600 .dbam_to_cs = f17_addr_mask_to_cs_size,
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05002601 }
2602 },
Michael Jin8960de42018-08-16 15:28:40 -04002603 [F17_M10H_CPUS] = {
2604 .ctl_name = "F17h_M10h",
2605 .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0,
2606 .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002607 .max_mcs = 2,
Michael Jin8960de42018-08-16 15:28:40 -04002608 .ops = {
2609 .early_channel_count = f17_early_channel_count,
Yazen Ghanname53a3b22019-08-21 23:59:59 +00002610 .dbam_to_cs = f17_addr_mask_to_cs_size,
Michael Jin8960de42018-08-16 15:28:40 -04002611 }
2612 },
Yazen Ghannam6e8462392019-02-28 15:36:09 +00002613 [F17_M30H_CPUS] = {
2614 .ctl_name = "F17h_M30h",
2615 .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0,
2616 .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002617 .max_mcs = 8,
Yazen Ghannam6e8462392019-02-28 15:36:09 +00002618 .ops = {
2619 .early_channel_count = f17_early_channel_count,
Yazen Ghanname53a3b22019-08-21 23:59:59 +00002620 .dbam_to_cs = f17_addr_mask_to_cs_size,
Yazen Ghannam6e8462392019-02-28 15:36:09 +00002621 }
2622 },
Alexander Monakovb6bea242020-05-10 20:48:42 +00002623 [F17_M60H_CPUS] = {
2624 .ctl_name = "F17h_M60h",
2625 .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0,
2626 .f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6,
2627 .max_mcs = 2,
2628 .ops = {
2629 .early_channel_count = f17_early_channel_count,
2630 .dbam_to_cs = f17_addr_mask_to_cs_size,
2631 }
2632 },
Isaac Vaughn3e443eb2019-09-06 23:21:38 +00002633 [F17_M70H_CPUS] = {
2634 .ctl_name = "F17h_M70h",
2635 .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0,
2636 .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6,
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00002637 .max_mcs = 2,
Isaac Vaughn3e443eb2019-09-06 23:21:38 +00002638 .ops = {
2639 .early_channel_count = f17_early_channel_count,
2640 .dbam_to_cs = f17_addr_mask_to_cs_size,
2641 }
2642 },
Yazen Ghannam2eb61c92020-01-10 01:56:50 +00002643 [F19_CPUS] = {
2644 .ctl_name = "F19h",
2645 .f0_id = PCI_DEVICE_ID_AMD_19H_DF_F0,
2646 .f6_id = PCI_DEVICE_ID_AMD_19H_DF_F6,
2647 .max_mcs = 8,
2648 .ops = {
2649 .early_channel_count = f17_early_channel_count,
2650 .dbam_to_cs = f17_addr_mask_to_cs_size,
2651 }
2652 },
Doug Thompson4d376072009-04-27 16:25:05 +02002653};
2654
Doug Thompsonb1289d62009-04-27 16:37:05 +02002655/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002656 * These are tables of eigenvectors (one per line) which can be used for the
2657 * construction of the syndrome tables. The modified syndrome search algorithm
2658 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02002659 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002660 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02002661 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002662static const u16 x4_vectors[] = {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002663 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2664 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2665 0x0001, 0x0002, 0x0004, 0x0008,
2666 0x1013, 0x3032, 0x4044, 0x8088,
2667 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2668 0x4857, 0xc4fe, 0x13cc, 0x3288,
2669 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2670 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2671 0x15c1, 0x2a42, 0x89ac, 0x4758,
2672 0x2b03, 0x1602, 0x4f0c, 0xca08,
2673 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2674 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2675 0x2b87, 0x164e, 0x642c, 0xdc18,
2676 0x40b9, 0x80de, 0x1094, 0x20e8,
2677 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2678 0x11c1, 0x2242, 0x84ac, 0x4c58,
2679 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2680 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2681 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2682 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2683 0x16b3, 0x3d62, 0x4f34, 0x8518,
2684 0x1e2f, 0x391a, 0x5cac, 0xf858,
2685 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2686 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2687 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2688 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2689 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2690 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2691 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2692 0x185d, 0x2ca6, 0x7914, 0x9e28,
2693 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2694 0x4199, 0x82ee, 0x19f4, 0x2e58,
2695 0x4807, 0xc40e, 0x130c, 0x3208,
2696 0x1905, 0x2e0a, 0x5804, 0xac08,
2697 0x213f, 0x132a, 0xadfc, 0x5ba8,
2698 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02002699};
2700
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002701static const u16 x8_vectors[] = {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002702 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2703 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2704 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2705 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2706 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2707 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2708 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2709 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2710 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2711 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2712 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2713 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2714 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2715 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2716 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2717 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2718 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2719 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2720 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2721};
2722
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002723static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01002724 unsigned v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02002725{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002726 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02002727
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002728 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2729 u16 s = syndrome;
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01002730 unsigned v_idx = err_sym * v_dim;
2731 unsigned v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02002732
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002733 /* walk over all 16 bits of the syndrome */
2734 for (i = 1; i < (1U << 16); i <<= 1) {
2735
2736 /* if bit is set in that eigenvector... */
2737 if (v_idx < v_end && vectors[v_idx] & i) {
2738 u16 ev_comp = vectors[v_idx++];
2739
2740 /* ... and bit set in the modified syndrome, */
2741 if (s & i) {
2742 /* remove it. */
2743 s ^= ev_comp;
2744
2745 if (!s)
2746 return err_sym;
2747 }
2748
2749 } else if (s & i)
2750 /* can't get to zero, move to next symbol */
2751 break;
2752 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02002753 }
2754
Joe Perches956b9ba12012-04-29 17:08:39 -03002755 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
Doug Thompsonb1289d62009-04-27 16:37:05 +02002756 return -1;
2757}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002758
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002759static int map_err_sym_to_channel(int err_sym, int sym_size)
2760{
2761 if (sym_size == 4)
2762 switch (err_sym) {
2763 case 0x20:
2764 case 0x21:
2765 return 0;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002766 case 0x22:
2767 case 0x23:
2768 return 1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002769 default:
2770 return err_sym >> 4;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002771 }
2772 /* x8 symbols */
2773 else
2774 switch (err_sym) {
2775 /* imaginary bits not in a DIMM */
2776 case 0x10:
2777 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2778 err_sym);
2779 return -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002780 case 0x11:
2781 return 0;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002782 case 0x12:
2783 return 1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002784 default:
2785 return err_sym >> 3;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002786 }
2787 return -1;
2788}
2789
2790static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2791{
2792 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002793 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002794
Borislav Petkova3b7db02011-01-19 20:35:12 +01002795 if (pvt->ecc_sym_sz == 8)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002796 err_sym = decode_syndrome(syndrome, x8_vectors,
2797 ARRAY_SIZE(x8_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01002798 pvt->ecc_sym_sz);
2799 else if (pvt->ecc_sym_sz == 4)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002800 err_sym = decode_syndrome(syndrome, x4_vectors,
2801 ARRAY_SIZE(x4_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01002802 pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002803 else {
Borislav Petkova3b7db02011-01-19 20:35:12 +01002804 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002805 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002806 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002807
Borislav Petkova3b7db02011-01-19 20:35:12 +01002808 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002809}
2810
Yazen Ghanname70984d2016-11-17 17:57:31 -05002811static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
Borislav Petkov33ca0642012-08-30 18:01:36 +02002812 u8 ecc_type)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002813{
Borislav Petkov33ca0642012-08-30 18:01:36 +02002814 enum hw_event_mc_err_type err_type;
2815 const char *string;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002816
Borislav Petkov33ca0642012-08-30 18:01:36 +02002817 if (ecc_type == 2)
2818 err_type = HW_EVENT_ERR_CORRECTED;
2819 else if (ecc_type == 1)
2820 err_type = HW_EVENT_ERR_UNCORRECTED;
Yazen Ghannamd12a9692016-11-17 17:57:32 -05002821 else if (ecc_type == 3)
2822 err_type = HW_EVENT_ERR_DEFERRED;
Borislav Petkov33ca0642012-08-30 18:01:36 +02002823 else {
2824 WARN(1, "Something is rotten in the state of Denmark.\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002825 return;
2826 }
2827
Borislav Petkov33ca0642012-08-30 18:01:36 +02002828 switch (err->err_code) {
2829 case DECODE_OK:
2830 string = "";
2831 break;
2832 case ERR_NODE:
2833 string = "Failed to map error addr to a node";
2834 break;
2835 case ERR_CSROW:
2836 string = "Failed to map error addr to a csrow";
2837 break;
2838 case ERR_CHANNEL:
Yazen Ghannam713ad542016-11-28 12:59:53 -06002839 string = "Unknown syndrome - possible error reporting race";
2840 break;
2841 case ERR_SYND:
2842 string = "MCA_SYND not valid - unknown syndrome and csrow";
2843 break;
2844 case ERR_NORM_ADDR:
2845 string = "Cannot decode normalized address";
Borislav Petkov33ca0642012-08-30 18:01:36 +02002846 break;
2847 default:
2848 string = "WTF error";
2849 break;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002850 }
Borislav Petkov33ca0642012-08-30 18:01:36 +02002851
2852 edac_mc_handle_error(err_type, mci, 1,
2853 err->page, err->offset, err->syndrome,
2854 err->csrow, err->channel, -1,
2855 string, "");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002856}
2857
Borislav Petkovdf781d02013-12-15 17:29:44 +01002858static inline void decode_bus_error(int node_id, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002859{
Daniel J Blueman0c510cc2015-02-17 11:34:38 +08002860 struct mem_ctl_info *mci;
2861 struct amd64_pvt *pvt;
Borislav Petkovf192c7b12011-01-10 14:24:32 +01002862 u8 ecc_type = (m->status >> 45) & 0x3;
Borislav Petkov66fed2d2012-08-09 18:41:07 +02002863 u8 xec = XEC(m->status, 0x1f);
2864 u16 ec = EC(m->status);
Borislav Petkov33ca0642012-08-30 18:01:36 +02002865 u64 sys_addr;
2866 struct err_info err;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002867
Daniel J Blueman0c510cc2015-02-17 11:34:38 +08002868 mci = edac_mc_find(node_id);
2869 if (!mci)
2870 return;
2871
2872 pvt = mci->pvt_info;
2873
Borislav Petkov66fed2d2012-08-09 18:41:07 +02002874 /* Bail out early if this was an 'observed' error */
Borislav Petkov5980bb92011-01-07 16:26:49 +01002875 if (PP(ec) == NBSL_PP_OBS)
Borislav Petkovb70ef012009-06-25 19:32:38 +02002876 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002877
Borislav Petkovecaf5602009-07-23 16:32:01 +02002878 /* Do only ECC errors */
2879 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002880 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002881
Borislav Petkov33ca0642012-08-30 18:01:36 +02002882 memset(&err, 0, sizeof(err));
2883
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002884 sys_addr = get_error_address(pvt, m);
Borislav Petkov33ca0642012-08-30 18:01:36 +02002885
Borislav Petkovecaf5602009-07-23 16:32:01 +02002886 if (ecc_type == 2)
Borislav Petkov33ca0642012-08-30 18:01:36 +02002887 err.syndrome = extract_syndrome(m->status);
2888
2889 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2890
Yazen Ghanname70984d2016-11-17 17:57:31 -05002891 __log_ecc_error(mci, &err, ecc_type);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002892}
2893
Doug Thompson0ec449e2009-04-27 19:41:25 +02002894/*
Yazen Ghannam713ad542016-11-28 12:59:53 -06002895 * To find the UMC channel represented by this bank we need to match on its
2896 * instance_id. The instance_id of a bank is held in the lower 32 bits of its
2897 * IPID.
Yazen Ghannambdcee772019-02-28 15:36:10 +00002898 *
2899 * Currently, we can derive the channel number by looking at the 6th nibble in
2900 * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
2901 * number.
Yazen Ghannam713ad542016-11-28 12:59:53 -06002902 */
Yazen Ghannambdcee772019-02-28 15:36:10 +00002903static int find_umc_channel(struct mce *m)
Yazen Ghannam713ad542016-11-28 12:59:53 -06002904{
Yazen Ghannambdcee772019-02-28 15:36:10 +00002905 return (m->ipid & GENMASK(31, 0)) >> 20;
Yazen Ghannam713ad542016-11-28 12:59:53 -06002906}
2907
2908static void decode_umc_error(int node_id, struct mce *m)
2909{
2910 u8 ecc_type = (m->status >> 45) & 0x3;
2911 struct mem_ctl_info *mci;
2912 struct amd64_pvt *pvt;
2913 struct err_info err;
2914 u64 sys_addr;
2915
2916 mci = edac_mc_find(node_id);
2917 if (!mci)
2918 return;
2919
2920 pvt = mci->pvt_info;
2921
2922 memset(&err, 0, sizeof(err));
2923
2924 if (m->status & MCI_STATUS_DEFERRED)
2925 ecc_type = 3;
2926
Yazen Ghannambdcee772019-02-28 15:36:10 +00002927 err.channel = find_umc_channel(m);
Yazen Ghannam713ad542016-11-28 12:59:53 -06002928
Yazen Ghannam713ad542016-11-28 12:59:53 -06002929 if (!(m->status & MCI_STATUS_SYNDV)) {
2930 err.err_code = ERR_SYND;
2931 goto log_error;
2932 }
2933
2934 if (ecc_type == 2) {
2935 u8 length = (m->synd >> 18) & 0x3f;
2936
2937 if (length)
2938 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2939 else
2940 err.err_code = ERR_CHANNEL;
2941 }
2942
2943 err.csrow = m->synd & 0x7;
2944
Yazen Ghannam8a2eaab2019-08-22 00:00:00 +00002945 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2946 err.err_code = ERR_NORM_ADDR;
2947 goto log_error;
2948 }
2949
2950 error_address_to_page_and_offset(sys_addr, &err);
2951
Yazen Ghannam713ad542016-11-28 12:59:53 -06002952log_error:
2953 __log_ecc_error(mci, &err, ecc_type);
2954}
2955
2956/*
Borislav Petkov3f37a362016-05-06 19:44:27 +02002957 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
2958 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002959 * Reserve F0 and F6 on systems with a UMC.
Doug Thompson0ec449e2009-04-27 19:41:25 +02002960 */
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002961static int
2962reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002963{
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002964 if (pvt->umc) {
2965 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2966 if (!pvt->F0) {
Yazen Ghannam6a4afe32020-12-15 17:01:31 +00002967 edac_dbg(1, "F0 not found, device 0x%x\n", pci_id1);
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002968 return -ENODEV;
2969 }
2970
2971 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2972 if (!pvt->F6) {
2973 pci_dev_put(pvt->F0);
2974 pvt->F0 = NULL;
2975
Yazen Ghannam6a4afe32020-12-15 17:01:31 +00002976 edac_dbg(1, "F6 not found: device 0x%x\n", pci_id2);
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002977 return -ENODEV;
2978 }
Borislav Petkov5246c542016-12-01 11:35:07 +01002979
Borislav Petkov706657b12020-11-22 15:57:21 +01002980 if (!pci_ctl_dev)
2981 pci_ctl_dev = &pvt->F0->dev;
2982
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002983 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2984 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2985 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2986
2987 return 0;
2988 }
2989
Doug Thompson0ec449e2009-04-27 19:41:25 +02002990 /* Reserve the ADDRESS MAP Device */
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002991 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002992 if (!pvt->F1) {
Yazen Ghannam6a4afe32020-12-15 17:01:31 +00002993 edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1);
Borislav Petkovbbd0c1f62010-10-01 19:27:58 +02002994 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002995 }
2996
Borislav Petkov3f37a362016-05-06 19:44:27 +02002997 /* Reserve the DCT Device */
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05002998 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
Borislav Petkov3f37a362016-05-06 19:44:27 +02002999 if (!pvt->F2) {
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02003000 pci_dev_put(pvt->F1);
3001 pvt->F1 = NULL;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003002
Yazen Ghannam6a4afe32020-12-15 17:01:31 +00003003 edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2);
Borislav Petkov5246c542016-12-01 11:35:07 +01003004 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003005 }
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05003006
Borislav Petkov706657b12020-11-22 15:57:21 +01003007 if (!pci_ctl_dev)
3008 pci_ctl_dev = &pvt->F2->dev;
3009
Joe Perches956b9ba12012-04-29 17:08:39 -03003010 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
3011 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
3012 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
Doug Thompson0ec449e2009-04-27 19:41:25 +02003013
3014 return 0;
3015}
3016
Borislav Petkov360b7f32010-10-15 19:25:38 +02003017static void free_mc_sibling_devs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02003018{
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05003019 if (pvt->umc) {
3020 pci_dev_put(pvt->F0);
3021 pci_dev_put(pvt->F6);
3022 } else {
3023 pci_dev_put(pvt->F1);
3024 pci_dev_put(pvt->F2);
3025 }
Doug Thompson0ec449e2009-04-27 19:41:25 +02003026}
3027
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003028static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
3029{
3030 pvt->ecc_sym_sz = 4;
3031
3032 if (pvt->umc) {
3033 u8 i;
3034
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +00003035 for_each_umc(i) {
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003036 /* Check enabled channels only: */
Yazen Ghannam78359612019-02-28 15:36:11 +00003037 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3038 if (pvt->umc[i].ecc_ctrl & BIT(9)) {
3039 pvt->ecc_sym_sz = 16;
3040 return;
3041 } else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
3042 pvt->ecc_sym_sz = 8;
3043 return;
3044 }
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003045 }
3046 }
Yazen Ghannam78359612019-02-28 15:36:11 +00003047 } else if (pvt->fam >= 0x10) {
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003048 u32 tmp;
3049
3050 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
3051 /* F16h has only DCT0, so no need to read dbam1. */
3052 if (pvt->fam != 0x16)
3053 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
3054
3055 /* F10h, revD and later can do x8 ECC too. */
3056 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
3057 pvt->ecc_sym_sz = 8;
3058 }
3059}
3060
3061/*
3062 * Retrieve the hardware registers of the memory controller.
3063 */
3064static void __read_mc_regs_df(struct amd64_pvt *pvt)
3065{
3066 u8 nid = pvt->mc_node_id;
3067 struct amd64_umc *umc;
3068 u32 i, umc_base;
3069
3070 /* Read registers from each UMC */
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +00003071 for_each_umc(i) {
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003072
3073 umc_base = get_umc_base(i);
3074 umc = &pvt->umc[i];
3075
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06003076 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
3077 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003078 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3079 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
Yazen Ghannam07ed82e2016-11-28 08:50:21 -06003080 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003081 }
3082}
3083
Doug Thompson0ec449e2009-04-27 19:41:25 +02003084/*
3085 * Retrieve the hardware registers of the memory controller (this includes the
3086 * 'Address Map' and 'Misc' device regs)
3087 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02003088static void read_mc_regs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02003089{
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003090 unsigned int range;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003091 u64 msr_val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003092
3093 /*
3094 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003095 * those are Read-As-Zero.
Doug Thompson0ec449e2009-04-27 19:41:25 +02003096 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02003097 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
Joe Perches956b9ba12012-04-29 17:08:39 -03003098 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003099
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003100 /* Check first whether TOP_MEM2 is enabled: */
Brijesh Singh059e5c32021-04-27 06:16:36 -05003101 rdmsrl(MSR_AMD64_SYSCFG, msr_val);
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003102 if (msr_val & BIT(21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02003103 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
Joe Perches956b9ba12012-04-29 17:08:39 -03003104 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003105 } else {
Joe Perches956b9ba12012-04-29 17:08:39 -03003106 edac_dbg(0, " TOP_MEM2 disabled\n");
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003107 }
3108
3109 if (pvt->umc) {
3110 __read_mc_regs_df(pvt);
3111 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
3112
3113 goto skip;
3114 }
Doug Thompson0ec449e2009-04-27 19:41:25 +02003115
Borislav Petkov5980bb92011-01-07 16:26:49 +01003116 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003117
Borislav Petkov5a5d2372011-01-17 17:52:57 +01003118 read_dram_ctl_register(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003119
Borislav Petkov7f19bf72010-10-21 18:52:53 +02003120 for (range = 0; range < DRAM_RANGES; range++) {
3121 u8 rw;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003122
Borislav Petkov7f19bf72010-10-21 18:52:53 +02003123 /* read settings for this DRAM range */
3124 read_dram_base_limit_regs(pvt, range);
Borislav Petkove97f8bb2009-10-12 15:27:45 +02003125
Borislav Petkov7f19bf72010-10-21 18:52:53 +02003126 rw = dram_rw(pvt, range);
3127 if (!rw)
3128 continue;
3129
Joe Perches956b9ba12012-04-29 17:08:39 -03003130 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
3131 range,
3132 get_dram_base(pvt, range),
3133 get_dram_limit(pvt, range));
Borislav Petkov7f19bf72010-10-21 18:52:53 +02003134
Joe Perches956b9ba12012-04-29 17:08:39 -03003135 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
3136 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
3137 (rw & 0x1) ? "R" : "-",
3138 (rw & 0x2) ? "W" : "-",
3139 dram_intlv_sel(pvt, range),
3140 dram_dst_node(pvt, range));
Doug Thompson0ec449e2009-04-27 19:41:25 +02003141 }
3142
Borislav Petkovbc21fa52010-11-11 17:29:13 +01003143 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05003144 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003145
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02003146 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003147
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05003148 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
3149 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003150
Borislav Petkov78da1212010-12-22 19:31:45 +01003151 if (!dct_ganging_enabled(pvt)) {
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05003152 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
3153 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003154 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01003155
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003156skip:
3157 read_dct_base_mask(pvt);
3158
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01003159 determine_memory_type(pvt);
3160 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
Borislav Petkova3b7db02011-01-19 20:35:12 +01003161
Yazen Ghannamb64ce7c2016-11-17 17:57:37 -05003162 determine_ecc_sym_sz(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003163}
3164
3165/*
3166 * NOTE: CPU Revision Dependent code
3167 *
3168 * Input:
Borislav Petkov11c75ea2010-11-29 19:49:02 +01003169 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02003170 * k8 private pointer to -->
3171 * DRAM Bank Address mapping register
3172 * node_id
3173 * DCL register where dual_channel_active is
3174 *
3175 * The DBAM register consists of 4 sets of 4 bits each definitions:
3176 *
3177 * Bits: CSROWs
3178 * 0-3 CSROWs 0 and 1
3179 * 4-7 CSROWs 2 and 3
3180 * 8-11 CSROWs 4 and 5
3181 * 12-15 CSROWs 6 and 7
3182 *
3183 * Values range from: 0 to 15
3184 * The meaning of the values depends on CPU revision and dual-channel state,
3185 * see relevant BKDG more info.
3186 *
3187 * The memory controller provides for total of only 8 CSROWs in its current
3188 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
3189 * single channel or two (2) DIMMs in dual channel mode.
3190 *
3191 * The following code logic collapses the various tables for CSROW based on CPU
3192 * revision.
3193 *
3194 * Returns:
3195 * The number of PAGE_SIZE pages on the specified CSROW number it
3196 * encompasses
3197 *
3198 */
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05003199static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
Doug Thompson0ec449e2009-04-27 19:41:25 +02003200{
Ashish Shenoyf92cae42012-02-22 17:20:38 -08003201 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05003202 int csrow_nr = csrow_nr_orig;
3203 u32 cs_mode, nr_pages;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003204
Yazen Ghanname53a3b22019-08-21 23:59:59 +00003205 if (!pvt->umc) {
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05003206 csrow_nr >>= 1;
Yazen Ghanname53a3b22019-08-21 23:59:59 +00003207 cs_mode = DBAM_DIMM(csrow_nr, dbam);
3208 } else {
3209 cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt);
3210 }
Doug Thompson0ec449e2009-04-27 19:41:25 +02003211
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05003212 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
3213 nr_pages <<= 20 - PAGE_SHIFT;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003214
Borislav Petkov10de6492012-09-12 19:00:38 +02003215 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
Yazen Ghannameb77e6b2017-04-27 12:11:54 -05003216 csrow_nr_orig, dct, cs_mode);
Borislav Petkov10de6492012-09-12 19:00:38 +02003217 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003218
3219 return nr_pages;
3220}
3221
Yazen Ghannam353a1fc2019-08-21 23:59:57 +00003222static int init_csrows_df(struct mem_ctl_info *mci)
3223{
3224 struct amd64_pvt *pvt = mci->pvt_info;
3225 enum edac_type edac_mode = EDAC_NONE;
3226 enum dev_type dev_type = DEV_UNKNOWN;
3227 struct dimm_info *dimm;
3228 int empty = 1;
3229 u8 umc, cs;
3230
3231 if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
3232 edac_mode = EDAC_S16ECD16ED;
3233 dev_type = DEV_X16;
3234 } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
3235 edac_mode = EDAC_S8ECD8ED;
3236 dev_type = DEV_X8;
3237 } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
3238 edac_mode = EDAC_S4ECD4ED;
3239 dev_type = DEV_X4;
3240 } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
3241 edac_mode = EDAC_SECDED;
3242 }
3243
3244 for_each_umc(umc) {
3245 for_each_chip_select(cs, umc, pvt) {
3246 if (!csrow_enabled(cs, umc, pvt))
3247 continue;
3248
3249 empty = 0;
3250 dimm = mci->csrows[cs]->channels[umc]->dimm;
3251
3252 edac_dbg(1, "MC node: %d, csrow: %d\n",
3253 pvt->mc_node_id, cs);
3254
3255 dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs);
3256 dimm->mtype = pvt->dram_type;
3257 dimm->edac_mode = edac_mode;
3258 dimm->dtype = dev_type;
Yazen Ghannam466503d2019-10-22 20:35:14 +00003259 dimm->grain = 64;
Yazen Ghannam353a1fc2019-08-21 23:59:57 +00003260 }
3261 }
3262
3263 return empty;
3264}
3265
Doug Thompson0ec449e2009-04-27 19:41:25 +02003266/*
3267 * Initialize the array of csrow attribute instances, based on the values
3268 * from pci config hardware registers.
3269 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02003270static int init_csrows(struct mem_ctl_info *mci)
Doug Thompson0ec449e2009-04-27 19:41:25 +02003271{
Borislav Petkov10de6492012-09-12 19:00:38 +02003272 struct amd64_pvt *pvt = mci->pvt_info;
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003273 enum edac_type edac_mode = EDAC_NONE;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003274 struct csrow_info *csrow;
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03003275 struct dimm_info *dimm;
Borislav Petkov10de6492012-09-12 19:00:38 +02003276 int i, j, empty = 1;
Mauro Carvalho Chehaba895bf82012-01-28 09:09:38 -03003277 int nr_pages = 0;
Borislav Petkov10de6492012-09-12 19:00:38 +02003278 u32 val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003279
Yazen Ghannam353a1fc2019-08-21 23:59:57 +00003280 if (pvt->umc)
3281 return init_csrows_df(mci);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003282
Yazen Ghannam353a1fc2019-08-21 23:59:57 +00003283 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003284
Yazen Ghannam353a1fc2019-08-21 23:59:57 +00003285 pvt->nbcfg = val;
3286
3287 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
3288 pvt->mc_node_id, val,
3289 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
Doug Thompson0ec449e2009-04-27 19:41:25 +02003290
Borislav Petkov10de6492012-09-12 19:00:38 +02003291 /*
3292 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
3293 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01003294 for_each_chip_select(i, 0, pvt) {
Borislav Petkov10de6492012-09-12 19:00:38 +02003295 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
3296 bool row_dct1 = false;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003297
Borislav Petkova4b4bed2013-08-10 13:54:48 +02003298 if (pvt->fam != 0xf)
Borislav Petkov10de6492012-09-12 19:00:38 +02003299 row_dct1 = !!csrow_enabled(i, 1, pvt);
3300
3301 if (!row_dct0 && !row_dct1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02003302 continue;
Doug Thompson0ec449e2009-04-27 19:41:25 +02003303
Borislav Petkov10de6492012-09-12 19:00:38 +02003304 csrow = mci->csrows[i];
Doug Thompson0ec449e2009-04-27 19:41:25 +02003305 empty = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01003306
Borislav Petkov10de6492012-09-12 19:00:38 +02003307 edac_dbg(1, "MC node: %d, csrow: %d\n",
3308 pvt->mc_node_id, i);
3309
Mauro Carvalho Chehab1eef1282013-03-11 09:07:46 -03003310 if (row_dct0) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003311 nr_pages = get_csrow_nr_pages(pvt, 0, i);
Mauro Carvalho Chehab1eef1282013-03-11 09:07:46 -03003312 csrow->channels[0]->dimm->nr_pages = nr_pages;
3313 }
Borislav Petkov10de6492012-09-12 19:00:38 +02003314
3315 /* K8 has only one DCT */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02003316 if (pvt->fam != 0xf && row_dct1) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003317 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
Mauro Carvalho Chehab1eef1282013-03-11 09:07:46 -03003318
3319 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
3320 nr_pages += row_dct1_pages;
3321 }
Doug Thompson0ec449e2009-04-27 19:41:25 +02003322
Borislav Petkov10de6492012-09-12 19:00:38 +02003323 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
Doug Thompson0ec449e2009-04-27 19:41:25 +02003324
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003325 /* Determine DIMM ECC mode: */
Yazen Ghannam353a1fc2019-08-21 23:59:57 +00003326 if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003327 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
3328 ? EDAC_S4ECD4ED
3329 : EDAC_SECDED;
3330 }
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03003331
3332 for (j = 0; j < pvt->channel_count; j++) {
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03003333 dimm = csrow->channels[j]->dimm;
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01003334 dimm->mtype = pvt->dram_type;
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03003335 dimm->edac_mode = edac_mode;
Yazen Ghannam466503d2019-10-22 20:35:14 +00003336 dimm->grain = 64;
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03003337 }
Doug Thompson0ec449e2009-04-27 19:41:25 +02003338 }
3339
3340 return empty;
3341}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02003342
Borislav Petkov06724532009-09-16 13:05:46 +02003343/* get all cores on this DCT */
Daniel J Blueman8b84c8d2012-11-27 14:32:10 +08003344static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02003345{
Borislav Petkov06724532009-09-16 13:05:46 +02003346 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02003347
Borislav Petkov06724532009-09-16 13:05:46 +02003348 for_each_online_cpu(cpu)
Yazen Ghannamdb970bd22020-11-09 21:06:57 +00003349 if (topology_die_id(cpu) == nid)
Borislav Petkov06724532009-09-16 13:05:46 +02003350 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02003351}
3352
3353/* check MCG_CTL on all the cpus on this node */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003354static bool nb_mce_bank_enabled_on_node(u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02003355{
Rusty Russellba578cb2009-11-03 14:56:35 +10303356 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01003357 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02003358 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02003359
Rusty Russellba578cb2009-11-03 14:56:35 +10303360 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02003361 amd64_warn("%s: Error allocating mask\n", __func__);
Rusty Russellba578cb2009-11-03 14:56:35 +10303362 return false;
3363 }
Borislav Petkov06724532009-09-16 13:05:46 +02003364
Rusty Russellba578cb2009-11-03 14:56:35 +10303365 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02003366
Rusty Russellba578cb2009-11-03 14:56:35 +10303367 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02003368
Rusty Russellba578cb2009-11-03 14:56:35 +10303369 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01003370 struct msr *reg = per_cpu_ptr(msrs, cpu);
Borislav Petkov5980bb92011-01-07 16:26:49 +01003371 nbe = reg->l & MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02003372
Joe Perches956b9ba12012-04-29 17:08:39 -03003373 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
3374 cpu, reg->q,
3375 (nbe ? "enabled" : "disabled"));
Borislav Petkov06724532009-09-16 13:05:46 +02003376
3377 if (!nbe)
3378 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02003379 }
3380 ret = true;
3381
3382out:
Rusty Russellba578cb2009-11-03 14:56:35 +10303383 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02003384 return ret;
3385}
3386
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08003387static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003388{
3389 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01003390 int cpu;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003391
3392 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02003393 amd64_warn("%s: error allocating mask\n", __func__);
Pan Bian0de278842016-12-04 14:07:18 +08003394 return -ENOMEM;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003395 }
3396
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003397 get_cpus_on_this_dct_cpumask(cmask, nid);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003398
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003399 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3400
3401 for_each_cpu(cpu, cmask) {
3402
Borislav Petkov50542252009-12-11 18:14:40 +01003403 struct msr *reg = per_cpu_ptr(msrs, cpu);
3404
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003405 if (on) {
Borislav Petkov5980bb92011-01-07 16:26:49 +01003406 if (reg->l & MSR_MCGCTL_NBE)
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003407 s->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003408
Borislav Petkov5980bb92011-01-07 16:26:49 +01003409 reg->l |= MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003410 } else {
3411 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01003412 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003413 */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003414 if (!s->flags.nb_mce_enable)
Borislav Petkov5980bb92011-01-07 16:26:49 +01003415 reg->l &= ~MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003416 }
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003417 }
3418 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3419
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003420 free_cpumask_var(cmask);
3421
3422 return 0;
3423}
3424
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08003425static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
Borislav Petkov2299ef72010-10-15 17:44:04 +02003426 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003427{
Borislav Petkov2299ef72010-10-15 17:44:04 +02003428 bool ret = true;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01003429 u32 value, mask = 0x3; /* UECC/CECC enable */
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003430
Borislav Petkov2299ef72010-10-15 17:44:04 +02003431 if (toggle_ecc_err_reporting(s, nid, ON)) {
3432 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3433 return false;
3434 }
3435
Borislav Petkovc9f4f262010-12-22 19:48:20 +01003436 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003437
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003438 s->old_nbctl = value & mask;
3439 s->nbctl_valid = true;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003440
3441 value |= mask;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01003442 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003443
Borislav Petkova97fa682010-12-23 14:07:18 +01003444 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003445
Joe Perches956b9ba12012-04-29 17:08:39 -03003446 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3447 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003448
Borislav Petkova97fa682010-12-23 14:07:18 +01003449 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02003450 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003451
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003452 s->flags.nb_ecc_prev = 0;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01003453
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003454 /* Attempt to turn on DRAM ECC Enable */
Borislav Petkova97fa682010-12-23 14:07:18 +01003455 value |= NBCFG_ECC_ENABLE;
3456 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003457
Borislav Petkova97fa682010-12-23 14:07:18 +01003458 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003459
Borislav Petkova97fa682010-12-23 14:07:18 +01003460 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02003461 amd64_warn("Hardware rejected DRAM ECC enable,"
3462 "check memory DIMM configuration.\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02003463 ret = false;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003464 } else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02003465 amd64_info("Hardware accepted DRAM ECC Enable\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003466 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01003467 } else {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003468 s->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003469 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01003470
Joe Perches956b9ba12012-04-29 17:08:39 -03003471 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3472 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003473
Borislav Petkov2299ef72010-10-15 17:44:04 +02003474 return ret;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003475}
3476
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08003477static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
Borislav Petkov360b7f32010-10-15 19:25:38 +02003478 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003479{
Borislav Petkovc9f4f262010-12-22 19:48:20 +01003480 u32 value, mask = 0x3; /* UECC/CECC enable */
3481
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003482 if (!s->nbctl_valid)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003483 return;
3484
Borislav Petkovc9f4f262010-12-22 19:48:20 +01003485 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003486 value &= ~mask;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003487 value |= s->old_nbctl;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003488
Borislav Petkovc9f4f262010-12-22 19:48:20 +01003489 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003490
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003491 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3492 if (!s->flags.nb_ecc_prev) {
Borislav Petkova97fa682010-12-23 14:07:18 +01003493 amd64_read_pci_cfg(F3, NBCFG, &value);
3494 value &= ~NBCFG_ECC_ENABLE;
3495 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01003496 }
3497
3498 /* restore the NB Enable MCGCTL bit */
Borislav Petkov2299ef72010-10-15 17:44:04 +02003499 if (toggle_ecc_err_reporting(s, nid, OFF))
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02003500 amd64_warn("Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01003501}
3502
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003503static bool ecc_enabled(struct amd64_pvt *pvt)
Doug Thompsonf9431992009-04-27 19:46:08 +02003504{
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003505 u16 nid = pvt->mc_node_id;
Borislav Petkov06724532009-09-16 13:05:46 +02003506 bool nb_mce_en = false;
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003507 u8 ecc_en = 0, i;
3508 u32 value;
Doug Thompsonf9431992009-04-27 19:46:08 +02003509
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003510 if (boot_cpu_data.x86 >= 0x17) {
3511 u8 umc_en_mask = 0, ecc_en_mask = 0;
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003512 struct amd64_umc *umc;
Doug Thompsonf9431992009-04-27 19:46:08 +02003513
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +00003514 for_each_umc(i) {
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003515 umc = &pvt->umc[i];
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003516
3517 /* Only check enabled UMCs. */
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003518 if (!(umc->sdp_ctrl & UMC_SDP_INIT))
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003519 continue;
3520
3521 umc_en_mask |= BIT(i);
3522
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003523 if (umc->umc_cap_hi & UMC_ECC_ENABLED)
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003524 ecc_en_mask |= BIT(i);
3525 }
3526
3527 /* Check whether at least one UMC is enabled: */
3528 if (umc_en_mask)
3529 ecc_en = umc_en_mask == ecc_en_mask;
Yazen Ghannam11ab1ca2017-01-27 11:24:19 -06003530 else
3531 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003532
3533 /* Assume UMC MCA banks are enabled. */
3534 nb_mce_en = true;
3535 } else {
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003536 amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003537
3538 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3539
3540 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3541 if (!nb_mce_en)
Yazen Ghannam11ab1ca2017-01-27 11:24:19 -06003542 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
Yazen Ghannam196b79f2016-11-17 17:57:34 -05003543 MSR_IA32_MCG_CTL, nid);
3544 }
3545
Borislav Petkov4cbcb732021-01-13 20:13:30 +01003546 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
Doug Thompsonf9431992009-04-27 19:46:08 +02003547
Borislav Petkov7fdfee92019-11-09 10:00:54 +01003548 if (!ecc_en || !nb_mce_en)
Borislav Petkov2299ef72010-10-15 17:44:04 +02003549 return false;
Borislav Petkov7fdfee92019-11-09 10:00:54 +01003550 else
3551 return true;
Doug Thompsonf9431992009-04-27 19:46:08 +02003552}
3553
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003554static inline void
3555f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3556{
Yazen Ghannamf8be8e52019-08-21 23:59:56 +00003557 u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003558
Yazen Ghannam4d30d2b2019-02-28 15:36:10 +00003559 for_each_umc(i) {
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003560 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3561 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3562 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
Yazen Ghannamf8be8e52019-08-21 23:59:56 +00003563
3564 dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6));
3565 dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003566 }
3567 }
3568
3569 /* Set chipkill only if ECC is enabled: */
3570 if (ecc_en) {
3571 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3572
Yazen Ghannamf8be8e52019-08-21 23:59:56 +00003573 if (!cpk_en)
3574 return;
3575
3576 if (dev_x4)
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003577 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
Yazen Ghannamf8be8e52019-08-21 23:59:56 +00003578 else if (dev_x16)
3579 mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
3580 else
3581 mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003582 }
3583}
3584
Yazen Ghannam38ddd4d2019-10-22 20:35:09 +00003585static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
Doug Thompson7d6034d2009-04-27 20:01:01 +02003586{
3587 struct amd64_pvt *pvt = mci->pvt_info;
3588
3589 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3590 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003591
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003592 if (pvt->umc) {
3593 f17h_determine_edac_ctl_cap(mci, pvt);
3594 } else {
3595 if (pvt->nbcap & NBCAP_SECDED)
3596 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003597
Yazen Ghannam2d09d8f2016-11-29 08:51:56 -06003598 if (pvt->nbcap & NBCAP_CHIPKILL)
3599 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3600 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02003601
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003602 mci->edac_cap = determine_edac_cap(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003603 mci->mod_name = EDAC_MOD_STR;
Yazen Ghannam38ddd4d2019-10-22 20:35:09 +00003604 mci->ctl_name = fam_type->ctl_name;
Yazen Ghanname7934b72016-11-17 17:57:30 -05003605 mci->dev_name = pci_name(pvt->F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003606 mci->ctl_page_to_phys = NULL;
3607
Doug Thompson7d6034d2009-04-27 20:01:01 +02003608 /* memory scrubber interface */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003609 mci->set_sdram_scrub_rate = set_scrub_rate;
3610 mci->get_sdram_scrub_rate = get_scrub_rate;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003611}
3612
Borislav Petkov0092b202010-10-01 19:20:05 +02003613/*
3614 * returns a pointer to the family descriptor on success, NULL otherwise.
3615 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003616static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
Borislav Petkov395ae782010-10-01 18:38:19 +02003617{
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05003618 pvt->ext_model = boot_cpu_data.x86_model >> 4;
Jia Zhangb3991512018-01-01 09:52:10 +08003619 pvt->stepping = boot_cpu_data.x86_stepping;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05003620 pvt->model = boot_cpu_data.x86_model;
3621 pvt->fam = boot_cpu_data.x86;
3622
3623 switch (pvt->fam) {
Borislav Petkov395ae782010-10-01 18:38:19 +02003624 case 0xf:
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003625 fam_type = &family_types[K8_CPUS];
3626 pvt->ops = &family_types[K8_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02003627 break;
Borislav Petkovdf71a052011-01-19 18:15:10 +01003628
Borislav Petkov395ae782010-10-01 18:38:19 +02003629 case 0x10:
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003630 fam_type = &family_types[F10_CPUS];
3631 pvt->ops = &family_types[F10_CPUS].ops;
Borislav Petkovdf71a052011-01-19 18:15:10 +01003632 break;
3633
3634 case 0x15:
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05003635 if (pvt->model == 0x30) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003636 fam_type = &family_types[F15_M30H_CPUS];
3637 pvt->ops = &family_types[F15_M30H_CPUS].ops;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05003638 break;
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01003639 } else if (pvt->model == 0x60) {
3640 fam_type = &family_types[F15_M60H_CPUS];
3641 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3642 break;
Borislav Petkov6c13d7f2020-12-12 15:20:28 +01003643 /* Richland is only client */
3644 } else if (pvt->model == 0x13) {
3645 return NULL;
3646 } else {
3647 fam_type = &family_types[F15_CPUS];
3648 pvt->ops = &family_types[F15_CPUS].ops;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05003649 }
Borislav Petkov395ae782010-10-01 18:38:19 +02003650 break;
3651
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05003652 case 0x16:
Aravind Gopalakrishnan85a88852014-02-20 10:28:46 -06003653 if (pvt->model == 0x30) {
3654 fam_type = &family_types[F16_M30H_CPUS];
3655 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3656 break;
3657 }
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003658 fam_type = &family_types[F16_CPUS];
3659 pvt->ops = &family_types[F16_CPUS].ops;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05003660 break;
3661
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05003662 case 0x17:
Michael Jin8960de42018-08-16 15:28:40 -04003663 if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
3664 fam_type = &family_types[F17_M10H_CPUS];
3665 pvt->ops = &family_types[F17_M10H_CPUS].ops;
3666 break;
Yazen Ghannam6e8462392019-02-28 15:36:09 +00003667 } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
3668 fam_type = &family_types[F17_M30H_CPUS];
3669 pvt->ops = &family_types[F17_M30H_CPUS].ops;
3670 break;
Alexander Monakovb6bea242020-05-10 20:48:42 +00003671 } else if (pvt->model >= 0x60 && pvt->model <= 0x6f) {
3672 fam_type = &family_types[F17_M60H_CPUS];
3673 pvt->ops = &family_types[F17_M60H_CPUS].ops;
3674 break;
Isaac Vaughn3e443eb2019-09-06 23:21:38 +00003675 } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
3676 fam_type = &family_types[F17_M70H_CPUS];
3677 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3678 break;
Michael Jin8960de42018-08-16 15:28:40 -04003679 }
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05003680 fallthrough;
Pu Wenc4a3e942018-09-27 16:31:28 +02003681 case 0x18:
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05003682 fam_type = &family_types[F17_CPUS];
3683 pvt->ops = &family_types[F17_CPUS].ops;
Pu Wenc4a3e942018-09-27 16:31:28 +02003684
3685 if (pvt->fam == 0x18)
3686 family_types[F17_CPUS].ctl_name = "F18h";
Yazen Ghannamf1cbbec2016-11-17 17:57:35 -05003687 break;
3688
Yazen Ghannam2eb61c92020-01-10 01:56:50 +00003689 case 0x19:
Yazen Ghannamb4210ea2020-10-09 17:18:03 +00003690 if (pvt->model >= 0x20 && pvt->model <= 0x2f) {
3691 fam_type = &family_types[F17_M70H_CPUS];
3692 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3693 fam_type->ctl_name = "F19h_M20h";
3694 break;
3695 }
Yazen Ghannam2eb61c92020-01-10 01:56:50 +00003696 fam_type = &family_types[F19_CPUS];
3697 pvt->ops = &family_types[F19_CPUS].ops;
3698 family_types[F19_CPUS].ctl_name = "F19h";
3699 break;
3700
Borislav Petkov395ae782010-10-01 18:38:19 +02003701 default:
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02003702 amd64_err("Unsupported family!\n");
Borislav Petkov0092b202010-10-01 19:20:05 +02003703 return NULL;
Borislav Petkov395ae782010-10-01 18:38:19 +02003704 }
Borislav Petkov0092b202010-10-01 19:20:05 +02003705
Borislav Petkov0092b202010-10-01 19:20:05 +02003706 return fam_type;
Borislav Petkov395ae782010-10-01 18:38:19 +02003707}
3708
Takashi Iwaie339f1e2015-02-04 11:48:53 +01003709static const struct attribute_group *amd64_edac_attr_groups[] = {
3710#ifdef CONFIG_EDAC_DEBUG
Borislav Petkov2a28ceef2020-12-14 20:47:11 +01003711 &dbg_group,
Borislav Petkov61810092020-12-15 09:18:44 +01003712 &inj_group,
Takashi Iwaie339f1e2015-02-04 11:48:53 +01003713#endif
3714 NULL
3715};
3716
Yazen Ghannam80355a32019-10-22 20:35:10 +00003717static int hw_info_get(struct amd64_pvt *pvt)
Doug Thompson7d6034d2009-04-27 20:01:01 +02003718{
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05003719 u16 pci_id1, pci_id2;
Colin Ian Kingf00eb5f2020-04-29 16:48:47 +01003720 int ret;
Borislav Petkov395ae782010-10-01 18:38:19 +02003721
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05003722 if (pvt->fam >= 0x17) {
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00003723 pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
Yazen Ghannam80355a32019-10-22 20:35:10 +00003724 if (!pvt->umc)
3725 return -ENOMEM;
Yazen Ghannam936fc3a2016-11-17 17:57:36 -05003726
3727 pci_id1 = fam_type->f0_id;
3728 pci_id2 = fam_type->f6_id;
3729 } else {
3730 pci_id1 = fam_type->f1_id;
3731 pci_id2 = fam_type->f2_id;
3732 }
3733
Yazen Ghannam80355a32019-10-22 20:35:10 +00003734 ret = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3735 if (ret)
3736 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003737
Borislav Petkov360b7f32010-10-15 19:25:38 +02003738 read_mc_regs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003739
Yazen Ghannam80355a32019-10-22 20:35:10 +00003740 return 0;
3741}
3742
3743static void hw_info_put(struct amd64_pvt *pvt)
3744{
3745 if (pvt->F0 || pvt->F1)
3746 free_mc_sibling_devs(pvt);
3747
3748 kfree(pvt->umc);
3749}
3750
3751static int init_one_instance(struct amd64_pvt *pvt)
3752{
3753 struct mem_ctl_info *mci = NULL;
3754 struct edac_mc_layer layers[2];
3755 int ret = -EINVAL;
3756
Doug Thompson7d6034d2009-04-27 20:01:01 +02003757 /*
3758 * We need to determine how many memory channels there are. Then use
3759 * that information for calculating the size of the dynamic instance
Borislav Petkov360b7f32010-10-15 19:25:38 +02003760 * tables in the 'mci' structure.
Doug Thompson7d6034d2009-04-27 20:01:01 +02003761 */
3762 pvt->channel_count = pvt->ops->early_channel_count(pvt);
3763 if (pvt->channel_count < 0)
Yazen Ghannam80355a32019-10-22 20:35:10 +00003764 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003765
3766 ret = -ENOMEM;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03003767 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3768 layers[0].size = pvt->csels[0].b_cnt;
3769 layers[0].is_virt_csrow = true;
3770 layers[1].type = EDAC_MC_LAYER_CHANNEL;
Borislav Petkovf0a56c42013-07-23 20:01:23 +02003771
3772 /*
3773 * Always allocate two channels since we can have setups with DIMMs on
3774 * only one channel. Also, this simplifies handling later for the price
3775 * of a couple of KBs tops.
3776 */
Yazen Ghannam5e4c5522019-10-22 20:35:11 +00003777 layers[1].size = fam_type->max_mcs;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03003778 layers[1].is_virt_csrow = false;
Borislav Petkovf0a56c42013-07-23 20:01:23 +02003779
Yazen Ghannam80355a32019-10-22 20:35:10 +00003780 mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003781 if (!mci)
Yazen Ghannam80355a32019-10-22 20:35:10 +00003782 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003783
3784 mci->pvt_info = pvt;
Borislav Petkov3f37a362016-05-06 19:44:27 +02003785 mci->pdev = &pvt->F3->dev;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003786
Yazen Ghannam38ddd4d2019-10-22 20:35:09 +00003787 setup_mci_misc_attrs(mci);
Borislav Petkov360b7f32010-10-15 19:25:38 +02003788
3789 if (init_csrows(mci))
Doug Thompson7d6034d2009-04-27 20:01:01 +02003790 mci->edac_cap = EDAC_FLAG_NONE;
3791
Doug Thompson7d6034d2009-04-27 20:01:01 +02003792 ret = -ENODEV;
Takashi Iwaie339f1e2015-02-04 11:48:53 +01003793 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
Joe Perches956b9ba12012-04-29 17:08:39 -03003794 edac_dbg(1, "failed edac_mc_add_mc()\n");
Yazen Ghannam80355a32019-10-22 20:35:10 +00003795 edac_mc_free(mci);
3796 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003797 }
3798
Doug Thompson7d6034d2009-04-27 20:01:01 +02003799 return 0;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003800}
3801
Yazen Ghannam582f94b2019-11-06 01:25:01 +00003802static bool instance_has_memory(struct amd64_pvt *pvt)
3803{
3804 bool cs_enabled = false;
3805 int cs = 0, dct = 0;
3806
3807 for (dct = 0; dct < fam_type->max_mcs; dct++) {
3808 for_each_chip_select(cs, dct, pvt)
3809 cs_enabled |= csrow_enabled(cs, dct, pvt);
3810 }
3811
3812 return cs_enabled;
3813}
3814
Borislav Petkov3f37a362016-05-06 19:44:27 +02003815static int probe_one_instance(unsigned int nid)
Doug Thompson7d6034d2009-04-27 20:01:01 +02003816{
Borislav Petkov2299ef72010-10-15 17:44:04 +02003817 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
Yazen Ghannam80355a32019-10-22 20:35:10 +00003818 struct amd64_pvt *pvt = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003819 struct ecc_settings *s;
Borislav Petkov3f37a362016-05-06 19:44:27 +02003820 int ret;
Borislav Petkovb8cfa022010-10-01 19:35:38 +02003821
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003822 ret = -ENOMEM;
3823 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3824 if (!s)
Borislav Petkov2299ef72010-10-15 17:44:04 +02003825 goto err_out;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003826
3827 ecc_stngs[nid] = s;
3828
Yazen Ghannam80355a32019-10-22 20:35:10 +00003829 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3830 if (!pvt)
3831 goto err_settings;
3832
3833 pvt->mc_node_id = nid;
3834 pvt->F3 = F3;
3835
Borislav Petkov6c13d7f2020-12-12 15:20:28 +01003836 ret = -ENODEV;
Yazen Ghannam80355a32019-10-22 20:35:10 +00003837 fam_type = per_family_init(pvt);
3838 if (!fam_type)
3839 goto err_enable;
3840
3841 ret = hw_info_get(pvt);
3842 if (ret < 0)
3843 goto err_enable;
3844
Yazen Ghannam582f94b2019-11-06 01:25:01 +00003845 ret = 0;
3846 if (!instance_has_memory(pvt)) {
3847 amd64_info("Node %d: No DIMMs detected.\n", nid);
3848 goto err_enable;
3849 }
3850
Yazen Ghannam1c9b08b2019-10-22 20:35:12 +00003851 if (!ecc_enabled(pvt)) {
Yazen Ghannam582f94b2019-11-06 01:25:01 +00003852 ret = -ENODEV;
Borislav Petkov2299ef72010-10-15 17:44:04 +02003853
3854 if (!ecc_enable_override)
3855 goto err_enable;
3856
Yazen Ghannam044e7a42016-11-22 15:40:16 -06003857 if (boot_cpu_data.x86 >= 0x17) {
3858 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3859 goto err_enable;
3860 } else
3861 amd64_warn("Forcing ECC on!\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02003862
3863 if (!enable_ecc_error_reporting(s, nid, F3))
3864 goto err_enable;
3865 }
3866
Yazen Ghannam80355a32019-10-22 20:35:10 +00003867 ret = init_one_instance(pvt);
Borislav Petkov360b7f32010-10-15 19:25:38 +02003868 if (ret < 0) {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003869 amd64_err("Error probing instance: %d\n", nid);
Yazen Ghannam044e7a42016-11-22 15:40:16 -06003870
3871 if (boot_cpu_data.x86 < 0x17)
3872 restore_ecc_error_reporting(s, nid, F3);
Yazen Ghannam2b9b2c42017-01-24 16:32:24 -06003873
3874 goto err_enable;
Borislav Petkov360b7f32010-10-15 19:25:38 +02003875 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02003876
Borislav Petkov4cbcb732021-01-13 20:13:30 +01003877 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3878 (pvt->fam == 0xf ?
3879 (pvt->ext_model >= K8_REV_F ? "revF or later "
3880 : "revE or earlier ")
3881 : ""), pvt->mc_node_id);
3882
Yazen Ghannam582f94b2019-11-06 01:25:01 +00003883 dump_misc_regs(pvt);
3884
Doug Thompson7d6034d2009-04-27 20:01:01 +02003885 return ret;
Borislav Petkov2299ef72010-10-15 17:44:04 +02003886
3887err_enable:
Yazen Ghannam80355a32019-10-22 20:35:10 +00003888 hw_info_put(pvt);
3889 kfree(pvt);
3890
3891err_settings:
Borislav Petkov2299ef72010-10-15 17:44:04 +02003892 kfree(s);
3893 ecc_stngs[nid] = NULL;
3894
3895err_out:
3896 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003897}
3898
Borislav Petkov3f37a362016-05-06 19:44:27 +02003899static void remove_one_instance(unsigned int nid)
Doug Thompson7d6034d2009-04-27 20:01:01 +02003900{
Borislav Petkov360b7f32010-10-15 19:25:38 +02003901 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3902 struct ecc_settings *s = ecc_stngs[nid];
Borislav Petkov3f37a362016-05-06 19:44:27 +02003903 struct mem_ctl_info *mci;
3904 struct amd64_pvt *pvt;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003905
3906 /* Remove from EDAC CORE tracking list */
Borislav Petkov3f37a362016-05-06 19:44:27 +02003907 mci = edac_mc_del_mc(&F3->dev);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003908 if (!mci)
3909 return;
3910
3911 pvt = mci->pvt_info;
3912
Borislav Petkov360b7f32010-10-15 19:25:38 +02003913 restore_ecc_error_reporting(s, nid, F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003914
Borislav Petkov360b7f32010-10-15 19:25:38 +02003915 kfree(ecc_stngs[nid]);
3916 ecc_stngs[nid] = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003917
Doug Thompson7d6034d2009-04-27 20:01:01 +02003918 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01003919 mci->pvt_info = NULL;
Borislav Petkov8f68ed92009-12-21 15:15:59 +01003920
Yazen Ghannam80355a32019-10-22 20:35:10 +00003921 hw_info_put(pvt);
Borislav Petkov8f68ed92009-12-21 15:15:59 +01003922 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003923 edac_mc_free(mci);
3924}
3925
Borislav Petkov360b7f32010-10-15 19:25:38 +02003926static void setup_pci_device(void)
Doug Thompson7d6034d2009-04-27 20:01:01 +02003927{
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003928 if (pci_ctl)
Doug Thompson7d6034d2009-04-27 20:01:01 +02003929 return;
3930
Borislav Petkov706657b12020-11-22 15:57:21 +01003931 pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR);
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003932 if (!pci_ctl) {
3933 pr_warn("%s(): Unable to create PCI control\n", __func__);
3934 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003935 }
3936}
3937
Yazen Ghannamd6efab72016-09-15 19:07:17 -05003938static const struct x86_cpu_id amd64_cpuids[] = {
Thomas Gleixner29842622020-03-20 14:13:55 +01003939 X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL),
3940 X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL),
3941 X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL),
3942 X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL),
3943 X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL),
3944 X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL),
3945 X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL),
Yazen Ghannamd6efab72016-09-15 19:07:17 -05003946 { }
3947};
3948MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3949
Doug Thompson7d6034d2009-04-27 20:01:01 +02003950static int __init amd64_edac_init(void)
3951{
Toshi Kani301375e2017-08-23 16:54:47 -06003952 const char *owner;
Borislav Petkov360b7f32010-10-15 19:25:38 +02003953 int err = -ENODEV;
Borislav Petkov3f37a362016-05-06 19:44:27 +02003954 int i;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003955
Toshi Kani301375e2017-08-23 16:54:47 -06003956 owner = edac_get_owner();
3957 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
3958 return -EBUSY;
3959
Yazen Ghannam1bd99002017-01-27 11:24:23 -06003960 if (!x86_match_cpu(amd64_cpuids))
3961 return -ENODEV;
3962
Hans Rosenfeld9653a5c2010-10-29 17:14:31 +02003963 if (amd_cache_northbridges() < 0)
Yazen Ghannam1bd99002017-01-27 11:24:23 -06003964 return -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003965
Borislav Petkov6ba92fe2016-06-16 01:13:18 +02003966 opstate_init();
3967
Borislav Petkovcc4d8862010-10-13 16:11:59 +02003968 err = -ENOMEM;
Kees Cook6396bb22018-06-12 14:03:40 -07003969 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
Borislav Petkov2ec591a2015-02-17 10:58:34 +01003970 if (!ecc_stngs)
Borislav Petkova9f0fbe2011-03-29 18:10:53 +02003971 goto err_free;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02003972
Borislav Petkov50542252009-12-11 18:14:40 +01003973 msrs = msrs_alloc();
Borislav Petkov56b34b92009-12-21 18:13:01 +01003974 if (!msrs)
Borislav Petkov360b7f32010-10-15 19:25:38 +02003975 goto err_free;
Borislav Petkov50542252009-12-11 18:14:40 +01003976
Yazen Ghannam2287c632017-01-13 09:52:19 -06003977 for (i = 0; i < amd_nb_num(); i++) {
3978 err = probe_one_instance(i);
3979 if (err) {
Borislav Petkov3f37a362016-05-06 19:44:27 +02003980 /* unwind properly */
3981 while (--i >= 0)
3982 remove_one_instance(i);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003983
Borislav Petkov3f37a362016-05-06 19:44:27 +02003984 goto err_pci;
3985 }
Yazen Ghannam2287c632017-01-13 09:52:19 -06003986 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02003987
Yazen Ghannam4688c9b2017-01-27 11:24:22 -06003988 if (!edac_has_mcs()) {
3989 err = -ENODEV;
3990 goto err_pci;
3991 }
3992
Yazen Ghannam234365f2017-01-24 16:32:25 -06003993 /* register stuff with EDAC MCE */
Yazen Ghannam234365f2017-01-24 16:32:25 -06003994 if (boot_cpu_data.x86 >= 0x17)
3995 amd_register_ecc_decoder(decode_umc_error);
3996 else
3997 amd_register_ecc_decoder(decode_bus_error);
3998
Borislav Petkov360b7f32010-10-15 19:25:38 +02003999 setup_pci_device();
Tomasz Palaf5b10c42014-11-02 11:22:12 +01004000
4001#ifdef CONFIG_X86_32
4002 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
4003#endif
4004
Borislav Petkovde0336b2016-04-27 12:21:21 +02004005 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
4006
Borislav Petkov360b7f32010-10-15 19:25:38 +02004007 return 0;
Borislav Petkov56b34b92009-12-21 18:13:01 +01004008
Borislav Petkov56b34b92009-12-21 18:13:01 +01004009err_pci:
Borislav Petkov706657b12020-11-22 15:57:21 +01004010 pci_ctl_dev = NULL;
4011
Borislav Petkov56b34b92009-12-21 18:13:01 +01004012 msrs_free(msrs);
4013 msrs = NULL;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02004014
Borislav Petkov360b7f32010-10-15 19:25:38 +02004015err_free:
Borislav Petkov360b7f32010-10-15 19:25:38 +02004016 kfree(ecc_stngs);
4017 ecc_stngs = NULL;
4018
Doug Thompson7d6034d2009-04-27 20:01:01 +02004019 return err;
4020}
4021
4022static void __exit amd64_edac_exit(void)
4023{
Borislav Petkov3f37a362016-05-06 19:44:27 +02004024 int i;
4025
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01004026 if (pci_ctl)
4027 edac_pci_release_generic_ctl(pci_ctl);
Doug Thompson7d6034d2009-04-27 20:01:01 +02004028
Yazen Ghannam234365f2017-01-24 16:32:25 -06004029 /* unregister from EDAC MCE */
Yazen Ghannam234365f2017-01-24 16:32:25 -06004030 if (boot_cpu_data.x86 >= 0x17)
4031 amd_unregister_ecc_decoder(decode_umc_error);
4032 else
4033 amd_unregister_ecc_decoder(decode_bus_error);
4034
Borislav Petkov3f37a362016-05-06 19:44:27 +02004035 for (i = 0; i < amd_nb_num(); i++)
4036 remove_one_instance(i);
Borislav Petkov50542252009-12-11 18:14:40 +01004037
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02004038 kfree(ecc_stngs);
4039 ecc_stngs = NULL;
4040
Borislav Petkov706657b12020-11-22 15:57:21 +01004041 pci_ctl_dev = NULL;
4042
Borislav Petkov50542252009-12-11 18:14:40 +01004043 msrs_free(msrs);
4044 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02004045}
4046
4047module_init(amd64_edac_init);
4048module_exit(amd64_edac_exit);
4049
4050MODULE_LICENSE("GPL");
4051MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
4052 "Dave Peterson, Thayne Harbaugh");
4053MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
4054 EDAC_AMD64_VERSION);
4055
4056module_param(edac_op_state, int, 0444);
4057MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");