blob: 3b181b22cd469b1990305a95c8164340f5441c7c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * pSeries NUMA support
3 *
4 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11#include <linux/threads.h>
12#include <linux/bootmem.h>
13#include <linux/init.h>
14#include <linux/mm.h>
15#include <linux/mmzone.h>
Paul Gortmaker4b16f8e2011-07-22 18:24:23 -040016#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/nodemask.h>
18#include <linux/cpu.h>
19#include <linux/notifier.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100020#include <linux/memblock.h>
Michael Ellerman6df16462008-02-14 11:37:49 +110021#include <linux/of.h>
Dave Hansen06eccea2009-02-12 12:36:04 +000022#include <linux/pfn.h>
Jesse Larrew9eff1a32010-12-01 12:31:15 +000023#include <linux/cpuset.h>
24#include <linux/node.h>
Nathan Fontenot30c05352013-04-24 06:02:13 +000025#include <linux/stop_machine.h>
Nathan Fontenote04fa612013-04-24 06:07:39 +000026#include <linux/proc_fs.h>
27#include <linux/seq_file.h>
28#include <linux/uaccess.h>
Linus Torvalds191a7122013-04-29 19:14:20 -070029#include <linux/slab.h>
Robert Jennings3be7db62013-07-24 20:13:21 -050030#include <asm/cputhreads.h>
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110031#include <asm/sparsemem.h>
David S. Millerd9b2b2a2008-02-13 16:56:49 -080032#include <asm/prom.h>
Paul Mackerras2249ca92005-11-07 13:18:13 +110033#include <asm/smp.h>
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +053034#include <asm/cputhreads.h>
35#include <asm/topology.h>
Jesse Larrew9eff1a32010-12-01 12:31:15 +000036#include <asm/firmware.h>
37#include <asm/paca.h>
Jesse Larrew39bf9902010-12-17 22:07:47 +000038#include <asm/hvcall.h>
David Howellsae3a197e2012-03-28 18:30:02 +010039#include <asm/setup.h>
Jesse Larrew176bbf12013-04-24 06:03:48 +000040#include <asm/vdso.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
42static int numa_enabled = 1;
43
Balbir Singh1daa6d02008-02-01 15:57:31 +110044static char *cmdline __initdata;
45
Linus Torvalds1da177e2005-04-16 15:20:36 -070046static int numa_debug;
47#define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
48
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110049int numa_cpu_lookup_table[NR_CPUS];
Anton Blanchard25863de2010-04-26 15:32:43 +000050cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
Linus Torvalds1da177e2005-04-16 15:20:36 -070051struct pglist_data *node_data[MAX_NUMNODES];
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110052
53EXPORT_SYMBOL(numa_cpu_lookup_table);
Anton Blanchard25863de2010-04-26 15:32:43 +000054EXPORT_SYMBOL(node_to_cpumask_map);
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110055EXPORT_SYMBOL(node_data);
56
Linus Torvalds1da177e2005-04-16 15:20:36 -070057static int min_common_depth;
Mike Kravetz237a09892005-12-05 12:06:42 -080058static int n_mem_addr_cells, n_mem_size_cells;
Anton Blanchard41eab6f2010-05-16 20:22:31 +000059static int form1_affinity;
60
61#define MAX_DISTANCE_REF_POINTS 4
62static int distance_ref_points_depth;
Alistair Poppleb08a2a12013-08-07 02:01:44 +100063static const __be32 *distance_ref_points;
Anton Blanchard41eab6f2010-05-16 20:22:31 +000064static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS];
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
Anton Blanchard25863de2010-04-26 15:32:43 +000066/*
67 * Allocate node_to_cpumask_map based on number of available nodes
68 * Requires node_possible_map to be valid.
69 *
Wanlong Gao95129382012-01-12 17:20:09 -080070 * Note: cpumask_of_node() is not valid until after this is done.
Anton Blanchard25863de2010-04-26 15:32:43 +000071 */
72static void __init setup_node_to_cpumask_map(void)
73{
Cody P Schaferf9d531b2013-04-29 15:08:03 -070074 unsigned int node;
Anton Blanchard25863de2010-04-26 15:32:43 +000075
76 /* setup nr_node_ids if not done yet */
Cody P Schaferf9d531b2013-04-29 15:08:03 -070077 if (nr_node_ids == MAX_NUMNODES)
78 setup_nr_node_ids();
Anton Blanchard25863de2010-04-26 15:32:43 +000079
80 /* allocate the map */
81 for (node = 0; node < nr_node_ids; node++)
82 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
83
84 /* cpumask_of_node() will now work */
85 dbg("Node to cpumask map for %d nodes\n", nr_node_ids);
86}
87
Stephen Rothwell55671f32013-03-25 18:44:44 +000088static int __init fake_numa_create_new_node(unsigned long end_pfn,
Balbir Singh1daa6d02008-02-01 15:57:31 +110089 unsigned int *nid)
90{
91 unsigned long long mem;
92 char *p = cmdline;
93 static unsigned int fake_nid;
94 static unsigned long long curr_boundary;
95
96 /*
97 * Modify node id, iff we started creating NUMA nodes
98 * We want to continue from where we left of the last time
99 */
100 if (fake_nid)
101 *nid = fake_nid;
102 /*
103 * In case there are no more arguments to parse, the
104 * node_id should be the same as the last fake node id
105 * (we've handled this above).
106 */
107 if (!p)
108 return 0;
109
110 mem = memparse(p, &p);
111 if (!mem)
112 return 0;
113
114 if (mem < curr_boundary)
115 return 0;
116
117 curr_boundary = mem;
118
119 if ((end_pfn << PAGE_SHIFT) > mem) {
120 /*
121 * Skip commas and spaces
122 */
123 while (*p == ',' || *p == ' ' || *p == '\t')
124 p++;
125
126 cmdline = p;
127 fake_nid++;
128 *nid = fake_nid;
129 dbg("created new fake_node with id %d\n", fake_nid);
130 return 1;
131 }
132 return 0;
133}
134
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000135/*
Tejun Heo5dfe8662011-07-14 09:46:10 +0200136 * get_node_active_region - Return active region containing pfn
Jon Tollefsone8170372008-10-16 18:59:43 +0000137 * Active range returned is empty if none found.
Tejun Heo5dfe8662011-07-14 09:46:10 +0200138 * @pfn: The page to return the region for
139 * @node_ar: Returned set to the active region containing @pfn
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000140 */
Tejun Heo5dfe8662011-07-14 09:46:10 +0200141static void __init get_node_active_region(unsigned long pfn,
142 struct node_active_region *node_ar)
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000143{
Tejun Heo5dfe8662011-07-14 09:46:10 +0200144 unsigned long start_pfn, end_pfn;
145 int i, nid;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000146
Tejun Heo5dfe8662011-07-14 09:46:10 +0200147 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
148 if (pfn >= start_pfn && pfn < end_pfn) {
149 node_ar->nid = nid;
150 node_ar->start_pfn = start_pfn;
151 node_ar->end_pfn = end_pfn;
152 break;
153 }
154 }
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000155}
156
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530157static void reset_numa_cpu_lookup_table(void)
158{
159 unsigned int cpu;
160
161 for_each_possible_cpu(cpu)
162 numa_cpu_lookup_table[cpu] = -1;
163}
164
165static void update_numa_cpu_lookup_table(unsigned int cpu, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166{
167 numa_cpu_lookup_table[cpu] = node;
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530168}
169
170static void map_cpu_to_node(int cpu, int node)
171{
172 update_numa_cpu_lookup_table(cpu, node);
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100173
Nathan Lynchbf4b85b2006-03-20 18:34:45 -0600174 dbg("adding cpu %d to node %d\n", cpu, node);
175
Anton Blanchard25863de2010-04-26 15:32:43 +0000176 if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node])))
177 cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178}
179
Jesse Larrew39bf9902010-12-17 22:07:47 +0000180#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181static void unmap_cpu_from_node(unsigned long cpu)
182{
183 int node = numa_cpu_lookup_table[cpu];
184
185 dbg("removing cpu %lu from node %d\n", cpu, node);
186
Anton Blanchard25863de2010-04-26 15:32:43 +0000187 if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) {
Anton Blanchard429f4d82011-01-29 12:37:16 +0000188 cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 } else {
190 printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n",
191 cpu, node);
192 }
193}
Jesse Larrew39bf9902010-12-17 22:07:47 +0000194#endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196/* must hold reference to node during call */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000197static const __be32 *of_get_associativity(struct device_node *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198{
Stephen Rothwelle2eb6392007-04-03 22:26:41 +1000199 return of_get_property(dev, "ibm,associativity", NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200}
201
Chandrucf000852008-08-30 00:28:16 +1000202/*
203 * Returns the property linux,drconf-usable-memory if
204 * it exists (the property exists only in kexec/kdump kernels,
205 * added by kexec-tools)
206 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000207static const __be32 *of_get_usable_memory(struct device_node *memory)
Chandrucf000852008-08-30 00:28:16 +1000208{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000209 const __be32 *prop;
Chandrucf000852008-08-30 00:28:16 +1000210 u32 len;
211 prop = of_get_property(memory, "linux,drconf-usable-memory", &len);
212 if (!prop || len < sizeof(unsigned int))
Robert Jenningsec32dd62013-10-28 09:20:50 -0500213 return NULL;
Chandrucf000852008-08-30 00:28:16 +1000214 return prop;
215}
216
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000217int __node_distance(int a, int b)
218{
219 int i;
220 int distance = LOCAL_DISTANCE;
221
222 if (!form1_affinity)
Vaidyanathan Srinivasan7122bee2013-03-22 05:49:35 +0000223 return ((a == b) ? LOCAL_DISTANCE : REMOTE_DISTANCE);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000224
225 for (i = 0; i < distance_ref_points_depth; i++) {
226 if (distance_lookup_table[a][i] == distance_lookup_table[b][i])
227 break;
228
229 /* Double the distance for each NUMA level */
230 distance *= 2;
231 }
232
233 return distance;
234}
Mike Qiu12c743e2014-04-18 15:07:14 -0700235EXPORT_SYMBOL(__node_distance);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000236
237static void initialize_distance_lookup_table(int nid,
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000238 const __be32 *associativity)
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000239{
240 int i;
241
242 if (!form1_affinity)
243 return;
244
245 for (i = 0; i < distance_ref_points_depth; i++) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000246 const __be32 *entry;
247
248 entry = &associativity[be32_to_cpu(distance_ref_points[i])];
249 distance_lookup_table[nid][i] = of_read_number(entry, 1);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000250 }
251}
252
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600253/* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
254 * info is found.
255 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000256static int associativity_to_nid(const __be32 *associativity)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257{
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600258 int nid = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
260 if (min_common_depth == -1)
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600261 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000263 if (of_read_number(associativity, 1) >= min_common_depth)
264 nid = of_read_number(&associativity[min_common_depth], 1);
Nathan Lynchbc16a752006-03-20 18:36:15 -0600265
266 /* POWER4 LPAR uses 0xffff as invalid node */
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600267 if (nid == 0xffff || nid >= MAX_NUMNODES)
268 nid = -1;
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000269
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000270 if (nid > 0 &&
271 of_read_number(associativity, 1) >= distance_ref_points_depth)
Jesse Larrew9eff1a32010-12-01 12:31:15 +0000272 initialize_distance_lookup_table(nid, associativity);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000273
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600274out:
Nathan Lynchcf950b72006-03-20 18:35:45 -0600275 return nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276}
277
Jesse Larrew9eff1a32010-12-01 12:31:15 +0000278/* Returns the nid associated with the given device tree node,
279 * or -1 if not found.
280 */
281static int of_node_to_nid_single(struct device_node *device)
282{
283 int nid = -1;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000284 const __be32 *tmp;
Jesse Larrew9eff1a32010-12-01 12:31:15 +0000285
286 tmp = of_get_associativity(device);
287 if (tmp)
288 nid = associativity_to_nid(tmp);
289 return nid;
290}
291
Jeremy Kerr953039c2006-05-01 12:16:12 -0700292/* Walk the device tree upwards, looking for an associativity id */
293int of_node_to_nid(struct device_node *device)
294{
295 struct device_node *tmp;
296 int nid = -1;
297
298 of_node_get(device);
299 while (device) {
300 nid = of_node_to_nid_single(device);
301 if (nid != -1)
302 break;
303
304 tmp = device;
305 device = of_get_parent(tmp);
306 of_node_put(tmp);
307 }
308 of_node_put(device);
309
310 return nid;
311}
312EXPORT_SYMBOL_GPL(of_node_to_nid);
313
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314static int __init find_min_common_depth(void)
315{
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000316 int depth;
Michael Ellermane70606e2011-04-10 20:42:05 +0000317 struct device_node *root;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318
Dipankar Sarma1c8ee732011-10-28 04:25:32 +0000319 if (firmware_has_feature(FW_FEATURE_OPAL))
320 root = of_find_node_by_path("/ibm,opal");
321 else
322 root = of_find_node_by_path("/rtas");
Michael Ellermane70606e2011-04-10 20:42:05 +0000323 if (!root)
324 root = of_find_node_by_path("/");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325
326 /*
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000327 * This property is a set of 32-bit integers, each representing
328 * an index into the ibm,associativity nodes.
329 *
330 * With form 0 affinity the first integer is for an SMP configuration
331 * (should be all 0's) and the second is for a normal NUMA
332 * configuration. We have only one level of NUMA.
333 *
334 * With form 1 affinity the first integer is the most significant
335 * NUMA boundary and the following are progressively less significant
336 * boundaries. There can be more than one level of NUMA.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 */
Michael Ellermane70606e2011-04-10 20:42:05 +0000338 distance_ref_points = of_get_property(root,
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000339 "ibm,associativity-reference-points",
340 &distance_ref_points_depth);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000342 if (!distance_ref_points) {
343 dbg("NUMA: ibm,associativity-reference-points not found.\n");
344 goto err;
345 }
346
347 distance_ref_points_depth /= sizeof(int);
348
Nathan Fontenot8002b0c2013-04-24 05:58:23 +0000349 if (firmware_has_feature(FW_FEATURE_OPAL) ||
350 firmware_has_feature(FW_FEATURE_TYPE1_AFFINITY)) {
351 dbg("Using form 1 affinity\n");
Dipankar Sarma1c8ee732011-10-28 04:25:32 +0000352 form1_affinity = 1;
Anton Blanchard4b83c332010-04-07 15:33:44 +0000353 }
354
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000355 if (form1_affinity) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000356 depth = of_read_number(distance_ref_points, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 } else {
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000358 if (distance_ref_points_depth < 2) {
359 printk(KERN_WARNING "NUMA: "
360 "short ibm,associativity-reference-points\n");
361 goto err;
362 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000364 depth = of_read_number(&distance_ref_points[1], 1);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000365 }
366
367 /*
368 * Warn and cap if the hardware supports more than
369 * MAX_DISTANCE_REF_POINTS domains.
370 */
371 if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) {
372 printk(KERN_WARNING "NUMA: distance array capped at "
373 "%d entries\n", MAX_DISTANCE_REF_POINTS);
374 distance_ref_points_depth = MAX_DISTANCE_REF_POINTS;
375 }
376
Michael Ellermane70606e2011-04-10 20:42:05 +0000377 of_node_put(root);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 return depth;
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000379
380err:
Michael Ellermane70606e2011-04-10 20:42:05 +0000381 of_node_put(root);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000382 return -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
Mike Kravetz84c9fdd2005-11-30 13:47:23 -0800385static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386{
387 struct device_node *memory = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388
389 memory = of_find_node_by_type(memory, "memory");
Paul Mackerras54c23312005-12-05 15:50:39 +1100390 if (!memory)
Mike Kravetz84c9fdd2005-11-30 13:47:23 -0800391 panic("numa.c: No memory nodes found!");
Paul Mackerras54c23312005-12-05 15:50:39 +1100392
Stephen Rothwella8bda5d2007-04-03 10:56:50 +1000393 *n_addr_cells = of_n_addr_cells(memory);
Stephen Rothwell9213fee2007-04-03 10:57:48 +1000394 *n_size_cells = of_n_size_cells(memory);
Mike Kravetz84c9fdd2005-11-30 13:47:23 -0800395 of_node_put(memory);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396}
397
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000398static unsigned long read_n_cells(int n, const __be32 **buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399{
400 unsigned long result = 0;
401
402 while (n--) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000403 result = (result << 32) | of_read_number(*buf, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 (*buf)++;
405 }
406 return result;
407}
408
Nathan Fontenot83426812008-07-03 13:35:54 +1000409/*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000410 * Read the next memblock list entry from the ibm,dynamic-memory property
Nathan Fontenot83426812008-07-03 13:35:54 +1000411 * and return the information in the provided of_drconf_cell structure.
412 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000413static void read_drconf_cell(struct of_drconf_cell *drmem, const __be32 **cellp)
Nathan Fontenot83426812008-07-03 13:35:54 +1000414{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000415 const __be32 *cp;
Nathan Fontenot83426812008-07-03 13:35:54 +1000416
417 drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp);
418
419 cp = *cellp;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000420 drmem->drc_index = of_read_number(cp, 1);
421 drmem->reserved = of_read_number(&cp[1], 1);
422 drmem->aa_index = of_read_number(&cp[2], 1);
423 drmem->flags = of_read_number(&cp[3], 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000424
425 *cellp = cp + 4;
426}
427
428/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300429 * Retrieve and validate the ibm,dynamic-memory property of the device tree.
Nathan Fontenot83426812008-07-03 13:35:54 +1000430 *
Yinghai Lu95f72d12010-07-12 14:36:09 +1000431 * The layout of the ibm,dynamic-memory property is a number N of memblock
432 * list entries followed by N memblock list entries. Each memblock list entry
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300433 * contains information as laid out in the of_drconf_cell struct above.
Nathan Fontenot83426812008-07-03 13:35:54 +1000434 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000435static int of_get_drconf_memory(struct device_node *memory, const __be32 **dm)
Nathan Fontenot83426812008-07-03 13:35:54 +1000436{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000437 const __be32 *prop;
Nathan Fontenot83426812008-07-03 13:35:54 +1000438 u32 len, entries;
439
440 prop = of_get_property(memory, "ibm,dynamic-memory", &len);
441 if (!prop || len < sizeof(unsigned int))
442 return 0;
443
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000444 entries = of_read_number(prop++, 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000445
446 /* Now that we know the number of entries, revalidate the size
447 * of the property read in to ensure we have everything
448 */
449 if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int))
450 return 0;
451
452 *dm = prop;
453 return entries;
454}
455
456/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300457 * Retrieve and validate the ibm,lmb-size property for drconf memory
Nathan Fontenot83426812008-07-03 13:35:54 +1000458 * from the device tree.
459 */
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000460static u64 of_get_lmb_size(struct device_node *memory)
Nathan Fontenot83426812008-07-03 13:35:54 +1000461{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000462 const __be32 *prop;
Nathan Fontenot83426812008-07-03 13:35:54 +1000463 u32 len;
464
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000465 prop = of_get_property(memory, "ibm,lmb-size", &len);
Nathan Fontenot83426812008-07-03 13:35:54 +1000466 if (!prop || len < sizeof(unsigned int))
467 return 0;
468
469 return read_n_cells(n_mem_size_cells, &prop);
470}
471
472struct assoc_arrays {
473 u32 n_arrays;
474 u32 array_sz;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000475 const __be32 *arrays;
Nathan Fontenot83426812008-07-03 13:35:54 +1000476};
477
478/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300479 * Retrieve and validate the list of associativity arrays for drconf
Nathan Fontenot83426812008-07-03 13:35:54 +1000480 * memory from the ibm,associativity-lookup-arrays property of the
481 * device tree..
482 *
483 * The layout of the ibm,associativity-lookup-arrays property is a number N
484 * indicating the number of associativity arrays, followed by a number M
485 * indicating the size of each associativity array, followed by a list
486 * of N associativity arrays.
487 */
488static int of_get_assoc_arrays(struct device_node *memory,
489 struct assoc_arrays *aa)
490{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000491 const __be32 *prop;
Nathan Fontenot83426812008-07-03 13:35:54 +1000492 u32 len;
493
494 prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len);
495 if (!prop || len < 2 * sizeof(unsigned int))
496 return -1;
497
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000498 aa->n_arrays = of_read_number(prop++, 1);
499 aa->array_sz = of_read_number(prop++, 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000500
Justin P. Mattock42b2aa82011-11-28 20:31:00 -0800501 /* Now that we know the number of arrays and size of each array,
Nathan Fontenot83426812008-07-03 13:35:54 +1000502 * revalidate the size of the property read in.
503 */
504 if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int))
505 return -1;
506
507 aa->arrays = prop;
508 return 0;
509}
510
511/*
512 * This is like of_node_to_nid_single() for memory represented in the
513 * ibm,dynamic-reconfiguration-memory node.
514 */
515static int of_drconf_to_nid_single(struct of_drconf_cell *drmem,
516 struct assoc_arrays *aa)
517{
518 int default_nid = 0;
519 int nid = default_nid;
520 int index;
521
522 if (min_common_depth > 0 && min_common_depth <= aa->array_sz &&
523 !(drmem->flags & DRCONF_MEM_AI_INVALID) &&
524 drmem->aa_index < aa->n_arrays) {
525 index = drmem->aa_index * aa->array_sz + min_common_depth - 1;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000526 nid = of_read_number(&aa->arrays[index], 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000527
528 if (nid == 0xffff || nid >= MAX_NUMNODES)
529 nid = default_nid;
530 }
531
532 return nid;
533}
534
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535/*
536 * Figure out to which domain a cpu belongs and stick it there.
537 * Return the id of the domain used.
538 */
Paul Gortmaker061d19f2013-06-24 15:30:09 -0400539static int numa_setup_cpu(unsigned long lcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540{
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530541 int nid;
542 struct device_node *cpu;
543
544 /*
545 * If a valid cpu-to-node mapping is already available, use it
546 * directly instead of querying the firmware, since it represents
547 * the most recent mapping notified to us by the platform (eg: VPHN).
548 */
549 if ((nid = numa_cpu_lookup_table[lcpu]) >= 0) {
550 map_cpu_to_node(lcpu, nid);
551 return nid;
552 }
553
554 cpu = of_get_cpu_node(lcpu, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555
556 if (!cpu) {
557 WARN_ON(1);
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530558 nid = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 goto out;
560 }
561
Jeremy Kerr953039c2006-05-01 12:16:12 -0700562 nid = of_node_to_nid_single(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600564 if (nid < 0 || !node_online(nid))
H Hartley Sweeten72c33682010-03-05 13:42:43 -0800565 nid = first_online_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566out:
Nathan Lynchcf950b72006-03-20 18:35:45 -0600567 map_cpu_to_node(lcpu, nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568
569 of_node_put(cpu);
570
Nathan Lynchcf950b72006-03-20 18:35:45 -0600571 return nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572}
573
Srivatsa S. Bhat68fb18aa2013-12-30 17:06:04 +0530574static void verify_cpu_node_mapping(int cpu, int node)
575{
576 int base, sibling, i;
577
578 /* Verify that all the threads in the core belong to the same node */
579 base = cpu_first_thread_sibling(cpu);
580
581 for (i = 0; i < threads_per_core; i++) {
582 sibling = base + i;
583
584 if (sibling == cpu || cpu_is_offline(sibling))
585 continue;
586
587 if (cpu_to_node(sibling) != node) {
588 WARN(1, "CPU thread siblings %d and %d don't belong"
589 " to the same node!\n", cpu, sibling);
590 break;
591 }
592 }
593}
594
Paul Gortmaker061d19f2013-06-24 15:30:09 -0400595static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 void *hcpu)
597{
598 unsigned long lcpu = (unsigned long)hcpu;
Srivatsa S. Bhat68fb18aa2013-12-30 17:06:04 +0530599 int ret = NOTIFY_DONE, nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
601 switch (action) {
602 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700603 case CPU_UP_PREPARE_FROZEN:
Srivatsa S. Bhat68fb18aa2013-12-30 17:06:04 +0530604 nid = numa_setup_cpu(lcpu);
605 verify_cpu_node_mapping((int)lcpu, nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 ret = NOTIFY_OK;
607 break;
608#ifdef CONFIG_HOTPLUG_CPU
609 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700610 case CPU_DEAD_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700612 case CPU_UP_CANCELED_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 unmap_cpu_from_node(lcpu);
614 break;
615 ret = NOTIFY_OK;
616#endif
617 }
618 return ret;
619}
620
621/*
622 * Check and possibly modify a memory region to enforce the memory limit.
623 *
624 * Returns the size the region should have to enforce the memory limit.
625 * This will either be the original value of size, a truncated value,
626 * or zero. If the returned value of size is 0 the region should be
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300627 * discarded as it lies wholly above the memory limit.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 */
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100629static unsigned long __init numa_enforce_memory_limit(unsigned long start,
630 unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631{
632 /*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000633 * We use memblock_end_of_DRAM() in here instead of memory_limit because
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 * we've already adjusted it for the limit and it takes care of
Milton Millerfe552492008-10-20 15:37:04 +0000635 * having memory holes below the limit. Also, in the case of
636 * iommu_is_off, memory_limit is not set but is implicitly enforced.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
Yinghai Lu95f72d12010-07-12 14:36:09 +1000639 if (start + size <= memblock_end_of_DRAM())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 return size;
641
Yinghai Lu95f72d12010-07-12 14:36:09 +1000642 if (start >= memblock_end_of_DRAM())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 return 0;
644
Yinghai Lu95f72d12010-07-12 14:36:09 +1000645 return memblock_end_of_DRAM() - start;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646}
647
Paul Mackerras02045682006-11-29 22:27:42 +1100648/*
Chandrucf000852008-08-30 00:28:16 +1000649 * Reads the counter for a given entry in
650 * linux,drconf-usable-memory property
651 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000652static inline int __init read_usm_ranges(const __be32 **usm)
Chandrucf000852008-08-30 00:28:16 +1000653{
654 /*
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000655 * For each lmb in ibm,dynamic-memory a corresponding
Chandrucf000852008-08-30 00:28:16 +1000656 * entry in linux,drconf-usable-memory property contains
657 * a counter followed by that many (base, size) duple.
658 * read the counter from linux,drconf-usable-memory
659 */
660 return read_n_cells(n_mem_size_cells, usm);
661}
662
663/*
Paul Mackerras02045682006-11-29 22:27:42 +1100664 * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
665 * node. This assumes n_mem_{addr,size}_cells have been set.
666 */
667static void __init parse_drconf_memory(struct device_node *memory)
668{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000669 const __be32 *uninitialized_var(dm), *usm;
Chandrucf000852008-08-30 00:28:16 +1000670 unsigned int n, rc, ranges, is_kexec_kdump = 0;
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000671 unsigned long lmb_size, base, size, sz;
Nathan Fontenot83426812008-07-03 13:35:54 +1000672 int nid;
Benjamin Herrenschmidtaa709f32012-07-05 16:30:33 +0000673 struct assoc_arrays aa = { .arrays = NULL };
Paul Mackerras02045682006-11-29 22:27:42 +1100674
Nathan Fontenot83426812008-07-03 13:35:54 +1000675 n = of_get_drconf_memory(memory, &dm);
676 if (!n)
Paul Mackerras02045682006-11-29 22:27:42 +1100677 return;
678
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000679 lmb_size = of_get_lmb_size(memory);
680 if (!lmb_size)
Nathan Fontenot83426812008-07-03 13:35:54 +1000681 return;
682
683 rc = of_get_assoc_arrays(memory, &aa);
684 if (rc)
Paul Mackerras02045682006-11-29 22:27:42 +1100685 return;
686
Chandrucf000852008-08-30 00:28:16 +1000687 /* check if this is a kexec/kdump kernel */
688 usm = of_get_usable_memory(memory);
689 if (usm != NULL)
690 is_kexec_kdump = 1;
691
Paul Mackerras02045682006-11-29 22:27:42 +1100692 for (; n != 0; --n) {
Nathan Fontenot83426812008-07-03 13:35:54 +1000693 struct of_drconf_cell drmem;
Balbir Singh1daa6d02008-02-01 15:57:31 +1100694
Nathan Fontenot83426812008-07-03 13:35:54 +1000695 read_drconf_cell(&drmem, &dm);
696
697 /* skip this block if the reserved bit is set in flags (0x80)
698 or if the block is not assigned to this partition (0x8) */
699 if ((drmem.flags & DRCONF_MEM_RESERVED)
700 || !(drmem.flags & DRCONF_MEM_ASSIGNED))
701 continue;
702
Chandrucf000852008-08-30 00:28:16 +1000703 base = drmem.base_addr;
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000704 size = lmb_size;
Chandrucf000852008-08-30 00:28:16 +1000705 ranges = 1;
Nathan Fontenot83426812008-07-03 13:35:54 +1000706
Chandrucf000852008-08-30 00:28:16 +1000707 if (is_kexec_kdump) {
708 ranges = read_usm_ranges(&usm);
709 if (!ranges) /* there are no (base, size) duple */
710 continue;
711 }
712 do {
713 if (is_kexec_kdump) {
714 base = read_n_cells(n_mem_addr_cells, &usm);
715 size = read_n_cells(n_mem_size_cells, &usm);
716 }
717 nid = of_drconf_to_nid_single(&drmem, &aa);
718 fake_numa_create_new_node(
719 ((base + size) >> PAGE_SHIFT),
Nathan Fontenot83426812008-07-03 13:35:54 +1000720 &nid);
Chandrucf000852008-08-30 00:28:16 +1000721 node_set_online(nid);
722 sz = numa_enforce_memory_limit(base, size);
723 if (sz)
Tang Chene7e8de52014-01-21 15:49:26 -0800724 memblock_set_node(base, sz,
725 &memblock.memory, nid);
Chandrucf000852008-08-30 00:28:16 +1000726 } while (--ranges);
Paul Mackerras02045682006-11-29 22:27:42 +1100727 }
728}
729
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730static int __init parse_numa_properties(void)
731{
Anton Blanchard94db7c52011-08-10 20:44:22 +0000732 struct device_node *memory;
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600733 int default_nid = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 unsigned long i;
735
736 if (numa_enabled == 0) {
737 printk(KERN_WARNING "NUMA disabled by user\n");
738 return -1;
739 }
740
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 min_common_depth = find_min_common_depth();
742
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 if (min_common_depth < 0)
744 return min_common_depth;
745
Nathan Lynchbf4b85b2006-03-20 18:34:45 -0600746 dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);
747
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 /*
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600749 * Even though we connect cpus to numa domains later in SMP
750 * init, we need to know the node ids now. This is because
751 * each node to be onlined must have NODE_DATA etc backing it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 */
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600753 for_each_present_cpu(i) {
Anton Blancharddfbe93a2011-08-10 20:44:23 +0000754 struct device_node *cpu;
Nathan Lynchcf950b72006-03-20 18:35:45 -0600755 int nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756
Milton Miller8b16cd22009-01-08 02:19:45 +0000757 cpu = of_get_cpu_node(i, NULL);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600758 BUG_ON(!cpu);
Jeremy Kerr953039c2006-05-01 12:16:12 -0700759 nid = of_node_to_nid_single(cpu);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600760 of_node_put(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600762 /*
763 * Don't fall back to default_nid yet -- we will plug
764 * cpus into nodes once the memory scan has discovered
765 * the topology.
766 */
767 if (nid < 0)
768 continue;
769 node_set_online(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 }
771
Mike Kravetz237a09892005-12-05 12:06:42 -0800772 get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells);
Anton Blanchard94db7c52011-08-10 20:44:22 +0000773
774 for_each_node_by_type(memory, "memory") {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 unsigned long start;
776 unsigned long size;
Nathan Lynchcf950b72006-03-20 18:35:45 -0600777 int nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 int ranges;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000779 const __be32 *memcell_buf;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 unsigned int len;
781
Stephen Rothwelle2eb6392007-04-03 22:26:41 +1000782 memcell_buf = of_get_property(memory,
Michael Ellermanba759482005-12-04 18:39:55 +1100783 "linux,usable-memory", &len);
784 if (!memcell_buf || len <= 0)
Stephen Rothwelle2eb6392007-04-03 22:26:41 +1000785 memcell_buf = of_get_property(memory, "reg", &len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 if (!memcell_buf || len <= 0)
787 continue;
788
Benjamin Herrenschmidtcc5d0182005-12-13 18:01:21 +1100789 /* ranges in cell */
790 ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791new_range:
792 /* these are order-sensitive, and modify the buffer pointer */
Mike Kravetz237a09892005-12-05 12:06:42 -0800793 start = read_n_cells(n_mem_addr_cells, &memcell_buf);
794 size = read_n_cells(n_mem_size_cells, &memcell_buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600796 /*
797 * Assumption: either all memory nodes or none will
798 * have associativity properties. If none, then
799 * everything goes to default_nid.
800 */
Jeremy Kerr953039c2006-05-01 12:16:12 -0700801 nid = of_node_to_nid_single(memory);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600802 if (nid < 0)
803 nid = default_nid;
Balbir Singh1daa6d02008-02-01 15:57:31 +1100804
805 fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600806 node_set_online(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100808 if (!(size = numa_enforce_memory_limit(start, size))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 if (--ranges)
810 goto new_range;
811 else
812 continue;
813 }
814
Tang Chene7e8de52014-01-21 15:49:26 -0800815 memblock_set_node(start, size, &memblock.memory, nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816
817 if (--ranges)
818 goto new_range;
819 }
820
Paul Mackerras02045682006-11-29 22:27:42 +1100821 /*
Anton Blancharddfbe93a2011-08-10 20:44:23 +0000822 * Now do the same thing for each MEMBLOCK listed in the
823 * ibm,dynamic-memory property in the
824 * ibm,dynamic-reconfiguration-memory node.
Paul Mackerras02045682006-11-29 22:27:42 +1100825 */
826 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
827 if (memory)
828 parse_drconf_memory(memory);
829
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 return 0;
831}
832
833static void __init setup_nonnuma(void)
834{
Yinghai Lu95f72d12010-07-12 14:36:09 +1000835 unsigned long top_of_ram = memblock_end_of_DRAM();
836 unsigned long total_ram = memblock_phys_mem_size();
Mel Gormanc67c3cb2006-09-27 01:49:49 -0700837 unsigned long start_pfn, end_pfn;
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000838 unsigned int nid = 0;
839 struct memblock_region *reg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840
Olof Johanssone110b282006-04-12 15:25:01 -0500841 printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842 top_of_ram, total_ram);
Olof Johanssone110b282006-04-12 15:25:01 -0500843 printk(KERN_DEBUG "Memory hole size: %ldMB\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844 (top_of_ram - total_ram) >> 20);
845
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000846 for_each_memblock(memory, reg) {
Yinghai Luc7fc2de2010-10-12 14:07:09 -0700847 start_pfn = memblock_region_memory_base_pfn(reg);
848 end_pfn = memblock_region_memory_end_pfn(reg);
Balbir Singh1daa6d02008-02-01 15:57:31 +1100849
850 fake_numa_create_new_node(end_pfn, &nid);
Tejun Heo1d7cfe12011-12-08 10:22:08 -0800851 memblock_set_node(PFN_PHYS(start_pfn),
Tang Chene7e8de52014-01-21 15:49:26 -0800852 PFN_PHYS(end_pfn - start_pfn),
853 &memblock.memory, nid);
Balbir Singh1daa6d02008-02-01 15:57:31 +1100854 node_set_online(nid);
Mel Gormanc67c3cb2006-09-27 01:49:49 -0700855 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856}
857
Anton Blanchard4b703a22005-12-13 06:56:47 +1100858void __init dump_numa_cpu_topology(void)
859{
860 unsigned int node;
861 unsigned int cpu, count;
862
863 if (min_common_depth == -1 || !numa_enabled)
864 return;
865
866 for_each_online_node(node) {
Olof Johanssone110b282006-04-12 15:25:01 -0500867 printk(KERN_DEBUG "Node %d CPUs:", node);
Anton Blanchard4b703a22005-12-13 06:56:47 +1100868
869 count = 0;
870 /*
871 * If we used a CPU iterator here we would miss printing
872 * the holes in the cpumap.
873 */
Anton Blanchard25863de2010-04-26 15:32:43 +0000874 for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
875 if (cpumask_test_cpu(cpu,
876 node_to_cpumask_map[node])) {
Anton Blanchard4b703a22005-12-13 06:56:47 +1100877 if (count == 0)
878 printk(" %u", cpu);
879 ++count;
880 } else {
881 if (count > 1)
882 printk("-%u", cpu - 1);
883 count = 0;
884 }
885 }
886
887 if (count > 1)
Anton Blanchard25863de2010-04-26 15:32:43 +0000888 printk("-%u", nr_cpu_ids - 1);
Anton Blanchard4b703a22005-12-13 06:56:47 +1100889 printk("\n");
890 }
891}
892
893static void __init dump_numa_memory_topology(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894{
895 unsigned int node;
896 unsigned int count;
897
898 if (min_common_depth == -1 || !numa_enabled)
899 return;
900
901 for_each_online_node(node) {
902 unsigned long i;
903
Olof Johanssone110b282006-04-12 15:25:01 -0500904 printk(KERN_DEBUG "Node %d Memory:", node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
906 count = 0;
907
Yinghai Lu95f72d12010-07-12 14:36:09 +1000908 for (i = 0; i < memblock_end_of_DRAM();
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100909 i += (1 << SECTION_SIZE_BITS)) {
910 if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 if (count == 0)
912 printk(" 0x%lx", i);
913 ++count;
914 } else {
915 if (count > 0)
916 printk("-0x%lx", i);
917 count = 0;
918 }
919 }
920
921 if (count > 0)
922 printk("-0x%lx", i);
923 printk("\n");
924 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925}
926
927/*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000928 * Allocate some memory, satisfying the memblock or bootmem allocator where
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 * required. nid is the preferred node and end is the physical address of
930 * the highest address in the node.
931 *
Dave Hansen0be210f2008-12-09 08:21:35 +0000932 * Returns the virtual address of the memory.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 */
Dave Hansen893473d2008-12-09 08:21:36 +0000934static void __init *careful_zallocation(int nid, unsigned long size,
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100935 unsigned long align,
936 unsigned long end_pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937{
Dave Hansen0be210f2008-12-09 08:21:35 +0000938 void *ret;
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100939 int new_nid;
Dave Hansen0be210f2008-12-09 08:21:35 +0000940 unsigned long ret_paddr;
941
Yinghai Lu95f72d12010-07-12 14:36:09 +1000942 ret_paddr = __memblock_alloc_base(size, align, end_pfn << PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943
944 /* retry over all memory */
Dave Hansen0be210f2008-12-09 08:21:35 +0000945 if (!ret_paddr)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000946 ret_paddr = __memblock_alloc_base(size, align, memblock_end_of_DRAM());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947
Dave Hansen0be210f2008-12-09 08:21:35 +0000948 if (!ret_paddr)
Dave Hansen5d21ea22008-12-09 08:21:33 +0000949 panic("numa.c: cannot allocate %lu bytes for node %d",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 size, nid);
951
Dave Hansen0be210f2008-12-09 08:21:35 +0000952 ret = __va(ret_paddr);
953
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 /*
Dave Hansenc555e5202008-12-09 08:21:32 +0000955 * We initialize the nodes in numeric order: 0, 1, 2...
Yinghai Lu95f72d12010-07-12 14:36:09 +1000956 * and hand over control from the MEMBLOCK allocator to the
Dave Hansenc555e5202008-12-09 08:21:32 +0000957 * bootmem allocator. If this function is called for
958 * node 5, then we know that all nodes <5 are using the
Yinghai Lu95f72d12010-07-12 14:36:09 +1000959 * bootmem allocator instead of the MEMBLOCK allocator.
Dave Hansenc555e5202008-12-09 08:21:32 +0000960 *
961 * So, check the nid from which this allocation came
962 * and double check to see if we need to use bootmem
Yinghai Lu95f72d12010-07-12 14:36:09 +1000963 * instead of the MEMBLOCK. We don't free the MEMBLOCK memory
Dave Hansenc555e5202008-12-09 08:21:32 +0000964 * since it would be useless.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 */
Dave Hansen0be210f2008-12-09 08:21:35 +0000966 new_nid = early_pfn_to_nid(ret_paddr >> PAGE_SHIFT);
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100967 if (new_nid < nid) {
Dave Hansen0be210f2008-12-09 08:21:35 +0000968 ret = __alloc_bootmem_node(NODE_DATA(new_nid),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 size, align, 0);
970
Dave Hansen0be210f2008-12-09 08:21:35 +0000971 dbg("alloc_bootmem %p %lx\n", ret, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 }
973
Dave Hansen893473d2008-12-09 08:21:36 +0000974 memset(ret, 0, size);
Dave Hansen0be210f2008-12-09 08:21:35 +0000975 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976}
977
Paul Gortmaker061d19f2013-06-24 15:30:09 -0400978static struct notifier_block ppc64_numa_nb = {
Chandra Seetharaman74b85f32006-06-27 02:54:09 -0700979 .notifier_call = cpu_numa_callback,
980 .priority = 1 /* Must run before sched domains notifier. */
981};
982
David Rientjes28e86bd2011-12-08 12:33:29 +0000983static void __init mark_reserved_regions_for_nid(int nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984{
Dave Hansen4a618662008-11-24 12:02:35 +0000985 struct pglist_data *node = NODE_DATA(nid);
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000986 struct memblock_region *reg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000988 for_each_memblock(reserved, reg) {
989 unsigned long physbase = reg->base;
990 unsigned long size = reg->size;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000991 unsigned long start_pfn = physbase >> PAGE_SHIFT;
Dave Hansen06eccea2009-02-12 12:36:04 +0000992 unsigned long end_pfn = PFN_UP(physbase + size);
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000993 struct node_active_region node_ar;
Xishi Qiu64080682013-11-12 15:07:17 -0800994 unsigned long node_end_pfn = pgdat_end_pfn(node);
Dave Hansen4a618662008-11-24 12:02:35 +0000995
996 /*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000997 * Check to make sure that this memblock.reserved area is
Dave Hansen4a618662008-11-24 12:02:35 +0000998 * within the bounds of the node that we care about.
999 * Checking the nid of the start and end points is not
1000 * sufficient because the reserved area could span the
1001 * entire node.
1002 */
1003 if (end_pfn <= node->node_start_pfn ||
1004 start_pfn >= node_end_pfn)
1005 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001007 get_node_active_region(start_pfn, &node_ar);
Jon Tollefsone8170372008-10-16 18:59:43 +00001008 while (start_pfn < end_pfn &&
1009 node_ar.start_pfn < node_ar.end_pfn) {
1010 unsigned long reserve_size = size;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001011 /*
1012 * if reserved region extends past active region
1013 * then trim size to active region
1014 */
1015 if (end_pfn > node_ar.end_pfn)
Jon Tollefsone8170372008-10-16 18:59:43 +00001016 reserve_size = (node_ar.end_pfn << PAGE_SHIFT)
Dave Hansen06eccea2009-02-12 12:36:04 +00001017 - physbase;
Dave Hansena4c74dd2008-12-11 08:36:06 +00001018 /*
1019 * Only worry about *this* node, others may not
1020 * yet have valid NODE_DATA().
1021 */
1022 if (node_ar.nid == nid) {
1023 dbg("reserve_bootmem %lx %lx nid=%d\n",
1024 physbase, reserve_size, node_ar.nid);
1025 reserve_bootmem_node(NODE_DATA(node_ar.nid),
1026 physbase, reserve_size,
1027 BOOTMEM_DEFAULT);
1028 }
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001029 /*
1030 * if reserved region is contained in the active region
1031 * then done.
1032 */
1033 if (end_pfn <= node_ar.end_pfn)
1034 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001036 /*
1037 * reserved region extends past the active region
1038 * get next active region that contains this
1039 * reserved region
1040 */
1041 start_pfn = node_ar.end_pfn;
1042 physbase = start_pfn << PAGE_SHIFT;
Jon Tollefsone8170372008-10-16 18:59:43 +00001043 size = size - reserve_size;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001044 get_node_active_region(start_pfn, &node_ar);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046 }
Dave Hansen4a618662008-11-24 12:02:35 +00001047}
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001048
Dave Hansen4a618662008-11-24 12:02:35 +00001049
1050void __init do_init_bootmem(void)
1051{
1052 int nid;
Dave Hansen4a618662008-11-24 12:02:35 +00001053
1054 min_low_pfn = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001055 max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
Dave Hansen4a618662008-11-24 12:02:35 +00001056 max_pfn = max_low_pfn;
1057
1058 if (parse_numa_properties())
1059 setup_nonnuma();
1060 else
1061 dump_numa_memory_topology();
1062
Dave Hansen4a618662008-11-24 12:02:35 +00001063 for_each_online_node(nid) {
1064 unsigned long start_pfn, end_pfn;
Dave Hansen0be210f2008-12-09 08:21:35 +00001065 void *bootmem_vaddr;
Dave Hansen4a618662008-11-24 12:02:35 +00001066 unsigned long bootmap_pages;
1067
1068 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
1069
1070 /*
1071 * Allocate the node structure node local if possible
1072 *
1073 * Be careful moving this around, as it relies on all
1074 * previous nodes' bootmem to be initialized and have
1075 * all reserved areas marked.
1076 */
Dave Hansen893473d2008-12-09 08:21:36 +00001077 NODE_DATA(nid) = careful_zallocation(nid,
Dave Hansen4a618662008-11-24 12:02:35 +00001078 sizeof(struct pglist_data),
1079 SMP_CACHE_BYTES, end_pfn);
Dave Hansen4a618662008-11-24 12:02:35 +00001080
1081 dbg("node %d\n", nid);
1082 dbg("NODE_DATA() = %p\n", NODE_DATA(nid));
1083
1084 NODE_DATA(nid)->bdata = &bootmem_node_data[nid];
1085 NODE_DATA(nid)->node_start_pfn = start_pfn;
1086 NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
1087
1088 if (NODE_DATA(nid)->node_spanned_pages == 0)
1089 continue;
1090
1091 dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT);
1092 dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT);
1093
1094 bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
Dave Hansen893473d2008-12-09 08:21:36 +00001095 bootmem_vaddr = careful_zallocation(nid,
Dave Hansen4a618662008-11-24 12:02:35 +00001096 bootmap_pages << PAGE_SHIFT,
1097 PAGE_SIZE, end_pfn);
Dave Hansen4a618662008-11-24 12:02:35 +00001098
Dave Hansen0be210f2008-12-09 08:21:35 +00001099 dbg("bootmap_vaddr = %p\n", bootmem_vaddr);
Dave Hansen4a618662008-11-24 12:02:35 +00001100
Dave Hansen0be210f2008-12-09 08:21:35 +00001101 init_bootmem_node(NODE_DATA(nid),
1102 __pa(bootmem_vaddr) >> PAGE_SHIFT,
Dave Hansen4a618662008-11-24 12:02:35 +00001103 start_pfn, end_pfn);
1104
1105 free_bootmem_with_active_regions(nid, end_pfn);
1106 /*
1107 * Be very careful about moving this around. Future
Dave Hansen893473d2008-12-09 08:21:36 +00001108 * calls to careful_zallocation() depend on this getting
Dave Hansen4a618662008-11-24 12:02:35 +00001109 * done correctly.
1110 */
1111 mark_reserved_regions_for_nid(nid);
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001112 sparse_memory_present_with_active_regions(nid);
Dave Hansen4a618662008-11-24 12:02:35 +00001113 }
Benjamin Herrenschmidtd3f62042009-06-02 21:16:38 +00001114
1115 init_bootmem_done = 1;
Anton Blanchard25863de2010-04-26 15:32:43 +00001116
1117 /*
1118 * Now bootmem is initialised we can create the node to cpumask
1119 * lookup tables and setup the cpu callback to populate them.
1120 */
1121 setup_node_to_cpumask_map();
1122
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +05301123 reset_numa_cpu_lookup_table();
Anton Blanchard25863de2010-04-26 15:32:43 +00001124 register_cpu_notifier(&ppc64_numa_nb);
1125 cpu_numa_callback(&ppc64_numa_nb, CPU_UP_PREPARE,
1126 (void *)(unsigned long)boot_cpuid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127}
1128
1129void __init paging_init(void)
1130{
Mel Gorman6391af12006-10-11 01:20:39 -07001131 unsigned long max_zone_pfns[MAX_NR_ZONES];
1132 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
Yinghai Lu95f72d12010-07-12 14:36:09 +10001133 max_zone_pfns[ZONE_DMA] = memblock_end_of_DRAM() >> PAGE_SHIFT;
Mel Gormanc67c3cb2006-09-27 01:49:49 -07001134 free_area_init_nodes(max_zone_pfns);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135}
1136
1137static int __init early_numa(char *p)
1138{
1139 if (!p)
1140 return 0;
1141
1142 if (strstr(p, "off"))
1143 numa_enabled = 0;
1144
1145 if (strstr(p, "debug"))
1146 numa_debug = 1;
1147
Balbir Singh1daa6d02008-02-01 15:57:31 +11001148 p = strstr(p, "fake=");
1149 if (p)
1150 cmdline = p + strlen("fake=");
1151
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 return 0;
1153}
1154early_param("numa", early_numa);
Mike Kravetz237a09892005-12-05 12:06:42 -08001155
1156#ifdef CONFIG_MEMORY_HOTPLUG
1157/*
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001158 * Find the node associated with a hot added memory section for
1159 * memory represented in the device tree by the property
1160 * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory.
Nathan Fontenot0db93602008-07-03 13:25:08 +10001161 */
1162static int hot_add_drconf_scn_to_nid(struct device_node *memory,
1163 unsigned long scn_addr)
1164{
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001165 const __be32 *dm;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001166 unsigned int drconf_cell_cnt, rc;
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +10001167 unsigned long lmb_size;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001168 struct assoc_arrays aa;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001169 int nid = -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001170
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001171 drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
1172 if (!drconf_cell_cnt)
1173 return -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001174
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +10001175 lmb_size = of_get_lmb_size(memory);
1176 if (!lmb_size)
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001177 return -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001178
1179 rc = of_get_assoc_arrays(memory, &aa);
1180 if (rc)
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001181 return -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001182
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001183 for (; drconf_cell_cnt != 0; --drconf_cell_cnt) {
Nathan Fontenot0db93602008-07-03 13:25:08 +10001184 struct of_drconf_cell drmem;
1185
1186 read_drconf_cell(&drmem, &dm);
1187
1188 /* skip this block if it is reserved or not assigned to
1189 * this partition */
1190 if ((drmem.flags & DRCONF_MEM_RESERVED)
1191 || !(drmem.flags & DRCONF_MEM_ASSIGNED))
1192 continue;
1193
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001194 if ((scn_addr < drmem.base_addr)
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +10001195 || (scn_addr >= (drmem.base_addr + lmb_size)))
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001196 continue;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001197
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001198 nid = of_drconf_to_nid_single(&drmem, &aa);
1199 break;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001200 }
1201
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001202 return nid;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001203}
1204
1205/*
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001206 * Find the node associated with a hot added memory section for memory
1207 * represented in the device tree as a node (i.e. memory@XXXX) for
Yinghai Lu95f72d12010-07-12 14:36:09 +10001208 * each memblock.
Mike Kravetz237a09892005-12-05 12:06:42 -08001209 */
Robert Jenningsec32dd62013-10-28 09:20:50 -05001210static int hot_add_node_scn_to_nid(unsigned long scn_addr)
Mike Kravetz237a09892005-12-05 12:06:42 -08001211{
Anton Blanchard94db7c52011-08-10 20:44:22 +00001212 struct device_node *memory;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001213 int nid = -1;
Mike Kravetz237a09892005-12-05 12:06:42 -08001214
Anton Blanchard94db7c52011-08-10 20:44:22 +00001215 for_each_node_by_type(memory, "memory") {
Mike Kravetz237a09892005-12-05 12:06:42 -08001216 unsigned long start, size;
Mike Kravetzb226e462005-12-16 14:30:35 -08001217 int ranges;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001218 const __be32 *memcell_buf;
Mike Kravetz237a09892005-12-05 12:06:42 -08001219 unsigned int len;
1220
Stephen Rothwelle2eb6392007-04-03 22:26:41 +10001221 memcell_buf = of_get_property(memory, "reg", &len);
Mike Kravetz237a09892005-12-05 12:06:42 -08001222 if (!memcell_buf || len <= 0)
1223 continue;
1224
Benjamin Herrenschmidtcc5d0182005-12-13 18:01:21 +11001225 /* ranges in cell */
1226 ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
Mike Kravetz237a09892005-12-05 12:06:42 -08001227
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001228 while (ranges--) {
1229 start = read_n_cells(n_mem_addr_cells, &memcell_buf);
1230 size = read_n_cells(n_mem_size_cells, &memcell_buf);
1231
1232 if ((scn_addr < start) || (scn_addr >= (start + size)))
1233 continue;
1234
1235 nid = of_node_to_nid_single(memory);
1236 break;
Mike Kravetz237a09892005-12-05 12:06:42 -08001237 }
1238
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001239 if (nid >= 0)
1240 break;
Mike Kravetz237a09892005-12-05 12:06:42 -08001241 }
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001242
Anton Blanchard60831842011-08-10 20:44:21 +00001243 of_node_put(memory);
1244
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001245 return nid;
Mike Kravetz237a09892005-12-05 12:06:42 -08001246}
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001247
1248/*
1249 * Find the node associated with a hot added memory section. Section
Yinghai Lu95f72d12010-07-12 14:36:09 +10001250 * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that
1251 * sections are fully contained within a single MEMBLOCK.
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001252 */
1253int hot_add_scn_to_nid(unsigned long scn_addr)
1254{
1255 struct device_node *memory = NULL;
1256 int nid, found = 0;
1257
1258 if (!numa_enabled || (min_common_depth < 0))
H Hartley Sweeten72c33682010-03-05 13:42:43 -08001259 return first_online_node;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001260
1261 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1262 if (memory) {
1263 nid = hot_add_drconf_scn_to_nid(memory, scn_addr);
1264 of_node_put(memory);
1265 } else {
1266 nid = hot_add_node_scn_to_nid(scn_addr);
1267 }
1268
1269 if (nid < 0 || !node_online(nid))
H Hartley Sweeten72c33682010-03-05 13:42:43 -08001270 nid = first_online_node;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001271
1272 if (NODE_DATA(nid)->node_spanned_pages)
1273 return nid;
1274
1275 for_each_online_node(nid) {
1276 if (NODE_DATA(nid)->node_spanned_pages) {
1277 found = 1;
1278 break;
1279 }
1280 }
1281
1282 BUG_ON(!found);
1283 return nid;
1284}
1285
Nishanth Aravamudancd342062010-10-26 17:35:12 +00001286static u64 hot_add_drconf_memory_max(void)
1287{
1288 struct device_node *memory = NULL;
1289 unsigned int drconf_cell_cnt = 0;
1290 u64 lmb_size = 0;
Robert Jenningsec32dd62013-10-28 09:20:50 -05001291 const __be32 *dm = NULL;
Nishanth Aravamudancd342062010-10-26 17:35:12 +00001292
1293 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1294 if (memory) {
1295 drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
1296 lmb_size = of_get_lmb_size(memory);
1297 of_node_put(memory);
1298 }
1299 return lmb_size * drconf_cell_cnt;
1300}
1301
1302/*
1303 * memory_hotplug_max - return max address of memory that may be added
1304 *
1305 * This is currently only used on systems that support drconfig memory
1306 * hotplug.
1307 */
1308u64 memory_hotplug_max(void)
1309{
1310 return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM());
1311}
Mike Kravetz237a09892005-12-05 12:06:42 -08001312#endif /* CONFIG_MEMORY_HOTPLUG */
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001313
Jesse Larrewbd034032011-01-20 19:00:51 +00001314/* Virtual Processor Home Node (VPHN) support */
Jesse Larrew39bf9902010-12-17 22:07:47 +00001315#ifdef CONFIG_PPC_SPLPAR
Nathan Fontenot30c05352013-04-24 06:02:13 +00001316struct topology_update_data {
1317 struct topology_update_data *next;
1318 unsigned int cpu;
1319 int old_nid;
1320 int new_nid;
1321};
1322
Anton Blanchard5de16692011-01-29 12:24:34 +00001323static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS];
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001324static cpumask_t cpu_associativity_changes_mask;
1325static int vphn_enabled;
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001326static int prrn_enabled;
1327static void reset_topology_timer(void);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001328
1329/*
1330 * Store the current values of the associativity change counters in the
1331 * hypervisor.
1332 */
1333static void setup_cpu_associativity_change_counters(void)
1334{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001335 int cpu;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001336
Anton Blanchard5de16692011-01-29 12:24:34 +00001337 /* The VPHN feature supports a maximum of 8 reference points */
1338 BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8);
1339
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001340 for_each_possible_cpu(cpu) {
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001341 int i;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001342 u8 *counts = vphn_cpu_change_counts[cpu];
1343 volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts;
1344
Anton Blanchard5de16692011-01-29 12:24:34 +00001345 for (i = 0; i < distance_ref_points_depth; i++)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001346 counts[i] = hypervisor_counts[i];
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001347 }
1348}
1349
1350/*
1351 * The hypervisor maintains a set of 8 associativity change counters in
1352 * the VPA of each cpu that correspond to the associativity levels in the
1353 * ibm,associativity-reference-points property. When an associativity
1354 * level changes, the corresponding counter is incremented.
1355 *
1356 * Set a bit in cpu_associativity_changes_mask for each cpu whose home
1357 * node associativity levels have changed.
1358 *
1359 * Returns the number of cpus with unhandled associativity changes.
1360 */
1361static int update_cpu_associativity_changes_mask(void)
1362{
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001363 int cpu;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001364 cpumask_t *changes = &cpu_associativity_changes_mask;
1365
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001366 for_each_possible_cpu(cpu) {
1367 int i, changed = 0;
1368 u8 *counts = vphn_cpu_change_counts[cpu];
1369 volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts;
1370
Anton Blanchard5de16692011-01-29 12:24:34 +00001371 for (i = 0; i < distance_ref_points_depth; i++) {
Anton Blanchardd69043e2011-01-29 12:26:19 +00001372 if (hypervisor_counts[i] != counts[i]) {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001373 counts[i] = hypervisor_counts[i];
1374 changed = 1;
1375 }
1376 }
1377 if (changed) {
Robert Jennings3be7db62013-07-24 20:13:21 -05001378 cpumask_or(changes, changes, cpu_sibling_mask(cpu));
1379 cpu = cpu_last_thread_sibling(cpu);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001380 }
1381 }
1382
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001383 return cpumask_weight(changes);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001384}
1385
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001386/*
1387 * 6 64-bit registers unpacked into 12 32-bit associativity values. To form
1388 * the complete property we have to add the length in the first cell.
1389 */
1390#define VPHN_ASSOC_BUFSIZE (6*sizeof(u64)/sizeof(u32) + 1)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001391
1392/*
1393 * Convert the associativity domain numbers returned from the hypervisor
1394 * to the sequence they would appear in the ibm,associativity property.
1395 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001396static int vphn_unpack_associativity(const long *packed, __be32 *unpacked)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001397{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001398 int i, nr_assoc_doms = 0;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001399 const __be16 *field = (const __be16 *) packed;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001400
1401#define VPHN_FIELD_UNUSED (0xffff)
1402#define VPHN_FIELD_MSB (0x8000)
1403#define VPHN_FIELD_MASK (~VPHN_FIELD_MSB)
1404
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001405 for (i = 1; i < VPHN_ASSOC_BUFSIZE; i++) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001406 if (be16_to_cpup(field) == VPHN_FIELD_UNUSED) {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001407 /* All significant fields processed, and remaining
1408 * fields contain the reserved value of all 1's.
1409 * Just store them.
1410 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001411 unpacked[i] = *((__be32 *)field);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001412 field += 2;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001413 } else if (be16_to_cpup(field) & VPHN_FIELD_MSB) {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001414 /* Data is in the lower 15 bits of this field */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001415 unpacked[i] = cpu_to_be32(
1416 be16_to_cpup(field) & VPHN_FIELD_MASK);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001417 field++;
1418 nr_assoc_doms++;
Jesse Larrew7639ada2011-01-20 19:01:13 +00001419 } else {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001420 /* Data is in the lower 15 bits of this field
1421 * concatenated with the next 16 bit field
1422 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001423 unpacked[i] = *((__be32 *)field);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001424 field += 2;
1425 nr_assoc_doms++;
1426 }
1427 }
1428
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001429 /* The first cell contains the length of the property */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001430 unpacked[0] = cpu_to_be32(nr_assoc_doms);
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001431
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001432 return nr_assoc_doms;
1433}
1434
1435/*
1436 * Retrieve the new associativity information for a virtual processor's
1437 * home node.
1438 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001439static long hcall_vphn(unsigned long cpu, __be32 *associativity)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001440{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001441 long rc;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001442 long retbuf[PLPAR_HCALL9_BUFSIZE] = {0};
1443 u64 flags = 1;
1444 int hwcpu = get_hard_smp_processor_id(cpu);
1445
1446 rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu);
1447 vphn_unpack_associativity(retbuf, associativity);
1448
1449 return rc;
1450}
1451
1452static long vphn_get_associativity(unsigned long cpu,
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001453 __be32 *associativity)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001454{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001455 long rc;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001456
1457 rc = hcall_vphn(cpu, associativity);
1458
1459 switch (rc) {
1460 case H_FUNCTION:
1461 printk(KERN_INFO
1462 "VPHN is not supported. Disabling polling...\n");
1463 stop_topology_update();
1464 break;
1465 case H_HARDWARE:
1466 printk(KERN_ERR
1467 "hcall_vphn() experienced a hardware fault "
1468 "preventing VPHN. Disabling polling...\n");
1469 stop_topology_update();
1470 }
1471
1472 return rc;
1473}
1474
1475/*
Nathan Fontenot30c05352013-04-24 06:02:13 +00001476 * Update the CPU maps and sysfs entries for a single CPU when its NUMA
1477 * characteristics change. This function doesn't perform any locking and is
1478 * only safe to call from stop_machine().
1479 */
1480static int update_cpu_topology(void *data)
1481{
1482 struct topology_update_data *update;
1483 unsigned long cpu;
1484
1485 if (!data)
1486 return -EINVAL;
1487
Robert Jennings3be7db62013-07-24 20:13:21 -05001488 cpu = smp_processor_id();
Nathan Fontenot30c05352013-04-24 06:02:13 +00001489
1490 for (update = data; update; update = update->next) {
1491 if (cpu != update->cpu)
1492 continue;
1493
Nathan Fontenot30c05352013-04-24 06:02:13 +00001494 unmap_cpu_from_node(update->cpu);
1495 map_cpu_to_node(update->cpu, update->new_nid);
Jesse Larrew176bbf12013-04-24 06:03:48 +00001496 vdso_getcpu_init();
Nathan Fontenot30c05352013-04-24 06:02:13 +00001497 }
1498
1499 return 0;
1500}
1501
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +05301502static int update_lookup_table(void *data)
1503{
1504 struct topology_update_data *update;
1505
1506 if (!data)
1507 return -EINVAL;
1508
1509 /*
1510 * Upon topology update, the numa-cpu lookup table needs to be updated
1511 * for all threads in the core, including offline CPUs, to ensure that
1512 * future hotplug operations respect the cpu-to-node associativity
1513 * properly.
1514 */
1515 for (update = data; update; update = update->next) {
1516 int nid, base, j;
1517
1518 nid = update->new_nid;
1519 base = cpu_first_thread_sibling(update->cpu);
1520
1521 for (j = 0; j < threads_per_core; j++) {
1522 update_numa_cpu_lookup_table(base + j, nid);
1523 }
1524 }
1525
1526 return 0;
1527}
1528
Nathan Fontenot30c05352013-04-24 06:02:13 +00001529/*
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001530 * Update the node maps and sysfs entries for each cpu whose home node
Jesse Larrew79c5fce2012-06-07 16:04:34 -05001531 * has changed. Returns 1 when the topology has changed, and 0 otherwise.
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001532 */
1533int arch_update_cpu_topology(void)
1534{
Robert Jennings3be7db62013-07-24 20:13:21 -05001535 unsigned int cpu, sibling, changed = 0;
Nathan Fontenot30c05352013-04-24 06:02:13 +00001536 struct topology_update_data *updates, *ud;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001537 __be32 associativity[VPHN_ASSOC_BUFSIZE] = {0};
Jesse Larrew176bbf12013-04-24 06:03:48 +00001538 cpumask_t updated_cpus;
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001539 struct device *dev;
Robert Jennings3be7db62013-07-24 20:13:21 -05001540 int weight, new_nid, i = 0;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001541
Nathan Fontenot30c05352013-04-24 06:02:13 +00001542 weight = cpumask_weight(&cpu_associativity_changes_mask);
1543 if (!weight)
1544 return 0;
1545
1546 updates = kzalloc(weight * (sizeof(*updates)), GFP_KERNEL);
1547 if (!updates)
1548 return 0;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001549
Jesse Larrew176bbf12013-04-24 06:03:48 +00001550 cpumask_clear(&updated_cpus);
1551
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001552 for_each_cpu(cpu, &cpu_associativity_changes_mask) {
Robert Jennings3be7db62013-07-24 20:13:21 -05001553 /*
1554 * If siblings aren't flagged for changes, updates list
1555 * will be too short. Skip on this update and set for next
1556 * update.
1557 */
1558 if (!cpumask_subset(cpu_sibling_mask(cpu),
1559 &cpu_associativity_changes_mask)) {
1560 pr_info("Sibling bits not set for associativity "
1561 "change, cpu%d\n", cpu);
1562 cpumask_or(&cpu_associativity_changes_mask,
1563 &cpu_associativity_changes_mask,
1564 cpu_sibling_mask(cpu));
1565 cpu = cpu_last_thread_sibling(cpu);
1566 continue;
1567 }
1568
1569 /* Use associativity from first thread for all siblings */
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001570 vphn_get_associativity(cpu, associativity);
Robert Jennings3be7db62013-07-24 20:13:21 -05001571 new_nid = associativity_to_nid(associativity);
1572 if (new_nid < 0 || !node_online(new_nid))
1573 new_nid = first_online_node;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001574
Robert Jennings3be7db62013-07-24 20:13:21 -05001575 if (new_nid == numa_cpu_lookup_table[cpu]) {
1576 cpumask_andnot(&cpu_associativity_changes_mask,
1577 &cpu_associativity_changes_mask,
1578 cpu_sibling_mask(cpu));
1579 cpu = cpu_last_thread_sibling(cpu);
1580 continue;
1581 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001582
Robert Jennings3be7db62013-07-24 20:13:21 -05001583 for_each_cpu(sibling, cpu_sibling_mask(cpu)) {
1584 ud = &updates[i++];
1585 ud->cpu = sibling;
1586 ud->new_nid = new_nid;
1587 ud->old_nid = numa_cpu_lookup_table[sibling];
1588 cpumask_set_cpu(sibling, &updated_cpus);
1589 if (i < weight)
1590 ud->next = &updates[i];
1591 }
1592 cpu = cpu_last_thread_sibling(cpu);
Nathan Fontenot30c05352013-04-24 06:02:13 +00001593 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001594
Michael Wang9a013362014-04-08 11:19:36 +08001595 /*
1596 * In cases where we have nothing to update (because the updates list
1597 * is too short or because the new topology is same as the old one),
1598 * skip invoking update_cpu_topology() via stop-machine(). This is
1599 * necessary (and not just a fast-path optimization) since stop-machine
1600 * can end up electing a random CPU to run update_cpu_topology(), and
1601 * thus trick us into setting up incorrect cpu-node mappings (since
1602 * 'updates' is kzalloc()'ed).
1603 *
1604 * And for the similar reason, we will skip all the following updating.
1605 */
1606 if (!cpumask_weight(&updated_cpus))
1607 goto out;
1608
Jesse Larrew176bbf12013-04-24 06:03:48 +00001609 stop_machine(update_cpu_topology, &updates[0], &updated_cpus);
Nathan Fontenot30c05352013-04-24 06:02:13 +00001610
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +05301611 /*
1612 * Update the numa-cpu lookup table with the new mappings, even for
1613 * offline CPUs. It is best to perform this update from the stop-
1614 * machine context.
1615 */
1616 stop_machine(update_lookup_table, &updates[0],
1617 cpumask_of(raw_smp_processor_id()));
1618
Nathan Fontenot30c05352013-04-24 06:02:13 +00001619 for (ud = &updates[0]; ud; ud = ud->next) {
Nathan Fontenotdd023212013-06-24 22:08:05 -05001620 unregister_cpu_under_node(ud->cpu, ud->old_nid);
1621 register_cpu_under_node(ud->cpu, ud->new_nid);
1622
Nathan Fontenot30c05352013-04-24 06:02:13 +00001623 dev = get_cpu_device(ud->cpu);
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001624 if (dev)
1625 kobject_uevent(&dev->kobj, KOBJ_CHANGE);
Nathan Fontenot30c05352013-04-24 06:02:13 +00001626 cpumask_clear_cpu(ud->cpu, &cpu_associativity_changes_mask);
Jesse Larrew79c5fce2012-06-07 16:04:34 -05001627 changed = 1;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001628 }
1629
Michael Wang9a013362014-04-08 11:19:36 +08001630out:
Nathan Fontenot30c05352013-04-24 06:02:13 +00001631 kfree(updates);
Jesse Larrew79c5fce2012-06-07 16:04:34 -05001632 return changed;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001633}
1634
1635static void topology_work_fn(struct work_struct *work)
1636{
1637 rebuild_sched_domains();
1638}
1639static DECLARE_WORK(topology_work, topology_work_fn);
1640
Robert Jenningsec32dd62013-10-28 09:20:50 -05001641static void topology_schedule_update(void)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001642{
1643 schedule_work(&topology_work);
1644}
1645
1646static void topology_timer_fn(unsigned long ignored)
1647{
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001648 if (prrn_enabled && cpumask_weight(&cpu_associativity_changes_mask))
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001649 topology_schedule_update();
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001650 else if (vphn_enabled) {
1651 if (update_cpu_associativity_changes_mask() > 0)
1652 topology_schedule_update();
1653 reset_topology_timer();
1654 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001655}
1656static struct timer_list topology_timer =
1657 TIMER_INITIALIZER(topology_timer_fn, 0, 0);
1658
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001659static void reset_topology_timer(void)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001660{
1661 topology_timer.data = 0;
1662 topology_timer.expires = jiffies + 60 * HZ;
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001663 mod_timer(&topology_timer, topology_timer.expires);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001664}
1665
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001666#ifdef CONFIG_SMP
1667
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001668static void stage_topology_update(int core_id)
1669{
1670 cpumask_or(&cpu_associativity_changes_mask,
1671 &cpu_associativity_changes_mask, cpu_sibling_mask(core_id));
1672 reset_topology_timer();
1673}
1674
1675static int dt_update_callback(struct notifier_block *nb,
1676 unsigned long action, void *data)
1677{
1678 struct of_prop_reconfig *update;
1679 int rc = NOTIFY_DONE;
1680
1681 switch (action) {
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001682 case OF_RECONFIG_UPDATE_PROPERTY:
1683 update = (struct of_prop_reconfig *)data;
Nathan Fontenot30c05352013-04-24 06:02:13 +00001684 if (!of_prop_cmp(update->dn->type, "cpu") &&
1685 !of_prop_cmp(update->prop->name, "ibm,associativity")) {
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001686 u32 core_id;
1687 of_property_read_u32(update->dn, "reg", &core_id);
1688 stage_topology_update(core_id);
1689 rc = NOTIFY_OK;
1690 }
1691 break;
1692 }
1693
1694 return rc;
1695}
1696
1697static struct notifier_block dt_update_nb = {
1698 .notifier_call = dt_update_callback,
1699};
1700
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001701#endif
1702
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001703/*
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001704 * Start polling for associativity changes.
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001705 */
1706int start_topology_update(void)
1707{
1708 int rc = 0;
1709
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001710 if (firmware_has_feature(FW_FEATURE_PRRN)) {
1711 if (!prrn_enabled) {
1712 prrn_enabled = 1;
1713 vphn_enabled = 0;
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001714#ifdef CONFIG_SMP
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001715 rc = of_reconfig_notifier_register(&dt_update_nb);
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001716#endif
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001717 }
Jesse Larrewb7abef02013-04-24 06:05:22 +00001718 } else if (firmware_has_feature(FW_FEATURE_VPHN) &&
Anton Blanchardf13c13a2013-08-07 02:01:26 +10001719 lppaca_shared_proc(get_lppaca())) {
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001720 if (!vphn_enabled) {
1721 prrn_enabled = 0;
1722 vphn_enabled = 1;
1723 setup_cpu_associativity_change_counters();
1724 init_timer_deferrable(&topology_timer);
1725 reset_topology_timer();
1726 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001727 }
1728
1729 return rc;
1730}
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001731
1732/*
1733 * Disable polling for VPHN associativity changes.
1734 */
1735int stop_topology_update(void)
1736{
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001737 int rc = 0;
1738
1739 if (prrn_enabled) {
1740 prrn_enabled = 0;
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001741#ifdef CONFIG_SMP
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001742 rc = of_reconfig_notifier_unregister(&dt_update_nb);
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001743#endif
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001744 } else if (vphn_enabled) {
1745 vphn_enabled = 0;
1746 rc = del_timer_sync(&topology_timer);
1747 }
1748
1749 return rc;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001750}
Nathan Fontenote04fa612013-04-24 06:07:39 +00001751
1752int prrn_is_enabled(void)
1753{
1754 return prrn_enabled;
1755}
1756
1757static int topology_read(struct seq_file *file, void *v)
1758{
1759 if (vphn_enabled || prrn_enabled)
1760 seq_puts(file, "on\n");
1761 else
1762 seq_puts(file, "off\n");
1763
1764 return 0;
1765}
1766
1767static int topology_open(struct inode *inode, struct file *file)
1768{
1769 return single_open(file, topology_read, NULL);
1770}
1771
1772static ssize_t topology_write(struct file *file, const char __user *buf,
1773 size_t count, loff_t *off)
1774{
1775 char kbuf[4]; /* "on" or "off" plus null. */
1776 int read_len;
1777
1778 read_len = count < 3 ? count : 3;
1779 if (copy_from_user(kbuf, buf, read_len))
1780 return -EINVAL;
1781
1782 kbuf[read_len] = '\0';
1783
1784 if (!strncmp(kbuf, "on", 2))
1785 start_topology_update();
1786 else if (!strncmp(kbuf, "off", 3))
1787 stop_topology_update();
1788 else
1789 return -EINVAL;
1790
1791 return count;
1792}
1793
1794static const struct file_operations topology_ops = {
1795 .read = seq_read,
1796 .write = topology_write,
1797 .open = topology_open,
1798 .release = single_release
1799};
1800
1801static int topology_update_init(void)
1802{
1803 start_topology_update();
Joe Perches316d71882014-01-28 10:22:22 -08001804 proc_create("powerpc/topology_updates", 0644, NULL, &topology_ops);
Nathan Fontenote04fa612013-04-24 06:07:39 +00001805
1806 return 0;
1807}
1808device_initcall(topology_update_init);
Jesse Larrew39bf9902010-12-17 22:07:47 +00001809#endif /* CONFIG_PPC_SPLPAR */