blob: 6847d509162f11b3b33bad62f1996d4a83d30549 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * pSeries NUMA support
3 *
4 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11#include <linux/threads.h>
12#include <linux/bootmem.h>
13#include <linux/init.h>
14#include <linux/mm.h>
15#include <linux/mmzone.h>
Paul Gortmaker4b16f8e2011-07-22 18:24:23 -040016#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/nodemask.h>
18#include <linux/cpu.h>
19#include <linux/notifier.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100020#include <linux/memblock.h>
Michael Ellerman6df16462008-02-14 11:37:49 +110021#include <linux/of.h>
Dave Hansen06eccea2009-02-12 12:36:04 +000022#include <linux/pfn.h>
Jesse Larrew9eff1a32010-12-01 12:31:15 +000023#include <linux/cpuset.h>
24#include <linux/node.h>
Nathan Fontenot30c05352013-04-24 06:02:13 +000025#include <linux/stop_machine.h>
Nathan Fontenote04fa612013-04-24 06:07:39 +000026#include <linux/proc_fs.h>
27#include <linux/seq_file.h>
28#include <linux/uaccess.h>
Linus Torvalds191a7122013-04-29 19:14:20 -070029#include <linux/slab.h>
Robert Jennings3be7db62013-07-24 20:13:21 -050030#include <asm/cputhreads.h>
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110031#include <asm/sparsemem.h>
David S. Millerd9b2b2a2008-02-13 16:56:49 -080032#include <asm/prom.h>
Paul Mackerras2249ca92005-11-07 13:18:13 +110033#include <asm/smp.h>
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +053034#include <asm/cputhreads.h>
35#include <asm/topology.h>
Jesse Larrew9eff1a32010-12-01 12:31:15 +000036#include <asm/firmware.h>
37#include <asm/paca.h>
Jesse Larrew39bf9902010-12-17 22:07:47 +000038#include <asm/hvcall.h>
David Howellsae3a197e2012-03-28 18:30:02 +010039#include <asm/setup.h>
Jesse Larrew176bbf12013-04-24 06:03:48 +000040#include <asm/vdso.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
42static int numa_enabled = 1;
43
Balbir Singh1daa6d02008-02-01 15:57:31 +110044static char *cmdline __initdata;
45
Linus Torvalds1da177e2005-04-16 15:20:36 -070046static int numa_debug;
47#define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
48
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110049int numa_cpu_lookup_table[NR_CPUS];
Anton Blanchard25863de2010-04-26 15:32:43 +000050cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
Linus Torvalds1da177e2005-04-16 15:20:36 -070051struct pglist_data *node_data[MAX_NUMNODES];
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110052
53EXPORT_SYMBOL(numa_cpu_lookup_table);
Anton Blanchard25863de2010-04-26 15:32:43 +000054EXPORT_SYMBOL(node_to_cpumask_map);
Anton Blanchard45fb6ce2005-11-11 14:22:35 +110055EXPORT_SYMBOL(node_data);
56
Linus Torvalds1da177e2005-04-16 15:20:36 -070057static int min_common_depth;
Mike Kravetz237a09892005-12-05 12:06:42 -080058static int n_mem_addr_cells, n_mem_size_cells;
Anton Blanchard41eab6f2010-05-16 20:22:31 +000059static int form1_affinity;
60
61#define MAX_DISTANCE_REF_POINTS 4
62static int distance_ref_points_depth;
Alistair Poppleb08a2a12013-08-07 02:01:44 +100063static const __be32 *distance_ref_points;
Anton Blanchard41eab6f2010-05-16 20:22:31 +000064static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS];
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
Anton Blanchard25863de2010-04-26 15:32:43 +000066/*
67 * Allocate node_to_cpumask_map based on number of available nodes
68 * Requires node_possible_map to be valid.
69 *
Wanlong Gao95129382012-01-12 17:20:09 -080070 * Note: cpumask_of_node() is not valid until after this is done.
Anton Blanchard25863de2010-04-26 15:32:43 +000071 */
72static void __init setup_node_to_cpumask_map(void)
73{
Cody P Schaferf9d531b2013-04-29 15:08:03 -070074 unsigned int node;
Anton Blanchard25863de2010-04-26 15:32:43 +000075
76 /* setup nr_node_ids if not done yet */
Cody P Schaferf9d531b2013-04-29 15:08:03 -070077 if (nr_node_ids == MAX_NUMNODES)
78 setup_nr_node_ids();
Anton Blanchard25863de2010-04-26 15:32:43 +000079
80 /* allocate the map */
81 for (node = 0; node < nr_node_ids; node++)
82 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
83
84 /* cpumask_of_node() will now work */
85 dbg("Node to cpumask map for %d nodes\n", nr_node_ids);
86}
87
Stephen Rothwell55671f32013-03-25 18:44:44 +000088static int __init fake_numa_create_new_node(unsigned long end_pfn,
Balbir Singh1daa6d02008-02-01 15:57:31 +110089 unsigned int *nid)
90{
91 unsigned long long mem;
92 char *p = cmdline;
93 static unsigned int fake_nid;
94 static unsigned long long curr_boundary;
95
96 /*
97 * Modify node id, iff we started creating NUMA nodes
98 * We want to continue from where we left of the last time
99 */
100 if (fake_nid)
101 *nid = fake_nid;
102 /*
103 * In case there are no more arguments to parse, the
104 * node_id should be the same as the last fake node id
105 * (we've handled this above).
106 */
107 if (!p)
108 return 0;
109
110 mem = memparse(p, &p);
111 if (!mem)
112 return 0;
113
114 if (mem < curr_boundary)
115 return 0;
116
117 curr_boundary = mem;
118
119 if ((end_pfn << PAGE_SHIFT) > mem) {
120 /*
121 * Skip commas and spaces
122 */
123 while (*p == ',' || *p == ' ' || *p == '\t')
124 p++;
125
126 cmdline = p;
127 fake_nid++;
128 *nid = fake_nid;
129 dbg("created new fake_node with id %d\n", fake_nid);
130 return 1;
131 }
132 return 0;
133}
134
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000135/*
Tejun Heo5dfe8662011-07-14 09:46:10 +0200136 * get_node_active_region - Return active region containing pfn
Jon Tollefsone8170372008-10-16 18:59:43 +0000137 * Active range returned is empty if none found.
Tejun Heo5dfe8662011-07-14 09:46:10 +0200138 * @pfn: The page to return the region for
139 * @node_ar: Returned set to the active region containing @pfn
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000140 */
Tejun Heo5dfe8662011-07-14 09:46:10 +0200141static void __init get_node_active_region(unsigned long pfn,
142 struct node_active_region *node_ar)
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000143{
Tejun Heo5dfe8662011-07-14 09:46:10 +0200144 unsigned long start_pfn, end_pfn;
145 int i, nid;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000146
Tejun Heo5dfe8662011-07-14 09:46:10 +0200147 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
148 if (pfn >= start_pfn && pfn < end_pfn) {
149 node_ar->nid = nid;
150 node_ar->start_pfn = start_pfn;
151 node_ar->end_pfn = end_pfn;
152 break;
153 }
154 }
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000155}
156
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530157static void reset_numa_cpu_lookup_table(void)
158{
159 unsigned int cpu;
160
161 for_each_possible_cpu(cpu)
162 numa_cpu_lookup_table[cpu] = -1;
163}
164
165static void update_numa_cpu_lookup_table(unsigned int cpu, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166{
167 numa_cpu_lookup_table[cpu] = node;
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530168}
169
170static void map_cpu_to_node(int cpu, int node)
171{
172 update_numa_cpu_lookup_table(cpu, node);
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100173
Nathan Lynchbf4b85b2006-03-20 18:34:45 -0600174 dbg("adding cpu %d to node %d\n", cpu, node);
175
Anton Blanchard25863de2010-04-26 15:32:43 +0000176 if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node])))
177 cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178}
179
Jesse Larrew39bf9902010-12-17 22:07:47 +0000180#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181static void unmap_cpu_from_node(unsigned long cpu)
182{
183 int node = numa_cpu_lookup_table[cpu];
184
185 dbg("removing cpu %lu from node %d\n", cpu, node);
186
Anton Blanchard25863de2010-04-26 15:32:43 +0000187 if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) {
Anton Blanchard429f4d82011-01-29 12:37:16 +0000188 cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 } else {
190 printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n",
191 cpu, node);
192 }
193}
Jesse Larrew39bf9902010-12-17 22:07:47 +0000194#endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196/* must hold reference to node during call */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000197static const __be32 *of_get_associativity(struct device_node *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198{
Stephen Rothwelle2eb6392007-04-03 22:26:41 +1000199 return of_get_property(dev, "ibm,associativity", NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200}
201
Chandrucf000852008-08-30 00:28:16 +1000202/*
203 * Returns the property linux,drconf-usable-memory if
204 * it exists (the property exists only in kexec/kdump kernels,
205 * added by kexec-tools)
206 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000207static const __be32 *of_get_usable_memory(struct device_node *memory)
Chandrucf000852008-08-30 00:28:16 +1000208{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000209 const __be32 *prop;
Chandrucf000852008-08-30 00:28:16 +1000210 u32 len;
211 prop = of_get_property(memory, "linux,drconf-usable-memory", &len);
212 if (!prop || len < sizeof(unsigned int))
Robert Jenningsec32dd62013-10-28 09:20:50 -0500213 return NULL;
Chandrucf000852008-08-30 00:28:16 +1000214 return prop;
215}
216
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000217int __node_distance(int a, int b)
218{
219 int i;
220 int distance = LOCAL_DISTANCE;
221
222 if (!form1_affinity)
Vaidyanathan Srinivasan7122bee2013-03-22 05:49:35 +0000223 return ((a == b) ? LOCAL_DISTANCE : REMOTE_DISTANCE);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000224
225 for (i = 0; i < distance_ref_points_depth; i++) {
226 if (distance_lookup_table[a][i] == distance_lookup_table[b][i])
227 break;
228
229 /* Double the distance for each NUMA level */
230 distance *= 2;
231 }
232
233 return distance;
234}
235
236static void initialize_distance_lookup_table(int nid,
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000237 const __be32 *associativity)
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000238{
239 int i;
240
241 if (!form1_affinity)
242 return;
243
244 for (i = 0; i < distance_ref_points_depth; i++) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000245 const __be32 *entry;
246
247 entry = &associativity[be32_to_cpu(distance_ref_points[i])];
248 distance_lookup_table[nid][i] = of_read_number(entry, 1);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000249 }
250}
251
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600252/* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
253 * info is found.
254 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000255static int associativity_to_nid(const __be32 *associativity)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256{
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600257 int nid = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258
259 if (min_common_depth == -1)
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600260 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000262 if (of_read_number(associativity, 1) >= min_common_depth)
263 nid = of_read_number(&associativity[min_common_depth], 1);
Nathan Lynchbc16a752006-03-20 18:36:15 -0600264
265 /* POWER4 LPAR uses 0xffff as invalid node */
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600266 if (nid == 0xffff || nid >= MAX_NUMNODES)
267 nid = -1;
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000268
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000269 if (nid > 0 &&
270 of_read_number(associativity, 1) >= distance_ref_points_depth)
Jesse Larrew9eff1a32010-12-01 12:31:15 +0000271 initialize_distance_lookup_table(nid, associativity);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000272
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600273out:
Nathan Lynchcf950b72006-03-20 18:35:45 -0600274 return nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275}
276
Jesse Larrew9eff1a32010-12-01 12:31:15 +0000277/* Returns the nid associated with the given device tree node,
278 * or -1 if not found.
279 */
280static int of_node_to_nid_single(struct device_node *device)
281{
282 int nid = -1;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000283 const __be32 *tmp;
Jesse Larrew9eff1a32010-12-01 12:31:15 +0000284
285 tmp = of_get_associativity(device);
286 if (tmp)
287 nid = associativity_to_nid(tmp);
288 return nid;
289}
290
Jeremy Kerr953039c2006-05-01 12:16:12 -0700291/* Walk the device tree upwards, looking for an associativity id */
292int of_node_to_nid(struct device_node *device)
293{
294 struct device_node *tmp;
295 int nid = -1;
296
297 of_node_get(device);
298 while (device) {
299 nid = of_node_to_nid_single(device);
300 if (nid != -1)
301 break;
302
303 tmp = device;
304 device = of_get_parent(tmp);
305 of_node_put(tmp);
306 }
307 of_node_put(device);
308
309 return nid;
310}
311EXPORT_SYMBOL_GPL(of_node_to_nid);
312
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313static int __init find_min_common_depth(void)
314{
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000315 int depth;
Michael Ellermane70606e2011-04-10 20:42:05 +0000316 struct device_node *root;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317
Dipankar Sarma1c8ee732011-10-28 04:25:32 +0000318 if (firmware_has_feature(FW_FEATURE_OPAL))
319 root = of_find_node_by_path("/ibm,opal");
320 else
321 root = of_find_node_by_path("/rtas");
Michael Ellermane70606e2011-04-10 20:42:05 +0000322 if (!root)
323 root = of_find_node_by_path("/");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
325 /*
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000326 * This property is a set of 32-bit integers, each representing
327 * an index into the ibm,associativity nodes.
328 *
329 * With form 0 affinity the first integer is for an SMP configuration
330 * (should be all 0's) and the second is for a normal NUMA
331 * configuration. We have only one level of NUMA.
332 *
333 * With form 1 affinity the first integer is the most significant
334 * NUMA boundary and the following are progressively less significant
335 * boundaries. There can be more than one level of NUMA.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 */
Michael Ellermane70606e2011-04-10 20:42:05 +0000337 distance_ref_points = of_get_property(root,
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000338 "ibm,associativity-reference-points",
339 &distance_ref_points_depth);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000341 if (!distance_ref_points) {
342 dbg("NUMA: ibm,associativity-reference-points not found.\n");
343 goto err;
344 }
345
346 distance_ref_points_depth /= sizeof(int);
347
Nathan Fontenot8002b0c2013-04-24 05:58:23 +0000348 if (firmware_has_feature(FW_FEATURE_OPAL) ||
349 firmware_has_feature(FW_FEATURE_TYPE1_AFFINITY)) {
350 dbg("Using form 1 affinity\n");
Dipankar Sarma1c8ee732011-10-28 04:25:32 +0000351 form1_affinity = 1;
Anton Blanchard4b83c332010-04-07 15:33:44 +0000352 }
353
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000354 if (form1_affinity) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000355 depth = of_read_number(distance_ref_points, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 } else {
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000357 if (distance_ref_points_depth < 2) {
358 printk(KERN_WARNING "NUMA: "
359 "short ibm,associativity-reference-points\n");
360 goto err;
361 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000363 depth = of_read_number(&distance_ref_points[1], 1);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000364 }
365
366 /*
367 * Warn and cap if the hardware supports more than
368 * MAX_DISTANCE_REF_POINTS domains.
369 */
370 if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) {
371 printk(KERN_WARNING "NUMA: distance array capped at "
372 "%d entries\n", MAX_DISTANCE_REF_POINTS);
373 distance_ref_points_depth = MAX_DISTANCE_REF_POINTS;
374 }
375
Michael Ellermane70606e2011-04-10 20:42:05 +0000376 of_node_put(root);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 return depth;
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000378
379err:
Michael Ellermane70606e2011-04-10 20:42:05 +0000380 of_node_put(root);
Anton Blanchard41eab6f2010-05-16 20:22:31 +0000381 return -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382}
383
Mike Kravetz84c9fdd2005-11-30 13:47:23 -0800384static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385{
386 struct device_node *memory = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387
388 memory = of_find_node_by_type(memory, "memory");
Paul Mackerras54c23312005-12-05 15:50:39 +1100389 if (!memory)
Mike Kravetz84c9fdd2005-11-30 13:47:23 -0800390 panic("numa.c: No memory nodes found!");
Paul Mackerras54c23312005-12-05 15:50:39 +1100391
Stephen Rothwella8bda5d2007-04-03 10:56:50 +1000392 *n_addr_cells = of_n_addr_cells(memory);
Stephen Rothwell9213fee2007-04-03 10:57:48 +1000393 *n_size_cells = of_n_size_cells(memory);
Mike Kravetz84c9fdd2005-11-30 13:47:23 -0800394 of_node_put(memory);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395}
396
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000397static unsigned long read_n_cells(int n, const __be32 **buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398{
399 unsigned long result = 0;
400
401 while (n--) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000402 result = (result << 32) | of_read_number(*buf, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403 (*buf)++;
404 }
405 return result;
406}
407
Nathan Fontenot83426812008-07-03 13:35:54 +1000408/*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000409 * Read the next memblock list entry from the ibm,dynamic-memory property
Nathan Fontenot83426812008-07-03 13:35:54 +1000410 * and return the information in the provided of_drconf_cell structure.
411 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000412static void read_drconf_cell(struct of_drconf_cell *drmem, const __be32 **cellp)
Nathan Fontenot83426812008-07-03 13:35:54 +1000413{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000414 const __be32 *cp;
Nathan Fontenot83426812008-07-03 13:35:54 +1000415
416 drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp);
417
418 cp = *cellp;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000419 drmem->drc_index = of_read_number(cp, 1);
420 drmem->reserved = of_read_number(&cp[1], 1);
421 drmem->aa_index = of_read_number(&cp[2], 1);
422 drmem->flags = of_read_number(&cp[3], 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000423
424 *cellp = cp + 4;
425}
426
427/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300428 * Retrieve and validate the ibm,dynamic-memory property of the device tree.
Nathan Fontenot83426812008-07-03 13:35:54 +1000429 *
Yinghai Lu95f72d12010-07-12 14:36:09 +1000430 * The layout of the ibm,dynamic-memory property is a number N of memblock
431 * list entries followed by N memblock list entries. Each memblock list entry
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300432 * contains information as laid out in the of_drconf_cell struct above.
Nathan Fontenot83426812008-07-03 13:35:54 +1000433 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000434static int of_get_drconf_memory(struct device_node *memory, const __be32 **dm)
Nathan Fontenot83426812008-07-03 13:35:54 +1000435{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000436 const __be32 *prop;
Nathan Fontenot83426812008-07-03 13:35:54 +1000437 u32 len, entries;
438
439 prop = of_get_property(memory, "ibm,dynamic-memory", &len);
440 if (!prop || len < sizeof(unsigned int))
441 return 0;
442
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000443 entries = of_read_number(prop++, 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000444
445 /* Now that we know the number of entries, revalidate the size
446 * of the property read in to ensure we have everything
447 */
448 if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int))
449 return 0;
450
451 *dm = prop;
452 return entries;
453}
454
455/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300456 * Retrieve and validate the ibm,lmb-size property for drconf memory
Nathan Fontenot83426812008-07-03 13:35:54 +1000457 * from the device tree.
458 */
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000459static u64 of_get_lmb_size(struct device_node *memory)
Nathan Fontenot83426812008-07-03 13:35:54 +1000460{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000461 const __be32 *prop;
Nathan Fontenot83426812008-07-03 13:35:54 +1000462 u32 len;
463
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000464 prop = of_get_property(memory, "ibm,lmb-size", &len);
Nathan Fontenot83426812008-07-03 13:35:54 +1000465 if (!prop || len < sizeof(unsigned int))
466 return 0;
467
468 return read_n_cells(n_mem_size_cells, &prop);
469}
470
471struct assoc_arrays {
472 u32 n_arrays;
473 u32 array_sz;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000474 const __be32 *arrays;
Nathan Fontenot83426812008-07-03 13:35:54 +1000475};
476
477/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300478 * Retrieve and validate the list of associativity arrays for drconf
Nathan Fontenot83426812008-07-03 13:35:54 +1000479 * memory from the ibm,associativity-lookup-arrays property of the
480 * device tree..
481 *
482 * The layout of the ibm,associativity-lookup-arrays property is a number N
483 * indicating the number of associativity arrays, followed by a number M
484 * indicating the size of each associativity array, followed by a list
485 * of N associativity arrays.
486 */
487static int of_get_assoc_arrays(struct device_node *memory,
488 struct assoc_arrays *aa)
489{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000490 const __be32 *prop;
Nathan Fontenot83426812008-07-03 13:35:54 +1000491 u32 len;
492
493 prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len);
494 if (!prop || len < 2 * sizeof(unsigned int))
495 return -1;
496
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000497 aa->n_arrays = of_read_number(prop++, 1);
498 aa->array_sz = of_read_number(prop++, 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000499
Justin P. Mattock42b2aa82011-11-28 20:31:00 -0800500 /* Now that we know the number of arrays and size of each array,
Nathan Fontenot83426812008-07-03 13:35:54 +1000501 * revalidate the size of the property read in.
502 */
503 if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int))
504 return -1;
505
506 aa->arrays = prop;
507 return 0;
508}
509
510/*
511 * This is like of_node_to_nid_single() for memory represented in the
512 * ibm,dynamic-reconfiguration-memory node.
513 */
514static int of_drconf_to_nid_single(struct of_drconf_cell *drmem,
515 struct assoc_arrays *aa)
516{
517 int default_nid = 0;
518 int nid = default_nid;
519 int index;
520
521 if (min_common_depth > 0 && min_common_depth <= aa->array_sz &&
522 !(drmem->flags & DRCONF_MEM_AI_INVALID) &&
523 drmem->aa_index < aa->n_arrays) {
524 index = drmem->aa_index * aa->array_sz + min_common_depth - 1;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000525 nid = of_read_number(&aa->arrays[index], 1);
Nathan Fontenot83426812008-07-03 13:35:54 +1000526
527 if (nid == 0xffff || nid >= MAX_NUMNODES)
528 nid = default_nid;
529 }
530
531 return nid;
532}
533
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534/*
535 * Figure out to which domain a cpu belongs and stick it there.
536 * Return the id of the domain used.
537 */
Paul Gortmaker061d19f2013-06-24 15:30:09 -0400538static int numa_setup_cpu(unsigned long lcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539{
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530540 int nid;
541 struct device_node *cpu;
542
543 /*
544 * If a valid cpu-to-node mapping is already available, use it
545 * directly instead of querying the firmware, since it represents
546 * the most recent mapping notified to us by the platform (eg: VPHN).
547 */
548 if ((nid = numa_cpu_lookup_table[lcpu]) >= 0) {
549 map_cpu_to_node(lcpu, nid);
550 return nid;
551 }
552
553 cpu = of_get_cpu_node(lcpu, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554
555 if (!cpu) {
556 WARN_ON(1);
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +0530557 nid = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558 goto out;
559 }
560
Jeremy Kerr953039c2006-05-01 12:16:12 -0700561 nid = of_node_to_nid_single(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600563 if (nid < 0 || !node_online(nid))
H Hartley Sweeten72c33682010-03-05 13:42:43 -0800564 nid = first_online_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565out:
Nathan Lynchcf950b72006-03-20 18:35:45 -0600566 map_cpu_to_node(lcpu, nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567
568 of_node_put(cpu);
569
Nathan Lynchcf950b72006-03-20 18:35:45 -0600570 return nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571}
572
Paul Gortmaker061d19f2013-06-24 15:30:09 -0400573static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 void *hcpu)
575{
576 unsigned long lcpu = (unsigned long)hcpu;
577 int ret = NOTIFY_DONE;
578
579 switch (action) {
580 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700581 case CPU_UP_PREPARE_FROZEN:
Nathan Lynch2b261222006-03-20 18:37:15 -0600582 numa_setup_cpu(lcpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 ret = NOTIFY_OK;
584 break;
585#ifdef CONFIG_HOTPLUG_CPU
586 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700587 case CPU_DEAD_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700589 case CPU_UP_CANCELED_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 unmap_cpu_from_node(lcpu);
591 break;
592 ret = NOTIFY_OK;
593#endif
594 }
595 return ret;
596}
597
598/*
599 * Check and possibly modify a memory region to enforce the memory limit.
600 *
601 * Returns the size the region should have to enforce the memory limit.
602 * This will either be the original value of size, a truncated value,
603 * or zero. If the returned value of size is 0 the region should be
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300604 * discarded as it lies wholly above the memory limit.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 */
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100606static unsigned long __init numa_enforce_memory_limit(unsigned long start,
607 unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608{
609 /*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000610 * We use memblock_end_of_DRAM() in here instead of memory_limit because
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 * we've already adjusted it for the limit and it takes care of
Milton Millerfe552492008-10-20 15:37:04 +0000612 * having memory holes below the limit. Also, in the case of
613 * iommu_is_off, memory_limit is not set but is implicitly enforced.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615
Yinghai Lu95f72d12010-07-12 14:36:09 +1000616 if (start + size <= memblock_end_of_DRAM())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 return size;
618
Yinghai Lu95f72d12010-07-12 14:36:09 +1000619 if (start >= memblock_end_of_DRAM())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 return 0;
621
Yinghai Lu95f72d12010-07-12 14:36:09 +1000622 return memblock_end_of_DRAM() - start;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623}
624
Paul Mackerras02045682006-11-29 22:27:42 +1100625/*
Chandrucf000852008-08-30 00:28:16 +1000626 * Reads the counter for a given entry in
627 * linux,drconf-usable-memory property
628 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000629static inline int __init read_usm_ranges(const __be32 **usm)
Chandrucf000852008-08-30 00:28:16 +1000630{
631 /*
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000632 * For each lmb in ibm,dynamic-memory a corresponding
Chandrucf000852008-08-30 00:28:16 +1000633 * entry in linux,drconf-usable-memory property contains
634 * a counter followed by that many (base, size) duple.
635 * read the counter from linux,drconf-usable-memory
636 */
637 return read_n_cells(n_mem_size_cells, usm);
638}
639
640/*
Paul Mackerras02045682006-11-29 22:27:42 +1100641 * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
642 * node. This assumes n_mem_{addr,size}_cells have been set.
643 */
644static void __init parse_drconf_memory(struct device_node *memory)
645{
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000646 const __be32 *uninitialized_var(dm), *usm;
Chandrucf000852008-08-30 00:28:16 +1000647 unsigned int n, rc, ranges, is_kexec_kdump = 0;
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000648 unsigned long lmb_size, base, size, sz;
Nathan Fontenot83426812008-07-03 13:35:54 +1000649 int nid;
Benjamin Herrenschmidtaa709f32012-07-05 16:30:33 +0000650 struct assoc_arrays aa = { .arrays = NULL };
Paul Mackerras02045682006-11-29 22:27:42 +1100651
Nathan Fontenot83426812008-07-03 13:35:54 +1000652 n = of_get_drconf_memory(memory, &dm);
653 if (!n)
Paul Mackerras02045682006-11-29 22:27:42 +1100654 return;
655
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000656 lmb_size = of_get_lmb_size(memory);
657 if (!lmb_size)
Nathan Fontenot83426812008-07-03 13:35:54 +1000658 return;
659
660 rc = of_get_assoc_arrays(memory, &aa);
661 if (rc)
Paul Mackerras02045682006-11-29 22:27:42 +1100662 return;
663
Chandrucf000852008-08-30 00:28:16 +1000664 /* check if this is a kexec/kdump kernel */
665 usm = of_get_usable_memory(memory);
666 if (usm != NULL)
667 is_kexec_kdump = 1;
668
Paul Mackerras02045682006-11-29 22:27:42 +1100669 for (; n != 0; --n) {
Nathan Fontenot83426812008-07-03 13:35:54 +1000670 struct of_drconf_cell drmem;
Balbir Singh1daa6d02008-02-01 15:57:31 +1100671
Nathan Fontenot83426812008-07-03 13:35:54 +1000672 read_drconf_cell(&drmem, &dm);
673
674 /* skip this block if the reserved bit is set in flags (0x80)
675 or if the block is not assigned to this partition (0x8) */
676 if ((drmem.flags & DRCONF_MEM_RESERVED)
677 || !(drmem.flags & DRCONF_MEM_ASSIGNED))
678 continue;
679
Chandrucf000852008-08-30 00:28:16 +1000680 base = drmem.base_addr;
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +1000681 size = lmb_size;
Chandrucf000852008-08-30 00:28:16 +1000682 ranges = 1;
Nathan Fontenot83426812008-07-03 13:35:54 +1000683
Chandrucf000852008-08-30 00:28:16 +1000684 if (is_kexec_kdump) {
685 ranges = read_usm_ranges(&usm);
686 if (!ranges) /* there are no (base, size) duple */
687 continue;
688 }
689 do {
690 if (is_kexec_kdump) {
691 base = read_n_cells(n_mem_addr_cells, &usm);
692 size = read_n_cells(n_mem_size_cells, &usm);
693 }
694 nid = of_drconf_to_nid_single(&drmem, &aa);
695 fake_numa_create_new_node(
696 ((base + size) >> PAGE_SHIFT),
Nathan Fontenot83426812008-07-03 13:35:54 +1000697 &nid);
Chandrucf000852008-08-30 00:28:16 +1000698 node_set_online(nid);
699 sz = numa_enforce_memory_limit(base, size);
700 if (sz)
Tejun Heo1d7cfe12011-12-08 10:22:08 -0800701 memblock_set_node(base, sz, nid);
Chandrucf000852008-08-30 00:28:16 +1000702 } while (--ranges);
Paul Mackerras02045682006-11-29 22:27:42 +1100703 }
704}
705
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706static int __init parse_numa_properties(void)
707{
Anton Blanchard94db7c52011-08-10 20:44:22 +0000708 struct device_node *memory;
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600709 int default_nid = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 unsigned long i;
711
712 if (numa_enabled == 0) {
713 printk(KERN_WARNING "NUMA disabled by user\n");
714 return -1;
715 }
716
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 min_common_depth = find_min_common_depth();
718
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 if (min_common_depth < 0)
720 return min_common_depth;
721
Nathan Lynchbf4b85b2006-03-20 18:34:45 -0600722 dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);
723
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 /*
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600725 * Even though we connect cpus to numa domains later in SMP
726 * init, we need to know the node ids now. This is because
727 * each node to be onlined must have NODE_DATA etc backing it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 */
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600729 for_each_present_cpu(i) {
Anton Blancharddfbe93a2011-08-10 20:44:23 +0000730 struct device_node *cpu;
Nathan Lynchcf950b72006-03-20 18:35:45 -0600731 int nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
Milton Miller8b16cd22009-01-08 02:19:45 +0000733 cpu = of_get_cpu_node(i, NULL);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600734 BUG_ON(!cpu);
Jeremy Kerr953039c2006-05-01 12:16:12 -0700735 nid = of_node_to_nid_single(cpu);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600736 of_node_put(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600738 /*
739 * Don't fall back to default_nid yet -- we will plug
740 * cpus into nodes once the memory scan has discovered
741 * the topology.
742 */
743 if (nid < 0)
744 continue;
745 node_set_online(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 }
747
Mike Kravetz237a09892005-12-05 12:06:42 -0800748 get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells);
Anton Blanchard94db7c52011-08-10 20:44:22 +0000749
750 for_each_node_by_type(memory, "memory") {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 unsigned long start;
752 unsigned long size;
Nathan Lynchcf950b72006-03-20 18:35:45 -0600753 int nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 int ranges;
Alistair Poppleb08a2a12013-08-07 02:01:44 +1000755 const __be32 *memcell_buf;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 unsigned int len;
757
Stephen Rothwelle2eb6392007-04-03 22:26:41 +1000758 memcell_buf = of_get_property(memory,
Michael Ellermanba759482005-12-04 18:39:55 +1100759 "linux,usable-memory", &len);
760 if (!memcell_buf || len <= 0)
Stephen Rothwelle2eb6392007-04-03 22:26:41 +1000761 memcell_buf = of_get_property(memory, "reg", &len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 if (!memcell_buf || len <= 0)
763 continue;
764
Benjamin Herrenschmidtcc5d0182005-12-13 18:01:21 +1100765 /* ranges in cell */
766 ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767new_range:
768 /* these are order-sensitive, and modify the buffer pointer */
Mike Kravetz237a09892005-12-05 12:06:42 -0800769 start = read_n_cells(n_mem_addr_cells, &memcell_buf);
770 size = read_n_cells(n_mem_size_cells, &memcell_buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600772 /*
773 * Assumption: either all memory nodes or none will
774 * have associativity properties. If none, then
775 * everything goes to default_nid.
776 */
Jeremy Kerr953039c2006-05-01 12:16:12 -0700777 nid = of_node_to_nid_single(memory);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600778 if (nid < 0)
779 nid = default_nid;
Balbir Singh1daa6d02008-02-01 15:57:31 +1100780
781 fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid);
Nathan Lynch482ec7c2006-03-20 18:36:45 -0600782 node_set_online(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100784 if (!(size = numa_enforce_memory_limit(start, size))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785 if (--ranges)
786 goto new_range;
787 else
788 continue;
789 }
790
Tejun Heo1d7cfe12011-12-08 10:22:08 -0800791 memblock_set_node(start, size, nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792
793 if (--ranges)
794 goto new_range;
795 }
796
Paul Mackerras02045682006-11-29 22:27:42 +1100797 /*
Anton Blancharddfbe93a2011-08-10 20:44:23 +0000798 * Now do the same thing for each MEMBLOCK listed in the
799 * ibm,dynamic-memory property in the
800 * ibm,dynamic-reconfiguration-memory node.
Paul Mackerras02045682006-11-29 22:27:42 +1100801 */
802 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
803 if (memory)
804 parse_drconf_memory(memory);
805
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 return 0;
807}
808
809static void __init setup_nonnuma(void)
810{
Yinghai Lu95f72d12010-07-12 14:36:09 +1000811 unsigned long top_of_ram = memblock_end_of_DRAM();
812 unsigned long total_ram = memblock_phys_mem_size();
Mel Gormanc67c3cb2006-09-27 01:49:49 -0700813 unsigned long start_pfn, end_pfn;
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000814 unsigned int nid = 0;
815 struct memblock_region *reg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816
Olof Johanssone110b282006-04-12 15:25:01 -0500817 printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818 top_of_ram, total_ram);
Olof Johanssone110b282006-04-12 15:25:01 -0500819 printk(KERN_DEBUG "Memory hole size: %ldMB\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 (top_of_ram - total_ram) >> 20);
821
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000822 for_each_memblock(memory, reg) {
Yinghai Luc7fc2de2010-10-12 14:07:09 -0700823 start_pfn = memblock_region_memory_base_pfn(reg);
824 end_pfn = memblock_region_memory_end_pfn(reg);
Balbir Singh1daa6d02008-02-01 15:57:31 +1100825
826 fake_numa_create_new_node(end_pfn, &nid);
Tejun Heo1d7cfe12011-12-08 10:22:08 -0800827 memblock_set_node(PFN_PHYS(start_pfn),
828 PFN_PHYS(end_pfn - start_pfn), nid);
Balbir Singh1daa6d02008-02-01 15:57:31 +1100829 node_set_online(nid);
Mel Gormanc67c3cb2006-09-27 01:49:49 -0700830 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831}
832
Anton Blanchard4b703a22005-12-13 06:56:47 +1100833void __init dump_numa_cpu_topology(void)
834{
835 unsigned int node;
836 unsigned int cpu, count;
837
838 if (min_common_depth == -1 || !numa_enabled)
839 return;
840
841 for_each_online_node(node) {
Olof Johanssone110b282006-04-12 15:25:01 -0500842 printk(KERN_DEBUG "Node %d CPUs:", node);
Anton Blanchard4b703a22005-12-13 06:56:47 +1100843
844 count = 0;
845 /*
846 * If we used a CPU iterator here we would miss printing
847 * the holes in the cpumap.
848 */
Anton Blanchard25863de2010-04-26 15:32:43 +0000849 for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
850 if (cpumask_test_cpu(cpu,
851 node_to_cpumask_map[node])) {
Anton Blanchard4b703a22005-12-13 06:56:47 +1100852 if (count == 0)
853 printk(" %u", cpu);
854 ++count;
855 } else {
856 if (count > 1)
857 printk("-%u", cpu - 1);
858 count = 0;
859 }
860 }
861
862 if (count > 1)
Anton Blanchard25863de2010-04-26 15:32:43 +0000863 printk("-%u", nr_cpu_ids - 1);
Anton Blanchard4b703a22005-12-13 06:56:47 +1100864 printk("\n");
865 }
866}
867
868static void __init dump_numa_memory_topology(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869{
870 unsigned int node;
871 unsigned int count;
872
873 if (min_common_depth == -1 || !numa_enabled)
874 return;
875
876 for_each_online_node(node) {
877 unsigned long i;
878
Olof Johanssone110b282006-04-12 15:25:01 -0500879 printk(KERN_DEBUG "Node %d Memory:", node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880
881 count = 0;
882
Yinghai Lu95f72d12010-07-12 14:36:09 +1000883 for (i = 0; i < memblock_end_of_DRAM();
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100884 i += (1 << SECTION_SIZE_BITS)) {
885 if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 if (count == 0)
887 printk(" 0x%lx", i);
888 ++count;
889 } else {
890 if (count > 0)
891 printk("-0x%lx", i);
892 count = 0;
893 }
894 }
895
896 if (count > 0)
897 printk("-0x%lx", i);
898 printk("\n");
899 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900}
901
902/*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000903 * Allocate some memory, satisfying the memblock or bootmem allocator where
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 * required. nid is the preferred node and end is the physical address of
905 * the highest address in the node.
906 *
Dave Hansen0be210f2008-12-09 08:21:35 +0000907 * Returns the virtual address of the memory.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 */
Dave Hansen893473d2008-12-09 08:21:36 +0000909static void __init *careful_zallocation(int nid, unsigned long size,
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100910 unsigned long align,
911 unsigned long end_pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912{
Dave Hansen0be210f2008-12-09 08:21:35 +0000913 void *ret;
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100914 int new_nid;
Dave Hansen0be210f2008-12-09 08:21:35 +0000915 unsigned long ret_paddr;
916
Yinghai Lu95f72d12010-07-12 14:36:09 +1000917 ret_paddr = __memblock_alloc_base(size, align, end_pfn << PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918
919 /* retry over all memory */
Dave Hansen0be210f2008-12-09 08:21:35 +0000920 if (!ret_paddr)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000921 ret_paddr = __memblock_alloc_base(size, align, memblock_end_of_DRAM());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922
Dave Hansen0be210f2008-12-09 08:21:35 +0000923 if (!ret_paddr)
Dave Hansen5d21ea22008-12-09 08:21:33 +0000924 panic("numa.c: cannot allocate %lu bytes for node %d",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925 size, nid);
926
Dave Hansen0be210f2008-12-09 08:21:35 +0000927 ret = __va(ret_paddr);
928
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 /*
Dave Hansenc555e5202008-12-09 08:21:32 +0000930 * We initialize the nodes in numeric order: 0, 1, 2...
Yinghai Lu95f72d12010-07-12 14:36:09 +1000931 * and hand over control from the MEMBLOCK allocator to the
Dave Hansenc555e5202008-12-09 08:21:32 +0000932 * bootmem allocator. If this function is called for
933 * node 5, then we know that all nodes <5 are using the
Yinghai Lu95f72d12010-07-12 14:36:09 +1000934 * bootmem allocator instead of the MEMBLOCK allocator.
Dave Hansenc555e5202008-12-09 08:21:32 +0000935 *
936 * So, check the nid from which this allocation came
937 * and double check to see if we need to use bootmem
Yinghai Lu95f72d12010-07-12 14:36:09 +1000938 * instead of the MEMBLOCK. We don't free the MEMBLOCK memory
Dave Hansenc555e5202008-12-09 08:21:32 +0000939 * since it would be useless.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 */
Dave Hansen0be210f2008-12-09 08:21:35 +0000941 new_nid = early_pfn_to_nid(ret_paddr >> PAGE_SHIFT);
Anton Blanchard45fb6ce2005-11-11 14:22:35 +1100942 if (new_nid < nid) {
Dave Hansen0be210f2008-12-09 08:21:35 +0000943 ret = __alloc_bootmem_node(NODE_DATA(new_nid),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 size, align, 0);
945
Dave Hansen0be210f2008-12-09 08:21:35 +0000946 dbg("alloc_bootmem %p %lx\n", ret, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947 }
948
Dave Hansen893473d2008-12-09 08:21:36 +0000949 memset(ret, 0, size);
Dave Hansen0be210f2008-12-09 08:21:35 +0000950 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951}
952
Paul Gortmaker061d19f2013-06-24 15:30:09 -0400953static struct notifier_block ppc64_numa_nb = {
Chandra Seetharaman74b85f32006-06-27 02:54:09 -0700954 .notifier_call = cpu_numa_callback,
955 .priority = 1 /* Must run before sched domains notifier. */
956};
957
David Rientjes28e86bd2011-12-08 12:33:29 +0000958static void __init mark_reserved_regions_for_nid(int nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959{
Dave Hansen4a618662008-11-24 12:02:35 +0000960 struct pglist_data *node = NODE_DATA(nid);
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000961 struct memblock_region *reg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
Benjamin Herrenschmidt28be7072010-08-04 13:43:53 +1000963 for_each_memblock(reserved, reg) {
964 unsigned long physbase = reg->base;
965 unsigned long size = reg->size;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000966 unsigned long start_pfn = physbase >> PAGE_SHIFT;
Dave Hansen06eccea2009-02-12 12:36:04 +0000967 unsigned long end_pfn = PFN_UP(physbase + size);
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000968 struct node_active_region node_ar;
Xishi Qiu64080682013-11-12 15:07:17 -0800969 unsigned long node_end_pfn = pgdat_end_pfn(node);
Dave Hansen4a618662008-11-24 12:02:35 +0000970
971 /*
Yinghai Lu95f72d12010-07-12 14:36:09 +1000972 * Check to make sure that this memblock.reserved area is
Dave Hansen4a618662008-11-24 12:02:35 +0000973 * within the bounds of the node that we care about.
974 * Checking the nid of the start and end points is not
975 * sufficient because the reserved area could span the
976 * entire node.
977 */
978 if (end_pfn <= node->node_start_pfn ||
979 start_pfn >= node_end_pfn)
980 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000982 get_node_active_region(start_pfn, &node_ar);
Jon Tollefsone8170372008-10-16 18:59:43 +0000983 while (start_pfn < end_pfn &&
984 node_ar.start_pfn < node_ar.end_pfn) {
985 unsigned long reserve_size = size;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +0000986 /*
987 * if reserved region extends past active region
988 * then trim size to active region
989 */
990 if (end_pfn > node_ar.end_pfn)
Jon Tollefsone8170372008-10-16 18:59:43 +0000991 reserve_size = (node_ar.end_pfn << PAGE_SHIFT)
Dave Hansen06eccea2009-02-12 12:36:04 +0000992 - physbase;
Dave Hansena4c74dd2008-12-11 08:36:06 +0000993 /*
994 * Only worry about *this* node, others may not
995 * yet have valid NODE_DATA().
996 */
997 if (node_ar.nid == nid) {
998 dbg("reserve_bootmem %lx %lx nid=%d\n",
999 physbase, reserve_size, node_ar.nid);
1000 reserve_bootmem_node(NODE_DATA(node_ar.nid),
1001 physbase, reserve_size,
1002 BOOTMEM_DEFAULT);
1003 }
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001004 /*
1005 * if reserved region is contained in the active region
1006 * then done.
1007 */
1008 if (end_pfn <= node_ar.end_pfn)
1009 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001011 /*
1012 * reserved region extends past the active region
1013 * get next active region that contains this
1014 * reserved region
1015 */
1016 start_pfn = node_ar.end_pfn;
1017 physbase = start_pfn << PAGE_SHIFT;
Jon Tollefsone8170372008-10-16 18:59:43 +00001018 size = size - reserve_size;
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001019 get_node_active_region(start_pfn, &node_ar);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021 }
Dave Hansen4a618662008-11-24 12:02:35 +00001022}
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001023
Dave Hansen4a618662008-11-24 12:02:35 +00001024
1025void __init do_init_bootmem(void)
1026{
1027 int nid;
Dave Hansen4a618662008-11-24 12:02:35 +00001028
1029 min_low_pfn = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001030 max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
Dave Hansen4a618662008-11-24 12:02:35 +00001031 max_pfn = max_low_pfn;
1032
1033 if (parse_numa_properties())
1034 setup_nonnuma();
1035 else
1036 dump_numa_memory_topology();
1037
Dave Hansen4a618662008-11-24 12:02:35 +00001038 for_each_online_node(nid) {
1039 unsigned long start_pfn, end_pfn;
Dave Hansen0be210f2008-12-09 08:21:35 +00001040 void *bootmem_vaddr;
Dave Hansen4a618662008-11-24 12:02:35 +00001041 unsigned long bootmap_pages;
1042
1043 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
1044
1045 /*
1046 * Allocate the node structure node local if possible
1047 *
1048 * Be careful moving this around, as it relies on all
1049 * previous nodes' bootmem to be initialized and have
1050 * all reserved areas marked.
1051 */
Dave Hansen893473d2008-12-09 08:21:36 +00001052 NODE_DATA(nid) = careful_zallocation(nid,
Dave Hansen4a618662008-11-24 12:02:35 +00001053 sizeof(struct pglist_data),
1054 SMP_CACHE_BYTES, end_pfn);
Dave Hansen4a618662008-11-24 12:02:35 +00001055
1056 dbg("node %d\n", nid);
1057 dbg("NODE_DATA() = %p\n", NODE_DATA(nid));
1058
1059 NODE_DATA(nid)->bdata = &bootmem_node_data[nid];
1060 NODE_DATA(nid)->node_start_pfn = start_pfn;
1061 NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
1062
1063 if (NODE_DATA(nid)->node_spanned_pages == 0)
1064 continue;
1065
1066 dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT);
1067 dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT);
1068
1069 bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
Dave Hansen893473d2008-12-09 08:21:36 +00001070 bootmem_vaddr = careful_zallocation(nid,
Dave Hansen4a618662008-11-24 12:02:35 +00001071 bootmap_pages << PAGE_SHIFT,
1072 PAGE_SIZE, end_pfn);
Dave Hansen4a618662008-11-24 12:02:35 +00001073
Dave Hansen0be210f2008-12-09 08:21:35 +00001074 dbg("bootmap_vaddr = %p\n", bootmem_vaddr);
Dave Hansen4a618662008-11-24 12:02:35 +00001075
Dave Hansen0be210f2008-12-09 08:21:35 +00001076 init_bootmem_node(NODE_DATA(nid),
1077 __pa(bootmem_vaddr) >> PAGE_SHIFT,
Dave Hansen4a618662008-11-24 12:02:35 +00001078 start_pfn, end_pfn);
1079
1080 free_bootmem_with_active_regions(nid, end_pfn);
1081 /*
1082 * Be very careful about moving this around. Future
Dave Hansen893473d2008-12-09 08:21:36 +00001083 * calls to careful_zallocation() depend on this getting
Dave Hansen4a618662008-11-24 12:02:35 +00001084 * done correctly.
1085 */
1086 mark_reserved_regions_for_nid(nid);
Jon Tollefson8f64e1f2008-10-09 10:18:40 +00001087 sparse_memory_present_with_active_regions(nid);
Dave Hansen4a618662008-11-24 12:02:35 +00001088 }
Benjamin Herrenschmidtd3f62042009-06-02 21:16:38 +00001089
1090 init_bootmem_done = 1;
Anton Blanchard25863de2010-04-26 15:32:43 +00001091
1092 /*
1093 * Now bootmem is initialised we can create the node to cpumask
1094 * lookup tables and setup the cpu callback to populate them.
1095 */
1096 setup_node_to_cpumask_map();
1097
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +05301098 reset_numa_cpu_lookup_table();
Anton Blanchard25863de2010-04-26 15:32:43 +00001099 register_cpu_notifier(&ppc64_numa_nb);
1100 cpu_numa_callback(&ppc64_numa_nb, CPU_UP_PREPARE,
1101 (void *)(unsigned long)boot_cpuid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102}
1103
1104void __init paging_init(void)
1105{
Mel Gorman6391af12006-10-11 01:20:39 -07001106 unsigned long max_zone_pfns[MAX_NR_ZONES];
1107 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
Yinghai Lu95f72d12010-07-12 14:36:09 +10001108 max_zone_pfns[ZONE_DMA] = memblock_end_of_DRAM() >> PAGE_SHIFT;
Mel Gormanc67c3cb2006-09-27 01:49:49 -07001109 free_area_init_nodes(max_zone_pfns);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110}
1111
1112static int __init early_numa(char *p)
1113{
1114 if (!p)
1115 return 0;
1116
1117 if (strstr(p, "off"))
1118 numa_enabled = 0;
1119
1120 if (strstr(p, "debug"))
1121 numa_debug = 1;
1122
Balbir Singh1daa6d02008-02-01 15:57:31 +11001123 p = strstr(p, "fake=");
1124 if (p)
1125 cmdline = p + strlen("fake=");
1126
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 return 0;
1128}
1129early_param("numa", early_numa);
Mike Kravetz237a09892005-12-05 12:06:42 -08001130
1131#ifdef CONFIG_MEMORY_HOTPLUG
1132/*
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001133 * Find the node associated with a hot added memory section for
1134 * memory represented in the device tree by the property
1135 * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory.
Nathan Fontenot0db93602008-07-03 13:25:08 +10001136 */
1137static int hot_add_drconf_scn_to_nid(struct device_node *memory,
1138 unsigned long scn_addr)
1139{
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001140 const __be32 *dm;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001141 unsigned int drconf_cell_cnt, rc;
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +10001142 unsigned long lmb_size;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001143 struct assoc_arrays aa;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001144 int nid = -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001145
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001146 drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
1147 if (!drconf_cell_cnt)
1148 return -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001149
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +10001150 lmb_size = of_get_lmb_size(memory);
1151 if (!lmb_size)
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001152 return -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001153
1154 rc = of_get_assoc_arrays(memory, &aa);
1155 if (rc)
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001156 return -1;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001157
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001158 for (; drconf_cell_cnt != 0; --drconf_cell_cnt) {
Nathan Fontenot0db93602008-07-03 13:25:08 +10001159 struct of_drconf_cell drmem;
1160
1161 read_drconf_cell(&drmem, &dm);
1162
1163 /* skip this block if it is reserved or not assigned to
1164 * this partition */
1165 if ((drmem.flags & DRCONF_MEM_RESERVED)
1166 || !(drmem.flags & DRCONF_MEM_ASSIGNED))
1167 continue;
1168
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001169 if ((scn_addr < drmem.base_addr)
Benjamin Herrenschmidt3fdfd992010-07-23 10:35:52 +10001170 || (scn_addr >= (drmem.base_addr + lmb_size)))
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001171 continue;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001172
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001173 nid = of_drconf_to_nid_single(&drmem, &aa);
1174 break;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001175 }
1176
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001177 return nid;
Nathan Fontenot0db93602008-07-03 13:25:08 +10001178}
1179
1180/*
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001181 * Find the node associated with a hot added memory section for memory
1182 * represented in the device tree as a node (i.e. memory@XXXX) for
Yinghai Lu95f72d12010-07-12 14:36:09 +10001183 * each memblock.
Mike Kravetz237a09892005-12-05 12:06:42 -08001184 */
Robert Jenningsec32dd62013-10-28 09:20:50 -05001185static int hot_add_node_scn_to_nid(unsigned long scn_addr)
Mike Kravetz237a09892005-12-05 12:06:42 -08001186{
Anton Blanchard94db7c52011-08-10 20:44:22 +00001187 struct device_node *memory;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001188 int nid = -1;
Mike Kravetz237a09892005-12-05 12:06:42 -08001189
Anton Blanchard94db7c52011-08-10 20:44:22 +00001190 for_each_node_by_type(memory, "memory") {
Mike Kravetz237a09892005-12-05 12:06:42 -08001191 unsigned long start, size;
Mike Kravetzb226e462005-12-16 14:30:35 -08001192 int ranges;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001193 const __be32 *memcell_buf;
Mike Kravetz237a09892005-12-05 12:06:42 -08001194 unsigned int len;
1195
Stephen Rothwelle2eb6392007-04-03 22:26:41 +10001196 memcell_buf = of_get_property(memory, "reg", &len);
Mike Kravetz237a09892005-12-05 12:06:42 -08001197 if (!memcell_buf || len <= 0)
1198 continue;
1199
Benjamin Herrenschmidtcc5d0182005-12-13 18:01:21 +11001200 /* ranges in cell */
1201 ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
Mike Kravetz237a09892005-12-05 12:06:42 -08001202
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001203 while (ranges--) {
1204 start = read_n_cells(n_mem_addr_cells, &memcell_buf);
1205 size = read_n_cells(n_mem_size_cells, &memcell_buf);
1206
1207 if ((scn_addr < start) || (scn_addr >= (start + size)))
1208 continue;
1209
1210 nid = of_node_to_nid_single(memory);
1211 break;
Mike Kravetz237a09892005-12-05 12:06:42 -08001212 }
1213
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001214 if (nid >= 0)
1215 break;
Mike Kravetz237a09892005-12-05 12:06:42 -08001216 }
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001217
Anton Blanchard60831842011-08-10 20:44:21 +00001218 of_node_put(memory);
1219
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001220 return nid;
Mike Kravetz237a09892005-12-05 12:06:42 -08001221}
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001222
1223/*
1224 * Find the node associated with a hot added memory section. Section
Yinghai Lu95f72d12010-07-12 14:36:09 +10001225 * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that
1226 * sections are fully contained within a single MEMBLOCK.
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001227 */
1228int hot_add_scn_to_nid(unsigned long scn_addr)
1229{
1230 struct device_node *memory = NULL;
1231 int nid, found = 0;
1232
1233 if (!numa_enabled || (min_common_depth < 0))
H Hartley Sweeten72c33682010-03-05 13:42:43 -08001234 return first_online_node;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001235
1236 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1237 if (memory) {
1238 nid = hot_add_drconf_scn_to_nid(memory, scn_addr);
1239 of_node_put(memory);
1240 } else {
1241 nid = hot_add_node_scn_to_nid(scn_addr);
1242 }
1243
1244 if (nid < 0 || !node_online(nid))
H Hartley Sweeten72c33682010-03-05 13:42:43 -08001245 nid = first_online_node;
Nathan Fontenot0f16ef72009-02-17 08:08:30 +00001246
1247 if (NODE_DATA(nid)->node_spanned_pages)
1248 return nid;
1249
1250 for_each_online_node(nid) {
1251 if (NODE_DATA(nid)->node_spanned_pages) {
1252 found = 1;
1253 break;
1254 }
1255 }
1256
1257 BUG_ON(!found);
1258 return nid;
1259}
1260
Nishanth Aravamudancd342062010-10-26 17:35:12 +00001261static u64 hot_add_drconf_memory_max(void)
1262{
1263 struct device_node *memory = NULL;
1264 unsigned int drconf_cell_cnt = 0;
1265 u64 lmb_size = 0;
Robert Jenningsec32dd62013-10-28 09:20:50 -05001266 const __be32 *dm = NULL;
Nishanth Aravamudancd342062010-10-26 17:35:12 +00001267
1268 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1269 if (memory) {
1270 drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
1271 lmb_size = of_get_lmb_size(memory);
1272 of_node_put(memory);
1273 }
1274 return lmb_size * drconf_cell_cnt;
1275}
1276
1277/*
1278 * memory_hotplug_max - return max address of memory that may be added
1279 *
1280 * This is currently only used on systems that support drconfig memory
1281 * hotplug.
1282 */
1283u64 memory_hotplug_max(void)
1284{
1285 return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM());
1286}
Mike Kravetz237a09892005-12-05 12:06:42 -08001287#endif /* CONFIG_MEMORY_HOTPLUG */
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001288
Jesse Larrewbd034032011-01-20 19:00:51 +00001289/* Virtual Processor Home Node (VPHN) support */
Jesse Larrew39bf9902010-12-17 22:07:47 +00001290#ifdef CONFIG_PPC_SPLPAR
Nathan Fontenot30c05352013-04-24 06:02:13 +00001291struct topology_update_data {
1292 struct topology_update_data *next;
1293 unsigned int cpu;
1294 int old_nid;
1295 int new_nid;
1296};
1297
Anton Blanchard5de16692011-01-29 12:24:34 +00001298static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS];
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001299static cpumask_t cpu_associativity_changes_mask;
1300static int vphn_enabled;
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001301static int prrn_enabled;
1302static void reset_topology_timer(void);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001303
1304/*
1305 * Store the current values of the associativity change counters in the
1306 * hypervisor.
1307 */
1308static void setup_cpu_associativity_change_counters(void)
1309{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001310 int cpu;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001311
Anton Blanchard5de16692011-01-29 12:24:34 +00001312 /* The VPHN feature supports a maximum of 8 reference points */
1313 BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8);
1314
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001315 for_each_possible_cpu(cpu) {
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001316 int i;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001317 u8 *counts = vphn_cpu_change_counts[cpu];
1318 volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts;
1319
Anton Blanchard5de16692011-01-29 12:24:34 +00001320 for (i = 0; i < distance_ref_points_depth; i++)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001321 counts[i] = hypervisor_counts[i];
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001322 }
1323}
1324
1325/*
1326 * The hypervisor maintains a set of 8 associativity change counters in
1327 * the VPA of each cpu that correspond to the associativity levels in the
1328 * ibm,associativity-reference-points property. When an associativity
1329 * level changes, the corresponding counter is incremented.
1330 *
1331 * Set a bit in cpu_associativity_changes_mask for each cpu whose home
1332 * node associativity levels have changed.
1333 *
1334 * Returns the number of cpus with unhandled associativity changes.
1335 */
1336static int update_cpu_associativity_changes_mask(void)
1337{
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001338 int cpu;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001339 cpumask_t *changes = &cpu_associativity_changes_mask;
1340
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001341 for_each_possible_cpu(cpu) {
1342 int i, changed = 0;
1343 u8 *counts = vphn_cpu_change_counts[cpu];
1344 volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts;
1345
Anton Blanchard5de16692011-01-29 12:24:34 +00001346 for (i = 0; i < distance_ref_points_depth; i++) {
Anton Blanchardd69043e2011-01-29 12:26:19 +00001347 if (hypervisor_counts[i] != counts[i]) {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001348 counts[i] = hypervisor_counts[i];
1349 changed = 1;
1350 }
1351 }
1352 if (changed) {
Robert Jennings3be7db62013-07-24 20:13:21 -05001353 cpumask_or(changes, changes, cpu_sibling_mask(cpu));
1354 cpu = cpu_last_thread_sibling(cpu);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001355 }
1356 }
1357
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001358 return cpumask_weight(changes);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001359}
1360
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001361/*
1362 * 6 64-bit registers unpacked into 12 32-bit associativity values. To form
1363 * the complete property we have to add the length in the first cell.
1364 */
1365#define VPHN_ASSOC_BUFSIZE (6*sizeof(u64)/sizeof(u32) + 1)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001366
1367/*
1368 * Convert the associativity domain numbers returned from the hypervisor
1369 * to the sequence they would appear in the ibm,associativity property.
1370 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001371static int vphn_unpack_associativity(const long *packed, __be32 *unpacked)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001372{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001373 int i, nr_assoc_doms = 0;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001374 const __be16 *field = (const __be16 *) packed;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001375
1376#define VPHN_FIELD_UNUSED (0xffff)
1377#define VPHN_FIELD_MSB (0x8000)
1378#define VPHN_FIELD_MASK (~VPHN_FIELD_MSB)
1379
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001380 for (i = 1; i < VPHN_ASSOC_BUFSIZE; i++) {
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001381 if (be16_to_cpup(field) == VPHN_FIELD_UNUSED) {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001382 /* All significant fields processed, and remaining
1383 * fields contain the reserved value of all 1's.
1384 * Just store them.
1385 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001386 unpacked[i] = *((__be32 *)field);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001387 field += 2;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001388 } else if (be16_to_cpup(field) & VPHN_FIELD_MSB) {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001389 /* Data is in the lower 15 bits of this field */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001390 unpacked[i] = cpu_to_be32(
1391 be16_to_cpup(field) & VPHN_FIELD_MASK);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001392 field++;
1393 nr_assoc_doms++;
Jesse Larrew7639ada2011-01-20 19:01:13 +00001394 } else {
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001395 /* Data is in the lower 15 bits of this field
1396 * concatenated with the next 16 bit field
1397 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001398 unpacked[i] = *((__be32 *)field);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001399 field += 2;
1400 nr_assoc_doms++;
1401 }
1402 }
1403
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001404 /* The first cell contains the length of the property */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001405 unpacked[0] = cpu_to_be32(nr_assoc_doms);
Anton Blanchardc0e5e462011-01-29 12:28:04 +00001406
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001407 return nr_assoc_doms;
1408}
1409
1410/*
1411 * Retrieve the new associativity information for a virtual processor's
1412 * home node.
1413 */
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001414static long hcall_vphn(unsigned long cpu, __be32 *associativity)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001415{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001416 long rc;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001417 long retbuf[PLPAR_HCALL9_BUFSIZE] = {0};
1418 u64 flags = 1;
1419 int hwcpu = get_hard_smp_processor_id(cpu);
1420
1421 rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu);
1422 vphn_unpack_associativity(retbuf, associativity);
1423
1424 return rc;
1425}
1426
1427static long vphn_get_associativity(unsigned long cpu,
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001428 __be32 *associativity)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001429{
Jesse Larrewcd9d6cc2011-01-20 19:01:35 +00001430 long rc;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001431
1432 rc = hcall_vphn(cpu, associativity);
1433
1434 switch (rc) {
1435 case H_FUNCTION:
1436 printk(KERN_INFO
1437 "VPHN is not supported. Disabling polling...\n");
1438 stop_topology_update();
1439 break;
1440 case H_HARDWARE:
1441 printk(KERN_ERR
1442 "hcall_vphn() experienced a hardware fault "
1443 "preventing VPHN. Disabling polling...\n");
1444 stop_topology_update();
1445 }
1446
1447 return rc;
1448}
1449
1450/*
Nathan Fontenot30c05352013-04-24 06:02:13 +00001451 * Update the CPU maps and sysfs entries for a single CPU when its NUMA
1452 * characteristics change. This function doesn't perform any locking and is
1453 * only safe to call from stop_machine().
1454 */
1455static int update_cpu_topology(void *data)
1456{
1457 struct topology_update_data *update;
1458 unsigned long cpu;
1459
1460 if (!data)
1461 return -EINVAL;
1462
Robert Jennings3be7db62013-07-24 20:13:21 -05001463 cpu = smp_processor_id();
Nathan Fontenot30c05352013-04-24 06:02:13 +00001464
1465 for (update = data; update; update = update->next) {
1466 if (cpu != update->cpu)
1467 continue;
1468
Nathan Fontenot30c05352013-04-24 06:02:13 +00001469 unmap_cpu_from_node(update->cpu);
1470 map_cpu_to_node(update->cpu, update->new_nid);
Jesse Larrew176bbf12013-04-24 06:03:48 +00001471 vdso_getcpu_init();
Nathan Fontenot30c05352013-04-24 06:02:13 +00001472 }
1473
1474 return 0;
1475}
1476
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +05301477static int update_lookup_table(void *data)
1478{
1479 struct topology_update_data *update;
1480
1481 if (!data)
1482 return -EINVAL;
1483
1484 /*
1485 * Upon topology update, the numa-cpu lookup table needs to be updated
1486 * for all threads in the core, including offline CPUs, to ensure that
1487 * future hotplug operations respect the cpu-to-node associativity
1488 * properly.
1489 */
1490 for (update = data; update; update = update->next) {
1491 int nid, base, j;
1492
1493 nid = update->new_nid;
1494 base = cpu_first_thread_sibling(update->cpu);
1495
1496 for (j = 0; j < threads_per_core; j++) {
1497 update_numa_cpu_lookup_table(base + j, nid);
1498 }
1499 }
1500
1501 return 0;
1502}
1503
Nathan Fontenot30c05352013-04-24 06:02:13 +00001504/*
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001505 * Update the node maps and sysfs entries for each cpu whose home node
Jesse Larrew79c5fce2012-06-07 16:04:34 -05001506 * has changed. Returns 1 when the topology has changed, and 0 otherwise.
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001507 */
1508int arch_update_cpu_topology(void)
1509{
Robert Jennings3be7db62013-07-24 20:13:21 -05001510 unsigned int cpu, sibling, changed = 0;
Nathan Fontenot30c05352013-04-24 06:02:13 +00001511 struct topology_update_data *updates, *ud;
Alistair Poppleb08a2a12013-08-07 02:01:44 +10001512 __be32 associativity[VPHN_ASSOC_BUFSIZE] = {0};
Jesse Larrew176bbf12013-04-24 06:03:48 +00001513 cpumask_t updated_cpus;
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001514 struct device *dev;
Robert Jennings3be7db62013-07-24 20:13:21 -05001515 int weight, new_nid, i = 0;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001516
Nathan Fontenot30c05352013-04-24 06:02:13 +00001517 weight = cpumask_weight(&cpu_associativity_changes_mask);
1518 if (!weight)
1519 return 0;
1520
1521 updates = kzalloc(weight * (sizeof(*updates)), GFP_KERNEL);
1522 if (!updates)
1523 return 0;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001524
Jesse Larrew176bbf12013-04-24 06:03:48 +00001525 cpumask_clear(&updated_cpus);
1526
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001527 for_each_cpu(cpu, &cpu_associativity_changes_mask) {
Robert Jennings3be7db62013-07-24 20:13:21 -05001528 /*
1529 * If siblings aren't flagged for changes, updates list
1530 * will be too short. Skip on this update and set for next
1531 * update.
1532 */
1533 if (!cpumask_subset(cpu_sibling_mask(cpu),
1534 &cpu_associativity_changes_mask)) {
1535 pr_info("Sibling bits not set for associativity "
1536 "change, cpu%d\n", cpu);
1537 cpumask_or(&cpu_associativity_changes_mask,
1538 &cpu_associativity_changes_mask,
1539 cpu_sibling_mask(cpu));
1540 cpu = cpu_last_thread_sibling(cpu);
1541 continue;
1542 }
1543
1544 /* Use associativity from first thread for all siblings */
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001545 vphn_get_associativity(cpu, associativity);
Robert Jennings3be7db62013-07-24 20:13:21 -05001546 new_nid = associativity_to_nid(associativity);
1547 if (new_nid < 0 || !node_online(new_nid))
1548 new_nid = first_online_node;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001549
Robert Jennings3be7db62013-07-24 20:13:21 -05001550 if (new_nid == numa_cpu_lookup_table[cpu]) {
1551 cpumask_andnot(&cpu_associativity_changes_mask,
1552 &cpu_associativity_changes_mask,
1553 cpu_sibling_mask(cpu));
1554 cpu = cpu_last_thread_sibling(cpu);
1555 continue;
1556 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001557
Robert Jennings3be7db62013-07-24 20:13:21 -05001558 for_each_cpu(sibling, cpu_sibling_mask(cpu)) {
1559 ud = &updates[i++];
1560 ud->cpu = sibling;
1561 ud->new_nid = new_nid;
1562 ud->old_nid = numa_cpu_lookup_table[sibling];
1563 cpumask_set_cpu(sibling, &updated_cpus);
1564 if (i < weight)
1565 ud->next = &updates[i];
1566 }
1567 cpu = cpu_last_thread_sibling(cpu);
Nathan Fontenot30c05352013-04-24 06:02:13 +00001568 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001569
Jesse Larrew176bbf12013-04-24 06:03:48 +00001570 stop_machine(update_cpu_topology, &updates[0], &updated_cpus);
Nathan Fontenot30c05352013-04-24 06:02:13 +00001571
Srivatsa S. Bhatd4edc5b2013-12-30 17:05:34 +05301572 /*
1573 * Update the numa-cpu lookup table with the new mappings, even for
1574 * offline CPUs. It is best to perform this update from the stop-
1575 * machine context.
1576 */
1577 stop_machine(update_lookup_table, &updates[0],
1578 cpumask_of(raw_smp_processor_id()));
1579
Nathan Fontenot30c05352013-04-24 06:02:13 +00001580 for (ud = &updates[0]; ud; ud = ud->next) {
Nathan Fontenotdd023212013-06-24 22:08:05 -05001581 unregister_cpu_under_node(ud->cpu, ud->old_nid);
1582 register_cpu_under_node(ud->cpu, ud->new_nid);
1583
Nathan Fontenot30c05352013-04-24 06:02:13 +00001584 dev = get_cpu_device(ud->cpu);
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001585 if (dev)
1586 kobject_uevent(&dev->kobj, KOBJ_CHANGE);
Nathan Fontenot30c05352013-04-24 06:02:13 +00001587 cpumask_clear_cpu(ud->cpu, &cpu_associativity_changes_mask);
Jesse Larrew79c5fce2012-06-07 16:04:34 -05001588 changed = 1;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001589 }
1590
Nathan Fontenot30c05352013-04-24 06:02:13 +00001591 kfree(updates);
Jesse Larrew79c5fce2012-06-07 16:04:34 -05001592 return changed;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001593}
1594
1595static void topology_work_fn(struct work_struct *work)
1596{
1597 rebuild_sched_domains();
1598}
1599static DECLARE_WORK(topology_work, topology_work_fn);
1600
Robert Jenningsec32dd62013-10-28 09:20:50 -05001601static void topology_schedule_update(void)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001602{
1603 schedule_work(&topology_work);
1604}
1605
1606static void topology_timer_fn(unsigned long ignored)
1607{
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001608 if (prrn_enabled && cpumask_weight(&cpu_associativity_changes_mask))
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001609 topology_schedule_update();
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001610 else if (vphn_enabled) {
1611 if (update_cpu_associativity_changes_mask() > 0)
1612 topology_schedule_update();
1613 reset_topology_timer();
1614 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001615}
1616static struct timer_list topology_timer =
1617 TIMER_INITIALIZER(topology_timer_fn, 0, 0);
1618
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001619static void reset_topology_timer(void)
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001620{
1621 topology_timer.data = 0;
1622 topology_timer.expires = jiffies + 60 * HZ;
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001623 mod_timer(&topology_timer, topology_timer.expires);
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001624}
1625
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001626#ifdef CONFIG_SMP
1627
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001628static void stage_topology_update(int core_id)
1629{
1630 cpumask_or(&cpu_associativity_changes_mask,
1631 &cpu_associativity_changes_mask, cpu_sibling_mask(core_id));
1632 reset_topology_timer();
1633}
1634
1635static int dt_update_callback(struct notifier_block *nb,
1636 unsigned long action, void *data)
1637{
1638 struct of_prop_reconfig *update;
1639 int rc = NOTIFY_DONE;
1640
1641 switch (action) {
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001642 case OF_RECONFIG_UPDATE_PROPERTY:
1643 update = (struct of_prop_reconfig *)data;
Nathan Fontenot30c05352013-04-24 06:02:13 +00001644 if (!of_prop_cmp(update->dn->type, "cpu") &&
1645 !of_prop_cmp(update->prop->name, "ibm,associativity")) {
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001646 u32 core_id;
1647 of_property_read_u32(update->dn, "reg", &core_id);
1648 stage_topology_update(core_id);
1649 rc = NOTIFY_OK;
1650 }
1651 break;
1652 }
1653
1654 return rc;
1655}
1656
1657static struct notifier_block dt_update_nb = {
1658 .notifier_call = dt_update_callback,
1659};
1660
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001661#endif
1662
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001663/*
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001664 * Start polling for associativity changes.
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001665 */
1666int start_topology_update(void)
1667{
1668 int rc = 0;
1669
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001670 if (firmware_has_feature(FW_FEATURE_PRRN)) {
1671 if (!prrn_enabled) {
1672 prrn_enabled = 1;
1673 vphn_enabled = 0;
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001674#ifdef CONFIG_SMP
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001675 rc = of_reconfig_notifier_register(&dt_update_nb);
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001676#endif
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001677 }
Jesse Larrewb7abef02013-04-24 06:05:22 +00001678 } else if (firmware_has_feature(FW_FEATURE_VPHN) &&
Anton Blanchardf13c13a2013-08-07 02:01:26 +10001679 lppaca_shared_proc(get_lppaca())) {
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001680 if (!vphn_enabled) {
1681 prrn_enabled = 0;
1682 vphn_enabled = 1;
1683 setup_cpu_associativity_change_counters();
1684 init_timer_deferrable(&topology_timer);
1685 reset_topology_timer();
1686 }
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001687 }
1688
1689 return rc;
1690}
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001691
1692/*
1693 * Disable polling for VPHN associativity changes.
1694 */
1695int stop_topology_update(void)
1696{
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001697 int rc = 0;
1698
1699 if (prrn_enabled) {
1700 prrn_enabled = 0;
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001701#ifdef CONFIG_SMP
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001702 rc = of_reconfig_notifier_unregister(&dt_update_nb);
Nathan Fontenot601abdc32013-04-29 03:45:36 +00001703#endif
Jesse Larrew5d88aa82013-04-24 06:00:35 +00001704 } else if (vphn_enabled) {
1705 vphn_enabled = 0;
1706 rc = del_timer_sync(&topology_timer);
1707 }
1708
1709 return rc;
Jesse Larrew9eff1a32010-12-01 12:31:15 +00001710}
Nathan Fontenote04fa612013-04-24 06:07:39 +00001711
1712int prrn_is_enabled(void)
1713{
1714 return prrn_enabled;
1715}
1716
1717static int topology_read(struct seq_file *file, void *v)
1718{
1719 if (vphn_enabled || prrn_enabled)
1720 seq_puts(file, "on\n");
1721 else
1722 seq_puts(file, "off\n");
1723
1724 return 0;
1725}
1726
1727static int topology_open(struct inode *inode, struct file *file)
1728{
1729 return single_open(file, topology_read, NULL);
1730}
1731
1732static ssize_t topology_write(struct file *file, const char __user *buf,
1733 size_t count, loff_t *off)
1734{
1735 char kbuf[4]; /* "on" or "off" plus null. */
1736 int read_len;
1737
1738 read_len = count < 3 ? count : 3;
1739 if (copy_from_user(kbuf, buf, read_len))
1740 return -EINVAL;
1741
1742 kbuf[read_len] = '\0';
1743
1744 if (!strncmp(kbuf, "on", 2))
1745 start_topology_update();
1746 else if (!strncmp(kbuf, "off", 3))
1747 stop_topology_update();
1748 else
1749 return -EINVAL;
1750
1751 return count;
1752}
1753
1754static const struct file_operations topology_ops = {
1755 .read = seq_read,
1756 .write = topology_write,
1757 .open = topology_open,
1758 .release = single_release
1759};
1760
1761static int topology_update_init(void)
1762{
1763 start_topology_update();
1764 proc_create("powerpc/topology_updates", 644, NULL, &topology_ops);
1765
1766 return 0;
1767}
1768device_initcall(topology_update_init);
Jesse Larrew39bf9902010-12-17 22:07:47 +00001769#endif /* CONFIG_PPC_SPLPAR */