blob: 4bc2c7d8bf42756d812c9ff74f647a6159a394a6 [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Yinghai Lu95f72d12010-07-12 14:36:09 +10002/*
3 * Procedures for maintaining information about logical memory blocks.
4 *
5 * Peter Bergner, IBM Corp. June 2001.
6 * Copyright (C) 2001 Peter Bergner.
Yinghai Lu95f72d12010-07-12 14:36:09 +10007 */
8
9#include <linux/kernel.h>
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -070010#include <linux/slab.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100011#include <linux/init.h>
12#include <linux/bitops.h>
Benjamin Herrenschmidt449e8df2010-07-06 15:39:07 -070013#include <linux/poison.h>
Benjamin Herrenschmidtc196f762010-07-06 15:39:16 -070014#include <linux/pfn.h>
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -070015#include <linux/debugfs.h>
Randy Dunlap514c6032018-04-05 16:25:34 -070016#include <linux/kmemleak.h>
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -070017#include <linux/seq_file.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100018#include <linux/memblock.h>
19
Christoph Hellwigc4c5ad62016-07-28 15:48:06 -070020#include <asm/sections.h>
Santosh Shilimkar26f09e92014-01-21 15:50:19 -080021#include <linux/io.h>
22
23#include "internal.h"
Tang Chen79442ed2013-11-12 15:07:59 -080024
Ard Biesheuvel8a5b403d2019-02-15 13:33:32 +010025#define INIT_MEMBLOCK_REGIONS 128
26#define INIT_PHYSMEM_REGIONS 4
27
28#ifndef INIT_MEMBLOCK_RESERVED_REGIONS
29# define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
30#endif
31
Mike Rapoport3e039c52018-06-30 17:55:05 +030032/**
33 * DOC: memblock overview
34 *
35 * Memblock is a method of managing memory regions during the early
36 * boot period when the usual kernel memory allocators are not up and
37 * running.
38 *
39 * Memblock views the system memory as collections of contiguous
40 * regions. There are several types of these collections:
41 *
42 * * ``memory`` - describes the physical memory available to the
43 * kernel; this may differ from the actual physical memory installed
44 * in the system, for instance when the memory is restricted with
45 * ``mem=`` command line parameter
46 * * ``reserved`` - describes the regions that were allocated
47 * * ``physmap`` - describes the actual physical memory regardless of
48 * the possible restrictions; the ``physmap`` type is only available
49 * on some architectures.
50 *
51 * Each region is represented by :c:type:`struct memblock_region` that
52 * defines the region extents, its attributes and NUMA node id on NUMA
53 * systems. Every memory type is described by the :c:type:`struct
54 * memblock_type` which contains an array of memory regions along with
55 * the allocator metadata. The memory types are nicely wrapped with
56 * :c:type:`struct memblock`. This structure is statically initialzed
57 * at build time. The region arrays for the "memory" and "reserved"
58 * types are initially sized to %INIT_MEMBLOCK_REGIONS and for the
59 * "physmap" type to %INIT_PHYSMEM_REGIONS.
Cao jin6e5af9a2019-11-30 17:56:21 -080060 * The memblock_allow_resize() enables automatic resizing of the region
61 * arrays during addition of new regions. This feature should be used
62 * with care so that memory allocated for the region array will not
63 * overlap with areas that should be reserved, for example initrd.
Mike Rapoport3e039c52018-06-30 17:55:05 +030064 *
65 * The early architecture setup should tell memblock what the physical
Cao jin6e5af9a2019-11-30 17:56:21 -080066 * memory layout is by using memblock_add() or memblock_add_node()
67 * functions. The first function does not assign the region to a NUMA
68 * node and it is appropriate for UMA systems. Yet, it is possible to
69 * use it on NUMA systems as well and assign the region to a NUMA node
70 * later in the setup process using memblock_set_node(). The
71 * memblock_add_node() performs such an assignment directly.
Mike Rapoport3e039c52018-06-30 17:55:05 +030072 *
Mike Rapoporta2974132019-03-11 23:30:54 -070073 * Once memblock is setup the memory can be allocated using one of the
74 * API variants:
75 *
Cao jin6e5af9a2019-11-30 17:56:21 -080076 * * memblock_phys_alloc*() - these functions return the **physical**
77 * address of the allocated memory
78 * * memblock_alloc*() - these functions return the **virtual** address
79 * of the allocated memory.
Mike Rapoporta2974132019-03-11 23:30:54 -070080 *
81 * Note, that both API variants use implict assumptions about allowed
82 * memory ranges and the fallback methods. Consult the documentation
Cao jin6e5af9a2019-11-30 17:56:21 -080083 * of memblock_alloc_internal() and memblock_alloc_range_nid()
84 * functions for more elaborate description.
Mike Rapoport3e039c52018-06-30 17:55:05 +030085 *
Cao jin6e5af9a2019-11-30 17:56:21 -080086 * As the system boot progresses, the architecture specific mem_init()
87 * function frees all the memory to the buddy page allocator.
Mike Rapoport3e039c52018-06-30 17:55:05 +030088 *
Cao jin6e5af9a2019-11-30 17:56:21 -080089 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
Mike Rapoport3e039c52018-06-30 17:55:05 +030090 * memblock data structures will be discarded after the system
Cao jin6e5af9a2019-11-30 17:56:21 -080091 * initialization completes.
Mike Rapoport3e039c52018-06-30 17:55:05 +030092 */
93
Mike Rapoportbda49a82018-10-30 15:09:40 -070094#ifndef CONFIG_NEED_MULTIPLE_NODES
95struct pglist_data __refdata contig_page_data;
96EXPORT_SYMBOL(contig_page_data);
97#endif
98
99unsigned long max_low_pfn;
100unsigned long min_low_pfn;
101unsigned long max_pfn;
102unsigned long long max_possible_pfn;
103
Tejun Heofe091c22011-12-08 10:22:07 -0800104static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
Ard Biesheuvel8a5b403d2019-02-15 13:33:32 +0100105static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
Philipp Hachtmann70210ed2014-01-29 18:16:01 +0100106#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
107static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS] __initdata_memblock;
108#endif
Tejun Heofe091c22011-12-08 10:22:07 -0800109
110struct memblock memblock __initdata_memblock = {
111 .memory.regions = memblock_memory_init_regions,
112 .memory.cnt = 1, /* empty dummy entry */
113 .memory.max = INIT_MEMBLOCK_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800114 .memory.name = "memory",
Tejun Heofe091c22011-12-08 10:22:07 -0800115
116 .reserved.regions = memblock_reserved_init_regions,
117 .reserved.cnt = 1, /* empty dummy entry */
Ard Biesheuvel8a5b403d2019-02-15 13:33:32 +0100118 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800119 .reserved.name = "reserved",
Tejun Heofe091c22011-12-08 10:22:07 -0800120
Philipp Hachtmann70210ed2014-01-29 18:16:01 +0100121#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
122 .physmem.regions = memblock_physmem_init_regions,
123 .physmem.cnt = 1, /* empty dummy entry */
124 .physmem.max = INIT_PHYSMEM_REGIONS,
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800125 .physmem.name = "physmem",
Philipp Hachtmann70210ed2014-01-29 18:16:01 +0100126#endif
127
Tang Chen79442ed2013-11-12 15:07:59 -0800128 .bottom_up = false,
Tejun Heofe091c22011-12-08 10:22:07 -0800129 .current_limit = MEMBLOCK_ALLOC_ANYWHERE,
130};
Yinghai Lu95f72d12010-07-12 14:36:09 +1000131
Yinghai Lu10d06432010-07-28 15:43:02 +1000132int memblock_debug __initdata_memblock;
Tony Lucka3f5baf2015-06-24 16:58:12 -0700133static bool system_has_some_mirror __initdata_memblock = false;
Tejun Heo1aadc052011-12-08 10:22:08 -0800134static int memblock_can_resize __initdata_memblock;
Gavin Shan181eb392012-05-29 15:06:50 -0700135static int memblock_memory_in_slab __initdata_memblock = 0;
136static int memblock_reserved_in_slab __initdata_memblock = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000137
Mike Rapoportc366ea82019-03-11 23:29:46 -0700138static enum memblock_flags __init_memblock choose_memblock_flags(void)
Tony Lucka3f5baf2015-06-24 16:58:12 -0700139{
140 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
141}
142
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800143/* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
144static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
145{
Stefan Agner1c4bc432018-06-07 17:06:15 -0700146 return *size = min(*size, PHYS_ADDR_MAX - base);
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800147}
148
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000149/*
150 * Address comparison utilities
151 */
Yinghai Lu10d06432010-07-28 15:43:02 +1000152static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1,
Benjamin Herrenschmidt2898cc42010-08-04 13:34:42 +1000153 phys_addr_t base2, phys_addr_t size2)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000154{
155 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
156}
157
Tang Chen95cf82e2015-09-08 15:02:03 -0700158bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
H Hartley Sweeten2d7d3eb2011-10-31 17:09:15 -0700159 phys_addr_t base, phys_addr_t size)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000160{
161 unsigned long i;
162
Alexander Kuleshovf14516f2016-01-14 15:20:39 -0800163 for (i = 0; i < type->cnt; i++)
164 if (memblock_addrs_overlap(base, size, type->regions[i].base,
165 type->regions[i].size))
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000166 break;
Tang Chenc5c5c9d2015-09-08 15:02:00 -0700167 return i < type->cnt;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000168}
169
Mike Rapoport47cec442018-06-30 17:55:02 +0300170/**
Tang Chen79442ed2013-11-12 15:07:59 -0800171 * __memblock_find_range_bottom_up - find free area utility in bottom-up
172 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300173 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
174 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tang Chen79442ed2013-11-12 15:07:59 -0800175 * @size: size of free area to find
176 * @align: alignment of free area to find
Grygorii Strashkob1154232014-01-21 15:50:16 -0800177 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700178 * @flags: pick from blocks based on memory attributes
Tang Chen79442ed2013-11-12 15:07:59 -0800179 *
180 * Utility called from memblock_find_in_range_node(), find free area bottom-up.
181 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300182 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800183 * Found address on success, 0 on failure.
184 */
185static phys_addr_t __init_memblock
186__memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700187 phys_addr_t size, phys_addr_t align, int nid,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300188 enum memblock_flags flags)
Tang Chen79442ed2013-11-12 15:07:59 -0800189{
190 phys_addr_t this_start, this_end, cand;
191 u64 i;
192
Tony Luckfc6daaf2015-06-24 16:58:09 -0700193 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
Tang Chen79442ed2013-11-12 15:07:59 -0800194 this_start = clamp(this_start, start, end);
195 this_end = clamp(this_end, start, end);
196
197 cand = round_up(this_start, align);
198 if (cand < this_end && this_end - cand >= size)
199 return cand;
200 }
201
202 return 0;
203}
204
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800205/**
Tang Chen14028992013-11-12 15:07:57 -0800206 * __memblock_find_range_top_down - find free area utility, in top-down
207 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300208 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
209 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tang Chen14028992013-11-12 15:07:57 -0800210 * @size: size of free area to find
211 * @align: alignment of free area to find
Grygorii Strashkob1154232014-01-21 15:50:16 -0800212 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700213 * @flags: pick from blocks based on memory attributes
Tang Chen14028992013-11-12 15:07:57 -0800214 *
215 * Utility called from memblock_find_in_range_node(), find free area top-down.
216 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300217 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800218 * Found address on success, 0 on failure.
Tang Chen14028992013-11-12 15:07:57 -0800219 */
220static phys_addr_t __init_memblock
221__memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700222 phys_addr_t size, phys_addr_t align, int nid,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300223 enum memblock_flags flags)
Tang Chen14028992013-11-12 15:07:57 -0800224{
225 phys_addr_t this_start, this_end, cand;
226 u64 i;
227
Tony Luckfc6daaf2015-06-24 16:58:09 -0700228 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
229 NULL) {
Tang Chen14028992013-11-12 15:07:57 -0800230 this_start = clamp(this_start, start, end);
231 this_end = clamp(this_end, start, end);
232
233 if (this_end < size)
234 continue;
235
236 cand = round_down(this_end - size, align);
237 if (cand >= this_start)
238 return cand;
239 }
240
241 return 0;
242}
243
244/**
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800245 * memblock_find_in_range_node - find free area in given range and node
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800246 * @size: size of free area to find
247 * @align: alignment of free area to find
Grygorii Strashko87029ee2014-01-21 15:50:14 -0800248 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300249 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
250 * %MEMBLOCK_ALLOC_ACCESSIBLE
Grygorii Strashkob1154232014-01-21 15:50:16 -0800251 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Tony Luckfc6daaf2015-06-24 16:58:09 -0700252 * @flags: pick from blocks based on memory attributes
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800253 *
254 * Find @size free area aligned to @align in the specified range and node.
255 *
Tang Chen79442ed2013-11-12 15:07:59 -0800256 * When allocation direction is bottom-up, the @start should be greater
257 * than the end of the kernel image. Otherwise, it will be trimmed. The
258 * reason is that we want the bottom-up allocation just near the kernel
259 * image so it is highly likely that the allocated memory and the kernel
260 * will reside in the same node.
261 *
262 * If bottom-up allocation failed, will try to allocate memory top-down.
263 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300264 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800265 * Found address on success, 0 on failure.
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000266 */
Mike Rapoportc366ea82019-03-11 23:29:46 -0700267static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
Grygorii Strashko87029ee2014-01-21 15:50:14 -0800268 phys_addr_t align, phys_addr_t start,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300269 phys_addr_t end, int nid,
270 enum memblock_flags flags)
Tang Chenf7210e62013-02-22 16:33:51 -0800271{
Tang Chen0cfb8f02014-08-29 15:18:31 -0700272 phys_addr_t kernel_end, ret;
Tang Chen79442ed2013-11-12 15:07:59 -0800273
Tang Chenf7210e62013-02-22 16:33:51 -0800274 /* pump up @end */
Qian Caifed84c72018-12-28 00:36:29 -0800275 if (end == MEMBLOCK_ALLOC_ACCESSIBLE ||
276 end == MEMBLOCK_ALLOC_KASAN)
Tang Chenf7210e62013-02-22 16:33:51 -0800277 end = memblock.current_limit;
278
279 /* avoid allocating the first page */
280 start = max_t(phys_addr_t, start, PAGE_SIZE);
281 end = max(start, end);
Tang Chen79442ed2013-11-12 15:07:59 -0800282 kernel_end = __pa_symbol(_end);
283
284 /*
285 * try bottom-up allocation only when bottom-up mode
286 * is set and @end is above the kernel image.
287 */
288 if (memblock_bottom_up() && end > kernel_end) {
289 phys_addr_t bottom_up_start;
290
291 /* make sure we will allocate above the kernel */
292 bottom_up_start = max(start, kernel_end);
293
294 /* ok, try bottom-up allocation first */
295 ret = __memblock_find_range_bottom_up(bottom_up_start, end,
Tony Luckfc6daaf2015-06-24 16:58:09 -0700296 size, align, nid, flags);
Tang Chen79442ed2013-11-12 15:07:59 -0800297 if (ret)
298 return ret;
299
300 /*
301 * we always limit bottom-up allocation above the kernel,
302 * but top-down allocation doesn't have the limit, so
303 * retrying top-down allocation may succeed when bottom-up
304 * allocation failed.
305 *
306 * bottom-up allocation is expected to be fail very rarely,
307 * so we use WARN_ONCE() here to see the stack trace if
308 * fail happens.
309 */
Michal Hockoe3d301c2018-07-13 16:59:16 -0700310 WARN_ONCE(IS_ENABLED(CONFIG_MEMORY_HOTREMOVE),
311 "memblock: bottom-up allocation failed, memory hotremove may be affected\n");
Tang Chen79442ed2013-11-12 15:07:59 -0800312 }
Tang Chenf7210e62013-02-22 16:33:51 -0800313
Tony Luckfc6daaf2015-06-24 16:58:09 -0700314 return __memblock_find_range_top_down(start, end, size, align, nid,
315 flags);
Tang Chenf7210e62013-02-22 16:33:51 -0800316}
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +1000317
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800318/**
319 * memblock_find_in_range - find free area in given range
320 * @start: start of candidate range
Mike Rapoport47cec442018-06-30 17:55:02 +0300321 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
322 * %MEMBLOCK_ALLOC_ACCESSIBLE
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800323 * @size: size of free area to find
324 * @align: alignment of free area to find
325 *
326 * Find @size free area aligned to @align in the specified range.
327 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300328 * Return:
Tang Chen79442ed2013-11-12 15:07:59 -0800329 * Found address on success, 0 on failure.
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800330 */
331phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
332 phys_addr_t end, phys_addr_t size,
333 phys_addr_t align)
334{
Tony Lucka3f5baf2015-06-24 16:58:12 -0700335 phys_addr_t ret;
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300336 enum memblock_flags flags = choose_memblock_flags();
Tony Lucka3f5baf2015-06-24 16:58:12 -0700337
338again:
339 ret = memblock_find_in_range_node(size, align, start, end,
340 NUMA_NO_NODE, flags);
341
342 if (!ret && (flags & MEMBLOCK_MIRROR)) {
343 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
344 &size);
345 flags &= ~MEMBLOCK_MIRROR;
346 goto again;
347 }
348
349 return ret;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -0800350}
351
Yinghai Lu10d06432010-07-28 15:43:02 +1000352static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000353{
Tejun Heo1440c4e2011-12-08 10:22:08 -0800354 type->total_size -= type->regions[r].size;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200355 memmove(&type->regions[r], &type->regions[r + 1],
356 (type->cnt - (r + 1)) * sizeof(type->regions[r]));
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +1000357 type->cnt--;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000358
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700359 /* Special case for empty arrays */
360 if (type->cnt == 0) {
Tejun Heo1440c4e2011-12-08 10:22:08 -0800361 WARN_ON(type->total_size != 0);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700362 type->cnt = 1;
363 type->regions[0].base = 0;
364 type->regions[0].size = 0;
Tang Chen66a20752014-01-21 15:49:20 -0800365 type->regions[0].flags = 0;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200366 memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700367 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000368}
369
Mike Rapoport350e88b2019-05-13 17:22:59 -0700370#ifndef CONFIG_ARCH_KEEP_MEMBLOCK
Pavel Tatashin3010f872017-08-18 15:16:05 -0700371/**
Mike Rapoport47cec442018-06-30 17:55:02 +0300372 * memblock_discard - discard memory and reserved arrays if they were allocated
Pavel Tatashin3010f872017-08-18 15:16:05 -0700373 */
374void __init memblock_discard(void)
Yinghai Lu29f67382012-07-11 14:02:56 -0700375{
Pavel Tatashin3010f872017-08-18 15:16:05 -0700376 phys_addr_t addr, size;
Yinghai Lu29f67382012-07-11 14:02:56 -0700377
Pavel Tatashin3010f872017-08-18 15:16:05 -0700378 if (memblock.reserved.regions != memblock_reserved_init_regions) {
379 addr = __pa(memblock.reserved.regions);
380 size = PAGE_ALIGN(sizeof(struct memblock_region) *
381 memblock.reserved.max);
382 __memblock_free_late(addr, size);
383 }
Yinghai Lu29f67382012-07-11 14:02:56 -0700384
Pavel Tatashin91b540f92017-08-25 15:55:46 -0700385 if (memblock.memory.regions != memblock_memory_init_regions) {
Pavel Tatashin3010f872017-08-18 15:16:05 -0700386 addr = __pa(memblock.memory.regions);
387 size = PAGE_ALIGN(sizeof(struct memblock_region) *
388 memblock.memory.max);
389 __memblock_free_late(addr, size);
390 }
Yinghai Lu29f67382012-07-11 14:02:56 -0700391}
Philipp Hachtmann5e270e22014-01-23 15:53:11 -0800392#endif
393
Greg Pearson48c3b582012-06-20 12:53:05 -0700394/**
395 * memblock_double_array - double the size of the memblock regions array
396 * @type: memblock type of the regions array being doubled
397 * @new_area_start: starting address of memory range to avoid overlap with
398 * @new_area_size: size of memory range to avoid overlap with
399 *
400 * Double the size of the @type regions array. If memblock is being used to
401 * allocate memory for a new reserved regions array and there is a previously
Mike Rapoport47cec442018-06-30 17:55:02 +0300402 * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
Greg Pearson48c3b582012-06-20 12:53:05 -0700403 * waiting to be reserved, ensure the memory used by the new array does
404 * not overlap.
405 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300406 * Return:
Greg Pearson48c3b582012-06-20 12:53:05 -0700407 * 0 on success, -1 on failure.
408 */
409static int __init_memblock memblock_double_array(struct memblock_type *type,
410 phys_addr_t new_area_start,
411 phys_addr_t new_area_size)
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700412{
413 struct memblock_region *new_array, *old_array;
Yinghai Lu29f67382012-07-11 14:02:56 -0700414 phys_addr_t old_alloc_size, new_alloc_size;
Mike Rapoporta36aab82018-08-17 15:47:17 -0700415 phys_addr_t old_size, new_size, addr, new_end;
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700416 int use_slab = slab_is_available();
Gavin Shan181eb392012-05-29 15:06:50 -0700417 int *in_slab;
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700418
419 /* We don't allow resizing until we know about the reserved regions
420 * of memory that aren't suitable for allocation
421 */
422 if (!memblock_can_resize)
423 return -1;
424
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700425 /* Calculate new doubled size */
426 old_size = type->max * sizeof(struct memblock_region);
427 new_size = old_size << 1;
Yinghai Lu29f67382012-07-11 14:02:56 -0700428 /*
429 * We need to allocated new one align to PAGE_SIZE,
430 * so we can free them completely later.
431 */
432 old_alloc_size = PAGE_ALIGN(old_size);
433 new_alloc_size = PAGE_ALIGN(new_size);
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700434
Gavin Shan181eb392012-05-29 15:06:50 -0700435 /* Retrieve the slab flag */
436 if (type == &memblock.memory)
437 in_slab = &memblock_memory_in_slab;
438 else
439 in_slab = &memblock_reserved_in_slab;
440
Mike Rapoporta2974132019-03-11 23:30:54 -0700441 /* Try to find some space for it */
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700442 if (use_slab) {
443 new_array = kmalloc(new_size, GFP_KERNEL);
Tejun Heo1f5026a2011-07-12 09:58:09 +0200444 addr = new_array ? __pa(new_array) : 0;
Gavin Shan4e2f0772012-05-29 15:06:50 -0700445 } else {
Greg Pearson48c3b582012-06-20 12:53:05 -0700446 /* only exclude range when trying to double reserved.regions */
447 if (type != &memblock.reserved)
448 new_area_start = new_area_size = 0;
449
450 addr = memblock_find_in_range(new_area_start + new_area_size,
451 memblock.current_limit,
Yinghai Lu29f67382012-07-11 14:02:56 -0700452 new_alloc_size, PAGE_SIZE);
Greg Pearson48c3b582012-06-20 12:53:05 -0700453 if (!addr && new_area_size)
454 addr = memblock_find_in_range(0,
Andrew Mortonfd073832012-07-31 16:42:40 -0700455 min(new_area_start, memblock.current_limit),
456 new_alloc_size, PAGE_SIZE);
Greg Pearson48c3b582012-06-20 12:53:05 -0700457
Sachin Kamat15674862012-09-04 13:55:05 +0530458 new_array = addr ? __va(addr) : NULL;
Gavin Shan4e2f0772012-05-29 15:06:50 -0700459 }
Tejun Heo1f5026a2011-07-12 09:58:09 +0200460 if (!addr) {
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700461 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
Heiko Carstens0262d9c2017-02-24 14:55:59 -0800462 type->name, type->max, type->max * 2);
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700463 return -1;
464 }
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700465
Mike Rapoporta36aab82018-08-17 15:47:17 -0700466 new_end = addr + new_size - 1;
467 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
468 type->name, type->max * 2, &addr, &new_end);
Yinghai Luea9e4372010-07-28 15:13:22 +1000469
Andrew Mortonfd073832012-07-31 16:42:40 -0700470 /*
471 * Found space, we now need to move the array over before we add the
472 * reserved region since it may be our reserved array itself that is
473 * full.
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700474 */
475 memcpy(new_array, type->regions, old_size);
476 memset(new_array + type->max, 0, old_size);
477 old_array = type->regions;
478 type->regions = new_array;
479 type->max <<= 1;
480
Andrew Mortonfd073832012-07-31 16:42:40 -0700481 /* Free old array. We needn't free it if the array is the static one */
Gavin Shan181eb392012-05-29 15:06:50 -0700482 if (*in_slab)
483 kfree(old_array);
484 else if (old_array != memblock_memory_init_regions &&
485 old_array != memblock_reserved_init_regions)
Yinghai Lu29f67382012-07-11 14:02:56 -0700486 memblock_free(__pa(old_array), old_alloc_size);
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700487
Andrew Mortonfd073832012-07-31 16:42:40 -0700488 /*
489 * Reserve the new array if that comes from the memblock. Otherwise, we
490 * needn't do it
Gavin Shan181eb392012-05-29 15:06:50 -0700491 */
492 if (!use_slab)
Yinghai Lu29f67382012-07-11 14:02:56 -0700493 BUG_ON(memblock_reserve(addr, new_alloc_size));
Gavin Shan181eb392012-05-29 15:06:50 -0700494
495 /* Update slab flag */
496 *in_slab = use_slab;
497
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700498 return 0;
499}
500
Tejun Heo784656f92011-07-12 11:15:55 +0200501/**
502 * memblock_merge_regions - merge neighboring compatible regions
503 * @type: memblock type to scan
504 *
505 * Scan @type and merge neighboring compatible regions.
506 */
507static void __init_memblock memblock_merge_regions(struct memblock_type *type)
508{
509 int i = 0;
510
511 /* cnt never goes below 1 */
512 while (i < type->cnt - 1) {
513 struct memblock_region *this = &type->regions[i];
514 struct memblock_region *next = &type->regions[i + 1];
515
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200516 if (this->base + this->size != next->base ||
517 memblock_get_region_node(this) !=
Tang Chen66a20752014-01-21 15:49:20 -0800518 memblock_get_region_node(next) ||
519 this->flags != next->flags) {
Tejun Heo784656f92011-07-12 11:15:55 +0200520 BUG_ON(this->base + this->size > next->base);
521 i++;
522 continue;
523 }
524
525 this->size += next->size;
Lin Fengc0232ae2013-01-11 14:31:44 -0800526 /* move forward from next + 1, index of which is i + 2 */
527 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
Tejun Heo784656f92011-07-12 11:15:55 +0200528 type->cnt--;
529 }
530}
531
532/**
533 * memblock_insert_region - insert new memblock region
Tang Chen209ff862013-04-29 15:08:41 -0700534 * @type: memblock type to insert into
535 * @idx: index for the insertion point
536 * @base: base address of the new region
537 * @size: size of the new region
538 * @nid: node id of the new region
Tang Chen66a20752014-01-21 15:49:20 -0800539 * @flags: flags of the new region
Tejun Heo784656f92011-07-12 11:15:55 +0200540 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300541 * Insert new memblock region [@base, @base + @size) into @type at @idx.
Alexander Kuleshov412d0002016-08-04 15:31:46 -0700542 * @type must already have extra room to accommodate the new region.
Tejun Heo784656f92011-07-12 11:15:55 +0200543 */
544static void __init_memblock memblock_insert_region(struct memblock_type *type,
545 int idx, phys_addr_t base,
Tang Chen66a20752014-01-21 15:49:20 -0800546 phys_addr_t size,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300547 int nid,
548 enum memblock_flags flags)
Tejun Heo784656f92011-07-12 11:15:55 +0200549{
550 struct memblock_region *rgn = &type->regions[idx];
551
552 BUG_ON(type->cnt >= type->max);
553 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
554 rgn->base = base;
555 rgn->size = size;
Tang Chen66a20752014-01-21 15:49:20 -0800556 rgn->flags = flags;
Tejun Heo7c0caeb2011-07-14 11:43:42 +0200557 memblock_set_region_node(rgn, nid);
Tejun Heo784656f92011-07-12 11:15:55 +0200558 type->cnt++;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800559 type->total_size += size;
Tejun Heo784656f92011-07-12 11:15:55 +0200560}
561
562/**
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100563 * memblock_add_range - add new memblock region
Tejun Heo784656f92011-07-12 11:15:55 +0200564 * @type: memblock type to add new region into
565 * @base: base address of the new region
566 * @size: size of the new region
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800567 * @nid: nid of the new region
Tang Chen66a20752014-01-21 15:49:20 -0800568 * @flags: flags of the new region
Tejun Heo784656f92011-07-12 11:15:55 +0200569 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300570 * Add new memblock region [@base, @base + @size) into @type. The new region
Tejun Heo784656f92011-07-12 11:15:55 +0200571 * is allowed to overlap with existing ones - overlaps don't affect already
572 * existing regions. @type is guaranteed to be minimal (all neighbouring
573 * compatible regions are merged) after the addition.
574 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300575 * Return:
Tejun Heo784656f92011-07-12 11:15:55 +0200576 * 0 on success, -errno on failure.
577 */
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100578int __init_memblock memblock_add_range(struct memblock_type *type,
Tang Chen66a20752014-01-21 15:49:20 -0800579 phys_addr_t base, phys_addr_t size,
Mike Rapoporte1720fe2018-06-30 17:55:01 +0300580 int nid, enum memblock_flags flags)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000581{
Tejun Heo784656f92011-07-12 11:15:55 +0200582 bool insert = false;
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800583 phys_addr_t obase = base;
584 phys_addr_t end = base + memblock_cap_size(base, &size);
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800585 int idx, nr_new;
586 struct memblock_region *rgn;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000587
Tejun Heob3dc6272012-04-20 08:31:34 -0700588 if (!size)
589 return 0;
590
Tejun Heo784656f92011-07-12 11:15:55 +0200591 /* special case for empty array */
592 if (type->regions[0].size == 0) {
Tejun Heo1440c4e2011-12-08 10:22:08 -0800593 WARN_ON(type->cnt != 1 || type->total_size);
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +1000594 type->regions[0].base = base;
595 type->regions[0].size = size;
Tang Chen66a20752014-01-21 15:49:20 -0800596 type->regions[0].flags = flags;
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800597 memblock_set_region_node(&type->regions[0], nid);
Tejun Heo1440c4e2011-12-08 10:22:08 -0800598 type->total_size = size;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000599 return 0;
600 }
Tejun Heo784656f92011-07-12 11:15:55 +0200601repeat:
602 /*
603 * The following is executed twice. Once with %false @insert and
604 * then with %true. The first counts the number of regions needed
Alexander Kuleshov412d0002016-08-04 15:31:46 -0700605 * to accommodate the new area. The second actually inserts them.
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700606 */
Tejun Heo784656f92011-07-12 11:15:55 +0200607 base = obase;
608 nr_new = 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000609
Gioh Kim66e8b432017-11-15 17:33:42 -0800610 for_each_memblock_type(idx, type, rgn) {
Tejun Heo784656f92011-07-12 11:15:55 +0200611 phys_addr_t rbase = rgn->base;
612 phys_addr_t rend = rbase + rgn->size;
613
614 if (rbase >= end)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000615 break;
Tejun Heo784656f92011-07-12 11:15:55 +0200616 if (rend <= base)
617 continue;
618 /*
619 * @rgn overlaps. If it separates the lower part of new
620 * area, insert that portion.
621 */
622 if (rbase > base) {
Wei Yangc0a29492015-09-04 15:47:38 -0700623#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
624 WARN_ON(nid != memblock_get_region_node(rgn));
625#endif
Wei Yang4fcab5f2015-09-08 14:59:53 -0700626 WARN_ON(flags != rgn->flags);
Tejun Heo784656f92011-07-12 11:15:55 +0200627 nr_new++;
628 if (insert)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800629 memblock_insert_region(type, idx++, base,
Tang Chen66a20752014-01-21 15:49:20 -0800630 rbase - base, nid,
631 flags);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000632 }
Tejun Heo784656f92011-07-12 11:15:55 +0200633 /* area below @rend is dealt with, forget about it */
634 base = min(rend, end);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000635 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000636
Tejun Heo784656f92011-07-12 11:15:55 +0200637 /* insert the remaining portion */
638 if (base < end) {
639 nr_new++;
640 if (insert)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800641 memblock_insert_region(type, idx, base, end - base,
Tang Chen66a20752014-01-21 15:49:20 -0800642 nid, flags);
Tejun Heo784656f92011-07-12 11:15:55 +0200643 }
644
nimisoloef3cc4d2016-07-26 15:24:56 -0700645 if (!nr_new)
646 return 0;
647
Tejun Heo784656f92011-07-12 11:15:55 +0200648 /*
649 * If this was the first round, resize array and repeat for actual
650 * insertions; otherwise, merge and return.
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700651 */
Tejun Heo784656f92011-07-12 11:15:55 +0200652 if (!insert) {
653 while (type->cnt + nr_new > type->max)
Greg Pearson48c3b582012-06-20 12:53:05 -0700654 if (memblock_double_array(type, obase, size) < 0)
Tejun Heo784656f92011-07-12 11:15:55 +0200655 return -ENOMEM;
656 insert = true;
657 goto repeat;
658 } else {
659 memblock_merge_regions(type);
660 return 0;
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -0700661 }
Yinghai Lu95f72d12010-07-12 14:36:09 +1000662}
663
Mike Rapoport48a833c2018-06-30 17:55:03 +0300664/**
665 * memblock_add_node - add new memblock region within a NUMA node
666 * @base: base address of the new region
667 * @size: size of the new region
668 * @nid: nid of the new region
669 *
670 * Add new memblock region [@base, @base + @size) to the "memory"
671 * type. See memblock_add_range() description for mode details
672 *
673 * Return:
674 * 0 on success, -errno on failure.
675 */
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800676int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
677 int nid)
678{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100679 return memblock_add_range(&memblock.memory, base, size, nid, 0);
Tejun Heo7fb0bc32011-12-08 10:22:08 -0800680}
681
Mike Rapoport48a833c2018-06-30 17:55:03 +0300682/**
683 * memblock_add - add new memblock region
684 * @base: base address of the new region
685 * @size: size of the new region
686 *
687 * Add new memblock region [@base, @base + @size) to the "memory"
688 * type. See memblock_add_range() description for mode details
689 *
690 * Return:
691 * 0 on success, -errno on failure.
692 */
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700693int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
Alexander Kuleshov6a4055b2015-04-15 16:14:44 -0700694{
Miles Chen5d63f812017-02-22 15:46:42 -0800695 phys_addr_t end = base + size - 1;
696
Sakari Ailusd75f7732019-03-25 21:32:28 +0200697 memblock_dbg("memblock_add: [%pa-%pa] %pS\n",
Miles Chen5d63f812017-02-22 15:46:42 -0800698 &base, &end, (void *)_RET_IP_);
Alexander Kuleshov6a4055b2015-04-15 16:14:44 -0700699
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700700 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000701}
702
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800703/**
704 * memblock_isolate_range - isolate given range into disjoint memblocks
705 * @type: memblock type to isolate range for
706 * @base: base of range to isolate
707 * @size: size of range to isolate
708 * @start_rgn: out parameter for the start of isolated region
709 * @end_rgn: out parameter for the end of isolated region
710 *
711 * Walk @type and ensure that regions don't cross the boundaries defined by
Mike Rapoport47cec442018-06-30 17:55:02 +0300712 * [@base, @base + @size). Crossing regions are split at the boundaries,
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800713 * which may create at most two more regions. The index of the first
714 * region inside the range is returned in *@start_rgn and end in *@end_rgn.
715 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300716 * Return:
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800717 * 0 on success, -errno on failure.
718 */
719static int __init_memblock memblock_isolate_range(struct memblock_type *type,
720 phys_addr_t base, phys_addr_t size,
721 int *start_rgn, int *end_rgn)
722{
Tejun Heoeb18f1b2011-12-08 10:22:07 -0800723 phys_addr_t end = base + memblock_cap_size(base, &size);
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800724 int idx;
725 struct memblock_region *rgn;
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800726
727 *start_rgn = *end_rgn = 0;
728
Tejun Heob3dc6272012-04-20 08:31:34 -0700729 if (!size)
730 return 0;
731
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800732 /* we'll create at most two more regions */
733 while (type->cnt + 2 > type->max)
Greg Pearson48c3b582012-06-20 12:53:05 -0700734 if (memblock_double_array(type, base, size) < 0)
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800735 return -ENOMEM;
736
Gioh Kim66e8b432017-11-15 17:33:42 -0800737 for_each_memblock_type(idx, type, rgn) {
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800738 phys_addr_t rbase = rgn->base;
739 phys_addr_t rend = rbase + rgn->size;
740
741 if (rbase >= end)
742 break;
743 if (rend <= base)
744 continue;
745
746 if (rbase < base) {
747 /*
748 * @rgn intersects from below. Split and continue
749 * to process the next region - the new top half.
750 */
751 rgn->base = base;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800752 rgn->size -= base - rbase;
753 type->total_size -= base - rbase;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800754 memblock_insert_region(type, idx, rbase, base - rbase,
Tang Chen66a20752014-01-21 15:49:20 -0800755 memblock_get_region_node(rgn),
756 rgn->flags);
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800757 } else if (rend > end) {
758 /*
759 * @rgn intersects from above. Split and redo the
760 * current region - the new bottom half.
761 */
762 rgn->base = end;
Tejun Heo1440c4e2011-12-08 10:22:08 -0800763 rgn->size -= end - rbase;
764 type->total_size -= end - rbase;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800765 memblock_insert_region(type, idx--, rbase, end - rbase,
Tang Chen66a20752014-01-21 15:49:20 -0800766 memblock_get_region_node(rgn),
767 rgn->flags);
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800768 } else {
769 /* @rgn is fully contained, record it */
770 if (!*end_rgn)
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -0800771 *start_rgn = idx;
772 *end_rgn = idx + 1;
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800773 }
774 }
775
776 return 0;
777}
Tejun Heo6a9ceb32011-12-08 10:22:07 -0800778
Alexander Kuleshov35bd16a2015-11-05 18:47:00 -0800779static int __init_memblock memblock_remove_range(struct memblock_type *type,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100780 phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000781{
Tejun Heo71936182011-12-08 10:22:07 -0800782 int start_rgn, end_rgn;
783 int i, ret;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000784
Tejun Heo71936182011-12-08 10:22:07 -0800785 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
786 if (ret)
787 return ret;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000788
Tejun Heo71936182011-12-08 10:22:07 -0800789 for (i = end_rgn - 1; i >= start_rgn; i--)
790 memblock_remove_region(type, i);
Benjamin Herrenschmidt8f7a6602011-03-22 16:33:43 -0700791 return 0;
Yinghai Lu95f72d12010-07-12 14:36:09 +1000792}
793
Tejun Heo581adcb2011-12-08 10:22:06 -0800794int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000795{
Minchan Kim25cf23d2018-06-07 17:07:35 -0700796 phys_addr_t end = base + size - 1;
797
798 memblock_dbg("memblock_remove: [%pa-%pa] %pS\n",
799 &base, &end, (void *)_RET_IP_);
800
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100801 return memblock_remove_range(&memblock.memory, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000802}
803
Mike Rapoport4d728682018-12-28 00:35:29 -0800804/**
805 * memblock_free - free boot memory block
806 * @base: phys starting address of the boot memory block
807 * @size: size of the boot memory block in bytes
808 *
809 * Free boot memory block previously allocated by memblock_alloc_xx() API.
810 * The freeing memory will not be released to the buddy allocator.
811 */
Tejun Heo581adcb2011-12-08 10:22:06 -0800812int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000813{
Miles Chen5d63f812017-02-22 15:46:42 -0800814 phys_addr_t end = base + size - 1;
815
Sakari Ailusd75f7732019-03-25 21:32:28 +0200816 memblock_dbg(" memblock_free: [%pa-%pa] %pS\n",
Miles Chen5d63f812017-02-22 15:46:42 -0800817 &base, &end, (void *)_RET_IP_);
Tejun Heo24aa0782011-07-12 11:16:06 +0200818
Catalin Marinas9099dae2016-10-11 13:55:11 -0700819 kmemleak_free_part_phys(base, size);
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100820 return memblock_remove_range(&memblock.reserved, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000821}
822
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700823int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +1000824{
Miles Chen5d63f812017-02-22 15:46:42 -0800825 phys_addr_t end = base + size - 1;
826
Sakari Ailusd75f7732019-03-25 21:32:28 +0200827 memblock_dbg("memblock_reserve: [%pa-%pa] %pS\n",
Miles Chen5d63f812017-02-22 15:46:42 -0800828 &base, &end, (void *)_RET_IP_);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000829
Alexander Kuleshovf705ac42016-05-20 16:57:35 -0700830 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
Yinghai Lu95f72d12010-07-12 14:36:09 +1000831}
832
Tejun Heo35fd0802011-07-12 11:15:59 +0200833/**
Mike Rapoport47cec442018-06-30 17:55:02 +0300834 * memblock_setclr_flag - set or clear flag for a memory region
835 * @base: base address of the region
836 * @size: size of the region
837 * @set: set or clear the flag
838 * @flag: the flag to udpate
Tang Chen66b16ed2014-01-21 15:49:23 -0800839 *
Tony Luck4308ce12014-12-12 16:54:59 -0800840 * This function isolates region [@base, @base + @size), and sets/clears flag
Tang Chen66b16ed2014-01-21 15:49:23 -0800841 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300842 * Return: 0 on success, -errno on failure.
Tang Chen66b16ed2014-01-21 15:49:23 -0800843 */
Tony Luck4308ce12014-12-12 16:54:59 -0800844static int __init_memblock memblock_setclr_flag(phys_addr_t base,
845 phys_addr_t size, int set, int flag)
Tang Chen66b16ed2014-01-21 15:49:23 -0800846{
847 struct memblock_type *type = &memblock.memory;
848 int i, ret, start_rgn, end_rgn;
849
850 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
851 if (ret)
852 return ret;
853
Mike Rapoportfe145122019-03-11 23:30:46 -0700854 for (i = start_rgn; i < end_rgn; i++) {
855 struct memblock_region *r = &type->regions[i];
856
Tony Luck4308ce12014-12-12 16:54:59 -0800857 if (set)
Mike Rapoportfe145122019-03-11 23:30:46 -0700858 r->flags |= flag;
Tony Luck4308ce12014-12-12 16:54:59 -0800859 else
Mike Rapoportfe145122019-03-11 23:30:46 -0700860 r->flags &= ~flag;
861 }
Tang Chen66b16ed2014-01-21 15:49:23 -0800862
863 memblock_merge_regions(type);
864 return 0;
865}
866
867/**
Tony Luck4308ce12014-12-12 16:54:59 -0800868 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
869 * @base: the base phys addr of the region
870 * @size: the size of the region
871 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300872 * Return: 0 on success, -errno on failure.
Tony Luck4308ce12014-12-12 16:54:59 -0800873 */
874int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
875{
876 return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
877}
878
879/**
Tang Chen66b16ed2014-01-21 15:49:23 -0800880 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
881 * @base: the base phys addr of the region
882 * @size: the size of the region
883 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300884 * Return: 0 on success, -errno on failure.
Tang Chen66b16ed2014-01-21 15:49:23 -0800885 */
886int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
887{
Tony Luck4308ce12014-12-12 16:54:59 -0800888 return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
Tang Chen66b16ed2014-01-21 15:49:23 -0800889}
890
891/**
Tony Lucka3f5baf2015-06-24 16:58:12 -0700892 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
893 * @base: the base phys addr of the region
894 * @size: the size of the region
895 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300896 * Return: 0 on success, -errno on failure.
Tony Lucka3f5baf2015-06-24 16:58:12 -0700897 */
898int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
899{
900 system_has_some_mirror = true;
901
902 return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
903}
904
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +0100905/**
906 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
907 * @base: the base phys addr of the region
908 * @size: the size of the region
909 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300910 * Return: 0 on success, -errno on failure.
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +0100911 */
912int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
913{
914 return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
915}
Tony Lucka3f5baf2015-06-24 16:58:12 -0700916
917/**
AKASHI Takahiro4c546b82017-04-03 11:23:54 +0900918 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
919 * @base: the base phys addr of the region
920 * @size: the size of the region
921 *
Mike Rapoport47cec442018-06-30 17:55:02 +0300922 * Return: 0 on success, -errno on failure.
AKASHI Takahiro4c546b82017-04-03 11:23:54 +0900923 */
924int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
925{
926 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
927}
928
929/**
Robin Holt8e7a7f82015-06-30 14:56:41 -0700930 * __next_reserved_mem_region - next function for for_each_reserved_region()
931 * @idx: pointer to u64 loop variable
932 * @out_start: ptr to phys_addr_t for start address of the region, can be %NULL
933 * @out_end: ptr to phys_addr_t for end address of the region, can be %NULL
934 *
935 * Iterate over all reserved memory regions.
936 */
937void __init_memblock __next_reserved_mem_region(u64 *idx,
938 phys_addr_t *out_start,
939 phys_addr_t *out_end)
940{
Alexander Kuleshov567d1172015-09-08 15:03:33 -0700941 struct memblock_type *type = &memblock.reserved;
Robin Holt8e7a7f82015-06-30 14:56:41 -0700942
Richard Leitnercd33a762016-05-20 16:58:33 -0700943 if (*idx < type->cnt) {
Alexander Kuleshov567d1172015-09-08 15:03:33 -0700944 struct memblock_region *r = &type->regions[*idx];
Robin Holt8e7a7f82015-06-30 14:56:41 -0700945 phys_addr_t base = r->base;
946 phys_addr_t size = r->size;
947
948 if (out_start)
949 *out_start = base;
950 if (out_end)
951 *out_end = base + size - 1;
952
953 *idx += 1;
954 return;
955 }
956
957 /* signal end of iteration */
958 *idx = ULLONG_MAX;
959}
960
Mike Rapoportc9a688a2019-03-11 23:30:50 -0700961static bool should_skip_region(struct memblock_region *m, int nid, int flags)
962{
963 int m_nid = memblock_get_region_node(m);
964
965 /* only memory regions are associated with nodes, check it */
966 if (nid != NUMA_NO_NODE && nid != m_nid)
967 return true;
968
969 /* skip hotpluggable memory regions if needed */
970 if (movable_node_is_enabled() && memblock_is_hotpluggable(m))
971 return true;
972
973 /* if we want mirror memory skip non-mirror memory regions */
974 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
975 return true;
976
977 /* skip nomap memory unless we were asked for it explicitly */
978 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
979 return true;
980
981 return false;
982}
983
Robin Holt8e7a7f82015-06-30 14:56:41 -0700984/**
Mike Rapoporta2974132019-03-11 23:30:54 -0700985 * __next_mem_range - next function for for_each_free_mem_range() etc.
Tejun Heo35fd0802011-07-12 11:15:59 +0200986 * @idx: pointer to u64 loop variable
Grygorii Strashkob1154232014-01-21 15:50:16 -0800987 * @nid: node selector, %NUMA_NO_NODE for all nodes
Tony Luckfc6daaf2015-06-24 16:58:09 -0700988 * @flags: pick from blocks based on memory attributes
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100989 * @type_a: pointer to memblock_type from where the range is taken
990 * @type_b: pointer to memblock_type which excludes memory from being taken
Wanpeng Lidad75572012-06-20 12:53:01 -0700991 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
992 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
993 * @out_nid: ptr to int for nid of the range, can be %NULL
Tejun Heo35fd0802011-07-12 11:15:59 +0200994 *
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100995 * Find the first area from *@idx which matches @nid, fill the out
Tejun Heo35fd0802011-07-12 11:15:59 +0200996 * parameters, and update *@idx for the next iteration. The lower 32bit of
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +0100997 * *@idx contains index into type_a and the upper 32bit indexes the
998 * areas before each region in type_b. For example, if type_b regions
Tejun Heo35fd0802011-07-12 11:15:59 +0200999 * look like the following,
1000 *
1001 * 0:[0-16), 1:[32-48), 2:[128-130)
1002 *
1003 * The upper 32bit indexes the following regions.
1004 *
1005 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
1006 *
1007 * As both region arrays are sorted, the function advances the two indices
1008 * in lockstep and returns each intersection.
1009 */
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001010void __init_memblock __next_mem_range(u64 *idx, int nid,
1011 enum memblock_flags flags,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001012 struct memblock_type *type_a,
1013 struct memblock_type *type_b,
1014 phys_addr_t *out_start,
1015 phys_addr_t *out_end, int *out_nid)
Tejun Heo35fd0802011-07-12 11:15:59 +02001016{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001017 int idx_a = *idx & 0xffffffff;
1018 int idx_b = *idx >> 32;
Grygorii Strashkob1154232014-01-21 15:50:16 -08001019
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001020 if (WARN_ONCE(nid == MAX_NUMNODES,
1021 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
Grygorii Strashko560dca272014-01-21 15:50:55 -08001022 nid = NUMA_NO_NODE;
Tejun Heo35fd0802011-07-12 11:15:59 +02001023
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001024 for (; idx_a < type_a->cnt; idx_a++) {
1025 struct memblock_region *m = &type_a->regions[idx_a];
1026
Tejun Heo35fd0802011-07-12 11:15:59 +02001027 phys_addr_t m_start = m->base;
1028 phys_addr_t m_end = m->base + m->size;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001029 int m_nid = memblock_get_region_node(m);
Tejun Heo35fd0802011-07-12 11:15:59 +02001030
Mike Rapoportc9a688a2019-03-11 23:30:50 -07001031 if (should_skip_region(m, nid, flags))
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001032 continue;
1033
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001034 if (!type_b) {
1035 if (out_start)
1036 *out_start = m_start;
1037 if (out_end)
1038 *out_end = m_end;
1039 if (out_nid)
1040 *out_nid = m_nid;
1041 idx_a++;
1042 *idx = (u32)idx_a | (u64)idx_b << 32;
1043 return;
1044 }
Tejun Heo35fd0802011-07-12 11:15:59 +02001045
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001046 /* scan areas before each reservation */
1047 for (; idx_b < type_b->cnt + 1; idx_b++) {
1048 struct memblock_region *r;
1049 phys_addr_t r_start;
1050 phys_addr_t r_end;
1051
1052 r = &type_b->regions[idx_b];
1053 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1054 r_end = idx_b < type_b->cnt ?
Stefan Agner1c4bc432018-06-07 17:06:15 -07001055 r->base : PHYS_ADDR_MAX;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001056
1057 /*
1058 * if idx_b advanced past idx_a,
1059 * break out to advance idx_a
1060 */
Tejun Heo35fd0802011-07-12 11:15:59 +02001061 if (r_start >= m_end)
1062 break;
1063 /* if the two regions intersect, we're done */
1064 if (m_start < r_end) {
1065 if (out_start)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001066 *out_start =
1067 max(m_start, r_start);
Tejun Heo35fd0802011-07-12 11:15:59 +02001068 if (out_end)
1069 *out_end = min(m_end, r_end);
1070 if (out_nid)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001071 *out_nid = m_nid;
Tejun Heo35fd0802011-07-12 11:15:59 +02001072 /*
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001073 * The region which ends first is
1074 * advanced for the next iteration.
Tejun Heo35fd0802011-07-12 11:15:59 +02001075 */
1076 if (m_end <= r_end)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001077 idx_a++;
Tejun Heo35fd0802011-07-12 11:15:59 +02001078 else
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001079 idx_b++;
1080 *idx = (u32)idx_a | (u64)idx_b << 32;
Tejun Heo35fd0802011-07-12 11:15:59 +02001081 return;
1082 }
1083 }
1084 }
1085
1086 /* signal end of iteration */
1087 *idx = ULLONG_MAX;
1088}
1089
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001090/**
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001091 * __next_mem_range_rev - generic next function for for_each_*_range_rev()
1092 *
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001093 * @idx: pointer to u64 loop variable
Alexander Kuleshovad5ea8c2015-09-08 15:04:22 -07001094 * @nid: node selector, %NUMA_NO_NODE for all nodes
Tony Luckfc6daaf2015-06-24 16:58:09 -07001095 * @flags: pick from blocks based on memory attributes
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001096 * @type_a: pointer to memblock_type from where the range is taken
1097 * @type_b: pointer to memblock_type which excludes memory from being taken
Wanpeng Lidad75572012-06-20 12:53:01 -07001098 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1099 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1100 * @out_nid: ptr to int for nid of the range, can be %NULL
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001101 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001102 * Finds the next range from type_a which is not marked as unsuitable
1103 * in type_b.
1104 *
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001105 * Reverse of __next_mem_range().
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001106 */
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001107void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
1108 enum memblock_flags flags,
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001109 struct memblock_type *type_a,
1110 struct memblock_type *type_b,
1111 phys_addr_t *out_start,
1112 phys_addr_t *out_end, int *out_nid)
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001113{
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001114 int idx_a = *idx & 0xffffffff;
1115 int idx_b = *idx >> 32;
Grygorii Strashkob1154232014-01-21 15:50:16 -08001116
Grygorii Strashko560dca272014-01-21 15:50:55 -08001117 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1118 nid = NUMA_NO_NODE;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001119
1120 if (*idx == (u64)ULLONG_MAX) {
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001121 idx_a = type_a->cnt - 1;
zijun_hue47608a2016-08-04 15:32:00 -07001122 if (type_b != NULL)
1123 idx_b = type_b->cnt;
1124 else
1125 idx_b = 0;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001126 }
1127
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001128 for (; idx_a >= 0; idx_a--) {
1129 struct memblock_region *m = &type_a->regions[idx_a];
1130
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001131 phys_addr_t m_start = m->base;
1132 phys_addr_t m_end = m->base + m->size;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001133 int m_nid = memblock_get_region_node(m);
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001134
Mike Rapoportc9a688a2019-03-11 23:30:50 -07001135 if (should_skip_region(m, nid, flags))
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001136 continue;
1137
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001138 if (!type_b) {
1139 if (out_start)
1140 *out_start = m_start;
1141 if (out_end)
1142 *out_end = m_end;
1143 if (out_nid)
1144 *out_nid = m_nid;
zijun_hufb399b42016-07-28 15:48:56 -07001145 idx_a--;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001146 *idx = (u32)idx_a | (u64)idx_b << 32;
1147 return;
1148 }
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001149
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001150 /* scan areas before each reservation */
1151 for (; idx_b >= 0; idx_b--) {
1152 struct memblock_region *r;
1153 phys_addr_t r_start;
1154 phys_addr_t r_end;
1155
1156 r = &type_b->regions[idx_b];
1157 r_start = idx_b ? r[-1].base + r[-1].size : 0;
1158 r_end = idx_b < type_b->cnt ?
Stefan Agner1c4bc432018-06-07 17:06:15 -07001159 r->base : PHYS_ADDR_MAX;
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001160 /*
1161 * if idx_b advanced past idx_a,
1162 * break out to advance idx_a
1163 */
1164
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001165 if (r_end <= m_start)
1166 break;
1167 /* if the two regions intersect, we're done */
1168 if (m_end > r_start) {
1169 if (out_start)
1170 *out_start = max(m_start, r_start);
1171 if (out_end)
1172 *out_end = min(m_end, r_end);
1173 if (out_nid)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001174 *out_nid = m_nid;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001175 if (m_start >= r_start)
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001176 idx_a--;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001177 else
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001178 idx_b--;
1179 *idx = (u32)idx_a | (u64)idx_b << 32;
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001180 return;
1181 }
1182 }
1183 }
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001184 /* signal end of iteration */
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001185 *idx = ULLONG_MAX;
1186}
1187
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001188#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1189/*
Chen Chang45e79812018-11-16 15:08:57 -08001190 * Common iterator interface used to define for_each_mem_pfn_range().
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001191 */
1192void __init_memblock __next_mem_pfn_range(int *idx, int nid,
1193 unsigned long *out_start_pfn,
1194 unsigned long *out_end_pfn, int *out_nid)
1195{
1196 struct memblock_type *type = &memblock.memory;
1197 struct memblock_region *r;
1198
1199 while (++*idx < type->cnt) {
1200 r = &type->regions[*idx];
1201
1202 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
1203 continue;
1204 if (nid == MAX_NUMNODES || nid == r->nid)
1205 break;
1206 }
1207 if (*idx >= type->cnt) {
1208 *idx = -1;
1209 return;
1210 }
1211
1212 if (out_start_pfn)
1213 *out_start_pfn = PFN_UP(r->base);
1214 if (out_end_pfn)
1215 *out_end_pfn = PFN_DOWN(r->base + r->size);
1216 if (out_nid)
1217 *out_nid = r->nid;
1218}
1219
1220/**
1221 * memblock_set_node - set node ID on memblock regions
1222 * @base: base of area to set node ID for
1223 * @size: size of area to set node ID for
Tang Chene7e8de52014-01-21 15:49:26 -08001224 * @type: memblock type to set node ID for
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001225 * @nid: node ID to set
1226 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001227 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001228 * Regions which cross the area boundaries are split as necessary.
1229 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001230 * Return:
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001231 * 0 on success, -errno on failure.
1232 */
1233int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
Tang Chene7e8de52014-01-21 15:49:26 -08001234 struct memblock_type *type, int nid)
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001235{
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001236 int start_rgn, end_rgn;
1237 int i, ret;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001238
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001239 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
1240 if (ret)
1241 return ret;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001242
Tejun Heo6a9ceb32011-12-08 10:22:07 -08001243 for (i = start_rgn; i < end_rgn; i++)
Wanpeng Lie9d24ad2012-10-08 16:32:21 -07001244 memblock_set_region_node(&type->regions[i], nid);
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001245
1246 memblock_merge_regions(type);
1247 return 0;
1248}
1249#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Alexander Duyck837566e2019-05-13 17:21:17 -07001250#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1251/**
1252 * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
1253 *
1254 * @idx: pointer to u64 loop variable
1255 * @zone: zone in which all of the memory blocks reside
1256 * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
1257 * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
1258 *
1259 * This function is meant to be a zone/pfn specific wrapper for the
1260 * for_each_mem_range type iterators. Specifically they are used in the
1261 * deferred memory init routines and as such we were duplicating much of
1262 * this logic throughout the code. So instead of having it in multiple
1263 * locations it seemed like it would make more sense to centralize this to
1264 * one new iterator that does everything they need.
1265 */
1266void __init_memblock
1267__next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
1268 unsigned long *out_spfn, unsigned long *out_epfn)
1269{
1270 int zone_nid = zone_to_nid(zone);
1271 phys_addr_t spa, epa;
1272 int nid;
1273
1274 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1275 &memblock.memory, &memblock.reserved,
1276 &spa, &epa, &nid);
1277
1278 while (*idx != U64_MAX) {
1279 unsigned long epfn = PFN_DOWN(epa);
1280 unsigned long spfn = PFN_UP(spa);
1281
1282 /*
1283 * Verify the end is at least past the start of the zone and
1284 * that we have at least one PFN to initialize.
1285 */
1286 if (zone->zone_start_pfn < epfn && spfn < epfn) {
1287 /* if we went too far just stop searching */
1288 if (zone_end_pfn(zone) <= spfn) {
1289 *idx = U64_MAX;
1290 break;
1291 }
1292
1293 if (out_spfn)
1294 *out_spfn = max(zone->zone_start_pfn, spfn);
1295 if (out_epfn)
1296 *out_epfn = min(zone_end_pfn(zone), epfn);
1297
1298 return;
1299 }
1300
1301 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1302 &memblock.memory, &memblock.reserved,
1303 &spa, &epa, &nid);
1304 }
1305
1306 /* signal end of iteration */
1307 if (out_spfn)
1308 *out_spfn = ULONG_MAX;
1309 if (out_epfn)
1310 *out_epfn = 0;
1311}
1312
1313#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001314
Mike Rapoport92d12f92019-03-11 23:29:41 -07001315/**
1316 * memblock_alloc_range_nid - allocate boot memory block
1317 * @size: size of memory block to be allocated in bytes
1318 * @align: alignment of the region and block's size
1319 * @start: the lower bound of the memory region to allocate (phys address)
1320 * @end: the upper bound of the memory region to allocate (phys address)
1321 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001322 * @exact_nid: control the allocation fall back to other nodes
Mike Rapoport92d12f92019-03-11 23:29:41 -07001323 *
1324 * The allocation is performed from memory region limited by
Cao jin95830662019-11-30 17:56:24 -08001325 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
Mike Rapoport92d12f92019-03-11 23:29:41 -07001326 *
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001327 * If the specified node can not hold the requested memory and @exact_nid
1328 * is false, the allocation falls back to any node in the system.
Mike Rapoport92d12f92019-03-11 23:29:41 -07001329 *
1330 * For systems with memory mirroring, the allocation is attempted first
1331 * from the regions with mirroring enabled and then retried from any
1332 * memory region.
1333 *
1334 * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for
1335 * allocated boot memory block, so that it is never reported as leaks.
1336 *
1337 * Return:
1338 * Physical address of allocated memory block on success, %0 on failure.
1339 */
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001340static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
1341 phys_addr_t align, phys_addr_t start,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001342 phys_addr_t end, int nid,
1343 bool exact_nid)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001344{
Mike Rapoport92d12f92019-03-11 23:29:41 -07001345 enum memblock_flags flags = choose_memblock_flags();
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001346 phys_addr_t found;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001347
Mike Rapoport92d12f92019-03-11 23:29:41 -07001348 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1349 nid = NUMA_NO_NODE;
1350
Mike Rapoport2f770802018-10-30 15:10:01 -07001351 if (!align) {
1352 /* Can't use WARNs this early in boot on powerpc */
1353 dump_stack();
1354 align = SMP_CACHE_BYTES;
1355 }
1356
Mike Rapoport92d12f92019-03-11 23:29:41 -07001357again:
Tony Luckfc6daaf2015-06-24 16:58:09 -07001358 found = memblock_find_in_range_node(size, align, start, end, nid,
1359 flags);
Mike Rapoport92d12f92019-03-11 23:29:41 -07001360 if (found && !memblock_reserve(found, size))
1361 goto done;
1362
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001363 if (nid != NUMA_NO_NODE && !exact_nid) {
Mike Rapoport92d12f92019-03-11 23:29:41 -07001364 found = memblock_find_in_range_node(size, align, start,
1365 end, NUMA_NO_NODE,
1366 flags);
1367 if (found && !memblock_reserve(found, size))
1368 goto done;
1369 }
1370
1371 if (flags & MEMBLOCK_MIRROR) {
1372 flags &= ~MEMBLOCK_MIRROR;
1373 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
1374 &size);
1375 goto again;
1376 }
1377
1378 return 0;
1379
1380done:
1381 /* Skip kmemleak for kasan_init() due to high volume. */
1382 if (end != MEMBLOCK_ALLOC_KASAN)
Catalin Marinasaedf95e2014-06-06 14:38:20 -07001383 /*
Mike Rapoport92d12f92019-03-11 23:29:41 -07001384 * The min_count is set to 0 so that memblock allocated
1385 * blocks are never reported as leaks. This is because many
1386 * of these blocks are only referred via the physical
1387 * address which is not looked up by kmemleak.
Catalin Marinasaedf95e2014-06-06 14:38:20 -07001388 */
Catalin Marinas9099dae2016-10-11 13:55:11 -07001389 kmemleak_alloc_phys(found, size, 0, 0);
Mike Rapoport92d12f92019-03-11 23:29:41 -07001390
1391 return found;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001392}
1393
Mike Rapoporta2974132019-03-11 23:30:54 -07001394/**
1395 * memblock_phys_alloc_range - allocate a memory block inside specified range
1396 * @size: size of memory block to be allocated in bytes
1397 * @align: alignment of the region and block's size
1398 * @start: the lower bound of the memory region to allocate (physical address)
1399 * @end: the upper bound of the memory region to allocate (physical address)
1400 *
1401 * Allocate @size bytes in the between @start and @end.
1402 *
1403 * Return: physical address of the allocated memory block on success,
1404 * %0 on failure.
1405 */
Mike Rapoport8a770c22019-03-11 23:29:16 -07001406phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
1407 phys_addr_t align,
1408 phys_addr_t start,
1409 phys_addr_t end)
Akinobu Mita2bfc2862014-06-04 16:06:53 -07001410{
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001411 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1412 false);
Tejun Heo7bd0b0f2011-12-08 10:22:09 -08001413}
1414
Mike Rapoporta2974132019-03-11 23:30:54 -07001415/**
1416 * memblock_phys_alloc_try_nid - allocate a memory block from specified MUMA node
1417 * @size: size of memory block to be allocated in bytes
1418 * @align: alignment of the region and block's size
1419 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1420 *
1421 * Allocates memory block from the specified NUMA node. If the node
1422 * has no available memory, attempts to allocated from any node in the
1423 * system.
1424 *
1425 * Return: physical address of the allocated memory block on success,
1426 * %0 on failure.
1427 */
Mike Rapoport9a8dd702018-10-30 15:07:59 -07001428phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
Benjamin Herrenschmidt9d1e2492010-07-06 15:39:17 -07001429{
Mike Rapoport33755572019-03-11 23:29:21 -07001430 return memblock_alloc_range_nid(size, align, 0,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001431 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001432}
1433
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001434/**
Mike Rapoporteb31d552018-10-30 15:08:04 -07001435 * memblock_alloc_internal - allocate boot memory block
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001436 * @size: size of memory block to be allocated in bytes
1437 * @align: alignment of the region and block's size
1438 * @min_addr: the lower bound of the memory region to allocate (phys address)
1439 * @max_addr: the upper bound of the memory region to allocate (phys address)
1440 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001441 * @exact_nid: control the allocation fall back to other nodes
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001442 *
Mike Rapoport92d12f92019-03-11 23:29:41 -07001443 * Allocates memory block using memblock_alloc_range_nid() and
1444 * converts the returned physical address to virtual.
1445 *
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001446 * The @min_addr limit is dropped if it can not be satisfied and the allocation
Mike Rapoport92d12f92019-03-11 23:29:41 -07001447 * will fall back to memory below @min_addr. Other constraints, such
1448 * as node and mirrored memory will be handled again in
1449 * memblock_alloc_range_nid().
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001450 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001451 * Return:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001452 * Virtual address of allocated memory block on success, NULL on failure.
1453 */
Mike Rapoporteb31d552018-10-30 15:08:04 -07001454static void * __init memblock_alloc_internal(
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001455 phys_addr_t size, phys_addr_t align,
1456 phys_addr_t min_addr, phys_addr_t max_addr,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001457 int nid, bool exact_nid)
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001458{
1459 phys_addr_t alloc;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001460
1461 /*
1462 * Detect any accidental use of these APIs after slab is ready, as at
1463 * this moment memblock may be deinitialized already and its
Mike Rapoportc6ffc5c2018-10-30 15:09:30 -07001464 * internal data may be destroyed (after execution of memblock_free_all)
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001465 */
1466 if (WARN_ON_ONCE(slab_is_available()))
1467 return kzalloc_node(size, GFP_NOWAIT, nid);
1468
Mike Rapoportf3057ad2019-10-18 20:20:01 -07001469 if (max_addr > memblock.current_limit)
1470 max_addr = memblock.current_limit;
1471
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001472 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
1473 exact_nid);
Mike Rapoport2f770802018-10-30 15:10:01 -07001474
Mike Rapoport92d12f92019-03-11 23:29:41 -07001475 /* retry allocation without lower limit */
1476 if (!alloc && min_addr)
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001477 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
1478 exact_nid);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001479
Mike Rapoport92d12f92019-03-11 23:29:41 -07001480 if (!alloc)
1481 return NULL;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001482
Mike Rapoport92d12f92019-03-11 23:29:41 -07001483 return phys_to_virt(alloc);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001484}
1485
1486/**
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001487 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
1488 * without zeroing memory
1489 * @size: size of memory block to be allocated in bytes
1490 * @align: alignment of the region and block's size
1491 * @min_addr: the lower bound of the memory region from where the allocation
1492 * is preferred (phys address)
1493 * @max_addr: the upper bound of the memory region from where the allocation
1494 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1495 * allocate only from memory limited by memblock.current_limit value
1496 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1497 *
1498 * Public function, provides additional debug information (including caller
1499 * info), if enabled. Does not zero allocated memory.
1500 *
1501 * Return:
1502 * Virtual address of allocated memory block on success, NULL on failure.
1503 */
1504void * __init memblock_alloc_exact_nid_raw(
1505 phys_addr_t size, phys_addr_t align,
1506 phys_addr_t min_addr, phys_addr_t max_addr,
1507 int nid)
1508{
1509 void *ptr;
1510
1511 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1512 __func__, (u64)size, (u64)align, nid, &min_addr,
1513 &max_addr, (void *)_RET_IP_);
1514
1515 ptr = memblock_alloc_internal(size, align,
1516 min_addr, max_addr, nid, true);
1517 if (ptr && size > 0)
1518 page_init_poison(ptr, size);
1519
1520 return ptr;
1521}
1522
1523/**
Mike Rapoporteb31d552018-10-30 15:08:04 -07001524 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001525 * memory and without panicking
1526 * @size: size of memory block to be allocated in bytes
1527 * @align: alignment of the region and block's size
1528 * @min_addr: the lower bound of the memory region from where the allocation
1529 * is preferred (phys address)
1530 * @max_addr: the upper bound of the memory region from where the allocation
Mike Rapoport97ad1082018-10-30 15:09:44 -07001531 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001532 * allocate only from memory limited by memblock.current_limit value
1533 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1534 *
1535 * Public function, provides additional debug information (including caller
1536 * info), if enabled. Does not zero allocated memory, does not panic if request
1537 * cannot be satisfied.
1538 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001539 * Return:
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001540 * Virtual address of allocated memory block on success, NULL on failure.
1541 */
Mike Rapoporteb31d552018-10-30 15:08:04 -07001542void * __init memblock_alloc_try_nid_raw(
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001543 phys_addr_t size, phys_addr_t align,
1544 phys_addr_t min_addr, phys_addr_t max_addr,
1545 int nid)
1546{
1547 void *ptr;
1548
Sakari Ailusd75f7732019-03-25 21:32:28 +02001549 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
Mike Rapoporta36aab82018-08-17 15:47:17 -07001550 __func__, (u64)size, (u64)align, nid, &min_addr,
1551 &max_addr, (void *)_RET_IP_);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001552
Mike Rapoporteb31d552018-10-30 15:08:04 -07001553 ptr = memblock_alloc_internal(size, align,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001554 min_addr, max_addr, nid, false);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001555 if (ptr && size > 0)
Alexander Duyckf682a972018-10-26 15:07:45 -07001556 page_init_poison(ptr, size);
1557
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001558 return ptr;
1559}
1560
1561/**
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001562 * memblock_alloc_try_nid - allocate boot memory block
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001563 * @size: size of memory block to be allocated in bytes
1564 * @align: alignment of the region and block's size
1565 * @min_addr: the lower bound of the memory region from where the allocation
1566 * is preferred (phys address)
1567 * @max_addr: the upper bound of the memory region from where the allocation
Mike Rapoport97ad1082018-10-30 15:09:44 -07001568 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001569 * allocate only from memory limited by memblock.current_limit value
1570 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1571 *
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001572 * Public function, provides additional debug information (including caller
1573 * info), if enabled. This function zeroes the allocated memory.
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001574 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001575 * Return:
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001576 * Virtual address of allocated memory block on success, NULL on failure.
1577 */
Mike Rapoporteb31d552018-10-30 15:08:04 -07001578void * __init memblock_alloc_try_nid(
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001579 phys_addr_t size, phys_addr_t align,
1580 phys_addr_t min_addr, phys_addr_t max_addr,
1581 int nid)
1582{
1583 void *ptr;
1584
Sakari Ailusd75f7732019-03-25 21:32:28 +02001585 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
Mike Rapoporta36aab82018-08-17 15:47:17 -07001586 __func__, (u64)size, (u64)align, nid, &min_addr,
1587 &max_addr, (void *)_RET_IP_);
Mike Rapoporteb31d552018-10-30 15:08:04 -07001588 ptr = memblock_alloc_internal(size, align,
Yunfeng Ye0ac398b2019-11-30 17:56:27 -08001589 min_addr, max_addr, nid, false);
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001590 if (ptr)
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08001591 memset(ptr, 0, size);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001592
Mike Rapoportc0dbe822019-03-11 23:30:37 -07001593 return ptr;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001594}
1595
1596/**
Mike Rapoporta2974132019-03-11 23:30:54 -07001597 * __memblock_free_late - free pages directly to buddy allocator
Mike Rapoport48a833c2018-06-30 17:55:03 +03001598 * @base: phys starting address of the boot memory block
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001599 * @size: size of the boot memory block in bytes
1600 *
Mike Rapoporta2974132019-03-11 23:30:54 -07001601 * This is only useful when the memblock allocator has already been torn
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001602 * down, but we are still initializing the system. Pages are released directly
Mike Rapoporta2974132019-03-11 23:30:54 -07001603 * to the buddy allocator.
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001604 */
1605void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)
1606{
Mike Rapoporta36aab82018-08-17 15:47:17 -07001607 phys_addr_t cursor, end;
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001608
Mike Rapoporta36aab82018-08-17 15:47:17 -07001609 end = base + size - 1;
Sakari Ailusd75f7732019-03-25 21:32:28 +02001610 memblock_dbg("%s: [%pa-%pa] %pS\n",
Mike Rapoporta36aab82018-08-17 15:47:17 -07001611 __func__, &base, &end, (void *)_RET_IP_);
Catalin Marinas9099dae2016-10-11 13:55:11 -07001612 kmemleak_free_part_phys(base, size);
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001613 cursor = PFN_UP(base);
1614 end = PFN_DOWN(base + size);
1615
1616 for (; cursor < end; cursor++) {
Mike Rapoport7c2ee342018-10-30 15:09:36 -07001617 memblock_free_pages(pfn_to_page(cursor), cursor, 0);
Arun KSca79b0c2018-12-28 00:34:29 -08001618 totalram_pages_inc();
Santosh Shilimkar26f09e92014-01-21 15:50:19 -08001619 }
1620}
Benjamin Herrenschmidt9d1e2492010-07-06 15:39:17 -07001621
1622/*
1623 * Remaining API functions
1624 */
1625
David Gibson1f1ffb8a2016-02-05 15:36:19 -08001626phys_addr_t __init_memblock memblock_phys_mem_size(void)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001627{
Tejun Heo1440c4e2011-12-08 10:22:08 -08001628 return memblock.memory.total_size;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001629}
1630
Srikar Dronamraju8907de52016-10-07 16:59:18 -07001631phys_addr_t __init_memblock memblock_reserved_size(void)
1632{
1633 return memblock.reserved.total_size;
1634}
1635
Yinghai Lu595ad9a2013-01-24 12:20:09 -08001636phys_addr_t __init memblock_mem_size(unsigned long limit_pfn)
1637{
1638 unsigned long pages = 0;
1639 struct memblock_region *r;
1640 unsigned long start_pfn, end_pfn;
1641
1642 for_each_memblock(memory, r) {
1643 start_pfn = memblock_region_memory_base_pfn(r);
1644 end_pfn = memblock_region_memory_end_pfn(r);
1645 start_pfn = min_t(unsigned long, start_pfn, limit_pfn);
1646 end_pfn = min_t(unsigned long, end_pfn, limit_pfn);
1647 pages += end_pfn - start_pfn;
1648 }
1649
Fabian Frederick16763232014-04-07 15:37:53 -07001650 return PFN_PHYS(pages);
Yinghai Lu595ad9a2013-01-24 12:20:09 -08001651}
1652
Sam Ravnborg0a93ebe2011-10-31 17:08:16 -07001653/* lowest address */
1654phys_addr_t __init_memblock memblock_start_of_DRAM(void)
1655{
1656 return memblock.memory.regions[0].base;
1657}
1658
Yinghai Lu10d06432010-07-28 15:43:02 +10001659phys_addr_t __init_memblock memblock_end_of_DRAM(void)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001660{
1661 int idx = memblock.memory.cnt - 1;
1662
Benjamin Herrenschmidte3239ff2010-08-04 14:06:41 +10001663 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001664}
1665
Dennis Chena571d4e2016-07-28 15:48:26 -07001666static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001667{
Stefan Agner1c4bc432018-06-07 17:06:15 -07001668 phys_addr_t max_addr = PHYS_ADDR_MAX;
Emil Medve136199f2014-04-07 15:37:52 -07001669 struct memblock_region *r;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001670
Dennis Chena571d4e2016-07-28 15:48:26 -07001671 /*
1672 * translate the memory @limit size into the max address within one of
1673 * the memory memblock regions, if the @limit exceeds the total size
Stefan Agner1c4bc432018-06-07 17:06:15 -07001674 * of those regions, max_addr will keep original value PHYS_ADDR_MAX
Dennis Chena571d4e2016-07-28 15:48:26 -07001675 */
Emil Medve136199f2014-04-07 15:37:52 -07001676 for_each_memblock(memory, r) {
Tejun Heoc0ce8fe2011-12-08 10:22:07 -08001677 if (limit <= r->size) {
1678 max_addr = r->base + limit;
1679 break;
1680 }
1681 limit -= r->size;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001682 }
1683
Dennis Chena571d4e2016-07-28 15:48:26 -07001684 return max_addr;
1685}
1686
1687void __init memblock_enforce_memory_limit(phys_addr_t limit)
1688{
Stefan Agner1c4bc432018-06-07 17:06:15 -07001689 phys_addr_t max_addr = PHYS_ADDR_MAX;
Dennis Chena571d4e2016-07-28 15:48:26 -07001690
1691 if (!limit)
1692 return;
1693
1694 max_addr = __find_max_addr(limit);
1695
1696 /* @limit exceeds the total size of the memory, do nothing */
Stefan Agner1c4bc432018-06-07 17:06:15 -07001697 if (max_addr == PHYS_ADDR_MAX)
Dennis Chena571d4e2016-07-28 15:48:26 -07001698 return;
1699
Tejun Heoc0ce8fe2011-12-08 10:22:07 -08001700 /* truncate both memory and reserved regions */
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001701 memblock_remove_range(&memblock.memory, max_addr,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001702 PHYS_ADDR_MAX);
Philipp Hachtmannf1af9d32014-01-29 18:16:01 +01001703 memblock_remove_range(&memblock.reserved, max_addr,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001704 PHYS_ADDR_MAX);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001705}
1706
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001707void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
1708{
1709 int start_rgn, end_rgn;
1710 int i, ret;
1711
1712 if (!size)
1713 return;
1714
1715 ret = memblock_isolate_range(&memblock.memory, base, size,
1716 &start_rgn, &end_rgn);
1717 if (ret)
1718 return;
1719
1720 /* remove all the MAP regions */
1721 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
1722 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1723 memblock_remove_region(&memblock.memory, i);
1724
1725 for (i = start_rgn - 1; i >= 0; i--)
1726 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1727 memblock_remove_region(&memblock.memory, i);
1728
1729 /* truncate the reserved regions */
1730 memblock_remove_range(&memblock.reserved, 0, base);
1731 memblock_remove_range(&memblock.reserved,
Stefan Agner1c4bc432018-06-07 17:06:15 -07001732 base + size, PHYS_ADDR_MAX);
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001733}
1734
Dennis Chena571d4e2016-07-28 15:48:26 -07001735void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1736{
Dennis Chena571d4e2016-07-28 15:48:26 -07001737 phys_addr_t max_addr;
Dennis Chena571d4e2016-07-28 15:48:26 -07001738
1739 if (!limit)
1740 return;
1741
1742 max_addr = __find_max_addr(limit);
1743
1744 /* @limit exceeds the total size of the memory, do nothing */
Stefan Agner1c4bc432018-06-07 17:06:15 -07001745 if (max_addr == PHYS_ADDR_MAX)
Dennis Chena571d4e2016-07-28 15:48:26 -07001746 return;
1747
AKASHI Takahiroc9ca9b42017-04-03 11:23:55 +09001748 memblock_cap_memory_range(0, max_addr);
Dennis Chena571d4e2016-07-28 15:48:26 -07001749}
1750
Yinghai Lucd794812010-10-11 12:34:09 -07001751static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001752{
1753 unsigned int left = 0, right = type->cnt;
1754
1755 do {
1756 unsigned int mid = (right + left) / 2;
1757
1758 if (addr < type->regions[mid].base)
1759 right = mid;
1760 else if (addr >= (type->regions[mid].base +
1761 type->regions[mid].size))
1762 left = mid + 1;
1763 else
1764 return mid;
1765 } while (left < right);
1766 return -1;
1767}
1768
Yueyi Lif5a222d2018-12-14 14:17:06 -08001769bool __init_memblock memblock_is_reserved(phys_addr_t addr)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001770{
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001771 return memblock_search(&memblock.reserved, addr) != -1;
1772}
Yinghai Lu95f72d12010-07-12 14:36:09 +10001773
Yaowei Baib4ad0c72016-01-14 15:18:54 -08001774bool __init_memblock memblock_is_memory(phys_addr_t addr)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001775{
1776 return memblock_search(&memblock.memory, addr) != -1;
1777}
1778
Yaowei Bai937f0c22018-02-06 15:41:18 -08001779bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
Ard Biesheuvelbf3d3cc2015-11-30 13:28:15 +01001780{
1781 int i = memblock_search(&memblock.memory, addr);
1782
1783 if (i == -1)
1784 return false;
1785 return !memblock_is_nomap(&memblock.memory.regions[i]);
1786}
1787
Yinghai Lue76b63f2013-09-11 14:22:17 -07001788#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1789int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
1790 unsigned long *start_pfn, unsigned long *end_pfn)
1791{
1792 struct memblock_type *type = &memblock.memory;
Fabian Frederick16763232014-04-07 15:37:53 -07001793 int mid = memblock_search(type, PFN_PHYS(pfn));
Yinghai Lue76b63f2013-09-11 14:22:17 -07001794
1795 if (mid == -1)
1796 return -1;
1797
Fabian Frederickf7e2f7e2014-06-04 16:07:51 -07001798 *start_pfn = PFN_DOWN(type->regions[mid].base);
1799 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
Yinghai Lue76b63f2013-09-11 14:22:17 -07001800
1801 return type->regions[mid].nid;
1802}
1803#endif
1804
Stephen Boydeab30942012-05-24 00:45:21 -07001805/**
1806 * memblock_is_region_memory - check if a region is a subset of memory
1807 * @base: base of region to check
1808 * @size: size of region to check
1809 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001810 * Check if the region [@base, @base + @size) is a subset of a memory block.
Stephen Boydeab30942012-05-24 00:45:21 -07001811 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001812 * Return:
Stephen Boydeab30942012-05-24 00:45:21 -07001813 * 0 if false, non-zero if true
1814 */
Yaowei Bai937f0c22018-02-06 15:41:18 -08001815bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001816{
Tomi Valkeinenabb65272011-01-20 14:44:20 -08001817 int idx = memblock_search(&memblock.memory, base);
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001818 phys_addr_t end = base + memblock_cap_size(base, &size);
Benjamin Herrenschmidt72d4b0b2010-08-04 14:38:47 +10001819
1820 if (idx == -1)
Yaowei Bai937f0c22018-02-06 15:41:18 -08001821 return false;
Wei Yangef415ef2017-02-22 15:45:04 -08001822 return (memblock.memory.regions[idx].base +
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001823 memblock.memory.regions[idx].size) >= end;
Yinghai Lu95f72d12010-07-12 14:36:09 +10001824}
1825
Stephen Boydeab30942012-05-24 00:45:21 -07001826/**
1827 * memblock_is_region_reserved - check if a region intersects reserved memory
1828 * @base: base of region to check
1829 * @size: size of region to check
1830 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001831 * Check if the region [@base, @base + @size) intersects a reserved
1832 * memory block.
Stephen Boydeab30942012-05-24 00:45:21 -07001833 *
Mike Rapoport47cec442018-06-30 17:55:02 +03001834 * Return:
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001835 * True if they intersect, false if not.
Stephen Boydeab30942012-05-24 00:45:21 -07001836 */
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001837bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
Yinghai Lu95f72d12010-07-12 14:36:09 +10001838{
Tejun Heoeb18f1b2011-12-08 10:22:07 -08001839 memblock_cap_size(base, &size);
Tang Chenc5c5c9d2015-09-08 15:02:00 -07001840 return memblock_overlaps_region(&memblock.reserved, base, size);
Yinghai Lu95f72d12010-07-12 14:36:09 +10001841}
1842
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001843void __init_memblock memblock_trim_memory(phys_addr_t align)
1844{
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001845 phys_addr_t start, end, orig_start, orig_end;
Emil Medve136199f2014-04-07 15:37:52 -07001846 struct memblock_region *r;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001847
Emil Medve136199f2014-04-07 15:37:52 -07001848 for_each_memblock(memory, r) {
1849 orig_start = r->base;
1850 orig_end = r->base + r->size;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001851 start = round_up(orig_start, align);
1852 end = round_down(orig_end, align);
1853
1854 if (start == orig_start && end == orig_end)
1855 continue;
1856
1857 if (start < end) {
Emil Medve136199f2014-04-07 15:37:52 -07001858 r->base = start;
1859 r->size = end - start;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001860 } else {
Emil Medve136199f2014-04-07 15:37:52 -07001861 memblock_remove_region(&memblock.memory,
1862 r - memblock.memory.regions);
1863 r--;
Yinghai Lu6ede1fd2012-10-22 16:35:18 -07001864 }
1865 }
1866}
Benjamin Herrenschmidte63075a2010-07-06 15:39:01 -07001867
Yinghai Lu3661ca62010-09-15 13:05:29 -07001868void __init_memblock memblock_set_current_limit(phys_addr_t limit)
Benjamin Herrenschmidte63075a2010-07-06 15:39:01 -07001869{
1870 memblock.current_limit = limit;
1871}
1872
Laura Abbottfec51012014-02-27 01:23:43 +01001873phys_addr_t __init_memblock memblock_get_current_limit(void)
1874{
1875 return memblock.current_limit;
1876}
1877
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001878static void __init_memblock memblock_dump(struct memblock_type *type)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001879{
Miles Chen5d63f812017-02-22 15:46:42 -08001880 phys_addr_t base, end, size;
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001881 enum memblock_flags flags;
Alexander Kuleshov8c9c1702016-01-14 15:20:42 -08001882 int idx;
1883 struct memblock_region *rgn;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001884
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001885 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001886
Gioh Kim66e8b432017-11-15 17:33:42 -08001887 for_each_memblock_type(idx, type, rgn) {
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001888 char nid_buf[32] = "";
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001889
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001890 base = rgn->base;
1891 size = rgn->size;
Miles Chen5d63f812017-02-22 15:46:42 -08001892 end = base + size - 1;
Tang Chen66a20752014-01-21 15:49:20 -08001893 flags = rgn->flags;
Tejun Heo7c0caeb2011-07-14 11:43:42 +02001894#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1895 if (memblock_get_region_node(rgn) != MAX_NUMNODES)
1896 snprintf(nid_buf, sizeof(nid_buf), " on node %d",
1897 memblock_get_region_node(rgn));
1898#endif
Mike Rapoporte1720fe2018-06-30 17:55:01 +03001899 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001900 type->name, idx, &base, &end, &size, nid_buf, flags);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001901 }
1902}
1903
Tejun Heo4ff7b822011-12-08 10:22:06 -08001904void __init_memblock __memblock_dump_all(void)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001905{
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001906 pr_info("MEMBLOCK configuration:\n");
Miles Chen5d63f812017-02-22 15:46:42 -08001907 pr_info(" memory size = %pa reserved size = %pa\n",
1908 &memblock.memory.total_size,
1909 &memblock.reserved.total_size);
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001910
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001911 memblock_dump(&memblock.memory);
1912 memblock_dump(&memblock.reserved);
Heiko Carstens409efd42017-02-24 14:55:56 -08001913#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
Heiko Carstens0262d9c2017-02-24 14:55:59 -08001914 memblock_dump(&memblock.physmem);
Heiko Carstens409efd42017-02-24 14:55:56 -08001915#endif
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001916}
1917
Tejun Heo1aadc052011-12-08 10:22:08 -08001918void __init memblock_allow_resize(void)
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001919{
Benjamin Herrenschmidt142b45a72010-07-06 15:39:13 -07001920 memblock_can_resize = 1;
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001921}
1922
Benjamin Herrenschmidt6ed311b2010-07-12 14:36:48 +10001923static int __init early_memblock(char *p)
1924{
1925 if (p && strstr(p, "debug"))
1926 memblock_debug = 1;
1927 return 0;
1928}
1929early_param("memblock", early_memblock);
1930
Mike Rapoportbda49a82018-10-30 15:09:40 -07001931static void __init __free_pages_memory(unsigned long start, unsigned long end)
1932{
1933 int order;
1934
1935 while (start < end) {
1936 order = min(MAX_ORDER - 1UL, __ffs(start));
1937
1938 while (start + (1UL << order) > end)
1939 order--;
1940
1941 memblock_free_pages(pfn_to_page(start), start, order);
1942
1943 start += (1UL << order);
1944 }
1945}
1946
1947static unsigned long __init __free_memory_core(phys_addr_t start,
1948 phys_addr_t end)
1949{
1950 unsigned long start_pfn = PFN_UP(start);
1951 unsigned long end_pfn = min_t(unsigned long,
1952 PFN_DOWN(end), max_low_pfn);
1953
1954 if (start_pfn >= end_pfn)
1955 return 0;
1956
1957 __free_pages_memory(start_pfn, end_pfn);
1958
1959 return end_pfn - start_pfn;
1960}
1961
1962static unsigned long __init free_low_memory_core_early(void)
1963{
1964 unsigned long count = 0;
1965 phys_addr_t start, end;
1966 u64 i;
1967
1968 memblock_clear_hotplug(0, -1);
1969
1970 for_each_reserved_mem_region(i, &start, &end)
1971 reserve_bootmem_region(start, end);
1972
1973 /*
1974 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
1975 * because in some case like Node0 doesn't have RAM installed
1976 * low ram will be on Node1
1977 */
1978 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
1979 NULL)
1980 count += __free_memory_core(start, end);
1981
1982 return count;
1983}
1984
1985static int reset_managed_pages_done __initdata;
1986
1987void reset_node_managed_pages(pg_data_t *pgdat)
1988{
1989 struct zone *z;
1990
1991 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
Arun KS9705bea2018-12-28 00:34:24 -08001992 atomic_long_set(&z->managed_pages, 0);
Mike Rapoportbda49a82018-10-30 15:09:40 -07001993}
1994
1995void __init reset_all_zones_managed_pages(void)
1996{
1997 struct pglist_data *pgdat;
1998
1999 if (reset_managed_pages_done)
2000 return;
2001
2002 for_each_online_pgdat(pgdat)
2003 reset_node_managed_pages(pgdat);
2004
2005 reset_managed_pages_done = 1;
2006}
2007
2008/**
2009 * memblock_free_all - release free pages to the buddy allocator
2010 *
2011 * Return: the number of pages actually released.
2012 */
2013unsigned long __init memblock_free_all(void)
2014{
2015 unsigned long pages;
2016
2017 reset_all_zones_managed_pages();
2018
2019 pages = free_low_memory_core_early();
Arun KSca79b0c2018-12-28 00:34:29 -08002020 totalram_pages_add(pages);
Mike Rapoportbda49a82018-10-30 15:09:40 -07002021
2022 return pages;
2023}
2024
Mike Rapoport350e88b2019-05-13 17:22:59 -07002025#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002026
2027static int memblock_debug_show(struct seq_file *m, void *private)
2028{
2029 struct memblock_type *type = m->private;
2030 struct memblock_region *reg;
2031 int i;
Miles Chen5d63f812017-02-22 15:46:42 -08002032 phys_addr_t end;
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002033
2034 for (i = 0; i < type->cnt; i++) {
2035 reg = &type->regions[i];
Miles Chen5d63f812017-02-22 15:46:42 -08002036 end = reg->base + reg->size - 1;
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002037
Miles Chen5d63f812017-02-22 15:46:42 -08002038 seq_printf(m, "%4d: ", i);
2039 seq_printf(m, "%pa..%pa\n", &reg->base, &end);
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002040 }
2041 return 0;
2042}
Andy Shevchenko5ad35092018-04-05 16:23:16 -07002043DEFINE_SHOW_ATTRIBUTE(memblock_debug);
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002044
2045static int __init memblock_init_debugfs(void)
2046{
2047 struct dentry *root = debugfs_create_dir("memblock", NULL);
Greg Kroah-Hartmand9f79792019-03-05 15:46:09 -08002048
Joe Perches0825a6f2018-06-14 15:27:58 -07002049 debugfs_create_file("memory", 0444, root,
2050 &memblock.memory, &memblock_debug_fops);
2051 debugfs_create_file("reserved", 0444, root,
2052 &memblock.reserved, &memblock_debug_fops);
Philipp Hachtmann70210ed2014-01-29 18:16:01 +01002053#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
Joe Perches0825a6f2018-06-14 15:27:58 -07002054 debugfs_create_file("physmem", 0444, root,
2055 &memblock.physmem, &memblock_debug_fops);
Philipp Hachtmann70210ed2014-01-29 18:16:01 +01002056#endif
Benjamin Herrenschmidt6d03b882010-07-06 15:39:19 -07002057
2058 return 0;
2059}
2060__initcall(memblock_init_debugfs);
2061
2062#endif /* CONFIG_DEBUG_FS */