blob: 84fde270ae74f2117caf70f3b9cc5516acdd8644 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Mel Gorman748446b2010-05-24 14:32:27 -07002/*
3 * linux/mm/compaction.c
4 *
5 * Memory compaction for the reduction of external fragmentation. Note that
6 * this heavily depends upon page migration to do all the real heavy
7 * lifting
8 *
9 * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
10 */
Vlastimil Babka698b1b32016-03-17 14:18:08 -070011#include <linux/cpu.h>
Mel Gorman748446b2010-05-24 14:32:27 -070012#include <linux/swap.h>
13#include <linux/migrate.h>
14#include <linux/compaction.h>
15#include <linux/mm_inline.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010016#include <linux/sched/signal.h>
Mel Gorman748446b2010-05-24 14:32:27 -070017#include <linux/backing-dev.h>
Mel Gorman76ab0f52010-05-24 14:32:28 -070018#include <linux/sysctl.h>
Mel Gormaned4a6d72010-05-24 14:32:29 -070019#include <linux/sysfs.h>
Minchan Kim194159f2013-02-22 16:33:58 -080020#include <linux/page-isolation.h>
Andrey Ryabininb8c73fc2015-02-13 14:39:28 -080021#include <linux/kasan.h>
Vlastimil Babka698b1b32016-03-17 14:18:08 -070022#include <linux/kthread.h>
23#include <linux/freezer.h>
Joonsoo Kim83358ec2016-07-26 15:23:43 -070024#include <linux/page_owner.h>
Johannes Weinereb414682018-10-26 15:06:27 -070025#include <linux/psi.h>
Mel Gorman748446b2010-05-24 14:32:27 -070026#include "internal.h"
27
Minchan Kim010fc292012-12-20 15:05:06 -080028#ifdef CONFIG_COMPACTION
29static inline void count_compact_event(enum vm_event_item item)
30{
31 count_vm_event(item);
32}
33
34static inline void count_compact_events(enum vm_event_item item, long delta)
35{
36 count_vm_events(item, delta);
37}
38#else
39#define count_compact_event(item) do { } while (0)
40#define count_compact_events(item, delta) do { } while (0)
41#endif
42
Michal Nazarewiczff9543f2011-12-29 13:09:50 +010043#if defined CONFIG_COMPACTION || defined CONFIG_CMA
44
Mel Gormanb7aba692011-01-13 15:45:54 -080045#define CREATE_TRACE_POINTS
46#include <trace/events/compaction.h>
47
Vlastimil Babka06b66402016-05-19 17:11:48 -070048#define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
49#define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
50#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order)
51#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
52
Nitin Guptafacdaa92020-08-11 18:31:00 -070053/*
54 * Fragmentation score check interval for proactive compaction purposes.
55 */
Nitin Guptad34c0a72020-08-11 18:31:07 -070056static const unsigned int HPAGE_FRAG_CHECK_INTERVAL_MSEC = 500;
Nitin Guptafacdaa92020-08-11 18:31:00 -070057
58/*
59 * Page order with-respect-to which proactive compaction
60 * calculates external fragmentation, which is used as
61 * the "fragmentation score" of a node/zone.
62 */
63#if defined CONFIG_TRANSPARENT_HUGEPAGE
64#define COMPACTION_HPAGE_ORDER HPAGE_PMD_ORDER
Nitin Gupta25788732020-08-11 18:31:04 -070065#elif defined CONFIG_HUGETLBFS
Nitin Guptafacdaa92020-08-11 18:31:00 -070066#define COMPACTION_HPAGE_ORDER HUGETLB_PAGE_ORDER
67#else
68#define COMPACTION_HPAGE_ORDER (PMD_SHIFT - PAGE_SHIFT)
69#endif
70
Mel Gorman748446b2010-05-24 14:32:27 -070071static unsigned long release_freepages(struct list_head *freelist)
72{
73 struct page *page, *next;
Vlastimil Babka6bace092014-12-10 15:43:31 -080074 unsigned long high_pfn = 0;
Mel Gorman748446b2010-05-24 14:32:27 -070075
76 list_for_each_entry_safe(page, next, freelist, lru) {
Vlastimil Babka6bace092014-12-10 15:43:31 -080077 unsigned long pfn = page_to_pfn(page);
Mel Gorman748446b2010-05-24 14:32:27 -070078 list_del(&page->lru);
79 __free_page(page);
Vlastimil Babka6bace092014-12-10 15:43:31 -080080 if (pfn > high_pfn)
81 high_pfn = pfn;
Mel Gorman748446b2010-05-24 14:32:27 -070082 }
83
Vlastimil Babka6bace092014-12-10 15:43:31 -080084 return high_pfn;
Mel Gorman748446b2010-05-24 14:32:27 -070085}
86
Mel Gorman4469ab92019-03-05 15:44:39 -080087static void split_map_pages(struct list_head *list)
Michal Nazarewiczff9543f2011-12-29 13:09:50 +010088{
Joonsoo Kim66c64222016-07-26 15:23:40 -070089 unsigned int i, order, nr_pages;
90 struct page *page, *next;
91 LIST_HEAD(tmp_list);
Michal Nazarewiczff9543f2011-12-29 13:09:50 +010092
Joonsoo Kim66c64222016-07-26 15:23:40 -070093 list_for_each_entry_safe(page, next, list, lru) {
94 list_del(&page->lru);
95
96 order = page_private(page);
97 nr_pages = 1 << order;
Joonsoo Kim66c64222016-07-26 15:23:40 -070098
Joonsoo Kim46f24fd2016-07-26 15:23:58 -070099 post_alloc_hook(page, order, __GFP_MOVABLE);
Joonsoo Kim66c64222016-07-26 15:23:40 -0700100 if (order)
101 split_page(page, order);
102
103 for (i = 0; i < nr_pages; i++) {
104 list_add(&page->lru, &tmp_list);
105 page++;
106 }
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100107 }
Joonsoo Kim66c64222016-07-26 15:23:40 -0700108
109 list_splice(&tmp_list, list);
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100110}
111
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700112#ifdef CONFIG_COMPACTION
Joonsoo Kim24e27162015-02-11 15:27:09 -0800113
Minchan Kimbda807d2016-07-26 15:23:05 -0700114int PageMovable(struct page *page)
115{
116 struct address_space *mapping;
117
118 VM_BUG_ON_PAGE(!PageLocked(page), page);
119 if (!__PageMovable(page))
120 return 0;
121
122 mapping = page_mapping(page);
123 if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
124 return 1;
125
126 return 0;
127}
128EXPORT_SYMBOL(PageMovable);
129
130void __SetPageMovable(struct page *page, struct address_space *mapping)
131{
132 VM_BUG_ON_PAGE(!PageLocked(page), page);
133 VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
134 page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
135}
136EXPORT_SYMBOL(__SetPageMovable);
137
138void __ClearPageMovable(struct page *page)
139{
Minchan Kimbda807d2016-07-26 15:23:05 -0700140 VM_BUG_ON_PAGE(!PageMovable(page), page);
141 /*
142 * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE
143 * flag so that VM can catch up released page by driver after isolation.
144 * With it, VM migration doesn't try to put it back.
145 */
146 page->mapping = (void *)((unsigned long)page->mapping &
147 PAGE_MAPPING_MOVABLE);
148}
149EXPORT_SYMBOL(__ClearPageMovable);
150
Joonsoo Kim24e27162015-02-11 15:27:09 -0800151/* Do not skip compaction more than 64 times */
152#define COMPACT_MAX_DEFER_SHIFT 6
153
154/*
155 * Compaction is deferred when compaction fails to result in a page
Alex Shi860b3272020-08-11 18:31:10 -0700156 * allocation success. 1 << compact_defer_shift, compactions are skipped up
Joonsoo Kim24e27162015-02-11 15:27:09 -0800157 * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
158 */
Hui Su2271b012020-12-14 19:12:46 -0800159static void defer_compaction(struct zone *zone, int order)
Joonsoo Kim24e27162015-02-11 15:27:09 -0800160{
161 zone->compact_considered = 0;
162 zone->compact_defer_shift++;
163
164 if (order < zone->compact_order_failed)
165 zone->compact_order_failed = order;
166
167 if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
168 zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
169
170 trace_mm_compaction_defer_compaction(zone, order);
171}
172
173/* Returns true if compaction should be skipped this time */
Hui Su2271b012020-12-14 19:12:46 -0800174static bool compaction_deferred(struct zone *zone, int order)
Joonsoo Kim24e27162015-02-11 15:27:09 -0800175{
176 unsigned long defer_limit = 1UL << zone->compact_defer_shift;
177
178 if (order < zone->compact_order_failed)
179 return false;
180
181 /* Avoid possible overflow */
Mateusz Nosek62b35fe2020-10-13 16:56:58 -0700182 if (++zone->compact_considered >= defer_limit) {
Joonsoo Kim24e27162015-02-11 15:27:09 -0800183 zone->compact_considered = defer_limit;
Joonsoo Kim24e27162015-02-11 15:27:09 -0800184 return false;
Mateusz Nosek62b35fe2020-10-13 16:56:58 -0700185 }
Joonsoo Kim24e27162015-02-11 15:27:09 -0800186
187 trace_mm_compaction_deferred(zone, order);
188
189 return true;
190}
191
192/*
193 * Update defer tracking counters after successful compaction of given order,
194 * which means an allocation either succeeded (alloc_success == true) or is
195 * expected to succeed.
196 */
197void compaction_defer_reset(struct zone *zone, int order,
198 bool alloc_success)
199{
200 if (alloc_success) {
201 zone->compact_considered = 0;
202 zone->compact_defer_shift = 0;
203 }
204 if (order >= zone->compact_order_failed)
205 zone->compact_order_failed = order + 1;
206
207 trace_mm_compaction_defer_reset(zone, order);
208}
209
210/* Returns true if restarting compaction after many failures */
Hui Su2271b012020-12-14 19:12:46 -0800211static bool compaction_restarting(struct zone *zone, int order)
Joonsoo Kim24e27162015-02-11 15:27:09 -0800212{
213 if (order < zone->compact_order_failed)
214 return false;
215
216 return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
217 zone->compact_considered >= 1UL << zone->compact_defer_shift;
218}
219
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700220/* Returns true if the pageblock should be scanned for pages to isolate. */
221static inline bool isolation_suitable(struct compact_control *cc,
222 struct page *page)
223{
224 if (cc->ignore_skip_hint)
225 return true;
226
227 return !get_pageblock_skip(page);
228}
229
Vlastimil Babka023336412015-09-08 15:02:42 -0700230static void reset_cached_positions(struct zone *zone)
231{
232 zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
233 zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
Joonsoo Kim623446e2016-03-15 14:57:45 -0700234 zone->compact_cached_free_pfn =
Vlastimil Babka06b66402016-05-19 17:11:48 -0700235 pageblock_start_pfn(zone_end_pfn(zone) - 1);
Vlastimil Babka023336412015-09-08 15:02:42 -0700236}
237
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700238/*
Hui Su2271b012020-12-14 19:12:46 -0800239 * Compound pages of >= pageblock_order should consistently be skipped until
Vlastimil Babkab527cfe2017-11-17 15:26:34 -0800240 * released. It is always pointless to compact pages of such order (if they are
241 * migratable), and the pageblocks they occupy cannot contain any free pages.
David Rientjes21dc7e02017-11-17 15:26:30 -0800242 */
Vlastimil Babkab527cfe2017-11-17 15:26:34 -0800243static bool pageblock_skip_persistent(struct page *page)
David Rientjes21dc7e02017-11-17 15:26:30 -0800244{
Vlastimil Babkab527cfe2017-11-17 15:26:34 -0800245 if (!PageCompound(page))
David Rientjes21dc7e02017-11-17 15:26:30 -0800246 return false;
Vlastimil Babkab527cfe2017-11-17 15:26:34 -0800247
248 page = compound_head(page);
249
250 if (compound_order(page) >= pageblock_order)
251 return true;
252
253 return false;
David Rientjes21dc7e02017-11-17 15:26:30 -0800254}
255
Mel Gormane332f742019-03-05 15:45:38 -0800256static bool
257__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
258 bool check_target)
259{
260 struct page *page = pfn_to_online_page(pfn);
Mel Gorman6b0868c2019-04-04 11:54:09 +0100261 struct page *block_page;
Mel Gormane332f742019-03-05 15:45:38 -0800262 struct page *end_page;
263 unsigned long block_pfn;
264
265 if (!page)
266 return false;
267 if (zone != page_zone(page))
268 return false;
269 if (pageblock_skip_persistent(page))
270 return false;
271
272 /*
273 * If skip is already cleared do no further checking once the
274 * restart points have been set.
275 */
276 if (check_source && check_target && !get_pageblock_skip(page))
277 return true;
278
279 /*
280 * If clearing skip for the target scanner, do not select a
281 * non-movable pageblock as the starting point.
282 */
283 if (!check_source && check_target &&
284 get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
285 return false;
286
Mel Gorman6b0868c2019-04-04 11:54:09 +0100287 /* Ensure the start of the pageblock or zone is online and valid */
288 block_pfn = pageblock_start_pfn(pfn);
Vlastimil Babkaa2e9a5a2019-10-14 14:12:07 -0700289 block_pfn = max(block_pfn, zone->zone_start_pfn);
290 block_page = pfn_to_online_page(block_pfn);
Mel Gorman6b0868c2019-04-04 11:54:09 +0100291 if (block_page) {
292 page = block_page;
293 pfn = block_pfn;
294 }
295
296 /* Ensure the end of the pageblock or zone is online and valid */
Vlastimil Babkaa2e9a5a2019-10-14 14:12:07 -0700297 block_pfn = pageblock_end_pfn(pfn) - 1;
Mel Gorman6b0868c2019-04-04 11:54:09 +0100298 block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
299 end_page = pfn_to_online_page(block_pfn);
300 if (!end_page)
301 return false;
302
Mel Gormane332f742019-03-05 15:45:38 -0800303 /*
304 * Only clear the hint if a sample indicates there is either a
305 * free page or an LRU page in the block. One or other condition
306 * is necessary for the block to be a migration source/target.
307 */
Mel Gormane332f742019-03-05 15:45:38 -0800308 do {
309 if (pfn_valid_within(pfn)) {
310 if (check_source && PageLRU(page)) {
311 clear_pageblock_skip(page);
312 return true;
313 }
314
315 if (check_target && PageBuddy(page)) {
316 clear_pageblock_skip(page);
317 return true;
318 }
319 }
320
321 page += (1 << PAGE_ALLOC_COSTLY_ORDER);
322 pfn += (1 << PAGE_ALLOC_COSTLY_ORDER);
Vlastimil Babkaa2e9a5a2019-10-14 14:12:07 -0700323 } while (page <= end_page);
Mel Gormane332f742019-03-05 15:45:38 -0800324
325 return false;
326}
327
David Rientjes21dc7e02017-11-17 15:26:30 -0800328/*
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700329 * This function is called to clear all cached information on pageblocks that
330 * should be skipped for page isolation when the migrate and free page scanner
331 * meet.
332 */
Mel Gorman62997022012-10-08 16:32:47 -0700333static void __reset_isolation_suitable(struct zone *zone)
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700334{
Mel Gormane332f742019-03-05 15:45:38 -0800335 unsigned long migrate_pfn = zone->zone_start_pfn;
Mel Gorman6b0868c2019-04-04 11:54:09 +0100336 unsigned long free_pfn = zone_end_pfn(zone) - 1;
Mel Gormane332f742019-03-05 15:45:38 -0800337 unsigned long reset_migrate = free_pfn;
338 unsigned long reset_free = migrate_pfn;
339 bool source_set = false;
340 bool free_set = false;
341
342 if (!zone->compact_blockskip_flush)
343 return;
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700344
Mel Gorman62997022012-10-08 16:32:47 -0700345 zone->compact_blockskip_flush = false;
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700346
Mel Gormane332f742019-03-05 15:45:38 -0800347 /*
348 * Walk the zone and update pageblock skip information. Source looks
349 * for PageLRU while target looks for PageBuddy. When the scanner
350 * is found, both PageBuddy and PageLRU are checked as the pageblock
351 * is suitable as both source and target.
352 */
353 for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
354 free_pfn -= pageblock_nr_pages) {
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700355 cond_resched();
356
Mel Gormane332f742019-03-05 15:45:38 -0800357 /* Update the migrate PFN */
358 if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
359 migrate_pfn < reset_migrate) {
360 source_set = true;
361 reset_migrate = migrate_pfn;
362 zone->compact_init_migrate_pfn = reset_migrate;
363 zone->compact_cached_migrate_pfn[0] = reset_migrate;
364 zone->compact_cached_migrate_pfn[1] = reset_migrate;
365 }
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700366
Mel Gormane332f742019-03-05 15:45:38 -0800367 /* Update the free PFN */
368 if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
369 free_pfn > reset_free) {
370 free_set = true;
371 reset_free = free_pfn;
372 zone->compact_init_free_pfn = reset_free;
373 zone->compact_cached_free_pfn = reset_free;
374 }
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700375 }
Vlastimil Babka023336412015-09-08 15:02:42 -0700376
Mel Gormane332f742019-03-05 15:45:38 -0800377 /* Leave no distance if no suitable block was reset */
378 if (reset_migrate >= reset_free) {
379 zone->compact_cached_migrate_pfn[0] = migrate_pfn;
380 zone->compact_cached_migrate_pfn[1] = migrate_pfn;
381 zone->compact_cached_free_pfn = free_pfn;
382 }
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700383}
384
Mel Gorman62997022012-10-08 16:32:47 -0700385void reset_isolation_suitable(pg_data_t *pgdat)
386{
387 int zoneid;
388
389 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
390 struct zone *zone = &pgdat->node_zones[zoneid];
391 if (!populated_zone(zone))
392 continue;
393
394 /* Only flush if a full compaction finished recently */
395 if (zone->compact_blockskip_flush)
396 __reset_isolation_suitable(zone);
397 }
398}
399
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700400/*
Mel Gormane380beb2019-03-05 15:44:58 -0800401 * Sets the pageblock skip bit if it was clear. Note that this is a hint as
402 * locks are not required for read/writers. Returns true if it was already set.
403 */
404static bool test_and_set_skip(struct compact_control *cc, struct page *page,
405 unsigned long pfn)
406{
407 bool skip;
408
409 /* Do no update if skip hint is being ignored */
410 if (cc->ignore_skip_hint)
411 return false;
412
413 if (!IS_ALIGNED(pfn, pageblock_nr_pages))
414 return false;
415
416 skip = get_pageblock_skip(page);
417 if (!skip && !cc->no_set_skip_hint)
418 set_pageblock_skip(page);
419
420 return skip;
421}
422
423static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
424{
425 struct zone *zone = cc->zone;
426
427 pfn = pageblock_end_pfn(pfn);
428
429 /* Set for isolation rather than compaction */
430 if (cc->no_set_skip_hint)
431 return;
432
433 if (pfn > zone->compact_cached_migrate_pfn[0])
434 zone->compact_cached_migrate_pfn[0] = pfn;
435 if (cc->mode != MIGRATE_ASYNC &&
436 pfn > zone->compact_cached_migrate_pfn[1])
437 zone->compact_cached_migrate_pfn[1] = pfn;
438}
439
440/*
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700441 * If no pages were isolated then mark this pageblock to be skipped in the
Mel Gorman62997022012-10-08 16:32:47 -0700442 * future. The information is later cleared by __reset_isolation_suitable().
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700443 */
Mel Gormanc89511a2012-10-08 16:32:45 -0700444static void update_pageblock_skip(struct compact_control *cc,
Mel Gormand097a6f2019-03-05 15:45:28 -0800445 struct page *page, unsigned long pfn)
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700446{
Mel Gormanc89511a2012-10-08 16:32:45 -0700447 struct zone *zone = cc->zone;
Joonsoo Kim6815bf32013-12-18 17:08:52 -0800448
Vlastimil Babka2583d672017-11-17 15:26:38 -0800449 if (cc->no_set_skip_hint)
Joonsoo Kim6815bf32013-12-18 17:08:52 -0800450 return;
451
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700452 if (!page)
453 return;
454
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -0700455 set_pageblock_skip(page);
Mel Gormanc89511a2012-10-08 16:32:45 -0700456
David Rientjes35979ef2014-06-04 16:08:27 -0700457 /* Update where async and sync compaction should restart */
Mel Gormane380beb2019-03-05 15:44:58 -0800458 if (pfn < zone->compact_cached_free_pfn)
459 zone->compact_cached_free_pfn = pfn;
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700460}
461#else
462static inline bool isolation_suitable(struct compact_control *cc,
463 struct page *page)
464{
465 return true;
466}
467
Vlastimil Babkab527cfe2017-11-17 15:26:34 -0800468static inline bool pageblock_skip_persistent(struct page *page)
David Rientjes21dc7e02017-11-17 15:26:30 -0800469{
470 return false;
471}
472
473static inline void update_pageblock_skip(struct compact_control *cc,
Mel Gormand097a6f2019-03-05 15:45:28 -0800474 struct page *page, unsigned long pfn)
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700475{
476}
Mel Gormane380beb2019-03-05 15:44:58 -0800477
478static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
479{
480}
481
482static bool test_and_set_skip(struct compact_control *cc, struct page *page,
483 unsigned long pfn)
484{
485 return false;
486}
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700487#endif /* CONFIG_COMPACTION */
488
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700489/*
490 * Compaction requires the taking of some coarse locks that are potentially
Mel Gormancb2dcaf2019-03-05 15:45:11 -0800491 * very heavily contended. For async compaction, trylock and record if the
492 * lock is contended. The lock will still be acquired but compaction will
493 * abort when the current block is finished regardless of success rate.
494 * Sync compaction acquires the lock.
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700495 *
Mel Gormancb2dcaf2019-03-05 15:45:11 -0800496 * Always returns true which makes it easier to track lock state in callers.
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700497 */
Mel Gormancb2dcaf2019-03-05 15:45:11 -0800498static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700499 struct compact_control *cc)
Jules Irenge77337ed2020-04-06 20:08:06 -0700500 __acquires(lock)
Mel Gorman2a1402aa2012-10-08 16:32:33 -0700501{
Mel Gormancb2dcaf2019-03-05 15:45:11 -0800502 /* Track if the lock is contended in async mode */
503 if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
504 if (spin_trylock_irqsave(lock, *flags))
505 return true;
506
507 cc->contended = true;
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700508 }
Vlastimil Babka1f9efde2014-10-09 15:27:14 -0700509
Mel Gormancb2dcaf2019-03-05 15:45:11 -0800510 spin_lock_irqsave(lock, *flags);
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700511 return true;
Mel Gorman2a1402aa2012-10-08 16:32:33 -0700512}
513
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100514/*
Mel Gormanc67fe372012-08-21 16:16:17 -0700515 * Compaction requires the taking of some coarse locks that are potentially
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700516 * very heavily contended. The lock should be periodically unlocked to avoid
517 * having disabled IRQs for a long time, even when there is nobody waiting on
518 * the lock. It might also be that allowing the IRQs will result in
519 * need_resched() becoming true. If scheduling is needed, async compaction
520 * aborts. Sync compaction schedules.
521 * Either compaction type will also abort if a fatal signal is pending.
522 * In either case if the lock was locked, it is dropped and not regained.
Mel Gormanc67fe372012-08-21 16:16:17 -0700523 *
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700524 * Returns true if compaction should abort due to fatal signal pending, or
525 * async compaction due to need_resched()
526 * Returns false when compaction can continue (sync compaction might have
527 * scheduled)
Mel Gormanc67fe372012-08-21 16:16:17 -0700528 */
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700529static bool compact_unlock_should_abort(spinlock_t *lock,
530 unsigned long flags, bool *locked, struct compact_control *cc)
Mel Gormanc67fe372012-08-21 16:16:17 -0700531{
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700532 if (*locked) {
533 spin_unlock_irqrestore(lock, flags);
534 *locked = false;
535 }
Vlastimil Babka1f9efde2014-10-09 15:27:14 -0700536
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700537 if (fatal_signal_pending(current)) {
Vlastimil Babkac3486f52016-07-28 15:49:30 -0700538 cc->contended = true;
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700539 return true;
540 }
Mel Gormanc67fe372012-08-21 16:16:17 -0700541
Mel Gormancf66f072019-03-05 15:45:24 -0800542 cond_resched();
Vlastimil Babkabe976572014-06-04 16:10:41 -0700543
544 return false;
545}
546
Mel Gormanc67fe372012-08-21 16:16:17 -0700547/*
Jerome Marchand9e4be472013-11-12 15:07:12 -0800548 * Isolate free pages onto a private freelist. If @strict is true, will abort
549 * returning 0 on any invalid PFNs or non-free pages inside of the pageblock
550 * (even though it may still end up isolating some pages).
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100551 */
Mel Gormanf40d1e42012-10-08 16:32:36 -0700552static unsigned long isolate_freepages_block(struct compact_control *cc,
Vlastimil Babkae14c7202014-10-09 15:27:20 -0700553 unsigned long *start_pfn,
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100554 unsigned long end_pfn,
555 struct list_head *freelist,
Mel Gorman4fca9732019-03-05 15:45:34 -0800556 unsigned int stride,
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100557 bool strict)
Mel Gorman748446b2010-05-24 14:32:27 -0700558{
Mel Gormanb7aba692011-01-13 15:45:54 -0800559 int nr_scanned = 0, total_isolated = 0;
Mel Gormand097a6f2019-03-05 15:45:28 -0800560 struct page *cursor;
Xiubo Lib8b2d822014-10-09 15:28:21 -0700561 unsigned long flags = 0;
Mel Gormanf40d1e42012-10-08 16:32:36 -0700562 bool locked = false;
Vlastimil Babkae14c7202014-10-09 15:27:20 -0700563 unsigned long blockpfn = *start_pfn;
Joonsoo Kim66c64222016-07-26 15:23:40 -0700564 unsigned int order;
Mel Gorman748446b2010-05-24 14:32:27 -0700565
Mel Gorman4fca9732019-03-05 15:45:34 -0800566 /* Strict mode is for isolation, speed is secondary */
567 if (strict)
568 stride = 1;
569
Mel Gorman748446b2010-05-24 14:32:27 -0700570 cursor = pfn_to_page(blockpfn);
571
Mel Gormanf40d1e42012-10-08 16:32:36 -0700572 /* Isolate free pages. */
Mel Gorman4fca9732019-03-05 15:45:34 -0800573 for (; blockpfn < end_pfn; blockpfn += stride, cursor += stride) {
Joonsoo Kim66c64222016-07-26 15:23:40 -0700574 int isolated;
Mel Gorman748446b2010-05-24 14:32:27 -0700575 struct page *page = cursor;
576
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700577 /*
578 * Periodically drop the lock (if held) regardless of its
579 * contention, to give chance to IRQs. Abort if fatal signal
580 * pending or async compaction detects need_resched()
581 */
582 if (!(blockpfn % SWAP_CLUSTER_MAX)
583 && compact_unlock_should_abort(&cc->zone->lock, flags,
584 &locked, cc))
585 break;
586
Mel Gormanb7aba692011-01-13 15:45:54 -0800587 nr_scanned++;
Mel Gormanf40d1e42012-10-08 16:32:36 -0700588 if (!pfn_valid_within(blockpfn))
Laura Abbott2af120b2014-03-10 15:49:44 -0700589 goto isolate_fail;
590
Vlastimil Babka9fcd6d22015-09-08 15:02:49 -0700591 /*
592 * For compound pages such as THP and hugetlbfs, we can save
593 * potentially a lot of iterations if we skip them at once.
594 * The check is racy, but we can consider only valid values
595 * and the only danger is skipping too much.
596 */
597 if (PageCompound(page)) {
David Rientjes21dc7e02017-11-17 15:26:30 -0800598 const unsigned int order = compound_order(page);
Vlastimil Babka9fcd6d22015-09-08 15:02:49 -0700599
Vlastimil Babkad3c85ba2017-11-17 15:26:41 -0800600 if (likely(order < MAX_ORDER)) {
David Rientjes21dc7e02017-11-17 15:26:30 -0800601 blockpfn += (1UL << order) - 1;
602 cursor += (1UL << order) - 1;
Vlastimil Babka9fcd6d22015-09-08 15:02:49 -0700603 }
Vlastimil Babka9fcd6d22015-09-08 15:02:49 -0700604 goto isolate_fail;
605 }
606
Mel Gormanf40d1e42012-10-08 16:32:36 -0700607 if (!PageBuddy(page))
Laura Abbott2af120b2014-03-10 15:49:44 -0700608 goto isolate_fail;
Mel Gormanf40d1e42012-10-08 16:32:36 -0700609
610 /*
Vlastimil Babka69b71892014-10-09 15:27:18 -0700611 * If we already hold the lock, we can skip some rechecking.
612 * Note that if we hold the lock now, checked_pageblock was
613 * already set in some previous iteration (or strict is true),
614 * so it is correct to skip the suitable migration target
615 * recheck as well.
Mel Gormanf40d1e42012-10-08 16:32:36 -0700616 */
Vlastimil Babka69b71892014-10-09 15:27:18 -0700617 if (!locked) {
Mel Gormancb2dcaf2019-03-05 15:45:11 -0800618 locked = compact_lock_irqsave(&cc->zone->lock,
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700619 &flags, cc);
Mel Gormanf40d1e42012-10-08 16:32:36 -0700620
Vlastimil Babka69b71892014-10-09 15:27:18 -0700621 /* Recheck this is a buddy page under lock */
622 if (!PageBuddy(page))
623 goto isolate_fail;
624 }
Mel Gorman748446b2010-05-24 14:32:27 -0700625
Joonsoo Kim66c64222016-07-26 15:23:40 -0700626 /* Found a free page, will break it into order-0 pages */
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700627 order = buddy_order(page);
Joonsoo Kim66c64222016-07-26 15:23:40 -0700628 isolated = __isolate_free_page(page, order);
David Rientjesa4f04f22016-06-24 14:50:10 -0700629 if (!isolated)
630 break;
Joonsoo Kim66c64222016-07-26 15:23:40 -0700631 set_page_private(page, order);
David Rientjesa4f04f22016-06-24 14:50:10 -0700632
Mel Gorman748446b2010-05-24 14:32:27 -0700633 total_isolated += isolated;
David Rientjesa4f04f22016-06-24 14:50:10 -0700634 cc->nr_freepages += isolated;
Joonsoo Kim66c64222016-07-26 15:23:40 -0700635 list_add_tail(&page->lru, freelist);
636
David Rientjesa4f04f22016-06-24 14:50:10 -0700637 if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
638 blockpfn += isolated;
639 break;
Mel Gorman748446b2010-05-24 14:32:27 -0700640 }
David Rientjesa4f04f22016-06-24 14:50:10 -0700641 /* Advance to the end of split page */
642 blockpfn += isolated - 1;
643 cursor += isolated - 1;
644 continue;
Laura Abbott2af120b2014-03-10 15:49:44 -0700645
646isolate_fail:
647 if (strict)
648 break;
649 else
650 continue;
651
Mel Gorman748446b2010-05-24 14:32:27 -0700652 }
653
David Rientjesa4f04f22016-06-24 14:50:10 -0700654 if (locked)
655 spin_unlock_irqrestore(&cc->zone->lock, flags);
656
Vlastimil Babka9fcd6d22015-09-08 15:02:49 -0700657 /*
658 * There is a tiny chance that we have read bogus compound_order(),
659 * so be careful to not go outside of the pageblock.
660 */
661 if (unlikely(blockpfn > end_pfn))
662 blockpfn = end_pfn;
663
Joonsoo Kime34d85f2015-02-11 15:27:04 -0800664 trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
665 nr_scanned, total_isolated);
666
Vlastimil Babkae14c7202014-10-09 15:27:20 -0700667 /* Record how far we have got within the block */
668 *start_pfn = blockpfn;
669
Mel Gormanf40d1e42012-10-08 16:32:36 -0700670 /*
671 * If strict isolation is requested by CMA then check that all the
672 * pages requested were isolated. If there were any failures, 0 is
673 * returned and CMA will fail.
674 */
Laura Abbott2af120b2014-03-10 15:49:44 -0700675 if (strict && blockpfn < end_pfn)
Mel Gormanf40d1e42012-10-08 16:32:36 -0700676 total_isolated = 0;
677
David Rientjes7f354a52017-02-22 15:44:50 -0800678 cc->total_free_scanned += nr_scanned;
Mel Gorman397487d2012-10-19 12:00:10 +0100679 if (total_isolated)
Minchan Kim010fc292012-12-20 15:05:06 -0800680 count_compact_events(COMPACTISOLATED, total_isolated);
Mel Gorman748446b2010-05-24 14:32:27 -0700681 return total_isolated;
682}
683
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100684/**
685 * isolate_freepages_range() - isolate free pages.
Mike Rapoporte8b098f2018-04-05 16:24:57 -0700686 * @cc: Compaction control structure.
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100687 * @start_pfn: The first PFN to start isolating.
688 * @end_pfn: The one-past-last PFN.
689 *
690 * Non-free pages, invalid PFNs, or zone boundaries within the
691 * [start_pfn, end_pfn) range are considered errors, cause function to
692 * undo its actions and return zero.
693 *
694 * Otherwise, function returns one-past-the-last PFN of isolated page
695 * (which may be greater then end_pfn if end fell in a middle of
696 * a free page).
697 */
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100698unsigned long
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700699isolate_freepages_range(struct compact_control *cc,
700 unsigned long start_pfn, unsigned long end_pfn)
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100701{
Joonsoo Kime1409c32016-03-15 14:57:48 -0700702 unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100703 LIST_HEAD(freelist);
704
Vlastimil Babka7d49d882014-10-09 15:27:11 -0700705 pfn = start_pfn;
Vlastimil Babka06b66402016-05-19 17:11:48 -0700706 block_start_pfn = pageblock_start_pfn(pfn);
Joonsoo Kime1409c32016-03-15 14:57:48 -0700707 if (block_start_pfn < cc->zone->zone_start_pfn)
708 block_start_pfn = cc->zone->zone_start_pfn;
Vlastimil Babka06b66402016-05-19 17:11:48 -0700709 block_end_pfn = pageblock_end_pfn(pfn);
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100710
Vlastimil Babka7d49d882014-10-09 15:27:11 -0700711 for (; pfn < end_pfn; pfn += isolated,
Joonsoo Kime1409c32016-03-15 14:57:48 -0700712 block_start_pfn = block_end_pfn,
Vlastimil Babka7d49d882014-10-09 15:27:11 -0700713 block_end_pfn += pageblock_nr_pages) {
Vlastimil Babkae14c7202014-10-09 15:27:20 -0700714 /* Protect pfn from changing by isolate_freepages_block */
715 unsigned long isolate_start_pfn = pfn;
Vlastimil Babka7d49d882014-10-09 15:27:11 -0700716
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100717 block_end_pfn = min(block_end_pfn, end_pfn);
718
Joonsoo Kim58420012014-11-13 15:19:07 -0800719 /*
720 * pfn could pass the block_end_pfn if isolated freepage
721 * is more than pageblock order. In this case, we adjust
722 * scanning range to right one.
723 */
724 if (pfn >= block_end_pfn) {
Vlastimil Babka06b66402016-05-19 17:11:48 -0700725 block_start_pfn = pageblock_start_pfn(pfn);
726 block_end_pfn = pageblock_end_pfn(pfn);
Joonsoo Kim58420012014-11-13 15:19:07 -0800727 block_end_pfn = min(block_end_pfn, end_pfn);
728 }
729
Joonsoo Kime1409c32016-03-15 14:57:48 -0700730 if (!pageblock_pfn_to_page(block_start_pfn,
731 block_end_pfn, cc->zone))
Vlastimil Babka7d49d882014-10-09 15:27:11 -0700732 break;
733
Vlastimil Babkae14c7202014-10-09 15:27:20 -0700734 isolated = isolate_freepages_block(cc, &isolate_start_pfn,
Mel Gorman4fca9732019-03-05 15:45:34 -0800735 block_end_pfn, &freelist, 0, true);
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100736
737 /*
738 * In strict mode, isolate_freepages_block() returns 0 if
739 * there are any holes in the block (ie. invalid PFNs or
740 * non-free pages).
741 */
742 if (!isolated)
743 break;
744
745 /*
746 * If we managed to isolate pages, it is always (1 << n) *
747 * pageblock_nr_pages for some non-negative n. (Max order
748 * page may span two pageblocks).
749 */
750 }
751
Joonsoo Kim66c64222016-07-26 15:23:40 -0700752 /* __isolate_free_page() does not map the pages */
Mel Gorman4469ab92019-03-05 15:44:39 -0800753 split_map_pages(&freelist);
Michal Nazarewicz85aa1252012-01-30 13:24:03 +0100754
755 if (pfn < end_pfn) {
756 /* Loop terminated early, cleanup. */
757 release_freepages(&freelist);
758 return 0;
759 }
760
761 /* We don't use freelists for anything. */
762 return pfn;
763}
764
Mel Gorman748446b2010-05-24 14:32:27 -0700765/* Similar to reclaim, but different enough that they don't share logic */
Andrey Ryabinin5f438ee2019-03-05 15:49:42 -0800766static bool too_many_isolated(pg_data_t *pgdat)
Mel Gorman748446b2010-05-24 14:32:27 -0700767{
Minchan Kimbc693042010-09-09 16:38:00 -0700768 unsigned long active, inactive, isolated;
Mel Gorman748446b2010-05-24 14:32:27 -0700769
Andrey Ryabinin5f438ee2019-03-05 15:49:42 -0800770 inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
771 node_page_state(pgdat, NR_INACTIVE_ANON);
772 active = node_page_state(pgdat, NR_ACTIVE_FILE) +
773 node_page_state(pgdat, NR_ACTIVE_ANON);
774 isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
775 node_page_state(pgdat, NR_ISOLATED_ANON);
Mel Gorman748446b2010-05-24 14:32:27 -0700776
Minchan Kimbc693042010-09-09 16:38:00 -0700777 return isolated > (inactive + active) / 2;
Mel Gorman748446b2010-05-24 14:32:27 -0700778}
779
Michal Nazarewicz2fe86e02012-01-30 13:16:26 +0100780/**
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -0700781 * isolate_migratepages_block() - isolate all migrate-able pages within
782 * a single pageblock
Michal Nazarewicz2fe86e02012-01-30 13:16:26 +0100783 * @cc: Compaction control structure.
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -0700784 * @low_pfn: The first PFN to isolate
785 * @end_pfn: The one-past-the-last PFN to isolate, within same pageblock
786 * @isolate_mode: Isolation mode to be used.
Michal Nazarewicz2fe86e02012-01-30 13:16:26 +0100787 *
788 * Isolate all pages that can be migrated from the range specified by
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -0700789 * [low_pfn, end_pfn). The range is expected to be within same pageblock.
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700790 * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion,
Oscar Salvador369fa222021-05-04 18:35:26 -0700791 * -ENOMEM in case we could not allocate a page, or 0.
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700792 * cc->migrate_pfn will contain the next pfn to scan.
Michal Nazarewicz2fe86e02012-01-30 13:16:26 +0100793 *
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -0700794 * The pages are isolated on cc->migratepages list (not required to be empty),
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700795 * and cc->nr_migratepages is updated accordingly.
Mel Gorman748446b2010-05-24 14:32:27 -0700796 */
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700797static int
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -0700798isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
799 unsigned long end_pfn, isolate_mode_t isolate_mode)
Mel Gorman748446b2010-05-24 14:32:27 -0700800{
Andrey Ryabinin5f438ee2019-03-05 15:49:42 -0800801 pg_data_t *pgdat = cc->zone->zone_pgdat;
Mel Gormanb7aba692011-01-13 15:45:54 -0800802 unsigned long nr_scanned = 0, nr_isolated = 0;
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700803 struct lruvec *lruvec;
Xiubo Lib8b2d822014-10-09 15:28:21 -0700804 unsigned long flags = 0;
Alex Shi6168d0d2020-12-15 12:34:29 -0800805 struct lruvec *locked = NULL;
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700806 struct page *page = NULL, *valid_page = NULL;
Joonsoo Kime34d85f2015-02-11 15:27:04 -0800807 unsigned long start_pfn = low_pfn;
Vlastimil Babkafdd048e2016-05-19 17:11:55 -0700808 bool skip_on_failure = false;
809 unsigned long next_skip_pfn = 0;
Mel Gormane380beb2019-03-05 15:44:58 -0800810 bool skip_updated = false;
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700811 int ret = 0;
812
813 cc->migrate_pfn = low_pfn;
Mel Gorman748446b2010-05-24 14:32:27 -0700814
Mel Gorman748446b2010-05-24 14:32:27 -0700815 /*
816 * Ensure that there are not too many pages isolated from the LRU
817 * list by either parallel reclaimers or compaction. If there are,
818 * delay for some time until fewer pages are isolated
819 */
Andrey Ryabinin5f438ee2019-03-05 15:49:42 -0800820 while (unlikely(too_many_isolated(pgdat))) {
Zi Yand20bdd572020-11-13 22:51:43 -0800821 /* stop isolation if there are still pages not migrated */
822 if (cc->nr_migratepages)
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700823 return -EAGAIN;
Zi Yand20bdd572020-11-13 22:51:43 -0800824
Mel Gormanf9e35b32011-06-15 15:08:52 -0700825 /* async migration should just abort */
David Rientjese0b9dae2014-06-04 16:08:28 -0700826 if (cc->mode == MIGRATE_ASYNC)
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700827 return -EAGAIN;
Mel Gormanf9e35b32011-06-15 15:08:52 -0700828
Mel Gorman748446b2010-05-24 14:32:27 -0700829 congestion_wait(BLK_RW_ASYNC, HZ/10);
830
831 if (fatal_signal_pending(current))
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700832 return -EINTR;
Mel Gorman748446b2010-05-24 14:32:27 -0700833 }
834
Mel Gormancf66f072019-03-05 15:45:24 -0800835 cond_resched();
David Rientjesaeef4b82014-06-04 16:08:31 -0700836
Vlastimil Babkafdd048e2016-05-19 17:11:55 -0700837 if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
838 skip_on_failure = true;
839 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
840 }
841
Mel Gorman748446b2010-05-24 14:32:27 -0700842 /* Time to isolate some pages for migration */
Mel Gorman748446b2010-05-24 14:32:27 -0700843 for (; low_pfn < end_pfn; low_pfn++) {
Vlastimil Babka29c0dde2015-09-08 15:02:46 -0700844
Vlastimil Babkafdd048e2016-05-19 17:11:55 -0700845 if (skip_on_failure && low_pfn >= next_skip_pfn) {
846 /*
847 * We have isolated all migration candidates in the
848 * previous order-aligned block, and did not skip it due
849 * to failure. We should migrate the pages now and
850 * hopefully succeed compaction.
851 */
852 if (nr_isolated)
853 break;
854
855 /*
856 * We failed to isolate in the previous order-aligned
857 * block. Set the new boundary to the end of the
858 * current block. Note we can't simply increase
859 * next_skip_pfn by 1 << order, as low_pfn might have
860 * been incremented by a higher number due to skipping
861 * a compound or a high-order buddy page in the
862 * previous loop iteration.
863 */
864 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
865 }
866
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700867 /*
868 * Periodically drop the lock (if held) regardless of its
Mel Gorman670105a2019-08-02 21:48:51 -0700869 * contention, to give chance to IRQs. Abort completely if
870 * a fatal signal is pending.
Vlastimil Babka8b44d272014-10-09 15:27:16 -0700871 */
Alex Shi6168d0d2020-12-15 12:34:29 -0800872 if (!(low_pfn % SWAP_CLUSTER_MAX)) {
873 if (locked) {
874 unlock_page_lruvec_irqrestore(locked, flags);
875 locked = NULL;
876 }
877
878 if (fatal_signal_pending(current)) {
879 cc->contended = true;
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -0700880 ret = -EINTR;
Alex Shi6168d0d2020-12-15 12:34:29 -0800881
Alex Shi6168d0d2020-12-15 12:34:29 -0800882 goto fatal_pending;
883 }
884
885 cond_resched();
Mel Gorman670105a2019-08-02 21:48:51 -0700886 }
Mel Gormanc67fe372012-08-21 16:16:17 -0700887
Mel Gorman748446b2010-05-24 14:32:27 -0700888 if (!pfn_valid_within(low_pfn))
Vlastimil Babkafdd048e2016-05-19 17:11:55 -0700889 goto isolate_fail;
Mel Gormanb7aba692011-01-13 15:45:54 -0800890 nr_scanned++;
Mel Gorman748446b2010-05-24 14:32:27 -0700891
Mel Gorman748446b2010-05-24 14:32:27 -0700892 page = pfn_to_page(low_pfn);
Mel Gormandc908602012-02-08 17:13:38 -0800893
Mel Gormane380beb2019-03-05 15:44:58 -0800894 /*
895 * Check if the pageblock has already been marked skipped.
896 * Only the aligned PFN is checked as the caller isolates
897 * COMPACT_CLUSTER_MAX at a time so the second call must
898 * not falsely conclude that the block should be skipped.
899 */
900 if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) {
901 if (!cc->ignore_skip_hint && get_pageblock_skip(page)) {
902 low_pfn = end_pfn;
Alex Shi9df41312020-12-15 12:34:20 -0800903 page = NULL;
Mel Gormane380beb2019-03-05 15:44:58 -0800904 goto isolate_abort;
905 }
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700906 valid_page = page;
Mel Gormane380beb2019-03-05 15:44:58 -0800907 }
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700908
Oscar Salvador369fa222021-05-04 18:35:26 -0700909 if (PageHuge(page) && cc->alloc_contig) {
Oscar Salvadorae37c7f2021-05-04 18:35:29 -0700910 ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);
Oscar Salvador369fa222021-05-04 18:35:26 -0700911
912 /*
913 * Fail isolation in case isolate_or_dissolve_huge_page()
914 * reports an error. In case of -ENOMEM, abort right away.
915 */
916 if (ret < 0) {
917 /* Do not report -EBUSY down the chain */
918 if (ret == -EBUSY)
919 ret = 0;
920 low_pfn += (1UL << compound_order(page)) - 1;
921 goto isolate_fail;
922 }
923
Oscar Salvadorae37c7f2021-05-04 18:35:29 -0700924 if (PageHuge(page)) {
925 /*
926 * Hugepage was successfully isolated and placed
927 * on the cc->migratepages list.
928 */
929 low_pfn += compound_nr(page) - 1;
930 goto isolate_success_no_list;
931 }
932
Oscar Salvador369fa222021-05-04 18:35:26 -0700933 /*
934 * Ok, the hugepage was dissolved. Now these pages are
935 * Buddy and cannot be re-allocated because they are
936 * isolated. Fall-through as the check below handles
937 * Buddy pages.
938 */
939 }
940
Mel Gorman6c144662014-01-23 15:53:38 -0800941 /*
Vlastimil Babka99c0fd52014-10-09 15:27:23 -0700942 * Skip if free. We read page order here without zone lock
943 * which is generally unsafe, but the race window is small and
944 * the worst thing that can happen is that we skip some
945 * potential isolation targets.
Mel Gorman6c144662014-01-23 15:53:38 -0800946 */
Vlastimil Babka99c0fd52014-10-09 15:27:23 -0700947 if (PageBuddy(page)) {
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700948 unsigned long freepage_order = buddy_order_unsafe(page);
Vlastimil Babka99c0fd52014-10-09 15:27:23 -0700949
950 /*
951 * Without lock, we cannot be sure that what we got is
952 * a valid page order. Consider only values in the
953 * valid order range to prevent low_pfn overflow.
954 */
955 if (freepage_order > 0 && freepage_order < MAX_ORDER)
956 low_pfn += (1UL << freepage_order) - 1;
Mel Gorman748446b2010-05-24 14:32:27 -0700957 continue;
Vlastimil Babka99c0fd52014-10-09 15:27:23 -0700958 }
Mel Gorman748446b2010-05-24 14:32:27 -0700959
Mel Gorman9927af742011-01-13 15:45:59 -0800960 /*
Vlastimil Babka29c0dde2015-09-08 15:02:46 -0700961 * Regardless of being on LRU, compound pages such as THP and
Rik van Riel1da2f322020-04-01 21:10:31 -0700962 * hugetlbfs are not to be compacted unless we are attempting
963 * an allocation much larger than the huge page size (eg CMA).
964 * We can potentially save a lot of iterations if we skip them
965 * at once. The check is racy, but we can consider only valid
966 * values and the only danger is skipping too much.
Andrea Arcangelibc835012011-01-13 15:47:08 -0800967 */
Rik van Riel1da2f322020-04-01 21:10:31 -0700968 if (PageCompound(page) && !cc->alloc_contig) {
David Rientjes21dc7e02017-11-17 15:26:30 -0800969 const unsigned int order = compound_order(page);
Vlastimil Babka29c0dde2015-09-08 15:02:46 -0700970
Vlastimil Babkad3c85ba2017-11-17 15:26:41 -0800971 if (likely(order < MAX_ORDER))
David Rientjes21dc7e02017-11-17 15:26:30 -0800972 low_pfn += (1UL << order) - 1;
Vlastimil Babkafdd048e2016-05-19 17:11:55 -0700973 goto isolate_fail;
Mel Gorman2a1402aa2012-10-08 16:32:33 -0700974 }
975
Minchan Kimbda807d2016-07-26 15:23:05 -0700976 /*
977 * Check may be lockless but that's ok as we recheck later.
978 * It's possible to migrate LRU and non-lru movable pages.
979 * Skip any other type of page
980 */
981 if (!PageLRU(page)) {
Minchan Kimbda807d2016-07-26 15:23:05 -0700982 /*
983 * __PageMovable can return false positive so we need
984 * to verify it under page_lock.
985 */
986 if (unlikely(__PageMovable(page)) &&
987 !PageIsolated(page)) {
988 if (locked) {
Alex Shi6168d0d2020-12-15 12:34:29 -0800989 unlock_page_lruvec_irqrestore(locked, flags);
990 locked = NULL;
Minchan Kimbda807d2016-07-26 15:23:05 -0700991 }
992
Yisheng Xie9e5bcd62017-02-24 14:57:29 -0800993 if (!isolate_movable_page(page, isolate_mode))
Minchan Kimbda807d2016-07-26 15:23:05 -0700994 goto isolate_success;
995 }
996
Vlastimil Babkafdd048e2016-05-19 17:11:55 -0700997 goto isolate_fail;
Minchan Kimbda807d2016-07-26 15:23:05 -0700998 }
Vlastimil Babka29c0dde2015-09-08 15:02:46 -0700999
David Rientjes119d6d52014-04-03 14:48:00 -07001000 /*
1001 * Migration will fail if an anonymous page is pinned in memory,
1002 * so avoid taking lru_lock and isolating it unnecessarily in an
1003 * admittedly racy check.
1004 */
1005 if (!page_mapping(page) &&
1006 page_count(page) > page_mapcount(page))
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001007 goto isolate_fail;
David Rientjes119d6d52014-04-03 14:48:00 -07001008
Michal Hocko73e64c512016-12-14 15:04:07 -08001009 /*
1010 * Only allow to migrate anonymous pages in GFP_NOFS context
1011 * because those do not depend on fs locks.
1012 */
1013 if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page))
1014 goto isolate_fail;
1015
Alex Shi9df41312020-12-15 12:34:20 -08001016 /*
1017 * Be careful not to clear PageLRU until after we're
1018 * sure the page is not being freed elsewhere -- the
1019 * page release code relies on it.
1020 */
1021 if (unlikely(!get_page_unless_zero(page)))
1022 goto isolate_fail;
1023
Alex Shic2135f72021-02-24 12:08:01 -08001024 if (!__isolate_lru_page_prepare(page, isolate_mode))
Alex Shi9df41312020-12-15 12:34:20 -08001025 goto isolate_fail_put;
1026
1027 /* Try isolate the page */
1028 if (!TestClearPageLRU(page))
1029 goto isolate_fail_put;
1030
Alex Shi6168d0d2020-12-15 12:34:29 -08001031 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1032
Vlastimil Babka69b71892014-10-09 15:27:18 -07001033 /* If we already hold the lock, we can skip some rechecking */
Alex Shi6168d0d2020-12-15 12:34:29 -08001034 if (lruvec != locked) {
1035 if (locked)
1036 unlock_page_lruvec_irqrestore(locked, flags);
1037
1038 compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
1039 locked = lruvec;
Alex Shi6168d0d2020-12-15 12:34:29 -08001040
1041 lruvec_memcg_debug(lruvec, page);
Mel Gormane380beb2019-03-05 15:44:58 -08001042
Mel Gormane380beb2019-03-05 15:44:58 -08001043 /* Try get exclusive access under lock */
1044 if (!skip_updated) {
1045 skip_updated = true;
1046 if (test_and_set_skip(cc, page, low_pfn))
1047 goto isolate_abort;
1048 }
Mel Gorman2a1402aa2012-10-08 16:32:33 -07001049
Vlastimil Babka29c0dde2015-09-08 15:02:46 -07001050 /*
1051 * Page become compound since the non-locked check,
1052 * and it's on LRU. It can only be a THP so the order
1053 * is safe to read and it's 0 for tail pages.
1054 */
Rik van Riel1da2f322020-04-01 21:10:31 -07001055 if (unlikely(PageCompound(page) && !cc->alloc_contig)) {
Matthew Wilcox (Oracle)d8c65462019-09-23 15:34:30 -07001056 low_pfn += compound_nr(page) - 1;
Alex Shi9df41312020-12-15 12:34:20 -08001057 SetPageLRU(page);
1058 goto isolate_fail_put;
Vlastimil Babka69b71892014-10-09 15:27:18 -07001059 }
Alex Shid99fd5f2021-02-24 12:09:25 -08001060 }
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001061
Rik van Riel1da2f322020-04-01 21:10:31 -07001062 /* The whole page is taken off the LRU; skip the tail pages. */
1063 if (PageCompound(page))
1064 low_pfn += compound_nr(page) - 1;
Andrea Arcangelibc835012011-01-13 15:47:08 -08001065
Mel Gorman748446b2010-05-24 14:32:27 -07001066 /* Successfully isolated */
Yu Zhao46ae6b22021-02-24 12:08:25 -08001067 del_page_from_lru_list(page, lruvec);
Rik van Riel1da2f322020-04-01 21:10:31 -07001068 mod_node_page_state(page_pgdat(page),
Huang Ying9de4f222020-04-06 20:04:41 -07001069 NR_ISOLATED_ANON + page_is_file_lru(page),
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07001070 thp_nr_pages(page));
Joonsoo Kimb6c75012014-04-07 15:37:07 -07001071
1072isolate_success:
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001073 list_add(&page->lru, &cc->migratepages);
Oscar Salvadorae37c7f2021-05-04 18:35:29 -07001074isolate_success_no_list:
Zi Yan38935862020-11-13 22:51:40 -08001075 cc->nr_migratepages += compound_nr(page);
1076 nr_isolated += compound_nr(page);
Mel Gorman748446b2010-05-24 14:32:27 -07001077
Mel Gorman804d3122019-03-05 15:45:07 -08001078 /*
1079 * Avoid isolating too much unless this block is being
Mel Gormancb2dcaf2019-03-05 15:45:11 -08001080 * rescanned (e.g. dirty/writeback pages, parallel allocation)
1081 * or a lock is contended. For contention, isolate quickly to
1082 * potentially remove one source of contention.
Mel Gorman804d3122019-03-05 15:45:07 -08001083 */
Zi Yan38935862020-11-13 22:51:40 -08001084 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX &&
Mel Gormancb2dcaf2019-03-05 15:45:11 -08001085 !cc->rescan && !cc->contended) {
Hillf Danton31b83842012-01-10 15:07:59 -08001086 ++low_pfn;
Mel Gorman748446b2010-05-24 14:32:27 -07001087 break;
Hillf Danton31b83842012-01-10 15:07:59 -08001088 }
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001089
1090 continue;
Alex Shi9df41312020-12-15 12:34:20 -08001091
1092isolate_fail_put:
1093 /* Avoid potential deadlock in freeing page under lru_lock */
1094 if (locked) {
Alex Shi6168d0d2020-12-15 12:34:29 -08001095 unlock_page_lruvec_irqrestore(locked, flags);
1096 locked = NULL;
Alex Shi9df41312020-12-15 12:34:20 -08001097 }
1098 put_page(page);
1099
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001100isolate_fail:
Oscar Salvador369fa222021-05-04 18:35:26 -07001101 if (!skip_on_failure && ret != -ENOMEM)
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001102 continue;
1103
1104 /*
1105 * We have isolated some pages, but then failed. Release them
1106 * instead of migrating, as we cannot form the cc->order buddy
1107 * page anyway.
1108 */
1109 if (nr_isolated) {
1110 if (locked) {
Alex Shi6168d0d2020-12-15 12:34:29 -08001111 unlock_page_lruvec_irqrestore(locked, flags);
1112 locked = NULL;
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001113 }
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001114 putback_movable_pages(&cc->migratepages);
1115 cc->nr_migratepages = 0;
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07001116 nr_isolated = 0;
1117 }
1118
1119 if (low_pfn < next_skip_pfn) {
1120 low_pfn = next_skip_pfn - 1;
1121 /*
1122 * The check near the loop beginning would have updated
1123 * next_skip_pfn too, but this is a bit simpler.
1124 */
1125 next_skip_pfn += 1UL << cc->order;
1126 }
Oscar Salvador369fa222021-05-04 18:35:26 -07001127
1128 if (ret == -ENOMEM)
1129 break;
Mel Gorman748446b2010-05-24 14:32:27 -07001130 }
1131
Vlastimil Babka99c0fd52014-10-09 15:27:23 -07001132 /*
1133 * The PageBuddy() check could have potentially brought us outside
1134 * the range to be scanned.
1135 */
1136 if (unlikely(low_pfn > end_pfn))
1137 low_pfn = end_pfn;
1138
Alex Shi9df41312020-12-15 12:34:20 -08001139 page = NULL;
1140
Mel Gormane380beb2019-03-05 15:44:58 -08001141isolate_abort:
Mel Gormanc67fe372012-08-21 16:16:17 -07001142 if (locked)
Alex Shi6168d0d2020-12-15 12:34:29 -08001143 unlock_page_lruvec_irqrestore(locked, flags);
Alex Shi9df41312020-12-15 12:34:20 -08001144 if (page) {
1145 SetPageLRU(page);
1146 put_page(page);
1147 }
Mel Gorman748446b2010-05-24 14:32:27 -07001148
Vlastimil Babka50b5b092014-01-21 15:51:10 -08001149 /*
Mel Gorman804d3122019-03-05 15:45:07 -08001150 * Updated the cached scanner pfn once the pageblock has been scanned
1151 * Pages will either be migrated in which case there is no point
1152 * scanning in the near future or migration failed in which case the
1153 * failure reason may persist. The block is marked for skipping if
1154 * there were no pages isolated in the block or if the block is
1155 * rescanned twice in a row.
Vlastimil Babka50b5b092014-01-21 15:51:10 -08001156 */
Mel Gorman804d3122019-03-05 15:45:07 -08001157 if (low_pfn == end_pfn && (!nr_isolated || cc->rescan)) {
Mel Gormane380beb2019-03-05 15:44:58 -08001158 if (valid_page && !skip_updated)
1159 set_pageblock_skip(valid_page);
1160 update_cached_migrate(cc, low_pfn);
1161 }
Mel Gormanbb13ffe2012-10-08 16:32:41 -07001162
Joonsoo Kime34d85f2015-02-11 15:27:04 -08001163 trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
1164 nr_scanned, nr_isolated);
Mel Gormanb7aba692011-01-13 15:45:54 -08001165
Mel Gorman670105a2019-08-02 21:48:51 -07001166fatal_pending:
David Rientjes7f354a52017-02-22 15:44:50 -08001167 cc->total_migrate_scanned += nr_scanned;
Mel Gorman397487d2012-10-19 12:00:10 +01001168 if (nr_isolated)
Minchan Kim010fc292012-12-20 15:05:06 -08001169 count_compact_events(COMPACTISOLATED, nr_isolated);
Mel Gorman397487d2012-10-19 12:00:10 +01001170
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001171 cc->migrate_pfn = low_pfn;
1172
1173 return ret;
Michal Nazarewicz2fe86e02012-01-30 13:16:26 +01001174}
1175
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001176/**
1177 * isolate_migratepages_range() - isolate migrate-able pages in a PFN range
1178 * @cc: Compaction control structure.
1179 * @start_pfn: The first PFN to start isolating.
1180 * @end_pfn: The one-past-last PFN.
1181 *
Oscar Salvador369fa222021-05-04 18:35:26 -07001182 * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM
1183 * in case we could not allocate a page, or 0.
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001184 */
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001185int
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001186isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
1187 unsigned long end_pfn)
1188{
Joonsoo Kime1409c32016-03-15 14:57:48 -07001189 unsigned long pfn, block_start_pfn, block_end_pfn;
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001190 int ret = 0;
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001191
1192 /* Scan block by block. First and last block may be incomplete */
1193 pfn = start_pfn;
Vlastimil Babka06b66402016-05-19 17:11:48 -07001194 block_start_pfn = pageblock_start_pfn(pfn);
Joonsoo Kime1409c32016-03-15 14:57:48 -07001195 if (block_start_pfn < cc->zone->zone_start_pfn)
1196 block_start_pfn = cc->zone->zone_start_pfn;
Vlastimil Babka06b66402016-05-19 17:11:48 -07001197 block_end_pfn = pageblock_end_pfn(pfn);
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001198
1199 for (; pfn < end_pfn; pfn = block_end_pfn,
Joonsoo Kime1409c32016-03-15 14:57:48 -07001200 block_start_pfn = block_end_pfn,
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001201 block_end_pfn += pageblock_nr_pages) {
1202
1203 block_end_pfn = min(block_end_pfn, end_pfn);
1204
Joonsoo Kime1409c32016-03-15 14:57:48 -07001205 if (!pageblock_pfn_to_page(block_start_pfn,
1206 block_end_pfn, cc->zone))
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001207 continue;
1208
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001209 ret = isolate_migratepages_block(cc, pfn, block_end_pfn,
1210 ISOLATE_UNEVICTABLE);
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001211
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001212 if (ret)
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001213 break;
Joonsoo Kim6ea41c02014-10-29 14:50:20 -07001214
Zi Yan38935862020-11-13 22:51:40 -08001215 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX)
Joonsoo Kim6ea41c02014-10-29 14:50:20 -07001216 break;
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001217 }
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001218
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001219 return ret;
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001220}
1221
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001222#endif /* CONFIG_COMPACTION || CONFIG_CMA */
1223#ifdef CONFIG_COMPACTION
Andrew Morton018e9a42015-04-15 16:15:20 -07001224
Vlastimil Babkab682deb2017-05-08 15:54:43 -07001225static bool suitable_migration_source(struct compact_control *cc,
1226 struct page *page)
1227{
Vlastimil Babka282722b2017-05-08 15:54:49 -07001228 int block_mt;
1229
Mel Gorman9bebefd2019-03-05 15:45:14 -08001230 if (pageblock_skip_persistent(page))
1231 return false;
1232
Vlastimil Babka282722b2017-05-08 15:54:49 -07001233 if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
Vlastimil Babkab682deb2017-05-08 15:54:43 -07001234 return true;
1235
Vlastimil Babka282722b2017-05-08 15:54:49 -07001236 block_mt = get_pageblock_migratetype(page);
1237
1238 if (cc->migratetype == MIGRATE_MOVABLE)
1239 return is_migrate_movable(block_mt);
1240 else
1241 return block_mt == cc->migratetype;
Vlastimil Babkab682deb2017-05-08 15:54:43 -07001242}
1243
Andrew Morton018e9a42015-04-15 16:15:20 -07001244/* Returns true if the page is within a block suitable for migration to */
Vlastimil Babka9f7e3382016-10-07 17:00:37 -07001245static bool suitable_migration_target(struct compact_control *cc,
1246 struct page *page)
Andrew Morton018e9a42015-04-15 16:15:20 -07001247{
1248 /* If the page is a large free page, then disallow migration */
1249 if (PageBuddy(page)) {
1250 /*
1251 * We are checking page_order without zone->lock taken. But
1252 * the only small danger is that we skip a potentially suitable
1253 * pageblock, so it's not worth to check order for valid range.
1254 */
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07001255 if (buddy_order_unsafe(page) >= pageblock_order)
Andrew Morton018e9a42015-04-15 16:15:20 -07001256 return false;
1257 }
1258
Yisheng Xie1ef36db2017-05-03 14:53:54 -07001259 if (cc->ignore_block_suitable)
1260 return true;
1261
Andrew Morton018e9a42015-04-15 16:15:20 -07001262 /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
Vlastimil Babkab682deb2017-05-08 15:54:43 -07001263 if (is_migrate_movable(get_pageblock_migratetype(page)))
Andrew Morton018e9a42015-04-15 16:15:20 -07001264 return true;
1265
1266 /* Otherwise skip the block */
1267 return false;
1268}
1269
Mel Gorman70b44592019-03-05 15:44:54 -08001270static inline unsigned int
1271freelist_scan_limit(struct compact_control *cc)
1272{
Qian Caidd7ef7b2019-05-13 17:17:38 -07001273 unsigned short shift = BITS_PER_LONG - 1;
1274
1275 return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;
Mel Gorman70b44592019-03-05 15:44:54 -08001276}
1277
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001278/*
Vlastimil Babkaf2849aa2015-09-08 15:02:36 -07001279 * Test whether the free scanner has reached the same or lower pageblock than
1280 * the migration scanner, and compaction should thus terminate.
1281 */
1282static inline bool compact_scanners_met(struct compact_control *cc)
1283{
1284 return (cc->free_pfn >> pageblock_order)
1285 <= (cc->migrate_pfn >> pageblock_order);
1286}
1287
Mel Gorman5a811882019-03-05 15:45:01 -08001288/*
1289 * Used when scanning for a suitable migration target which scans freelists
1290 * in reverse. Reorders the list such as the unscanned pages are scanned
1291 * first on the next iteration of the free scanner
1292 */
1293static void
1294move_freelist_head(struct list_head *freelist, struct page *freepage)
1295{
1296 LIST_HEAD(sublist);
1297
1298 if (!list_is_last(freelist, &freepage->lru)) {
1299 list_cut_before(&sublist, freelist, &freepage->lru);
1300 if (!list_empty(&sublist))
1301 list_splice_tail(&sublist, freelist);
1302 }
1303}
1304
1305/*
1306 * Similar to move_freelist_head except used by the migration scanner
1307 * when scanning forward. It's possible for these list operations to
1308 * move against each other if they search the free list exactly in
1309 * lockstep.
1310 */
Mel Gorman70b44592019-03-05 15:44:54 -08001311static void
1312move_freelist_tail(struct list_head *freelist, struct page *freepage)
1313{
1314 LIST_HEAD(sublist);
1315
1316 if (!list_is_first(freelist, &freepage->lru)) {
1317 list_cut_position(&sublist, freelist, &freepage->lru);
1318 if (!list_empty(&sublist))
1319 list_splice_tail(&sublist, freelist);
1320 }
1321}
1322
Mel Gorman5a811882019-03-05 15:45:01 -08001323static void
1324fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long nr_isolated)
1325{
1326 unsigned long start_pfn, end_pfn;
Vlastimil Babka6e2b7042021-02-24 12:09:39 -08001327 struct page *page;
Mel Gorman5a811882019-03-05 15:45:01 -08001328
1329 /* Do not search around if there are enough pages already */
1330 if (cc->nr_freepages >= cc->nr_migratepages)
1331 return;
1332
1333 /* Minimise scanning during async compaction */
1334 if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
1335 return;
1336
1337 /* Pageblock boundaries */
Vlastimil Babka6e2b7042021-02-24 12:09:39 -08001338 start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);
1339 end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));
1340
1341 page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);
1342 if (!page)
1343 return;
Mel Gorman5a811882019-03-05 15:45:01 -08001344
1345 /* Scan before */
1346 if (start_pfn != pfn) {
Mel Gorman4fca9732019-03-05 15:45:34 -08001347 isolate_freepages_block(cc, &start_pfn, pfn, &cc->freepages, 1, false);
Mel Gorman5a811882019-03-05 15:45:01 -08001348 if (cc->nr_freepages >= cc->nr_migratepages)
1349 return;
1350 }
1351
1352 /* Scan after */
1353 start_pfn = pfn + nr_isolated;
Mel Gorman60fce362019-05-17 14:31:41 -07001354 if (start_pfn < end_pfn)
Mel Gorman4fca9732019-03-05 15:45:34 -08001355 isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);
Mel Gorman5a811882019-03-05 15:45:01 -08001356
1357 /* Skip this pageblock in the future as it's full or nearly full */
1358 if (cc->nr_freepages < cc->nr_migratepages)
1359 set_pageblock_skip(page);
1360}
1361
Mel Gormandbe2d4e2019-03-05 15:45:31 -08001362/* Search orders in round-robin fashion */
1363static int next_search_order(struct compact_control *cc, int order)
1364{
1365 order--;
1366 if (order < 0)
1367 order = cc->order - 1;
1368
1369 /* Search wrapped around? */
1370 if (order == cc->search_order) {
1371 cc->search_order--;
1372 if (cc->search_order < 0)
1373 cc->search_order = cc->order - 1;
1374 return -1;
1375 }
1376
1377 return order;
1378}
1379
Mel Gorman5a811882019-03-05 15:45:01 -08001380static unsigned long
1381fast_isolate_freepages(struct compact_control *cc)
1382{
1383 unsigned int limit = min(1U, freelist_scan_limit(cc) >> 1);
1384 unsigned int nr_scanned = 0;
Rokudo Yan74e21482021-02-04 18:32:20 -08001385 unsigned long low_pfn, min_pfn, highest = 0;
Mel Gorman5a811882019-03-05 15:45:01 -08001386 unsigned long nr_isolated = 0;
1387 unsigned long distance;
1388 struct page *page = NULL;
1389 bool scan_start = false;
1390 int order;
1391
1392 /* Full compaction passes in a negative order */
1393 if (cc->order <= 0)
1394 return cc->free_pfn;
1395
1396 /*
1397 * If starting the scan, use a deeper search and use the highest
1398 * PFN found if a suitable one is not found.
1399 */
Mel Gormane332f742019-03-05 15:45:38 -08001400 if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
Mel Gorman5a811882019-03-05 15:45:01 -08001401 limit = pageblock_nr_pages >> 1;
1402 scan_start = true;
1403 }
1404
1405 /*
1406 * Preferred point is in the top quarter of the scan space but take
1407 * a pfn from the top half if the search is problematic.
1408 */
1409 distance = (cc->free_pfn - cc->migrate_pfn);
1410 low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
1411 min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
1412
1413 if (WARN_ON_ONCE(min_pfn > low_pfn))
1414 low_pfn = min_pfn;
1415
Mel Gormandbe2d4e2019-03-05 15:45:31 -08001416 /*
1417 * Search starts from the last successful isolation order or the next
1418 * order to search after a previous failure
1419 */
1420 cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
1421
1422 for (order = cc->search_order;
1423 !page && order >= 0;
1424 order = next_search_order(cc, order)) {
Mel Gorman5a811882019-03-05 15:45:01 -08001425 struct free_area *area = &cc->zone->free_area[order];
1426 struct list_head *freelist;
1427 struct page *freepage;
1428 unsigned long flags;
1429 unsigned int order_scanned = 0;
Rokudo Yan74e21482021-02-04 18:32:20 -08001430 unsigned long high_pfn = 0;
Mel Gorman5a811882019-03-05 15:45:01 -08001431
1432 if (!area->nr_free)
1433 continue;
1434
1435 spin_lock_irqsave(&cc->zone->lock, flags);
1436 freelist = &area->free_list[MIGRATE_MOVABLE];
1437 list_for_each_entry_reverse(freepage, freelist, lru) {
1438 unsigned long pfn;
1439
1440 order_scanned++;
1441 nr_scanned++;
1442 pfn = page_to_pfn(freepage);
1443
1444 if (pfn >= highest)
Vlastimil Babka6e2b7042021-02-24 12:09:39 -08001445 highest = max(pageblock_start_pfn(pfn),
1446 cc->zone->zone_start_pfn);
Mel Gorman5a811882019-03-05 15:45:01 -08001447
1448 if (pfn >= low_pfn) {
1449 cc->fast_search_fail = 0;
Mel Gormandbe2d4e2019-03-05 15:45:31 -08001450 cc->search_order = order;
Mel Gorman5a811882019-03-05 15:45:01 -08001451 page = freepage;
1452 break;
1453 }
1454
1455 if (pfn >= min_pfn && pfn > high_pfn) {
1456 high_pfn = pfn;
1457
1458 /* Shorten the scan if a candidate is found */
1459 limit >>= 1;
1460 }
1461
1462 if (order_scanned >= limit)
1463 break;
1464 }
1465
1466 /* Use a minimum pfn if a preferred one was not found */
1467 if (!page && high_pfn) {
1468 page = pfn_to_page(high_pfn);
1469
1470 /* Update freepage for the list reorder below */
1471 freepage = page;
1472 }
1473
1474 /* Reorder to so a future search skips recent pages */
1475 move_freelist_head(freelist, freepage);
1476
1477 /* Isolate the page if available */
1478 if (page) {
1479 if (__isolate_free_page(page, order)) {
1480 set_page_private(page, order);
1481 nr_isolated = 1 << order;
1482 cc->nr_freepages += nr_isolated;
1483 list_add_tail(&page->lru, &cc->freepages);
1484 count_compact_events(COMPACTISOLATED, nr_isolated);
1485 } else {
1486 /* If isolation fails, abort the search */
Qian Cai5b56d992019-04-04 11:54:41 +01001487 order = cc->search_order + 1;
Mel Gorman5a811882019-03-05 15:45:01 -08001488 page = NULL;
1489 }
1490 }
1491
1492 spin_unlock_irqrestore(&cc->zone->lock, flags);
1493
1494 /*
1495 * Smaller scan on next order so the total scan ig related
1496 * to freelist_scan_limit.
1497 */
1498 if (order_scanned >= limit)
1499 limit = min(1U, limit >> 1);
1500 }
1501
1502 if (!page) {
1503 cc->fast_search_fail++;
1504 if (scan_start) {
1505 /*
1506 * Use the highest PFN found above min. If one was
Ethon Paulf3867752020-06-04 16:49:13 -07001507 * not found, be pessimistic for direct compaction
Mel Gorman5a811882019-03-05 15:45:01 -08001508 * and use the min mark.
1509 */
1510 if (highest) {
1511 page = pfn_to_page(highest);
1512 cc->free_pfn = highest;
1513 } else {
Suzuki K Poulosee577c8b2019-05-31 22:30:59 -07001514 if (cc->direct_compaction && pfn_valid(min_pfn)) {
Baoquan He73a6e472020-06-03 15:57:55 -07001515 page = pageblock_pfn_to_page(min_pfn,
Vlastimil Babka6e2b7042021-02-24 12:09:39 -08001516 min(pageblock_end_pfn(min_pfn),
1517 zone_end_pfn(cc->zone)),
Baoquan He73a6e472020-06-03 15:57:55 -07001518 cc->zone);
Mel Gorman5a811882019-03-05 15:45:01 -08001519 cc->free_pfn = min_pfn;
1520 }
1521 }
1522 }
1523 }
1524
Mel Gormand097a6f2019-03-05 15:45:28 -08001525 if (highest && highest >= cc->zone->compact_cached_free_pfn) {
1526 highest -= pageblock_nr_pages;
Mel Gorman5a811882019-03-05 15:45:01 -08001527 cc->zone->compact_cached_free_pfn = highest;
Mel Gormand097a6f2019-03-05 15:45:28 -08001528 }
Mel Gorman5a811882019-03-05 15:45:01 -08001529
1530 cc->total_free_scanned += nr_scanned;
1531 if (!page)
1532 return cc->free_pfn;
1533
1534 low_pfn = page_to_pfn(page);
1535 fast_isolate_around(cc, low_pfn, nr_isolated);
1536 return low_pfn;
1537}
1538
Vlastimil Babkaf2849aa2015-09-08 15:02:36 -07001539/*
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001540 * Based on information in the current compact_control, find blocks
1541 * suitable for isolating free pages from and then isolate them.
1542 */
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001543static void isolate_freepages(struct compact_control *cc)
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001544{
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001545 struct zone *zone = cc->zone;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001546 struct page *page;
Vlastimil Babkac96b9e52014-06-04 16:07:26 -07001547 unsigned long block_start_pfn; /* start of current pageblock */
Vlastimil Babkae14c7202014-10-09 15:27:20 -07001548 unsigned long isolate_start_pfn; /* exact pfn we start at */
Vlastimil Babkac96b9e52014-06-04 16:07:26 -07001549 unsigned long block_end_pfn; /* end of current pageblock */
1550 unsigned long low_pfn; /* lowest pfn scanner is able to scan */
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001551 struct list_head *freelist = &cc->freepages;
Mel Gorman4fca9732019-03-05 15:45:34 -08001552 unsigned int stride;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001553
Mel Gorman5a811882019-03-05 15:45:01 -08001554 /* Try a small search of the free lists for a candidate */
1555 isolate_start_pfn = fast_isolate_freepages(cc);
1556 if (cc->nr_freepages)
1557 goto splitmap;
1558
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001559 /*
1560 * Initialise the free scanner. The starting point is where we last
Vlastimil Babka49e068f2014-05-06 12:50:03 -07001561 * successfully isolated from, zone-cached value, or the end of the
Vlastimil Babkae14c7202014-10-09 15:27:20 -07001562 * zone when isolating for the first time. For looping we also need
1563 * this pfn aligned down to the pageblock boundary, because we do
Vlastimil Babkac96b9e52014-06-04 16:07:26 -07001564 * block_start_pfn -= pageblock_nr_pages in the for loop.
1565 * For ending point, take care when isolating in last pageblock of a
Randy Dunlapa1c1dbe2020-08-11 18:32:49 -07001566 * zone which ends in the middle of a pageblock.
Vlastimil Babka49e068f2014-05-06 12:50:03 -07001567 * The low boundary is the end of the pageblock the migration scanner
1568 * is using.
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001569 */
Vlastimil Babkae14c7202014-10-09 15:27:20 -07001570 isolate_start_pfn = cc->free_pfn;
Mel Gorman5a811882019-03-05 15:45:01 -08001571 block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
Vlastimil Babkac96b9e52014-06-04 16:07:26 -07001572 block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
1573 zone_end_pfn(zone));
Vlastimil Babka06b66402016-05-19 17:11:48 -07001574 low_pfn = pageblock_end_pfn(cc->migrate_pfn);
Mel Gorman4fca9732019-03-05 15:45:34 -08001575 stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001576
1577 /*
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001578 * Isolate free pages until enough are available to migrate the
1579 * pages on cc->migratepages. We stop searching if the migrate
1580 * and free page scanners meet or enough free pages are isolated.
1581 */
Vlastimil Babkaf5f61a32015-09-08 15:02:39 -07001582 for (; block_start_pfn >= low_pfn;
Vlastimil Babkac96b9e52014-06-04 16:07:26 -07001583 block_end_pfn = block_start_pfn,
Vlastimil Babkae14c7202014-10-09 15:27:20 -07001584 block_start_pfn -= pageblock_nr_pages,
1585 isolate_start_pfn = block_start_pfn) {
Mel Gorman4fca9732019-03-05 15:45:34 -08001586 unsigned long nr_isolated;
1587
David Rientjesf6ea3ad2013-09-30 13:45:03 -07001588 /*
1589 * This can iterate a massively long zone without finding any
Mel Gormancb810ad2019-03-05 15:45:21 -08001590 * suitable migration targets, so periodically check resched.
David Rientjesf6ea3ad2013-09-30 13:45:03 -07001591 */
Mel Gormancb810ad2019-03-05 15:45:21 -08001592 if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
Mel Gormancf66f072019-03-05 15:45:24 -08001593 cond_resched();
David Rientjesf6ea3ad2013-09-30 13:45:03 -07001594
Vlastimil Babka7d49d882014-10-09 15:27:11 -07001595 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1596 zone);
1597 if (!page)
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001598 continue;
1599
1600 /* Check the block is suitable for migration */
Vlastimil Babka9f7e3382016-10-07 17:00:37 -07001601 if (!suitable_migration_target(cc, page))
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001602 continue;
Linus Torvalds68e3e922012-06-03 20:05:57 -07001603
Mel Gormanbb13ffe2012-10-08 16:32:41 -07001604 /* If isolation recently failed, do not retry */
1605 if (!isolation_suitable(cc, page))
1606 continue;
1607
Vlastimil Babkae14c7202014-10-09 15:27:20 -07001608 /* Found a block suitable for isolating free pages from. */
Mel Gorman4fca9732019-03-05 15:45:34 -08001609 nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
1610 block_end_pfn, freelist, stride, false);
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001611
Mel Gormand097a6f2019-03-05 15:45:28 -08001612 /* Update the skip hint if the full pageblock was scanned */
1613 if (isolate_start_pfn == block_end_pfn)
1614 update_pageblock_skip(cc, page, block_start_pfn);
1615
Mel Gormancb2dcaf2019-03-05 15:45:11 -08001616 /* Are enough freepages isolated? */
1617 if (cc->nr_freepages >= cc->nr_migratepages) {
David Rientjesa46cbf32016-07-14 12:06:50 -07001618 if (isolate_start_pfn >= block_end_pfn) {
1619 /*
1620 * Restart at previous pageblock if more
1621 * freepages can be isolated next time.
1622 */
Vlastimil Babkaf5f61a32015-09-08 15:02:39 -07001623 isolate_start_pfn =
1624 block_start_pfn - pageblock_nr_pages;
David Rientjesa46cbf32016-07-14 12:06:50 -07001625 }
Vlastimil Babkabe976572014-06-04 16:10:41 -07001626 break;
David Rientjesa46cbf32016-07-14 12:06:50 -07001627 } else if (isolate_start_pfn < block_end_pfn) {
Vlastimil Babkaf5f61a32015-09-08 15:02:39 -07001628 /*
David Rientjesa46cbf32016-07-14 12:06:50 -07001629 * If isolation failed early, do not continue
1630 * needlessly.
Vlastimil Babkaf5f61a32015-09-08 15:02:39 -07001631 */
David Rientjesa46cbf32016-07-14 12:06:50 -07001632 break;
Vlastimil Babkaf5f61a32015-09-08 15:02:39 -07001633 }
Mel Gorman4fca9732019-03-05 15:45:34 -08001634
1635 /* Adjust stride depending on isolation */
1636 if (nr_isolated) {
1637 stride = 1;
1638 continue;
1639 }
1640 stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
Michal Nazarewicz2fe86e02012-01-30 13:16:26 +01001641 }
1642
Vlastimil Babka7ed695e2014-01-21 15:51:09 -08001643 /*
Vlastimil Babkaf5f61a32015-09-08 15:02:39 -07001644 * Record where the free scanner will restart next time. Either we
1645 * broke from the loop and set isolate_start_pfn based on the last
1646 * call to isolate_freepages_block(), or we met the migration scanner
1647 * and the loop terminated due to isolate_start_pfn < low_pfn
Vlastimil Babka7ed695e2014-01-21 15:51:09 -08001648 */
Vlastimil Babkaf5f61a32015-09-08 15:02:39 -07001649 cc->free_pfn = isolate_start_pfn;
Mel Gorman5a811882019-03-05 15:45:01 -08001650
1651splitmap:
1652 /* __isolate_free_page() does not map the pages */
1653 split_map_pages(freelist);
Mel Gorman748446b2010-05-24 14:32:27 -07001654}
1655
1656/*
1657 * This is a migrate-callback that "allocates" freepages by taking pages
1658 * from the isolated freelists in the block we are migrating to.
1659 */
1660static struct page *compaction_alloc(struct page *migratepage,
Michal Hocko666feb22018-04-10 16:30:03 -07001661 unsigned long data)
Mel Gorman748446b2010-05-24 14:32:27 -07001662{
1663 struct compact_control *cc = (struct compact_control *)data;
1664 struct page *freepage;
1665
Mel Gorman748446b2010-05-24 14:32:27 -07001666 if (list_empty(&cc->freepages)) {
Mel Gormancb2dcaf2019-03-05 15:45:11 -08001667 isolate_freepages(cc);
Mel Gorman748446b2010-05-24 14:32:27 -07001668
1669 if (list_empty(&cc->freepages))
1670 return NULL;
1671 }
1672
1673 freepage = list_entry(cc->freepages.next, struct page, lru);
1674 list_del(&freepage->lru);
1675 cc->nr_freepages--;
1676
1677 return freepage;
1678}
1679
1680/*
David Rientjesd53aea32014-06-04 16:08:26 -07001681 * This is a migrate-callback that "frees" freepages back to the isolated
1682 * freelist. All pages on the freelist are from the same zone, so there is no
1683 * special handling needed for NUMA.
1684 */
1685static void compaction_free(struct page *page, unsigned long data)
1686{
1687 struct compact_control *cc = (struct compact_control *)data;
1688
1689 list_add(&page->lru, &cc->freepages);
1690 cc->nr_freepages++;
1691}
1692
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001693/* possible outcome of isolate_migratepages */
1694typedef enum {
1695 ISOLATE_ABORT, /* Abort compaction now */
1696 ISOLATE_NONE, /* No pages isolated, continue scanning */
1697 ISOLATE_SUCCESS, /* Pages isolated, migrate */
1698} isolate_migrate_t;
1699
1700/*
Eric B Munson5bbe3542015-04-15 16:13:20 -07001701 * Allow userspace to control policy on scanning the unevictable LRU for
1702 * compactable pages.
1703 */
Sebastian Andrzej Siewior6923aa02020-04-01 21:10:42 -07001704#ifdef CONFIG_PREEMPT_RT
1705int sysctl_compact_unevictable_allowed __read_mostly = 0;
1706#else
Eric B Munson5bbe3542015-04-15 16:13:20 -07001707int sysctl_compact_unevictable_allowed __read_mostly = 1;
Sebastian Andrzej Siewior6923aa02020-04-01 21:10:42 -07001708#endif
Eric B Munson5bbe3542015-04-15 16:13:20 -07001709
Mel Gorman70b44592019-03-05 15:44:54 -08001710static inline void
1711update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
1712{
1713 if (cc->fast_start_pfn == ULONG_MAX)
1714 return;
1715
1716 if (!cc->fast_start_pfn)
1717 cc->fast_start_pfn = pfn;
1718
1719 cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
1720}
1721
1722static inline unsigned long
1723reinit_migrate_pfn(struct compact_control *cc)
1724{
1725 if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
1726 return cc->migrate_pfn;
1727
1728 cc->migrate_pfn = cc->fast_start_pfn;
1729 cc->fast_start_pfn = ULONG_MAX;
1730
1731 return cc->migrate_pfn;
1732}
1733
1734/*
1735 * Briefly search the free lists for a migration source that already has
1736 * some free pages to reduce the number of pages that need migration
1737 * before a pageblock is free.
1738 */
1739static unsigned long fast_find_migrateblock(struct compact_control *cc)
1740{
1741 unsigned int limit = freelist_scan_limit(cc);
1742 unsigned int nr_scanned = 0;
1743 unsigned long distance;
1744 unsigned long pfn = cc->migrate_pfn;
1745 unsigned long high_pfn;
1746 int order;
Wonhyuk Yang15d28d02021-02-24 12:09:36 -08001747 bool found_block = false;
Mel Gorman70b44592019-03-05 15:44:54 -08001748
1749 /* Skip hints are relied on to avoid repeats on the fast search */
1750 if (cc->ignore_skip_hint)
1751 return pfn;
1752
1753 /*
1754 * If the migrate_pfn is not at the start of a zone or the start
1755 * of a pageblock then assume this is a continuation of a previous
1756 * scan restarted due to COMPACT_CLUSTER_MAX.
1757 */
1758 if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
1759 return pfn;
1760
1761 /*
1762 * For smaller orders, just linearly scan as the number of pages
1763 * to migrate should be relatively small and does not necessarily
1764 * justify freeing up a large block for a small allocation.
1765 */
1766 if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
1767 return pfn;
1768
1769 /*
1770 * Only allow kcompactd and direct requests for movable pages to
1771 * quickly clear out a MOVABLE pageblock for allocation. This
1772 * reduces the risk that a large movable pageblock is freed for
1773 * an unmovable/reclaimable small allocation.
1774 */
1775 if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
1776 return pfn;
1777
1778 /*
1779 * When starting the migration scanner, pick any pageblock within the
1780 * first half of the search space. Otherwise try and pick a pageblock
1781 * within the first eighth to reduce the chances that a migration
1782 * target later becomes a source.
1783 */
1784 distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
1785 if (cc->migrate_pfn != cc->zone->zone_start_pfn)
1786 distance >>= 2;
1787 high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
1788
1789 for (order = cc->order - 1;
Wonhyuk Yang15d28d02021-02-24 12:09:36 -08001790 order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit;
Mel Gorman70b44592019-03-05 15:44:54 -08001791 order--) {
1792 struct free_area *area = &cc->zone->free_area[order];
1793 struct list_head *freelist;
1794 unsigned long flags;
1795 struct page *freepage;
1796
1797 if (!area->nr_free)
1798 continue;
1799
1800 spin_lock_irqsave(&cc->zone->lock, flags);
1801 freelist = &area->free_list[MIGRATE_MOVABLE];
1802 list_for_each_entry(freepage, freelist, lru) {
1803 unsigned long free_pfn;
1804
Wonhyuk Yang15d28d02021-02-24 12:09:36 -08001805 if (nr_scanned++ >= limit) {
1806 move_freelist_tail(freelist, freepage);
1807 break;
1808 }
1809
Mel Gorman70b44592019-03-05 15:44:54 -08001810 free_pfn = page_to_pfn(freepage);
1811 if (free_pfn < high_pfn) {
Mel Gorman70b44592019-03-05 15:44:54 -08001812 /*
1813 * Avoid if skipped recently. Ideally it would
1814 * move to the tail but even safe iteration of
1815 * the list assumes an entry is deleted, not
1816 * reordered.
1817 */
Wonhyuk Yang15d28d02021-02-24 12:09:36 -08001818 if (get_pageblock_skip(freepage))
Mel Gorman70b44592019-03-05 15:44:54 -08001819 continue;
Mel Gorman70b44592019-03-05 15:44:54 -08001820
1821 /* Reorder to so a future search skips recent pages */
1822 move_freelist_tail(freelist, freepage);
1823
Mel Gormane380beb2019-03-05 15:44:58 -08001824 update_fast_start_pfn(cc, free_pfn);
Mel Gorman70b44592019-03-05 15:44:54 -08001825 pfn = pageblock_start_pfn(free_pfn);
1826 cc->fast_search_fail = 0;
Wonhyuk Yang15d28d02021-02-24 12:09:36 -08001827 found_block = true;
Mel Gorman70b44592019-03-05 15:44:54 -08001828 set_pageblock_skip(freepage);
1829 break;
1830 }
Mel Gorman70b44592019-03-05 15:44:54 -08001831 }
1832 spin_unlock_irqrestore(&cc->zone->lock, flags);
1833 }
1834
1835 cc->total_migrate_scanned += nr_scanned;
1836
1837 /*
1838 * If fast scanning failed then use a cached entry for a page block
1839 * that had free pages as the basis for starting a linear scan.
1840 */
Wonhyuk Yang15d28d02021-02-24 12:09:36 -08001841 if (!found_block) {
1842 cc->fast_search_fail++;
Mel Gorman70b44592019-03-05 15:44:54 -08001843 pfn = reinit_migrate_pfn(cc);
Wonhyuk Yang15d28d02021-02-24 12:09:36 -08001844 }
Mel Gorman70b44592019-03-05 15:44:54 -08001845 return pfn;
1846}
1847
Eric B Munson5bbe3542015-04-15 16:13:20 -07001848/*
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001849 * Isolate all pages that can be migrated from the first suitable block,
1850 * starting at the block pointed to by the migrate scanner pfn within
1851 * compact_control.
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001852 */
Pengfei Li32aaf052019-09-23 15:36:58 -07001853static isolate_migrate_t isolate_migratepages(struct compact_control *cc)
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001854{
Joonsoo Kime1409c32016-03-15 14:57:48 -07001855 unsigned long block_start_pfn;
1856 unsigned long block_end_pfn;
1857 unsigned long low_pfn;
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001858 struct page *page;
1859 const isolate_mode_t isolate_mode =
Eric B Munson5bbe3542015-04-15 16:13:20 -07001860 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
Hugh Dickins1d2047f2016-07-28 15:48:41 -07001861 (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
Mel Gorman70b44592019-03-05 15:44:54 -08001862 bool fast_find_block;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001863
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001864 /*
1865 * Start at where we last stopped, or beginning of the zone as
Mel Gorman70b44592019-03-05 15:44:54 -08001866 * initialized by compact_zone(). The first failure will use
1867 * the lowest PFN as the starting point for linear scanning.
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001868 */
Mel Gorman70b44592019-03-05 15:44:54 -08001869 low_pfn = fast_find_migrateblock(cc);
Vlastimil Babka06b66402016-05-19 17:11:48 -07001870 block_start_pfn = pageblock_start_pfn(low_pfn);
Pengfei Li32aaf052019-09-23 15:36:58 -07001871 if (block_start_pfn < cc->zone->zone_start_pfn)
1872 block_start_pfn = cc->zone->zone_start_pfn;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001873
Mel Gorman70b44592019-03-05 15:44:54 -08001874 /*
1875 * fast_find_migrateblock marks a pageblock skipped so to avoid
1876 * the isolation_suitable check below, check whether the fast
1877 * search was successful.
1878 */
1879 fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
1880
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001881 /* Only scan within a pageblock boundary */
Vlastimil Babka06b66402016-05-19 17:11:48 -07001882 block_end_pfn = pageblock_end_pfn(low_pfn);
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001883
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001884 /*
1885 * Iterate over whole pageblocks until we find the first suitable.
1886 * Do not cross the free scanner.
1887 */
Joonsoo Kime1409c32016-03-15 14:57:48 -07001888 for (; block_end_pfn <= cc->free_pfn;
Mel Gorman70b44592019-03-05 15:44:54 -08001889 fast_find_block = false,
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001890 cc->migrate_pfn = low_pfn = block_end_pfn,
Joonsoo Kime1409c32016-03-15 14:57:48 -07001891 block_start_pfn = block_end_pfn,
1892 block_end_pfn += pageblock_nr_pages) {
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001893
1894 /*
1895 * This can potentially iterate a massively long zone with
1896 * many pageblocks unsuitable, so periodically check if we
Mel Gormancb810ad2019-03-05 15:45:21 -08001897 * need to schedule.
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001898 */
Mel Gormancb810ad2019-03-05 15:45:21 -08001899 if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
Mel Gormancf66f072019-03-05 15:45:24 -08001900 cond_resched();
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001901
Pengfei Li32aaf052019-09-23 15:36:58 -07001902 page = pageblock_pfn_to_page(block_start_pfn,
1903 block_end_pfn, cc->zone);
Vlastimil Babka7d49d882014-10-09 15:27:11 -07001904 if (!page)
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001905 continue;
1906
Mel Gormane380beb2019-03-05 15:44:58 -08001907 /*
1908 * If isolation recently failed, do not retry. Only check the
1909 * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock
1910 * to be visited multiple times. Assume skip was checked
1911 * before making it "skip" so other compaction instances do
1912 * not scan the same block.
1913 */
1914 if (IS_ALIGNED(low_pfn, pageblock_nr_pages) &&
1915 !fast_find_block && !isolation_suitable(cc, page))
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001916 continue;
1917
1918 /*
Mel Gorman9bebefd2019-03-05 15:45:14 -08001919 * For async compaction, also only scan in MOVABLE blocks
1920 * without huge pages. Async compaction is optimistic to see
1921 * if the minimum amount of work satisfies the allocation.
1922 * The cached PFN is updated as it's possible that all
1923 * remaining blocks between source and target are unsuitable
1924 * and the compaction scanners fail to meet.
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001925 */
Mel Gorman9bebefd2019-03-05 15:45:14 -08001926 if (!suitable_migration_source(cc, page)) {
1927 update_cached_migrate(cc, block_end_pfn);
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001928 continue;
Mel Gorman9bebefd2019-03-05 15:45:14 -08001929 }
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001930
1931 /* Perform the isolation */
Oscar Salvadorc2ad7a12021-05-04 18:35:17 -07001932 if (isolate_migratepages_block(cc, low_pfn, block_end_pfn,
1933 isolate_mode))
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001934 return ISOLATE_ABORT;
1935
1936 /*
1937 * Either we isolated something and proceed with migration. Or
1938 * we failed and compact_zone should decide if we should
1939 * continue or not.
1940 */
1941 break;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001942 }
1943
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07001944 return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01001945}
1946
Yaowei Bai21c527a2015-11-05 18:47:20 -08001947/*
1948 * order == -1 is expected when compacting via
1949 * /proc/sys/vm/compact_memory
1950 */
1951static inline bool is_via_compact_memory(int order)
1952{
1953 return order == -1;
1954}
1955
Nitin Guptafacdaa92020-08-11 18:31:00 -07001956static bool kswapd_is_running(pg_data_t *pgdat)
1957{
1958 return pgdat->kswapd && (pgdat->kswapd->state == TASK_RUNNING);
1959}
1960
1961/*
1962 * A zone's fragmentation score is the external fragmentation wrt to the
Charan Teja Reddy40d7e202021-02-24 12:09:32 -08001963 * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100].
1964 */
1965static unsigned int fragmentation_score_zone(struct zone *zone)
1966{
1967 return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);
1968}
1969
1970/*
1971 * A weighted zone's fragmentation score is the external fragmentation
1972 * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It
1973 * returns a value in the range [0, 100].
Nitin Guptafacdaa92020-08-11 18:31:00 -07001974 *
1975 * The scaling factor ensures that proactive compaction focuses on larger
1976 * zones like ZONE_NORMAL, rather than smaller, specialized zones like
1977 * ZONE_DMA32. For smaller zones, the score value remains close to zero,
1978 * and thus never exceeds the high threshold for proactive compaction.
1979 */
Charan Teja Reddy40d7e202021-02-24 12:09:32 -08001980static unsigned int fragmentation_score_zone_weighted(struct zone *zone)
Nitin Guptafacdaa92020-08-11 18:31:00 -07001981{
1982 unsigned long score;
1983
Charan Teja Reddy40d7e202021-02-24 12:09:32 -08001984 score = zone->present_pages * fragmentation_score_zone(zone);
Nitin Guptafacdaa92020-08-11 18:31:00 -07001985 return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);
1986}
1987
1988/*
1989 * The per-node proactive (background) compaction process is started by its
1990 * corresponding kcompactd thread when the node's fragmentation score
1991 * exceeds the high threshold. The compaction process remains active till
1992 * the node's score falls below the low threshold, or one of the back-off
1993 * conditions is met.
1994 */
Nitin Guptad34c0a72020-08-11 18:31:07 -07001995static unsigned int fragmentation_score_node(pg_data_t *pgdat)
Nitin Guptafacdaa92020-08-11 18:31:00 -07001996{
Nitin Guptad34c0a72020-08-11 18:31:07 -07001997 unsigned int score = 0;
Nitin Guptafacdaa92020-08-11 18:31:00 -07001998 int zoneid;
1999
2000 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2001 struct zone *zone;
2002
2003 zone = &pgdat->node_zones[zoneid];
Charan Teja Reddy40d7e202021-02-24 12:09:32 -08002004 score += fragmentation_score_zone_weighted(zone);
Nitin Guptafacdaa92020-08-11 18:31:00 -07002005 }
2006
2007 return score;
2008}
2009
Nitin Guptad34c0a72020-08-11 18:31:07 -07002010static unsigned int fragmentation_score_wmark(pg_data_t *pgdat, bool low)
Nitin Guptafacdaa92020-08-11 18:31:00 -07002011{
Nitin Guptad34c0a72020-08-11 18:31:07 -07002012 unsigned int wmark_low;
Nitin Guptafacdaa92020-08-11 18:31:00 -07002013
2014 /*
Ingo Molnarf0953a12021-05-06 18:06:47 -07002015 * Cap the low watermark to avoid excessive compaction
2016 * activity in case a user sets the proactiveness tunable
Nitin Guptafacdaa92020-08-11 18:31:00 -07002017 * close to 100 (maximum).
2018 */
Nitin Guptad34c0a72020-08-11 18:31:07 -07002019 wmark_low = max(100U - sysctl_compaction_proactiveness, 5U);
2020 return low ? wmark_low : min(wmark_low + 10, 100U);
Nitin Guptafacdaa92020-08-11 18:31:00 -07002021}
2022
2023static bool should_proactive_compact_node(pg_data_t *pgdat)
2024{
2025 int wmark_high;
2026
2027 if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat))
2028 return false;
2029
2030 wmark_high = fragmentation_score_wmark(pgdat, false);
2031 return fragmentation_score_node(pgdat) > wmark_high;
2032}
2033
Mel Gorman40cacbc2019-03-05 15:44:36 -08002034static enum compact_result __compact_finished(struct compact_control *cc)
Mel Gorman748446b2010-05-24 14:32:27 -07002035{
Mel Gorman8fb74b92013-01-11 14:32:16 -08002036 unsigned int order;
Vlastimil Babkad39773a2017-05-08 15:54:46 -07002037 const int migratetype = cc->migratetype;
Mel Gormancb2dcaf2019-03-05 15:45:11 -08002038 int ret;
Mel Gorman748446b2010-05-24 14:32:27 -07002039
Mel Gorman753341a2012-10-08 16:32:40 -07002040 /* Compaction run completes if the migrate and free scanner meet */
Vlastimil Babkaf2849aa2015-09-08 15:02:36 -07002041 if (compact_scanners_met(cc)) {
Vlastimil Babka55b7c4c2014-01-21 15:51:11 -08002042 /* Let the next compaction start anew. */
Mel Gorman40cacbc2019-03-05 15:44:36 -08002043 reset_cached_positions(cc->zone);
Vlastimil Babka55b7c4c2014-01-21 15:51:11 -08002044
Mel Gorman62997022012-10-08 16:32:47 -07002045 /*
2046 * Mark that the PG_migrate_skip information should be cleared
Vlastimil Babkaaccf6242016-03-17 14:18:15 -07002047 * by kswapd when it goes to sleep. kcompactd does not set the
Mel Gorman62997022012-10-08 16:32:47 -07002048 * flag itself as the decision to be clear should be directly
2049 * based on an allocation request.
2050 */
Vlastimil Babkaaccf6242016-03-17 14:18:15 -07002051 if (cc->direct_compaction)
Mel Gorman40cacbc2019-03-05 15:44:36 -08002052 cc->zone->compact_blockskip_flush = true;
Mel Gorman62997022012-10-08 16:32:47 -07002053
Michal Hockoc8f7de02016-05-20 16:56:47 -07002054 if (cc->whole_zone)
2055 return COMPACT_COMPLETE;
2056 else
2057 return COMPACT_PARTIAL_SKIPPED;
Mel Gormanbb13ffe2012-10-08 16:32:41 -07002058 }
Mel Gorman748446b2010-05-24 14:32:27 -07002059
Nitin Guptafacdaa92020-08-11 18:31:00 -07002060 if (cc->proactive_compaction) {
2061 int score, wmark_low;
2062 pg_data_t *pgdat;
2063
2064 pgdat = cc->zone->zone_pgdat;
2065 if (kswapd_is_running(pgdat))
2066 return COMPACT_PARTIAL_SKIPPED;
2067
2068 score = fragmentation_score_zone(cc->zone);
2069 wmark_low = fragmentation_score_wmark(pgdat, true);
2070
2071 if (score > wmark_low)
2072 ret = COMPACT_CONTINUE;
2073 else
2074 ret = COMPACT_SUCCESS;
2075
2076 goto out;
2077 }
2078
Yaowei Bai21c527a2015-11-05 18:47:20 -08002079 if (is_via_compact_memory(cc->order))
Mel Gorman56de7262010-05-24 14:32:30 -07002080 return COMPACT_CONTINUE;
2081
Mel Gormanefe771c2019-03-05 15:44:46 -08002082 /*
2083 * Always finish scanning a pageblock to reduce the possibility of
2084 * fallbacks in the future. This is particularly important when
2085 * migration source is unmovable/reclaimable but it's not worth
2086 * special casing.
2087 */
2088 if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
2089 return COMPACT_CONTINUE;
Vlastimil Babkabaf6a9a2017-05-08 15:54:52 -07002090
Mel Gorman56de7262010-05-24 14:32:30 -07002091 /* Direct compactor: Is a suitable page free? */
Mel Gormancb2dcaf2019-03-05 15:45:11 -08002092 ret = COMPACT_NO_SUITABLE_PAGE;
Mel Gorman8fb74b92013-01-11 14:32:16 -08002093 for (order = cc->order; order < MAX_ORDER; order++) {
Mel Gorman40cacbc2019-03-05 15:44:36 -08002094 struct free_area *area = &cc->zone->free_area[order];
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002095 bool can_steal;
Mel Gorman56de7262010-05-24 14:32:30 -07002096
Mel Gorman8fb74b92013-01-11 14:32:16 -08002097 /* Job done if page is free of the right migratetype */
Dan Williamsb03641a2019-05-14 15:41:32 -07002098 if (!free_area_empty(area, migratetype))
Vlastimil Babkacf378312016-10-07 16:57:41 -07002099 return COMPACT_SUCCESS;
Mel Gorman8fb74b92013-01-11 14:32:16 -08002100
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002101#ifdef CONFIG_CMA
2102 /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */
2103 if (migratetype == MIGRATE_MOVABLE &&
Dan Williamsb03641a2019-05-14 15:41:32 -07002104 !free_area_empty(area, MIGRATE_CMA))
Vlastimil Babkacf378312016-10-07 16:57:41 -07002105 return COMPACT_SUCCESS;
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002106#endif
2107 /*
2108 * Job done if allocation would steal freepages from
2109 * other migratetype buddy lists.
2110 */
2111 if (find_suitable_fallback(area, order, migratetype,
Vlastimil Babkabaf6a9a2017-05-08 15:54:52 -07002112 true, &can_steal) != -1) {
2113
2114 /* movable pages are OK in any pageblock */
2115 if (migratetype == MIGRATE_MOVABLE)
2116 return COMPACT_SUCCESS;
2117
2118 /*
2119 * We are stealing for a non-movable allocation. Make
2120 * sure we finish compacting the current pageblock
2121 * first so it is as free as possible and we won't
2122 * have to steal another one soon. This only applies
2123 * to sync compaction, as async compaction operates
2124 * on pageblocks of the same migratetype.
2125 */
2126 if (cc->mode == MIGRATE_ASYNC ||
2127 IS_ALIGNED(cc->migrate_pfn,
2128 pageblock_nr_pages)) {
2129 return COMPACT_SUCCESS;
2130 }
2131
Mel Gormancb2dcaf2019-03-05 15:45:11 -08002132 ret = COMPACT_CONTINUE;
2133 break;
Vlastimil Babkabaf6a9a2017-05-08 15:54:52 -07002134 }
Mel Gorman56de7262010-05-24 14:32:30 -07002135 }
2136
Nitin Guptafacdaa92020-08-11 18:31:00 -07002137out:
Mel Gormancb2dcaf2019-03-05 15:45:11 -08002138 if (cc->contended || fatal_signal_pending(current))
2139 ret = COMPACT_CONTENDED;
2140
2141 return ret;
Joonsoo Kim837d0262015-02-11 15:27:06 -08002142}
2143
Mel Gorman40cacbc2019-03-05 15:44:36 -08002144static enum compact_result compact_finished(struct compact_control *cc)
Joonsoo Kim837d0262015-02-11 15:27:06 -08002145{
2146 int ret;
2147
Mel Gorman40cacbc2019-03-05 15:44:36 -08002148 ret = __compact_finished(cc);
2149 trace_mm_compaction_finished(cc->zone, cc->order, ret);
Joonsoo Kim837d0262015-02-11 15:27:06 -08002150 if (ret == COMPACT_NO_SUITABLE_PAGE)
2151 ret = COMPACT_CONTINUE;
2152
2153 return ret;
Mel Gorman748446b2010-05-24 14:32:27 -07002154}
2155
Michal Hockoea7ab982016-05-20 16:56:38 -07002156static enum compact_result __compaction_suitable(struct zone *zone, int order,
Mel Gormanc6038442016-05-19 17:13:38 -07002157 unsigned int alloc_flags,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002158 int highest_zoneidx,
Michal Hocko86a294a2016-05-20 16:57:12 -07002159 unsigned long wmark_target)
Mel Gorman3e7d3442011-01-13 15:45:56 -08002160{
Mel Gorman3e7d3442011-01-13 15:45:56 -08002161 unsigned long watermark;
2162
Yaowei Bai21c527a2015-11-05 18:47:20 -08002163 if (is_via_compact_memory(order))
Michal Hocko3957c772011-06-15 15:08:25 -07002164 return COMPACT_CONTINUE;
2165
Mel Gormana9214442018-12-28 00:35:44 -08002166 watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
Vlastimil Babkaebff3982014-12-10 15:43:22 -08002167 /*
2168 * If watermarks for high-order allocation are already met, there
2169 * should be no need for compaction at all.
2170 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002171 if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,
Vlastimil Babkaebff3982014-12-10 15:43:22 -08002172 alloc_flags))
Vlastimil Babkacf378312016-10-07 16:57:41 -07002173 return COMPACT_SUCCESS;
Vlastimil Babkaebff3982014-12-10 15:43:22 -08002174
Michal Hocko3957c772011-06-15 15:08:25 -07002175 /*
Vlastimil Babka9861a622016-10-07 16:57:53 -07002176 * Watermarks for order-0 must be met for compaction to be able to
Vlastimil Babka984fdba2016-10-07 16:57:57 -07002177 * isolate free pages for migration targets. This means that the
2178 * watermark and alloc_flags have to match, or be more pessimistic than
2179 * the check in __isolate_free_page(). We don't use the direct
2180 * compactor's alloc_flags, as they are not relevant for freepage
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002181 * isolation. We however do use the direct compactor's highest_zoneidx
2182 * to skip over zones where lowmem reserves would prevent allocation
2183 * even if compaction succeeds.
Vlastimil Babka8348faf2016-10-07 16:58:00 -07002184 * For costly orders, we require low watermark instead of min for
2185 * compaction to proceed to increase its chances.
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09002186 * ALLOC_CMA is used, as pages in CMA pageblocks are considered
2187 * suitable migration targets
Mel Gorman3e7d3442011-01-13 15:45:56 -08002188 */
Vlastimil Babka8348faf2016-10-07 16:58:00 -07002189 watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
2190 low_wmark_pages(zone) : min_wmark_pages(zone);
2191 watermark += compact_gap(order);
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002192 if (!__zone_watermark_ok(zone, 0, watermark, highest_zoneidx,
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09002193 ALLOC_CMA, wmark_target))
Mel Gorman3e7d3442011-01-13 15:45:56 -08002194 return COMPACT_SKIPPED;
2195
Vlastimil Babkacc5c9f02016-10-07 17:00:43 -07002196 return COMPACT_CONTINUE;
2197}
2198
Hui Su2b1a20c2020-12-14 19:12:42 -08002199/*
2200 * compaction_suitable: Is this suitable to run compaction on this zone now?
2201 * Returns
2202 * COMPACT_SKIPPED - If there are too few free pages for compaction
2203 * COMPACT_SUCCESS - If the allocation would succeed without compaction
2204 * COMPACT_CONTINUE - If compaction should run now
2205 */
Vlastimil Babkacc5c9f02016-10-07 17:00:43 -07002206enum compact_result compaction_suitable(struct zone *zone, int order,
2207 unsigned int alloc_flags,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002208 int highest_zoneidx)
Vlastimil Babkacc5c9f02016-10-07 17:00:43 -07002209{
2210 enum compact_result ret;
2211 int fragindex;
2212
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002213 ret = __compaction_suitable(zone, order, alloc_flags, highest_zoneidx,
Vlastimil Babkacc5c9f02016-10-07 17:00:43 -07002214 zone_page_state(zone, NR_FREE_PAGES));
Mel Gorman3e7d3442011-01-13 15:45:56 -08002215 /*
2216 * fragmentation index determines if allocation failures are due to
2217 * low memory or external fragmentation
2218 *
Vlastimil Babkaebff3982014-12-10 15:43:22 -08002219 * index of -1000 would imply allocations might succeed depending on
2220 * watermarks, but we already failed the high-order watermark check
Mel Gorman3e7d3442011-01-13 15:45:56 -08002221 * index towards 0 implies failure is due to lack of memory
2222 * index towards 1000 implies failure is due to fragmentation
2223 *
Vlastimil Babka20311422016-10-07 17:00:46 -07002224 * Only compact if a failure would be due to fragmentation. Also
2225 * ignore fragindex for non-costly orders where the alternative to
2226 * a successful reclaim/compaction is OOM. Fragindex and the
2227 * vm.extfrag_threshold sysctl is meant as a heuristic to prevent
2228 * excessive compaction for costly orders, but it should not be at the
2229 * expense of system stability.
Mel Gorman3e7d3442011-01-13 15:45:56 -08002230 */
Vlastimil Babka20311422016-10-07 17:00:46 -07002231 if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) {
Vlastimil Babkacc5c9f02016-10-07 17:00:43 -07002232 fragindex = fragmentation_index(zone, order);
2233 if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold)
2234 ret = COMPACT_NOT_SUITABLE_ZONE;
2235 }
Mel Gorman3e7d3442011-01-13 15:45:56 -08002236
Joonsoo Kim837d0262015-02-11 15:27:06 -08002237 trace_mm_compaction_suitable(zone, order, ret);
2238 if (ret == COMPACT_NOT_SUITABLE_ZONE)
2239 ret = COMPACT_SKIPPED;
2240
2241 return ret;
2242}
2243
Michal Hocko86a294a2016-05-20 16:57:12 -07002244bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
2245 int alloc_flags)
2246{
2247 struct zone *zone;
2248 struct zoneref *z;
2249
2250 /*
2251 * Make sure at least one zone would pass __compaction_suitable if we continue
2252 * retrying the reclaim.
2253 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002254 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
2255 ac->highest_zoneidx, ac->nodemask) {
Michal Hocko86a294a2016-05-20 16:57:12 -07002256 unsigned long available;
2257 enum compact_result compact_result;
2258
2259 /*
2260 * Do not consider all the reclaimable memory because we do not
2261 * want to trash just for a single high order allocation which
2262 * is even not guaranteed to appear even if __compaction_suitable
2263 * is happy about the watermark check.
2264 */
Mel Gorman5a1c84b2016-07-28 15:47:31 -07002265 available = zone_reclaimable_pages(zone) / order;
Michal Hocko86a294a2016-05-20 16:57:12 -07002266 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
2267 compact_result = __compaction_suitable(zone, order, alloc_flags,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002268 ac->highest_zoneidx, available);
Vlastimil Babkacc5c9f02016-10-07 17:00:43 -07002269 if (compact_result != COMPACT_SKIPPED)
Michal Hocko86a294a2016-05-20 16:57:12 -07002270 return true;
2271 }
2272
2273 return false;
2274}
2275
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002276static enum compact_result
2277compact_zone(struct compact_control *cc, struct capture_control *capc)
Mel Gorman748446b2010-05-24 14:32:27 -07002278{
Michal Hockoea7ab982016-05-20 16:56:38 -07002279 enum compact_result ret;
Mel Gorman40cacbc2019-03-05 15:44:36 -08002280 unsigned long start_pfn = cc->zone->zone_start_pfn;
2281 unsigned long end_pfn = zone_end_pfn(cc->zone);
Mel Gorman566e54e2019-03-05 15:44:32 -08002282 unsigned long last_migrated_pfn;
David Rientjese0b9dae2014-06-04 16:08:28 -07002283 const bool sync = cc->mode != MIGRATE_ASYNC;
Mel Gorman8854c552019-03-05 15:45:18 -08002284 bool update_cached;
Mel Gorman748446b2010-05-24 14:32:27 -07002285
Yafang Shaoa94b5252019-09-23 15:36:54 -07002286 /*
2287 * These counters track activities during zone compaction. Initialize
2288 * them before compacting a new zone.
2289 */
2290 cc->total_migrate_scanned = 0;
2291 cc->total_free_scanned = 0;
2292 cc->nr_migratepages = 0;
2293 cc->nr_freepages = 0;
2294 INIT_LIST_HEAD(&cc->freepages);
2295 INIT_LIST_HEAD(&cc->migratepages);
2296
Wei Yang01c0bfe2020-06-03 15:59:08 -07002297 cc->migratetype = gfp_migratetype(cc->gfp_mask);
Mel Gorman40cacbc2019-03-05 15:44:36 -08002298 ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002299 cc->highest_zoneidx);
Michal Hockoc46649d2016-05-20 16:56:41 -07002300 /* Compaction is likely to fail */
Vlastimil Babkacf378312016-10-07 16:57:41 -07002301 if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
Mel Gorman3e7d3442011-01-13 15:45:56 -08002302 return ret;
Michal Hockoc46649d2016-05-20 16:56:41 -07002303
2304 /* huh, compaction_suitable is returning something unexpected */
2305 VM_BUG_ON(ret != COMPACT_CONTINUE);
Mel Gorman3e7d3442011-01-13 15:45:56 -08002306
Mel Gormanc89511a2012-10-08 16:32:45 -07002307 /*
Vlastimil Babkad3132e42014-01-21 15:51:08 -08002308 * Clear pageblock skip if there were failures recently and compaction
Vlastimil Babkaaccf6242016-03-17 14:18:15 -07002309 * is about to be retried after being deferred.
Vlastimil Babkad3132e42014-01-21 15:51:08 -08002310 */
Mel Gorman40cacbc2019-03-05 15:44:36 -08002311 if (compaction_restarting(cc->zone, cc->order))
2312 __reset_isolation_suitable(cc->zone);
Vlastimil Babkad3132e42014-01-21 15:51:08 -08002313
2314 /*
Mel Gormanc89511a2012-10-08 16:32:45 -07002315 * Setup to move all movable pages to the end of the zone. Used cached
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002316 * information on where the scanners should start (unless we explicitly
2317 * want to compact the whole zone), but check that it is initialised
2318 * by ensuring the values are within zone boundaries.
Mel Gormanc89511a2012-10-08 16:32:45 -07002319 */
Mel Gorman70b44592019-03-05 15:44:54 -08002320 cc->fast_start_pfn = 0;
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002321 if (cc->whole_zone) {
Mel Gormanc89511a2012-10-08 16:32:45 -07002322 cc->migrate_pfn = start_pfn;
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002323 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2324 } else {
Mel Gorman40cacbc2019-03-05 15:44:36 -08002325 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
2326 cc->free_pfn = cc->zone->compact_cached_free_pfn;
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002327 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
2328 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
Mel Gorman40cacbc2019-03-05 15:44:36 -08002329 cc->zone->compact_cached_free_pfn = cc->free_pfn;
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002330 }
2331 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
2332 cc->migrate_pfn = start_pfn;
Mel Gorman40cacbc2019-03-05 15:44:36 -08002333 cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
2334 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002335 }
Michal Hockoc8f7de02016-05-20 16:56:47 -07002336
Mel Gormane332f742019-03-05 15:45:38 -08002337 if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002338 cc->whole_zone = true;
2339 }
Michal Hockoc8f7de02016-05-20 16:56:47 -07002340
Mel Gorman566e54e2019-03-05 15:44:32 -08002341 last_migrated_pfn = 0;
Mel Gorman748446b2010-05-24 14:32:27 -07002342
Mel Gorman8854c552019-03-05 15:45:18 -08002343 /*
2344 * Migrate has separate cached PFNs for ASYNC and SYNC* migration on
2345 * the basis that some migrations will fail in ASYNC mode. However,
2346 * if the cached PFNs match and pageblocks are skipped due to having
2347 * no isolation candidates, then the sync state does not matter.
2348 * Until a pageblock with isolation candidates is found, keep the
2349 * cached PFNs in sync to avoid revisiting the same blocks.
2350 */
2351 update_cached = !sync &&
2352 cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
2353
Joonsoo Kim16c4a092015-02-11 15:27:01 -08002354 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
2355 cc->free_pfn, end_pfn, sync);
Mel Gorman0eb927c2014-01-21 15:51:05 -08002356
Minchan Kim361a2a22021-05-04 18:36:57 -07002357 /* lru_add_drain_all could be expensive with involving other CPUs */
2358 lru_add_drain();
Mel Gorman748446b2010-05-24 14:32:27 -07002359
Mel Gorman40cacbc2019-03-05 15:44:36 -08002360 while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
Minchan Kim9d502c12011-03-22 16:30:39 -07002361 int err;
Yanfei Xu19d3cf92020-12-14 19:12:39 -08002362 unsigned long iteration_start_pfn = cc->migrate_pfn;
Mel Gorman748446b2010-05-24 14:32:27 -07002363
Mel Gorman804d3122019-03-05 15:45:07 -08002364 /*
2365 * Avoid multiple rescans which can happen if a page cannot be
2366 * isolated (dirty/writeback in async mode) or if the migrated
2367 * pages are being allocated before the pageblock is cleared.
2368 * The first rescan will capture the entire pageblock for
2369 * migration. If it fails, it'll be marked skip and scanning
2370 * will proceed as normal.
2371 */
2372 cc->rescan = false;
2373 if (pageblock_start_pfn(last_migrated_pfn) ==
Yanfei Xu19d3cf92020-12-14 19:12:39 -08002374 pageblock_start_pfn(iteration_start_pfn)) {
Mel Gorman804d3122019-03-05 15:45:07 -08002375 cc->rescan = true;
2376 }
2377
Pengfei Li32aaf052019-09-23 15:36:58 -07002378 switch (isolate_migratepages(cc)) {
Mel Gormanf9e35b32011-06-15 15:08:52 -07002379 case ISOLATE_ABORT:
Vlastimil Babka2d1e1042015-11-05 18:48:02 -08002380 ret = COMPACT_CONTENDED;
Rafael Aquini5733c7d2012-12-11 16:02:47 -08002381 putback_movable_pages(&cc->migratepages);
Shaohua Lie64c5232012-10-08 16:32:27 -07002382 cc->nr_migratepages = 0;
Mel Gormanf9e35b32011-06-15 15:08:52 -07002383 goto out;
2384 case ISOLATE_NONE:
Mel Gorman8854c552019-03-05 15:45:18 -08002385 if (update_cached) {
2386 cc->zone->compact_cached_migrate_pfn[1] =
2387 cc->zone->compact_cached_migrate_pfn[0];
2388 }
2389
Vlastimil Babkafdaf7f52014-12-10 15:43:34 -08002390 /*
2391 * We haven't isolated and migrated anything, but
2392 * there might still be unflushed migrations from
2393 * previous cc->order aligned block.
2394 */
2395 goto check_drain;
Mel Gormanf9e35b32011-06-15 15:08:52 -07002396 case ISOLATE_SUCCESS:
Mel Gorman8854c552019-03-05 15:45:18 -08002397 update_cached = false;
Yanfei Xu19d3cf92020-12-14 19:12:39 -08002398 last_migrated_pfn = iteration_start_pfn;
Mel Gormanf9e35b32011-06-15 15:08:52 -07002399 }
Mel Gorman748446b2010-05-24 14:32:27 -07002400
David Rientjesd53aea32014-06-04 16:08:26 -07002401 err = migrate_pages(&cc->migratepages, compaction_alloc,
David Rientjese0b9dae2014-06-04 16:08:28 -07002402 compaction_free, (unsigned long)cc, cc->mode,
Mel Gorman7b2a2d42012-10-19 14:07:31 +01002403 MR_COMPACTION);
Mel Gorman748446b2010-05-24 14:32:27 -07002404
Vlastimil Babkaf8c93012014-06-04 16:08:32 -07002405 trace_mm_compaction_migratepages(cc->nr_migratepages, err,
2406 &cc->migratepages);
Mel Gorman748446b2010-05-24 14:32:27 -07002407
Vlastimil Babkaf8c93012014-06-04 16:08:32 -07002408 /* All pages were either migrated or will be released */
2409 cc->nr_migratepages = 0;
Minchan Kim9d502c12011-03-22 16:30:39 -07002410 if (err) {
Rafael Aquini5733c7d2012-12-11 16:02:47 -08002411 putback_movable_pages(&cc->migratepages);
Vlastimil Babka7ed695e2014-01-21 15:51:09 -08002412 /*
2413 * migrate_pages() may return -ENOMEM when scanners meet
2414 * and we want compact_finished() to detect it
2415 */
Vlastimil Babkaf2849aa2015-09-08 15:02:36 -07002416 if (err == -ENOMEM && !compact_scanners_met(cc)) {
Vlastimil Babka2d1e1042015-11-05 18:48:02 -08002417 ret = COMPACT_CONTENDED;
David Rientjes4bf2bba2012-07-11 14:02:13 -07002418 goto out;
2419 }
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07002420 /*
2421 * We failed to migrate at least one page in the current
2422 * order-aligned block, so skip the rest of it.
2423 */
2424 if (cc->direct_compaction &&
2425 (cc->mode == MIGRATE_ASYNC)) {
2426 cc->migrate_pfn = block_end_pfn(
2427 cc->migrate_pfn - 1, cc->order);
2428 /* Draining pcplists is useless in this case */
Mel Gorman566e54e2019-03-05 15:44:32 -08002429 last_migrated_pfn = 0;
Vlastimil Babkafdd048e2016-05-19 17:11:55 -07002430 }
Mel Gorman748446b2010-05-24 14:32:27 -07002431 }
Vlastimil Babkafdaf7f52014-12-10 15:43:34 -08002432
Vlastimil Babkafdaf7f52014-12-10 15:43:34 -08002433check_drain:
2434 /*
2435 * Has the migration scanner moved away from the previous
2436 * cc->order aligned block where we migrated from? If yes,
2437 * flush the pages that were freed, so that they can merge and
2438 * compact_finished() can detect immediately if allocation
2439 * would succeed.
2440 */
Mel Gorman566e54e2019-03-05 15:44:32 -08002441 if (cc->order > 0 && last_migrated_pfn) {
Vlastimil Babkafdaf7f52014-12-10 15:43:34 -08002442 unsigned long current_block_start =
Vlastimil Babka06b66402016-05-19 17:11:48 -07002443 block_start_pfn(cc->migrate_pfn, cc->order);
Vlastimil Babkafdaf7f52014-12-10 15:43:34 -08002444
Mel Gorman566e54e2019-03-05 15:44:32 -08002445 if (last_migrated_pfn < current_block_start) {
Ingo Molnarb01b2142020-05-27 22:11:15 +02002446 lru_add_drain_cpu_zone(cc->zone);
Vlastimil Babkafdaf7f52014-12-10 15:43:34 -08002447 /* No more flushing until we migrate again */
Mel Gorman566e54e2019-03-05 15:44:32 -08002448 last_migrated_pfn = 0;
Vlastimil Babkafdaf7f52014-12-10 15:43:34 -08002449 }
2450 }
2451
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002452 /* Stop if a page has been captured */
2453 if (capc && capc->page) {
2454 ret = COMPACT_SUCCESS;
2455 break;
2456 }
Mel Gorman748446b2010-05-24 14:32:27 -07002457 }
2458
Mel Gormanf9e35b32011-06-15 15:08:52 -07002459out:
Vlastimil Babka6bace092014-12-10 15:43:31 -08002460 /*
2461 * Release free pages and update where the free scanner should restart,
2462 * so we don't leave any returned pages behind in the next attempt.
2463 */
2464 if (cc->nr_freepages > 0) {
2465 unsigned long free_pfn = release_freepages(&cc->freepages);
2466
2467 cc->nr_freepages = 0;
2468 VM_BUG_ON(free_pfn == 0);
2469 /* The cached pfn is always the first in a pageblock */
Vlastimil Babka06b66402016-05-19 17:11:48 -07002470 free_pfn = pageblock_start_pfn(free_pfn);
Vlastimil Babka6bace092014-12-10 15:43:31 -08002471 /*
2472 * Only go back, not forward. The cached pfn might have been
2473 * already reset to zone end in compact_finished()
2474 */
Mel Gorman40cacbc2019-03-05 15:44:36 -08002475 if (free_pfn > cc->zone->compact_cached_free_pfn)
2476 cc->zone->compact_cached_free_pfn = free_pfn;
Vlastimil Babka6bace092014-12-10 15:43:31 -08002477 }
Mel Gorman748446b2010-05-24 14:32:27 -07002478
David Rientjes7f354a52017-02-22 15:44:50 -08002479 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
2480 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
2481
Joonsoo Kim16c4a092015-02-11 15:27:01 -08002482 trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
2483 cc->free_pfn, end_pfn, sync, ret);
Mel Gorman0eb927c2014-01-21 15:51:05 -08002484
Mel Gorman748446b2010-05-24 14:32:27 -07002485 return ret;
2486}
Mel Gorman76ab0f52010-05-24 14:32:28 -07002487
Michal Hockoea7ab982016-05-20 16:56:38 -07002488static enum compact_result compact_zone_order(struct zone *zone, int order,
Vlastimil Babkac3486f52016-07-28 15:49:30 -07002489 gfp_t gfp_mask, enum compact_priority prio,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002490 unsigned int alloc_flags, int highest_zoneidx,
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002491 struct page **capture)
Mel Gorman56de7262010-05-24 14:32:30 -07002492{
Michal Hockoea7ab982016-05-20 16:56:38 -07002493 enum compact_result ret;
Mel Gorman56de7262010-05-24 14:32:30 -07002494 struct compact_control cc = {
Mel Gorman56de7262010-05-24 14:32:30 -07002495 .order = order,
Mel Gormandbe2d4e2019-03-05 15:45:31 -08002496 .search_order = order,
David Rientjes6d7ce552014-10-09 15:27:27 -07002497 .gfp_mask = gfp_mask,
Mel Gorman56de7262010-05-24 14:32:30 -07002498 .zone = zone,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07002499 .mode = (prio == COMPACT_PRIO_ASYNC) ?
2500 MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT,
Vlastimil Babkaebff3982014-12-10 15:43:22 -08002501 .alloc_flags = alloc_flags,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002502 .highest_zoneidx = highest_zoneidx,
Vlastimil Babkaaccf6242016-03-17 14:18:15 -07002503 .direct_compaction = true,
Vlastimil Babkaa8e025e2016-10-07 16:57:47 -07002504 .whole_zone = (prio == MIN_COMPACT_PRIORITY),
Vlastimil Babka9f7e3382016-10-07 17:00:37 -07002505 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
2506 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
Mel Gorman56de7262010-05-24 14:32:30 -07002507 };
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002508 struct capture_control capc = {
2509 .cc = &cc,
2510 .page = NULL,
2511 };
2512
Vlastimil Babkab9e20f02020-06-25 20:29:24 -07002513 /*
2514 * Make sure the structs are really initialized before we expose the
2515 * capture control, in case we are interrupted and the interrupt handler
2516 * frees a page.
2517 */
2518 barrier();
2519 WRITE_ONCE(current->capture_control, &capc);
Mel Gorman56de7262010-05-24 14:32:30 -07002520
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002521 ret = compact_zone(&cc, &capc);
Shaohua Lie64c5232012-10-08 16:32:27 -07002522
2523 VM_BUG_ON(!list_empty(&cc.freepages));
2524 VM_BUG_ON(!list_empty(&cc.migratepages));
2525
Vlastimil Babkab9e20f02020-06-25 20:29:24 -07002526 /*
2527 * Make sure we hide capture control first before we read the captured
2528 * page pointer, otherwise an interrupt could free and capture a page
2529 * and we would leak it.
2530 */
2531 WRITE_ONCE(current->capture_control, NULL);
2532 *capture = READ_ONCE(capc.page);
Charan Teja Reddy06dac2f2021-05-04 18:36:51 -07002533 /*
2534 * Technically, it is also possible that compaction is skipped but
2535 * the page is still captured out of luck(IRQ came and freed the page).
2536 * Returning COMPACT_SUCCESS in such cases helps in properly accounting
2537 * the COMPACT[STALL|FAIL] when compaction is skipped.
2538 */
2539 if (*capture)
2540 ret = COMPACT_SUCCESS;
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002541
Shaohua Lie64c5232012-10-08 16:32:27 -07002542 return ret;
Mel Gorman56de7262010-05-24 14:32:30 -07002543}
2544
Mel Gorman5e771902010-05-24 14:32:31 -07002545int sysctl_extfrag_threshold = 500;
2546
Mel Gorman56de7262010-05-24 14:32:30 -07002547/**
2548 * try_to_compact_pages - Direct compact to satisfy a high-order allocation
Mel Gorman56de7262010-05-24 14:32:30 -07002549 * @gfp_mask: The GFP mask of the current allocation
Vlastimil Babka1a6d53a2015-02-11 15:25:44 -08002550 * @order: The order of the current allocation
2551 * @alloc_flags: The allocation flags of the current allocation
2552 * @ac: The context of current allocation
Yang Shi112d2d22018-01-31 16:20:23 -08002553 * @prio: Determines how hard direct compaction should try to succeed
Vlastimil Babka64675522020-04-01 21:10:35 -07002554 * @capture: Pointer to free page created by compaction will be stored here
Mel Gorman56de7262010-05-24 14:32:30 -07002555 *
2556 * This is the main entry point for direct page compaction.
2557 */
Michal Hockoea7ab982016-05-20 16:56:38 -07002558enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07002559 unsigned int alloc_flags, const struct alloc_context *ac,
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002560 enum compact_priority prio, struct page **capture)
Mel Gorman56de7262010-05-24 14:32:30 -07002561{
Mel Gorman56de7262010-05-24 14:32:30 -07002562 int may_perform_io = gfp_mask & __GFP_IO;
Mel Gorman56de7262010-05-24 14:32:30 -07002563 struct zoneref *z;
2564 struct zone *zone;
Michal Hocko1d4746d2016-05-20 16:56:44 -07002565 enum compact_result rc = COMPACT_SKIPPED;
Mel Gorman56de7262010-05-24 14:32:30 -07002566
Michal Hocko73e64c512016-12-14 15:04:07 -08002567 /*
2568 * Check if the GFP flags allow compaction - GFP_NOIO is really
2569 * tricky context because the migration might require IO
2570 */
2571 if (!may_perform_io)
Vlastimil Babka53853e22014-10-09 15:27:02 -07002572 return COMPACT_SKIPPED;
Mel Gorman56de7262010-05-24 14:32:30 -07002573
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07002574 trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
Joonsoo Kim837d0262015-02-11 15:27:06 -08002575
Mel Gorman56de7262010-05-24 14:32:30 -07002576 /* Compact each zone in the list */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002577 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
2578 ac->highest_zoneidx, ac->nodemask) {
Michal Hockoea7ab982016-05-20 16:56:38 -07002579 enum compact_result status;
Mel Gorman56de7262010-05-24 14:32:30 -07002580
Vlastimil Babkaa8e025e2016-10-07 16:57:47 -07002581 if (prio > MIN_COMPACT_PRIORITY
2582 && compaction_deferred(zone, order)) {
Michal Hocko1d4746d2016-05-20 16:56:44 -07002583 rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
Vlastimil Babka53853e22014-10-09 15:27:02 -07002584 continue;
Michal Hocko1d4746d2016-05-20 16:56:44 -07002585 }
Vlastimil Babka53853e22014-10-09 15:27:02 -07002586
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07002587 status = compact_zone_order(zone, order, gfp_mask, prio,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002588 alloc_flags, ac->highest_zoneidx, capture);
Mel Gorman56de7262010-05-24 14:32:30 -07002589 rc = max(status, rc);
2590
Vlastimil Babka7ceb0092016-10-07 16:57:44 -07002591 /* The allocation should succeed, stop compacting */
2592 if (status == COMPACT_SUCCESS) {
Vlastimil Babka53853e22014-10-09 15:27:02 -07002593 /*
2594 * We think the allocation will succeed in this zone,
2595 * but it is not certain, hence the false. The caller
2596 * will repeat this with true if allocation indeed
2597 * succeeds in this zone.
2598 */
2599 compaction_defer_reset(zone, order, false);
Vlastimil Babka1f9efde2014-10-09 15:27:14 -07002600
Vlastimil Babkac3486f52016-07-28 15:49:30 -07002601 break;
Vlastimil Babka1f9efde2014-10-09 15:27:14 -07002602 }
2603
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07002604 if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
Vlastimil Babkac3486f52016-07-28 15:49:30 -07002605 status == COMPACT_PARTIAL_SKIPPED))
Vlastimil Babka53853e22014-10-09 15:27:02 -07002606 /*
2607 * We think that allocation won't succeed in this zone
2608 * so we defer compaction there. If it ends up
2609 * succeeding after all, it will be reset.
2610 */
2611 defer_compaction(zone, order);
Vlastimil Babka1f9efde2014-10-09 15:27:14 -07002612
2613 /*
2614 * We might have stopped compacting due to need_resched() in
2615 * async compaction, or due to a fatal signal detected. In that
Vlastimil Babkac3486f52016-07-28 15:49:30 -07002616 * case do not try further zones
Vlastimil Babka1f9efde2014-10-09 15:27:14 -07002617 */
Vlastimil Babkac3486f52016-07-28 15:49:30 -07002618 if ((prio == COMPACT_PRIO_ASYNC && need_resched())
2619 || fatal_signal_pending(current))
2620 break;
Mel Gorman56de7262010-05-24 14:32:30 -07002621 }
2622
2623 return rc;
2624}
2625
Nitin Guptafacdaa92020-08-11 18:31:00 -07002626/*
2627 * Compact all zones within a node till each zone's fragmentation score
2628 * reaches within proactive compaction thresholds (as determined by the
2629 * proactiveness tunable).
2630 *
2631 * It is possible that the function returns before reaching score targets
2632 * due to various back-off conditions, such as, contention on per-node or
2633 * per-zone locks.
2634 */
2635static void proactive_compact_node(pg_data_t *pgdat)
2636{
2637 int zoneid;
2638 struct zone *zone;
2639 struct compact_control cc = {
2640 .order = -1,
2641 .mode = MIGRATE_SYNC_LIGHT,
2642 .ignore_skip_hint = true,
2643 .whole_zone = true,
2644 .gfp_mask = GFP_KERNEL,
2645 .proactive_compaction = true,
2646 };
2647
2648 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2649 zone = &pgdat->node_zones[zoneid];
2650 if (!populated_zone(zone))
2651 continue;
2652
2653 cc.zone = zone;
2654
2655 compact_zone(&cc, NULL);
2656
2657 VM_BUG_ON(!list_empty(&cc.freepages));
2658 VM_BUG_ON(!list_empty(&cc.migratepages));
2659 }
2660}
Mel Gorman56de7262010-05-24 14:32:30 -07002661
Mel Gorman76ab0f52010-05-24 14:32:28 -07002662/* Compact all zones within a node */
Andrew Morton7103f162013-02-22 16:32:33 -08002663static void compact_node(int nid)
Rik van Riel7be62de2012-03-21 16:33:52 -07002664{
Vlastimil Babka791cae92016-10-07 16:57:38 -07002665 pg_data_t *pgdat = NODE_DATA(nid);
2666 int zoneid;
2667 struct zone *zone;
Rik van Riel7be62de2012-03-21 16:33:52 -07002668 struct compact_control cc = {
2669 .order = -1,
David Rientjese0b9dae2014-06-04 16:08:28 -07002670 .mode = MIGRATE_SYNC,
David Rientjes91ca9182014-04-03 14:47:23 -07002671 .ignore_skip_hint = true,
Vlastimil Babka06ed2992016-10-07 16:57:35 -07002672 .whole_zone = true,
Michal Hocko73e64c512016-12-14 15:04:07 -08002673 .gfp_mask = GFP_KERNEL,
Rik van Riel7be62de2012-03-21 16:33:52 -07002674 };
2675
Vlastimil Babka791cae92016-10-07 16:57:38 -07002676
2677 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2678
2679 zone = &pgdat->node_zones[zoneid];
2680 if (!populated_zone(zone))
2681 continue;
2682
Vlastimil Babka791cae92016-10-07 16:57:38 -07002683 cc.zone = zone;
Vlastimil Babka791cae92016-10-07 16:57:38 -07002684
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002685 compact_zone(&cc, NULL);
Vlastimil Babka791cae92016-10-07 16:57:38 -07002686
2687 VM_BUG_ON(!list_empty(&cc.freepages));
2688 VM_BUG_ON(!list_empty(&cc.migratepages));
2689 }
Rik van Riel7be62de2012-03-21 16:33:52 -07002690}
2691
Mel Gorman76ab0f52010-05-24 14:32:28 -07002692/* Compact all nodes in the system */
Jason Liu7964c062013-01-11 14:31:47 -08002693static void compact_nodes(void)
Mel Gorman76ab0f52010-05-24 14:32:28 -07002694{
2695 int nid;
2696
Hugh Dickins8575ec22012-03-21 16:33:53 -07002697 /* Flush pending updates to the LRU lists */
2698 lru_add_drain_all();
2699
Mel Gorman76ab0f52010-05-24 14:32:28 -07002700 for_each_online_node(nid)
2701 compact_node(nid);
Mel Gorman76ab0f52010-05-24 14:32:28 -07002702}
2703
Yaowei Baifec4eb22016-01-14 15:20:09 -08002704/*
Nitin Guptafacdaa92020-08-11 18:31:00 -07002705 * Tunable for proactive compaction. It determines how
2706 * aggressively the kernel should compact memory in the
2707 * background. It takes values in the range [0, 100].
2708 */
Nitin Guptad34c0a72020-08-11 18:31:07 -07002709unsigned int __read_mostly sysctl_compaction_proactiveness = 20;
Nitin Guptafacdaa92020-08-11 18:31:00 -07002710
2711/*
Yaowei Baifec4eb22016-01-14 15:20:09 -08002712 * This is the entry point for compacting all nodes via
2713 * /proc/sys/vm/compact_memory
2714 */
Mel Gorman76ab0f52010-05-24 14:32:28 -07002715int sysctl_compaction_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02002716 void *buffer, size_t *length, loff_t *ppos)
Mel Gorman76ab0f52010-05-24 14:32:28 -07002717{
2718 if (write)
Jason Liu7964c062013-01-11 14:31:47 -08002719 compact_nodes();
Mel Gorman76ab0f52010-05-24 14:32:28 -07002720
2721 return 0;
2722}
Mel Gormaned4a6d72010-05-24 14:32:29 -07002723
2724#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
Rashika Kheria74e77fb2014-04-03 14:48:01 -07002725static ssize_t sysfs_compact_node(struct device *dev,
Kay Sievers10fbcf42011-12-21 14:48:43 -08002726 struct device_attribute *attr,
Mel Gormaned4a6d72010-05-24 14:32:29 -07002727 const char *buf, size_t count)
2728{
Hugh Dickins8575ec22012-03-21 16:33:53 -07002729 int nid = dev->id;
2730
2731 if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
2732 /* Flush pending updates to the LRU lists */
2733 lru_add_drain_all();
2734
2735 compact_node(nid);
2736 }
Mel Gormaned4a6d72010-05-24 14:32:29 -07002737
2738 return count;
2739}
Joe Perches0825a6f2018-06-14 15:27:58 -07002740static DEVICE_ATTR(compact, 0200, NULL, sysfs_compact_node);
Mel Gormaned4a6d72010-05-24 14:32:29 -07002741
2742int compaction_register_node(struct node *node)
2743{
Kay Sievers10fbcf42011-12-21 14:48:43 -08002744 return device_create_file(&node->dev, &dev_attr_compact);
Mel Gormaned4a6d72010-05-24 14:32:29 -07002745}
2746
2747void compaction_unregister_node(struct node *node)
2748{
Kay Sievers10fbcf42011-12-21 14:48:43 -08002749 return device_remove_file(&node->dev, &dev_attr_compact);
Mel Gormaned4a6d72010-05-24 14:32:29 -07002750}
2751#endif /* CONFIG_SYSFS && CONFIG_NUMA */
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01002752
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002753static inline bool kcompactd_work_requested(pg_data_t *pgdat)
2754{
Vlastimil Babka172400c2016-05-05 16:22:32 -07002755 return pgdat->kcompactd_max_order > 0 || kthread_should_stop();
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002756}
2757
2758static bool kcompactd_node_suitable(pg_data_t *pgdat)
2759{
2760 int zoneid;
2761 struct zone *zone;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002762 enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002763
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002764 for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) {
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002765 zone = &pgdat->node_zones[zoneid];
2766
2767 if (!populated_zone(zone))
2768 continue;
2769
2770 if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002771 highest_zoneidx) == COMPACT_CONTINUE)
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002772 return true;
2773 }
2774
2775 return false;
2776}
2777
2778static void kcompactd_do_work(pg_data_t *pgdat)
2779{
2780 /*
2781 * With no special task, compact all zones so that a page of requested
2782 * order is allocatable.
2783 */
2784 int zoneid;
2785 struct zone *zone;
2786 struct compact_control cc = {
2787 .order = pgdat->kcompactd_max_order,
Mel Gormandbe2d4e2019-03-05 15:45:31 -08002788 .search_order = pgdat->kcompactd_max_order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002789 .highest_zoneidx = pgdat->kcompactd_highest_zoneidx,
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002790 .mode = MIGRATE_SYNC_LIGHT,
David Rientjesa0647dc2017-11-17 15:26:27 -08002791 .ignore_skip_hint = false,
Michal Hocko73e64c512016-12-14 15:04:07 -08002792 .gfp_mask = GFP_KERNEL,
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002793 };
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002794 trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002795 cc.highest_zoneidx);
David Rientjes7f354a52017-02-22 15:44:50 -08002796 count_compact_event(KCOMPACTD_WAKE);
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002797
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002798 for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) {
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002799 int status;
2800
2801 zone = &pgdat->node_zones[zoneid];
2802 if (!populated_zone(zone))
2803 continue;
2804
2805 if (compaction_deferred(zone, cc.order))
2806 continue;
2807
2808 if (compaction_suitable(zone, cc.order, 0, zoneid) !=
2809 COMPACT_CONTINUE)
2810 continue;
2811
Vlastimil Babka172400c2016-05-05 16:22:32 -07002812 if (kthread_should_stop())
2813 return;
Yafang Shaoa94b5252019-09-23 15:36:54 -07002814
2815 cc.zone = zone;
Mel Gorman5e1f0f02019-03-05 15:45:41 -08002816 status = compact_zone(&cc, NULL);
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002817
Vlastimil Babka7ceb0092016-10-07 16:57:44 -07002818 if (status == COMPACT_SUCCESS) {
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002819 compaction_defer_reset(zone, cc.order, false);
Michal Hockoc8f7de02016-05-20 16:56:47 -07002820 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002821 /*
David Rientjesbc3106b2018-04-05 16:24:02 -07002822 * Buddy pages may become stranded on pcps that could
2823 * otherwise coalesce on the zone's free area for
2824 * order >= cc.order. This is ratelimited by the
2825 * upcoming deferral.
2826 */
2827 drain_all_pages(zone);
2828
2829 /*
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002830 * We use sync migration mode here, so we defer like
2831 * sync direct compaction does.
2832 */
2833 defer_compaction(zone, cc.order);
2834 }
2835
David Rientjes7f354a52017-02-22 15:44:50 -08002836 count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
2837 cc.total_migrate_scanned);
2838 count_compact_events(KCOMPACTD_FREE_SCANNED,
2839 cc.total_free_scanned);
2840
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002841 VM_BUG_ON(!list_empty(&cc.freepages));
2842 VM_BUG_ON(!list_empty(&cc.migratepages));
2843 }
2844
2845 /*
2846 * Regardless of success, we are done until woken up next. But remember
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002847 * the requested order/highest_zoneidx in case it was higher/tighter
2848 * than our current ones
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002849 */
2850 if (pgdat->kcompactd_max_order <= cc.order)
2851 pgdat->kcompactd_max_order = 0;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002852 if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx)
2853 pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002854}
2855
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002856void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002857{
2858 if (!order)
2859 return;
2860
2861 if (pgdat->kcompactd_max_order < order)
2862 pgdat->kcompactd_max_order = order;
2863
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002864 if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx)
2865 pgdat->kcompactd_highest_zoneidx = highest_zoneidx;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002866
Davidlohr Bueso68186002017-10-03 16:15:03 -07002867 /*
2868 * Pairs with implicit barrier in wait_event_freezable()
2869 * such that wakeups are not missed.
2870 */
2871 if (!wq_has_sleeper(&pgdat->kcompactd_wait))
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002872 return;
2873
2874 if (!kcompactd_node_suitable(pgdat))
2875 return;
2876
2877 trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002878 highest_zoneidx);
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002879 wake_up_interruptible(&pgdat->kcompactd_wait);
2880}
2881
2882/*
2883 * The background compaction daemon, started as a kernel thread
2884 * from the init process.
2885 */
2886static int kcompactd(void *p)
2887{
Zhiyuan Dai68d68ff2021-05-04 18:40:12 -07002888 pg_data_t *pgdat = (pg_data_t *)p;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002889 struct task_struct *tsk = current;
Nitin Guptafacdaa92020-08-11 18:31:00 -07002890 unsigned int proactive_defer = 0;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002891
2892 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2893
2894 if (!cpumask_empty(cpumask))
2895 set_cpus_allowed_ptr(tsk, cpumask);
2896
2897 set_freezable();
2898
2899 pgdat->kcompactd_max_order = 0;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002900 pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002901
2902 while (!kthread_should_stop()) {
Johannes Weinereb414682018-10-26 15:06:27 -07002903 unsigned long pflags;
2904
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002905 trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
Nitin Guptafacdaa92020-08-11 18:31:00 -07002906 if (wait_event_freezable_timeout(pgdat->kcompactd_wait,
2907 kcompactd_work_requested(pgdat),
2908 msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC))) {
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002909
Nitin Guptafacdaa92020-08-11 18:31:00 -07002910 psi_memstall_enter(&pflags);
2911 kcompactd_do_work(pgdat);
2912 psi_memstall_leave(&pflags);
2913 continue;
2914 }
2915
2916 /* kcompactd wait timeout */
2917 if (should_proactive_compact_node(pgdat)) {
2918 unsigned int prev_score, score;
2919
2920 if (proactive_defer) {
2921 proactive_defer--;
2922 continue;
2923 }
2924 prev_score = fragmentation_score_node(pgdat);
2925 proactive_compact_node(pgdat);
2926 score = fragmentation_score_node(pgdat);
2927 /*
2928 * Defer proactive compaction if the fragmentation
2929 * score did not go down i.e. no progress made.
2930 */
2931 proactive_defer = score < prev_score ?
2932 0 : 1 << COMPACT_MAX_DEFER_SHIFT;
2933 }
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002934 }
2935
2936 return 0;
2937}
2938
2939/*
2940 * This kcompactd start function will be called by init and node-hot-add.
2941 * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
2942 */
2943int kcompactd_run(int nid)
2944{
2945 pg_data_t *pgdat = NODE_DATA(nid);
2946 int ret = 0;
2947
2948 if (pgdat->kcompactd)
2949 return 0;
2950
2951 pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
2952 if (IS_ERR(pgdat->kcompactd)) {
2953 pr_err("Failed to start kcompactd on node %d\n", nid);
2954 ret = PTR_ERR(pgdat->kcompactd);
2955 pgdat->kcompactd = NULL;
2956 }
2957 return ret;
2958}
2959
2960/*
2961 * Called by memory hotplug when all memory in a node is offlined. Caller must
2962 * hold mem_hotplug_begin/end().
2963 */
2964void kcompactd_stop(int nid)
2965{
2966 struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
2967
2968 if (kcompactd) {
2969 kthread_stop(kcompactd);
2970 NODE_DATA(nid)->kcompactd = NULL;
2971 }
2972}
2973
2974/*
2975 * It's optimal to keep kcompactd on the same CPUs as their memory, but
2976 * not required for correctness. So if the last cpu in a node goes
2977 * away, we get changed to run anywhere: as the first one comes back,
2978 * restore their cpu bindings.
2979 */
Anna-Maria Gleixnere46b1db2016-11-27 00:13:42 +01002980static int kcompactd_cpu_online(unsigned int cpu)
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002981{
2982 int nid;
2983
Anna-Maria Gleixnere46b1db2016-11-27 00:13:42 +01002984 for_each_node_state(nid, N_MEMORY) {
2985 pg_data_t *pgdat = NODE_DATA(nid);
2986 const struct cpumask *mask;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002987
Anna-Maria Gleixnere46b1db2016-11-27 00:13:42 +01002988 mask = cpumask_of_node(pgdat->node_id);
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002989
Anna-Maria Gleixnere46b1db2016-11-27 00:13:42 +01002990 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
2991 /* One of our CPUs online: restore mask */
2992 set_cpus_allowed_ptr(pgdat->kcompactd, mask);
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002993 }
Anna-Maria Gleixnere46b1db2016-11-27 00:13:42 +01002994 return 0;
Vlastimil Babka698b1b32016-03-17 14:18:08 -07002995}
2996
2997static int __init kcompactd_init(void)
2998{
2999 int nid;
Anna-Maria Gleixnere46b1db2016-11-27 00:13:42 +01003000 int ret;
3001
3002 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
3003 "mm/compaction:online",
3004 kcompactd_cpu_online, NULL);
3005 if (ret < 0) {
3006 pr_err("kcompactd: failed to register hotplug callbacks.\n");
3007 return ret;
3008 }
Vlastimil Babka698b1b32016-03-17 14:18:08 -07003009
3010 for_each_node_state(nid, N_MEMORY)
3011 kcompactd_run(nid);
Vlastimil Babka698b1b32016-03-17 14:18:08 -07003012 return 0;
3013}
3014subsys_initcall(kcompactd_init)
3015
Michal Nazarewiczff9543f2011-12-29 13:09:50 +01003016#endif /* CONFIG_COMPACTION */