Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds |
| 4 | * |
| 5 | * Swap reorganised 29.12.95, Stephen Tweedie. |
| 6 | * kswapd added: 7.1.96 sct |
| 7 | * Removed kswapd_ctl limits, and swap out as many pages as needed |
| 8 | * to bring the system back to freepages.high: 2.4.97, Rik van Riel. |
| 9 | * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). |
| 10 | * Multiqueue VM started 5.8.00, Rik van Riel. |
| 11 | */ |
| 12 | |
Mitchel Humpherys | b1de0d1 | 2014-06-06 14:38:30 -0700 | [diff] [blame] | 13 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 14 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <linux/mm.h> |
Ingo Molnar | 5b3cc15 | 2017-02-02 20:43:54 +0100 | [diff] [blame] | 16 | #include <linux/sched/mm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <linux/module.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 18 | #include <linux/gfp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 19 | #include <linux/kernel_stat.h> |
| 20 | #include <linux/swap.h> |
| 21 | #include <linux/pagemap.h> |
| 22 | #include <linux/init.h> |
| 23 | #include <linux/highmem.h> |
Anton Vorontsov | 70ddf63 | 2013-04-29 15:08:31 -0700 | [diff] [blame] | 24 | #include <linux/vmpressure.h> |
Andrew Morton | e129b5c | 2006-09-27 01:50:00 -0700 | [diff] [blame] | 25 | #include <linux/vmstat.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #include <linux/file.h> |
| 27 | #include <linux/writeback.h> |
| 28 | #include <linux/blkdev.h> |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 29 | #include <linux/buffer_head.h> /* for buffer_heads_over_limit */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | #include <linux/mm_inline.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | #include <linux/backing-dev.h> |
| 32 | #include <linux/rmap.h> |
| 33 | #include <linux/topology.h> |
| 34 | #include <linux/cpu.h> |
| 35 | #include <linux/cpuset.h> |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 36 | #include <linux/compaction.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | #include <linux/notifier.h> |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 38 | #include <linux/rwsem.h> |
Rafael J. Wysocki | 248a030 | 2006-03-22 00:09:04 -0800 | [diff] [blame] | 39 | #include <linux/delay.h> |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 40 | #include <linux/kthread.h> |
Nigel Cunningham | 7dfb710 | 2006-12-06 20:34:23 -0800 | [diff] [blame] | 41 | #include <linux/freezer.h> |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 42 | #include <linux/memcontrol.h> |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 43 | #include <linux/migrate.h> |
Keika Kobayashi | 873b477 | 2008-07-25 01:48:52 -0700 | [diff] [blame] | 44 | #include <linux/delayacct.h> |
Lee Schermerhorn | af936a1 | 2008-10-18 20:26:53 -0700 | [diff] [blame] | 45 | #include <linux/sysctl.h> |
Aneesh Kumar K.V | 9195244 | 2022-08-18 18:40:34 +0530 | [diff] [blame] | 46 | #include <linux/memory-tiers.h> |
KOSAKI Motohiro | 929bea7 | 2011-04-14 15:22:12 -0700 | [diff] [blame] | 47 | #include <linux/oom.h> |
Kuo-Hsin Yang | 64e3d12 | 2018-11-06 13:23:24 +0000 | [diff] [blame] | 48 | #include <linux/pagevec.h> |
Linus Torvalds | 268bb0c | 2011-05-20 12:50:29 -0700 | [diff] [blame] | 49 | #include <linux/prefetch.h> |
Mitchel Humpherys | b1de0d1 | 2014-06-06 14:38:30 -0700 | [diff] [blame] | 50 | #include <linux/printk.h> |
Ross Zwisler | f9fe48b | 2016-01-22 15:10:40 -0800 | [diff] [blame] | 51 | #include <linux/dax.h> |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 52 | #include <linux/psi.h> |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 53 | #include <linux/pagewalk.h> |
| 54 | #include <linux/shmem_fs.h> |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 55 | #include <linux/ctype.h> |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 56 | #include <linux/debugfs.h> |
Johannes Weiner | 57e9cc5 | 2022-10-26 14:01:33 -0400 | [diff] [blame] | 57 | #include <linux/khugepaged.h> |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 58 | #include <linux/rculist_nulls.h> |
| 59 | #include <linux/random.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | |
| 61 | #include <asm/tlbflush.h> |
| 62 | #include <asm/div64.h> |
| 63 | |
| 64 | #include <linux/swapops.h> |
Rafael Aquini | 117aad1 | 2013-09-30 13:45:16 -0700 | [diff] [blame] | 65 | #include <linux/balloon_compaction.h> |
Huang Ying | c574bbe | 2022-03-22 14:46:23 -0700 | [diff] [blame] | 66 | #include <linux/sched/sysctl.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | |
Nick Piggin | 0f8053a | 2006-03-22 00:08:33 -0800 | [diff] [blame] | 68 | #include "internal.h" |
NeilBrown | 014bb1d | 2022-05-09 18:20:47 -0700 | [diff] [blame] | 69 | #include "swap.h" |
Nick Piggin | 0f8053a | 2006-03-22 00:08:33 -0800 | [diff] [blame] | 70 | |
Mel Gorman | 33906bc | 2010-08-09 17:19:16 -0700 | [diff] [blame] | 71 | #define CREATE_TRACE_POINTS |
| 72 | #include <trace/events/vmscan.h> |
| 73 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | struct scan_control { |
KOSAKI Motohiro | 22fba33 | 2009-12-14 17:59:10 -0800 | [diff] [blame] | 75 | /* How many pages shrink_list() should reclaim */ |
| 76 | unsigned long nr_to_reclaim; |
| 77 | |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 78 | /* |
| 79 | * Nodemask of nodes allowed by the caller. If NULL, all nodes |
| 80 | * are scanned. |
| 81 | */ |
| 82 | nodemask_t *nodemask; |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 83 | |
KOSAKI Motohiro | 5f53e76 | 2010-05-24 14:32:37 -0700 | [diff] [blame] | 84 | /* |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 85 | * The memory cgroup that hit its limit and as a result is the |
| 86 | * primary target of this reclaim invocation. |
| 87 | */ |
| 88 | struct mem_cgroup *target_mem_cgroup; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 89 | |
Johannes Weiner | 7cf111b | 2020-06-03 16:03:06 -0700 | [diff] [blame] | 90 | /* |
| 91 | * Scan pressure balancing between anon and file LRUs |
| 92 | */ |
| 93 | unsigned long anon_cost; |
| 94 | unsigned long file_cost; |
| 95 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 96 | /* Can active folios be deactivated as part of reclaim? */ |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 97 | #define DEACTIVATE_ANON 1 |
| 98 | #define DEACTIVATE_FILE 2 |
| 99 | unsigned int may_deactivate:2; |
| 100 | unsigned int force_deactivate:1; |
| 101 | unsigned int skipped_deactivate:1; |
| 102 | |
Johannes Weiner | 1276ad6 | 2017-02-24 14:56:11 -0800 | [diff] [blame] | 103 | /* Writepage batching in laptop mode; RECLAIM_WRITE */ |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 104 | unsigned int may_writepage:1; |
| 105 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 106 | /* Can mapped folios be reclaimed? */ |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 107 | unsigned int may_unmap:1; |
| 108 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 109 | /* Can folios be swapped as part of reclaim? */ |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 110 | unsigned int may_swap:1; |
| 111 | |
Yosry Ahmed | 73b73ba | 2022-07-14 06:49:18 +0000 | [diff] [blame] | 112 | /* Proactive reclaim invoked by userspace through memory.reclaim */ |
| 113 | unsigned int proactive:1; |
| 114 | |
Yisheng Xie | d6622f6 | 2017-05-03 14:53:57 -0700 | [diff] [blame] | 115 | /* |
Johannes Weiner | f56ce41 | 2021-08-19 19:04:21 -0700 | [diff] [blame] | 116 | * Cgroup memory below memory.low is protected as long as we |
| 117 | * don't threaten to OOM. If any cgroup is reclaimed at |
| 118 | * reduced force or passed over entirely due to its memory.low |
| 119 | * setting (memcg_low_skipped), and nothing is reclaimed as a |
| 120 | * result, then go back for one more cycle that reclaims the protected |
| 121 | * memory (memcg_low_reclaim) to avert OOM. |
Yisheng Xie | d6622f6 | 2017-05-03 14:53:57 -0700 | [diff] [blame] | 122 | */ |
| 123 | unsigned int memcg_low_reclaim:1; |
| 124 | unsigned int memcg_low_skipped:1; |
Johannes Weiner | 241994ed | 2015-02-11 15:26:06 -0800 | [diff] [blame] | 125 | |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 126 | unsigned int hibernation_mode:1; |
| 127 | |
| 128 | /* One of the zones is ready for compaction */ |
| 129 | unsigned int compaction_ready:1; |
| 130 | |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 131 | /* There is easily reclaimable cold cache in the current node */ |
| 132 | unsigned int cache_trim_mode:1; |
| 133 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 134 | /* The file folios on the current node are dangerously low */ |
Johannes Weiner | 53138ce | 2019-11-30 17:55:56 -0800 | [diff] [blame] | 135 | unsigned int file_is_tiny:1; |
| 136 | |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 137 | /* Always discard instead of demoting to lower tier memory */ |
| 138 | unsigned int no_demotion:1; |
| 139 | |
Greg Thelen | bb451fd | 2018-08-17 15:45:19 -0700 | [diff] [blame] | 140 | /* Allocation order */ |
| 141 | s8 order; |
| 142 | |
| 143 | /* Scan (total_size >> priority) pages at once */ |
| 144 | s8 priority; |
| 145 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 146 | /* The highest zone to isolate folios for reclaim from */ |
Greg Thelen | bb451fd | 2018-08-17 15:45:19 -0700 | [diff] [blame] | 147 | s8 reclaim_idx; |
| 148 | |
| 149 | /* This context's GFP mask */ |
| 150 | gfp_t gfp_mask; |
| 151 | |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 152 | /* Incremented by the number of inactive pages that were scanned */ |
| 153 | unsigned long nr_scanned; |
| 154 | |
| 155 | /* Number of pages freed so far during a call to shrink_zones() */ |
| 156 | unsigned long nr_reclaimed; |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 157 | |
| 158 | struct { |
| 159 | unsigned int dirty; |
| 160 | unsigned int unqueued_dirty; |
| 161 | unsigned int congested; |
| 162 | unsigned int writeback; |
| 163 | unsigned int immediate; |
| 164 | unsigned int file_taken; |
| 165 | unsigned int taken; |
| 166 | } nr; |
Yafang Shao | e5ca807 | 2019-07-16 16:26:09 -0700 | [diff] [blame] | 167 | |
| 168 | /* for recording the reclaimed slab by now */ |
| 169 | struct reclaim_state reclaim_state; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | }; |
| 171 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 172 | #ifdef ARCH_HAS_PREFETCHW |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 173 | #define prefetchw_prev_lru_folio(_folio, _base, _field) \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | do { \ |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 175 | if ((_folio)->lru.prev != _base) { \ |
| 176 | struct folio *prev; \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | \ |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 178 | prev = lru_to_folio(&(_folio->lru)); \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | prefetchw(&prev->_field); \ |
| 180 | } \ |
| 181 | } while (0) |
| 182 | #else |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 183 | #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | #endif |
| 185 | |
| 186 | /* |
Johannes Weiner | c843966 | 2020-06-03 16:02:37 -0700 | [diff] [blame] | 187 | * From 0 .. 200. Higher means more swappy. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | */ |
| 189 | int vm_swappiness = 60; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 190 | |
Roman Gushchin | 5035ebc | 2022-05-31 20:22:23 -0700 | [diff] [blame] | 191 | LIST_HEAD(shrinker_list); |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 192 | DECLARE_RWSEM(shrinker_rwsem); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 194 | #ifdef CONFIG_MEMCG |
Yang Shi | a2fb126 | 2021-05-04 18:36:17 -0700 | [diff] [blame] | 195 | static int shrinker_nr_max; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 196 | |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 197 | /* The shrinker_info is expanded in a batch of BITS_PER_LONG */ |
Yang Shi | a2fb126 | 2021-05-04 18:36:17 -0700 | [diff] [blame] | 198 | static inline int shrinker_map_size(int nr_items) |
| 199 | { |
| 200 | return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long)); |
| 201 | } |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 202 | |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 203 | static inline int shrinker_defer_size(int nr_items) |
| 204 | { |
| 205 | return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t)); |
| 206 | } |
| 207 | |
Yang Shi | 468ab84 | 2021-05-04 18:36:26 -0700 | [diff] [blame] | 208 | static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg, |
| 209 | int nid) |
| 210 | { |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 211 | return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info, |
| 212 | lockdep_is_held(&shrinker_rwsem)); |
Yang Shi | 468ab84 | 2021-05-04 18:36:26 -0700 | [diff] [blame] | 213 | } |
| 214 | |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 215 | static int expand_one_shrinker_info(struct mem_cgroup *memcg, |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 216 | int map_size, int defer_size, |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 217 | int old_map_size, int old_defer_size, |
| 218 | int new_nr_max) |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 219 | { |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 220 | struct shrinker_info *new, *old; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 221 | struct mem_cgroup_per_node *pn; |
| 222 | int nid; |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 223 | int size = map_size + defer_size; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 224 | |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 225 | for_each_node(nid) { |
| 226 | pn = memcg->nodeinfo[nid]; |
Yang Shi | 468ab84 | 2021-05-04 18:36:26 -0700 | [diff] [blame] | 227 | old = shrinker_info_protected(memcg, nid); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 228 | /* Not yet online memcg */ |
| 229 | if (!old) |
| 230 | return 0; |
| 231 | |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 232 | /* Already expanded this shrinker_info */ |
| 233 | if (new_nr_max <= old->map_nr_max) |
| 234 | continue; |
| 235 | |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 236 | new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); |
| 237 | if (!new) |
| 238 | return -ENOMEM; |
| 239 | |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 240 | new->nr_deferred = (atomic_long_t *)(new + 1); |
| 241 | new->map = (void *)new->nr_deferred + defer_size; |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 242 | new->map_nr_max = new_nr_max; |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 243 | |
| 244 | /* map: set all old bits, clear all new bits */ |
| 245 | memset(new->map, (int)0xff, old_map_size); |
| 246 | memset((void *)new->map + old_map_size, 0, map_size - old_map_size); |
| 247 | /* nr_deferred: copy old values, clear all new values */ |
| 248 | memcpy(new->nr_deferred, old->nr_deferred, old_defer_size); |
| 249 | memset((void *)new->nr_deferred + old_defer_size, 0, |
| 250 | defer_size - old_defer_size); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 251 | |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 252 | rcu_assign_pointer(pn->shrinker_info, new); |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 253 | kvfree_rcu(old, rcu); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 254 | } |
| 255 | |
| 256 | return 0; |
| 257 | } |
| 258 | |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 259 | void free_shrinker_info(struct mem_cgroup *memcg) |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 260 | { |
| 261 | struct mem_cgroup_per_node *pn; |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 262 | struct shrinker_info *info; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 263 | int nid; |
| 264 | |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 265 | for_each_node(nid) { |
| 266 | pn = memcg->nodeinfo[nid]; |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 267 | info = rcu_dereference_protected(pn->shrinker_info, true); |
| 268 | kvfree(info); |
| 269 | rcu_assign_pointer(pn->shrinker_info, NULL); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 270 | } |
| 271 | } |
| 272 | |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 273 | int alloc_shrinker_info(struct mem_cgroup *memcg) |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 274 | { |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 275 | struct shrinker_info *info; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 276 | int nid, size, ret = 0; |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 277 | int map_size, defer_size = 0; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 278 | |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 279 | down_write(&shrinker_rwsem); |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 280 | map_size = shrinker_map_size(shrinker_nr_max); |
| 281 | defer_size = shrinker_defer_size(shrinker_nr_max); |
| 282 | size = map_size + defer_size; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 283 | for_each_node(nid) { |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 284 | info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid); |
| 285 | if (!info) { |
| 286 | free_shrinker_info(memcg); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 287 | ret = -ENOMEM; |
| 288 | break; |
| 289 | } |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 290 | info->nr_deferred = (atomic_long_t *)(info + 1); |
| 291 | info->map = (void *)info->nr_deferred + defer_size; |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 292 | info->map_nr_max = shrinker_nr_max; |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 293 | rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 294 | } |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 295 | up_write(&shrinker_rwsem); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 296 | |
| 297 | return ret; |
| 298 | } |
| 299 | |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 300 | static int expand_shrinker_info(int new_id) |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 301 | { |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 302 | int ret = 0; |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 303 | int new_nr_max = round_up(new_id + 1, BITS_PER_LONG); |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 304 | int map_size, defer_size = 0; |
| 305 | int old_map_size, old_defer_size = 0; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 306 | struct mem_cgroup *memcg; |
| 307 | |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 308 | if (!root_mem_cgroup) |
Yang Shi | d27cf2a | 2021-05-04 18:36:14 -0700 | [diff] [blame] | 309 | goto out; |
| 310 | |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 311 | lockdep_assert_held(&shrinker_rwsem); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 312 | |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 313 | map_size = shrinker_map_size(new_nr_max); |
| 314 | defer_size = shrinker_defer_size(new_nr_max); |
| 315 | old_map_size = shrinker_map_size(shrinker_nr_max); |
| 316 | old_defer_size = shrinker_defer_size(shrinker_nr_max); |
| 317 | |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 318 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 319 | do { |
Yang Shi | 3c6f17e | 2021-05-04 18:36:33 -0700 | [diff] [blame] | 320 | ret = expand_one_shrinker_info(memcg, map_size, defer_size, |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 321 | old_map_size, old_defer_size, |
| 322 | new_nr_max); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 323 | if (ret) { |
| 324 | mem_cgroup_iter_break(NULL, memcg); |
Yang Shi | d27cf2a | 2021-05-04 18:36:14 -0700 | [diff] [blame] | 325 | goto out; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 326 | } |
| 327 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); |
Yang Shi | d27cf2a | 2021-05-04 18:36:14 -0700 | [diff] [blame] | 328 | out: |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 329 | if (!ret) |
Yang Shi | a2fb126 | 2021-05-04 18:36:17 -0700 | [diff] [blame] | 330 | shrinker_nr_max = new_nr_max; |
Yang Shi | d27cf2a | 2021-05-04 18:36:14 -0700 | [diff] [blame] | 331 | |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 332 | return ret; |
| 333 | } |
| 334 | |
| 335 | void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) |
| 336 | { |
| 337 | if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) { |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 338 | struct shrinker_info *info; |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 339 | |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 340 | rcu_read_lock(); |
| 341 | info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 342 | if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) { |
| 343 | /* Pairs with smp mb in shrink_slab() */ |
| 344 | smp_mb__before_atomic(); |
| 345 | set_bit(shrinker_id, info->map); |
| 346 | } |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 347 | rcu_read_unlock(); |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 348 | } |
| 349 | } |
| 350 | |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 351 | static DEFINE_IDR(shrinker_idr); |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 352 | |
| 353 | static int prealloc_memcg_shrinker(struct shrinker *shrinker) |
| 354 | { |
| 355 | int id, ret = -ENOMEM; |
| 356 | |
Yang Shi | 476b30a | 2021-05-04 18:36:39 -0700 | [diff] [blame] | 357 | if (mem_cgroup_disabled()) |
| 358 | return -ENOSYS; |
| 359 | |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 360 | down_write(&shrinker_rwsem); |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 361 | /* This may call shrinker, so it must use down_read_trylock() */ |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 362 | id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL); |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 363 | if (id < 0) |
| 364 | goto unlock; |
| 365 | |
Kirill Tkhai | 0a4465d | 2018-08-17 15:47:37 -0700 | [diff] [blame] | 366 | if (id >= shrinker_nr_max) { |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 367 | if (expand_shrinker_info(id)) { |
Kirill Tkhai | 0a4465d | 2018-08-17 15:47:37 -0700 | [diff] [blame] | 368 | idr_remove(&shrinker_idr, id); |
| 369 | goto unlock; |
| 370 | } |
Kirill Tkhai | 0a4465d | 2018-08-17 15:47:37 -0700 | [diff] [blame] | 371 | } |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 372 | shrinker->id = id; |
| 373 | ret = 0; |
| 374 | unlock: |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 375 | up_write(&shrinker_rwsem); |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 376 | return ret; |
| 377 | } |
| 378 | |
| 379 | static void unregister_memcg_shrinker(struct shrinker *shrinker) |
| 380 | { |
| 381 | int id = shrinker->id; |
| 382 | |
| 383 | BUG_ON(id < 0); |
| 384 | |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 385 | lockdep_assert_held(&shrinker_rwsem); |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 386 | |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 387 | idr_remove(&shrinker_idr, id); |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 388 | } |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 389 | |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 390 | static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, |
| 391 | struct mem_cgroup *memcg) |
| 392 | { |
| 393 | struct shrinker_info *info; |
| 394 | |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 395 | info = shrinker_info_protected(memcg, nid); |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 396 | return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0); |
| 397 | } |
| 398 | |
| 399 | static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, |
| 400 | struct mem_cgroup *memcg) |
| 401 | { |
| 402 | struct shrinker_info *info; |
| 403 | |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 404 | info = shrinker_info_protected(memcg, nid); |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 405 | return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]); |
| 406 | } |
| 407 | |
Yang Shi | a178015 | 2021-05-04 18:36:42 -0700 | [diff] [blame] | 408 | void reparent_shrinker_deferred(struct mem_cgroup *memcg) |
| 409 | { |
| 410 | int i, nid; |
| 411 | long nr; |
| 412 | struct mem_cgroup *parent; |
| 413 | struct shrinker_info *child_info, *parent_info; |
| 414 | |
| 415 | parent = parent_mem_cgroup(memcg); |
| 416 | if (!parent) |
| 417 | parent = root_mem_cgroup; |
| 418 | |
| 419 | /* Prevent from concurrent shrinker_info expand */ |
Qi Zheng | c534f7c | 2023-06-09 08:15:14 +0000 | [diff] [blame] | 420 | down_read(&shrinker_rwsem); |
Yang Shi | a178015 | 2021-05-04 18:36:42 -0700 | [diff] [blame] | 421 | for_each_node(nid) { |
| 422 | child_info = shrinker_info_protected(memcg, nid); |
| 423 | parent_info = shrinker_info_protected(parent, nid); |
Qi Zheng | 42c9db3 | 2023-03-13 19:28:12 +0800 | [diff] [blame] | 424 | for (i = 0; i < child_info->map_nr_max; i++) { |
Yang Shi | a178015 | 2021-05-04 18:36:42 -0700 | [diff] [blame] | 425 | nr = atomic_long_read(&child_info->nr_deferred[i]); |
| 426 | atomic_long_add(nr, &parent_info->nr_deferred[i]); |
| 427 | } |
| 428 | } |
Qi Zheng | c534f7c | 2023-06-09 08:15:14 +0000 | [diff] [blame] | 429 | up_read(&shrinker_rwsem); |
Yang Shi | a178015 | 2021-05-04 18:36:42 -0700 | [diff] [blame] | 430 | } |
| 431 | |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 432 | /* Returns true for reclaim through cgroup limits or cgroup interfaces. */ |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 433 | static bool cgroup_reclaim(struct scan_control *sc) |
Johannes Weiner | 89b5fae | 2012-01-12 17:17:50 -0800 | [diff] [blame] | 434 | { |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 435 | return sc->target_mem_cgroup; |
Johannes Weiner | 89b5fae | 2012-01-12 17:17:50 -0800 | [diff] [blame] | 436 | } |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 437 | |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 438 | /* |
| 439 | * Returns true for reclaim on the root cgroup. This is true for direct |
| 440 | * allocator reclaim and reclaim through cgroup interfaces on the root cgroup. |
| 441 | */ |
| 442 | static bool root_reclaim(struct scan_control *sc) |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 443 | { |
| 444 | return !sc->target_mem_cgroup || mem_cgroup_is_root(sc->target_mem_cgroup); |
| 445 | } |
| 446 | |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 447 | /** |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 448 | * writeback_throttling_sane - is the usual dirty throttling mechanism available? |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 449 | * @sc: scan_control in question |
| 450 | * |
| 451 | * The normal page dirty throttling mechanism in balance_dirty_pages() is |
| 452 | * completely broken with the legacy memcg and direct stalling in |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 453 | * shrink_folio_list() is used for throttling instead, which lacks all the |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 454 | * niceties such as fairness, adaptive pausing, bandwidth proportional |
| 455 | * allocation and configurability. |
| 456 | * |
| 457 | * This function tests whether the vmscan currently in progress can assume |
| 458 | * that the normal dirty throttling mechanism is operational. |
| 459 | */ |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 460 | static bool writeback_throttling_sane(struct scan_control *sc) |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 461 | { |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 462 | if (!cgroup_reclaim(sc)) |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 463 | return true; |
| 464 | #ifdef CONFIG_CGROUP_WRITEBACK |
Linus Torvalds | 69234ac | 2015-11-05 14:51:32 -0800 | [diff] [blame] | 465 | if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 466 | return true; |
| 467 | #endif |
| 468 | return false; |
| 469 | } |
KAMEZAWA Hiroyuki | 91a4547 | 2008-02-07 00:14:29 -0800 | [diff] [blame] | 470 | #else |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 471 | static int prealloc_memcg_shrinker(struct shrinker *shrinker) |
| 472 | { |
Yang Shi | 476b30a | 2021-05-04 18:36:39 -0700 | [diff] [blame] | 473 | return -ENOSYS; |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 474 | } |
| 475 | |
| 476 | static void unregister_memcg_shrinker(struct shrinker *shrinker) |
| 477 | { |
| 478 | } |
| 479 | |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 480 | static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, |
| 481 | struct mem_cgroup *memcg) |
| 482 | { |
| 483 | return 0; |
| 484 | } |
| 485 | |
| 486 | static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, |
| 487 | struct mem_cgroup *memcg) |
| 488 | { |
| 489 | return 0; |
| 490 | } |
| 491 | |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 492 | static bool cgroup_reclaim(struct scan_control *sc) |
Johannes Weiner | 89b5fae | 2012-01-12 17:17:50 -0800 | [diff] [blame] | 493 | { |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 494 | return false; |
Johannes Weiner | 89b5fae | 2012-01-12 17:17:50 -0800 | [diff] [blame] | 495 | } |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 496 | |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 497 | static bool root_reclaim(struct scan_control *sc) |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 498 | { |
| 499 | return true; |
| 500 | } |
| 501 | |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 502 | static bool writeback_throttling_sane(struct scan_control *sc) |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 503 | { |
| 504 | return true; |
| 505 | } |
KAMEZAWA Hiroyuki | 91a4547 | 2008-02-07 00:14:29 -0800 | [diff] [blame] | 506 | #endif |
| 507 | |
Yosry Ahmed | ef05e68 | 2023-04-13 10:40:33 +0000 | [diff] [blame] | 508 | static void set_task_reclaim_state(struct task_struct *task, |
| 509 | struct reclaim_state *rs) |
| 510 | { |
| 511 | /* Check for an overwrite */ |
| 512 | WARN_ON_ONCE(rs && task->reclaim_state); |
| 513 | |
| 514 | /* Check for the nulling of an already-nulled member */ |
| 515 | WARN_ON_ONCE(!rs && !task->reclaim_state); |
| 516 | |
| 517 | task->reclaim_state = rs; |
| 518 | } |
| 519 | |
Yosry Ahmed | 583c27a | 2023-04-13 10:40:32 +0000 | [diff] [blame] | 520 | /* |
| 521 | * flush_reclaim_state(): add pages reclaimed outside of LRU-based reclaim to |
| 522 | * scan_control->nr_reclaimed. |
| 523 | */ |
| 524 | static void flush_reclaim_state(struct scan_control *sc) |
| 525 | { |
| 526 | /* |
| 527 | * Currently, reclaim_state->reclaimed includes three types of pages |
| 528 | * freed outside of vmscan: |
| 529 | * (1) Slab pages. |
| 530 | * (2) Clean file pages from pruned inodes (on highmem systems). |
| 531 | * (3) XFS freed buffer pages. |
| 532 | * |
| 533 | * For all of these cases, we cannot universally link the pages to a |
| 534 | * single memcg. For example, a memcg-aware shrinker can free one object |
| 535 | * charged to the target memcg, causing an entire page to be freed. |
| 536 | * If we count the entire page as reclaimed from the memcg, we end up |
| 537 | * overestimating the reclaimed amount (potentially under-reclaiming). |
| 538 | * |
| 539 | * Only count such pages for global reclaim to prevent under-reclaiming |
| 540 | * from the target memcg; preventing unnecessary retries during memcg |
| 541 | * charging and false positives from proactive reclaim. |
| 542 | * |
| 543 | * For uncommon cases where the freed pages were actually mostly |
| 544 | * charged to the target memcg, we end up underestimating the reclaimed |
| 545 | * amount. This should be fine. The freed pages will be uncharged |
| 546 | * anyway, even if they are not counted here properly, and we will be |
| 547 | * able to make forward progress in charging (which is usually in a |
| 548 | * retry loop). |
| 549 | * |
| 550 | * We can go one step further, and report the uncharged objcg pages in |
| 551 | * memcg reclaim, to make reporting more accurate and reduce |
| 552 | * underestimation, but it's probably not worth the complexity for now. |
| 553 | */ |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 554 | if (current->reclaim_state && root_reclaim(sc)) { |
Yosry Ahmed | 583c27a | 2023-04-13 10:40:32 +0000 | [diff] [blame] | 555 | sc->nr_reclaimed += current->reclaim_state->reclaimed; |
| 556 | current->reclaim_state->reclaimed = 0; |
| 557 | } |
| 558 | } |
| 559 | |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 560 | static long xchg_nr_deferred(struct shrinker *shrinker, |
| 561 | struct shrink_control *sc) |
| 562 | { |
| 563 | int nid = sc->nid; |
| 564 | |
| 565 | if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) |
| 566 | nid = 0; |
| 567 | |
| 568 | if (sc->memcg && |
| 569 | (shrinker->flags & SHRINKER_MEMCG_AWARE)) |
| 570 | return xchg_nr_deferred_memcg(nid, shrinker, |
| 571 | sc->memcg); |
| 572 | |
| 573 | return atomic_long_xchg(&shrinker->nr_deferred[nid], 0); |
| 574 | } |
| 575 | |
| 576 | |
| 577 | static long add_nr_deferred(long nr, struct shrinker *shrinker, |
| 578 | struct shrink_control *sc) |
| 579 | { |
| 580 | int nid = sc->nid; |
| 581 | |
| 582 | if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) |
| 583 | nid = 0; |
| 584 | |
| 585 | if (sc->memcg && |
| 586 | (shrinker->flags & SHRINKER_MEMCG_AWARE)) |
| 587 | return add_nr_deferred_memcg(nr, nid, shrinker, |
| 588 | sc->memcg); |
| 589 | |
| 590 | return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]); |
| 591 | } |
| 592 | |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 593 | static bool can_demote(int nid, struct scan_control *sc) |
| 594 | { |
Huang Ying | 20b51af1 | 2021-09-02 14:59:33 -0700 | [diff] [blame] | 595 | if (!numa_demotion_enabled) |
| 596 | return false; |
Johannes Weiner | 3f1509c | 2022-05-18 15:09:11 -0400 | [diff] [blame] | 597 | if (sc && sc->no_demotion) |
| 598 | return false; |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 599 | if (next_demotion_node(nid) == NUMA_NO_NODE) |
| 600 | return false; |
| 601 | |
Huang Ying | 20b51af1 | 2021-09-02 14:59:33 -0700 | [diff] [blame] | 602 | return true; |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 603 | } |
| 604 | |
Keith Busch | a2a3648 | 2021-09-02 14:59:26 -0700 | [diff] [blame] | 605 | static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, |
| 606 | int nid, |
| 607 | struct scan_control *sc) |
| 608 | { |
| 609 | if (memcg == NULL) { |
| 610 | /* |
| 611 | * For non-memcg reclaim, is there |
| 612 | * space in any swap device? |
| 613 | */ |
| 614 | if (get_nr_swap_pages() > 0) |
| 615 | return true; |
| 616 | } else { |
| 617 | /* Is the memcg below its swap limit? */ |
| 618 | if (mem_cgroup_get_nr_swap_pages(memcg) > 0) |
| 619 | return true; |
| 620 | } |
| 621 | |
| 622 | /* |
| 623 | * The page can not be swapped. |
| 624 | * |
| 625 | * Can it be reclaimed from this node via demotion? |
| 626 | */ |
| 627 | return can_demote(nid, sc); |
| 628 | } |
| 629 | |
Mel Gorman | 5a1c84b | 2016-07-28 15:47:31 -0700 | [diff] [blame] | 630 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 631 | * This misses isolated folios which are not accounted for to save counters. |
Mel Gorman | 5a1c84b | 2016-07-28 15:47:31 -0700 | [diff] [blame] | 632 | * As the data only determines if reclaim or compaction continues, it is |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 633 | * not expected that isolated folios will be a dominating factor. |
Mel Gorman | 5a1c84b | 2016-07-28 15:47:31 -0700 | [diff] [blame] | 634 | */ |
| 635 | unsigned long zone_reclaimable_pages(struct zone *zone) |
| 636 | { |
| 637 | unsigned long nr; |
| 638 | |
| 639 | nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) + |
| 640 | zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE); |
Keith Busch | a2a3648 | 2021-09-02 14:59:26 -0700 | [diff] [blame] | 641 | if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL)) |
Mel Gorman | 5a1c84b | 2016-07-28 15:47:31 -0700 | [diff] [blame] | 642 | nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) + |
| 643 | zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON); |
| 644 | |
| 645 | return nr; |
| 646 | } |
| 647 | |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 648 | /** |
| 649 | * lruvec_lru_size - Returns the number of pages on the given LRU list. |
| 650 | * @lruvec: lru vector |
| 651 | * @lru: lru to use |
Wei Yang | 8b3a899 | 2022-04-28 23:16:04 -0700 | [diff] [blame] | 652 | * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list) |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 653 | */ |
Yu Zhao | 2091339 | 2021-02-24 12:08:44 -0800 | [diff] [blame] | 654 | static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, |
| 655 | int zone_idx) |
KOSAKI Motohiro | c9f299d | 2009-01-07 18:08:16 -0800 | [diff] [blame] | 656 | { |
Johannes Weiner | de3b015 | 2019-11-30 17:55:31 -0800 | [diff] [blame] | 657 | unsigned long size = 0; |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 658 | int zid; |
| 659 | |
Wei Yang | 8b3a899 | 2022-04-28 23:16:04 -0700 | [diff] [blame] | 660 | for (zid = 0; zid <= zone_idx; zid++) { |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 661 | struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid]; |
KOSAKI Motohiro | c9f299d | 2009-01-07 18:08:16 -0800 | [diff] [blame] | 662 | |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 663 | if (!managed_zone(zone)) |
| 664 | continue; |
Michal Hocko | b4536f0c8 | 2017-01-10 16:58:04 -0800 | [diff] [blame] | 665 | |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 666 | if (!mem_cgroup_disabled()) |
Johannes Weiner | de3b015 | 2019-11-30 17:55:31 -0800 | [diff] [blame] | 667 | size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 668 | else |
Johannes Weiner | de3b015 | 2019-11-30 17:55:31 -0800 | [diff] [blame] | 669 | size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru); |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 670 | } |
Johannes Weiner | de3b015 | 2019-11-30 17:55:31 -0800 | [diff] [blame] | 671 | return size; |
Michal Hocko | b4536f0c8 | 2017-01-10 16:58:04 -0800 | [diff] [blame] | 672 | } |
| 673 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 674 | /* |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 675 | * Add a shrinker callback to be called from the vm. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 676 | */ |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 677 | static int __prealloc_shrinker(struct shrinker *shrinker) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 678 | { |
Yang Shi | 476b30a | 2021-05-04 18:36:39 -0700 | [diff] [blame] | 679 | unsigned int size; |
| 680 | int err; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 681 | |
Yang Shi | 476b30a | 2021-05-04 18:36:39 -0700 | [diff] [blame] | 682 | if (shrinker->flags & SHRINKER_MEMCG_AWARE) { |
| 683 | err = prealloc_memcg_shrinker(shrinker); |
| 684 | if (err != -ENOSYS) |
| 685 | return err; |
| 686 | |
| 687 | shrinker->flags &= ~SHRINKER_MEMCG_AWARE; |
| 688 | } |
| 689 | |
| 690 | size = sizeof(*shrinker->nr_deferred); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 691 | if (shrinker->flags & SHRINKER_NUMA_AWARE) |
| 692 | size *= nr_node_ids; |
| 693 | |
| 694 | shrinker->nr_deferred = kzalloc(size, GFP_KERNEL); |
| 695 | if (!shrinker->nr_deferred) |
| 696 | return -ENOMEM; |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 697 | |
Tetsuo Handa | 8e04944 | 2018-04-04 19:53:07 +0900 | [diff] [blame] | 698 | return 0; |
| 699 | } |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 700 | |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 701 | #ifdef CONFIG_SHRINKER_DEBUG |
| 702 | int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 703 | { |
| 704 | va_list ap; |
| 705 | int err; |
| 706 | |
| 707 | va_start(ap, fmt); |
| 708 | shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); |
| 709 | va_end(ap); |
| 710 | if (!shrinker->name) |
| 711 | return -ENOMEM; |
| 712 | |
| 713 | err = __prealloc_shrinker(shrinker); |
Tetsuo Handa | 14773bf | 2022-07-20 23:47:55 +0900 | [diff] [blame] | 714 | if (err) { |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 715 | kfree_const(shrinker->name); |
Tetsuo Handa | 14773bf | 2022-07-20 23:47:55 +0900 | [diff] [blame] | 716 | shrinker->name = NULL; |
| 717 | } |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 718 | |
| 719 | return err; |
| 720 | } |
| 721 | #else |
| 722 | int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 723 | { |
| 724 | return __prealloc_shrinker(shrinker); |
| 725 | } |
| 726 | #endif |
| 727 | |
Tetsuo Handa | 8e04944 | 2018-04-04 19:53:07 +0900 | [diff] [blame] | 728 | void free_prealloced_shrinker(struct shrinker *shrinker) |
| 729 | { |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 730 | #ifdef CONFIG_SHRINKER_DEBUG |
| 731 | kfree_const(shrinker->name); |
Tetsuo Handa | 14773bf | 2022-07-20 23:47:55 +0900 | [diff] [blame] | 732 | shrinker->name = NULL; |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 733 | #endif |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 734 | if (shrinker->flags & SHRINKER_MEMCG_AWARE) { |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 735 | down_write(&shrinker_rwsem); |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 736 | unregister_memcg_shrinker(shrinker); |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 737 | up_write(&shrinker_rwsem); |
Yang Shi | 476b30a | 2021-05-04 18:36:39 -0700 | [diff] [blame] | 738 | return; |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 739 | } |
Kirill Tkhai | b4c2b23 | 2018-08-17 15:47:29 -0700 | [diff] [blame] | 740 | |
Tetsuo Handa | 8e04944 | 2018-04-04 19:53:07 +0900 | [diff] [blame] | 741 | kfree(shrinker->nr_deferred); |
| 742 | shrinker->nr_deferred = NULL; |
| 743 | } |
| 744 | |
| 745 | void register_shrinker_prepared(struct shrinker *shrinker) |
| 746 | { |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 747 | down_write(&shrinker_rwsem); |
Qi Zheng | 71c3ad6 | 2023-06-09 08:15:18 +0000 | [diff] [blame] | 748 | list_add_tail(&shrinker->list, &shrinker_list); |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 749 | shrinker->flags |= SHRINKER_REGISTERED; |
Roman Gushchin | 5035ebc | 2022-05-31 20:22:23 -0700 | [diff] [blame] | 750 | shrinker_debugfs_add(shrinker); |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 751 | up_write(&shrinker_rwsem); |
Tetsuo Handa | 8e04944 | 2018-04-04 19:53:07 +0900 | [diff] [blame] | 752 | } |
| 753 | |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 754 | static int __register_shrinker(struct shrinker *shrinker) |
Tetsuo Handa | 8e04944 | 2018-04-04 19:53:07 +0900 | [diff] [blame] | 755 | { |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 756 | int err = __prealloc_shrinker(shrinker); |
Tetsuo Handa | 8e04944 | 2018-04-04 19:53:07 +0900 | [diff] [blame] | 757 | |
| 758 | if (err) |
| 759 | return err; |
| 760 | register_shrinker_prepared(shrinker); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 761 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 762 | } |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 763 | |
| 764 | #ifdef CONFIG_SHRINKER_DEBUG |
| 765 | int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 766 | { |
| 767 | va_list ap; |
| 768 | int err; |
| 769 | |
| 770 | va_start(ap, fmt); |
| 771 | shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); |
| 772 | va_end(ap); |
| 773 | if (!shrinker->name) |
| 774 | return -ENOMEM; |
| 775 | |
| 776 | err = __register_shrinker(shrinker); |
Tetsuo Handa | 14773bf | 2022-07-20 23:47:55 +0900 | [diff] [blame] | 777 | if (err) { |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 778 | kfree_const(shrinker->name); |
Tetsuo Handa | 14773bf | 2022-07-20 23:47:55 +0900 | [diff] [blame] | 779 | shrinker->name = NULL; |
| 780 | } |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 781 | return err; |
| 782 | } |
| 783 | #else |
| 784 | int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) |
| 785 | { |
| 786 | return __register_shrinker(shrinker); |
| 787 | } |
| 788 | #endif |
Rusty Russell | 8e1f936 | 2007-07-17 04:03:17 -0700 | [diff] [blame] | 789 | EXPORT_SYMBOL(register_shrinker); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 790 | |
| 791 | /* |
| 792 | * Remove one |
| 793 | */ |
Rusty Russell | 8e1f936 | 2007-07-17 04:03:17 -0700 | [diff] [blame] | 794 | void unregister_shrinker(struct shrinker *shrinker) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 795 | { |
Qi Zheng | badc28d | 2023-02-02 18:56:12 +0800 | [diff] [blame] | 796 | struct dentry *debugfs_entry; |
Joan Bruguera Micó | 26e239b | 2023-05-03 01:32:32 +0000 | [diff] [blame] | 797 | int debugfs_id; |
Qi Zheng | badc28d | 2023-02-02 18:56:12 +0800 | [diff] [blame] | 798 | |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 799 | if (!(shrinker->flags & SHRINKER_REGISTERED)) |
Tetsuo Handa | bb422a7 | 2017-12-18 20:31:41 +0900 | [diff] [blame] | 800 | return; |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 801 | |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 802 | down_write(&shrinker_rwsem); |
Qi Zheng | 71c3ad6 | 2023-06-09 08:15:18 +0000 | [diff] [blame] | 803 | list_del(&shrinker->list); |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 804 | shrinker->flags &= ~SHRINKER_REGISTERED; |
| 805 | if (shrinker->flags & SHRINKER_MEMCG_AWARE) |
| 806 | unregister_memcg_shrinker(shrinker); |
Joan Bruguera Micó | 26e239b | 2023-05-03 01:32:32 +0000 | [diff] [blame] | 807 | debugfs_entry = shrinker_debugfs_detach(shrinker, &debugfs_id); |
Qi Zheng | 47a7c01 | 2023-06-09 08:15:12 +0000 | [diff] [blame] | 808 | up_write(&shrinker_rwsem); |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 809 | |
Joan Bruguera Micó | 26e239b | 2023-05-03 01:32:32 +0000 | [diff] [blame] | 810 | shrinker_debugfs_remove(debugfs_entry, debugfs_id); |
Qi Zheng | badc28d | 2023-02-02 18:56:12 +0800 | [diff] [blame] | 811 | |
Andrew Vagin | ae39332 | 2013-10-16 13:46:46 -0700 | [diff] [blame] | 812 | kfree(shrinker->nr_deferred); |
Tetsuo Handa | bb422a7 | 2017-12-18 20:31:41 +0900 | [diff] [blame] | 813 | shrinker->nr_deferred = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 814 | } |
Rusty Russell | 8e1f936 | 2007-07-17 04:03:17 -0700 | [diff] [blame] | 815 | EXPORT_SYMBOL(unregister_shrinker); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 816 | |
Christian König | 880121b | 2021-04-15 13:56:23 +0200 | [diff] [blame] | 817 | /** |
| 818 | * synchronize_shrinkers - Wait for all running shrinkers to complete. |
| 819 | * |
Qi Zheng | 07252b0 | 2023-06-09 08:15:13 +0000 | [diff] [blame] | 820 | * This is equivalent to calling unregister_shrink() and register_shrinker(), |
| 821 | * but atomically and with less overhead. This is useful to guarantee that all |
| 822 | * shrinker invocations have seen an update, before freeing memory, similar to |
| 823 | * rcu. |
Christian König | 880121b | 2021-04-15 13:56:23 +0200 | [diff] [blame] | 824 | */ |
| 825 | void synchronize_shrinkers(void) |
| 826 | { |
Qi Zheng | 07252b0 | 2023-06-09 08:15:13 +0000 | [diff] [blame] | 827 | down_write(&shrinker_rwsem); |
| 828 | up_write(&shrinker_rwsem); |
Christian König | 880121b | 2021-04-15 13:56:23 +0200 | [diff] [blame] | 829 | } |
| 830 | EXPORT_SYMBOL(synchronize_shrinkers); |
| 831 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 832 | #define SHRINK_BATCH 128 |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 833 | |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 834 | static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, |
Josef Bacik | 9092c71 | 2018-01-31 16:16:26 -0800 | [diff] [blame] | 835 | struct shrinker *shrinker, int priority) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 836 | { |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 837 | unsigned long freed = 0; |
| 838 | unsigned long long delta; |
| 839 | long total_scan; |
Vladimir Davydov | d5bc5fd | 2014-04-03 14:47:32 -0700 | [diff] [blame] | 840 | long freeable; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 841 | long nr; |
| 842 | long new_nr; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 843 | long batch_size = shrinker->batch ? shrinker->batch |
| 844 | : SHRINK_BATCH; |
Shaohua Li | 5f33a08 | 2016-12-12 16:41:50 -0800 | [diff] [blame] | 845 | long scanned = 0, next_deferred; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 846 | |
Vladimir Davydov | d5bc5fd | 2014-04-03 14:47:32 -0700 | [diff] [blame] | 847 | freeable = shrinker->count_objects(shrinker, shrinkctl); |
Kirill Tkhai | 9b99646 | 2018-08-17 15:48:21 -0700 | [diff] [blame] | 848 | if (freeable == 0 || freeable == SHRINK_EMPTY) |
| 849 | return freeable; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 850 | |
| 851 | /* |
| 852 | * copy the current shrinker scan count into a local variable |
| 853 | * and zero it so that other concurrent shrinker invocations |
| 854 | * don't also do this scanning work. |
| 855 | */ |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 856 | nr = xchg_nr_deferred(shrinker, shrinkctl); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 857 | |
Johannes Weiner | 4b85afb | 2018-10-26 15:06:42 -0700 | [diff] [blame] | 858 | if (shrinker->seeks) { |
| 859 | delta = freeable >> priority; |
| 860 | delta *= 4; |
| 861 | do_div(delta, shrinker->seeks); |
| 862 | } else { |
| 863 | /* |
| 864 | * These objects don't require any IO to create. Trim |
| 865 | * them aggressively under memory pressure to keep |
| 866 | * them from causing refetches in the IO caches. |
| 867 | */ |
| 868 | delta = freeable / 2; |
| 869 | } |
Roman Gushchin | 172b06c3 | 2018-09-20 12:22:46 -0700 | [diff] [blame] | 870 | |
Yang Shi | 18bb473 | 2021-05-04 18:36:45 -0700 | [diff] [blame] | 871 | total_scan = nr >> priority; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 872 | total_scan += delta; |
Yang Shi | 18bb473 | 2021-05-04 18:36:45 -0700 | [diff] [blame] | 873 | total_scan = min(total_scan, (2 * freeable)); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 874 | |
| 875 | trace_mm_shrink_slab_start(shrinker, shrinkctl, nr, |
Josef Bacik | 9092c71 | 2018-01-31 16:16:26 -0800 | [diff] [blame] | 876 | freeable, delta, total_scan, priority); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 877 | |
Vladimir Davydov | 0b1fb40 | 2014-01-23 15:53:22 -0800 | [diff] [blame] | 878 | /* |
| 879 | * Normally, we should not scan less than batch_size objects in one |
| 880 | * pass to avoid too frequent shrinker calls, but if the slab has less |
| 881 | * than batch_size objects in total and we are really tight on memory, |
| 882 | * we will try to reclaim all available objects, otherwise we can end |
| 883 | * up failing allocations although there are plenty of reclaimable |
| 884 | * objects spread over several slabs with usage less than the |
| 885 | * batch_size. |
| 886 | * |
| 887 | * We detect the "tight on memory" situations by looking at the total |
| 888 | * number of objects we want to scan (total_scan). If it is greater |
Vladimir Davydov | d5bc5fd | 2014-04-03 14:47:32 -0700 | [diff] [blame] | 889 | * than the total number of objects on slab (freeable), we must be |
Vladimir Davydov | 0b1fb40 | 2014-01-23 15:53:22 -0800 | [diff] [blame] | 890 | * scanning at high prio and therefore should try to reclaim as much as |
| 891 | * possible. |
| 892 | */ |
| 893 | while (total_scan >= batch_size || |
Vladimir Davydov | d5bc5fd | 2014-04-03 14:47:32 -0700 | [diff] [blame] | 894 | total_scan >= freeable) { |
Dave Chinner | a0b0213 | 2013-08-28 10:18:16 +1000 | [diff] [blame] | 895 | unsigned long ret; |
Vladimir Davydov | 0b1fb40 | 2014-01-23 15:53:22 -0800 | [diff] [blame] | 896 | unsigned long nr_to_scan = min(batch_size, total_scan); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 897 | |
Vladimir Davydov | 0b1fb40 | 2014-01-23 15:53:22 -0800 | [diff] [blame] | 898 | shrinkctl->nr_to_scan = nr_to_scan; |
Chris Wilson | d460acb | 2017-09-06 16:19:26 -0700 | [diff] [blame] | 899 | shrinkctl->nr_scanned = nr_to_scan; |
Dave Chinner | a0b0213 | 2013-08-28 10:18:16 +1000 | [diff] [blame] | 900 | ret = shrinker->scan_objects(shrinker, shrinkctl); |
| 901 | if (ret == SHRINK_STOP) |
| 902 | break; |
| 903 | freed += ret; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 904 | |
Chris Wilson | d460acb | 2017-09-06 16:19:26 -0700 | [diff] [blame] | 905 | count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned); |
| 906 | total_scan -= shrinkctl->nr_scanned; |
| 907 | scanned += shrinkctl->nr_scanned; |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 908 | |
| 909 | cond_resched(); |
| 910 | } |
| 911 | |
Yang Shi | 18bb473 | 2021-05-04 18:36:45 -0700 | [diff] [blame] | 912 | /* |
| 913 | * The deferred work is increased by any new work (delta) that wasn't |
| 914 | * done, decreased by old deferred work that was done now. |
| 915 | * |
| 916 | * And it is capped to two times of the freeable items. |
| 917 | */ |
| 918 | next_deferred = max_t(long, (nr + delta - scanned), 0); |
| 919 | next_deferred = min(next_deferred, (2 * freeable)); |
| 920 | |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 921 | /* |
| 922 | * move the unused scan count back into the shrinker in a |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 923 | * manner that handles concurrent updates. |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 924 | */ |
Yang Shi | 8675083 | 2021-05-04 18:36:36 -0700 | [diff] [blame] | 925 | new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 926 | |
Yang Shi | 8efb4b5 | 2021-05-04 18:36:08 -0700 | [diff] [blame] | 927 | trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan); |
Glauber Costa | 1d3d443 | 2013-08-28 10:18:04 +1000 | [diff] [blame] | 928 | return freed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 929 | } |
| 930 | |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 931 | #ifdef CONFIG_MEMCG |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 932 | static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, |
| 933 | struct mem_cgroup *memcg, int priority) |
| 934 | { |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 935 | struct shrinker_info *info; |
Kirill Tkhai | b8e57ef | 2018-10-05 15:52:10 -0700 | [diff] [blame] | 936 | unsigned long ret, freed = 0; |
Qi Zheng | d6ecbcd | 2023-06-09 08:15:16 +0000 | [diff] [blame] | 937 | int i; |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 938 | |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 939 | if (!mem_cgroup_online(memcg)) |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 940 | return 0; |
| 941 | |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 942 | if (!down_read_trylock(&shrinker_rwsem)) |
| 943 | return 0; |
| 944 | |
| 945 | info = shrinker_info_protected(memcg, nid); |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 946 | if (unlikely(!info)) |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 947 | goto unlock; |
| 948 | |
Qi Zheng | d6ecbcd | 2023-06-09 08:15:16 +0000 | [diff] [blame] | 949 | for_each_set_bit(i, info->map, info->map_nr_max) { |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 950 | struct shrink_control sc = { |
| 951 | .gfp_mask = gfp_mask, |
| 952 | .nid = nid, |
| 953 | .memcg = memcg, |
| 954 | }; |
| 955 | struct shrinker *shrinker; |
| 956 | |
| 957 | shrinker = idr_find(&shrinker_idr, i); |
Yang Shi | 41ca668 | 2021-05-04 18:36:29 -0700 | [diff] [blame] | 958 | if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) { |
Kirill Tkhai | 7e010df | 2018-08-17 15:48:34 -0700 | [diff] [blame] | 959 | if (!shrinker) |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 960 | clear_bit(i, info->map); |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 961 | continue; |
| 962 | } |
| 963 | |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 964 | /* Call non-slab shrinkers even though kmem is disabled */ |
Roman Gushchin | f7a449f | 2023-02-13 11:29:22 -0800 | [diff] [blame] | 965 | if (!memcg_kmem_online() && |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 966 | !(shrinker->flags & SHRINKER_NONSLAB)) |
| 967 | continue; |
| 968 | |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 969 | ret = do_shrink_slab(&sc, shrinker, priority); |
Kirill Tkhai | f90280d | 2018-08-17 15:48:25 -0700 | [diff] [blame] | 970 | if (ret == SHRINK_EMPTY) { |
Yang Shi | e4262c4 | 2021-05-04 18:36:23 -0700 | [diff] [blame] | 971 | clear_bit(i, info->map); |
Kirill Tkhai | f90280d | 2018-08-17 15:48:25 -0700 | [diff] [blame] | 972 | /* |
| 973 | * After the shrinker reported that it had no objects to |
| 974 | * free, but before we cleared the corresponding bit in |
| 975 | * the memcg shrinker map, a new object might have been |
| 976 | * added. To make sure, we have the bit set in this |
| 977 | * case, we invoke the shrinker one more time and reset |
| 978 | * the bit if it reports that it is not empty anymore. |
| 979 | * The memory barrier here pairs with the barrier in |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 980 | * set_shrinker_bit(): |
Kirill Tkhai | f90280d | 2018-08-17 15:48:25 -0700 | [diff] [blame] | 981 | * |
| 982 | * list_lru_add() shrink_slab_memcg() |
| 983 | * list_add_tail() clear_bit() |
| 984 | * <MB> <MB> |
| 985 | * set_bit() do_shrink_slab() |
| 986 | */ |
| 987 | smp_mb__after_atomic(); |
| 988 | ret = do_shrink_slab(&sc, shrinker, priority); |
| 989 | if (ret == SHRINK_EMPTY) |
| 990 | ret = 0; |
| 991 | else |
Yang Shi | 2bfd363 | 2021-05-04 18:36:11 -0700 | [diff] [blame] | 992 | set_shrinker_bit(memcg, nid, i); |
Kirill Tkhai | f90280d | 2018-08-17 15:48:25 -0700 | [diff] [blame] | 993 | } |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 994 | freed += ret; |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 995 | |
| 996 | if (rwsem_is_contended(&shrinker_rwsem)) { |
| 997 | freed = freed ? : 1; |
| 998 | break; |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 999 | } |
| 1000 | } |
| 1001 | unlock: |
Qi Zheng | 7cee360 | 2023-06-09 08:15:17 +0000 | [diff] [blame] | 1002 | up_read(&shrinker_rwsem); |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 1003 | return freed; |
| 1004 | } |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 1005 | #else /* CONFIG_MEMCG */ |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 1006 | static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, |
| 1007 | struct mem_cgroup *memcg, int priority) |
| 1008 | { |
| 1009 | return 0; |
| 1010 | } |
Yang Shi | 0a432dcb | 2019-09-23 15:38:12 -0700 | [diff] [blame] | 1011 | #endif /* CONFIG_MEMCG */ |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 1012 | |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1013 | /** |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1014 | * shrink_slab - shrink slab caches |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1015 | * @gfp_mask: allocation context |
| 1016 | * @nid: node whose slab caches to target |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1017 | * @memcg: memory cgroup whose slab caches to target |
Josef Bacik | 9092c71 | 2018-01-31 16:16:26 -0800 | [diff] [blame] | 1018 | * @priority: the reclaim priority |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1019 | * |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1020 | * Call the shrink functions to age shrinkable caches. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1021 | * |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1022 | * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set, |
| 1023 | * unaware shrinkers will receive a node id of 0 instead. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1024 | * |
Vladimir Davydov | aeed1d32 | 2018-08-17 15:48:17 -0700 | [diff] [blame] | 1025 | * @memcg specifies the memory cgroup to target. Unaware shrinkers |
| 1026 | * are called only if it is the root cgroup. |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1027 | * |
Josef Bacik | 9092c71 | 2018-01-31 16:16:26 -0800 | [diff] [blame] | 1028 | * @priority is sc->priority, we take the number of objects and >> by priority |
| 1029 | * in order to get the scan target. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1030 | * |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1031 | * Returns the number of reclaimed slab objects. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1032 | */ |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1033 | static unsigned long shrink_slab(gfp_t gfp_mask, int nid, |
| 1034 | struct mem_cgroup *memcg, |
Josef Bacik | 9092c71 | 2018-01-31 16:16:26 -0800 | [diff] [blame] | 1035 | int priority) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1036 | { |
Kirill Tkhai | b8e57ef | 2018-10-05 15:52:10 -0700 | [diff] [blame] | 1037 | unsigned long ret, freed = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1038 | struct shrinker *shrinker; |
| 1039 | |
Yang Shi | fa1e512 | 2019-08-02 21:48:44 -0700 | [diff] [blame] | 1040 | /* |
| 1041 | * The root memcg might be allocated even though memcg is disabled |
| 1042 | * via "cgroup_disable=memory" boot parameter. This could make |
| 1043 | * mem_cgroup_is_root() return false, then just run memcg slab |
| 1044 | * shrink, but skip global shrink. This may result in premature |
| 1045 | * oom. |
| 1046 | */ |
| 1047 | if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg)) |
Kirill Tkhai | b0dedc4 | 2018-08-17 15:48:14 -0700 | [diff] [blame] | 1048 | return shrink_slab_memcg(gfp_mask, nid, memcg, priority); |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1049 | |
Qi Zheng | 71c3ad6 | 2023-06-09 08:15:18 +0000 | [diff] [blame] | 1050 | if (!down_read_trylock(&shrinker_rwsem)) |
| 1051 | goto out; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1052 | |
Qi Zheng | 71c3ad6 | 2023-06-09 08:15:18 +0000 | [diff] [blame] | 1053 | list_for_each_entry(shrinker, &shrinker_list, list) { |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1054 | struct shrink_control sc = { |
| 1055 | .gfp_mask = gfp_mask, |
| 1056 | .nid = nid, |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1057 | .memcg = memcg, |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1058 | }; |
Vladimir Davydov | ec97097 | 2014-01-23 15:53:23 -0800 | [diff] [blame] | 1059 | |
Kirill Tkhai | 9b99646 | 2018-08-17 15:48:21 -0700 | [diff] [blame] | 1060 | ret = do_shrink_slab(&sc, shrinker, priority); |
| 1061 | if (ret == SHRINK_EMPTY) |
| 1062 | ret = 0; |
| 1063 | freed += ret; |
Qi Zheng | 71c3ad6 | 2023-06-09 08:15:18 +0000 | [diff] [blame] | 1064 | /* |
| 1065 | * Bail out if someone want to register a new shrinker to |
| 1066 | * prevent the registration from being stalled for long periods |
| 1067 | * by parallel ongoing shrinking. |
| 1068 | */ |
| 1069 | if (rwsem_is_contended(&shrinker_rwsem)) { |
Minchan Kim | e496612 | 2018-01-31 16:16:55 -0800 | [diff] [blame] | 1070 | freed = freed ? : 1; |
| 1071 | break; |
| 1072 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1073 | } |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 1074 | |
Qi Zheng | 71c3ad6 | 2023-06-09 08:15:18 +0000 | [diff] [blame] | 1075 | up_read(&shrinker_rwsem); |
| 1076 | out: |
Minchan Kim | f06590b | 2011-05-24 17:11:11 -0700 | [diff] [blame] | 1077 | cond_resched(); |
Dave Chinner | 24f7c6b | 2013-08-28 10:17:56 +1000 | [diff] [blame] | 1078 | return freed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1079 | } |
| 1080 | |
Jan Kara | e83b39d6 | 2022-11-15 13:32:55 +0100 | [diff] [blame] | 1081 | static unsigned long drop_slab_node(int nid) |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1082 | { |
Jan Kara | e83b39d6 | 2022-11-15 13:32:55 +0100 | [diff] [blame] | 1083 | unsigned long freed = 0; |
| 1084 | struct mem_cgroup *memcg = NULL; |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1085 | |
Jan Kara | e83b39d6 | 2022-11-15 13:32:55 +0100 | [diff] [blame] | 1086 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1087 | do { |
Jan Kara | e83b39d6 | 2022-11-15 13:32:55 +0100 | [diff] [blame] | 1088 | freed += shrink_slab(GFP_KERNEL, nid, memcg, 0); |
| 1089 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1090 | |
Jan Kara | e83b39d6 | 2022-11-15 13:32:55 +0100 | [diff] [blame] | 1091 | return freed; |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1092 | } |
| 1093 | |
| 1094 | void drop_slab(void) |
| 1095 | { |
| 1096 | int nid; |
Jan Kara | e83b39d6 | 2022-11-15 13:32:55 +0100 | [diff] [blame] | 1097 | int shift = 0; |
| 1098 | unsigned long freed; |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1099 | |
Jan Kara | e83b39d6 | 2022-11-15 13:32:55 +0100 | [diff] [blame] | 1100 | do { |
| 1101 | freed = 0; |
| 1102 | for_each_online_node(nid) { |
| 1103 | if (fatal_signal_pending(current)) |
| 1104 | return; |
| 1105 | |
| 1106 | freed += drop_slab_node(nid); |
| 1107 | } |
| 1108 | } while ((freed >> shift++) > 1); |
Vladimir Davydov | cb731d6 | 2015-02-12 14:58:54 -0800 | [diff] [blame] | 1109 | } |
| 1110 | |
Johannes Weiner | 57e9cc5 | 2022-10-26 14:01:33 -0400 | [diff] [blame] | 1111 | static int reclaimer_offset(void) |
| 1112 | { |
| 1113 | BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD != |
| 1114 | PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD); |
| 1115 | BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD != |
| 1116 | PGSCAN_DIRECT - PGSCAN_KSWAPD); |
| 1117 | BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD != |
| 1118 | PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD); |
| 1119 | BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD != |
| 1120 | PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD); |
| 1121 | |
| 1122 | if (current_is_kswapd()) |
| 1123 | return 0; |
| 1124 | if (current_is_khugepaged()) |
| 1125 | return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD; |
| 1126 | return PGSTEAL_DIRECT - PGSTEAL_KSWAPD; |
| 1127 | } |
| 1128 | |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1129 | static inline int is_page_cache_freeable(struct folio *folio) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1130 | { |
Johannes Weiner | ceddc3a | 2009-09-21 17:03:00 -0700 | [diff] [blame] | 1131 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1132 | * A freeable page cache folio is referenced only by the caller |
| 1133 | * that isolated the folio, the page cache and optional filesystem |
| 1134 | * private data at folio->private. |
Johannes Weiner | ceddc3a | 2009-09-21 17:03:00 -0700 | [diff] [blame] | 1135 | */ |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1136 | return folio_ref_count(folio) - folio_test_private(folio) == |
| 1137 | 1 + folio_nr_pages(folio); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1138 | } |
| 1139 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1140 | /* |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1141 | * We detected a synchronous write error writing a folio out. Probably |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1142 | * -ENOSPC. We need to propagate that into the address_space for a subsequent |
| 1143 | * fsync(), msync() or close(). |
| 1144 | * |
| 1145 | * The tricky part is that after writepage we cannot touch the mapping: nothing |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1146 | * prevents it from being freed up. But we have a ref on the folio and once |
| 1147 | * that folio is locked, the mapping is pinned. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1148 | * |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1149 | * We're allowed to run sleeping folio_lock() here because we know the caller has |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1150 | * __GFP_FS. |
| 1151 | */ |
| 1152 | static void handle_write_error(struct address_space *mapping, |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1153 | struct folio *folio, int error) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1154 | { |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1155 | folio_lock(folio); |
| 1156 | if (folio_mapping(folio) == mapping) |
Guillaume Chazarain | 3e9f45b | 2007-05-08 00:23:25 -0700 | [diff] [blame] | 1157 | mapping_set_error(mapping, error); |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1158 | folio_unlock(folio); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1159 | } |
| 1160 | |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 1161 | static bool skip_throttle_noprogress(pg_data_t *pgdat) |
| 1162 | { |
| 1163 | int reclaimable = 0, write_pending = 0; |
| 1164 | int i; |
| 1165 | |
| 1166 | /* |
| 1167 | * If kswapd is disabled, reschedule if necessary but do not |
| 1168 | * throttle as the system is likely near OOM. |
| 1169 | */ |
| 1170 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) |
| 1171 | return true; |
| 1172 | |
| 1173 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1174 | * If there are a lot of dirty/writeback folios then do not |
| 1175 | * throttle as throttling will occur when the folios cycle |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 1176 | * towards the end of the LRU if still under writeback. |
| 1177 | */ |
| 1178 | for (i = 0; i < MAX_NR_ZONES; i++) { |
| 1179 | struct zone *zone = pgdat->node_zones + i; |
| 1180 | |
Wei Yang | 36c2612 | 2022-04-28 23:16:03 -0700 | [diff] [blame] | 1181 | if (!managed_zone(zone)) |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 1182 | continue; |
| 1183 | |
| 1184 | reclaimable += zone_reclaimable_pages(zone); |
| 1185 | write_pending += zone_page_state_snapshot(zone, |
| 1186 | NR_ZONE_WRITE_PENDING); |
| 1187 | } |
| 1188 | if (2 * write_pending <= reclaimable) |
| 1189 | return true; |
| 1190 | |
| 1191 | return false; |
| 1192 | } |
| 1193 | |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 1194 | void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1195 | { |
| 1196 | wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 1197 | long timeout, ret; |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1198 | DEFINE_WAIT(wait); |
| 1199 | |
| 1200 | /* |
Mike Christie | 54e6842 | 2023-03-10 16:03:26 -0600 | [diff] [blame] | 1201 | * Do not throttle user workers, kthreads other than kswapd or |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1202 | * workqueues. They may be required for reclaim to make |
| 1203 | * forward progress (e.g. journalling workqueues or kthreads). |
| 1204 | */ |
| 1205 | if (!current_is_kswapd() && |
Mike Christie | 54e6842 | 2023-03-10 16:03:26 -0600 | [diff] [blame] | 1206 | current->flags & (PF_USER_WORKER|PF_KTHREAD)) { |
Mel Gorman | b485c6f | 2022-02-11 16:32:29 -0800 | [diff] [blame] | 1207 | cond_resched(); |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1208 | return; |
Mel Gorman | b485c6f | 2022-02-11 16:32:29 -0800 | [diff] [blame] | 1209 | } |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1210 | |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 1211 | /* |
| 1212 | * These figures are pulled out of thin air. |
| 1213 | * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many |
| 1214 | * parallel reclaimers which is a short-lived event so the timeout is |
| 1215 | * short. Failing to make progress or waiting on writeback are |
| 1216 | * potentially long-lived events so use a longer timeout. This is shaky |
| 1217 | * logic as a failure to make progress could be due to anything from |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1218 | * writeback to a slow device to excessive referenced folios at the tail |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 1219 | * of the inactive LRU. |
| 1220 | */ |
| 1221 | switch(reason) { |
| 1222 | case VMSCAN_THROTTLE_WRITEBACK: |
| 1223 | timeout = HZ/10; |
| 1224 | |
| 1225 | if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { |
| 1226 | WRITE_ONCE(pgdat->nr_reclaim_start, |
| 1227 | node_page_state(pgdat, NR_THROTTLED_WRITTEN)); |
| 1228 | } |
| 1229 | |
| 1230 | break; |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 1231 | case VMSCAN_THROTTLE_CONGESTED: |
| 1232 | fallthrough; |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 1233 | case VMSCAN_THROTTLE_NOPROGRESS: |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 1234 | if (skip_throttle_noprogress(pgdat)) { |
| 1235 | cond_resched(); |
| 1236 | return; |
| 1237 | } |
| 1238 | |
| 1239 | timeout = 1; |
| 1240 | |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 1241 | break; |
| 1242 | case VMSCAN_THROTTLE_ISOLATED: |
| 1243 | timeout = HZ/50; |
| 1244 | break; |
| 1245 | default: |
| 1246 | WARN_ON_ONCE(1); |
| 1247 | timeout = HZ; |
| 1248 | break; |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1249 | } |
| 1250 | |
| 1251 | prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); |
| 1252 | ret = schedule_timeout(timeout); |
| 1253 | finish_wait(wqh, &wait); |
Mel Gorman | d818fca | 2021-11-05 13:42:29 -0700 | [diff] [blame] | 1254 | |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 1255 | if (reason == VMSCAN_THROTTLE_WRITEBACK) |
Mel Gorman | d818fca | 2021-11-05 13:42:29 -0700 | [diff] [blame] | 1256 | atomic_dec(&pgdat->nr_writeback_throttled); |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1257 | |
| 1258 | trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout), |
| 1259 | jiffies_to_usecs(timeout - ret), |
| 1260 | reason); |
| 1261 | } |
| 1262 | |
| 1263 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1264 | * Account for folios written if tasks are throttled waiting on dirty |
| 1265 | * folios to clean. If enough folios have been cleaned since throttling |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1266 | * started then wakeup the throttled tasks. |
| 1267 | */ |
Linus Torvalds | 512b7931 | 2021-11-06 14:08:17 -0700 | [diff] [blame] | 1268 | void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1269 | int nr_throttled) |
| 1270 | { |
| 1271 | unsigned long nr_written; |
| 1272 | |
Linus Torvalds | 512b7931 | 2021-11-06 14:08:17 -0700 | [diff] [blame] | 1273 | node_stat_add_folio(folio, NR_THROTTLED_WRITTEN); |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1274 | |
| 1275 | /* |
| 1276 | * This is an inaccurate read as the per-cpu deltas may not |
| 1277 | * be synchronised. However, given that the system is |
| 1278 | * writeback throttled, it is not worth taking the penalty |
| 1279 | * of getting an accurate count. At worst, the throttle |
| 1280 | * timeout guarantees forward progress. |
| 1281 | */ |
| 1282 | nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) - |
| 1283 | READ_ONCE(pgdat->nr_reclaim_start); |
| 1284 | |
| 1285 | if (nr_written > SWAP_CLUSTER_MAX * nr_throttled) |
| 1286 | wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]); |
| 1287 | } |
| 1288 | |
Christoph Lameter | 04e62a2 | 2006-06-23 02:03:38 -0700 | [diff] [blame] | 1289 | /* possible outcome of pageout() */ |
| 1290 | typedef enum { |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1291 | /* failed to write folio out, folio is locked */ |
Christoph Lameter | 04e62a2 | 2006-06-23 02:03:38 -0700 | [diff] [blame] | 1292 | PAGE_KEEP, |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1293 | /* move folio to the active list, folio is locked */ |
Christoph Lameter | 04e62a2 | 2006-06-23 02:03:38 -0700 | [diff] [blame] | 1294 | PAGE_ACTIVATE, |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1295 | /* folio has been sent to the disk successfully, folio is unlocked */ |
Christoph Lameter | 04e62a2 | 2006-06-23 02:03:38 -0700 | [diff] [blame] | 1296 | PAGE_SUCCESS, |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1297 | /* folio is clean and locked */ |
Christoph Lameter | 04e62a2 | 2006-06-23 02:03:38 -0700 | [diff] [blame] | 1298 | PAGE_CLEAN, |
| 1299 | } pageout_t; |
| 1300 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1301 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1302 | * pageout is called by shrink_folio_list() for each dirty folio. |
Andrew Morton | 1742f19 | 2006-03-22 00:08:21 -0800 | [diff] [blame] | 1303 | * Calls ->writepage(). |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1304 | */ |
NeilBrown | 2282679 | 2022-05-09 18:20:49 -0700 | [diff] [blame] | 1305 | static pageout_t pageout(struct folio *folio, struct address_space *mapping, |
| 1306 | struct swap_iocb **plug) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1307 | { |
| 1308 | /* |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1309 | * If the folio is dirty, only perform writeback if that write |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1310 | * will be non-blocking. To prevent this allocation from being |
| 1311 | * stalled by pagecache activity. But note that there may be |
| 1312 | * stalls if we need to run get_block(). We could test |
| 1313 | * PagePrivate for that. |
| 1314 | * |
Al Viro | 8174202 | 2014-04-03 03:17:43 -0400 | [diff] [blame] | 1315 | * If this process is currently in __generic_file_write_iter() against |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1316 | * this folio's queue, we can perform writeback even if that |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1317 | * will block. |
| 1318 | * |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1319 | * If the folio is swapcache, write it back even if that would |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1320 | * block, for some throttling. This happens by accident, because |
| 1321 | * swap_backing_dev_info is bust: it doesn't reflect the |
| 1322 | * congestion state of the swapdevs. Easy to fix, if needed. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1323 | */ |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1324 | if (!is_page_cache_freeable(folio)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1325 | return PAGE_KEEP; |
| 1326 | if (!mapping) { |
| 1327 | /* |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1328 | * Some data journaling orphaned folios can have |
| 1329 | * folio->mapping == NULL while being dirty with clean buffers. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1330 | */ |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1331 | if (folio_test_private(folio)) { |
Matthew Wilcox (Oracle) | 68189fe | 2022-05-01 01:08:08 -0400 | [diff] [blame] | 1332 | if (try_to_free_buffers(folio)) { |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1333 | folio_clear_dirty(folio); |
| 1334 | pr_info("%s: orphaned folio\n", __func__); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1335 | return PAGE_CLEAN; |
| 1336 | } |
| 1337 | } |
| 1338 | return PAGE_KEEP; |
| 1339 | } |
| 1340 | if (mapping->a_ops->writepage == NULL) |
| 1341 | return PAGE_ACTIVATE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1342 | |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1343 | if (folio_clear_dirty_for_io(folio)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1344 | int res; |
| 1345 | struct writeback_control wbc = { |
| 1346 | .sync_mode = WB_SYNC_NONE, |
| 1347 | .nr_to_write = SWAP_CLUSTER_MAX, |
OGAWA Hirofumi | 111ebb6 | 2006-06-23 02:03:26 -0700 | [diff] [blame] | 1348 | .range_start = 0, |
| 1349 | .range_end = LLONG_MAX, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1350 | .for_reclaim = 1, |
NeilBrown | 2282679 | 2022-05-09 18:20:49 -0700 | [diff] [blame] | 1351 | .swap_plug = plug, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1352 | }; |
| 1353 | |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1354 | folio_set_reclaim(folio); |
| 1355 | res = mapping->a_ops->writepage(&folio->page, &wbc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1356 | if (res < 0) |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1357 | handle_write_error(mapping, folio, res); |
Zach Brown | 994fc28c | 2005-12-15 14:28:17 -0800 | [diff] [blame] | 1358 | if (res == AOP_WRITEPAGE_ACTIVATE) { |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1359 | folio_clear_reclaim(folio); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1360 | return PAGE_ACTIVATE; |
| 1361 | } |
Andy Whitcroft | c661b07 | 2007-08-22 14:01:26 -0700 | [diff] [blame] | 1362 | |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1363 | if (!folio_test_writeback(folio)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1364 | /* synchronous write or broken a_ops? */ |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1365 | folio_clear_reclaim(folio); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1366 | } |
Matthew Wilcox (Oracle) | e0cd5e7 | 2022-01-17 23:35:57 -0500 | [diff] [blame] | 1367 | trace_mm_vmscan_write_folio(folio); |
| 1368 | node_stat_add_folio(folio, NR_VMSCAN_WRITE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1369 | return PAGE_SUCCESS; |
| 1370 | } |
| 1371 | |
| 1372 | return PAGE_CLEAN; |
| 1373 | } |
| 1374 | |
Andrew Morton | a649fd9 | 2006-10-17 00:09:36 -0700 | [diff] [blame] | 1375 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1376 | * Same as remove_mapping, but if the folio is removed from the mapping, it |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1377 | * gets returned with a refcount of 0. |
Andrew Morton | a649fd9 | 2006-10-17 00:09:36 -0700 | [diff] [blame] | 1378 | */ |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1379 | static int __remove_mapping(struct address_space *mapping, struct folio *folio, |
Johannes Weiner | b910718 | 2019-11-30 17:55:59 -0800 | [diff] [blame] | 1380 | bool reclaimed, struct mem_cgroup *target_memcg) |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1381 | { |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1382 | int refcount; |
Joonsoo Kim | aae466b | 2020-08-11 18:30:50 -0700 | [diff] [blame] | 1383 | void *shadow = NULL; |
Greg Thelen | c4843a7 | 2015-05-22 17:13:16 -0400 | [diff] [blame] | 1384 | |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1385 | BUG_ON(!folio_test_locked(folio)); |
| 1386 | BUG_ON(mapping != folio_mapping(folio)); |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1387 | |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1388 | if (!folio_test_swapcache(folio)) |
Johannes Weiner | 51b8c1f | 2021-11-08 18:31:24 -0800 | [diff] [blame] | 1389 | spin_lock(&mapping->host->i_lock); |
Johannes Weiner | 3047250 | 2021-09-02 14:53:18 -0700 | [diff] [blame] | 1390 | xa_lock_irq(&mapping->i_pages); |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1391 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1392 | * The non racy check for a busy folio. |
Nick Piggin | 0fd0e6b | 2006-09-27 01:50:02 -0700 | [diff] [blame] | 1393 | * |
| 1394 | * Must be careful with the order of the tests. When someone has |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1395 | * a ref to the folio, it may be possible that they dirty it then |
| 1396 | * drop the reference. So if the dirty flag is tested before the |
| 1397 | * refcount here, then the following race may occur: |
Nick Piggin | 0fd0e6b | 2006-09-27 01:50:02 -0700 | [diff] [blame] | 1398 | * |
| 1399 | * get_user_pages(&page); |
| 1400 | * [user mapping goes away] |
| 1401 | * write_to(page); |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1402 | * !folio_test_dirty(folio) [good] |
| 1403 | * folio_set_dirty(folio); |
| 1404 | * folio_put(folio); |
| 1405 | * !refcount(folio) [good, discard it] |
Nick Piggin | 0fd0e6b | 2006-09-27 01:50:02 -0700 | [diff] [blame] | 1406 | * |
| 1407 | * [oops, our write_to data is lost] |
| 1408 | * |
| 1409 | * Reversing the order of the tests ensures such a situation cannot |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1410 | * escape unnoticed. The smp_rmb is needed to ensure the folio->flags |
| 1411 | * load is not satisfied before that of folio->_refcount. |
Nick Piggin | 0fd0e6b | 2006-09-27 01:50:02 -0700 | [diff] [blame] | 1412 | * |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1413 | * Note that if the dirty flag is always set via folio_mark_dirty, |
Matthew Wilcox | b93b016 | 2018-04-10 16:36:56 -0700 | [diff] [blame] | 1414 | * and thus under the i_pages lock, then this ordering is not required. |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1415 | */ |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1416 | refcount = 1 + folio_nr_pages(folio); |
| 1417 | if (!folio_ref_freeze(folio, refcount)) |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1418 | goto cannot_free; |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1419 | /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */ |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1420 | if (unlikely(folio_test_dirty(folio))) { |
| 1421 | folio_ref_unfreeze(folio, refcount); |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1422 | goto cannot_free; |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1423 | } |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1424 | |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1425 | if (folio_test_swapcache(folio)) { |
David Hildenbrand | 3d2c908 | 2023-08-21 18:08:48 +0200 | [diff] [blame] | 1426 | swp_entry_t swap = folio->swap; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 1427 | |
Joonsoo Kim | aae466b | 2020-08-11 18:30:50 -0700 | [diff] [blame] | 1428 | if (reclaimed && !mapping_exiting(mapping)) |
Matthew Wilcox (Oracle) | 8927f64 | 2021-12-23 16:39:05 -0500 | [diff] [blame] | 1429 | shadow = workingset_eviction(folio, target_memcg); |
Matthew Wilcox (Oracle) | ceff9d3 | 2022-06-17 18:50:20 +0100 | [diff] [blame] | 1430 | __delete_from_swap_cache(folio, swap, shadow); |
Hugh Dickins | c449deb | 2022-12-04 17:01:03 -0800 | [diff] [blame] | 1431 | mem_cgroup_swapout(folio, swap); |
Johannes Weiner | 3047250 | 2021-09-02 14:53:18 -0700 | [diff] [blame] | 1432 | xa_unlock_irq(&mapping->i_pages); |
Matthew Wilcox (Oracle) | 4081f74 | 2022-09-02 20:46:09 +0100 | [diff] [blame] | 1433 | put_swap_folio(folio, swap); |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1434 | } else { |
Matthew Wilcox (Oracle) | d2329aa | 2022-05-01 07:35:31 -0400 | [diff] [blame] | 1435 | void (*free_folio)(struct folio *); |
Linus Torvalds | 6072d13 | 2010-12-01 13:35:19 -0500 | [diff] [blame] | 1436 | |
Matthew Wilcox (Oracle) | d2329aa | 2022-05-01 07:35:31 -0400 | [diff] [blame] | 1437 | free_folio = mapping->a_ops->free_folio; |
Johannes Weiner | a528910 | 2014-04-03 14:47:51 -0700 | [diff] [blame] | 1438 | /* |
| 1439 | * Remember a shadow entry for reclaimed file cache in |
| 1440 | * order to detect refaults, thus thrashing, later on. |
| 1441 | * |
| 1442 | * But don't store shadows in an address space that is |
dylan-meiners | 238c304 | 2020-08-06 23:26:29 -0700 | [diff] [blame] | 1443 | * already exiting. This is not just an optimization, |
Johannes Weiner | a528910 | 2014-04-03 14:47:51 -0700 | [diff] [blame] | 1444 | * inode reclaim needs to empty out the radix tree or |
| 1445 | * the nodes are lost. Don't plant shadows behind its |
| 1446 | * back. |
Ross Zwisler | f9fe48b | 2016-01-22 15:10:40 -0800 | [diff] [blame] | 1447 | * |
| 1448 | * We also don't store shadows for DAX mappings because the |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1449 | * only page cache folios found in these are zero pages |
Ross Zwisler | f9fe48b | 2016-01-22 15:10:40 -0800 | [diff] [blame] | 1450 | * covering holes, and because we don't want to mix DAX |
| 1451 | * exceptional entries and shadow exceptional entries in the |
Matthew Wilcox | b93b016 | 2018-04-10 16:36:56 -0700 | [diff] [blame] | 1452 | * same address_space. |
Johannes Weiner | a528910 | 2014-04-03 14:47:51 -0700 | [diff] [blame] | 1453 | */ |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1454 | if (reclaimed && folio_is_file_lru(folio) && |
Ross Zwisler | f9fe48b | 2016-01-22 15:10:40 -0800 | [diff] [blame] | 1455 | !mapping_exiting(mapping) && !dax_mapping(mapping)) |
Matthew Wilcox (Oracle) | 8927f64 | 2021-12-23 16:39:05 -0500 | [diff] [blame] | 1456 | shadow = workingset_eviction(folio, target_memcg); |
| 1457 | __filemap_remove_folio(folio, shadow); |
Johannes Weiner | 3047250 | 2021-09-02 14:53:18 -0700 | [diff] [blame] | 1458 | xa_unlock_irq(&mapping->i_pages); |
Johannes Weiner | 51b8c1f | 2021-11-08 18:31:24 -0800 | [diff] [blame] | 1459 | if (mapping_shrinkable(mapping)) |
| 1460 | inode_add_lru(mapping->host); |
| 1461 | spin_unlock(&mapping->host->i_lock); |
Linus Torvalds | 6072d13 | 2010-12-01 13:35:19 -0500 | [diff] [blame] | 1462 | |
Matthew Wilcox (Oracle) | d2329aa | 2022-05-01 07:35:31 -0400 | [diff] [blame] | 1463 | if (free_folio) |
| 1464 | free_folio(folio); |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1465 | } |
| 1466 | |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1467 | return 1; |
| 1468 | |
| 1469 | cannot_free: |
Johannes Weiner | 3047250 | 2021-09-02 14:53:18 -0700 | [diff] [blame] | 1470 | xa_unlock_irq(&mapping->i_pages); |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1471 | if (!folio_test_swapcache(folio)) |
Johannes Weiner | 51b8c1f | 2021-11-08 18:31:24 -0800 | [diff] [blame] | 1472 | spin_unlock(&mapping->host->i_lock); |
Christoph Lameter | 49d2e9c | 2006-01-08 01:00:48 -0800 | [diff] [blame] | 1473 | return 0; |
| 1474 | } |
| 1475 | |
Matthew Wilcox (Oracle) | 5100da3 | 2022-02-12 22:48:55 -0500 | [diff] [blame] | 1476 | /** |
| 1477 | * remove_mapping() - Attempt to remove a folio from its mapping. |
| 1478 | * @mapping: The address space. |
| 1479 | * @folio: The folio to remove. |
| 1480 | * |
| 1481 | * If the folio is dirty, under writeback or if someone else has a ref |
| 1482 | * on it, removal will fail. |
| 1483 | * Return: The number of pages removed from the mapping. 0 if the folio |
| 1484 | * could not be removed. |
| 1485 | * Context: The caller should have a single refcount on the folio and |
| 1486 | * hold its lock. |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1487 | */ |
Matthew Wilcox (Oracle) | 5100da3 | 2022-02-12 22:48:55 -0500 | [diff] [blame] | 1488 | long remove_mapping(struct address_space *mapping, struct folio *folio) |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1489 | { |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1490 | if (__remove_mapping(mapping, folio, false, NULL)) { |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1491 | /* |
Matthew Wilcox (Oracle) | 5100da3 | 2022-02-12 22:48:55 -0500 | [diff] [blame] | 1492 | * Unfreezing the refcount with 1 effectively |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1493 | * drops the pagecache ref for us without requiring another |
| 1494 | * atomic operation. |
| 1495 | */ |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1496 | folio_ref_unfreeze(folio, 1); |
Matthew Wilcox (Oracle) | 5100da3 | 2022-02-12 22:48:55 -0500 | [diff] [blame] | 1497 | return folio_nr_pages(folio); |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1498 | } |
| 1499 | return 0; |
| 1500 | } |
| 1501 | |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 1502 | /** |
Matthew Wilcox (Oracle) | ca6d60f3 | 2022-01-21 08:41:46 -0500 | [diff] [blame] | 1503 | * folio_putback_lru - Put previously isolated folio onto appropriate LRU list. |
| 1504 | * @folio: Folio to be returned to an LRU list. |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 1505 | * |
Matthew Wilcox (Oracle) | ca6d60f3 | 2022-01-21 08:41:46 -0500 | [diff] [blame] | 1506 | * Add previously isolated @folio to appropriate LRU list. |
| 1507 | * The folio may still be unevictable for other reasons. |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 1508 | * |
Matthew Wilcox (Oracle) | ca6d60f3 | 2022-01-21 08:41:46 -0500 | [diff] [blame] | 1509 | * Context: lru_lock must not be held, interrupts must be enabled. |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 1510 | */ |
Matthew Wilcox (Oracle) | ca6d60f3 | 2022-01-21 08:41:46 -0500 | [diff] [blame] | 1511 | void folio_putback_lru(struct folio *folio) |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 1512 | { |
Matthew Wilcox (Oracle) | ca6d60f3 | 2022-01-21 08:41:46 -0500 | [diff] [blame] | 1513 | folio_add_lru(folio); |
| 1514 | folio_put(folio); /* drop ref from isolate */ |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 1515 | } |
| 1516 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1517 | enum folio_references { |
| 1518 | FOLIOREF_RECLAIM, |
| 1519 | FOLIOREF_RECLAIM_CLEAN, |
| 1520 | FOLIOREF_KEEP, |
| 1521 | FOLIOREF_ACTIVATE, |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1522 | }; |
| 1523 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1524 | static enum folio_references folio_check_references(struct folio *folio, |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1525 | struct scan_control *sc) |
| 1526 | { |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1527 | int referenced_ptes, referenced_folio; |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1528 | unsigned long vm_flags; |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1529 | |
Matthew Wilcox (Oracle) | b3ac041 | 2022-01-21 11:27:31 -0500 | [diff] [blame] | 1530 | referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, |
| 1531 | &vm_flags); |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1532 | referenced_folio = folio_test_clear_referenced(folio); |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1533 | |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1534 | /* |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1535 | * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. |
| 1536 | * Let the folio, now marked Mlocked, be moved to the unevictable list. |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1537 | */ |
| 1538 | if (vm_flags & VM_LOCKED) |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1539 | return FOLIOREF_ACTIVATE; |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1540 | |
Minchan Kim | 6d4675e | 2022-05-19 14:08:54 -0700 | [diff] [blame] | 1541 | /* rmap lock contention: rotate */ |
| 1542 | if (referenced_ptes == -1) |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1543 | return FOLIOREF_KEEP; |
Minchan Kim | 6d4675e | 2022-05-19 14:08:54 -0700 | [diff] [blame] | 1544 | |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1545 | if (referenced_ptes) { |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1546 | /* |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1547 | * All mapped folios start out with page table |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1548 | * references from the instantiating fault, so we need |
Linus Torvalds | 9030fb0 | 2022-03-22 17:03:12 -0700 | [diff] [blame] | 1549 | * to look twice if a mapped file/anon folio is used more |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1550 | * than once. |
| 1551 | * |
| 1552 | * Mark it and spare it for another trip around the |
| 1553 | * inactive list. Another page table reference will |
| 1554 | * lead to its activation. |
| 1555 | * |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1556 | * Note: the mark is set for activated folios as well |
| 1557 | * so that recently deactivated but used folios are |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1558 | * quickly recovered. |
| 1559 | */ |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1560 | folio_set_referenced(folio); |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1561 | |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1562 | if (referenced_folio || referenced_ptes > 1) |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1563 | return FOLIOREF_ACTIVATE; |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1564 | |
Konstantin Khlebnikov | c909e99 | 2012-01-10 15:07:03 -0800 | [diff] [blame] | 1565 | /* |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1566 | * Activate file-backed executable folios after first usage. |
Konstantin Khlebnikov | c909e99 | 2012-01-10 15:07:03 -0800 | [diff] [blame] | 1567 | */ |
Miaohe Lin | f19a27e | 2022-05-12 20:23:00 -0700 | [diff] [blame] | 1568 | if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1569 | return FOLIOREF_ACTIVATE; |
Konstantin Khlebnikov | c909e99 | 2012-01-10 15:07:03 -0800 | [diff] [blame] | 1570 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1571 | return FOLIOREF_KEEP; |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1572 | } |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1573 | |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1574 | /* Reclaim if clean, defer dirty folios to writeback */ |
Miaohe Lin | f19a27e | 2022-05-12 20:23:00 -0700 | [diff] [blame] | 1575 | if (referenced_folio && folio_is_file_lru(folio)) |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1576 | return FOLIOREF_RECLAIM_CLEAN; |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1577 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1578 | return FOLIOREF_RECLAIM; |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1579 | } |
| 1580 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1581 | /* Check if a folio is dirty or under writeback */ |
Matthew Wilcox (Oracle) | e20c41b | 2022-01-17 14:35:22 -0500 | [diff] [blame] | 1582 | static void folio_check_dirty_writeback(struct folio *folio, |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1583 | bool *dirty, bool *writeback) |
| 1584 | { |
Mel Gorman | b459722 | 2013-07-03 15:02:05 -0700 | [diff] [blame] | 1585 | struct address_space *mapping; |
| 1586 | |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1587 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1588 | * Anonymous folios are not handled by flushers and must be written |
Miaohe Lin | 32a331a | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 1589 | * from reclaim context. Do not stall reclaim based on them. |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1590 | * MADV_FREE anonymous folios are put into inactive file list too. |
Miaohe Lin | 32a331a | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 1591 | * They could be mistakenly treated as file lru. So further anon |
| 1592 | * test is needed. |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1593 | */ |
Matthew Wilcox (Oracle) | e20c41b | 2022-01-17 14:35:22 -0500 | [diff] [blame] | 1594 | if (!folio_is_file_lru(folio) || |
| 1595 | (folio_test_anon(folio) && !folio_test_swapbacked(folio))) { |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1596 | *dirty = false; |
| 1597 | *writeback = false; |
| 1598 | return; |
| 1599 | } |
| 1600 | |
Matthew Wilcox (Oracle) | e20c41b | 2022-01-17 14:35:22 -0500 | [diff] [blame] | 1601 | /* By default assume that the folio flags are accurate */ |
| 1602 | *dirty = folio_test_dirty(folio); |
| 1603 | *writeback = folio_test_writeback(folio); |
Mel Gorman | b459722 | 2013-07-03 15:02:05 -0700 | [diff] [blame] | 1604 | |
| 1605 | /* Verify dirty/writeback state if the filesystem supports it */ |
Matthew Wilcox (Oracle) | e20c41b | 2022-01-17 14:35:22 -0500 | [diff] [blame] | 1606 | if (!folio_test_private(folio)) |
Mel Gorman | b459722 | 2013-07-03 15:02:05 -0700 | [diff] [blame] | 1607 | return; |
| 1608 | |
Matthew Wilcox (Oracle) | e20c41b | 2022-01-17 14:35:22 -0500 | [diff] [blame] | 1609 | mapping = folio_mapping(folio); |
Mel Gorman | b459722 | 2013-07-03 15:02:05 -0700 | [diff] [blame] | 1610 | if (mapping && mapping->a_ops->is_dirty_writeback) |
Matthew Wilcox (Oracle) | 520f301 | 2022-01-17 14:35:22 -0500 | [diff] [blame] | 1611 | mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1612 | } |
| 1613 | |
Matthew Wilcox (Oracle) | 4e096ae | 2023-05-13 01:11:01 +0100 | [diff] [blame] | 1614 | static struct folio *alloc_demote_folio(struct folio *src, |
| 1615 | unsigned long private) |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1616 | { |
Matthew Wilcox (Oracle) | 4e096ae | 2023-05-13 01:11:01 +0100 | [diff] [blame] | 1617 | struct folio *dst; |
Jagdish Gediya | 3200802 | 2022-08-18 18:40:40 +0530 | [diff] [blame] | 1618 | nodemask_t *allowed_mask; |
| 1619 | struct migration_target_control *mtc; |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1620 | |
Jagdish Gediya | 3200802 | 2022-08-18 18:40:40 +0530 | [diff] [blame] | 1621 | mtc = (struct migration_target_control *)private; |
| 1622 | |
| 1623 | allowed_mask = mtc->nmask; |
| 1624 | /* |
| 1625 | * make sure we allocate from the target node first also trying to |
| 1626 | * demote or reclaim pages from the target node via kswapd if we are |
| 1627 | * low on free memory on target node. If we don't do this and if |
| 1628 | * we have free memory on the slower(lower) memtier, we would start |
| 1629 | * allocating pages from slower(lower) memory tiers without even forcing |
| 1630 | * a demotion of cold pages from the target memtier. This can result |
| 1631 | * in the kernel placing hot pages in slower(lower) memory tiers. |
| 1632 | */ |
| 1633 | mtc->nmask = NULL; |
| 1634 | mtc->gfp_mask |= __GFP_THISNODE; |
Matthew Wilcox (Oracle) | 4e096ae | 2023-05-13 01:11:01 +0100 | [diff] [blame] | 1635 | dst = alloc_migration_target(src, (unsigned long)mtc); |
| 1636 | if (dst) |
| 1637 | return dst; |
Jagdish Gediya | 3200802 | 2022-08-18 18:40:40 +0530 | [diff] [blame] | 1638 | |
| 1639 | mtc->gfp_mask &= ~__GFP_THISNODE; |
| 1640 | mtc->nmask = allowed_mask; |
| 1641 | |
Matthew Wilcox (Oracle) | 4e096ae | 2023-05-13 01:11:01 +0100 | [diff] [blame] | 1642 | return alloc_migration_target(src, (unsigned long)mtc); |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1643 | } |
| 1644 | |
| 1645 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1646 | * Take folios on @demote_folios and attempt to demote them to another node. |
| 1647 | * Folios which are not demoted are left on @demote_folios. |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1648 | */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1649 | static unsigned int demote_folio_list(struct list_head *demote_folios, |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1650 | struct pglist_data *pgdat) |
| 1651 | { |
| 1652 | int target_nid = next_demotion_node(pgdat->node_id); |
| 1653 | unsigned int nr_succeeded; |
Jagdish Gediya | 3200802 | 2022-08-18 18:40:40 +0530 | [diff] [blame] | 1654 | nodemask_t allowed_mask; |
| 1655 | |
| 1656 | struct migration_target_control mtc = { |
| 1657 | /* |
| 1658 | * Allocate from 'node', or fail quickly and quietly. |
| 1659 | * When this happens, 'page' will likely just be discarded |
| 1660 | * instead of migrated. |
| 1661 | */ |
| 1662 | .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | |
| 1663 | __GFP_NOMEMALLOC | GFP_NOWAIT, |
| 1664 | .nid = target_nid, |
| 1665 | .nmask = &allowed_mask |
| 1666 | }; |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1667 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1668 | if (list_empty(demote_folios)) |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1669 | return 0; |
| 1670 | |
| 1671 | if (target_nid == NUMA_NO_NODE) |
| 1672 | return 0; |
| 1673 | |
Jagdish Gediya | 3200802 | 2022-08-18 18:40:40 +0530 | [diff] [blame] | 1674 | node_get_allowed_targets(pgdat, &allowed_mask); |
| 1675 | |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1676 | /* Demotion ignores all cpuset and mempolicy settings */ |
Matthew Wilcox (Oracle) | 4e096ae | 2023-05-13 01:11:01 +0100 | [diff] [blame] | 1677 | migrate_pages(demote_folios, alloc_demote_folio, NULL, |
Jagdish Gediya | 3200802 | 2022-08-18 18:40:40 +0530 | [diff] [blame] | 1678 | (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, |
| 1679 | &nr_succeeded); |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1680 | |
Johannes Weiner | 57e9cc5 | 2022-10-26 14:01:33 -0400 | [diff] [blame] | 1681 | __count_vm_events(PGDEMOTE_KSWAPD + reclaimer_offset(), nr_succeeded); |
Yang Shi | 668e414 | 2021-09-02 14:59:19 -0700 | [diff] [blame] | 1682 | |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1683 | return nr_succeeded; |
| 1684 | } |
| 1685 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1686 | static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) |
NeilBrown | d791ea6 | 2022-05-09 18:20:48 -0700 | [diff] [blame] | 1687 | { |
| 1688 | if (gfp_mask & __GFP_FS) |
| 1689 | return true; |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1690 | if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) |
NeilBrown | d791ea6 | 2022-05-09 18:20:48 -0700 | [diff] [blame] | 1691 | return false; |
| 1692 | /* |
| 1693 | * We can "enter_fs" for swap-cache with only __GFP_IO |
| 1694 | * providing this isn't SWP_FS_OPS. |
| 1695 | * ->flags can be updated non-atomicially (scan_swap_map_slots), |
| 1696 | * but that will never affect SWP_FS_OPS, so the data_race |
| 1697 | * is safe. |
| 1698 | */ |
Matthew Wilcox (Oracle) | b98c359 | 2022-06-17 18:50:18 +0100 | [diff] [blame] | 1699 | return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); |
NeilBrown | d791ea6 | 2022-05-09 18:20:48 -0700 | [diff] [blame] | 1700 | } |
| 1701 | |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 1702 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1703 | * shrink_folio_list() returns the number of reclaimed pages |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1704 | */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1705 | static unsigned int shrink_folio_list(struct list_head *folio_list, |
| 1706 | struct pglist_data *pgdat, struct scan_control *sc, |
| 1707 | struct reclaim_stat *stat, bool ignore_references) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1708 | { |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1709 | LIST_HEAD(ret_folios); |
| 1710 | LIST_HEAD(free_folios); |
| 1711 | LIST_HEAD(demote_folios); |
Maninder Singh | 730ec8c | 2020-06-03 16:01:18 -0700 | [diff] [blame] | 1712 | unsigned int nr_reclaimed = 0; |
| 1713 | unsigned int pgactivate = 0; |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1714 | bool do_demote_pass; |
NeilBrown | 2282679 | 2022-05-09 18:20:49 -0700 | [diff] [blame] | 1715 | struct swap_iocb *plug = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1716 | |
Kirill Tkhai | 060f005 | 2019-03-05 15:48:15 -0800 | [diff] [blame] | 1717 | memset(stat, 0, sizeof(*stat)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1718 | cond_resched(); |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1719 | do_demote_pass = can_demote(pgdat->node_id, sc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1720 | |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1721 | retry: |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1722 | while (!list_empty(folio_list)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1723 | struct address_space *mapping; |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1724 | struct folio *folio; |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1725 | enum folio_references references = FOLIOREF_RECLAIM; |
NeilBrown | d791ea6 | 2022-05-09 18:20:48 -0700 | [diff] [blame] | 1726 | bool dirty, writeback; |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1727 | unsigned int nr_pages; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1728 | |
| 1729 | cond_resched(); |
| 1730 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1731 | folio = lru_to_folio(folio_list); |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 1732 | list_del(&folio->lru); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1733 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1734 | if (!folio_trylock(folio)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1735 | goto keep; |
| 1736 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1737 | VM_BUG_ON_FOLIO(folio_test_active(folio), folio); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1738 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1739 | nr_pages = folio_nr_pages(folio); |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1740 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1741 | /* Account the number of base pages */ |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1742 | sc->nr_scanned += nr_pages; |
Christoph Lameter | 80e4342 | 2006-02-11 17:55:53 -0800 | [diff] [blame] | 1743 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1744 | if (unlikely(!folio_evictable(folio))) |
Minchan Kim | ad6b670 | 2017-05-03 14:54:13 -0700 | [diff] [blame] | 1745 | goto activate_locked; |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 1746 | |
Matthew Wilcox (Oracle) | 1bee2c1 | 2022-05-12 20:23:01 -0700 | [diff] [blame] | 1747 | if (!sc->may_unmap && folio_mapped(folio)) |
Christoph Lameter | 80e4342 | 2006-02-11 17:55:53 -0800 | [diff] [blame] | 1748 | goto keep_locked; |
| 1749 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 1750 | /* folio_update_gen() tried to promote this page? */ |
| 1751 | if (lru_gen_enabled() && !ignore_references && |
| 1752 | folio_mapped(folio) && folio_test_referenced(folio)) |
| 1753 | goto keep_locked; |
| 1754 | |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1755 | /* |
Andrey Ryabinin | 894befe | 2018-04-10 16:27:51 -0700 | [diff] [blame] | 1756 | * The number of dirty pages determines if a node is marked |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 1757 | * reclaim_congested. kswapd will stall and start writing |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1758 | * folios if the tail of the LRU is all dirty unqueued folios. |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1759 | */ |
Matthew Wilcox (Oracle) | e20c41b | 2022-01-17 14:35:22 -0500 | [diff] [blame] | 1760 | folio_check_dirty_writeback(folio, &dirty, &writeback); |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1761 | if (dirty || writeback) |
Matthew Wilcox (Oracle) | c79b7b9 | 2022-01-17 14:46:55 -0500 | [diff] [blame] | 1762 | stat->nr_dirty += nr_pages; |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1763 | |
| 1764 | if (dirty && !writeback) |
Matthew Wilcox (Oracle) | c79b7b9 | 2022-01-17 14:46:55 -0500 | [diff] [blame] | 1765 | stat->nr_unqueued_dirty += nr_pages; |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1766 | |
Mel Gorman | d04e8ac | 2013-07-03 15:02:03 -0700 | [diff] [blame] | 1767 | /* |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1768 | * Treat this folio as congested if folios are cycling |
| 1769 | * through the LRU so quickly that the folios marked |
| 1770 | * for immediate reclaim are making it to the end of |
| 1771 | * the LRU a second time. |
Mel Gorman | d04e8ac | 2013-07-03 15:02:03 -0700 | [diff] [blame] | 1772 | */ |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1773 | if (writeback && folio_test_reclaim(folio)) |
Matthew Wilcox (Oracle) | c79b7b9 | 2022-01-17 14:46:55 -0500 | [diff] [blame] | 1774 | stat->nr_congested += nr_pages; |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1775 | |
| 1776 | /* |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1777 | * If a folio at the tail of the LRU is under writeback, there |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1778 | * are three cases to consider. |
| 1779 | * |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1780 | * 1) If reclaim is encountering an excessive number |
| 1781 | * of folios under writeback and this folio has both |
| 1782 | * the writeback and reclaim flags set, then it |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1783 | * indicates that folios are being queued for I/O but |
| 1784 | * are being recycled through the LRU before the I/O |
| 1785 | * can complete. Waiting on the folio itself risks an |
| 1786 | * indefinite stall if it is impossible to writeback |
| 1787 | * the folio due to I/O error or disconnected storage |
| 1788 | * so instead note that the LRU is being scanned too |
| 1789 | * quickly and the caller can stall after the folio |
| 1790 | * list has been processed. |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1791 | * |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1792 | * 2) Global or new memcg reclaim encounters a folio that is |
Michal Hocko | ecf5fc6 | 2015-08-04 14:36:58 -0700 | [diff] [blame] | 1793 | * not marked for immediate reclaim, or the caller does not |
| 1794 | * have __GFP_FS (or __GFP_IO if it's simply going to swap, |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1795 | * not to fs). In this case mark the folio for immediate |
Tejun Heo | 97c9341f | 2015-05-22 18:23:36 -0400 | [diff] [blame] | 1796 | * reclaim and continue scanning. |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1797 | * |
NeilBrown | d791ea6 | 2022-05-09 18:20:48 -0700 | [diff] [blame] | 1798 | * Require may_enter_fs() because we would wait on fs, which |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1799 | * may not have submitted I/O yet. And the loop driver might |
| 1800 | * enter reclaim, and deadlock if it waits on a folio for |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1801 | * which it is needed to do the write (loop masks off |
| 1802 | * __GFP_IO|__GFP_FS for this reason); but more thought |
| 1803 | * would probably show more reasons. |
| 1804 | * |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1805 | * 3) Legacy memcg encounters a folio that already has the |
| 1806 | * reclaim flag set. memcg does not have any dirty folio |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1807 | * throttling so we could easily OOM just because too many |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1808 | * folios are in writeback and there is nothing else to |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1809 | * reclaim. Wait for the writeback to complete. |
Johannes Weiner | c55e8d0 | 2017-02-24 14:56:23 -0800 | [diff] [blame] | 1810 | * |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1811 | * In cases 1) and 2) we activate the folios to get them out of |
| 1812 | * the way while we continue scanning for clean folios on the |
Johannes Weiner | c55e8d0 | 2017-02-24 14:56:23 -0800 | [diff] [blame] | 1813 | * inactive list and refilling from the active list. The |
| 1814 | * observation here is that waiting for disk writes is more |
| 1815 | * expensive than potentially causing reloads down the line. |
| 1816 | * Since they're marked for immediate reclaim, they won't put |
| 1817 | * memory pressure on the cache working set any longer than it |
| 1818 | * takes to write them to disk. |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1819 | */ |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1820 | if (folio_test_writeback(folio)) { |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1821 | /* Case 1 above */ |
| 1822 | if (current_is_kswapd() && |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1823 | folio_test_reclaim(folio) && |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 1824 | test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { |
Matthew Wilcox (Oracle) | c79b7b9 | 2022-01-17 14:46:55 -0500 | [diff] [blame] | 1825 | stat->nr_immediate += nr_pages; |
Johannes Weiner | c55e8d0 | 2017-02-24 14:56:23 -0800 | [diff] [blame] | 1826 | goto activate_locked; |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1827 | |
| 1828 | /* Case 2 above */ |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 1829 | } else if (writeback_throttling_sane(sc) || |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1830 | !folio_test_reclaim(folio) || |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1831 | !may_enter_fs(folio, sc->gfp_mask)) { |
Hugh Dickins | c3b94f4 | 2012-07-31 16:45:59 -0700 | [diff] [blame] | 1832 | /* |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1833 | * This is slightly racy - |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1834 | * folio_end_writeback() might have |
| 1835 | * just cleared the reclaim flag, then |
| 1836 | * setting the reclaim flag here ends up |
| 1837 | * interpreted as the readahead flag - but |
| 1838 | * that does not matter enough to care. |
| 1839 | * What we do want is for this folio to |
| 1840 | * have the reclaim flag set next time |
| 1841 | * memcg reclaim reaches the tests above, |
| 1842 | * so it will then wait for writeback to |
| 1843 | * avoid OOM; and it's also appropriate |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1844 | * in global reclaim. |
Hugh Dickins | c3b94f4 | 2012-07-31 16:45:59 -0700 | [diff] [blame] | 1845 | */ |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1846 | folio_set_reclaim(folio); |
Matthew Wilcox (Oracle) | c79b7b9 | 2022-01-17 14:46:55 -0500 | [diff] [blame] | 1847 | stat->nr_writeback += nr_pages; |
Johannes Weiner | c55e8d0 | 2017-02-24 14:56:23 -0800 | [diff] [blame] | 1848 | goto activate_locked; |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 1849 | |
| 1850 | /* Case 3 above */ |
| 1851 | } else { |
Matthew Wilcox (Oracle) | d33e4e1 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1852 | folio_unlock(folio); |
| 1853 | folio_wait_writeback(folio); |
| 1854 | /* then go back and try same folio again */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1855 | list_add_tail(&folio->lru, folio_list); |
Hugh Dickins | 7fadc82 | 2015-09-08 15:03:46 -0700 | [diff] [blame] | 1856 | continue; |
Michal Hocko | e62e384 | 2012-07-31 16:45:55 -0700 | [diff] [blame] | 1857 | } |
Andy Whitcroft | c661b07 | 2007-08-22 14:01:26 -0700 | [diff] [blame] | 1858 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1859 | |
Minchan Kim | 8940b34 | 2019-09-25 16:49:11 -0700 | [diff] [blame] | 1860 | if (!ignore_references) |
Matthew Wilcox (Oracle) | d92013d | 2022-02-15 13:44:40 -0500 | [diff] [blame] | 1861 | references = folio_check_references(folio, sc); |
Minchan Kim | 02c6de8 | 2012-10-08 16:31:55 -0700 | [diff] [blame] | 1862 | |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1863 | switch (references) { |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1864 | case FOLIOREF_ACTIVATE: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1865 | goto activate_locked; |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1866 | case FOLIOREF_KEEP: |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1867 | stat->nr_ref_keep += nr_pages; |
Johannes Weiner | 64574746 | 2010-03-05 13:42:22 -0800 | [diff] [blame] | 1868 | goto keep_locked; |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1869 | case FOLIOREF_RECLAIM: |
| 1870 | case FOLIOREF_RECLAIM_CLEAN: |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1871 | ; /* try to reclaim the folio below */ |
Johannes Weiner | dfc8d63 | 2010-03-05 13:42:19 -0800 | [diff] [blame] | 1872 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1873 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1874 | /* |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1875 | * Before reclaiming the folio, try to relocate |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1876 | * its contents to another node. |
| 1877 | */ |
| 1878 | if (do_demote_pass && |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1879 | (thp_migration_supported() || !folio_test_large(folio))) { |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1880 | list_add(&folio->lru, &demote_folios); |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1881 | folio_unlock(folio); |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 1882 | continue; |
| 1883 | } |
| 1884 | |
| 1885 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1886 | * Anonymous process memory has backing store? |
| 1887 | * Try to allocate it some swap space here. |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1888 | * Lazyfree folio could be freed directly |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1889 | */ |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1890 | if (folio_test_anon(folio) && folio_test_swapbacked(folio)) { |
| 1891 | if (!folio_test_swapcache(folio)) { |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1892 | if (!(sc->gfp_mask & __GFP_IO)) |
| 1893 | goto keep_locked; |
Matthew Wilcox (Oracle) | d4b4084 | 2022-02-04 14:13:31 -0500 | [diff] [blame] | 1894 | if (folio_maybe_dma_pinned(folio)) |
Linus Torvalds | feb889f | 2021-01-16 15:34:57 -0800 | [diff] [blame] | 1895 | goto keep_locked; |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1896 | if (folio_test_large(folio)) { |
| 1897 | /* cannot split folio, skip it */ |
Matthew Wilcox (Oracle) | d4b4084 | 2022-02-04 14:13:31 -0500 | [diff] [blame] | 1898 | if (!can_split_folio(folio, NULL)) |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1899 | goto activate_locked; |
| 1900 | /* |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1901 | * Split folios without a PMD map right |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1902 | * away. Chances are some or all of the |
| 1903 | * tail pages can be freed without IO. |
| 1904 | */ |
Matthew Wilcox (Oracle) | d4b4084 | 2022-02-04 14:13:31 -0500 | [diff] [blame] | 1905 | if (!folio_entire_mapcount(folio) && |
Matthew Wilcox (Oracle) | 346cf613 | 2022-01-18 08:56:47 -0500 | [diff] [blame] | 1906 | split_folio_to_list(folio, |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1907 | folio_list)) |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1908 | goto activate_locked; |
| 1909 | } |
Matthew Wilcox (Oracle) | 09c02e5 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1910 | if (!add_to_swap(folio)) { |
| 1911 | if (!folio_test_large(folio)) |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1912 | goto activate_locked_split; |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1913 | /* Fallback to swap normal pages */ |
Matthew Wilcox (Oracle) | 346cf613 | 2022-01-18 08:56:47 -0500 | [diff] [blame] | 1914 | if (split_folio_to_list(folio, |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1915 | folio_list)) |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1916 | goto activate_locked; |
Huang Ying | fe490cc | 2017-09-06 16:22:52 -0700 | [diff] [blame] | 1917 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 1918 | count_vm_event(THP_SWPOUT_FALLBACK); |
| 1919 | #endif |
Matthew Wilcox (Oracle) | 09c02e5 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1920 | if (!add_to_swap(folio)) |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1921 | goto activate_locked_split; |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1922 | } |
Minchan Kim | 0f07465 | 2017-07-06 15:37:24 -0700 | [diff] [blame] | 1923 | } |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1924 | } else if (folio_test_swapbacked(folio) && |
| 1925 | folio_test_large(folio)) { |
| 1926 | /* Split shmem folio */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 1927 | if (split_folio_to_list(folio, folio_list)) |
Kirill A. Shutemov | 7751b2d | 2016-07-26 15:25:56 -0700 | [diff] [blame] | 1928 | goto keep_locked; |
Mel Gorman | e2be15f | 2013-07-03 15:01:57 -0700 | [diff] [blame] | 1929 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1930 | |
| 1931 | /* |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1932 | * If the folio was split above, the tail pages will make |
| 1933 | * their own pass through this function and be accounted |
| 1934 | * then. |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1935 | */ |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1936 | if ((nr_pages > 1) && !folio_test_large(folio)) { |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1937 | sc->nr_scanned -= (nr_pages - 1); |
| 1938 | nr_pages = 1; |
| 1939 | } |
| 1940 | |
| 1941 | /* |
Matthew Wilcox (Oracle) | 1bee2c1 | 2022-05-12 20:23:01 -0700 | [diff] [blame] | 1942 | * The folio is mapped into the page tables of one or more |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1943 | * processes. Try to unmap it here. |
| 1944 | */ |
Matthew Wilcox (Oracle) | 1bee2c1 | 2022-05-12 20:23:01 -0700 | [diff] [blame] | 1945 | if (folio_mapped(folio)) { |
Shakeel Butt | 013339d | 2020-12-14 19:06:39 -0800 | [diff] [blame] | 1946 | enum ttu_flags flags = TTU_BATCH_FLUSH; |
Matthew Wilcox (Oracle) | 1bee2c1 | 2022-05-12 20:23:01 -0700 | [diff] [blame] | 1947 | bool was_swapbacked = folio_test_swapbacked(folio); |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1948 | |
Matthew Wilcox (Oracle) | 1bee2c1 | 2022-05-12 20:23:01 -0700 | [diff] [blame] | 1949 | if (folio_test_pmd_mappable(folio)) |
Huang Ying | bd4c82c2 | 2017-09-06 16:22:49 -0700 | [diff] [blame] | 1950 | flags |= TTU_SPLIT_HUGE_PMD; |
Jaewon Kim | 1f318a9b | 2020-06-03 16:01:15 -0700 | [diff] [blame] | 1951 | |
Matthew Wilcox (Oracle) | 869f7ee | 2022-02-15 09:28:49 -0500 | [diff] [blame] | 1952 | try_to_unmap(folio, flags); |
Matthew Wilcox (Oracle) | 1bee2c1 | 2022-05-12 20:23:01 -0700 | [diff] [blame] | 1953 | if (folio_mapped(folio)) { |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 1954 | stat->nr_unmap_fail += nr_pages; |
Matthew Wilcox (Oracle) | 1bee2c1 | 2022-05-12 20:23:01 -0700 | [diff] [blame] | 1955 | if (!was_swapbacked && |
| 1956 | folio_test_swapbacked(folio)) |
Jaewon Kim | 1f318a9b | 2020-06-03 16:01:15 -0700 | [diff] [blame] | 1957 | stat->nr_lazyfree_fail += nr_pages; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1958 | goto activate_locked; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1959 | } |
| 1960 | } |
| 1961 | |
Jan Kara | d824ec2 | 2023-04-28 14:41:40 +0200 | [diff] [blame] | 1962 | /* |
| 1963 | * Folio is unmapped now so it cannot be newly pinned anymore. |
| 1964 | * No point in trying to reclaim folio if it is pinned. |
| 1965 | * Furthermore we don't want to reclaim underlying fs metadata |
| 1966 | * if the folio is pinned and thus potentially modified by the |
| 1967 | * pinning process as that may upset the filesystem. |
| 1968 | */ |
| 1969 | if (folio_maybe_dma_pinned(folio)) |
| 1970 | goto activate_locked; |
| 1971 | |
Matthew Wilcox (Oracle) | 5441d49 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 1972 | mapping = folio_mapping(folio); |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1973 | if (folio_test_dirty(folio)) { |
Mel Gorman | ee72886 | 2011-10-31 17:07:38 -0700 | [diff] [blame] | 1974 | /* |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1975 | * Only kswapd can writeback filesystem folios |
Johannes Weiner | 4eda482 | 2017-02-24 14:56:20 -0800 | [diff] [blame] | 1976 | * to avoid risk of stack overflow. But avoid |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1977 | * injecting inefficient single-folio I/O into |
Johannes Weiner | 4eda482 | 2017-02-24 14:56:20 -0800 | [diff] [blame] | 1978 | * flusher writeback as much as possible: only |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1979 | * write folios when we've encountered many |
| 1980 | * dirty folios, and when we've already scanned |
| 1981 | * the rest of the LRU for clean folios and see |
| 1982 | * the same dirty folios again (with the reclaim |
| 1983 | * flag set). |
Mel Gorman | ee72886 | 2011-10-31 17:07:38 -0700 | [diff] [blame] | 1984 | */ |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1985 | if (folio_is_file_lru(folio) && |
| 1986 | (!current_is_kswapd() || |
| 1987 | !folio_test_reclaim(folio) || |
Johannes Weiner | 4eda482 | 2017-02-24 14:56:20 -0800 | [diff] [blame] | 1988 | !test_bit(PGDAT_DIRTY, &pgdat->flags))) { |
Mel Gorman | 49ea7eb | 2011-10-31 17:07:59 -0700 | [diff] [blame] | 1989 | /* |
| 1990 | * Immediately reclaim when written back. |
Vishal Moola (Oracle) | 5a9e347 | 2022-12-21 10:08:48 -0800 | [diff] [blame] | 1991 | * Similar in principle to folio_deactivate() |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1992 | * except we already have the folio isolated |
Mel Gorman | 49ea7eb | 2011-10-31 17:07:59 -0700 | [diff] [blame] | 1993 | * and know it's dirty |
| 1994 | */ |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 1995 | node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE, |
| 1996 | nr_pages); |
| 1997 | folio_set_reclaim(folio); |
Mel Gorman | 49ea7eb | 2011-10-31 17:07:59 -0700 | [diff] [blame] | 1998 | |
Johannes Weiner | c55e8d0 | 2017-02-24 14:56:23 -0800 | [diff] [blame] | 1999 | goto activate_locked; |
Mel Gorman | ee72886 | 2011-10-31 17:07:38 -0700 | [diff] [blame] | 2000 | } |
| 2001 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2002 | if (references == FOLIOREF_RECLAIM_CLEAN) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2003 | goto keep_locked; |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2004 | if (!may_enter_fs(folio, sc->gfp_mask)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2005 | goto keep_locked; |
Christoph Lameter | 52a8363 | 2006-02-01 03:05:28 -0800 | [diff] [blame] | 2006 | if (!sc->may_writepage) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2007 | goto keep_locked; |
| 2008 | |
Mel Gorman | d950c94 | 2015-09-04 15:47:35 -0700 | [diff] [blame] | 2009 | /* |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2010 | * Folio is dirty. Flush the TLB if a writable entry |
| 2011 | * potentially exists to avoid CPU writes after I/O |
Mel Gorman | d950c94 | 2015-09-04 15:47:35 -0700 | [diff] [blame] | 2012 | * starts and then write it out here. |
| 2013 | */ |
| 2014 | try_to_unmap_flush_dirty(); |
NeilBrown | 2282679 | 2022-05-09 18:20:49 -0700 | [diff] [blame] | 2015 | switch (pageout(folio, mapping, &plug)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2016 | case PAGE_KEEP: |
| 2017 | goto keep_locked; |
| 2018 | case PAGE_ACTIVATE: |
| 2019 | goto activate_locked; |
| 2020 | case PAGE_SUCCESS: |
Matthew Wilcox (Oracle) | c79b7b9 | 2022-01-17 14:46:55 -0500 | [diff] [blame] | 2021 | stat->nr_pageout += nr_pages; |
Johannes Weiner | 96f8bf4 | 2020-06-03 16:03:09 -0700 | [diff] [blame] | 2022 | |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2023 | if (folio_test_writeback(folio)) |
Mel Gorman | 41ac199 | 2012-05-29 15:06:19 -0700 | [diff] [blame] | 2024 | goto keep; |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2025 | if (folio_test_dirty(folio)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2026 | goto keep; |
KOSAKI Motohiro | 7d3579e | 2010-10-26 14:21:42 -0700 | [diff] [blame] | 2027 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2028 | /* |
| 2029 | * A synchronous write - probably a ramdisk. Go |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2030 | * ahead and try to reclaim the folio. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2031 | */ |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2032 | if (!folio_trylock(folio)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2033 | goto keep; |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2034 | if (folio_test_dirty(folio) || |
| 2035 | folio_test_writeback(folio)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2036 | goto keep_locked; |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2037 | mapping = folio_mapping(folio); |
Gustavo A. R. Silva | 01359eb | 2020-12-14 19:15:00 -0800 | [diff] [blame] | 2038 | fallthrough; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2039 | case PAGE_CLEAN: |
Matthew Wilcox (Oracle) | 49bd2bf | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2040 | ; /* try to free the folio below */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2041 | } |
| 2042 | } |
| 2043 | |
| 2044 | /* |
Matthew Wilcox (Oracle) | 0a36111 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2045 | * If the folio has buffers, try to free the buffer |
| 2046 | * mappings associated with this folio. If we succeed |
| 2047 | * we try to free the folio as well. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2048 | * |
Matthew Wilcox (Oracle) | 0a36111 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2049 | * We do this even if the folio is dirty. |
| 2050 | * filemap_release_folio() does not perform I/O, but it |
| 2051 | * is possible for a folio to have the dirty flag set, |
| 2052 | * but it is actually clean (all its buffers are clean). |
| 2053 | * This happens if the buffers were written out directly, |
| 2054 | * with submit_bh(). ext3 will do this, as well as |
| 2055 | * the blockdev mapping. filemap_release_folio() will |
| 2056 | * discover that cleanness and will drop the buffers |
| 2057 | * and mark the folio clean - it can be freed. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2058 | * |
Matthew Wilcox (Oracle) | 0a36111 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2059 | * Rarely, folios can have buffers and no ->mapping. |
| 2060 | * These are the folios which were not successfully |
| 2061 | * invalidated in truncate_cleanup_folio(). We try to |
| 2062 | * drop those buffers here and if that worked, and the |
| 2063 | * folio is no longer mapped into process address space |
| 2064 | * (refcount == 1) it can be freed. Otherwise, leave |
| 2065 | * the folio on the LRU so it is swappable. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2066 | */ |
David Howells | 0201ebf | 2023-06-28 11:48:51 +0100 | [diff] [blame] | 2067 | if (folio_needs_release(folio)) { |
Matthew Wilcox (Oracle) | 0a36111 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2068 | if (!filemap_release_folio(folio, sc->gfp_mask)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2069 | goto activate_locked; |
Matthew Wilcox (Oracle) | 0a36111 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2070 | if (!mapping && folio_ref_count(folio) == 1) { |
| 2071 | folio_unlock(folio); |
| 2072 | if (folio_put_testzero(folio)) |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 2073 | goto free_it; |
| 2074 | else { |
| 2075 | /* |
| 2076 | * rare race with speculative reference. |
| 2077 | * the speculative reference will free |
Matthew Wilcox (Oracle) | 0a36111 | 2022-05-12 20:23:02 -0700 | [diff] [blame] | 2078 | * this folio shortly, so we may |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 2079 | * increment nr_reclaimed here (and |
| 2080 | * leave it off the LRU). |
| 2081 | */ |
Miaohe Lin | 9aafcff | 2022-05-12 20:23:00 -0700 | [diff] [blame] | 2082 | nr_reclaimed += nr_pages; |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 2083 | continue; |
| 2084 | } |
| 2085 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2086 | } |
| 2087 | |
Matthew Wilcox (Oracle) | 64daa5d | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2088 | if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) { |
Shaohua Li | 802a3a9 | 2017-05-03 14:52:32 -0700 | [diff] [blame] | 2089 | /* follow __remove_mapping for reference */ |
Matthew Wilcox (Oracle) | 64daa5d | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2090 | if (!folio_ref_freeze(folio, 1)) |
Shaohua Li | 802a3a9 | 2017-05-03 14:52:32 -0700 | [diff] [blame] | 2091 | goto keep_locked; |
Miaohe Lin | d17be2d | 2021-09-02 14:59:39 -0700 | [diff] [blame] | 2092 | /* |
Matthew Wilcox (Oracle) | 64daa5d | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2093 | * The folio has only one reference left, which is |
Miaohe Lin | d17be2d | 2021-09-02 14:59:39 -0700 | [diff] [blame] | 2094 | * from the isolation. After the caller puts the |
Matthew Wilcox (Oracle) | 64daa5d | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2095 | * folio back on the lru and drops the reference, the |
| 2096 | * folio will be freed anyway. It doesn't matter |
| 2097 | * which lru it goes on. So we don't bother checking |
| 2098 | * the dirty flag here. |
Miaohe Lin | d17be2d | 2021-09-02 14:59:39 -0700 | [diff] [blame] | 2099 | */ |
Matthew Wilcox (Oracle) | 64daa5d | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2100 | count_vm_events(PGLAZYFREED, nr_pages); |
| 2101 | count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); |
Matthew Wilcox (Oracle) | be7c07d | 2021-12-23 16:50:23 -0500 | [diff] [blame] | 2102 | } else if (!mapping || !__remove_mapping(mapping, folio, true, |
Johannes Weiner | b910718 | 2019-11-30 17:55:59 -0800 | [diff] [blame] | 2103 | sc->target_mem_cgroup)) |
Shaohua Li | 802a3a9 | 2017-05-03 14:52:32 -0700 | [diff] [blame] | 2104 | goto keep_locked; |
Hugh Dickins | 9a1ea43 | 2018-12-28 00:36:14 -0800 | [diff] [blame] | 2105 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2106 | folio_unlock(folio); |
Nick Piggin | e286781 | 2008-07-25 19:45:30 -0700 | [diff] [blame] | 2107 | free_it: |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2108 | /* |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2109 | * Folio may get swapped out as a whole, need to account |
| 2110 | * all pages in it. |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2111 | */ |
| 2112 | nr_reclaimed += nr_pages; |
Mel Gorman | abe4c3b | 2010-08-09 17:19:31 -0700 | [diff] [blame] | 2113 | |
| 2114 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2115 | * Is there need to periodically free_folio_list? It would |
Mel Gorman | abe4c3b | 2010-08-09 17:19:31 -0700 | [diff] [blame] | 2116 | * appear not as the counts should be low |
| 2117 | */ |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2118 | if (unlikely(folio_test_large(folio))) |
Matthew Wilcox (Oracle) | 5375336 | 2022-06-17 18:50:17 +0100 | [diff] [blame] | 2119 | destroy_large_folio(folio); |
Yang Shi | 7ae8853 | 2019-09-23 15:38:09 -0700 | [diff] [blame] | 2120 | else |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2121 | list_add(&folio->lru, &free_folios); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2122 | continue; |
| 2123 | |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2124 | activate_locked_split: |
| 2125 | /* |
| 2126 | * The tail pages that are failed to add into swap cache |
| 2127 | * reach here. Fixup nr_scanned and nr_pages. |
| 2128 | */ |
| 2129 | if (nr_pages > 1) { |
| 2130 | sc->nr_scanned -= (nr_pages - 1); |
| 2131 | nr_pages = 1; |
| 2132 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2133 | activate_locked: |
Rik van Riel | 68a22394 | 2008-10-18 20:26:23 -0700 | [diff] [blame] | 2134 | /* Not a candidate for swapping, so reclaim swap space. */ |
Matthew Wilcox (Oracle) | 246b648 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2135 | if (folio_test_swapcache(folio) && |
Matthew Wilcox (Oracle) | 9202d527b | 2022-09-02 20:46:43 +0100 | [diff] [blame] | 2136 | (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio))) |
Matthew Wilcox (Oracle) | bdb0ed5 | 2022-09-02 20:46:06 +0100 | [diff] [blame] | 2137 | folio_free_swap(folio); |
Matthew Wilcox (Oracle) | 246b648 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2138 | VM_BUG_ON_FOLIO(folio_test_active(folio), folio); |
| 2139 | if (!folio_test_mlocked(folio)) { |
| 2140 | int type = folio_is_file_lru(folio); |
| 2141 | folio_set_active(folio); |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2142 | stat->nr_activate[type] += nr_pages; |
Matthew Wilcox (Oracle) | 246b648 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2143 | count_memcg_folio_events(folio, PGACTIVATE, nr_pages); |
Minchan Kim | ad6b670 | 2017-05-03 14:54:13 -0700 | [diff] [blame] | 2144 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2145 | keep_locked: |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2146 | folio_unlock(folio); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2147 | keep: |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2148 | list_add(&folio->lru, &ret_folios); |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2149 | VM_BUG_ON_FOLIO(folio_test_lru(folio) || |
| 2150 | folio_test_unevictable(folio), folio); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2151 | } |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2152 | /* 'folio_list' is always empty here */ |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 2153 | |
Matthew Wilcox (Oracle) | c28a0e9 | 2022-05-12 20:23:03 -0700 | [diff] [blame] | 2154 | /* Migrate folios selected for demotion */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2155 | nr_reclaimed += demote_folio_list(&demote_folios, pgdat); |
| 2156 | /* Folios that could not be demoted are still in @demote_folios */ |
| 2157 | if (!list_empty(&demote_folios)) { |
Mina Almasry | 6b426d0 | 2022-12-01 15:33:17 -0800 | [diff] [blame] | 2158 | /* Folios which weren't demoted go back on @folio_list */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2159 | list_splice_init(&demote_folios, folio_list); |
Mina Almasry | 6b426d0 | 2022-12-01 15:33:17 -0800 | [diff] [blame] | 2160 | |
| 2161 | /* |
| 2162 | * goto retry to reclaim the undemoted folios in folio_list if |
| 2163 | * desired. |
| 2164 | * |
| 2165 | * Reclaiming directly from top tier nodes is not often desired |
| 2166 | * due to it breaking the LRU ordering: in general memory |
| 2167 | * should be reclaimed from lower tier nodes and demoted from |
| 2168 | * top tier nodes. |
| 2169 | * |
| 2170 | * However, disabling reclaim from top tier nodes entirely |
| 2171 | * would cause ooms in edge scenarios where lower tier memory |
| 2172 | * is unreclaimable for whatever reason, eg memory being |
| 2173 | * mlocked or too hot to reclaim. We can disable reclaim |
| 2174 | * from top tier nodes in proactive reclaim though as that is |
| 2175 | * not real memory pressure. |
| 2176 | */ |
| 2177 | if (!sc->proactive) { |
| 2178 | do_demote_pass = false; |
| 2179 | goto retry; |
| 2180 | } |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 2181 | } |
Mel Gorman | abe4c3b | 2010-08-09 17:19:31 -0700 | [diff] [blame] | 2182 | |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2183 | pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; |
| 2184 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2185 | mem_cgroup_uncharge_list(&free_folios); |
Mel Gorman | 72b252a | 2015-09-04 15:47:32 -0700 | [diff] [blame] | 2186 | try_to_unmap_flush(); |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2187 | free_unref_page_list(&free_folios); |
Mel Gorman | abe4c3b | 2010-08-09 17:19:31 -0700 | [diff] [blame] | 2188 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2189 | list_splice(&ret_folios, folio_list); |
Kirill Tkhai | 886cf19 | 2019-05-13 17:16:51 -0700 | [diff] [blame] | 2190 | count_vm_events(PGACTIVATE, pgactivate); |
Johannes Weiner | 0a31bc9 | 2014-08-08 14:19:22 -0700 | [diff] [blame] | 2191 | |
NeilBrown | 2282679 | 2022-05-09 18:20:49 -0700 | [diff] [blame] | 2192 | if (plug) |
| 2193 | swap_write_unplug(plug); |
Andrew Morton | 05ff513 | 2006-03-22 00:08:20 -0800 | [diff] [blame] | 2194 | return nr_reclaimed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2195 | } |
| 2196 | |
Maninder Singh | 730ec8c | 2020-06-03 16:01:18 -0700 | [diff] [blame] | 2197 | unsigned int reclaim_clean_pages_from_list(struct zone *zone, |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2198 | struct list_head *folio_list) |
Minchan Kim | 02c6de8 | 2012-10-08 16:31:55 -0700 | [diff] [blame] | 2199 | { |
| 2200 | struct scan_control sc = { |
| 2201 | .gfp_mask = GFP_KERNEL, |
Minchan Kim | 02c6de8 | 2012-10-08 16:31:55 -0700 | [diff] [blame] | 2202 | .may_unmap = 1, |
| 2203 | }; |
Jaewon Kim | 1f318a9b | 2020-06-03 16:01:15 -0700 | [diff] [blame] | 2204 | struct reclaim_stat stat; |
Maninder Singh | 730ec8c | 2020-06-03 16:01:18 -0700 | [diff] [blame] | 2205 | unsigned int nr_reclaimed; |
Matthew Wilcox (Oracle) | b8cecb9 | 2022-06-17 16:42:44 +0100 | [diff] [blame] | 2206 | struct folio *folio, *next; |
| 2207 | LIST_HEAD(clean_folios); |
Yu Zhao | 2d2b8d2 | 2021-06-30 18:49:48 -0700 | [diff] [blame] | 2208 | unsigned int noreclaim_flag; |
Minchan Kim | 02c6de8 | 2012-10-08 16:31:55 -0700 | [diff] [blame] | 2209 | |
Matthew Wilcox (Oracle) | b8cecb9 | 2022-06-17 16:42:44 +0100 | [diff] [blame] | 2210 | list_for_each_entry_safe(folio, next, folio_list, lru) { |
| 2211 | if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && |
| 2212 | !folio_test_dirty(folio) && !__folio_test_movable(folio) && |
| 2213 | !folio_test_unevictable(folio)) { |
| 2214 | folio_clear_active(folio); |
| 2215 | list_move(&folio->lru, &clean_folios); |
Minchan Kim | 02c6de8 | 2012-10-08 16:31:55 -0700 | [diff] [blame] | 2216 | } |
| 2217 | } |
| 2218 | |
Yu Zhao | 2d2b8d2 | 2021-06-30 18:49:48 -0700 | [diff] [blame] | 2219 | /* |
| 2220 | * We should be safe here since we are only dealing with file pages and |
| 2221 | * we are not kswapd and therefore cannot write dirty file pages. But |
| 2222 | * call memalloc_noreclaim_save() anyway, just in case these conditions |
| 2223 | * change in the future. |
| 2224 | */ |
| 2225 | noreclaim_flag = memalloc_noreclaim_save(); |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2226 | nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc, |
Shakeel Butt | 013339d | 2020-12-14 19:06:39 -0800 | [diff] [blame] | 2227 | &stat, true); |
Yu Zhao | 2d2b8d2 | 2021-06-30 18:49:48 -0700 | [diff] [blame] | 2228 | memalloc_noreclaim_restore(noreclaim_flag); |
| 2229 | |
Matthew Wilcox (Oracle) | b8cecb9 | 2022-06-17 16:42:44 +0100 | [diff] [blame] | 2230 | list_splice(&clean_folios, folio_list); |
Nicholas Piggin | 2da9f63 | 2020-11-13 22:51:46 -0800 | [diff] [blame] | 2231 | mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, |
| 2232 | -(long)nr_reclaimed); |
Jaewon Kim | 1f318a9b | 2020-06-03 16:01:15 -0700 | [diff] [blame] | 2233 | /* |
| 2234 | * Since lazyfree pages are isolated from file LRU from the beginning, |
| 2235 | * they will rotate back to anonymous LRU in the end if it failed to |
| 2236 | * discard so isolated count will be mismatched. |
| 2237 | * Compensate the isolated count for both LRU lists. |
| 2238 | */ |
| 2239 | mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, |
| 2240 | stat.nr_lazyfree_fail); |
| 2241 | mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, |
Nicholas Piggin | 2da9f63 | 2020-11-13 22:51:46 -0800 | [diff] [blame] | 2242 | -(long)stat.nr_lazyfree_fail); |
Jaewon Kim | 1f318a9b | 2020-06-03 16:01:15 -0700 | [diff] [blame] | 2243 | return nr_reclaimed; |
Minchan Kim | 02c6de8 | 2012-10-08 16:31:55 -0700 | [diff] [blame] | 2244 | } |
| 2245 | |
Andy Whitcroft | 5ad333e | 2007-07-17 04:03:16 -0700 | [diff] [blame] | 2246 | /* |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2247 | * Update LRU sizes after isolating pages. The LRU size updates must |
Ethon Paul | 55b65a5 | 2020-06-04 16:49:10 -0700 | [diff] [blame] | 2248 | * be complete before mem_cgroup_update_lru_size due to a sanity check. |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2249 | */ |
| 2250 | static __always_inline void update_lru_sizes(struct lruvec *lruvec, |
Michal Hocko | b4536f0c8 | 2017-01-10 16:58:04 -0800 | [diff] [blame] | 2251 | enum lru_list lru, unsigned long *nr_zone_taken) |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2252 | { |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2253 | int zid; |
| 2254 | |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2255 | for (zid = 0; zid < MAX_NR_ZONES; zid++) { |
| 2256 | if (!nr_zone_taken[zid]) |
| 2257 | continue; |
| 2258 | |
Wei Yang | a892cb6 | 2020-06-03 16:01:12 -0700 | [diff] [blame] | 2259 | update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2260 | } |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2261 | |
Mel Gorman | 7ee36a1 | 2016-07-28 15:47:17 -0700 | [diff] [blame] | 2262 | } |
| 2263 | |
Zhaoyang Huang | 5da226d | 2023-05-31 10:51:01 +0800 | [diff] [blame] | 2264 | #ifdef CONFIG_CMA |
| 2265 | /* |
| 2266 | * It is waste of effort to scan and reclaim CMA pages if it is not available |
| 2267 | * for current allocation context. Kswapd can not be enrolled as it can not |
| 2268 | * distinguish this scenario by using sc->gfp_mask = GFP_KERNEL |
| 2269 | */ |
| 2270 | static bool skip_cma(struct folio *folio, struct scan_control *sc) |
| 2271 | { |
| 2272 | return !current_is_kswapd() && |
| 2273 | gfp_migratetype(sc->gfp_mask) != MIGRATE_MOVABLE && |
| 2274 | get_pageblock_migratetype(&folio->page) == MIGRATE_CMA; |
| 2275 | } |
| 2276 | #else |
| 2277 | static bool skip_cma(struct folio *folio, struct scan_control *sc) |
| 2278 | { |
| 2279 | return false; |
| 2280 | } |
| 2281 | #endif |
| 2282 | |
Mel Gorman | f611fab | 2021-06-30 18:53:19 -0700 | [diff] [blame] | 2283 | /* |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2284 | * Isolating page from the lruvec to fill in @dst list by nr_to_scan times. |
| 2285 | * |
| 2286 | * lruvec->lru_lock is heavily contended. Some of the functions that |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2287 | * shrink the lists perform better by taking out a batch of pages |
| 2288 | * and working on them outside the LRU lock. |
| 2289 | * |
| 2290 | * For pagecache intensive workloads, this function is the hottest |
| 2291 | * spot in the kernel (apart from copy_*_user functions). |
| 2292 | * |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2293 | * Lru_lock must be held before calling this function. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2294 | * |
Minchan Kim | 791b48b | 2017-05-12 15:47:06 -0700 | [diff] [blame] | 2295 | * @nr_to_scan: The number of eligible pages to look through on the list. |
Konstantin Khlebnikov | 5dc3597 | 2012-05-29 15:06:58 -0700 | [diff] [blame] | 2296 | * @lruvec: The LRU vector to pull pages from. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2297 | * @dst: The temp list to put pages on to. |
Hugh Dickins | f626012 | 2012-01-12 17:20:06 -0800 | [diff] [blame] | 2298 | * @nr_scanned: The number of pages that were scanned. |
Rik van Riel | fe2c2a1 | 2012-03-21 16:33:51 -0700 | [diff] [blame] | 2299 | * @sc: The scan_control struct for this reclaim session |
Konstantin Khlebnikov | 3cb9945 | 2012-05-29 15:06:53 -0700 | [diff] [blame] | 2300 | * @lru: LRU list id for isolating |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2301 | * |
| 2302 | * returns how many pages were moved onto *@dst. |
| 2303 | */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2304 | static unsigned long isolate_lru_folios(unsigned long nr_to_scan, |
Konstantin Khlebnikov | 5dc3597 | 2012-05-29 15:06:58 -0700 | [diff] [blame] | 2305 | struct lruvec *lruvec, struct list_head *dst, |
Rik van Riel | fe2c2a1 | 2012-03-21 16:33:51 -0700 | [diff] [blame] | 2306 | unsigned long *nr_scanned, struct scan_control *sc, |
Kirill Tkhai | a9e7c39 | 2019-03-05 15:46:55 -0800 | [diff] [blame] | 2307 | enum lru_list lru) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2308 | { |
Hugh Dickins | 75b00af | 2012-05-29 15:07:09 -0700 | [diff] [blame] | 2309 | struct list_head *src = &lruvec->lists[lru]; |
Andrew Morton | 69e0594 | 2006-03-22 00:08:19 -0800 | [diff] [blame] | 2310 | unsigned long nr_taken = 0; |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2311 | unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 }; |
Mel Gorman | 7cc30fc | 2016-07-28 15:46:59 -0700 | [diff] [blame] | 2312 | unsigned long nr_skipped[MAX_NR_ZONES] = { 0, }; |
Johannes Weiner | 3db6581 | 2017-05-03 14:52:13 -0700 | [diff] [blame] | 2313 | unsigned long skipped = 0; |
Minchan Kim | 791b48b | 2017-05-12 15:47:06 -0700 | [diff] [blame] | 2314 | unsigned long scan, total_scan, nr_pages; |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2315 | LIST_HEAD(folios_skipped); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2316 | |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2317 | total_scan = 0; |
Minchan Kim | 791b48b | 2017-05-12 15:47:06 -0700 | [diff] [blame] | 2318 | scan = 0; |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2319 | while (scan < nr_to_scan && !list_empty(src)) { |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2320 | struct list_head *move_to = src; |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2321 | struct folio *folio; |
Andy Whitcroft | 5ad333e | 2007-07-17 04:03:16 -0700 | [diff] [blame] | 2322 | |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2323 | folio = lru_to_folio(src); |
| 2324 | prefetchw_prev_lru_folio(folio, src, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2325 | |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2326 | nr_pages = folio_nr_pages(folio); |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2327 | total_scan += nr_pages; |
| 2328 | |
Zhaoyang Huang | 5da226d | 2023-05-31 10:51:01 +0800 | [diff] [blame] | 2329 | if (folio_zonenum(folio) > sc->reclaim_idx || |
| 2330 | skip_cma(folio, sc)) { |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2331 | nr_skipped[folio_zonenum(folio)] += nr_pages; |
| 2332 | move_to = &folios_skipped; |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2333 | goto move; |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 2334 | } |
| 2335 | |
Minchan Kim | 791b48b | 2017-05-12 15:47:06 -0700 | [diff] [blame] | 2336 | /* |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2337 | * Do not count skipped folios because that makes the function |
| 2338 | * return with no isolated folios if the LRU mostly contains |
| 2339 | * ineligible folios. This causes the VM to not reclaim any |
| 2340 | * folios, triggering a premature OOM. |
| 2341 | * Account all pages in a folio. |
Minchan Kim | 791b48b | 2017-05-12 15:47:06 -0700 | [diff] [blame] | 2342 | */ |
Yang Shi | 98879b3 | 2019-07-11 20:59:30 -0700 | [diff] [blame] | 2343 | scan += nr_pages; |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2344 | |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2345 | if (!folio_test_lru(folio)) |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2346 | goto move; |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2347 | if (!sc->may_unmap && folio_mapped(folio)) |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2348 | goto move; |
| 2349 | |
Alex Shi | c2135f7 | 2021-02-24 12:08:01 -0800 | [diff] [blame] | 2350 | /* |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2351 | * Be careful not to clear the lru flag until after we're |
| 2352 | * sure the folio is not being freed elsewhere -- the |
| 2353 | * folio release code relies on it. |
Alex Shi | c2135f7 | 2021-02-24 12:08:01 -0800 | [diff] [blame] | 2354 | */ |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2355 | if (unlikely(!folio_try_get(folio))) |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2356 | goto move; |
Alex Shi | c2135f7 | 2021-02-24 12:08:01 -0800 | [diff] [blame] | 2357 | |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2358 | if (!folio_test_clear_lru(folio)) { |
| 2359 | /* Another thread is already isolating this folio */ |
| 2360 | folio_put(folio); |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2361 | goto move; |
Alex Shi | c2135f7 | 2021-02-24 12:08:01 -0800 | [diff] [blame] | 2362 | } |
| 2363 | |
| 2364 | nr_taken += nr_pages; |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2365 | nr_zone_taken[folio_zonenum(folio)] += nr_pages; |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2366 | move_to = dst; |
| 2367 | move: |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2368 | list_move(&folio->lru, move_to); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2369 | } |
| 2370 | |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 2371 | /* |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2372 | * Splice any skipped folios to the start of the LRU list. Note that |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 2373 | * this disrupts the LRU order when reclaiming for lower zones but |
| 2374 | * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2375 | * scanning would soon rescan the same folios to skip and waste lots |
Miaohe Lin | b2cb682 | 2022-04-28 23:16:04 -0700 | [diff] [blame] | 2376 | * of cpu cycles. |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 2377 | */ |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2378 | if (!list_empty(&folios_skipped)) { |
Mel Gorman | 7cc30fc | 2016-07-28 15:46:59 -0700 | [diff] [blame] | 2379 | int zid; |
| 2380 | |
Matthew Wilcox (Oracle) | 166e3d3 | 2022-06-17 16:42:45 +0100 | [diff] [blame] | 2381 | list_splice(&folios_skipped, src); |
Mel Gorman | 7cc30fc | 2016-07-28 15:46:59 -0700 | [diff] [blame] | 2382 | for (zid = 0; zid < MAX_NR_ZONES; zid++) { |
| 2383 | if (!nr_skipped[zid]) |
| 2384 | continue; |
| 2385 | |
| 2386 | __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]); |
Michal Hocko | 1265e3a | 2017-02-22 15:44:21 -0800 | [diff] [blame] | 2387 | skipped += nr_skipped[zid]; |
Mel Gorman | 7cc30fc | 2016-07-28 15:46:59 -0700 | [diff] [blame] | 2388 | } |
| 2389 | } |
Minchan Kim | 791b48b | 2017-05-12 15:47:06 -0700 | [diff] [blame] | 2390 | *nr_scanned = total_scan; |
Michal Hocko | 1265e3a | 2017-02-22 15:44:21 -0800 | [diff] [blame] | 2391 | trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, |
Hugh Dickins | 89f6c88 | 2022-03-22 14:45:41 -0700 | [diff] [blame] | 2392 | total_scan, skipped, nr_taken, |
| 2393 | sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru); |
Michal Hocko | b4536f0c8 | 2017-01-10 16:58:04 -0800 | [diff] [blame] | 2394 | update_lru_sizes(lruvec, lru, nr_zone_taken); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2395 | return nr_taken; |
| 2396 | } |
| 2397 | |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2398 | /** |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2399 | * folio_isolate_lru() - Try to isolate a folio from its LRU list. |
| 2400 | * @folio: Folio to isolate from its LRU list. |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2401 | * |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2402 | * Isolate a @folio from an LRU list and adjust the vmstat statistic |
| 2403 | * corresponding to whatever LRU list the folio was on. |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2404 | * |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2405 | * The folio will have its LRU flag cleared. If it was found on the |
| 2406 | * active list, it will have the Active flag set. If it was found on the |
| 2407 | * unevictable list, it will have the Unevictable flag set. These flags |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 2408 | * may need to be cleared by the caller before letting the page go. |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2409 | * |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2410 | * Context: |
Mike Rapoport | a5d09be | 2018-02-06 15:42:19 -0800 | [diff] [blame] | 2411 | * |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2412 | * (1) Must be called with an elevated refcount on the folio. This is a |
| 2413 | * fundamental difference from isolate_lru_folios() (which is called |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2414 | * without a stable reference). |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2415 | * (2) The lru_lock must not be held. |
| 2416 | * (3) Interrupts must be enabled. |
| 2417 | * |
Baolin Wang | be2d575 | 2023-02-15 18:39:34 +0800 | [diff] [blame] | 2418 | * Return: true if the folio was removed from an LRU list. |
| 2419 | * false if the folio was not on an LRU list. |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2420 | */ |
Baolin Wang | be2d575 | 2023-02-15 18:39:34 +0800 | [diff] [blame] | 2421 | bool folio_isolate_lru(struct folio *folio) |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2422 | { |
Baolin Wang | be2d575 | 2023-02-15 18:39:34 +0800 | [diff] [blame] | 2423 | bool ret = false; |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2424 | |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2425 | VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio); |
Konstantin Khlebnikov | 0c91731 | 2011-05-24 17:12:21 -0700 | [diff] [blame] | 2426 | |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2427 | if (folio_test_clear_lru(folio)) { |
Hugh Dickins | fa9add6 | 2012-05-29 15:07:09 -0700 | [diff] [blame] | 2428 | struct lruvec *lruvec; |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2429 | |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2430 | folio_get(folio); |
Matthew Wilcox (Oracle) | e809c3f | 2021-06-28 21:59:47 -0400 | [diff] [blame] | 2431 | lruvec = folio_lruvec_lock_irq(folio); |
Matthew Wilcox (Oracle) | d1d8a3b | 2021-12-24 13:26:22 -0500 | [diff] [blame] | 2432 | lruvec_del_folio(lruvec, folio); |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2433 | unlock_page_lruvec_irq(lruvec); |
Baolin Wang | be2d575 | 2023-02-15 18:39:34 +0800 | [diff] [blame] | 2434 | ret = true; |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2435 | } |
Alex Shi | d25b5bd | 2020-12-15 12:34:16 -0800 | [diff] [blame] | 2436 | |
Nick Piggin | 62695a8 | 2008-10-18 20:26:09 -0700 | [diff] [blame] | 2437 | return ret; |
| 2438 | } |
| 2439 | |
Andy Whitcroft | 5ad333e | 2007-07-17 04:03:16 -0700 | [diff] [blame] | 2440 | /* |
Fengguang Wu | d37dd5d | 2012-12-18 14:23:28 -0800 | [diff] [blame] | 2441 | * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and |
Xianting Tian | 178821b | 2019-11-30 17:56:05 -0800 | [diff] [blame] | 2442 | * then get rescheduled. When there are massive number of tasks doing page |
Fengguang Wu | d37dd5d | 2012-12-18 14:23:28 -0800 | [diff] [blame] | 2443 | * allocation, such sleeping direct reclaimers may keep piling up on each CPU, |
| 2444 | * the LRU list will go small and be scanned faster than necessary, leading to |
| 2445 | * unnecessary swapping, thrashing and OOM. |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2446 | */ |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2447 | static int too_many_isolated(struct pglist_data *pgdat, int file, |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2448 | struct scan_control *sc) |
| 2449 | { |
| 2450 | unsigned long inactive, isolated; |
Mel Gorman | d818fca | 2021-11-05 13:42:29 -0700 | [diff] [blame] | 2451 | bool too_many; |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2452 | |
| 2453 | if (current_is_kswapd()) |
| 2454 | return 0; |
| 2455 | |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 2456 | if (!writeback_throttling_sane(sc)) |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2457 | return 0; |
| 2458 | |
| 2459 | if (file) { |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2460 | inactive = node_page_state(pgdat, NR_INACTIVE_FILE); |
| 2461 | isolated = node_page_state(pgdat, NR_ISOLATED_FILE); |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2462 | } else { |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2463 | inactive = node_page_state(pgdat, NR_INACTIVE_ANON); |
| 2464 | isolated = node_page_state(pgdat, NR_ISOLATED_ANON); |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2465 | } |
| 2466 | |
Fengguang Wu | 3cf2384 | 2012-12-18 14:23:31 -0800 | [diff] [blame] | 2467 | /* |
| 2468 | * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they |
| 2469 | * won't get blocked by normal direct-reclaimers, forming a circular |
| 2470 | * deadlock. |
| 2471 | */ |
Kefeng Wang | 5221b5a | 2023-05-16 14:38:19 +0800 | [diff] [blame] | 2472 | if (gfp_has_io_fs(sc->gfp_mask)) |
Fengguang Wu | 3cf2384 | 2012-12-18 14:23:31 -0800 | [diff] [blame] | 2473 | inactive >>= 3; |
| 2474 | |
Mel Gorman | d818fca | 2021-11-05 13:42:29 -0700 | [diff] [blame] | 2475 | too_many = isolated > inactive; |
| 2476 | |
| 2477 | /* Wake up tasks throttled due to too_many_isolated. */ |
| 2478 | if (!too_many) |
| 2479 | wake_throttle_isolated(pgdat); |
| 2480 | |
| 2481 | return too_many; |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2482 | } |
| 2483 | |
Kirill Tkhai | a222f34 | 2019-05-13 17:17:00 -0700 | [diff] [blame] | 2484 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2485 | * move_folios_to_lru() moves folios from private @list to appropriate LRU list. |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2486 | * On return, @list is reused as a list of folios to be freed by the caller. |
Kirill Tkhai | a222f34 | 2019-05-13 17:17:00 -0700 | [diff] [blame] | 2487 | * |
| 2488 | * Returns the number of pages moved to the given lruvec. |
| 2489 | */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2490 | static unsigned int move_folios_to_lru(struct lruvec *lruvec, |
| 2491 | struct list_head *list) |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2492 | { |
Kirill Tkhai | a222f34 | 2019-05-13 17:17:00 -0700 | [diff] [blame] | 2493 | int nr_pages, nr_moved = 0; |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2494 | LIST_HEAD(folios_to_free); |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2495 | |
Kirill Tkhai | a222f34 | 2019-05-13 17:17:00 -0700 | [diff] [blame] | 2496 | while (!list_empty(list)) { |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2497 | struct folio *folio = lru_to_folio(list); |
| 2498 | |
| 2499 | VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); |
| 2500 | list_del(&folio->lru); |
| 2501 | if (unlikely(!folio_evictable(folio))) { |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2502 | spin_unlock_irq(&lruvec->lru_lock); |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2503 | folio_putback_lru(folio); |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2504 | spin_lock_irq(&lruvec->lru_lock); |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2505 | continue; |
| 2506 | } |
Hugh Dickins | fa9add6 | 2012-05-29 15:07:09 -0700 | [diff] [blame] | 2507 | |
Alex Shi | 3d06afa | 2020-12-15 12:33:37 -0800 | [diff] [blame] | 2508 | /* |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2509 | * The folio_set_lru needs to be kept here for list integrity. |
Alex Shi | 3d06afa | 2020-12-15 12:33:37 -0800 | [diff] [blame] | 2510 | * Otherwise: |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2511 | * #0 move_folios_to_lru #1 release_pages |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2512 | * if (!folio_put_testzero()) |
| 2513 | * if (folio_put_testzero()) |
| 2514 | * !lru //skip lru_lock |
| 2515 | * folio_set_lru() |
| 2516 | * list_add(&folio->lru,) |
| 2517 | * list_add(&folio->lru,) |
Alex Shi | 3d06afa | 2020-12-15 12:33:37 -0800 | [diff] [blame] | 2518 | */ |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2519 | folio_set_lru(folio); |
Kirill Tkhai | a222f34 | 2019-05-13 17:17:00 -0700 | [diff] [blame] | 2520 | |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2521 | if (unlikely(folio_put_testzero(folio))) { |
| 2522 | __folio_clear_lru_flags(folio); |
Hugh Dickins | 2bcf887 | 2012-01-12 17:19:56 -0800 | [diff] [blame] | 2523 | |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2524 | if (unlikely(folio_test_large(folio))) { |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2525 | spin_unlock_irq(&lruvec->lru_lock); |
Matthew Wilcox (Oracle) | 5375336 | 2022-06-17 18:50:17 +0100 | [diff] [blame] | 2526 | destroy_large_folio(folio); |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2527 | spin_lock_irq(&lruvec->lru_lock); |
Hugh Dickins | 2bcf887 | 2012-01-12 17:19:56 -0800 | [diff] [blame] | 2528 | } else |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2529 | list_add(&folio->lru, &folios_to_free); |
Alex Shi | 3d06afa | 2020-12-15 12:33:37 -0800 | [diff] [blame] | 2530 | |
| 2531 | continue; |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2532 | } |
Alex Shi | 3d06afa | 2020-12-15 12:33:37 -0800 | [diff] [blame] | 2533 | |
Alex Shi | afca915 | 2020-12-15 12:34:02 -0800 | [diff] [blame] | 2534 | /* |
| 2535 | * All pages were isolated from the same lruvec (and isolation |
| 2536 | * inhibits memcg migration). |
| 2537 | */ |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2538 | VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); |
| 2539 | lruvec_add_folio(lruvec, folio); |
| 2540 | nr_pages = folio_nr_pages(folio); |
Alex Shi | 3d06afa | 2020-12-15 12:33:37 -0800 | [diff] [blame] | 2541 | nr_moved += nr_pages; |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2542 | if (folio_test_active(folio)) |
Alex Shi | 3d06afa | 2020-12-15 12:33:37 -0800 | [diff] [blame] | 2543 | workingset_age_nonresident(lruvec, nr_pages); |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2544 | } |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2545 | |
Hugh Dickins | 3f79768 | 2012-01-12 17:20:07 -0800 | [diff] [blame] | 2546 | /* |
| 2547 | * To save our caller's stack, now use input list for pages to free. |
| 2548 | */ |
Matthew Wilcox (Oracle) | ff00a17 | 2022-06-17 16:42:46 +0100 | [diff] [blame] | 2549 | list_splice(&folios_to_free, list); |
Kirill Tkhai | a222f34 | 2019-05-13 17:17:00 -0700 | [diff] [blame] | 2550 | |
| 2551 | return nr_moved; |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2552 | } |
| 2553 | |
| 2554 | /* |
Miaohe Lin | 5829f7d | 2022-04-28 23:16:04 -0700 | [diff] [blame] | 2555 | * If a kernel thread (such as nfsd for loop-back mounts) services a backing |
| 2556 | * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case |
| 2557 | * we should not throttle. Otherwise it is safe to do so. |
NeilBrown | 399ba0b | 2014-06-04 16:07:42 -0700 | [diff] [blame] | 2558 | */ |
| 2559 | static int current_may_throttle(void) |
| 2560 | { |
NeilBrown | b9b1335 | 2022-03-22 14:39:10 -0700 | [diff] [blame] | 2561 | return !(current->flags & PF_LOCAL_THROTTLE); |
NeilBrown | 399ba0b | 2014-06-04 16:07:42 -0700 | [diff] [blame] | 2562 | } |
| 2563 | |
| 2564 | /* |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 2565 | * shrink_inactive_list() is a helper for shrink_node(). It returns the number |
Andrew Morton | 1742f19 | 2006-03-22 00:08:21 -0800 | [diff] [blame] | 2566 | * of reclaimed pages |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2567 | */ |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2568 | static unsigned long shrink_inactive_list(unsigned long nr_to_scan, |
| 2569 | struct lruvec *lruvec, struct scan_control *sc, |
| 2570 | enum lru_list lru) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2571 | { |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2572 | LIST_HEAD(folio_list); |
KOSAKI Motohiro | e247dbc | 2010-08-09 17:19:28 -0700 | [diff] [blame] | 2573 | unsigned long nr_scanned; |
Maninder Singh | 730ec8c | 2020-06-03 16:01:18 -0700 | [diff] [blame] | 2574 | unsigned int nr_reclaimed = 0; |
KOSAKI Motohiro | e247dbc | 2010-08-09 17:19:28 -0700 | [diff] [blame] | 2575 | unsigned long nr_taken; |
Kirill Tkhai | 060f005 | 2019-03-05 15:48:15 -0800 | [diff] [blame] | 2576 | struct reclaim_stat stat; |
Johannes Weiner | 497a6c1 | 2020-06-03 16:02:34 -0700 | [diff] [blame] | 2577 | bool file = is_file_lru(lru); |
Kirill Tkhai | f46b791 | 2019-05-13 17:22:33 -0700 | [diff] [blame] | 2578 | enum vm_event_item item; |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2579 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
Michal Hocko | db73ee0 | 2017-09-06 16:21:11 -0700 | [diff] [blame] | 2580 | bool stalled = false; |
KOSAKI Motohiro | 78dc583 | 2009-06-16 15:31:40 -0700 | [diff] [blame] | 2581 | |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2582 | while (unlikely(too_many_isolated(pgdat, file, sc))) { |
Michal Hocko | db73ee0 | 2017-09-06 16:21:11 -0700 | [diff] [blame] | 2583 | if (stalled) |
| 2584 | return 0; |
| 2585 | |
| 2586 | /* wait a bit for the reclaimer. */ |
Michal Hocko | db73ee0 | 2017-09-06 16:21:11 -0700 | [diff] [blame] | 2587 | stalled = true; |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 2588 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); |
Rik van Riel | 35cd781 | 2009-09-21 17:01:38 -0700 | [diff] [blame] | 2589 | |
| 2590 | /* We are about to die and free our memory. Return now. */ |
| 2591 | if (fatal_signal_pending(current)) |
| 2592 | return SWAP_CLUSTER_MAX; |
| 2593 | } |
| 2594 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2595 | lru_add_drain(); |
Minchan Kim | f80c067 | 2011-10-31 17:06:55 -0700 | [diff] [blame] | 2596 | |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2597 | spin_lock_irq(&lruvec->lru_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2598 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2599 | nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, |
Kirill Tkhai | a9e7c39 | 2019-03-05 15:46:55 -0800 | [diff] [blame] | 2600 | &nr_scanned, sc, lru); |
Konstantin Khlebnikov | 95d918f | 2012-05-29 15:06:59 -0700 | [diff] [blame] | 2601 | |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2602 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); |
Johannes Weiner | 57e9cc5 | 2022-10-26 14:01:33 -0400 | [diff] [blame] | 2603 | item = PGSCAN_KSWAPD + reclaimer_offset(); |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 2604 | if (!cgroup_reclaim(sc)) |
Kirill Tkhai | f46b791 | 2019-05-13 17:22:33 -0700 | [diff] [blame] | 2605 | __count_vm_events(item, nr_scanned); |
| 2606 | __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); |
Johannes Weiner | 497a6c1 | 2020-06-03 16:02:34 -0700 | [diff] [blame] | 2607 | __count_vm_events(PGSCAN_ANON + file, nr_scanned); |
| 2608 | |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2609 | spin_unlock_irq(&lruvec->lru_lock); |
KOSAKI Motohiro | a731286 | 2009-09-21 17:01:37 -0700 | [diff] [blame] | 2610 | |
Hillf Danton | d563c05 | 2012-03-21 16:34:02 -0700 | [diff] [blame] | 2611 | if (nr_taken == 0) |
Mel Gorman | 6663562 | 2010-08-09 17:19:30 -0700 | [diff] [blame] | 2612 | return 0; |
KOSAKI Motohiro | e247dbc | 2010-08-09 17:19:28 -0700 | [diff] [blame] | 2613 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2614 | nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false); |
KOSAKI Motohiro | e247dbc | 2010-08-09 17:19:28 -0700 | [diff] [blame] | 2615 | |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2616 | spin_lock_irq(&lruvec->lru_lock); |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2617 | move_folios_to_lru(lruvec, &folio_list); |
Johannes Weiner | 497a6c1 | 2020-06-03 16:02:34 -0700 | [diff] [blame] | 2618 | |
| 2619 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); |
Johannes Weiner | 57e9cc5 | 2022-10-26 14:01:33 -0400 | [diff] [blame] | 2620 | item = PGSTEAL_KSWAPD + reclaimer_offset(); |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 2621 | if (!cgroup_reclaim(sc)) |
Kirill Tkhai | f46b791 | 2019-05-13 17:22:33 -0700 | [diff] [blame] | 2622 | __count_vm_events(item, nr_reclaimed); |
| 2623 | __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); |
Johannes Weiner | 497a6c1 | 2020-06-03 16:02:34 -0700 | [diff] [blame] | 2624 | __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2625 | spin_unlock_irq(&lruvec->lru_lock); |
Hugh Dickins | 3f79768 | 2012-01-12 17:20:07 -0800 | [diff] [blame] | 2626 | |
Johannes Weiner | 0538a82c | 2022-10-13 15:31:13 -0400 | [diff] [blame] | 2627 | lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed); |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2628 | mem_cgroup_uncharge_list(&folio_list); |
| 2629 | free_unref_page_list(&folio_list); |
Mel Gorman | e11da5b | 2010-10-26 14:21:40 -0700 | [diff] [blame] | 2630 | |
Mel Gorman | 92df3a7 | 2011-10-31 17:07:56 -0700 | [diff] [blame] | 2631 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2632 | * If dirty folios are scanned that are not queued for IO, it |
Andrey Ryabinin | 1c610d5 | 2018-03-22 16:17:42 -0700 | [diff] [blame] | 2633 | * implies that flushers are not doing their job. This can |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2634 | * happen when memory pressure pushes dirty folios to the end of |
Andrey Ryabinin | 1c610d5 | 2018-03-22 16:17:42 -0700 | [diff] [blame] | 2635 | * the LRU before the dirty limits are breached and the dirty |
| 2636 | * data has expired. It can also happen when the proportion of |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2637 | * dirty folios grows not through writes but through memory |
Andrey Ryabinin | 1c610d5 | 2018-03-22 16:17:42 -0700 | [diff] [blame] | 2638 | * pressure reclaiming all the clean cache. And in some cases, |
| 2639 | * the flushers simply cannot keep up with the allocation |
| 2640 | * rate. Nudge the flusher threads in case they are asleep. |
| 2641 | */ |
Aneesh Kumar K.V | 81a70c2 | 2022-11-18 12:36:03 +0530 | [diff] [blame] | 2642 | if (stat.nr_unqueued_dirty == nr_taken) { |
Andrey Ryabinin | 1c610d5 | 2018-03-22 16:17:42 -0700 | [diff] [blame] | 2643 | wakeup_flusher_threads(WB_REASON_VMSCAN); |
Aneesh Kumar K.V | 81a70c2 | 2022-11-18 12:36:03 +0530 | [diff] [blame] | 2644 | /* |
| 2645 | * For cgroupv1 dirty throttling is achieved by waking up |
| 2646 | * the kernel flusher here and later waiting on folios |
| 2647 | * which are in writeback to finish (see shrink_folio_list()). |
| 2648 | * |
| 2649 | * Flusher may not be able to issue writeback quickly |
| 2650 | * enough for cgroupv1 writeback throttling to work |
| 2651 | * on a large system. |
| 2652 | */ |
| 2653 | if (!writeback_throttling_sane(sc)) |
| 2654 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); |
| 2655 | } |
Andrey Ryabinin | 1c610d5 | 2018-03-22 16:17:42 -0700 | [diff] [blame] | 2656 | |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 2657 | sc->nr.dirty += stat.nr_dirty; |
| 2658 | sc->nr.congested += stat.nr_congested; |
| 2659 | sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; |
| 2660 | sc->nr.writeback += stat.nr_writeback; |
| 2661 | sc->nr.immediate += stat.nr_immediate; |
| 2662 | sc->nr.taken += nr_taken; |
| 2663 | if (file) |
| 2664 | sc->nr.file_taken += nr_taken; |
Mel Gorman | 8e95028 | 2013-07-03 15:02:02 -0700 | [diff] [blame] | 2665 | |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2666 | trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, |
Steven Rostedt | d51d1e6 | 2018-04-10 16:28:07 -0700 | [diff] [blame] | 2667 | nr_scanned, nr_reclaimed, &stat, sc->priority, file); |
Andrew Morton | 05ff513 | 2006-03-22 00:08:20 -0800 | [diff] [blame] | 2668 | return nr_reclaimed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2669 | } |
| 2670 | |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2671 | /* |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2672 | * shrink_active_list() moves folios from the active LRU to the inactive LRU. |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2673 | * |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2674 | * We move them the other way if the folio is referenced by one or more |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2675 | * processes. |
| 2676 | * |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2677 | * If the folios are mostly unmapped, the processing is fast and it is |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2678 | * appropriate to hold lru_lock across the whole operation. But if |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2679 | * the folios are mapped, the processing is slow (folio_referenced()), so |
| 2680 | * we should drop lru_lock around each folio. It's impossible to balance |
| 2681 | * this, so instead we remove the folios from the LRU while processing them. |
| 2682 | * It is safe to rely on the active flag against the non-LRU folios in here |
| 2683 | * because nobody will play with that bit on a non-LRU folio. |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2684 | * |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2685 | * The downside is that we have to touch folio->_refcount against each folio. |
| 2686 | * But we had to alter folio->flags anyway. |
Hugh Dickins | 15b4473 | 2020-12-15 14:21:31 -0800 | [diff] [blame] | 2687 | */ |
Hugh Dickins | f626012 | 2012-01-12 17:20:06 -0800 | [diff] [blame] | 2688 | static void shrink_active_list(unsigned long nr_to_scan, |
Konstantin Khlebnikov | 1a93be0 | 2012-05-29 15:07:01 -0700 | [diff] [blame] | 2689 | struct lruvec *lruvec, |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 2690 | struct scan_control *sc, |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 2691 | enum lru_list lru) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2692 | { |
KOSAKI Motohiro | 44c241f | 2009-09-21 17:01:35 -0700 | [diff] [blame] | 2693 | unsigned long nr_taken; |
Hugh Dickins | f626012 | 2012-01-12 17:20:06 -0800 | [diff] [blame] | 2694 | unsigned long nr_scanned; |
Wu Fengguang | 6fe6b7e | 2009-06-16 15:33:05 -0700 | [diff] [blame] | 2695 | unsigned long vm_flags; |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2696 | LIST_HEAD(l_hold); /* The folios which were snipped off */ |
Wu Fengguang | 8cab475 | 2009-06-16 15:33:12 -0700 | [diff] [blame] | 2697 | LIST_HEAD(l_active); |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 2698 | LIST_HEAD(l_inactive); |
Michal Hocko | 9d998b4 | 2017-02-22 15:44:18 -0800 | [diff] [blame] | 2699 | unsigned nr_deactivate, nr_activate; |
| 2700 | unsigned nr_rotated = 0; |
Konstantin Khlebnikov | 3cb9945 | 2012-05-29 15:06:53 -0700 | [diff] [blame] | 2701 | int file = is_file_lru(lru); |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2702 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2703 | |
| 2704 | lru_add_drain(); |
Minchan Kim | f80c067 | 2011-10-31 17:06:55 -0700 | [diff] [blame] | 2705 | |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2706 | spin_lock_irq(&lruvec->lru_lock); |
Johannes Weiner | 925b767 | 2012-01-12 17:18:15 -0800 | [diff] [blame] | 2707 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2708 | nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, |
Kirill Tkhai | a9e7c39 | 2019-03-05 15:46:55 -0800 | [diff] [blame] | 2709 | &nr_scanned, sc, lru); |
Johannes Weiner | 89b5fae | 2012-01-12 17:17:50 -0800 | [diff] [blame] | 2710 | |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2711 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); |
KAMEZAWA Hiroyuki | 1cfb419 | 2008-02-07 00:14:37 -0800 | [diff] [blame] | 2712 | |
Shakeel Butt | 912c057 | 2020-08-06 23:26:32 -0700 | [diff] [blame] | 2713 | if (!cgroup_reclaim(sc)) |
| 2714 | __count_vm_events(PGREFILL, nr_scanned); |
Yafang Shao | 2fa2690 | 2019-05-13 17:23:02 -0700 | [diff] [blame] | 2715 | __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); |
Hugh Dickins | 9d5e6a9 | 2016-05-19 17:12:38 -0700 | [diff] [blame] | 2716 | |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2717 | spin_unlock_irq(&lruvec->lru_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2718 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2719 | while (!list_empty(&l_hold)) { |
Matthew Wilcox (Oracle) | b3ac041 | 2022-01-21 11:27:31 -0500 | [diff] [blame] | 2720 | struct folio *folio; |
Matthew Wilcox (Oracle) | b3ac041 | 2022-01-21 11:27:31 -0500 | [diff] [blame] | 2721 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2722 | cond_resched(); |
Matthew Wilcox (Oracle) | b3ac041 | 2022-01-21 11:27:31 -0500 | [diff] [blame] | 2723 | folio = lru_to_folio(&l_hold); |
| 2724 | list_del(&folio->lru); |
Rik van Riel | 7e9cd48 | 2008-10-18 20:26:35 -0700 | [diff] [blame] | 2725 | |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2726 | if (unlikely(!folio_evictable(folio))) { |
| 2727 | folio_putback_lru(folio); |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 2728 | continue; |
| 2729 | } |
| 2730 | |
Mel Gorman | cc715d9 | 2012-03-21 16:34:00 -0700 | [diff] [blame] | 2731 | if (unlikely(buffer_heads_over_limit)) { |
David Howells | 0201ebf | 2023-06-28 11:48:51 +0100 | [diff] [blame] | 2732 | if (folio_needs_release(folio) && |
| 2733 | folio_trylock(folio)) { |
| 2734 | filemap_release_folio(folio, 0); |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2735 | folio_unlock(folio); |
Mel Gorman | cc715d9 | 2012-03-21 16:34:00 -0700 | [diff] [blame] | 2736 | } |
| 2737 | } |
| 2738 | |
Minchan Kim | 6d4675e | 2022-05-19 14:08:54 -0700 | [diff] [blame] | 2739 | /* Referenced or rmap lock contention: rotate */ |
Matthew Wilcox (Oracle) | b3ac041 | 2022-01-21 11:27:31 -0500 | [diff] [blame] | 2740 | if (folio_referenced(folio, 0, sc->target_mem_cgroup, |
Minchan Kim | 6d4675e | 2022-05-19 14:08:54 -0700 | [diff] [blame] | 2741 | &vm_flags) != 0) { |
Wu Fengguang | 8cab475 | 2009-06-16 15:33:12 -0700 | [diff] [blame] | 2742 | /* |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2743 | * Identify referenced, file-backed active folios and |
Wu Fengguang | 8cab475 | 2009-06-16 15:33:12 -0700 | [diff] [blame] | 2744 | * give them one more trip around the active list. So |
| 2745 | * that executable code get better chances to stay in |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2746 | * memory under moderate memory pressure. Anon folios |
Wu Fengguang | 8cab475 | 2009-06-16 15:33:12 -0700 | [diff] [blame] | 2747 | * are not likely to be evicted by use-once streaming |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2748 | * IO, plus JVM can create lots of anon VM_EXEC folios, |
Wu Fengguang | 8cab475 | 2009-06-16 15:33:12 -0700 | [diff] [blame] | 2749 | * so we ignore them here. |
| 2750 | */ |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2751 | if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { |
| 2752 | nr_rotated += folio_nr_pages(folio); |
| 2753 | list_add(&folio->lru, &l_active); |
Wu Fengguang | 8cab475 | 2009-06-16 15:33:12 -0700 | [diff] [blame] | 2754 | continue; |
| 2755 | } |
| 2756 | } |
Rik van Riel | 7e9cd48 | 2008-10-18 20:26:35 -0700 | [diff] [blame] | 2757 | |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2758 | folio_clear_active(folio); /* we are de-activating */ |
| 2759 | folio_set_workingset(folio); |
| 2760 | list_add(&folio->lru, &l_inactive); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2761 | } |
| 2762 | |
Andrew Morton | b555749 | 2009-01-06 14:40:13 -0800 | [diff] [blame] | 2763 | /* |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2764 | * Move folios back to the lru list. |
Andrew Morton | b555749 | 2009-01-06 14:40:13 -0800 | [diff] [blame] | 2765 | */ |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2766 | spin_lock_irq(&lruvec->lru_lock); |
Rik van Riel | 556adec | 2008-10-18 20:26:34 -0700 | [diff] [blame] | 2767 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2768 | nr_activate = move_folios_to_lru(lruvec, &l_active); |
| 2769 | nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); |
Matthew Wilcox (Oracle) | 07f67a8 | 2022-06-17 16:42:47 +0100 | [diff] [blame] | 2770 | /* Keep all free folios in l_active list */ |
Kirill Tkhai | f372d89 | 2019-05-13 17:16:57 -0700 | [diff] [blame] | 2771 | list_splice(&l_inactive, &l_active); |
Kirill Tkhai | 9851ac1 | 2019-05-13 17:16:54 -0700 | [diff] [blame] | 2772 | |
| 2773 | __count_vm_events(PGDEACTIVATE, nr_deactivate); |
| 2774 | __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); |
| 2775 | |
Mel Gorman | 599d0c9 | 2016-07-28 15:45:31 -0700 | [diff] [blame] | 2776 | __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 2777 | spin_unlock_irq(&lruvec->lru_lock); |
Hugh Dickins | 2bcf887 | 2012-01-12 17:19:56 -0800 | [diff] [blame] | 2778 | |
Johannes Weiner | 0538a82c | 2022-10-13 15:31:13 -0400 | [diff] [blame] | 2779 | if (nr_rotated) |
| 2780 | lru_note_cost(lruvec, file, 0, nr_rotated); |
Kirill Tkhai | f372d89 | 2019-05-13 17:16:57 -0700 | [diff] [blame] | 2781 | mem_cgroup_uncharge_list(&l_active); |
| 2782 | free_unref_page_list(&l_active); |
Michal Hocko | 9d998b4 | 2017-02-22 15:44:18 -0800 | [diff] [blame] | 2783 | trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, |
| 2784 | nr_deactivate, nr_rotated, sc->priority, file); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2785 | } |
| 2786 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2787 | static unsigned int reclaim_folio_list(struct list_head *folio_list, |
Miaohe Lin | 1fe47c0 | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 2788 | struct pglist_data *pgdat) |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2789 | { |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2790 | struct reclaim_stat dummy_stat; |
Miaohe Lin | 1fe47c0 | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 2791 | unsigned int nr_reclaimed; |
| 2792 | struct folio *folio; |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2793 | struct scan_control sc = { |
| 2794 | .gfp_mask = GFP_KERNEL, |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2795 | .may_writepage = 1, |
| 2796 | .may_unmap = 1, |
| 2797 | .may_swap = 1, |
Dave Hansen | 26aa2d1 | 2021-09-02 14:59:16 -0700 | [diff] [blame] | 2798 | .no_demotion = 1, |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2799 | }; |
| 2800 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2801 | nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false); |
| 2802 | while (!list_empty(folio_list)) { |
| 2803 | folio = lru_to_folio(folio_list); |
Miaohe Lin | 1fe47c0 | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 2804 | list_del(&folio->lru); |
| 2805 | folio_putback_lru(folio); |
| 2806 | } |
| 2807 | |
| 2808 | return nr_reclaimed; |
| 2809 | } |
| 2810 | |
Matthew Wilcox (Oracle) | a83f055 | 2022-06-17 16:42:48 +0100 | [diff] [blame] | 2811 | unsigned long reclaim_pages(struct list_head *folio_list) |
Miaohe Lin | 1fe47c0 | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 2812 | { |
Wei Yang | ed657e5 | 2022-05-12 20:23:00 -0700 | [diff] [blame] | 2813 | int nid; |
Miaohe Lin | 1fe47c0 | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 2814 | unsigned int nr_reclaimed = 0; |
Matthew Wilcox (Oracle) | a83f055 | 2022-06-17 16:42:48 +0100 | [diff] [blame] | 2815 | LIST_HEAD(node_folio_list); |
Miaohe Lin | 1fe47c0 | 2022-05-12 20:22:59 -0700 | [diff] [blame] | 2816 | unsigned int noreclaim_flag; |
| 2817 | |
Matthew Wilcox (Oracle) | a83f055 | 2022-06-17 16:42:48 +0100 | [diff] [blame] | 2818 | if (list_empty(folio_list)) |
Wei Yang | 1ae65e2 | 2022-05-12 20:23:00 -0700 | [diff] [blame] | 2819 | return nr_reclaimed; |
| 2820 | |
Yu Zhao | 2d2b8d2 | 2021-06-30 18:49:48 -0700 | [diff] [blame] | 2821 | noreclaim_flag = memalloc_noreclaim_save(); |
| 2822 | |
Matthew Wilcox (Oracle) | a83f055 | 2022-06-17 16:42:48 +0100 | [diff] [blame] | 2823 | nid = folio_nid(lru_to_folio(folio_list)); |
Wei Yang | 1ae65e2 | 2022-05-12 20:23:00 -0700 | [diff] [blame] | 2824 | do { |
Matthew Wilcox (Oracle) | a83f055 | 2022-06-17 16:42:48 +0100 | [diff] [blame] | 2825 | struct folio *folio = lru_to_folio(folio_list); |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2826 | |
Matthew Wilcox (Oracle) | a83f055 | 2022-06-17 16:42:48 +0100 | [diff] [blame] | 2827 | if (nid == folio_nid(folio)) { |
| 2828 | folio_clear_active(folio); |
| 2829 | list_move(&folio->lru, &node_folio_list); |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2830 | continue; |
| 2831 | } |
| 2832 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2833 | nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); |
Matthew Wilcox (Oracle) | a83f055 | 2022-06-17 16:42:48 +0100 | [diff] [blame] | 2834 | nid = folio_nid(lru_to_folio(folio_list)); |
| 2835 | } while (!list_empty(folio_list)); |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2836 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2837 | nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2838 | |
Yu Zhao | 2d2b8d2 | 2021-06-30 18:49:48 -0700 | [diff] [blame] | 2839 | memalloc_noreclaim_restore(noreclaim_flag); |
| 2840 | |
Minchan Kim | 1a4e58c | 2019-09-25 16:49:15 -0700 | [diff] [blame] | 2841 | return nr_reclaimed; |
| 2842 | } |
| 2843 | |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 2844 | static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, |
| 2845 | struct lruvec *lruvec, struct scan_control *sc) |
| 2846 | { |
| 2847 | if (is_active_lru(lru)) { |
| 2848 | if (sc->may_deactivate & (1 << is_file_lru(lru))) |
| 2849 | shrink_active_list(nr_to_scan, lruvec, sc, lru); |
| 2850 | else |
| 2851 | sc->skipped_deactivate = 1; |
| 2852 | return 0; |
| 2853 | } |
| 2854 | |
| 2855 | return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); |
| 2856 | } |
| 2857 | |
Rik van Riel | 59dc76b | 2016-05-20 16:56:31 -0700 | [diff] [blame] | 2858 | /* |
| 2859 | * The inactive anon list should be small enough that the VM never has |
| 2860 | * to do too much work. |
KOSAKI Motohiro | 14797e2 | 2009-01-07 18:08:18 -0800 | [diff] [blame] | 2861 | * |
Rik van Riel | 59dc76b | 2016-05-20 16:56:31 -0700 | [diff] [blame] | 2862 | * The inactive file list should be small enough to leave most memory |
| 2863 | * to the established workingset on the scan-resistant active list, |
| 2864 | * but large enough to avoid thrashing the aggregate readahead window. |
| 2865 | * |
| 2866 | * Both inactive lists should also be large enough that each inactive |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2867 | * folio has a chance to be referenced again before it is reclaimed. |
Rik van Riel | 59dc76b | 2016-05-20 16:56:31 -0700 | [diff] [blame] | 2868 | * |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 2869 | * If that fails and refaulting is observed, the inactive list grows. |
| 2870 | * |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2871 | * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios |
Andrey Ryabinin | 3a50d14 | 2017-11-15 17:34:15 -0800 | [diff] [blame] | 2872 | * on this LRU, maintained by the pageout code. An inactive_ratio |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 2873 | * of 3 means 3:1 or 25% of the folios are kept on the inactive list. |
Rik van Riel | 59dc76b | 2016-05-20 16:56:31 -0700 | [diff] [blame] | 2874 | * |
| 2875 | * total target max |
| 2876 | * memory ratio inactive |
| 2877 | * ------------------------------------- |
| 2878 | * 10MB 1 5MB |
| 2879 | * 100MB 1 50MB |
| 2880 | * 1GB 3 250MB |
| 2881 | * 10GB 10 0.9GB |
| 2882 | * 100GB 31 3GB |
| 2883 | * 1TB 101 10GB |
| 2884 | * 10TB 320 32GB |
KOSAKI Motohiro | 14797e2 | 2009-01-07 18:08:18 -0800 | [diff] [blame] | 2885 | */ |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 2886 | static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) |
KOSAKI Motohiro | 14797e2 | 2009-01-07 18:08:18 -0800 | [diff] [blame] | 2887 | { |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 2888 | enum lru_list active_lru = inactive_lru + LRU_ACTIVE; |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 2889 | unsigned long inactive, active; |
| 2890 | unsigned long inactive_ratio; |
Rik van Riel | 59dc76b | 2016-05-20 16:56:31 -0700 | [diff] [blame] | 2891 | unsigned long gb; |
| 2892 | |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 2893 | inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); |
| 2894 | active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); |
Mel Gorman | f8d1a31 | 2016-07-28 15:47:34 -0700 | [diff] [blame] | 2895 | |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 2896 | gb = (inactive + active) >> (30 - PAGE_SHIFT); |
Joonsoo Kim | 4002570 | 2020-08-11 18:30:54 -0700 | [diff] [blame] | 2897 | if (gb) |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 2898 | inactive_ratio = int_sqrt(10 * gb); |
| 2899 | else |
| 2900 | inactive_ratio = 1; |
Michal Hocko | fd53880 | 2017-02-22 15:45:58 -0800 | [diff] [blame] | 2901 | |
Rik van Riel | 59dc76b | 2016-05-20 16:56:31 -0700 | [diff] [blame] | 2902 | return inactive * inactive_ratio < active; |
Rik van Riel | b39415b | 2009-12-14 17:59:48 -0800 | [diff] [blame] | 2903 | } |
| 2904 | |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 2905 | enum scan_balance { |
| 2906 | SCAN_EQUAL, |
| 2907 | SCAN_FRACT, |
| 2908 | SCAN_ANON, |
| 2909 | SCAN_FILE, |
| 2910 | }; |
| 2911 | |
Yu Zhao | f1e1a7b | 2022-09-18 02:00:00 -0600 | [diff] [blame] | 2912 | static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) |
| 2913 | { |
| 2914 | unsigned long file; |
| 2915 | struct lruvec *target_lruvec; |
| 2916 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 2917 | if (lru_gen_enabled()) |
| 2918 | return; |
| 2919 | |
Yu Zhao | f1e1a7b | 2022-09-18 02:00:00 -0600 | [diff] [blame] | 2920 | target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); |
| 2921 | |
| 2922 | /* |
| 2923 | * Flush the memory cgroup stats, so that we read accurate per-memcg |
| 2924 | * lruvec stats for heuristics. |
| 2925 | */ |
| 2926 | mem_cgroup_flush_stats(); |
| 2927 | |
| 2928 | /* |
| 2929 | * Determine the scan balance between anon and file LRUs. |
| 2930 | */ |
| 2931 | spin_lock_irq(&target_lruvec->lru_lock); |
| 2932 | sc->anon_cost = target_lruvec->anon_cost; |
| 2933 | sc->file_cost = target_lruvec->file_cost; |
| 2934 | spin_unlock_irq(&target_lruvec->lru_lock); |
| 2935 | |
| 2936 | /* |
| 2937 | * Target desirable inactive:active list ratios for the anon |
| 2938 | * and file LRU lists. |
| 2939 | */ |
| 2940 | if (!sc->force_deactivate) { |
| 2941 | unsigned long refaults; |
| 2942 | |
| 2943 | /* |
| 2944 | * When refaults are being observed, it means a new |
| 2945 | * workingset is being established. Deactivate to get |
| 2946 | * rid of any stale active pages quickly. |
| 2947 | */ |
| 2948 | refaults = lruvec_page_state(target_lruvec, |
| 2949 | WORKINGSET_ACTIVATE_ANON); |
| 2950 | if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || |
| 2951 | inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) |
| 2952 | sc->may_deactivate |= DEACTIVATE_ANON; |
| 2953 | else |
| 2954 | sc->may_deactivate &= ~DEACTIVATE_ANON; |
| 2955 | |
| 2956 | refaults = lruvec_page_state(target_lruvec, |
| 2957 | WORKINGSET_ACTIVATE_FILE); |
| 2958 | if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || |
| 2959 | inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) |
| 2960 | sc->may_deactivate |= DEACTIVATE_FILE; |
| 2961 | else |
| 2962 | sc->may_deactivate &= ~DEACTIVATE_FILE; |
| 2963 | } else |
| 2964 | sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; |
| 2965 | |
| 2966 | /* |
| 2967 | * If we have plenty of inactive file pages that aren't |
| 2968 | * thrashing, try to reclaim those first before touching |
| 2969 | * anonymous pages. |
| 2970 | */ |
| 2971 | file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); |
| 2972 | if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) |
| 2973 | sc->cache_trim_mode = 1; |
| 2974 | else |
| 2975 | sc->cache_trim_mode = 0; |
| 2976 | |
| 2977 | /* |
| 2978 | * Prevent the reclaimer from falling into the cache trap: as |
| 2979 | * cache pages start out inactive, every cache fault will tip |
| 2980 | * the scan balance towards the file LRU. And as the file LRU |
| 2981 | * shrinks, so does the window for rotation from references. |
| 2982 | * This means we have a runaway feedback loop where a tiny |
| 2983 | * thrashing file LRU becomes infinitely more attractive than |
| 2984 | * anon pages. Try to detect this based on file LRU size. |
| 2985 | */ |
| 2986 | if (!cgroup_reclaim(sc)) { |
| 2987 | unsigned long total_high_wmark = 0; |
| 2988 | unsigned long free, anon; |
| 2989 | int z; |
| 2990 | |
| 2991 | free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); |
| 2992 | file = node_page_state(pgdat, NR_ACTIVE_FILE) + |
| 2993 | node_page_state(pgdat, NR_INACTIVE_FILE); |
| 2994 | |
| 2995 | for (z = 0; z < MAX_NR_ZONES; z++) { |
| 2996 | struct zone *zone = &pgdat->node_zones[z]; |
| 2997 | |
| 2998 | if (!managed_zone(zone)) |
| 2999 | continue; |
| 3000 | |
| 3001 | total_high_wmark += high_wmark_pages(zone); |
| 3002 | } |
| 3003 | |
| 3004 | /* |
| 3005 | * Consider anon: if that's low too, this isn't a |
| 3006 | * runaway file reclaim problem, but rather just |
| 3007 | * extreme pressure. Reclaim as per usual then. |
| 3008 | */ |
| 3009 | anon = node_page_state(pgdat, NR_INACTIVE_ANON); |
| 3010 | |
| 3011 | sc->file_is_tiny = |
| 3012 | file + free <= total_high_wmark && |
| 3013 | !(sc->may_deactivate & DEACTIVATE_ANON) && |
| 3014 | anon >> sc->priority; |
| 3015 | } |
| 3016 | } |
| 3017 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3018 | /* |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3019 | * Determine how aggressively the anon and file LRU lists should be |
Miaohe Lin | 02e458d | 2022-04-28 23:16:04 -0700 | [diff] [blame] | 3020 | * scanned. |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3021 | * |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 3022 | * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan |
| 3023 | * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3024 | */ |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 3025 | static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, |
| 3026 | unsigned long *nr) |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3027 | { |
Keith Busch | a2a3648 | 2021-09-02 14:59:26 -0700 | [diff] [blame] | 3028 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 3029 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
Johannes Weiner | d483a5d | 2020-06-03 16:03:13 -0700 | [diff] [blame] | 3030 | unsigned long anon_cost, file_cost, total_cost; |
Vladimir Davydov | 3337767 | 2016-01-20 15:02:59 -0800 | [diff] [blame] | 3031 | int swappiness = mem_cgroup_swappiness(memcg); |
Yu Zhao | ed01737 | 2020-10-15 20:09:55 -0700 | [diff] [blame] | 3032 | u64 fraction[ANON_AND_FILE]; |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3033 | u64 denominator = 0; /* gcc */ |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3034 | enum scan_balance scan_balance; |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3035 | unsigned long ap, fp; |
| 3036 | enum lru_list lru; |
Shaohua Li | 76a33fc | 2010-05-24 14:32:36 -0700 | [diff] [blame] | 3037 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 3038 | /* If we have no swap space, do not bother scanning anon folios. */ |
Keith Busch | a2a3648 | 2021-09-02 14:59:26 -0700 | [diff] [blame] | 3039 | if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3040 | scan_balance = SCAN_FILE; |
Shaohua Li | 76a33fc | 2010-05-24 14:32:36 -0700 | [diff] [blame] | 3041 | goto out; |
| 3042 | } |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3043 | |
Johannes Weiner | 10316b3 | 2013-02-22 16:32:14 -0800 | [diff] [blame] | 3044 | /* |
| 3045 | * Global reclaim will swap to prevent OOM even with no |
| 3046 | * swappiness, but memcg users want to use this knob to |
| 3047 | * disable swapping for individual groups completely when |
| 3048 | * using the memory controller's swap limit feature would be |
| 3049 | * too expensive. |
| 3050 | */ |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 3051 | if (cgroup_reclaim(sc) && !swappiness) { |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3052 | scan_balance = SCAN_FILE; |
Johannes Weiner | 10316b3 | 2013-02-22 16:32:14 -0800 | [diff] [blame] | 3053 | goto out; |
| 3054 | } |
| 3055 | |
| 3056 | /* |
| 3057 | * Do not apply any pressure balancing cleverness when the |
| 3058 | * system is close to OOM, scan both anon and file equally |
| 3059 | * (unless the swappiness setting disagrees with swapping). |
| 3060 | */ |
Johannes Weiner | 0269517 | 2014-08-06 16:06:17 -0700 | [diff] [blame] | 3061 | if (!sc->priority && swappiness) { |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3062 | scan_balance = SCAN_EQUAL; |
Johannes Weiner | 10316b3 | 2013-02-22 16:32:14 -0800 | [diff] [blame] | 3063 | goto out; |
| 3064 | } |
| 3065 | |
Johannes Weiner | 11d16c2 | 2013-02-22 16:32:15 -0800 | [diff] [blame] | 3066 | /* |
Johannes Weiner | 53138ce | 2019-11-30 17:55:56 -0800 | [diff] [blame] | 3067 | * If the system is almost out of file pages, force-scan anon. |
Johannes Weiner | 6237625 | 2014-05-06 12:50:07 -0700 | [diff] [blame] | 3068 | */ |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 3069 | if (sc->file_is_tiny) { |
Johannes Weiner | 53138ce | 2019-11-30 17:55:56 -0800 | [diff] [blame] | 3070 | scan_balance = SCAN_ANON; |
| 3071 | goto out; |
Johannes Weiner | 6237625 | 2014-05-06 12:50:07 -0700 | [diff] [blame] | 3072 | } |
| 3073 | |
| 3074 | /* |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 3075 | * If there is enough inactive page cache, we do not reclaim |
| 3076 | * anything from the anonymous working right now. |
Johannes Weiner | 7c5bd70 | 2013-02-22 16:32:10 -0800 | [diff] [blame] | 3077 | */ |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 3078 | if (sc->cache_trim_mode) { |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3079 | scan_balance = SCAN_FILE; |
Johannes Weiner | 7c5bd70 | 2013-02-22 16:32:10 -0800 | [diff] [blame] | 3080 | goto out; |
| 3081 | } |
| 3082 | |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3083 | scan_balance = SCAN_FRACT; |
Johannes Weiner | 7c5bd70 | 2013-02-22 16:32:10 -0800 | [diff] [blame] | 3084 | /* |
Johannes Weiner | 314b57f | 2020-06-03 16:03:03 -0700 | [diff] [blame] | 3085 | * Calculate the pressure balance between anon and file pages. |
| 3086 | * |
| 3087 | * The amount of pressure we put on each LRU is inversely |
| 3088 | * proportional to the cost of reclaiming each list, as |
| 3089 | * determined by the share of pages that are refaulting, times |
| 3090 | * the relative IO cost of bringing back a swapped out |
| 3091 | * anonymous page vs reloading a filesystem page (swappiness). |
| 3092 | * |
Johannes Weiner | d483a5d | 2020-06-03 16:03:13 -0700 | [diff] [blame] | 3093 | * Although we limit that influence to ensure no list gets |
| 3094 | * left behind completely: at least a third of the pressure is |
| 3095 | * applied, before swappiness. |
| 3096 | * |
Johannes Weiner | 314b57f | 2020-06-03 16:03:03 -0700 | [diff] [blame] | 3097 | * With swappiness at 100, anon and file have equal IO cost. |
KOSAKI Motohiro | 58c37f6 | 2010-08-09 17:19:51 -0700 | [diff] [blame] | 3098 | */ |
Johannes Weiner | d483a5d | 2020-06-03 16:03:13 -0700 | [diff] [blame] | 3099 | total_cost = sc->anon_cost + sc->file_cost; |
| 3100 | anon_cost = total_cost + sc->anon_cost; |
| 3101 | file_cost = total_cost + sc->file_cost; |
| 3102 | total_cost = anon_cost + file_cost; |
KOSAKI Motohiro | 58c37f6 | 2010-08-09 17:19:51 -0700 | [diff] [blame] | 3103 | |
Johannes Weiner | d483a5d | 2020-06-03 16:03:13 -0700 | [diff] [blame] | 3104 | ap = swappiness * (total_cost + 1); |
| 3105 | ap /= anon_cost + 1; |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3106 | |
Johannes Weiner | d483a5d | 2020-06-03 16:03:13 -0700 | [diff] [blame] | 3107 | fp = (200 - swappiness) * (total_cost + 1); |
| 3108 | fp /= file_cost + 1; |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3109 | |
Shaohua Li | 76a33fc | 2010-05-24 14:32:36 -0700 | [diff] [blame] | 3110 | fraction[0] = ap; |
| 3111 | fraction[1] = fp; |
Johannes Weiner | a4fe163 | 2020-06-03 16:02:50 -0700 | [diff] [blame] | 3112 | denominator = ap + fp; |
Shaohua Li | 76a33fc | 2010-05-24 14:32:36 -0700 | [diff] [blame] | 3113 | out: |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3114 | for_each_evictable_lru(lru) { |
| 3115 | int file = is_file_lru(lru); |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3116 | unsigned long lruvec_size; |
Johannes Weiner | f56ce41 | 2021-08-19 19:04:21 -0700 | [diff] [blame] | 3117 | unsigned long low, min; |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3118 | unsigned long scan; |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3119 | |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3120 | lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); |
Johannes Weiner | f56ce41 | 2021-08-19 19:04:21 -0700 | [diff] [blame] | 3121 | mem_cgroup_protection(sc->target_mem_cgroup, memcg, |
| 3122 | &min, &low); |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3123 | |
Johannes Weiner | f56ce41 | 2021-08-19 19:04:21 -0700 | [diff] [blame] | 3124 | if (min || low) { |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3125 | /* |
| 3126 | * Scale a cgroup's reclaim pressure by proportioning |
| 3127 | * its current usage to its memory.low or memory.min |
| 3128 | * setting. |
| 3129 | * |
| 3130 | * This is important, as otherwise scanning aggression |
| 3131 | * becomes extremely binary -- from nothing as we |
| 3132 | * approach the memory protection threshold, to totally |
| 3133 | * nominal as we exceed it. This results in requiring |
| 3134 | * setting extremely liberal protection thresholds. It |
| 3135 | * also means we simply get no protection at all if we |
| 3136 | * set it too low, which is not ideal. |
Chris Down | 1bc63fb | 2019-10-06 17:58:38 -0700 | [diff] [blame] | 3137 | * |
| 3138 | * If there is any protection in place, we reduce scan |
| 3139 | * pressure by how much of the total memory used is |
| 3140 | * within protection thresholds. |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3141 | * |
Chris Down | 9de7ca4 | 2019-10-06 17:58:35 -0700 | [diff] [blame] | 3142 | * There is one special case: in the first reclaim pass, |
| 3143 | * we skip over all groups that are within their low |
| 3144 | * protection. If that fails to reclaim enough pages to |
| 3145 | * satisfy the reclaim goal, we come back and override |
| 3146 | * the best-effort low protection. However, we still |
| 3147 | * ideally want to honor how well-behaved groups are in |
| 3148 | * that case instead of simply punishing them all |
| 3149 | * equally. As such, we reclaim them based on how much |
Chris Down | 1bc63fb | 2019-10-06 17:58:38 -0700 | [diff] [blame] | 3150 | * memory they are using, reducing the scan pressure |
| 3151 | * again by how much of the total memory used is under |
| 3152 | * hard protection. |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3153 | */ |
Chris Down | 1bc63fb | 2019-10-06 17:58:38 -0700 | [diff] [blame] | 3154 | unsigned long cgroup_size = mem_cgroup_size(memcg); |
Johannes Weiner | f56ce41 | 2021-08-19 19:04:21 -0700 | [diff] [blame] | 3155 | unsigned long protection; |
| 3156 | |
| 3157 | /* memory.low scaling, make sure we retry before OOM */ |
| 3158 | if (!sc->memcg_low_reclaim && low > min) { |
| 3159 | protection = low; |
| 3160 | sc->memcg_low_skipped = 1; |
| 3161 | } else { |
| 3162 | protection = min; |
| 3163 | } |
Chris Down | 1bc63fb | 2019-10-06 17:58:38 -0700 | [diff] [blame] | 3164 | |
| 3165 | /* Avoid TOCTOU with earlier protection check */ |
| 3166 | cgroup_size = max(cgroup_size, protection); |
| 3167 | |
| 3168 | scan = lruvec_size - lruvec_size * protection / |
Rik van Riel | 32d4f4b | 2021-09-08 18:10:08 -0700 | [diff] [blame] | 3169 | (cgroup_size + 1); |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3170 | |
| 3171 | /* |
Chris Down | 1bc63fb | 2019-10-06 17:58:38 -0700 | [diff] [blame] | 3172 | * Minimally target SWAP_CLUSTER_MAX pages to keep |
Ethon Paul | 55b65a5 | 2020-06-04 16:49:10 -0700 | [diff] [blame] | 3173 | * reclaim moving forwards, avoiding decrementing |
Chris Down | 9de7ca4 | 2019-10-06 17:58:35 -0700 | [diff] [blame] | 3174 | * sc->priority further than desirable. |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3175 | */ |
Chris Down | 1bc63fb | 2019-10-06 17:58:38 -0700 | [diff] [blame] | 3176 | scan = max(scan, SWAP_CLUSTER_MAX); |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3177 | } else { |
| 3178 | scan = lruvec_size; |
| 3179 | } |
| 3180 | |
| 3181 | scan >>= sc->priority; |
| 3182 | |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3183 | /* |
| 3184 | * If the cgroup's already been deleted, make sure to |
| 3185 | * scrape out the remaining cache. |
| 3186 | */ |
| 3187 | if (!scan && !mem_cgroup_online(memcg)) |
Chris Down | 9783aa9 | 2019-10-06 17:58:32 -0700 | [diff] [blame] | 3188 | scan = min(lruvec_size, SWAP_CLUSTER_MAX); |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3189 | |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3190 | switch (scan_balance) { |
| 3191 | case SCAN_EQUAL: |
| 3192 | /* Scan lists relative to size */ |
| 3193 | break; |
| 3194 | case SCAN_FRACT: |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3195 | /* |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3196 | * Scan types proportional to swappiness and |
| 3197 | * their relative recent reclaim efficiency. |
Gavin Shan | 76073c6 | 2020-02-20 20:04:24 -0800 | [diff] [blame] | 3198 | * Make sure we don't miss the last page on |
| 3199 | * the offlined memory cgroups because of a |
| 3200 | * round-off error. |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3201 | */ |
Gavin Shan | 76073c6 | 2020-02-20 20:04:24 -0800 | [diff] [blame] | 3202 | scan = mem_cgroup_online(memcg) ? |
| 3203 | div64_u64(scan * fraction[file], denominator) : |
| 3204 | DIV64_U64_ROUND_UP(scan * fraction[file], |
Roman Gushchin | 68600f6 | 2018-10-26 15:03:27 -0700 | [diff] [blame] | 3205 | denominator); |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3206 | break; |
| 3207 | case SCAN_FILE: |
| 3208 | case SCAN_ANON: |
| 3209 | /* Scan one type exclusively */ |
Mateusz Nosek | e072bff | 2020-04-01 21:10:15 -0700 | [diff] [blame] | 3210 | if ((scan_balance == SCAN_FILE) != file) |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3211 | scan = 0; |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3212 | break; |
| 3213 | default: |
| 3214 | /* Look ma, no brain */ |
| 3215 | BUG(); |
Johannes Weiner | 9a26511 | 2013-02-22 16:32:17 -0800 | [diff] [blame] | 3216 | } |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3217 | |
Johannes Weiner | 688035f | 2017-05-03 14:52:07 -0700 | [diff] [blame] | 3218 | nr[lru] = scan; |
Shaohua Li | 76a33fc | 2010-05-24 14:32:36 -0700 | [diff] [blame] | 3219 | } |
Wu Fengguang | 6e08a36 | 2009-06-16 15:32:29 -0700 | [diff] [blame] | 3220 | } |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 3221 | |
Dave Hansen | 2f368a9 | 2021-09-02 14:59:23 -0700 | [diff] [blame] | 3222 | /* |
| 3223 | * Anonymous LRU management is a waste if there is |
| 3224 | * ultimately no way to reclaim the memory. |
| 3225 | */ |
| 3226 | static bool can_age_anon_pages(struct pglist_data *pgdat, |
| 3227 | struct scan_control *sc) |
| 3228 | { |
| 3229 | /* Aging the anon LRU is valuable if swap is present: */ |
| 3230 | if (total_swap_pages > 0) |
| 3231 | return true; |
| 3232 | |
| 3233 | /* Also valuable if anon pages can be demoted: */ |
| 3234 | return can_demote(pgdat->node_id, sc); |
| 3235 | } |
| 3236 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 3237 | #ifdef CONFIG_LRU_GEN |
| 3238 | |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 3239 | #ifdef CONFIG_LRU_GEN_ENABLED |
| 3240 | DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); |
| 3241 | #define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) |
| 3242 | #else |
| 3243 | DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); |
| 3244 | #define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) |
| 3245 | #endif |
| 3246 | |
T.J. Alumbaugh | bd02df4 | 2023-05-22 11:20:57 +0000 | [diff] [blame] | 3247 | static bool should_walk_mmu(void) |
| 3248 | { |
| 3249 | return arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK); |
| 3250 | } |
| 3251 | |
| 3252 | static bool should_clear_pmd_young(void) |
| 3253 | { |
| 3254 | return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG); |
| 3255 | } |
| 3256 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 3257 | /****************************************************************************** |
| 3258 | * shorthand helpers |
| 3259 | ******************************************************************************/ |
| 3260 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3261 | #define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) |
| 3262 | |
| 3263 | #define DEFINE_MAX_SEQ(lruvec) \ |
| 3264 | unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) |
| 3265 | |
| 3266 | #define DEFINE_MIN_SEQ(lruvec) \ |
| 3267 | unsigned long min_seq[ANON_AND_FILE] = { \ |
| 3268 | READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ |
| 3269 | READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ |
| 3270 | } |
| 3271 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 3272 | #define for_each_gen_type_zone(gen, type, zone) \ |
| 3273 | for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ |
| 3274 | for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ |
| 3275 | for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) |
| 3276 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 3277 | #define get_memcg_gen(seq) ((seq) % MEMCG_NR_GENS) |
| 3278 | #define get_memcg_bin(bin) ((bin) % MEMCG_NR_BINS) |
| 3279 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3280 | static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 3281 | { |
| 3282 | struct pglist_data *pgdat = NODE_DATA(nid); |
| 3283 | |
| 3284 | #ifdef CONFIG_MEMCG |
| 3285 | if (memcg) { |
| 3286 | struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; |
| 3287 | |
Yu Zhao | 931b6a8 | 2022-11-15 18:38:08 -0700 | [diff] [blame] | 3288 | /* see the comment in mem_cgroup_lruvec() */ |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 3289 | if (!lruvec->pgdat) |
| 3290 | lruvec->pgdat = pgdat; |
| 3291 | |
| 3292 | return lruvec; |
| 3293 | } |
| 3294 | #endif |
| 3295 | VM_WARN_ON_ONCE(!mem_cgroup_disabled()); |
| 3296 | |
Yu Zhao | 931b6a8 | 2022-11-15 18:38:08 -0700 | [diff] [blame] | 3297 | return &pgdat->__lruvec; |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 3298 | } |
| 3299 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3300 | static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) |
| 3301 | { |
| 3302 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 3303 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 3304 | |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 3305 | if (!sc->may_swap) |
| 3306 | return 0; |
| 3307 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3308 | if (!can_demote(pgdat->node_id, sc) && |
| 3309 | mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) |
| 3310 | return 0; |
| 3311 | |
| 3312 | return mem_cgroup_swappiness(memcg); |
| 3313 | } |
| 3314 | |
| 3315 | static int get_nr_gens(struct lruvec *lruvec, int type) |
| 3316 | { |
| 3317 | return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; |
| 3318 | } |
| 3319 | |
| 3320 | static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) |
| 3321 | { |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 3322 | /* see the comment on lru_gen_folio */ |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3323 | return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && |
| 3324 | get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && |
| 3325 | get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; |
| 3326 | } |
| 3327 | |
| 3328 | /****************************************************************************** |
T.J. Alumbaugh | ccbbbb8 | 2023-01-18 00:18:23 +0000 | [diff] [blame] | 3329 | * Bloom filters |
| 3330 | ******************************************************************************/ |
| 3331 | |
| 3332 | /* |
| 3333 | * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when |
| 3334 | * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of |
| 3335 | * bits in a bitmap, k is the number of hash functions and n is the number of |
| 3336 | * inserted items. |
| 3337 | * |
| 3338 | * Page table walkers use one of the two filters to reduce their search space. |
| 3339 | * To get rid of non-leaf entries that no longer have enough leaf entries, the |
| 3340 | * aging uses the double-buffering technique to flip to the other filter each |
| 3341 | * time it produces a new generation. For non-leaf entries that have enough |
| 3342 | * leaf entries, the aging carries them over to the next generation in |
| 3343 | * walk_pmd_range(); the eviction also report them when walking the rmap |
| 3344 | * in lru_gen_look_around(). |
| 3345 | * |
| 3346 | * For future optimizations: |
| 3347 | * 1. It's not necessary to keep both filters all the time. The spare one can be |
| 3348 | * freed after the RCU grace period and reallocated if needed again. |
| 3349 | * 2. And when reallocating, it's worth scaling its size according to the number |
| 3350 | * of inserted entries in the other filter, to reduce the memory overhead on |
| 3351 | * small systems and false positives on large systems. |
| 3352 | * 3. Jenkins' hash function is an alternative to Knuth's. |
| 3353 | */ |
| 3354 | #define BLOOM_FILTER_SHIFT 15 |
| 3355 | |
| 3356 | static inline int filter_gen_from_seq(unsigned long seq) |
| 3357 | { |
| 3358 | return seq % NR_BLOOM_FILTERS; |
| 3359 | } |
| 3360 | |
| 3361 | static void get_item_key(void *item, int *key) |
| 3362 | { |
| 3363 | u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); |
| 3364 | |
| 3365 | BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); |
| 3366 | |
| 3367 | key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); |
| 3368 | key[1] = hash >> BLOOM_FILTER_SHIFT; |
| 3369 | } |
| 3370 | |
| 3371 | static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) |
| 3372 | { |
| 3373 | int key[2]; |
| 3374 | unsigned long *filter; |
| 3375 | int gen = filter_gen_from_seq(seq); |
| 3376 | |
| 3377 | filter = READ_ONCE(lruvec->mm_state.filters[gen]); |
| 3378 | if (!filter) |
| 3379 | return true; |
| 3380 | |
| 3381 | get_item_key(item, key); |
| 3382 | |
| 3383 | return test_bit(key[0], filter) && test_bit(key[1], filter); |
| 3384 | } |
| 3385 | |
| 3386 | static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) |
| 3387 | { |
| 3388 | int key[2]; |
| 3389 | unsigned long *filter; |
| 3390 | int gen = filter_gen_from_seq(seq); |
| 3391 | |
| 3392 | filter = READ_ONCE(lruvec->mm_state.filters[gen]); |
| 3393 | if (!filter) |
| 3394 | return; |
| 3395 | |
| 3396 | get_item_key(item, key); |
| 3397 | |
| 3398 | if (!test_bit(key[0], filter)) |
| 3399 | set_bit(key[0], filter); |
| 3400 | if (!test_bit(key[1], filter)) |
| 3401 | set_bit(key[1], filter); |
| 3402 | } |
| 3403 | |
| 3404 | static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) |
| 3405 | { |
| 3406 | unsigned long *filter; |
| 3407 | int gen = filter_gen_from_seq(seq); |
| 3408 | |
| 3409 | filter = lruvec->mm_state.filters[gen]; |
| 3410 | if (filter) { |
| 3411 | bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); |
| 3412 | return; |
| 3413 | } |
| 3414 | |
| 3415 | filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), |
| 3416 | __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); |
| 3417 | WRITE_ONCE(lruvec->mm_state.filters[gen], filter); |
| 3418 | } |
| 3419 | |
| 3420 | /****************************************************************************** |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3421 | * mm_struct list |
| 3422 | ******************************************************************************/ |
| 3423 | |
| 3424 | static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) |
| 3425 | { |
| 3426 | static struct lru_gen_mm_list mm_list = { |
| 3427 | .fifo = LIST_HEAD_INIT(mm_list.fifo), |
| 3428 | .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), |
| 3429 | }; |
| 3430 | |
| 3431 | #ifdef CONFIG_MEMCG |
| 3432 | if (memcg) |
| 3433 | return &memcg->mm_list; |
| 3434 | #endif |
| 3435 | VM_WARN_ON_ONCE(!mem_cgroup_disabled()); |
| 3436 | |
| 3437 | return &mm_list; |
| 3438 | } |
| 3439 | |
| 3440 | void lru_gen_add_mm(struct mm_struct *mm) |
| 3441 | { |
| 3442 | int nid; |
| 3443 | struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); |
| 3444 | struct lru_gen_mm_list *mm_list = get_mm_list(memcg); |
| 3445 | |
| 3446 | VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); |
| 3447 | #ifdef CONFIG_MEMCG |
| 3448 | VM_WARN_ON_ONCE(mm->lru_gen.memcg); |
| 3449 | mm->lru_gen.memcg = memcg; |
| 3450 | #endif |
| 3451 | spin_lock(&mm_list->lock); |
| 3452 | |
| 3453 | for_each_node_state(nid, N_MEMORY) { |
| 3454 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 3455 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3456 | /* the first addition since the last iteration */ |
| 3457 | if (lruvec->mm_state.tail == &mm_list->fifo) |
| 3458 | lruvec->mm_state.tail = &mm->lru_gen.list; |
| 3459 | } |
| 3460 | |
| 3461 | list_add_tail(&mm->lru_gen.list, &mm_list->fifo); |
| 3462 | |
| 3463 | spin_unlock(&mm_list->lock); |
| 3464 | } |
| 3465 | |
| 3466 | void lru_gen_del_mm(struct mm_struct *mm) |
| 3467 | { |
| 3468 | int nid; |
| 3469 | struct lru_gen_mm_list *mm_list; |
| 3470 | struct mem_cgroup *memcg = NULL; |
| 3471 | |
| 3472 | if (list_empty(&mm->lru_gen.list)) |
| 3473 | return; |
| 3474 | |
| 3475 | #ifdef CONFIG_MEMCG |
| 3476 | memcg = mm->lru_gen.memcg; |
| 3477 | #endif |
| 3478 | mm_list = get_mm_list(memcg); |
| 3479 | |
| 3480 | spin_lock(&mm_list->lock); |
| 3481 | |
| 3482 | for_each_node(nid) { |
| 3483 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 3484 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3485 | /* where the current iteration continues after */ |
| 3486 | if (lruvec->mm_state.head == &mm->lru_gen.list) |
| 3487 | lruvec->mm_state.head = lruvec->mm_state.head->prev; |
| 3488 | |
| 3489 | /* where the last iteration ended before */ |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3490 | if (lruvec->mm_state.tail == &mm->lru_gen.list) |
| 3491 | lruvec->mm_state.tail = lruvec->mm_state.tail->next; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3492 | } |
| 3493 | |
| 3494 | list_del_init(&mm->lru_gen.list); |
| 3495 | |
| 3496 | spin_unlock(&mm_list->lock); |
| 3497 | |
| 3498 | #ifdef CONFIG_MEMCG |
| 3499 | mem_cgroup_put(mm->lru_gen.memcg); |
| 3500 | mm->lru_gen.memcg = NULL; |
| 3501 | #endif |
| 3502 | } |
| 3503 | |
| 3504 | #ifdef CONFIG_MEMCG |
| 3505 | void lru_gen_migrate_mm(struct mm_struct *mm) |
| 3506 | { |
| 3507 | struct mem_cgroup *memcg; |
| 3508 | struct task_struct *task = rcu_dereference_protected(mm->owner, true); |
| 3509 | |
| 3510 | VM_WARN_ON_ONCE(task->mm != mm); |
| 3511 | lockdep_assert_held(&task->alloc_lock); |
| 3512 | |
| 3513 | /* for mm_update_next_owner() */ |
| 3514 | if (mem_cgroup_disabled()) |
| 3515 | return; |
| 3516 | |
Yu Zhao | de08eaa | 2023-01-15 20:44:05 -0700 | [diff] [blame] | 3517 | /* migration can happen before addition */ |
| 3518 | if (!mm->lru_gen.memcg) |
| 3519 | return; |
| 3520 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3521 | rcu_read_lock(); |
| 3522 | memcg = mem_cgroup_from_task(task); |
| 3523 | rcu_read_unlock(); |
| 3524 | if (memcg == mm->lru_gen.memcg) |
| 3525 | return; |
| 3526 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3527 | VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); |
| 3528 | |
| 3529 | lru_gen_del_mm(mm); |
| 3530 | lru_gen_add_mm(mm); |
| 3531 | } |
| 3532 | #endif |
| 3533 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3534 | static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) |
| 3535 | { |
| 3536 | int i; |
| 3537 | int hist; |
| 3538 | |
| 3539 | lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); |
| 3540 | |
| 3541 | if (walk) { |
| 3542 | hist = lru_hist_from_seq(walk->max_seq); |
| 3543 | |
| 3544 | for (i = 0; i < NR_MM_STATS; i++) { |
| 3545 | WRITE_ONCE(lruvec->mm_state.stats[hist][i], |
| 3546 | lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); |
| 3547 | walk->mm_stats[i] = 0; |
| 3548 | } |
| 3549 | } |
| 3550 | |
| 3551 | if (NR_HIST_GENS > 1 && last) { |
| 3552 | hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); |
| 3553 | |
| 3554 | for (i = 0; i < NR_MM_STATS; i++) |
| 3555 | WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); |
| 3556 | } |
| 3557 | } |
| 3558 | |
| 3559 | static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) |
| 3560 | { |
| 3561 | int type; |
| 3562 | unsigned long size = 0; |
| 3563 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 3564 | int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); |
| 3565 | |
| 3566 | if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) |
| 3567 | return true; |
| 3568 | |
| 3569 | clear_bit(key, &mm->lru_gen.bitmap); |
| 3570 | |
| 3571 | for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { |
| 3572 | size += type ? get_mm_counter(mm, MM_FILEPAGES) : |
| 3573 | get_mm_counter(mm, MM_ANONPAGES) + |
| 3574 | get_mm_counter(mm, MM_SHMEMPAGES); |
| 3575 | } |
| 3576 | |
| 3577 | if (size < MIN_LRU_BATCH) |
| 3578 | return true; |
| 3579 | |
| 3580 | return !mmget_not_zero(mm); |
| 3581 | } |
| 3582 | |
| 3583 | static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, |
| 3584 | struct mm_struct **iter) |
| 3585 | { |
| 3586 | bool first = false; |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3587 | bool last = false; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3588 | struct mm_struct *mm = NULL; |
| 3589 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 3590 | struct lru_gen_mm_list *mm_list = get_mm_list(memcg); |
| 3591 | struct lru_gen_mm_state *mm_state = &lruvec->mm_state; |
| 3592 | |
| 3593 | /* |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3594 | * mm_state->seq is incremented after each iteration of mm_list. There |
| 3595 | * are three interesting cases for this page table walker: |
| 3596 | * 1. It tries to start a new iteration with a stale max_seq: there is |
| 3597 | * nothing left to do. |
| 3598 | * 2. It started the next iteration: it needs to reset the Bloom filter |
| 3599 | * so that a fresh set of PTE tables can be recorded. |
| 3600 | * 3. It ended the current iteration: it needs to reset the mm stats |
| 3601 | * counters and tell its caller to increment max_seq. |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3602 | */ |
| 3603 | spin_lock(&mm_list->lock); |
| 3604 | |
| 3605 | VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3606 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3607 | if (walk->max_seq <= mm_state->seq) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3608 | goto done; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3609 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3610 | if (!mm_state->head) |
| 3611 | mm_state->head = &mm_list->fifo; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3612 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3613 | if (mm_state->head == &mm_list->fifo) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3614 | first = true; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3615 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3616 | do { |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3617 | mm_state->head = mm_state->head->next; |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3618 | if (mm_state->head == &mm_list->fifo) { |
| 3619 | WRITE_ONCE(mm_state->seq, mm_state->seq + 1); |
| 3620 | last = true; |
| 3621 | break; |
| 3622 | } |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3623 | |
| 3624 | /* force scan for those added after the last iteration */ |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3625 | if (!mm_state->tail || mm_state->tail == mm_state->head) { |
| 3626 | mm_state->tail = mm_state->head->next; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3627 | walk->force_scan = true; |
| 3628 | } |
| 3629 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3630 | mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3631 | if (should_skip_mm(mm, walk)) |
| 3632 | mm = NULL; |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3633 | } while (!mm); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3634 | done: |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3635 | if (*iter || last) |
| 3636 | reset_mm_stats(lruvec, walk, last); |
| 3637 | |
| 3638 | spin_unlock(&mm_list->lock); |
| 3639 | |
| 3640 | if (mm && first) |
| 3641 | reset_bloom_filter(lruvec, walk->max_seq + 1); |
| 3642 | |
| 3643 | if (*iter) |
| 3644 | mmput_async(*iter); |
| 3645 | |
| 3646 | *iter = mm; |
| 3647 | |
| 3648 | return last; |
| 3649 | } |
| 3650 | |
| 3651 | static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) |
| 3652 | { |
| 3653 | bool success = false; |
| 3654 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 3655 | struct lru_gen_mm_list *mm_list = get_mm_list(memcg); |
| 3656 | struct lru_gen_mm_state *mm_state = &lruvec->mm_state; |
| 3657 | |
| 3658 | spin_lock(&mm_list->lock); |
| 3659 | |
| 3660 | VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); |
| 3661 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 3662 | if (max_seq > mm_state->seq) { |
| 3663 | mm_state->head = NULL; |
| 3664 | mm_state->tail = NULL; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3665 | WRITE_ONCE(mm_state->seq, mm_state->seq + 1); |
| 3666 | reset_mm_stats(lruvec, NULL, true); |
| 3667 | success = true; |
| 3668 | } |
| 3669 | |
| 3670 | spin_unlock(&mm_list->lock); |
| 3671 | |
| 3672 | return success; |
| 3673 | } |
| 3674 | |
| 3675 | /****************************************************************************** |
T.J. Alumbaugh | 32d32ef | 2023-02-14 03:54:45 +0000 | [diff] [blame] | 3676 | * PID controller |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3677 | ******************************************************************************/ |
| 3678 | |
| 3679 | /* |
| 3680 | * A feedback loop based on Proportional-Integral-Derivative (PID) controller. |
| 3681 | * |
| 3682 | * The P term is refaulted/(evicted+protected) from a tier in the generation |
| 3683 | * currently being evicted; the I term is the exponential moving average of the |
| 3684 | * P term over the generations previously evicted, using the smoothing factor |
| 3685 | * 1/2; the D term isn't supported. |
| 3686 | * |
| 3687 | * The setpoint (SP) is always the first tier of one type; the process variable |
| 3688 | * (PV) is either any tier of the other type or any other tier of the same |
| 3689 | * type. |
| 3690 | * |
| 3691 | * The error is the difference between the SP and the PV; the correction is to |
| 3692 | * turn off protection when SP>PV or turn on protection when SP<PV. |
| 3693 | * |
| 3694 | * For future optimizations: |
| 3695 | * 1. The D term may discount the other two terms over time so that long-lived |
| 3696 | * generations can resist stale information. |
| 3697 | */ |
| 3698 | struct ctrl_pos { |
| 3699 | unsigned long refaulted; |
| 3700 | unsigned long total; |
| 3701 | int gain; |
| 3702 | }; |
| 3703 | |
| 3704 | static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, |
| 3705 | struct ctrl_pos *pos) |
| 3706 | { |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 3707 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3708 | int hist = lru_hist_from_seq(lrugen->min_seq[type]); |
| 3709 | |
| 3710 | pos->refaulted = lrugen->avg_refaulted[type][tier] + |
| 3711 | atomic_long_read(&lrugen->refaulted[hist][type][tier]); |
| 3712 | pos->total = lrugen->avg_total[type][tier] + |
| 3713 | atomic_long_read(&lrugen->evicted[hist][type][tier]); |
| 3714 | if (tier) |
| 3715 | pos->total += lrugen->protected[hist][type][tier - 1]; |
| 3716 | pos->gain = gain; |
| 3717 | } |
| 3718 | |
| 3719 | static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) |
| 3720 | { |
| 3721 | int hist, tier; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 3722 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3723 | bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; |
| 3724 | unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; |
| 3725 | |
| 3726 | lockdep_assert_held(&lruvec->lru_lock); |
| 3727 | |
| 3728 | if (!carryover && !clear) |
| 3729 | return; |
| 3730 | |
| 3731 | hist = lru_hist_from_seq(seq); |
| 3732 | |
| 3733 | for (tier = 0; tier < MAX_NR_TIERS; tier++) { |
| 3734 | if (carryover) { |
| 3735 | unsigned long sum; |
| 3736 | |
| 3737 | sum = lrugen->avg_refaulted[type][tier] + |
| 3738 | atomic_long_read(&lrugen->refaulted[hist][type][tier]); |
| 3739 | WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); |
| 3740 | |
| 3741 | sum = lrugen->avg_total[type][tier] + |
| 3742 | atomic_long_read(&lrugen->evicted[hist][type][tier]); |
| 3743 | if (tier) |
| 3744 | sum += lrugen->protected[hist][type][tier - 1]; |
| 3745 | WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); |
| 3746 | } |
| 3747 | |
| 3748 | if (clear) { |
| 3749 | atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); |
| 3750 | atomic_long_set(&lrugen->evicted[hist][type][tier], 0); |
| 3751 | if (tier) |
| 3752 | WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); |
| 3753 | } |
| 3754 | } |
| 3755 | } |
| 3756 | |
| 3757 | static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) |
| 3758 | { |
| 3759 | /* |
| 3760 | * Return true if the PV has a limited number of refaults or a lower |
| 3761 | * refaulted/total than the SP. |
| 3762 | */ |
| 3763 | return pv->refaulted < MIN_LRU_BATCH || |
| 3764 | pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= |
| 3765 | (sp->refaulted + 1) * pv->total * pv->gain; |
| 3766 | } |
| 3767 | |
| 3768 | /****************************************************************************** |
| 3769 | * the aging |
| 3770 | ******************************************************************************/ |
| 3771 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 3772 | /* promote pages accessed through page tables */ |
| 3773 | static int folio_update_gen(struct folio *folio, int gen) |
| 3774 | { |
| 3775 | unsigned long new_flags, old_flags = READ_ONCE(folio->flags); |
| 3776 | |
| 3777 | VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); |
| 3778 | VM_WARN_ON_ONCE(!rcu_read_lock_held()); |
| 3779 | |
| 3780 | do { |
| 3781 | /* lru_gen_del_folio() has isolated this page? */ |
| 3782 | if (!(old_flags & LRU_GEN_MASK)) { |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 3783 | /* for shrink_folio_list() */ |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 3784 | new_flags = old_flags | BIT(PG_referenced); |
| 3785 | continue; |
| 3786 | } |
| 3787 | |
| 3788 | new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); |
| 3789 | new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; |
| 3790 | } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); |
| 3791 | |
| 3792 | return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; |
| 3793 | } |
| 3794 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3795 | /* protect pages accessed multiple times through file descriptors */ |
| 3796 | static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) |
| 3797 | { |
| 3798 | int type = folio_is_file_lru(folio); |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 3799 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3800 | int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); |
| 3801 | unsigned long new_flags, old_flags = READ_ONCE(folio->flags); |
| 3802 | |
| 3803 | VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); |
| 3804 | |
| 3805 | do { |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 3806 | new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; |
| 3807 | /* folio_update_gen() has promoted this page? */ |
| 3808 | if (new_gen >= 0 && new_gen != old_gen) |
| 3809 | return new_gen; |
| 3810 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 3811 | new_gen = (old_gen + 1) % MAX_NR_GENS; |
| 3812 | |
| 3813 | new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); |
| 3814 | new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; |
| 3815 | /* for folio_end_writeback() */ |
| 3816 | if (reclaiming) |
| 3817 | new_flags |= BIT(PG_reclaim); |
| 3818 | } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); |
| 3819 | |
| 3820 | lru_gen_update_size(lruvec, folio, old_gen, new_gen); |
| 3821 | |
| 3822 | return new_gen; |
| 3823 | } |
| 3824 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3825 | static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, |
| 3826 | int old_gen, int new_gen) |
| 3827 | { |
| 3828 | int type = folio_is_file_lru(folio); |
| 3829 | int zone = folio_zonenum(folio); |
| 3830 | int delta = folio_nr_pages(folio); |
| 3831 | |
| 3832 | VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); |
| 3833 | VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); |
| 3834 | |
| 3835 | walk->batched++; |
| 3836 | |
| 3837 | walk->nr_pages[old_gen][type][zone] -= delta; |
| 3838 | walk->nr_pages[new_gen][type][zone] += delta; |
| 3839 | } |
| 3840 | |
| 3841 | static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) |
| 3842 | { |
| 3843 | int gen, type, zone; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 3844 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3845 | |
| 3846 | walk->batched = 0; |
| 3847 | |
| 3848 | for_each_gen_type_zone(gen, type, zone) { |
| 3849 | enum lru_list lru = type * LRU_INACTIVE_FILE; |
| 3850 | int delta = walk->nr_pages[gen][type][zone]; |
| 3851 | |
| 3852 | if (!delta) |
| 3853 | continue; |
| 3854 | |
| 3855 | walk->nr_pages[gen][type][zone] = 0; |
| 3856 | WRITE_ONCE(lrugen->nr_pages[gen][type][zone], |
| 3857 | lrugen->nr_pages[gen][type][zone] + delta); |
| 3858 | |
| 3859 | if (lru_gen_is_active(lruvec, gen)) |
| 3860 | lru += LRU_ACTIVE; |
| 3861 | __update_lru_size(lruvec, lru, zone, delta); |
| 3862 | } |
| 3863 | } |
| 3864 | |
| 3865 | static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) |
| 3866 | { |
| 3867 | struct address_space *mapping; |
| 3868 | struct vm_area_struct *vma = args->vma; |
| 3869 | struct lru_gen_mm_walk *walk = args->private; |
| 3870 | |
| 3871 | if (!vma_is_accessible(vma)) |
| 3872 | return true; |
| 3873 | |
| 3874 | if (is_vm_hugetlb_page(vma)) |
| 3875 | return true; |
| 3876 | |
Yu Zhao | 8788f67 | 2022-12-30 14:52:51 -0700 | [diff] [blame] | 3877 | if (!vma_has_recency(vma)) |
| 3878 | return true; |
| 3879 | |
| 3880 | if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3881 | return true; |
| 3882 | |
| 3883 | if (vma == get_gate_vma(vma->vm_mm)) |
| 3884 | return true; |
| 3885 | |
| 3886 | if (vma_is_anonymous(vma)) |
| 3887 | return !walk->can_swap; |
| 3888 | |
| 3889 | if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) |
| 3890 | return true; |
| 3891 | |
| 3892 | mapping = vma->vm_file->f_mapping; |
| 3893 | if (mapping_unevictable(mapping)) |
| 3894 | return true; |
| 3895 | |
| 3896 | if (shmem_mapping(mapping)) |
| 3897 | return !walk->can_swap; |
| 3898 | |
| 3899 | /* to exclude special mappings like dax, etc. */ |
| 3900 | return !mapping->a_ops->read_folio; |
| 3901 | } |
| 3902 | |
| 3903 | /* |
| 3904 | * Some userspace memory allocators map many single-page VMAs. Instead of |
| 3905 | * returning back to the PGD table for each of such VMAs, finish an entire PMD |
| 3906 | * table to reduce zigzags and improve cache performance. |
| 3907 | */ |
| 3908 | static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, |
| 3909 | unsigned long *vm_start, unsigned long *vm_end) |
| 3910 | { |
| 3911 | unsigned long start = round_up(*vm_end, size); |
| 3912 | unsigned long end = (start | ~mask) + 1; |
Liam R. Howlett | 78ba531 | 2022-09-06 19:49:05 +0000 | [diff] [blame] | 3913 | VMA_ITERATOR(vmi, args->mm, start); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3914 | |
| 3915 | VM_WARN_ON_ONCE(mask & size); |
| 3916 | VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); |
| 3917 | |
Liam R. Howlett | 78ba531 | 2022-09-06 19:49:05 +0000 | [diff] [blame] | 3918 | for_each_vma(vmi, args->vma) { |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3919 | if (end && end <= args->vma->vm_start) |
| 3920 | return false; |
| 3921 | |
Liam R. Howlett | 78ba531 | 2022-09-06 19:49:05 +0000 | [diff] [blame] | 3922 | if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3923 | continue; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3924 | |
| 3925 | *vm_start = max(start, args->vma->vm_start); |
| 3926 | *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; |
| 3927 | |
| 3928 | return true; |
| 3929 | } |
| 3930 | |
| 3931 | return false; |
| 3932 | } |
| 3933 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 3934 | static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) |
| 3935 | { |
| 3936 | unsigned long pfn = pte_pfn(pte); |
| 3937 | |
| 3938 | VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); |
| 3939 | |
| 3940 | if (!pte_present(pte) || is_zero_pfn(pfn)) |
| 3941 | return -1; |
| 3942 | |
| 3943 | if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) |
| 3944 | return -1; |
| 3945 | |
| 3946 | if (WARN_ON_ONCE(!pfn_valid(pfn))) |
| 3947 | return -1; |
| 3948 | |
| 3949 | return pfn; |
| 3950 | } |
| 3951 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3952 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) |
| 3953 | static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) |
| 3954 | { |
| 3955 | unsigned long pfn = pmd_pfn(pmd); |
| 3956 | |
| 3957 | VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); |
| 3958 | |
| 3959 | if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) |
| 3960 | return -1; |
| 3961 | |
| 3962 | if (WARN_ON_ONCE(pmd_devmap(pmd))) |
| 3963 | return -1; |
| 3964 | |
| 3965 | if (WARN_ON_ONCE(!pfn_valid(pfn))) |
| 3966 | return -1; |
| 3967 | |
| 3968 | return pfn; |
| 3969 | } |
| 3970 | #endif |
| 3971 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 3972 | static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3973 | struct pglist_data *pgdat, bool can_swap) |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 3974 | { |
| 3975 | struct folio *folio; |
| 3976 | |
| 3977 | /* try to avoid unnecessary memory loads */ |
| 3978 | if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) |
| 3979 | return NULL; |
| 3980 | |
| 3981 | folio = pfn_folio(pfn); |
| 3982 | if (folio_nid(folio) != pgdat->node_id) |
| 3983 | return NULL; |
| 3984 | |
| 3985 | if (folio_memcg_rcu(folio) != memcg) |
| 3986 | return NULL; |
| 3987 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3988 | /* file VMAs can contain anon pages from COW */ |
| 3989 | if (!folio_is_file_lru(folio) && !can_swap) |
| 3990 | return NULL; |
| 3991 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 3992 | return folio; |
| 3993 | } |
| 3994 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 3995 | static bool suitable_to_scan(int total, int young) |
| 3996 | { |
| 3997 | int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); |
| 3998 | |
| 3999 | /* suitable if the average number of young PTEs per cacheline is >=1 */ |
| 4000 | return young * n >= total; |
| 4001 | } |
| 4002 | |
| 4003 | static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, |
| 4004 | struct mm_walk *args) |
| 4005 | { |
| 4006 | int i; |
| 4007 | pte_t *pte; |
| 4008 | spinlock_t *ptl; |
| 4009 | unsigned long addr; |
| 4010 | int total = 0; |
| 4011 | int young = 0; |
| 4012 | struct lru_gen_mm_walk *walk = args->private; |
| 4013 | struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); |
| 4014 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 4015 | int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); |
| 4016 | |
Hugh Dickins | 52fc048 | 2023-06-08 18:37:12 -0700 | [diff] [blame] | 4017 | pte = pte_offset_map_nolock(args->mm, pmd, start & PMD_MASK, &ptl); |
| 4018 | if (!pte) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4019 | return false; |
Hugh Dickins | 52fc048 | 2023-06-08 18:37:12 -0700 | [diff] [blame] | 4020 | if (!spin_trylock(ptl)) { |
| 4021 | pte_unmap(pte); |
| 4022 | return false; |
| 4023 | } |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4024 | |
| 4025 | arch_enter_lazy_mmu_mode(); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4026 | restart: |
| 4027 | for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { |
| 4028 | unsigned long pfn; |
| 4029 | struct folio *folio; |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4030 | pte_t ptent = ptep_get(pte + i); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4031 | |
| 4032 | total++; |
| 4033 | walk->mm_stats[MM_LEAF_TOTAL]++; |
| 4034 | |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4035 | pfn = get_pte_pfn(ptent, args->vma, addr); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4036 | if (pfn == -1) |
| 4037 | continue; |
| 4038 | |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4039 | if (!pte_young(ptent)) { |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4040 | walk->mm_stats[MM_LEAF_OLD]++; |
| 4041 | continue; |
| 4042 | } |
| 4043 | |
| 4044 | folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); |
| 4045 | if (!folio) |
| 4046 | continue; |
| 4047 | |
| 4048 | if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) |
| 4049 | VM_WARN_ON_ONCE(true); |
| 4050 | |
| 4051 | young++; |
| 4052 | walk->mm_stats[MM_LEAF_YOUNG]++; |
| 4053 | |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4054 | if (pte_dirty(ptent) && !folio_test_dirty(folio) && |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4055 | !(folio_test_anon(folio) && folio_test_swapbacked(folio) && |
| 4056 | !folio_test_swapcache(folio))) |
| 4057 | folio_mark_dirty(folio); |
| 4058 | |
| 4059 | old_gen = folio_update_gen(folio, new_gen); |
| 4060 | if (old_gen >= 0 && old_gen != new_gen) |
| 4061 | update_batch_size(walk, folio, old_gen, new_gen); |
| 4062 | } |
| 4063 | |
| 4064 | if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) |
| 4065 | goto restart; |
| 4066 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4067 | arch_leave_lazy_mmu_mode(); |
Hugh Dickins | 52fc048 | 2023-06-08 18:37:12 -0700 | [diff] [blame] | 4068 | pte_unmap_unlock(pte, ptl); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4069 | |
| 4070 | return suitable_to_scan(total, young); |
| 4071 | } |
| 4072 | |
| 4073 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4074 | static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma, |
| 4075 | struct mm_walk *args, unsigned long *bitmap, unsigned long *first) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4076 | { |
| 4077 | int i; |
| 4078 | pmd_t *pmd; |
| 4079 | spinlock_t *ptl; |
| 4080 | struct lru_gen_mm_walk *walk = args->private; |
| 4081 | struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); |
| 4082 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 4083 | int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); |
| 4084 | |
| 4085 | VM_WARN_ON_ONCE(pud_leaf(*pud)); |
| 4086 | |
| 4087 | /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4088 | if (*first == -1) { |
| 4089 | *first = addr; |
| 4090 | bitmap_zero(bitmap, MIN_LRU_BATCH); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4091 | return; |
| 4092 | } |
| 4093 | |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4094 | i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4095 | if (i && i <= MIN_LRU_BATCH) { |
| 4096 | __set_bit(i - 1, bitmap); |
| 4097 | return; |
| 4098 | } |
| 4099 | |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4100 | pmd = pmd_offset(pud, *first); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4101 | |
| 4102 | ptl = pmd_lockptr(args->mm, pmd); |
| 4103 | if (!spin_trylock(ptl)) |
| 4104 | goto done; |
| 4105 | |
| 4106 | arch_enter_lazy_mmu_mode(); |
| 4107 | |
| 4108 | do { |
| 4109 | unsigned long pfn; |
| 4110 | struct folio *folio; |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4111 | |
| 4112 | /* don't round down the first address */ |
| 4113 | addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4114 | |
| 4115 | pfn = get_pmd_pfn(pmd[i], vma, addr); |
| 4116 | if (pfn == -1) |
| 4117 | goto next; |
| 4118 | |
| 4119 | if (!pmd_trans_huge(pmd[i])) { |
T.J. Alumbaugh | bd02df4 | 2023-05-22 11:20:57 +0000 | [diff] [blame] | 4120 | if (should_clear_pmd_young()) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4121 | pmdp_test_and_clear_young(vma, addr, pmd + i); |
| 4122 | goto next; |
| 4123 | } |
| 4124 | |
| 4125 | folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); |
| 4126 | if (!folio) |
| 4127 | goto next; |
| 4128 | |
| 4129 | if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) |
| 4130 | goto next; |
| 4131 | |
| 4132 | walk->mm_stats[MM_LEAF_YOUNG]++; |
| 4133 | |
| 4134 | if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && |
| 4135 | !(folio_test_anon(folio) && folio_test_swapbacked(folio) && |
| 4136 | !folio_test_swapcache(folio))) |
| 4137 | folio_mark_dirty(folio); |
| 4138 | |
| 4139 | old_gen = folio_update_gen(folio, new_gen); |
| 4140 | if (old_gen >= 0 && old_gen != new_gen) |
| 4141 | update_batch_size(walk, folio, old_gen, new_gen); |
| 4142 | next: |
| 4143 | i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; |
| 4144 | } while (i <= MIN_LRU_BATCH); |
| 4145 | |
| 4146 | arch_leave_lazy_mmu_mode(); |
| 4147 | spin_unlock(ptl); |
| 4148 | done: |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4149 | *first = -1; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4150 | } |
| 4151 | #else |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4152 | static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma, |
| 4153 | struct mm_walk *args, unsigned long *bitmap, unsigned long *first) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4154 | { |
| 4155 | } |
| 4156 | #endif |
| 4157 | |
| 4158 | static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, |
| 4159 | struct mm_walk *args) |
| 4160 | { |
| 4161 | int i; |
| 4162 | pmd_t *pmd; |
| 4163 | unsigned long next; |
| 4164 | unsigned long addr; |
| 4165 | struct vm_area_struct *vma; |
T.J. Alumbaugh | 0285762 | 2023-05-22 11:20:55 +0000 | [diff] [blame] | 4166 | DECLARE_BITMAP(bitmap, MIN_LRU_BATCH); |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4167 | unsigned long first = -1; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4168 | struct lru_gen_mm_walk *walk = args->private; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4169 | |
| 4170 | VM_WARN_ON_ONCE(pud_leaf(*pud)); |
| 4171 | |
| 4172 | /* |
| 4173 | * Finish an entire PMD in two passes: the first only reaches to PTE |
| 4174 | * tables to avoid taking the PMD lock; the second, if necessary, takes |
| 4175 | * the PMD lock to clear the accessed bit in PMD entries. |
| 4176 | */ |
| 4177 | pmd = pmd_offset(pud, start & PUD_MASK); |
| 4178 | restart: |
| 4179 | /* walk_pte_range() may call get_next_vma() */ |
| 4180 | vma = args->vma; |
| 4181 | for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { |
Peter Zijlstra | dab6e71 | 2020-11-26 17:20:28 +0100 | [diff] [blame] | 4182 | pmd_t val = pmdp_get_lockless(pmd + i); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4183 | |
| 4184 | next = pmd_addr_end(addr, end); |
| 4185 | |
| 4186 | if (!pmd_present(val) || is_huge_zero_pmd(val)) { |
| 4187 | walk->mm_stats[MM_LEAF_TOTAL]++; |
| 4188 | continue; |
| 4189 | } |
| 4190 | |
| 4191 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 4192 | if (pmd_trans_huge(val)) { |
| 4193 | unsigned long pfn = pmd_pfn(val); |
| 4194 | struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); |
| 4195 | |
| 4196 | walk->mm_stats[MM_LEAF_TOTAL]++; |
| 4197 | |
| 4198 | if (!pmd_young(val)) { |
| 4199 | walk->mm_stats[MM_LEAF_OLD]++; |
| 4200 | continue; |
| 4201 | } |
| 4202 | |
| 4203 | /* try to avoid unnecessary memory loads */ |
| 4204 | if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) |
| 4205 | continue; |
| 4206 | |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4207 | walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4208 | continue; |
| 4209 | } |
| 4210 | #endif |
| 4211 | walk->mm_stats[MM_NONLEAF_TOTAL]++; |
| 4212 | |
T.J. Alumbaugh | bd02df4 | 2023-05-22 11:20:57 +0000 | [diff] [blame] | 4213 | if (should_clear_pmd_young()) { |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 4214 | if (!pmd_young(val)) |
| 4215 | continue; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4216 | |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4217 | walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 4218 | } |
Juergen Gross | 4aaf269 | 2022-11-23 07:45:10 +0100 | [diff] [blame] | 4219 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4220 | if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) |
| 4221 | continue; |
| 4222 | |
| 4223 | walk->mm_stats[MM_NONLEAF_FOUND]++; |
| 4224 | |
| 4225 | if (!walk_pte_range(&val, addr, next, args)) |
| 4226 | continue; |
| 4227 | |
| 4228 | walk->mm_stats[MM_NONLEAF_ADDED]++; |
| 4229 | |
| 4230 | /* carry over to the next generation */ |
| 4231 | update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); |
| 4232 | } |
| 4233 | |
T.J. Alumbaugh | b5ff413 | 2023-01-18 00:18:26 +0000 | [diff] [blame] | 4234 | walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4235 | |
| 4236 | if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) |
| 4237 | goto restart; |
| 4238 | } |
| 4239 | |
| 4240 | static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, |
| 4241 | struct mm_walk *args) |
| 4242 | { |
| 4243 | int i; |
| 4244 | pud_t *pud; |
| 4245 | unsigned long addr; |
| 4246 | unsigned long next; |
| 4247 | struct lru_gen_mm_walk *walk = args->private; |
| 4248 | |
| 4249 | VM_WARN_ON_ONCE(p4d_leaf(*p4d)); |
| 4250 | |
| 4251 | pud = pud_offset(p4d, start & P4D_MASK); |
| 4252 | restart: |
| 4253 | for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { |
| 4254 | pud_t val = READ_ONCE(pud[i]); |
| 4255 | |
| 4256 | next = pud_addr_end(addr, end); |
| 4257 | |
| 4258 | if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) |
| 4259 | continue; |
| 4260 | |
| 4261 | walk_pmd_range(&val, addr, next, args); |
| 4262 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4263 | if (need_resched() || walk->batched >= MAX_LRU_BATCH) { |
| 4264 | end = (addr | ~PUD_MASK) + 1; |
| 4265 | goto done; |
| 4266 | } |
| 4267 | } |
| 4268 | |
| 4269 | if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) |
| 4270 | goto restart; |
| 4271 | |
| 4272 | end = round_up(end, P4D_SIZE); |
| 4273 | done: |
| 4274 | if (!end || !args->vma) |
| 4275 | return 1; |
| 4276 | |
| 4277 | walk->next_addr = max(end, args->vma->vm_start); |
| 4278 | |
| 4279 | return -EAGAIN; |
| 4280 | } |
| 4281 | |
| 4282 | static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) |
| 4283 | { |
| 4284 | static const struct mm_walk_ops mm_walk_ops = { |
| 4285 | .test_walk = should_skip_vma, |
| 4286 | .p4d_entry = walk_pud_range, |
Suren Baghdasaryan | 49b0638 | 2023-08-04 08:27:19 -0700 | [diff] [blame] | 4287 | .walk_lock = PGWALK_RDLOCK, |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4288 | }; |
| 4289 | |
| 4290 | int err; |
| 4291 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 4292 | |
| 4293 | walk->next_addr = FIRST_USER_ADDRESS; |
| 4294 | |
| 4295 | do { |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 4296 | DEFINE_MAX_SEQ(lruvec); |
| 4297 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4298 | err = -EBUSY; |
| 4299 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 4300 | /* another thread might have called inc_max_seq() */ |
| 4301 | if (walk->max_seq != max_seq) |
| 4302 | break; |
| 4303 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4304 | /* folio_update_gen() requires stable folio_memcg() */ |
| 4305 | if (!mem_cgroup_trylock_pages(memcg)) |
| 4306 | break; |
| 4307 | |
| 4308 | /* the caller might be holding the lock for write */ |
| 4309 | if (mmap_read_trylock(mm)) { |
| 4310 | err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); |
| 4311 | |
| 4312 | mmap_read_unlock(mm); |
| 4313 | } |
| 4314 | |
| 4315 | mem_cgroup_unlock_pages(); |
| 4316 | |
| 4317 | if (walk->batched) { |
| 4318 | spin_lock_irq(&lruvec->lru_lock); |
| 4319 | reset_batch_size(lruvec, walk); |
| 4320 | spin_unlock_irq(&lruvec->lru_lock); |
| 4321 | } |
| 4322 | |
| 4323 | cond_resched(); |
| 4324 | } while (err == -EAGAIN); |
| 4325 | } |
| 4326 | |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 4327 | static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4328 | { |
| 4329 | struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; |
| 4330 | |
| 4331 | if (pgdat && current_is_kswapd()) { |
| 4332 | VM_WARN_ON_ONCE(walk); |
| 4333 | |
| 4334 | walk = &pgdat->mm_walk; |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 4335 | } else if (!walk && force_alloc) { |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4336 | VM_WARN_ON_ONCE(current_is_kswapd()); |
| 4337 | |
| 4338 | walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); |
| 4339 | } |
| 4340 | |
| 4341 | current->reclaim_state->mm_walk = walk; |
| 4342 | |
| 4343 | return walk; |
| 4344 | } |
| 4345 | |
| 4346 | static void clear_mm_walk(void) |
| 4347 | { |
| 4348 | struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; |
| 4349 | |
| 4350 | VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); |
| 4351 | VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); |
| 4352 | |
| 4353 | current->reclaim_state->mm_walk = NULL; |
| 4354 | |
| 4355 | if (!current_is_kswapd()) |
| 4356 | kfree(walk); |
| 4357 | } |
| 4358 | |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4359 | static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4360 | { |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4361 | int zone; |
| 4362 | int remaining = MAX_LRU_BATCH; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 4363 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4364 | int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4365 | |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4366 | if (type == LRU_GEN_ANON && !can_swap) |
| 4367 | goto done; |
| 4368 | |
| 4369 | /* prevent cold/hot inversion if force_scan is true */ |
| 4370 | for (zone = 0; zone < MAX_NR_ZONES; zone++) { |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 4371 | struct list_head *head = &lrugen->folios[old_gen][type][zone]; |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4372 | |
| 4373 | while (!list_empty(head)) { |
| 4374 | struct folio *folio = lru_to_folio(head); |
| 4375 | |
| 4376 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 4377 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); |
| 4378 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 4379 | VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); |
| 4380 | |
| 4381 | new_gen = folio_inc_gen(lruvec, folio, false); |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 4382 | list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4383 | |
| 4384 | if (!--remaining) |
| 4385 | return false; |
| 4386 | } |
| 4387 | } |
| 4388 | done: |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4389 | reset_ctrl_pos(lruvec, type, true); |
| 4390 | WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4391 | |
| 4392 | return true; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4393 | } |
| 4394 | |
| 4395 | static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) |
| 4396 | { |
| 4397 | int gen, type, zone; |
| 4398 | bool success = false; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 4399 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4400 | DEFINE_MIN_SEQ(lruvec); |
| 4401 | |
| 4402 | VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); |
| 4403 | |
| 4404 | /* find the oldest populated generation */ |
| 4405 | for (type = !can_swap; type < ANON_AND_FILE; type++) { |
| 4406 | while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { |
| 4407 | gen = lru_gen_from_seq(min_seq[type]); |
| 4408 | |
| 4409 | for (zone = 0; zone < MAX_NR_ZONES; zone++) { |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 4410 | if (!list_empty(&lrugen->folios[gen][type][zone])) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4411 | goto next; |
| 4412 | } |
| 4413 | |
| 4414 | min_seq[type]++; |
| 4415 | } |
| 4416 | next: |
| 4417 | ; |
| 4418 | } |
| 4419 | |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 4420 | /* see the comment on lru_gen_folio */ |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4421 | if (can_swap) { |
| 4422 | min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); |
| 4423 | min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); |
| 4424 | } |
| 4425 | |
| 4426 | for (type = !can_swap; type < ANON_AND_FILE; type++) { |
| 4427 | if (min_seq[type] == lrugen->min_seq[type]) |
| 4428 | continue; |
| 4429 | |
| 4430 | reset_ctrl_pos(lruvec, type, true); |
| 4431 | WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); |
| 4432 | success = true; |
| 4433 | } |
| 4434 | |
| 4435 | return success; |
| 4436 | } |
| 4437 | |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4438 | static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4439 | { |
| 4440 | int prev, next; |
| 4441 | int type, zone; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 4442 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Kalesh Singh | bb5e7f2 | 2023-08-01 19:56:03 -0700 | [diff] [blame] | 4443 | restart: |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4444 | spin_lock_irq(&lruvec->lru_lock); |
| 4445 | |
| 4446 | VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); |
| 4447 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4448 | for (type = ANON_AND_FILE - 1; type >= 0; type--) { |
| 4449 | if (get_nr_gens(lruvec, type) != MAX_NR_GENS) |
| 4450 | continue; |
| 4451 | |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4452 | VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4453 | |
Kalesh Singh | bb5e7f2 | 2023-08-01 19:56:03 -0700 | [diff] [blame] | 4454 | if (inc_min_seq(lruvec, type, can_swap)) |
| 4455 | continue; |
| 4456 | |
| 4457 | spin_unlock_irq(&lruvec->lru_lock); |
| 4458 | cond_resched(); |
| 4459 | goto restart; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4460 | } |
| 4461 | |
| 4462 | /* |
| 4463 | * Update the active/inactive LRU sizes for compatibility. Both sides of |
| 4464 | * the current max_seq need to be covered, since max_seq+1 can overlap |
| 4465 | * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do |
| 4466 | * overlap, cold/hot inversion happens. |
| 4467 | */ |
| 4468 | prev = lru_gen_from_seq(lrugen->max_seq - 1); |
| 4469 | next = lru_gen_from_seq(lrugen->max_seq + 1); |
| 4470 | |
| 4471 | for (type = 0; type < ANON_AND_FILE; type++) { |
| 4472 | for (zone = 0; zone < MAX_NR_ZONES; zone++) { |
| 4473 | enum lru_list lru = type * LRU_INACTIVE_FILE; |
| 4474 | long delta = lrugen->nr_pages[prev][type][zone] - |
| 4475 | lrugen->nr_pages[next][type][zone]; |
| 4476 | |
| 4477 | if (!delta) |
| 4478 | continue; |
| 4479 | |
| 4480 | __update_lru_size(lruvec, lru, zone, delta); |
| 4481 | __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); |
| 4482 | } |
| 4483 | } |
| 4484 | |
| 4485 | for (type = 0; type < ANON_AND_FILE; type++) |
| 4486 | reset_ctrl_pos(lruvec, type, false); |
| 4487 | |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 4488 | WRITE_ONCE(lrugen->timestamps[next], jiffies); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4489 | /* make sure preceding modifications appear */ |
| 4490 | smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4491 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4492 | spin_unlock_irq(&lruvec->lru_lock); |
| 4493 | } |
| 4494 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4495 | static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4496 | struct scan_control *sc, bool can_swap, bool force_scan) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4497 | { |
| 4498 | bool success; |
| 4499 | struct lru_gen_mm_walk *walk; |
| 4500 | struct mm_struct *mm = NULL; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 4501 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4502 | |
| 4503 | VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); |
| 4504 | |
| 4505 | /* see the comment in iterate_mm_list() */ |
| 4506 | if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { |
| 4507 | success = false; |
| 4508 | goto done; |
| 4509 | } |
| 4510 | |
| 4511 | /* |
| 4512 | * If the hardware doesn't automatically set the accessed bit, fallback |
| 4513 | * to lru_gen_look_around(), which only clears the accessed bit in a |
| 4514 | * handful of PTEs. Spreading the work out over a period of time usually |
| 4515 | * is less efficient, but it avoids bursty page faults. |
| 4516 | */ |
T.J. Alumbaugh | bd02df4 | 2023-05-22 11:20:57 +0000 | [diff] [blame] | 4517 | if (!should_walk_mmu()) { |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4518 | success = iterate_mm_list_nowalk(lruvec, max_seq); |
| 4519 | goto done; |
| 4520 | } |
| 4521 | |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 4522 | walk = set_mm_walk(NULL, true); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4523 | if (!walk) { |
| 4524 | success = iterate_mm_list_nowalk(lruvec, max_seq); |
| 4525 | goto done; |
| 4526 | } |
| 4527 | |
| 4528 | walk->lruvec = lruvec; |
| 4529 | walk->max_seq = max_seq; |
| 4530 | walk->can_swap = can_swap; |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 4531 | walk->force_scan = force_scan; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4532 | |
| 4533 | do { |
| 4534 | success = iterate_mm_list(lruvec, walk, &mm); |
| 4535 | if (mm) |
| 4536 | walk_mm(lruvec, mm, walk); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4537 | } while (mm); |
| 4538 | done: |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 4539 | if (success) |
| 4540 | inc_max_seq(lruvec, can_swap, force_scan); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4541 | |
Kalesh Singh | 7f63cf2 | 2023-04-13 14:43:26 -0700 | [diff] [blame] | 4542 | return success; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4543 | } |
| 4544 | |
T.J. Alumbaugh | 7b8144e | 2023-01-18 00:18:21 +0000 | [diff] [blame] | 4545 | /****************************************************************************** |
| 4546 | * working set protection |
| 4547 | ******************************************************************************/ |
| 4548 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4549 | static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4550 | { |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4551 | int gen, type, zone; |
| 4552 | unsigned long total = 0; |
| 4553 | bool can_swap = get_swappiness(lruvec, sc); |
| 4554 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4555 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 4556 | DEFINE_MAX_SEQ(lruvec); |
| 4557 | DEFINE_MIN_SEQ(lruvec); |
| 4558 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4559 | for (type = !can_swap; type < ANON_AND_FILE; type++) { |
| 4560 | unsigned long seq; |
| 4561 | |
| 4562 | for (seq = min_seq[type]; seq <= max_seq; seq++) { |
| 4563 | gen = lru_gen_from_seq(seq); |
| 4564 | |
| 4565 | for (zone = 0; zone < MAX_NR_ZONES; zone++) |
| 4566 | total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); |
| 4567 | } |
| 4568 | } |
| 4569 | |
| 4570 | /* whether the size is big enough to be helpful */ |
| 4571 | return mem_cgroup_online(memcg) ? (total >> sc->priority) : total; |
| 4572 | } |
| 4573 | |
| 4574 | static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc, |
| 4575 | unsigned long min_ttl) |
| 4576 | { |
| 4577 | int gen; |
| 4578 | unsigned long birth; |
| 4579 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 4580 | DEFINE_MIN_SEQ(lruvec); |
| 4581 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4582 | /* see the comment on lru_gen_folio */ |
| 4583 | gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); |
| 4584 | birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); |
| 4585 | |
| 4586 | if (time_is_after_jiffies(birth + min_ttl)) |
| 4587 | return false; |
| 4588 | |
| 4589 | if (!lruvec_is_sizable(lruvec, sc)) |
| 4590 | return false; |
| 4591 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4592 | mem_cgroup_calculate_protection(NULL, memcg); |
| 4593 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4594 | return !mem_cgroup_below_min(NULL, memcg); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4595 | } |
| 4596 | |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 4597 | /* to protect the working set of the last N jiffies */ |
| 4598 | static unsigned long lru_gen_min_ttl __read_mostly; |
| 4599 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4600 | static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) |
| 4601 | { |
| 4602 | struct mem_cgroup *memcg; |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 4603 | unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4604 | |
| 4605 | VM_WARN_ON_ONCE(!current_is_kswapd()); |
| 4606 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4607 | /* check the order to exclude compaction-induced reclaim */ |
| 4608 | if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY) |
Yu Zhao | f76c833 | 2022-09-18 02:00:06 -0600 | [diff] [blame] | 4609 | return; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4610 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4611 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 4612 | do { |
| 4613 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 4614 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4615 | if (lruvec_is_reclaimable(lruvec, sc, min_ttl)) { |
| 4616 | mem_cgroup_iter_break(NULL, memcg); |
| 4617 | return; |
| 4618 | } |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4619 | |
| 4620 | cond_resched(); |
| 4621 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4622 | |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 4623 | /* |
| 4624 | * The main goal is to OOM kill if every generation from all memcgs is |
| 4625 | * younger than min_ttl. However, another possibility is all memcgs are |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 4626 | * either too small or below min. |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 4627 | */ |
| 4628 | if (mutex_trylock(&oom_lock)) { |
| 4629 | struct oom_control oc = { |
| 4630 | .gfp_mask = sc->gfp_mask, |
| 4631 | }; |
| 4632 | |
| 4633 | out_of_memory(&oc); |
| 4634 | |
| 4635 | mutex_unlock(&oom_lock); |
| 4636 | } |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4637 | } |
| 4638 | |
T.J. Alumbaugh | db19a43 | 2023-01-18 00:18:22 +0000 | [diff] [blame] | 4639 | /****************************************************************************** |
| 4640 | * rmap/PT walk feedback |
| 4641 | ******************************************************************************/ |
| 4642 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4643 | /* |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 4644 | * This function exploits spatial locality when shrink_folio_list() walks the |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4645 | * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If |
| 4646 | * the scan was done cacheline efficiently, it adds the PMD entry pointing to |
| 4647 | * the PTE table to the Bloom filter. This forms a feedback loop between the |
| 4648 | * eviction and the aging. |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4649 | */ |
| 4650 | void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) |
| 4651 | { |
| 4652 | int i; |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4653 | unsigned long start; |
| 4654 | unsigned long end; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4655 | struct lru_gen_mm_walk *walk; |
| 4656 | int young = 0; |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4657 | pte_t *pte = pvmw->pte; |
| 4658 | unsigned long addr = pvmw->address; |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4659 | struct folio *folio = pfn_folio(pvmw->pfn); |
Kalesh Singh | a3235ea | 2023-08-01 19:56:04 -0700 | [diff] [blame] | 4660 | bool can_swap = !folio_is_file_lru(folio); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4661 | struct mem_cgroup *memcg = folio_memcg(folio); |
| 4662 | struct pglist_data *pgdat = folio_pgdat(folio); |
| 4663 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 4664 | DEFINE_MAX_SEQ(lruvec); |
| 4665 | int old_gen, new_gen = lru_gen_from_seq(max_seq); |
| 4666 | |
| 4667 | lockdep_assert_held(pvmw->ptl); |
| 4668 | VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); |
| 4669 | |
| 4670 | if (spin_is_contended(pvmw->ptl)) |
| 4671 | return; |
| 4672 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4673 | /* avoid taking the LRU lock under the PTL when possible */ |
| 4674 | walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; |
| 4675 | |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4676 | start = max(addr & PMD_MASK, pvmw->vma->vm_start); |
| 4677 | end = min(addr | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4678 | |
| 4679 | if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4680 | if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4681 | end = start + MIN_LRU_BATCH * PAGE_SIZE; |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4682 | else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4683 | start = end - MIN_LRU_BATCH * PAGE_SIZE; |
| 4684 | else { |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4685 | start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; |
| 4686 | end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4687 | } |
| 4688 | } |
| 4689 | |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4690 | /* folio_update_gen() requires stable folio_memcg() */ |
| 4691 | if (!mem_cgroup_trylock_pages(memcg)) |
| 4692 | return; |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4693 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4694 | arch_enter_lazy_mmu_mode(); |
| 4695 | |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4696 | pte -= (addr - start) / PAGE_SIZE; |
| 4697 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4698 | for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { |
| 4699 | unsigned long pfn; |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4700 | pte_t ptent = ptep_get(pte + i); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4701 | |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4702 | pfn = get_pte_pfn(ptent, pvmw->vma, addr); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4703 | if (pfn == -1) |
| 4704 | continue; |
| 4705 | |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4706 | if (!pte_young(ptent)) |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4707 | continue; |
| 4708 | |
Kalesh Singh | a3235ea | 2023-08-01 19:56:04 -0700 | [diff] [blame] | 4709 | folio = get_pfn_folio(pfn, memcg, pgdat, can_swap); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4710 | if (!folio) |
| 4711 | continue; |
| 4712 | |
| 4713 | if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) |
| 4714 | VM_WARN_ON_ONCE(true); |
| 4715 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4716 | young++; |
| 4717 | |
Ryan Roberts | c33c794 | 2023-06-12 16:15:45 +0100 | [diff] [blame] | 4718 | if (pte_dirty(ptent) && !folio_test_dirty(folio) && |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4719 | !(folio_test_anon(folio) && folio_test_swapbacked(folio) && |
| 4720 | !folio_test_swapcache(folio))) |
| 4721 | folio_mark_dirty(folio); |
| 4722 | |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4723 | if (walk) { |
| 4724 | old_gen = folio_update_gen(folio, new_gen); |
| 4725 | if (old_gen >= 0 && old_gen != new_gen) |
| 4726 | update_batch_size(walk, folio, old_gen, new_gen); |
| 4727 | |
| 4728 | continue; |
| 4729 | } |
| 4730 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4731 | old_gen = folio_lru_gen(folio); |
| 4732 | if (old_gen < 0) |
| 4733 | folio_set_referenced(folio); |
| 4734 | else if (old_gen != new_gen) |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4735 | folio_activate(folio); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4736 | } |
| 4737 | |
| 4738 | arch_leave_lazy_mmu_mode(); |
T.J. Alumbaugh | abf0867 | 2023-01-18 00:18:27 +0000 | [diff] [blame] | 4739 | mem_cgroup_unlock_pages(); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4740 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 4741 | /* feedback from rmap walkers to page table walkers */ |
| 4742 | if (suitable_to_scan(i, young)) |
| 4743 | update_bloom_filter(lruvec, max_seq, pvmw->pmd); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4744 | } |
| 4745 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4746 | /****************************************************************************** |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4747 | * memcg LRU |
| 4748 | ******************************************************************************/ |
| 4749 | |
| 4750 | /* see the comment on MEMCG_NR_GENS */ |
| 4751 | enum { |
| 4752 | MEMCG_LRU_NOP, |
| 4753 | MEMCG_LRU_HEAD, |
| 4754 | MEMCG_LRU_TAIL, |
| 4755 | MEMCG_LRU_OLD, |
| 4756 | MEMCG_LRU_YOUNG, |
| 4757 | }; |
| 4758 | |
| 4759 | #ifdef CONFIG_MEMCG |
| 4760 | |
| 4761 | static int lru_gen_memcg_seg(struct lruvec *lruvec) |
| 4762 | { |
| 4763 | return READ_ONCE(lruvec->lrugen.seg); |
| 4764 | } |
| 4765 | |
| 4766 | static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) |
| 4767 | { |
| 4768 | int seg; |
| 4769 | int old, new; |
Yu Zhao | 814bc1d | 2023-06-19 13:38:21 -0600 | [diff] [blame] | 4770 | unsigned long flags; |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4771 | int bin = get_random_u32_below(MEMCG_NR_BINS); |
| 4772 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 4773 | |
Yu Zhao | 814bc1d | 2023-06-19 13:38:21 -0600 | [diff] [blame] | 4774 | spin_lock_irqsave(&pgdat->memcg_lru.lock, flags); |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4775 | |
| 4776 | VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list)); |
| 4777 | |
| 4778 | seg = 0; |
| 4779 | new = old = lruvec->lrugen.gen; |
| 4780 | |
| 4781 | /* see the comment on MEMCG_NR_GENS */ |
| 4782 | if (op == MEMCG_LRU_HEAD) |
| 4783 | seg = MEMCG_LRU_HEAD; |
| 4784 | else if (op == MEMCG_LRU_TAIL) |
| 4785 | seg = MEMCG_LRU_TAIL; |
| 4786 | else if (op == MEMCG_LRU_OLD) |
| 4787 | new = get_memcg_gen(pgdat->memcg_lru.seq); |
| 4788 | else if (op == MEMCG_LRU_YOUNG) |
| 4789 | new = get_memcg_gen(pgdat->memcg_lru.seq + 1); |
| 4790 | else |
| 4791 | VM_WARN_ON_ONCE(true); |
| 4792 | |
| 4793 | hlist_nulls_del_rcu(&lruvec->lrugen.list); |
| 4794 | |
| 4795 | if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD) |
| 4796 | hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); |
| 4797 | else |
| 4798 | hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); |
| 4799 | |
| 4800 | pgdat->memcg_lru.nr_memcgs[old]--; |
| 4801 | pgdat->memcg_lru.nr_memcgs[new]++; |
| 4802 | |
| 4803 | lruvec->lrugen.gen = new; |
| 4804 | WRITE_ONCE(lruvec->lrugen.seg, seg); |
| 4805 | |
| 4806 | if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq)) |
| 4807 | WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); |
| 4808 | |
Yu Zhao | 814bc1d | 2023-06-19 13:38:21 -0600 | [diff] [blame] | 4809 | spin_unlock_irqrestore(&pgdat->memcg_lru.lock, flags); |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4810 | } |
| 4811 | |
| 4812 | void lru_gen_online_memcg(struct mem_cgroup *memcg) |
| 4813 | { |
| 4814 | int gen; |
| 4815 | int nid; |
| 4816 | int bin = get_random_u32_below(MEMCG_NR_BINS); |
| 4817 | |
| 4818 | for_each_node(nid) { |
| 4819 | struct pglist_data *pgdat = NODE_DATA(nid); |
| 4820 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 4821 | |
Yu Zhao | 814bc1d | 2023-06-19 13:38:21 -0600 | [diff] [blame] | 4822 | spin_lock_irq(&pgdat->memcg_lru.lock); |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4823 | |
| 4824 | VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list)); |
| 4825 | |
| 4826 | gen = get_memcg_gen(pgdat->memcg_lru.seq); |
| 4827 | |
| 4828 | hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]); |
| 4829 | pgdat->memcg_lru.nr_memcgs[gen]++; |
| 4830 | |
| 4831 | lruvec->lrugen.gen = gen; |
| 4832 | |
Yu Zhao | 814bc1d | 2023-06-19 13:38:21 -0600 | [diff] [blame] | 4833 | spin_unlock_irq(&pgdat->memcg_lru.lock); |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4834 | } |
| 4835 | } |
| 4836 | |
| 4837 | void lru_gen_offline_memcg(struct mem_cgroup *memcg) |
| 4838 | { |
| 4839 | int nid; |
| 4840 | |
| 4841 | for_each_node(nid) { |
| 4842 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 4843 | |
| 4844 | lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD); |
| 4845 | } |
| 4846 | } |
| 4847 | |
| 4848 | void lru_gen_release_memcg(struct mem_cgroup *memcg) |
| 4849 | { |
| 4850 | int gen; |
| 4851 | int nid; |
| 4852 | |
| 4853 | for_each_node(nid) { |
| 4854 | struct pglist_data *pgdat = NODE_DATA(nid); |
| 4855 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 4856 | |
Yu Zhao | 814bc1d | 2023-06-19 13:38:21 -0600 | [diff] [blame] | 4857 | spin_lock_irq(&pgdat->memcg_lru.lock); |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4858 | |
T.J. Mercier | 6867c7a | 2023-08-14 15:16:36 +0000 | [diff] [blame] | 4859 | if (hlist_nulls_unhashed(&lruvec->lrugen.list)) |
| 4860 | goto unlock; |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4861 | |
| 4862 | gen = lruvec->lrugen.gen; |
| 4863 | |
T.J. Mercier | 6867c7a | 2023-08-14 15:16:36 +0000 | [diff] [blame] | 4864 | hlist_nulls_del_init_rcu(&lruvec->lrugen.list); |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4865 | pgdat->memcg_lru.nr_memcgs[gen]--; |
| 4866 | |
| 4867 | if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq)) |
| 4868 | WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); |
T.J. Mercier | 6867c7a | 2023-08-14 15:16:36 +0000 | [diff] [blame] | 4869 | unlock: |
Yu Zhao | 814bc1d | 2023-06-19 13:38:21 -0600 | [diff] [blame] | 4870 | spin_unlock_irq(&pgdat->memcg_lru.lock); |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4871 | } |
| 4872 | } |
| 4873 | |
T.J. Alumbaugh | 5c7e7a0 | 2023-05-22 11:20:56 +0000 | [diff] [blame] | 4874 | void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4875 | { |
T.J. Alumbaugh | 5c7e7a0 | 2023-05-22 11:20:56 +0000 | [diff] [blame] | 4876 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 4877 | |
T.J. Alumbaugh | 36c7b4d | 2023-01-18 00:18:24 +0000 | [diff] [blame] | 4878 | /* see the comment on MEMCG_NR_GENS */ |
| 4879 | if (lru_gen_memcg_seg(lruvec) != MEMCG_LRU_HEAD) |
| 4880 | lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD); |
| 4881 | } |
| 4882 | |
| 4883 | #else /* !CONFIG_MEMCG */ |
| 4884 | |
| 4885 | static int lru_gen_memcg_seg(struct lruvec *lruvec) |
| 4886 | { |
| 4887 | return 0; |
| 4888 | } |
| 4889 | |
| 4890 | #endif |
| 4891 | |
| 4892 | /****************************************************************************** |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4893 | * the eviction |
| 4894 | ******************************************************************************/ |
| 4895 | |
Kalesh Singh | 669281e | 2023-08-01 19:56:02 -0700 | [diff] [blame] | 4896 | static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, |
| 4897 | int tier_idx) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4898 | { |
| 4899 | bool success; |
| 4900 | int gen = folio_lru_gen(folio); |
| 4901 | int type = folio_is_file_lru(folio); |
| 4902 | int zone = folio_zonenum(folio); |
| 4903 | int delta = folio_nr_pages(folio); |
| 4904 | int refs = folio_lru_refs(folio); |
| 4905 | int tier = lru_tier_from_refs(refs); |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 4906 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4907 | |
| 4908 | VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); |
| 4909 | |
| 4910 | /* unevictable */ |
| 4911 | if (!folio_evictable(folio)) { |
| 4912 | success = lru_gen_del_folio(lruvec, folio, true); |
| 4913 | VM_WARN_ON_ONCE_FOLIO(!success, folio); |
| 4914 | folio_set_unevictable(folio); |
| 4915 | lruvec_add_folio(lruvec, folio); |
| 4916 | __count_vm_events(UNEVICTABLE_PGCULLED, delta); |
| 4917 | return true; |
| 4918 | } |
| 4919 | |
| 4920 | /* dirty lazyfree */ |
| 4921 | if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { |
| 4922 | success = lru_gen_del_folio(lruvec, folio, true); |
| 4923 | VM_WARN_ON_ONCE_FOLIO(!success, folio); |
| 4924 | folio_set_swapbacked(folio); |
| 4925 | lruvec_add_folio_tail(lruvec, folio); |
| 4926 | return true; |
| 4927 | } |
| 4928 | |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4929 | /* promoted */ |
| 4930 | if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 4931 | list_move(&folio->lru, &lrugen->folios[gen][type][zone]); |
Yu Zhao | 018ee47 | 2022-09-18 02:00:04 -0600 | [diff] [blame] | 4932 | return true; |
| 4933 | } |
| 4934 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4935 | /* protected */ |
| 4936 | if (tier > tier_idx) { |
| 4937 | int hist = lru_hist_from_seq(lrugen->min_seq[type]); |
| 4938 | |
| 4939 | gen = folio_inc_gen(lruvec, folio, false); |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 4940 | list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4941 | |
| 4942 | WRITE_ONCE(lrugen->protected[hist][type][tier - 1], |
| 4943 | lrugen->protected[hist][type][tier - 1] + delta); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4944 | return true; |
| 4945 | } |
| 4946 | |
Kalesh Singh | 669281e | 2023-08-01 19:56:02 -0700 | [diff] [blame] | 4947 | /* ineligible */ |
Charan Teja Kalla | b7108d6 | 2023-08-09 13:35:44 +0530 | [diff] [blame] | 4948 | if (zone > sc->reclaim_idx || skip_cma(folio, sc)) { |
Kalesh Singh | 669281e | 2023-08-01 19:56:02 -0700 | [diff] [blame] | 4949 | gen = folio_inc_gen(lruvec, folio, false); |
| 4950 | list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); |
| 4951 | return true; |
| 4952 | } |
| 4953 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4954 | /* waiting for writeback */ |
| 4955 | if (folio_test_locked(folio) || folio_test_writeback(folio) || |
| 4956 | (type == LRU_GEN_FILE && folio_test_dirty(folio))) { |
| 4957 | gen = folio_inc_gen(lruvec, folio, true); |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 4958 | list_move(&folio->lru, &lrugen->folios[gen][type][zone]); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4959 | return true; |
| 4960 | } |
| 4961 | |
| 4962 | return false; |
| 4963 | } |
| 4964 | |
| 4965 | static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) |
| 4966 | { |
| 4967 | bool success; |
| 4968 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4969 | /* swapping inhibited */ |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 4970 | if (!(sc->gfp_mask & __GFP_IO) && |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4971 | (folio_test_dirty(folio) || |
| 4972 | (folio_test_anon(folio) && !folio_test_swapcache(folio)))) |
| 4973 | return false; |
| 4974 | |
| 4975 | /* raced with release_pages() */ |
| 4976 | if (!folio_try_get(folio)) |
| 4977 | return false; |
| 4978 | |
| 4979 | /* raced with another isolation */ |
| 4980 | if (!folio_test_clear_lru(folio)) { |
| 4981 | folio_put(folio); |
| 4982 | return false; |
| 4983 | } |
| 4984 | |
| 4985 | /* see the comment on MAX_NR_TIERS */ |
| 4986 | if (!folio_test_referenced(folio)) |
| 4987 | set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); |
| 4988 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 4989 | /* for shrink_folio_list() */ |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 4990 | folio_clear_reclaim(folio); |
| 4991 | folio_clear_referenced(folio); |
| 4992 | |
| 4993 | success = lru_gen_del_folio(lruvec, folio, true); |
| 4994 | VM_WARN_ON_ONCE_FOLIO(!success, folio); |
| 4995 | |
| 4996 | return true; |
| 4997 | } |
| 4998 | |
| 4999 | static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, |
| 5000 | int type, int tier, struct list_head *list) |
| 5001 | { |
Kalesh Singh | 669281e | 2023-08-01 19:56:02 -0700 | [diff] [blame] | 5002 | int i; |
| 5003 | int gen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5004 | enum vm_event_item item; |
| 5005 | int sorted = 0; |
| 5006 | int scanned = 0; |
| 5007 | int isolated = 0; |
| 5008 | int remaining = MAX_LRU_BATCH; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 5009 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5010 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5011 | |
| 5012 | VM_WARN_ON_ONCE(!list_empty(list)); |
| 5013 | |
| 5014 | if (get_nr_gens(lruvec, type) == MIN_NR_GENS) |
| 5015 | return 0; |
| 5016 | |
| 5017 | gen = lru_gen_from_seq(lrugen->min_seq[type]); |
| 5018 | |
Kalesh Singh | 669281e | 2023-08-01 19:56:02 -0700 | [diff] [blame] | 5019 | for (i = MAX_NR_ZONES; i > 0; i--) { |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5020 | LIST_HEAD(moved); |
| 5021 | int skipped = 0; |
Kalesh Singh | 669281e | 2023-08-01 19:56:02 -0700 | [diff] [blame] | 5022 | int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES; |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 5023 | struct list_head *head = &lrugen->folios[gen][type][zone]; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5024 | |
| 5025 | while (!list_empty(head)) { |
| 5026 | struct folio *folio = lru_to_folio(head); |
| 5027 | int delta = folio_nr_pages(folio); |
| 5028 | |
| 5029 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 5030 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); |
| 5031 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 5032 | VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); |
| 5033 | |
| 5034 | scanned += delta; |
| 5035 | |
Kalesh Singh | 669281e | 2023-08-01 19:56:02 -0700 | [diff] [blame] | 5036 | if (sort_folio(lruvec, folio, sc, tier)) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5037 | sorted += delta; |
| 5038 | else if (isolate_folio(lruvec, folio, sc)) { |
| 5039 | list_add(&folio->lru, list); |
| 5040 | isolated += delta; |
| 5041 | } else { |
| 5042 | list_move(&folio->lru, &moved); |
| 5043 | skipped += delta; |
| 5044 | } |
| 5045 | |
| 5046 | if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) |
| 5047 | break; |
| 5048 | } |
| 5049 | |
| 5050 | if (skipped) { |
| 5051 | list_splice(&moved, head); |
| 5052 | __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); |
| 5053 | } |
| 5054 | |
| 5055 | if (!remaining || isolated >= MIN_LRU_BATCH) |
| 5056 | break; |
| 5057 | } |
| 5058 | |
Johannes Weiner | 57e9cc5 | 2022-10-26 14:01:33 -0400 | [diff] [blame] | 5059 | item = PGSCAN_KSWAPD + reclaimer_offset(); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5060 | if (!cgroup_reclaim(sc)) { |
| 5061 | __count_vm_events(item, isolated); |
| 5062 | __count_vm_events(PGREFILL, sorted); |
| 5063 | } |
| 5064 | __count_memcg_events(memcg, item, isolated); |
| 5065 | __count_memcg_events(memcg, PGREFILL, sorted); |
| 5066 | __count_vm_events(PGSCAN_ANON + type, isolated); |
| 5067 | |
| 5068 | /* |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5069 | * There might not be eligible folios due to reclaim_idx. Check the |
| 5070 | * remaining to prevent livelock if it's not making progress. |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5071 | */ |
| 5072 | return isolated || !remaining ? scanned : 0; |
| 5073 | } |
| 5074 | |
| 5075 | static int get_tier_idx(struct lruvec *lruvec, int type) |
| 5076 | { |
| 5077 | int tier; |
| 5078 | struct ctrl_pos sp, pv; |
| 5079 | |
| 5080 | /* |
| 5081 | * To leave a margin for fluctuations, use a larger gain factor (1:2). |
| 5082 | * This value is chosen because any other tier would have at least twice |
| 5083 | * as many refaults as the first tier. |
| 5084 | */ |
| 5085 | read_ctrl_pos(lruvec, type, 0, 1, &sp); |
| 5086 | for (tier = 1; tier < MAX_NR_TIERS; tier++) { |
| 5087 | read_ctrl_pos(lruvec, type, tier, 2, &pv); |
| 5088 | if (!positive_ctrl_err(&sp, &pv)) |
| 5089 | break; |
| 5090 | } |
| 5091 | |
| 5092 | return tier - 1; |
| 5093 | } |
| 5094 | |
| 5095 | static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) |
| 5096 | { |
| 5097 | int type, tier; |
| 5098 | struct ctrl_pos sp, pv; |
| 5099 | int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; |
| 5100 | |
| 5101 | /* |
| 5102 | * Compare the first tier of anon with that of file to determine which |
| 5103 | * type to scan. Also need to compare other tiers of the selected type |
| 5104 | * with the first tier of the other type to determine the last tier (of |
| 5105 | * the selected type) to evict. |
| 5106 | */ |
| 5107 | read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); |
| 5108 | read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); |
| 5109 | type = positive_ctrl_err(&sp, &pv); |
| 5110 | |
| 5111 | read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); |
| 5112 | for (tier = 1; tier < MAX_NR_TIERS; tier++) { |
| 5113 | read_ctrl_pos(lruvec, type, tier, gain[type], &pv); |
| 5114 | if (!positive_ctrl_err(&sp, &pv)) |
| 5115 | break; |
| 5116 | } |
| 5117 | |
| 5118 | *tier_idx = tier - 1; |
| 5119 | |
| 5120 | return type; |
| 5121 | } |
| 5122 | |
| 5123 | static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, |
| 5124 | int *type_scanned, struct list_head *list) |
| 5125 | { |
| 5126 | int i; |
| 5127 | int type; |
| 5128 | int scanned; |
| 5129 | int tier = -1; |
| 5130 | DEFINE_MIN_SEQ(lruvec); |
| 5131 | |
| 5132 | /* |
| 5133 | * Try to make the obvious choice first. When anon and file are both |
| 5134 | * available from the same generation, interpret swappiness 1 as file |
| 5135 | * first and 200 as anon first. |
| 5136 | */ |
| 5137 | if (!swappiness) |
| 5138 | type = LRU_GEN_FILE; |
| 5139 | else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) |
| 5140 | type = LRU_GEN_ANON; |
| 5141 | else if (swappiness == 1) |
| 5142 | type = LRU_GEN_FILE; |
| 5143 | else if (swappiness == 200) |
| 5144 | type = LRU_GEN_ANON; |
| 5145 | else |
| 5146 | type = get_type_to_scan(lruvec, swappiness, &tier); |
| 5147 | |
| 5148 | for (i = !swappiness; i < ANON_AND_FILE; i++) { |
| 5149 | if (tier < 0) |
| 5150 | tier = get_tier_idx(lruvec, type); |
| 5151 | |
| 5152 | scanned = scan_folios(lruvec, sc, type, tier, list); |
| 5153 | if (scanned) |
| 5154 | break; |
| 5155 | |
| 5156 | type = !type; |
| 5157 | tier = -1; |
| 5158 | } |
| 5159 | |
| 5160 | *type_scanned = type; |
| 5161 | |
| 5162 | return scanned; |
| 5163 | } |
| 5164 | |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5165 | static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5166 | { |
| 5167 | int type; |
| 5168 | int scanned; |
| 5169 | int reclaimed; |
| 5170 | LIST_HEAD(list); |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5171 | LIST_HEAD(clean); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5172 | struct folio *folio; |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5173 | struct folio *next; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5174 | enum vm_event_item item; |
| 5175 | struct reclaim_stat stat; |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 5176 | struct lru_gen_mm_walk *walk; |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5177 | bool skip_retry = false; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5178 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5179 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 5180 | |
| 5181 | spin_lock_irq(&lruvec->lru_lock); |
| 5182 | |
| 5183 | scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); |
| 5184 | |
| 5185 | scanned += try_to_inc_min_seq(lruvec, swappiness); |
| 5186 | |
| 5187 | if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) |
| 5188 | scanned = 0; |
| 5189 | |
| 5190 | spin_unlock_irq(&lruvec->lru_lock); |
| 5191 | |
| 5192 | if (list_empty(&list)) |
| 5193 | return scanned; |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5194 | retry: |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 5195 | reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false); |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5196 | sc->nr_reclaimed += reclaimed; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5197 | |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5198 | list_for_each_entry_safe_reverse(folio, next, &list, lru) { |
| 5199 | if (!folio_evictable(folio)) { |
| 5200 | list_del(&folio->lru); |
| 5201 | folio_putback_lru(folio); |
| 5202 | continue; |
| 5203 | } |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5204 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5205 | if (folio_test_reclaim(folio) && |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5206 | (folio_test_dirty(folio) || folio_test_writeback(folio))) { |
| 5207 | /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ |
| 5208 | if (folio_test_workingset(folio)) |
| 5209 | folio_set_referenced(folio); |
| 5210 | continue; |
| 5211 | } |
| 5212 | |
| 5213 | if (skip_retry || folio_test_active(folio) || folio_test_referenced(folio) || |
| 5214 | folio_mapped(folio) || folio_test_locked(folio) || |
| 5215 | folio_test_dirty(folio) || folio_test_writeback(folio)) { |
| 5216 | /* don't add rejected folios to the oldest generation */ |
| 5217 | set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, |
| 5218 | BIT(PG_active)); |
| 5219 | continue; |
| 5220 | } |
| 5221 | |
| 5222 | /* retry folios that may have missed folio_rotate_reclaimable() */ |
| 5223 | list_move(&folio->lru, &clean); |
| 5224 | sc->nr_scanned -= folio_nr_pages(folio); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5225 | } |
| 5226 | |
| 5227 | spin_lock_irq(&lruvec->lru_lock); |
| 5228 | |
Matthew Wilcox (Oracle) | 49fd9b6 | 2022-09-02 20:45:57 +0100 | [diff] [blame] | 5229 | move_folios_to_lru(lruvec, &list); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5230 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 5231 | walk = current->reclaim_state->mm_walk; |
| 5232 | if (walk && walk->batched) |
| 5233 | reset_batch_size(lruvec, walk); |
| 5234 | |
Johannes Weiner | 57e9cc5 | 2022-10-26 14:01:33 -0400 | [diff] [blame] | 5235 | item = PGSTEAL_KSWAPD + reclaimer_offset(); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5236 | if (!cgroup_reclaim(sc)) |
| 5237 | __count_vm_events(item, reclaimed); |
| 5238 | __count_memcg_events(memcg, item, reclaimed); |
| 5239 | __count_vm_events(PGSTEAL_ANON + type, reclaimed); |
| 5240 | |
| 5241 | spin_unlock_irq(&lruvec->lru_lock); |
| 5242 | |
| 5243 | mem_cgroup_uncharge_list(&list); |
| 5244 | free_unref_page_list(&list); |
| 5245 | |
Yu Zhao | 359a5e1 | 2022-11-15 18:38:07 -0700 | [diff] [blame] | 5246 | INIT_LIST_HEAD(&list); |
| 5247 | list_splice_init(&clean, &list); |
| 5248 | |
| 5249 | if (!list_empty(&list)) { |
| 5250 | skip_retry = true; |
| 5251 | goto retry; |
| 5252 | } |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5253 | |
| 5254 | return scanned; |
| 5255 | } |
| 5256 | |
Yu Zhao | 77d4459 | 2022-12-21 21:19:03 -0700 | [diff] [blame] | 5257 | static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, |
| 5258 | struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) |
| 5259 | { |
| 5260 | int gen, type, zone; |
| 5261 | unsigned long old = 0; |
| 5262 | unsigned long young = 0; |
| 5263 | unsigned long total = 0; |
| 5264 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
| 5265 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5266 | DEFINE_MIN_SEQ(lruvec); |
| 5267 | |
| 5268 | /* whether this lruvec is completely out of cold folios */ |
| 5269 | if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) { |
| 5270 | *nr_to_scan = 0; |
| 5271 | return true; |
| 5272 | } |
| 5273 | |
| 5274 | for (type = !can_swap; type < ANON_AND_FILE; type++) { |
| 5275 | unsigned long seq; |
| 5276 | |
| 5277 | for (seq = min_seq[type]; seq <= max_seq; seq++) { |
| 5278 | unsigned long size = 0; |
| 5279 | |
| 5280 | gen = lru_gen_from_seq(seq); |
| 5281 | |
| 5282 | for (zone = 0; zone < MAX_NR_ZONES; zone++) |
| 5283 | size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); |
| 5284 | |
| 5285 | total += size; |
| 5286 | if (seq == max_seq) |
| 5287 | young += size; |
| 5288 | else if (seq + MIN_NR_GENS == max_seq) |
| 5289 | old += size; |
| 5290 | } |
| 5291 | } |
| 5292 | |
| 5293 | /* try to scrape all its memory if this memcg was deleted */ |
| 5294 | *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; |
| 5295 | |
| 5296 | /* |
| 5297 | * The aging tries to be lazy to reduce the overhead, while the eviction |
| 5298 | * stalls when the number of generations reaches MIN_NR_GENS. Hence, the |
| 5299 | * ideal number of generations is MIN_NR_GENS+1. |
| 5300 | */ |
| 5301 | if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) |
| 5302 | return false; |
| 5303 | |
| 5304 | /* |
| 5305 | * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) |
| 5306 | * of the total number of pages for each generation. A reasonable range |
| 5307 | * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The |
| 5308 | * aging cares about the upper bound of hot pages, while the eviction |
| 5309 | * cares about the lower bound of cold pages. |
| 5310 | */ |
| 5311 | if (young * MIN_NR_GENS > total) |
| 5312 | return true; |
| 5313 | if (old * (MIN_NR_GENS + 2) < total) |
| 5314 | return true; |
| 5315 | |
| 5316 | return false; |
| 5317 | } |
| 5318 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 5319 | /* |
| 5320 | * For future optimizations: |
| 5321 | * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg |
| 5322 | * reclaim. |
| 5323 | */ |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5324 | static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5325 | { |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5326 | unsigned long nr_to_scan; |
| 5327 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5328 | DEFINE_MAX_SEQ(lruvec); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5329 | |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5330 | if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5331 | return 0; |
| 5332 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 5333 | if (!should_run_aging(lruvec, max_seq, sc, can_swap, &nr_to_scan)) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5334 | return nr_to_scan; |
| 5335 | |
| 5336 | /* skip the aging path at the default priority */ |
| 5337 | if (sc->priority == DEF_PRIORITY) |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 5338 | return nr_to_scan; |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 5339 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 5340 | /* skip this lruvec as it's low on cold folios */ |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5341 | return try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false) ? -1 : 0; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5342 | } |
| 5343 | |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5344 | static unsigned long get_nr_to_reclaim(struct scan_control *sc) |
Yu Zhao | f76c833 | 2022-09-18 02:00:06 -0600 | [diff] [blame] | 5345 | { |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5346 | /* don't abort memcg reclaim to ensure fairness */ |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 5347 | if (!root_reclaim(sc)) |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5348 | return -1; |
Yu Zhao | f76c833 | 2022-09-18 02:00:06 -0600 | [diff] [blame] | 5349 | |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5350 | return max(sc->nr_to_reclaim, compact_gap(sc->order)); |
Yu Zhao | f76c833 | 2022-09-18 02:00:06 -0600 | [diff] [blame] | 5351 | } |
| 5352 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5353 | static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5354 | { |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5355 | long nr_to_scan; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5356 | unsigned long scanned = 0; |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5357 | unsigned long nr_to_reclaim = get_nr_to_reclaim(sc); |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5358 | int swappiness = get_swappiness(lruvec, sc); |
| 5359 | |
| 5360 | /* clean file folios are more likely to exist */ |
| 5361 | if (swappiness && !(sc->gfp_mask & __GFP_IO)) |
| 5362 | swappiness = 1; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5363 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5364 | while (true) { |
| 5365 | int delta; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5366 | |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 5367 | nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5368 | if (nr_to_scan <= 0) |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 5369 | break; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5370 | |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5371 | delta = evict_folios(lruvec, sc, swappiness); |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5372 | if (!delta) |
Yu Zhao | 7348cc9 | 2022-12-21 21:19:02 -0700 | [diff] [blame] | 5373 | break; |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5374 | |
| 5375 | scanned += delta; |
| 5376 | if (scanned >= nr_to_scan) |
| 5377 | break; |
| 5378 | |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 5379 | if (sc->nr_reclaimed >= nr_to_reclaim) |
Yu Zhao | f76c833 | 2022-09-18 02:00:06 -0600 | [diff] [blame] | 5380 | break; |
| 5381 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5382 | cond_resched(); |
| 5383 | } |
| 5384 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5385 | /* whether try_to_inc_max_seq() was successful */ |
| 5386 | return nr_to_scan < 0; |
| 5387 | } |
| 5388 | |
| 5389 | static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) |
| 5390 | { |
| 5391 | bool success; |
| 5392 | unsigned long scanned = sc->nr_scanned; |
| 5393 | unsigned long reclaimed = sc->nr_reclaimed; |
| 5394 | int seg = lru_gen_memcg_seg(lruvec); |
| 5395 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5396 | struct pglist_data *pgdat = lruvec_pgdat(lruvec); |
| 5397 | |
| 5398 | /* see the comment on MEMCG_NR_GENS */ |
| 5399 | if (!lruvec_is_sizable(lruvec, sc)) |
| 5400 | return seg != MEMCG_LRU_TAIL ? MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG; |
| 5401 | |
| 5402 | mem_cgroup_calculate_protection(NULL, memcg); |
| 5403 | |
| 5404 | if (mem_cgroup_below_min(NULL, memcg)) |
| 5405 | return MEMCG_LRU_YOUNG; |
| 5406 | |
| 5407 | if (mem_cgroup_below_low(NULL, memcg)) { |
| 5408 | /* see the comment on MEMCG_NR_GENS */ |
| 5409 | if (seg != MEMCG_LRU_TAIL) |
| 5410 | return MEMCG_LRU_TAIL; |
| 5411 | |
| 5412 | memcg_memory_event(memcg, MEMCG_LOW); |
| 5413 | } |
| 5414 | |
| 5415 | success = try_to_shrink_lruvec(lruvec, sc); |
| 5416 | |
| 5417 | shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority); |
| 5418 | |
| 5419 | if (!sc->proactive) |
| 5420 | vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned, |
| 5421 | sc->nr_reclaimed - reclaimed); |
| 5422 | |
Yosry Ahmed | 583c27a | 2023-04-13 10:40:32 +0000 | [diff] [blame] | 5423 | flush_reclaim_state(sc); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5424 | |
| 5425 | return success ? MEMCG_LRU_YOUNG : 0; |
| 5426 | } |
| 5427 | |
| 5428 | #ifdef CONFIG_MEMCG |
| 5429 | |
| 5430 | static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) |
| 5431 | { |
Yu Zhao | 9f550d7 | 2023-02-13 00:53:22 -0700 | [diff] [blame] | 5432 | int op; |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5433 | int gen; |
| 5434 | int bin; |
| 5435 | int first_bin; |
| 5436 | struct lruvec *lruvec; |
| 5437 | struct lru_gen_folio *lrugen; |
Yu Zhao | 9f550d7 | 2023-02-13 00:53:22 -0700 | [diff] [blame] | 5438 | struct mem_cgroup *memcg; |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5439 | const struct hlist_nulls_node *pos; |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5440 | unsigned long nr_to_reclaim = get_nr_to_reclaim(sc); |
| 5441 | |
| 5442 | bin = first_bin = get_random_u32_below(MEMCG_NR_BINS); |
| 5443 | restart: |
Yu Zhao | 9f550d7 | 2023-02-13 00:53:22 -0700 | [diff] [blame] | 5444 | op = 0; |
| 5445 | memcg = NULL; |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5446 | gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq)); |
| 5447 | |
| 5448 | rcu_read_lock(); |
| 5449 | |
| 5450 | hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) { |
T.J. Mercier | 6867c7a | 2023-08-14 15:16:36 +0000 | [diff] [blame] | 5451 | if (op) { |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5452 | lru_gen_rotate_memcg(lruvec, op); |
T.J. Mercier | 6867c7a | 2023-08-14 15:16:36 +0000 | [diff] [blame] | 5453 | op = 0; |
| 5454 | } |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5455 | |
| 5456 | mem_cgroup_put(memcg); |
| 5457 | |
| 5458 | lruvec = container_of(lrugen, struct lruvec, lrugen); |
| 5459 | memcg = lruvec_memcg(lruvec); |
| 5460 | |
| 5461 | if (!mem_cgroup_tryget(memcg)) { |
T.J. Mercier | 6867c7a | 2023-08-14 15:16:36 +0000 | [diff] [blame] | 5462 | lru_gen_release_memcg(memcg); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5463 | memcg = NULL; |
| 5464 | continue; |
| 5465 | } |
| 5466 | |
| 5467 | rcu_read_unlock(); |
| 5468 | |
| 5469 | op = shrink_one(lruvec, sc); |
| 5470 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5471 | rcu_read_lock(); |
Yu Zhao | 9f550d7 | 2023-02-13 00:53:22 -0700 | [diff] [blame] | 5472 | |
| 5473 | if (sc->nr_reclaimed >= nr_to_reclaim) |
| 5474 | break; |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5475 | } |
| 5476 | |
| 5477 | rcu_read_unlock(); |
| 5478 | |
Yu Zhao | 9f550d7 | 2023-02-13 00:53:22 -0700 | [diff] [blame] | 5479 | if (op) |
| 5480 | lru_gen_rotate_memcg(lruvec, op); |
| 5481 | |
| 5482 | mem_cgroup_put(memcg); |
| 5483 | |
| 5484 | if (sc->nr_reclaimed >= nr_to_reclaim) |
| 5485 | return; |
| 5486 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5487 | /* restart if raced with lru_gen_rotate_memcg() */ |
| 5488 | if (gen != get_nulls_value(pos)) |
| 5489 | goto restart; |
| 5490 | |
| 5491 | /* try the rest of the bins of the current generation */ |
| 5492 | bin = get_memcg_bin(bin + 1); |
| 5493 | if (bin != first_bin) |
| 5494 | goto restart; |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5495 | } |
| 5496 | |
| 5497 | static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
| 5498 | { |
| 5499 | struct blk_plug plug; |
| 5500 | |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 5501 | VM_WARN_ON_ONCE(root_reclaim(sc)); |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5502 | VM_WARN_ON_ONCE(!sc->may_writepage || !sc->may_unmap); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5503 | |
| 5504 | lru_add_drain(); |
| 5505 | |
| 5506 | blk_start_plug(&plug); |
| 5507 | |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5508 | set_mm_walk(NULL, sc->proactive); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5509 | |
| 5510 | if (try_to_shrink_lruvec(lruvec, sc)) |
| 5511 | lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG); |
| 5512 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 5513 | clear_mm_walk(); |
| 5514 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 5515 | blk_finish_plug(&plug); |
| 5516 | } |
| 5517 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5518 | #else /* !CONFIG_MEMCG */ |
| 5519 | |
| 5520 | static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) |
| 5521 | { |
| 5522 | BUILD_BUG(); |
| 5523 | } |
| 5524 | |
| 5525 | static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
| 5526 | { |
| 5527 | BUILD_BUG(); |
| 5528 | } |
| 5529 | |
| 5530 | #endif |
| 5531 | |
| 5532 | static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc) |
| 5533 | { |
| 5534 | int priority; |
| 5535 | unsigned long reclaimable; |
| 5536 | struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); |
| 5537 | |
| 5538 | if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH) |
| 5539 | return; |
| 5540 | /* |
| 5541 | * Determine the initial priority based on ((total / MEMCG_NR_GENS) >> |
| 5542 | * priority) * reclaimed_to_scanned_ratio = nr_to_reclaim, where the |
| 5543 | * estimated reclaimed_to_scanned_ratio = inactive / total. |
| 5544 | */ |
| 5545 | reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE); |
| 5546 | if (get_swappiness(lruvec, sc)) |
| 5547 | reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON); |
| 5548 | |
| 5549 | reclaimable /= MEMCG_NR_GENS; |
| 5550 | |
| 5551 | /* round down reclaimable and round up sc->nr_to_reclaim */ |
| 5552 | priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1); |
| 5553 | |
| 5554 | sc->priority = clamp(priority, 0, DEF_PRIORITY); |
| 5555 | } |
| 5556 | |
| 5557 | static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) |
| 5558 | { |
| 5559 | struct blk_plug plug; |
| 5560 | unsigned long reclaimed = sc->nr_reclaimed; |
| 5561 | |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 5562 | VM_WARN_ON_ONCE(!root_reclaim(sc)); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5563 | |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5564 | /* |
| 5565 | * Unmapped clean folios are already prioritized. Scanning for more of |
| 5566 | * them is likely futile and can cause high reclaim latency when there |
| 5567 | * is a large number of memcgs. |
| 5568 | */ |
| 5569 | if (!sc->may_writepage || !sc->may_unmap) |
| 5570 | goto done; |
| 5571 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5572 | lru_add_drain(); |
| 5573 | |
| 5574 | blk_start_plug(&plug); |
| 5575 | |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5576 | set_mm_walk(pgdat, sc->proactive); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5577 | |
| 5578 | set_initial_priority(pgdat, sc); |
| 5579 | |
| 5580 | if (current_is_kswapd()) |
| 5581 | sc->nr_reclaimed = 0; |
| 5582 | |
| 5583 | if (mem_cgroup_disabled()) |
| 5584 | shrink_one(&pgdat->__lruvec, sc); |
| 5585 | else |
| 5586 | shrink_many(pgdat, sc); |
| 5587 | |
| 5588 | if (current_is_kswapd()) |
| 5589 | sc->nr_reclaimed += reclaimed; |
| 5590 | |
| 5591 | clear_mm_walk(); |
| 5592 | |
| 5593 | blk_finish_plug(&plug); |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 5594 | done: |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 5595 | /* kswapd should never fail */ |
| 5596 | pgdat->kswapd_failures = 0; |
| 5597 | } |
| 5598 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 5599 | /****************************************************************************** |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5600 | * state change |
| 5601 | ******************************************************************************/ |
| 5602 | |
| 5603 | static bool __maybe_unused state_is_valid(struct lruvec *lruvec) |
| 5604 | { |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 5605 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5606 | |
| 5607 | if (lrugen->enabled) { |
| 5608 | enum lru_list lru; |
| 5609 | |
| 5610 | for_each_evictable_lru(lru) { |
| 5611 | if (!list_empty(&lruvec->lists[lru])) |
| 5612 | return false; |
| 5613 | } |
| 5614 | } else { |
| 5615 | int gen, type, zone; |
| 5616 | |
| 5617 | for_each_gen_type_zone(gen, type, zone) { |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 5618 | if (!list_empty(&lrugen->folios[gen][type][zone])) |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5619 | return false; |
| 5620 | } |
| 5621 | } |
| 5622 | |
| 5623 | return true; |
| 5624 | } |
| 5625 | |
| 5626 | static bool fill_evictable(struct lruvec *lruvec) |
| 5627 | { |
| 5628 | enum lru_list lru; |
| 5629 | int remaining = MAX_LRU_BATCH; |
| 5630 | |
| 5631 | for_each_evictable_lru(lru) { |
| 5632 | int type = is_file_lru(lru); |
| 5633 | bool active = is_active_lru(lru); |
| 5634 | struct list_head *head = &lruvec->lists[lru]; |
| 5635 | |
| 5636 | while (!list_empty(head)) { |
| 5637 | bool success; |
| 5638 | struct folio *folio = lru_to_folio(head); |
| 5639 | |
| 5640 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 5641 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); |
| 5642 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 5643 | VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); |
| 5644 | |
| 5645 | lruvec_del_folio(lruvec, folio); |
| 5646 | success = lru_gen_add_folio(lruvec, folio, false); |
| 5647 | VM_WARN_ON_ONCE(!success); |
| 5648 | |
| 5649 | if (!--remaining) |
| 5650 | return false; |
| 5651 | } |
| 5652 | } |
| 5653 | |
| 5654 | return true; |
| 5655 | } |
| 5656 | |
| 5657 | static bool drain_evictable(struct lruvec *lruvec) |
| 5658 | { |
| 5659 | int gen, type, zone; |
| 5660 | int remaining = MAX_LRU_BATCH; |
| 5661 | |
| 5662 | for_each_gen_type_zone(gen, type, zone) { |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 5663 | struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5664 | |
| 5665 | while (!list_empty(head)) { |
| 5666 | bool success; |
| 5667 | struct folio *folio = lru_to_folio(head); |
| 5668 | |
| 5669 | VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); |
| 5670 | VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); |
| 5671 | VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); |
| 5672 | VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); |
| 5673 | |
| 5674 | success = lru_gen_del_folio(lruvec, folio, false); |
| 5675 | VM_WARN_ON_ONCE(!success); |
| 5676 | lruvec_add_folio(lruvec, folio); |
| 5677 | |
| 5678 | if (!--remaining) |
| 5679 | return false; |
| 5680 | } |
| 5681 | } |
| 5682 | |
| 5683 | return true; |
| 5684 | } |
| 5685 | |
| 5686 | static void lru_gen_change_state(bool enabled) |
| 5687 | { |
| 5688 | static DEFINE_MUTEX(state_mutex); |
| 5689 | |
| 5690 | struct mem_cgroup *memcg; |
| 5691 | |
| 5692 | cgroup_lock(); |
| 5693 | cpus_read_lock(); |
| 5694 | get_online_mems(); |
| 5695 | mutex_lock(&state_mutex); |
| 5696 | |
| 5697 | if (enabled == lru_gen_enabled()) |
| 5698 | goto unlock; |
| 5699 | |
| 5700 | if (enabled) |
| 5701 | static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); |
| 5702 | else |
| 5703 | static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); |
| 5704 | |
| 5705 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 5706 | do { |
| 5707 | int nid; |
| 5708 | |
| 5709 | for_each_node(nid) { |
| 5710 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 5711 | |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5712 | spin_lock_irq(&lruvec->lru_lock); |
| 5713 | |
| 5714 | VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); |
| 5715 | VM_WARN_ON_ONCE(!state_is_valid(lruvec)); |
| 5716 | |
| 5717 | lruvec->lrugen.enabled = enabled; |
| 5718 | |
| 5719 | while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { |
| 5720 | spin_unlock_irq(&lruvec->lru_lock); |
| 5721 | cond_resched(); |
| 5722 | spin_lock_irq(&lruvec->lru_lock); |
| 5723 | } |
| 5724 | |
| 5725 | spin_unlock_irq(&lruvec->lru_lock); |
| 5726 | } |
| 5727 | |
| 5728 | cond_resched(); |
| 5729 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); |
| 5730 | unlock: |
| 5731 | mutex_unlock(&state_mutex); |
| 5732 | put_online_mems(); |
| 5733 | cpus_read_unlock(); |
| 5734 | cgroup_unlock(); |
| 5735 | } |
| 5736 | |
| 5737 | /****************************************************************************** |
| 5738 | * sysfs interface |
| 5739 | ******************************************************************************/ |
| 5740 | |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5741 | static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 5742 | { |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5743 | return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 5744 | } |
| 5745 | |
Yu Zhao | 07017ac | 2022-09-18 02:00:10 -0600 | [diff] [blame] | 5746 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5747 | static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr, |
| 5748 | const char *buf, size_t len) |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 5749 | { |
| 5750 | unsigned int msecs; |
| 5751 | |
| 5752 | if (kstrtouint(buf, 0, &msecs)) |
| 5753 | return -EINVAL; |
| 5754 | |
| 5755 | WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); |
| 5756 | |
| 5757 | return len; |
| 5758 | } |
| 5759 | |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5760 | static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms); |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 5761 | |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5762 | static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5763 | { |
| 5764 | unsigned int caps = 0; |
| 5765 | |
| 5766 | if (get_cap(LRU_GEN_CORE)) |
| 5767 | caps |= BIT(LRU_GEN_CORE); |
| 5768 | |
T.J. Alumbaugh | bd02df4 | 2023-05-22 11:20:57 +0000 | [diff] [blame] | 5769 | if (should_walk_mmu()) |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5770 | caps |= BIT(LRU_GEN_MM_WALK); |
| 5771 | |
T.J. Alumbaugh | bd02df4 | 2023-05-22 11:20:57 +0000 | [diff] [blame] | 5772 | if (should_clear_pmd_young()) |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5773 | caps |= BIT(LRU_GEN_NONLEAF_YOUNG); |
| 5774 | |
Xu Panda | 8ef9c32 | 2022-11-24 19:29:01 +0800 | [diff] [blame] | 5775 | return sysfs_emit(buf, "0x%04x\n", caps); |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5776 | } |
| 5777 | |
Yu Zhao | 07017ac | 2022-09-18 02:00:10 -0600 | [diff] [blame] | 5778 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5779 | static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5780 | const char *buf, size_t len) |
| 5781 | { |
| 5782 | int i; |
| 5783 | unsigned int caps; |
| 5784 | |
| 5785 | if (tolower(*buf) == 'n') |
| 5786 | caps = 0; |
| 5787 | else if (tolower(*buf) == 'y') |
| 5788 | caps = -1; |
| 5789 | else if (kstrtouint(buf, 0, &caps)) |
| 5790 | return -EINVAL; |
| 5791 | |
| 5792 | for (i = 0; i < NR_LRU_GEN_CAPS; i++) { |
| 5793 | bool enabled = caps & BIT(i); |
| 5794 | |
| 5795 | if (i == LRU_GEN_CORE) |
| 5796 | lru_gen_change_state(enabled); |
| 5797 | else if (enabled) |
| 5798 | static_branch_enable(&lru_gen_caps[i]); |
| 5799 | else |
| 5800 | static_branch_disable(&lru_gen_caps[i]); |
| 5801 | } |
| 5802 | |
| 5803 | return len; |
| 5804 | } |
| 5805 | |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5806 | static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled); |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5807 | |
| 5808 | static struct attribute *lru_gen_attrs[] = { |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 5809 | &lru_gen_min_ttl_attr.attr, |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5810 | &lru_gen_enabled_attr.attr, |
| 5811 | NULL |
| 5812 | }; |
| 5813 | |
T.J. Alumbaugh | 9a52b2f | 2023-02-14 03:54:44 +0000 | [diff] [blame] | 5814 | static const struct attribute_group lru_gen_attr_group = { |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 5815 | .name = "lru_gen", |
| 5816 | .attrs = lru_gen_attrs, |
| 5817 | }; |
| 5818 | |
| 5819 | /****************************************************************************** |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 5820 | * debugfs interface |
| 5821 | ******************************************************************************/ |
| 5822 | |
| 5823 | static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) |
| 5824 | { |
| 5825 | struct mem_cgroup *memcg; |
| 5826 | loff_t nr_to_skip = *pos; |
| 5827 | |
| 5828 | m->private = kvmalloc(PATH_MAX, GFP_KERNEL); |
| 5829 | if (!m->private) |
| 5830 | return ERR_PTR(-ENOMEM); |
| 5831 | |
| 5832 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 5833 | do { |
| 5834 | int nid; |
| 5835 | |
| 5836 | for_each_node_state(nid, N_MEMORY) { |
| 5837 | if (!nr_to_skip--) |
| 5838 | return get_lruvec(memcg, nid); |
| 5839 | } |
| 5840 | } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); |
| 5841 | |
| 5842 | return NULL; |
| 5843 | } |
| 5844 | |
| 5845 | static void lru_gen_seq_stop(struct seq_file *m, void *v) |
| 5846 | { |
| 5847 | if (!IS_ERR_OR_NULL(v)) |
| 5848 | mem_cgroup_iter_break(NULL, lruvec_memcg(v)); |
| 5849 | |
| 5850 | kvfree(m->private); |
| 5851 | m->private = NULL; |
| 5852 | } |
| 5853 | |
| 5854 | static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) |
| 5855 | { |
| 5856 | int nid = lruvec_pgdat(v)->node_id; |
| 5857 | struct mem_cgroup *memcg = lruvec_memcg(v); |
| 5858 | |
| 5859 | ++*pos; |
| 5860 | |
| 5861 | nid = next_memory_node(nid); |
| 5862 | if (nid == MAX_NUMNODES) { |
| 5863 | memcg = mem_cgroup_iter(NULL, memcg, NULL); |
| 5864 | if (!memcg) |
| 5865 | return NULL; |
| 5866 | |
| 5867 | nid = first_memory_node; |
| 5868 | } |
| 5869 | |
| 5870 | return get_lruvec(memcg, nid); |
| 5871 | } |
| 5872 | |
| 5873 | static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, |
| 5874 | unsigned long max_seq, unsigned long *min_seq, |
| 5875 | unsigned long seq) |
| 5876 | { |
| 5877 | int i; |
| 5878 | int type, tier; |
| 5879 | int hist = lru_hist_from_seq(seq); |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 5880 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 5881 | |
| 5882 | for (tier = 0; tier < MAX_NR_TIERS; tier++) { |
| 5883 | seq_printf(m, " %10d", tier); |
| 5884 | for (type = 0; type < ANON_AND_FILE; type++) { |
| 5885 | const char *s = " "; |
| 5886 | unsigned long n[3] = {}; |
| 5887 | |
| 5888 | if (seq == max_seq) { |
| 5889 | s = "RT "; |
| 5890 | n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); |
| 5891 | n[1] = READ_ONCE(lrugen->avg_total[type][tier]); |
| 5892 | } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { |
| 5893 | s = "rep"; |
| 5894 | n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); |
| 5895 | n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); |
| 5896 | if (tier) |
| 5897 | n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); |
| 5898 | } |
| 5899 | |
| 5900 | for (i = 0; i < 3; i++) |
| 5901 | seq_printf(m, " %10lu%c", n[i], s[i]); |
| 5902 | } |
| 5903 | seq_putc(m, '\n'); |
| 5904 | } |
| 5905 | |
| 5906 | seq_puts(m, " "); |
| 5907 | for (i = 0; i < NR_MM_STATS; i++) { |
| 5908 | const char *s = " "; |
| 5909 | unsigned long n = 0; |
| 5910 | |
| 5911 | if (seq == max_seq && NR_HIST_GENS == 1) { |
| 5912 | s = "LOYNFA"; |
| 5913 | n = READ_ONCE(lruvec->mm_state.stats[hist][i]); |
| 5914 | } else if (seq != max_seq && NR_HIST_GENS > 1) { |
| 5915 | s = "loynfa"; |
| 5916 | n = READ_ONCE(lruvec->mm_state.stats[hist][i]); |
| 5917 | } |
| 5918 | |
| 5919 | seq_printf(m, " %10lu%c", n, s[i]); |
| 5920 | } |
| 5921 | seq_putc(m, '\n'); |
| 5922 | } |
| 5923 | |
Yu Zhao | 07017ac | 2022-09-18 02:00:10 -0600 | [diff] [blame] | 5924 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 5925 | static int lru_gen_seq_show(struct seq_file *m, void *v) |
| 5926 | { |
| 5927 | unsigned long seq; |
| 5928 | bool full = !debugfs_real_fops(m->file)->write; |
| 5929 | struct lruvec *lruvec = v; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 5930 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 5931 | int nid = lruvec_pgdat(lruvec)->node_id; |
| 5932 | struct mem_cgroup *memcg = lruvec_memcg(lruvec); |
| 5933 | DEFINE_MAX_SEQ(lruvec); |
| 5934 | DEFINE_MIN_SEQ(lruvec); |
| 5935 | |
| 5936 | if (nid == first_memory_node) { |
| 5937 | const char *path = memcg ? m->private : ""; |
| 5938 | |
| 5939 | #ifdef CONFIG_MEMCG |
| 5940 | if (memcg) |
| 5941 | cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); |
| 5942 | #endif |
| 5943 | seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); |
| 5944 | } |
| 5945 | |
| 5946 | seq_printf(m, " node %5d\n", nid); |
| 5947 | |
| 5948 | if (!full) |
| 5949 | seq = min_seq[LRU_GEN_ANON]; |
| 5950 | else if (max_seq >= MAX_NR_GENS) |
| 5951 | seq = max_seq - MAX_NR_GENS + 1; |
| 5952 | else |
| 5953 | seq = 0; |
| 5954 | |
| 5955 | for (; seq <= max_seq; seq++) { |
| 5956 | int type, zone; |
| 5957 | int gen = lru_gen_from_seq(seq); |
| 5958 | unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); |
| 5959 | |
| 5960 | seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); |
| 5961 | |
| 5962 | for (type = 0; type < ANON_AND_FILE; type++) { |
| 5963 | unsigned long size = 0; |
| 5964 | char mark = full && seq < min_seq[type] ? 'x' : ' '; |
| 5965 | |
| 5966 | for (zone = 0; zone < MAX_NR_ZONES; zone++) |
| 5967 | size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); |
| 5968 | |
| 5969 | seq_printf(m, " %10lu%c", size, mark); |
| 5970 | } |
| 5971 | |
| 5972 | seq_putc(m, '\n'); |
| 5973 | |
| 5974 | if (full) |
| 5975 | lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); |
| 5976 | } |
| 5977 | |
| 5978 | return 0; |
| 5979 | } |
| 5980 | |
| 5981 | static const struct seq_operations lru_gen_seq_ops = { |
| 5982 | .start = lru_gen_seq_start, |
| 5983 | .stop = lru_gen_seq_stop, |
| 5984 | .next = lru_gen_seq_next, |
| 5985 | .show = lru_gen_seq_show, |
| 5986 | }; |
| 5987 | |
| 5988 | static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, |
| 5989 | bool can_swap, bool force_scan) |
| 5990 | { |
| 5991 | DEFINE_MAX_SEQ(lruvec); |
| 5992 | DEFINE_MIN_SEQ(lruvec); |
| 5993 | |
| 5994 | if (seq < max_seq) |
| 5995 | return 0; |
| 5996 | |
| 5997 | if (seq > max_seq) |
| 5998 | return -EINVAL; |
| 5999 | |
| 6000 | if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) |
| 6001 | return -ERANGE; |
| 6002 | |
| 6003 | try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); |
| 6004 | |
| 6005 | return 0; |
| 6006 | } |
| 6007 | |
| 6008 | static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, |
| 6009 | int swappiness, unsigned long nr_to_reclaim) |
| 6010 | { |
| 6011 | DEFINE_MAX_SEQ(lruvec); |
| 6012 | |
| 6013 | if (seq + MIN_NR_GENS > max_seq) |
| 6014 | return -EINVAL; |
| 6015 | |
| 6016 | sc->nr_reclaimed = 0; |
| 6017 | |
| 6018 | while (!signal_pending(current)) { |
| 6019 | DEFINE_MIN_SEQ(lruvec); |
| 6020 | |
| 6021 | if (seq < min_seq[!swappiness]) |
| 6022 | return 0; |
| 6023 | |
| 6024 | if (sc->nr_reclaimed >= nr_to_reclaim) |
| 6025 | return 0; |
| 6026 | |
Yu Zhao | a579086 | 2022-12-21 21:19:01 -0700 | [diff] [blame] | 6027 | if (!evict_folios(lruvec, sc, swappiness)) |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 6028 | return 0; |
| 6029 | |
| 6030 | cond_resched(); |
| 6031 | } |
| 6032 | |
| 6033 | return -EINTR; |
| 6034 | } |
| 6035 | |
| 6036 | static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, |
| 6037 | struct scan_control *sc, int swappiness, unsigned long opt) |
| 6038 | { |
| 6039 | struct lruvec *lruvec; |
| 6040 | int err = -EINVAL; |
| 6041 | struct mem_cgroup *memcg = NULL; |
| 6042 | |
| 6043 | if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) |
| 6044 | return -EINVAL; |
| 6045 | |
| 6046 | if (!mem_cgroup_disabled()) { |
| 6047 | rcu_read_lock(); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6048 | |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 6049 | memcg = mem_cgroup_from_id(memcg_id); |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6050 | if (!mem_cgroup_tryget(memcg)) |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 6051 | memcg = NULL; |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6052 | |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 6053 | rcu_read_unlock(); |
| 6054 | |
| 6055 | if (!memcg) |
| 6056 | return -EINVAL; |
| 6057 | } |
| 6058 | |
| 6059 | if (memcg_id != mem_cgroup_id(memcg)) |
| 6060 | goto done; |
| 6061 | |
| 6062 | lruvec = get_lruvec(memcg, nid); |
| 6063 | |
| 6064 | if (swappiness < 0) |
| 6065 | swappiness = get_swappiness(lruvec, sc); |
| 6066 | else if (swappiness > 200) |
| 6067 | goto done; |
| 6068 | |
| 6069 | switch (cmd) { |
| 6070 | case '+': |
| 6071 | err = run_aging(lruvec, seq, sc, swappiness, opt); |
| 6072 | break; |
| 6073 | case '-': |
| 6074 | err = run_eviction(lruvec, seq, sc, swappiness, opt); |
| 6075 | break; |
| 6076 | } |
| 6077 | done: |
| 6078 | mem_cgroup_put(memcg); |
| 6079 | |
| 6080 | return err; |
| 6081 | } |
| 6082 | |
Yu Zhao | 07017ac | 2022-09-18 02:00:10 -0600 | [diff] [blame] | 6083 | /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 6084 | static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, |
| 6085 | size_t len, loff_t *pos) |
| 6086 | { |
| 6087 | void *buf; |
| 6088 | char *cur, *next; |
| 6089 | unsigned int flags; |
| 6090 | struct blk_plug plug; |
| 6091 | int err = -EINVAL; |
| 6092 | struct scan_control sc = { |
| 6093 | .may_writepage = true, |
| 6094 | .may_unmap = true, |
| 6095 | .may_swap = true, |
| 6096 | .reclaim_idx = MAX_NR_ZONES - 1, |
| 6097 | .gfp_mask = GFP_KERNEL, |
| 6098 | }; |
| 6099 | |
| 6100 | buf = kvmalloc(len + 1, GFP_KERNEL); |
| 6101 | if (!buf) |
| 6102 | return -ENOMEM; |
| 6103 | |
| 6104 | if (copy_from_user(buf, src, len)) { |
| 6105 | kvfree(buf); |
| 6106 | return -EFAULT; |
| 6107 | } |
| 6108 | |
| 6109 | set_task_reclaim_state(current, &sc.reclaim_state); |
| 6110 | flags = memalloc_noreclaim_save(); |
| 6111 | blk_start_plug(&plug); |
Yu Zhao | e9d4e1e | 2022-12-21 21:19:05 -0700 | [diff] [blame] | 6112 | if (!set_mm_walk(NULL, true)) { |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 6113 | err = -ENOMEM; |
| 6114 | goto done; |
| 6115 | } |
| 6116 | |
| 6117 | next = buf; |
| 6118 | next[len] = '\0'; |
| 6119 | |
| 6120 | while ((cur = strsep(&next, ",;\n"))) { |
| 6121 | int n; |
| 6122 | int end; |
| 6123 | char cmd; |
| 6124 | unsigned int memcg_id; |
| 6125 | unsigned int nid; |
| 6126 | unsigned long seq; |
| 6127 | unsigned int swappiness = -1; |
| 6128 | unsigned long opt = -1; |
| 6129 | |
| 6130 | cur = skip_spaces(cur); |
| 6131 | if (!*cur) |
| 6132 | continue; |
| 6133 | |
| 6134 | n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, |
| 6135 | &seq, &end, &swappiness, &end, &opt, &end); |
| 6136 | if (n < 4 || cur[end]) { |
| 6137 | err = -EINVAL; |
| 6138 | break; |
| 6139 | } |
| 6140 | |
| 6141 | err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); |
| 6142 | if (err) |
| 6143 | break; |
| 6144 | } |
| 6145 | done: |
| 6146 | clear_mm_walk(); |
| 6147 | blk_finish_plug(&plug); |
| 6148 | memalloc_noreclaim_restore(flags); |
| 6149 | set_task_reclaim_state(current, NULL); |
| 6150 | |
| 6151 | kvfree(buf); |
| 6152 | |
| 6153 | return err ? : len; |
| 6154 | } |
| 6155 | |
| 6156 | static int lru_gen_seq_open(struct inode *inode, struct file *file) |
| 6157 | { |
| 6158 | return seq_open(file, &lru_gen_seq_ops); |
| 6159 | } |
| 6160 | |
| 6161 | static const struct file_operations lru_gen_rw_fops = { |
| 6162 | .open = lru_gen_seq_open, |
| 6163 | .read = seq_read, |
| 6164 | .write = lru_gen_seq_write, |
| 6165 | .llseek = seq_lseek, |
| 6166 | .release = seq_release, |
| 6167 | }; |
| 6168 | |
| 6169 | static const struct file_operations lru_gen_ro_fops = { |
| 6170 | .open = lru_gen_seq_open, |
| 6171 | .read = seq_read, |
| 6172 | .llseek = seq_lseek, |
| 6173 | .release = seq_release, |
| 6174 | }; |
| 6175 | |
| 6176 | /****************************************************************************** |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6177 | * initialization |
| 6178 | ******************************************************************************/ |
| 6179 | |
| 6180 | void lru_gen_init_lruvec(struct lruvec *lruvec) |
| 6181 | { |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 6182 | int i; |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6183 | int gen, type, zone; |
Yu Zhao | 391655f | 2022-12-21 21:18:59 -0700 | [diff] [blame] | 6184 | struct lru_gen_folio *lrugen = &lruvec->lrugen; |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6185 | |
| 6186 | lrugen->max_seq = MIN_NR_GENS + 1; |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 6187 | lrugen->enabled = lru_gen_enabled(); |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6188 | |
Yu Zhao | 1332a80 | 2022-09-18 02:00:08 -0600 | [diff] [blame] | 6189 | for (i = 0; i <= MIN_NR_GENS + 1; i++) |
| 6190 | lrugen->timestamps[i] = jiffies; |
| 6191 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6192 | for_each_gen_type_zone(gen, type, zone) |
Yu Zhao | 6df1b22 | 2022-12-21 21:19:00 -0700 | [diff] [blame] | 6193 | INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 6194 | |
| 6195 | lruvec->mm_state.seq = MIN_NR_GENS; |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6196 | } |
| 6197 | |
| 6198 | #ifdef CONFIG_MEMCG |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6199 | |
| 6200 | void lru_gen_init_pgdat(struct pglist_data *pgdat) |
| 6201 | { |
| 6202 | int i, j; |
| 6203 | |
| 6204 | spin_lock_init(&pgdat->memcg_lru.lock); |
| 6205 | |
| 6206 | for (i = 0; i < MEMCG_NR_GENS; i++) { |
| 6207 | for (j = 0; j < MEMCG_NR_BINS; j++) |
| 6208 | INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i); |
| 6209 | } |
| 6210 | } |
| 6211 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6212 | void lru_gen_init_memcg(struct mem_cgroup *memcg) |
| 6213 | { |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 6214 | INIT_LIST_HEAD(&memcg->mm_list.fifo); |
| 6215 | spin_lock_init(&memcg->mm_list.lock); |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6216 | } |
| 6217 | |
| 6218 | void lru_gen_exit_memcg(struct mem_cgroup *memcg) |
| 6219 | { |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 6220 | int i; |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6221 | int nid; |
| 6222 | |
T.J. Alumbaugh | 37cc999 | 2023-01-18 00:18:25 +0000 | [diff] [blame] | 6223 | VM_WARN_ON_ONCE(!list_empty(&memcg->mm_list.fifo)); |
| 6224 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6225 | for_each_node(nid) { |
| 6226 | struct lruvec *lruvec = get_lruvec(memcg, nid); |
| 6227 | |
| 6228 | VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, |
| 6229 | sizeof(lruvec->lrugen.nr_pages))); |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 6230 | |
T.J. Alumbaugh | 37cc999 | 2023-01-18 00:18:25 +0000 | [diff] [blame] | 6231 | lruvec->lrugen.list.next = LIST_POISON1; |
| 6232 | |
Yu Zhao | bd74fda | 2022-09-18 02:00:05 -0600 | [diff] [blame] | 6233 | for (i = 0; i < NR_BLOOM_FILTERS; i++) { |
| 6234 | bitmap_free(lruvec->mm_state.filters[i]); |
| 6235 | lruvec->mm_state.filters[i] = NULL; |
| 6236 | } |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6237 | } |
| 6238 | } |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6239 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6240 | #endif /* CONFIG_MEMCG */ |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6241 | |
| 6242 | static int __init init_lru_gen(void) |
| 6243 | { |
| 6244 | BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); |
| 6245 | BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); |
| 6246 | |
Yu Zhao | 354ed59 | 2022-09-18 02:00:07 -0600 | [diff] [blame] | 6247 | if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) |
| 6248 | pr_err("lru_gen: failed to create sysfs group\n"); |
| 6249 | |
Yu Zhao | d6c3af7 | 2022-09-18 02:00:09 -0600 | [diff] [blame] | 6250 | debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); |
| 6251 | debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); |
| 6252 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6253 | return 0; |
| 6254 | }; |
| 6255 | late_initcall(init_lru_gen); |
| 6256 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 6257 | #else /* !CONFIG_LRU_GEN */ |
| 6258 | |
| 6259 | static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) |
| 6260 | { |
| 6261 | } |
| 6262 | |
| 6263 | static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
| 6264 | { |
| 6265 | } |
| 6266 | |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6267 | static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) |
| 6268 | { |
| 6269 | } |
| 6270 | |
Yu Zhao | ec1c86b | 2022-09-18 02:00:02 -0600 | [diff] [blame] | 6271 | #endif /* CONFIG_LRU_GEN */ |
| 6272 | |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 6273 | static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6274 | { |
| 6275 | unsigned long nr[NR_LRU_LISTS]; |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6276 | unsigned long targets[NR_LRU_LISTS]; |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6277 | unsigned long nr_to_scan; |
| 6278 | enum lru_list lru; |
| 6279 | unsigned long nr_reclaimed = 0; |
| 6280 | unsigned long nr_to_reclaim = sc->nr_to_reclaim; |
Johannes Weiner | f53af42 | 2022-08-02 12:28:11 -0400 | [diff] [blame] | 6281 | bool proportional_reclaim; |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6282 | struct blk_plug plug; |
| 6283 | |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 6284 | if (lru_gen_enabled() && !root_reclaim(sc)) { |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 6285 | lru_gen_shrink_lruvec(lruvec, sc); |
| 6286 | return; |
| 6287 | } |
| 6288 | |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 6289 | get_scan_count(lruvec, sc, nr); |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6290 | |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6291 | /* Record the original scan target for proportional adjustments later */ |
| 6292 | memcpy(targets, nr, sizeof(nr)); |
| 6293 | |
Mel Gorman | 1a50190 | 2014-06-04 16:10:49 -0700 | [diff] [blame] | 6294 | /* |
| 6295 | * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal |
| 6296 | * event that can occur when there is little memory pressure e.g. |
| 6297 | * multiple streaming readers/writers. Hence, we do not abort scanning |
| 6298 | * when the requested number of pages are reclaimed when scanning at |
| 6299 | * DEF_PRIORITY on the assumption that the fact we are direct |
| 6300 | * reclaiming implies that kswapd is not keeping up and it is best to |
| 6301 | * do a batch of work at once. For memcg reclaim one check is made to |
| 6302 | * abort proportional reclaim if either the file or anon lru has already |
| 6303 | * dropped to zero at the first pass. |
| 6304 | */ |
Johannes Weiner | f53af42 | 2022-08-02 12:28:11 -0400 | [diff] [blame] | 6305 | proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && |
| 6306 | sc->priority == DEF_PRIORITY); |
Mel Gorman | 1a50190 | 2014-06-04 16:10:49 -0700 | [diff] [blame] | 6307 | |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6308 | blk_start_plug(&plug); |
| 6309 | while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || |
| 6310 | nr[LRU_INACTIVE_FILE]) { |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6311 | unsigned long nr_anon, nr_file, percentage; |
| 6312 | unsigned long nr_scanned; |
| 6313 | |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6314 | for_each_evictable_lru(lru) { |
| 6315 | if (nr[lru]) { |
| 6316 | nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); |
| 6317 | nr[lru] -= nr_to_scan; |
| 6318 | |
| 6319 | nr_reclaimed += shrink_list(lru, nr_to_scan, |
Johannes Weiner | 3b99120 | 2019-04-18 17:50:34 -0700 | [diff] [blame] | 6320 | lruvec, sc); |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6321 | } |
| 6322 | } |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6323 | |
Michal Hocko | bd04173 | 2016-12-02 17:26:48 -0800 | [diff] [blame] | 6324 | cond_resched(); |
| 6325 | |
Johannes Weiner | f53af42 | 2022-08-02 12:28:11 -0400 | [diff] [blame] | 6326 | if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6327 | continue; |
| 6328 | |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6329 | /* |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6330 | * For kswapd and memcg, reclaim at least the number of pages |
Mel Gorman | 1a50190 | 2014-06-04 16:10:49 -0700 | [diff] [blame] | 6331 | * requested. Ensure that the anon and file LRUs are scanned |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6332 | * proportionally what was requested by get_scan_count(). We |
| 6333 | * stop reclaiming one LRU and reduce the amount scanning |
| 6334 | * proportional to the original scan target. |
| 6335 | */ |
| 6336 | nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; |
| 6337 | nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; |
| 6338 | |
Mel Gorman | 1a50190 | 2014-06-04 16:10:49 -0700 | [diff] [blame] | 6339 | /* |
| 6340 | * It's just vindictive to attack the larger once the smaller |
| 6341 | * has gone to zero. And given the way we stop scanning the |
| 6342 | * smaller below, this makes sure that we only make one nudge |
| 6343 | * towards proportionality once we've got nr_to_reclaim. |
| 6344 | */ |
| 6345 | if (!nr_file || !nr_anon) |
| 6346 | break; |
| 6347 | |
Mel Gorman | e82e056 | 2013-07-03 15:01:44 -0700 | [diff] [blame] | 6348 | if (nr_file > nr_anon) { |
| 6349 | unsigned long scan_target = targets[LRU_INACTIVE_ANON] + |
| 6350 | targets[LRU_ACTIVE_ANON] + 1; |
| 6351 | lru = LRU_BASE; |
| 6352 | percentage = nr_anon * 100 / scan_target; |
| 6353 | } else { |
| 6354 | unsigned long scan_target = targets[LRU_INACTIVE_FILE] + |
| 6355 | targets[LRU_ACTIVE_FILE] + 1; |
| 6356 | lru = LRU_FILE; |
| 6357 | percentage = nr_file * 100 / scan_target; |
| 6358 | } |
| 6359 | |
| 6360 | /* Stop scanning the smaller of the LRU */ |
| 6361 | nr[lru] = 0; |
| 6362 | nr[lru + LRU_ACTIVE] = 0; |
| 6363 | |
| 6364 | /* |
| 6365 | * Recalculate the other LRU scan count based on its original |
| 6366 | * scan target and the percentage scanning already complete |
| 6367 | */ |
| 6368 | lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; |
| 6369 | nr_scanned = targets[lru] - nr[lru]; |
| 6370 | nr[lru] = targets[lru] * (100 - percentage) / 100; |
| 6371 | nr[lru] -= min(nr[lru], nr_scanned); |
| 6372 | |
| 6373 | lru += LRU_ACTIVE; |
| 6374 | nr_scanned = targets[lru] - nr[lru]; |
| 6375 | nr[lru] = targets[lru] * (100 - percentage) / 100; |
| 6376 | nr[lru] -= min(nr[lru], nr_scanned); |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6377 | } |
| 6378 | blk_finish_plug(&plug); |
| 6379 | sc->nr_reclaimed += nr_reclaimed; |
| 6380 | |
| 6381 | /* |
| 6382 | * Even if we did not try to evict anon pages at all, we want to |
| 6383 | * rebalance the anon lru active/inactive ratio. |
| 6384 | */ |
Dave Hansen | 2f368a9 | 2021-09-02 14:59:23 -0700 | [diff] [blame] | 6385 | if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && |
| 6386 | inactive_is_low(lruvec, LRU_INACTIVE_ANON)) |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6387 | shrink_active_list(SWAP_CLUSTER_MAX, lruvec, |
| 6388 | sc, LRU_ACTIVE_ANON); |
Johannes Weiner | 9b4f98c | 2013-02-22 16:32:19 -0800 | [diff] [blame] | 6389 | } |
| 6390 | |
Mel Gorman | 23b9da5 | 2012-05-29 15:06:20 -0700 | [diff] [blame] | 6391 | /* Use reclaim/compaction for costly allocs or under memory pressure */ |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 6392 | static bool in_reclaim_compaction(struct scan_control *sc) |
Mel Gorman | 23b9da5 | 2012-05-29 15:06:20 -0700 | [diff] [blame] | 6393 | { |
Kirill A. Shutemov | d84da3f | 2012-12-11 16:00:31 -0800 | [diff] [blame] | 6394 | if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && |
Mel Gorman | 23b9da5 | 2012-05-29 15:06:20 -0700 | [diff] [blame] | 6395 | (sc->order > PAGE_ALLOC_COSTLY_ORDER || |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 6396 | sc->priority < DEF_PRIORITY - 2)) |
Mel Gorman | 23b9da5 | 2012-05-29 15:06:20 -0700 | [diff] [blame] | 6397 | return true; |
| 6398 | |
| 6399 | return false; |
| 6400 | } |
| 6401 | |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 6402 | /* |
Mel Gorman | 23b9da5 | 2012-05-29 15:06:20 -0700 | [diff] [blame] | 6403 | * Reclaim/compaction is used for high-order allocation requests. It reclaims |
| 6404 | * order-0 pages before compacting the zone. should_continue_reclaim() returns |
| 6405 | * true if more pages should be reclaimed such that when the page allocator |
Qiwu Chen | df3a45f | 2020-06-03 16:01:21 -0700 | [diff] [blame] | 6406 | * calls try_to_compact_pages() that it will have enough free pages to succeed. |
Mel Gorman | 23b9da5 | 2012-05-29 15:06:20 -0700 | [diff] [blame] | 6407 | * It will give up earlier than that if there is difficulty reclaiming pages. |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6408 | */ |
Mel Gorman | a9dd0a8 | 2016-07-28 15:46:02 -0700 | [diff] [blame] | 6409 | static inline bool should_continue_reclaim(struct pglist_data *pgdat, |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6410 | unsigned long nr_reclaimed, |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6411 | struct scan_control *sc) |
| 6412 | { |
| 6413 | unsigned long pages_for_compaction; |
| 6414 | unsigned long inactive_lru_pages; |
Mel Gorman | a9dd0a8 | 2016-07-28 15:46:02 -0700 | [diff] [blame] | 6415 | int z; |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6416 | |
| 6417 | /* If not in reclaim/compaction mode, stop */ |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 6418 | if (!in_reclaim_compaction(sc)) |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6419 | return false; |
| 6420 | |
Vlastimil Babka | 5ee0471 | 2019-09-23 15:37:29 -0700 | [diff] [blame] | 6421 | /* |
| 6422 | * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX |
| 6423 | * number of pages that were scanned. This will return to the caller |
| 6424 | * with the risk reclaim/compaction and the resulting allocation attempt |
| 6425 | * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL |
| 6426 | * allocations through requiring that the full LRU list has been scanned |
| 6427 | * first, by assuming that zero delta of sc->nr_scanned means full LRU |
| 6428 | * scan, but that approximation was wrong, and there were corner cases |
| 6429 | * where always a non-zero amount of pages were scanned. |
| 6430 | */ |
| 6431 | if (!nr_reclaimed) |
| 6432 | return false; |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6433 | |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6434 | /* If compaction would go ahead or the allocation would succeed, stop */ |
Mel Gorman | a9dd0a8 | 2016-07-28 15:46:02 -0700 | [diff] [blame] | 6435 | for (z = 0; z <= sc->reclaim_idx; z++) { |
| 6436 | struct zone *zone = &pgdat->node_zones[z]; |
Mel Gorman | 6aa303d | 2016-09-01 16:14:55 -0700 | [diff] [blame] | 6437 | if (!managed_zone(zone)) |
Mel Gorman | a9dd0a8 | 2016-07-28 15:46:02 -0700 | [diff] [blame] | 6438 | continue; |
| 6439 | |
Johannes Weiner | e860632 | 2023-05-19 14:39:57 +0200 | [diff] [blame] | 6440 | /* Allocation can already succeed, nothing to do */ |
| 6441 | if (zone_watermark_ok(zone, sc->order, min_wmark_pages(zone), |
| 6442 | sc->reclaim_idx, 0)) |
| 6443 | return false; |
| 6444 | |
Johannes Weiner | 3cf0493 | 2023-06-02 11:12:04 -0400 | [diff] [blame] | 6445 | if (compaction_suitable(zone, sc->order, sc->reclaim_idx)) |
Johannes Weiner | e860632 | 2023-05-19 14:39:57 +0200 | [diff] [blame] | 6446 | return false; |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6447 | } |
Hillf Danton | 1c6c159 | 2019-09-23 15:37:26 -0700 | [diff] [blame] | 6448 | |
| 6449 | /* |
| 6450 | * If we have not reclaimed enough pages for compaction and the |
| 6451 | * inactive lists are large enough, continue reclaiming |
| 6452 | */ |
| 6453 | pages_for_compaction = compact_gap(sc->order); |
| 6454 | inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE); |
Keith Busch | a2a3648 | 2021-09-02 14:59:26 -0700 | [diff] [blame] | 6455 | if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) |
Hillf Danton | 1c6c159 | 2019-09-23 15:37:26 -0700 | [diff] [blame] | 6456 | inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON); |
| 6457 | |
Vlastimil Babka | 5ee0471 | 2019-09-23 15:37:29 -0700 | [diff] [blame] | 6458 | return inactive_lru_pages > pages_for_compaction; |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 6459 | } |
| 6460 | |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6461 | static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 6462 | { |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6463 | struct mem_cgroup *target_memcg = sc->target_mem_cgroup; |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6464 | struct mem_cgroup *memcg; |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 6465 | |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6466 | memcg = mem_cgroup_iter(target_memcg, NULL, NULL); |
Johannes Weiner | 5660048 | 2012-01-12 17:17:59 -0800 | [diff] [blame] | 6467 | do { |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 6468 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6469 | unsigned long reclaimed; |
| 6470 | unsigned long scanned; |
Johannes Weiner | 5660048 | 2012-01-12 17:17:59 -0800 | [diff] [blame] | 6471 | |
Xunlei Pang | e3336ca | 2020-09-04 16:35:27 -0700 | [diff] [blame] | 6472 | /* |
| 6473 | * This loop can become CPU-bound when target memcgs |
| 6474 | * aren't eligible for reclaim - either because they |
| 6475 | * don't have any reclaimable pages, or because their |
| 6476 | * memory is explicitly protected. Avoid soft lockups. |
| 6477 | */ |
| 6478 | cond_resched(); |
| 6479 | |
Chris Down | 45c7f7e | 2020-08-06 23:22:05 -0700 | [diff] [blame] | 6480 | mem_cgroup_calculate_protection(target_memcg, memcg); |
| 6481 | |
Yosry Ahmed | adb8213 | 2022-12-02 03:15:10 +0000 | [diff] [blame] | 6482 | if (mem_cgroup_below_min(target_memcg, memcg)) { |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6483 | /* |
| 6484 | * Hard protection. |
| 6485 | * If there is no reclaimable memory, OOM. |
| 6486 | */ |
| 6487 | continue; |
Yosry Ahmed | adb8213 | 2022-12-02 03:15:10 +0000 | [diff] [blame] | 6488 | } else if (mem_cgroup_below_low(target_memcg, memcg)) { |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6489 | /* |
| 6490 | * Soft protection. |
| 6491 | * Respect the protection only as long as |
| 6492 | * there is an unprotected supply |
| 6493 | * of reclaimable memory from other cgroups. |
| 6494 | */ |
| 6495 | if (!sc->memcg_low_reclaim) { |
| 6496 | sc->memcg_low_skipped = 1; |
Roman Gushchin | bf8d5d5 | 2018-06-07 17:07:46 -0700 | [diff] [blame] | 6497 | continue; |
Johannes Weiner | 241994ed | 2015-02-11 15:26:06 -0800 | [diff] [blame] | 6498 | } |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6499 | memcg_memory_event(memcg, MEMCG_LOW); |
Johannes Weiner | 6b4f779 | 2014-12-12 16:56:13 -0800 | [diff] [blame] | 6500 | } |
| 6501 | |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6502 | reclaimed = sc->nr_reclaimed; |
| 6503 | scanned = sc->nr_scanned; |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 6504 | |
| 6505 | shrink_lruvec(lruvec, sc); |
Anton Vorontsov | 70ddf63 | 2013-04-29 15:08:31 -0700 | [diff] [blame] | 6506 | |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6507 | shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, |
| 6508 | sc->priority); |
Johannes Weiner | 2344d7e | 2014-08-06 16:06:15 -0700 | [diff] [blame] | 6509 | |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6510 | /* Record the group's reclaim efficiency */ |
Yosry Ahmed | 73b73ba | 2022-07-14 06:49:18 +0000 | [diff] [blame] | 6511 | if (!sc->proactive) |
| 6512 | vmpressure(sc->gfp_mask, memcg, false, |
| 6513 | sc->nr_scanned - scanned, |
| 6514 | sc->nr_reclaimed - reclaimed); |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 6515 | |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6516 | } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL))); |
| 6517 | } |
| 6518 | |
Liu Song | 6c9e0907 | 2020-01-30 22:14:08 -0800 | [diff] [blame] | 6519 | static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6520 | { |
Haifeng Xu | 54c4fe0 | 2023-04-11 06:17:57 +0000 | [diff] [blame] | 6521 | unsigned long nr_reclaimed, nr_scanned, nr_node_reclaimed; |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 6522 | struct lruvec *target_lruvec; |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6523 | bool reclaimable = false; |
| 6524 | |
Yosry Ahmed | 7a70447 | 2023-06-21 02:30:53 +0000 | [diff] [blame] | 6525 | if (lru_gen_enabled() && root_reclaim(sc)) { |
Yu Zhao | e4dde56 | 2022-12-21 21:19:04 -0700 | [diff] [blame] | 6526 | lru_gen_shrink_node(pgdat, sc); |
| 6527 | return; |
| 6528 | } |
| 6529 | |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 6530 | target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); |
| 6531 | |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6532 | again: |
| 6533 | memset(&sc->nr, 0, sizeof(sc->nr)); |
| 6534 | |
| 6535 | nr_reclaimed = sc->nr_reclaimed; |
| 6536 | nr_scanned = sc->nr_scanned; |
| 6537 | |
Yu Zhao | f1e1a7b | 2022-09-18 02:00:00 -0600 | [diff] [blame] | 6538 | prepare_scan_count(pgdat, sc); |
Johannes Weiner | 53138ce | 2019-11-30 17:55:56 -0800 | [diff] [blame] | 6539 | |
Johannes Weiner | 0f6a5cf | 2019-11-30 17:55:49 -0800 | [diff] [blame] | 6540 | shrink_node_memcgs(pgdat, sc); |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 6541 | |
Yosry Ahmed | 583c27a | 2023-04-13 10:40:32 +0000 | [diff] [blame] | 6542 | flush_reclaim_state(sc); |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 6543 | |
Haifeng Xu | 54c4fe0 | 2023-04-11 06:17:57 +0000 | [diff] [blame] | 6544 | nr_node_reclaimed = sc->nr_reclaimed - nr_reclaimed; |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 6545 | |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6546 | /* Record the subtree's reclaim efficiency */ |
Yosry Ahmed | 73b73ba | 2022-07-14 06:49:18 +0000 | [diff] [blame] | 6547 | if (!sc->proactive) |
| 6548 | vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, |
Haifeng Xu | 54c4fe0 | 2023-04-11 06:17:57 +0000 | [diff] [blame] | 6549 | sc->nr_scanned - nr_scanned, nr_node_reclaimed); |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6550 | |
Haifeng Xu | 54c4fe0 | 2023-04-11 06:17:57 +0000 | [diff] [blame] | 6551 | if (nr_node_reclaimed) |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6552 | reclaimable = true; |
| 6553 | |
| 6554 | if (current_is_kswapd()) { |
| 6555 | /* |
| 6556 | * If reclaim is isolating dirty pages under writeback, |
| 6557 | * it implies that the long-lived page allocation rate |
| 6558 | * is exceeding the page laundering rate. Either the |
| 6559 | * global limits are not being effective at throttling |
| 6560 | * processes due to the page distribution throughout |
| 6561 | * zones or there is heavy usage of a slow backing |
| 6562 | * device. The only option is to throttle from reclaim |
| 6563 | * context which is not ideal as there is no guarantee |
| 6564 | * the dirtying process is throttled in the same way |
| 6565 | * balance_dirty_pages() manages. |
| 6566 | * |
| 6567 | * Once a node is flagged PGDAT_WRITEBACK, kswapd will |
| 6568 | * count the number of pages under pages flagged for |
| 6569 | * immediate reclaim and stall if any are encountered |
| 6570 | * in the nr_immediate check below. |
| 6571 | */ |
| 6572 | if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) |
| 6573 | set_bit(PGDAT_WRITEBACK, &pgdat->flags); |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 6574 | |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6575 | /* Allow kswapd to start writing pages during reclaim.*/ |
| 6576 | if (sc->nr.unqueued_dirty == sc->nr.file_taken) |
| 6577 | set_bit(PGDAT_DIRTY, &pgdat->flags); |
Andrey Ryabinin | e3c1ac5 | 2018-04-10 16:28:03 -0700 | [diff] [blame] | 6578 | |
| 6579 | /* |
Randy Dunlap | 1eba09c | 2020-08-11 18:33:26 -0700 | [diff] [blame] | 6580 | * If kswapd scans pages marked for immediate |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6581 | * reclaim and under writeback (nr_immediate), it |
| 6582 | * implies that pages are cycling through the LRU |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 6583 | * faster than they are written so forcibly stall |
| 6584 | * until some pages complete writeback. |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 6585 | */ |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6586 | if (sc->nr.immediate) |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 6587 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6588 | } |
Andrey Ryabinin | d108c77 | 2018-04-10 16:27:59 -0700 | [diff] [blame] | 6589 | |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6590 | /* |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 6591 | * Tag a node/memcg as congested if all the dirty pages were marked |
| 6592 | * for writeback and immediate reclaim (counted in nr.congested). |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 6593 | * |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6594 | * Legacy memcg will stall in page writeback so avoid forcibly |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 6595 | * stalling in reclaim_throttle(). |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6596 | */ |
Yosry Ahmed | 1bc545b | 2023-06-21 02:31:01 +0000 | [diff] [blame] | 6597 | if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested) { |
| 6598 | if (cgroup_reclaim(sc) && writeback_throttling_sane(sc)) |
| 6599 | set_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags); |
| 6600 | |
| 6601 | if (current_is_kswapd()) |
| 6602 | set_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags); |
| 6603 | } |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6604 | |
| 6605 | /* |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 6606 | * Stall direct reclaim for IO completions if the lruvec is |
| 6607 | * node is congested. Allow kswapd to continue until it |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6608 | * starts encountering unqueued dirty pages or cycling through |
| 6609 | * the LRU too quickly. |
| 6610 | */ |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 6611 | if (!current_is_kswapd() && current_may_throttle() && |
| 6612 | !sc->hibernation_mode && |
Yosry Ahmed | 1bc545b | 2023-06-21 02:31:01 +0000 | [diff] [blame] | 6613 | (test_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags) || |
| 6614 | test_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags))) |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 6615 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED); |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6616 | |
Haifeng Xu | 54c4fe0 | 2023-04-11 06:17:57 +0000 | [diff] [blame] | 6617 | if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc)) |
Johannes Weiner | d2af339 | 2019-11-30 17:55:43 -0800 | [diff] [blame] | 6618 | goto again; |
Johannes Weiner | 2344d7e | 2014-08-06 16:06:15 -0700 | [diff] [blame] | 6619 | |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 6620 | /* |
| 6621 | * Kswapd gives up on balancing particular nodes after too |
| 6622 | * many failures to reclaim anything from them and goes to |
| 6623 | * sleep. On reclaim progress, reset the failure counter. A |
| 6624 | * successful direct reclaim run will revive a dormant kswapd. |
| 6625 | */ |
| 6626 | if (reclaimable) |
| 6627 | pgdat->kswapd_failures = 0; |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 6628 | } |
| 6629 | |
Vlastimil Babka | 53853e2 | 2014-10-09 15:27:02 -0700 | [diff] [blame] | 6630 | /* |
Vlastimil Babka | fdd4c614 | 2016-10-07 16:58:03 -0700 | [diff] [blame] | 6631 | * Returns true if compaction should go ahead for a costly-order request, or |
| 6632 | * the allocation would already succeed without compaction. Return false if we |
| 6633 | * should reclaim first. |
Vlastimil Babka | 53853e2 | 2014-10-09 15:27:02 -0700 | [diff] [blame] | 6634 | */ |
Mel Gorman | 4f58833 | 2016-07-28 15:46:38 -0700 | [diff] [blame] | 6635 | static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) |
Mel Gorman | fe4b1b2 | 2012-01-12 17:19:45 -0800 | [diff] [blame] | 6636 | { |
Mel Gorman | 31483b6 | 2016-07-28 15:45:46 -0700 | [diff] [blame] | 6637 | unsigned long watermark; |
Mel Gorman | fe4b1b2 | 2012-01-12 17:19:45 -0800 | [diff] [blame] | 6638 | |
Johannes Weiner | e860632 | 2023-05-19 14:39:57 +0200 | [diff] [blame] | 6639 | /* Allocation can already succeed, nothing to do */ |
| 6640 | if (zone_watermark_ok(zone, sc->order, min_wmark_pages(zone), |
| 6641 | sc->reclaim_idx, 0)) |
Vlastimil Babka | fdd4c614 | 2016-10-07 16:58:03 -0700 | [diff] [blame] | 6642 | return true; |
Mel Gorman | fe4b1b2 | 2012-01-12 17:19:45 -0800 | [diff] [blame] | 6643 | |
Johannes Weiner | e860632 | 2023-05-19 14:39:57 +0200 | [diff] [blame] | 6644 | /* Compaction cannot yet proceed. Do reclaim. */ |
Johannes Weiner | 3cf0493 | 2023-06-02 11:12:04 -0400 | [diff] [blame] | 6645 | if (!compaction_suitable(zone, sc->order, sc->reclaim_idx)) |
Johannes Weiner | e860632 | 2023-05-19 14:39:57 +0200 | [diff] [blame] | 6646 | return false; |
Johannes Weiner | f98a497 | 2023-05-19 14:39:58 +0200 | [diff] [blame] | 6647 | |
Vlastimil Babka | fdd4c614 | 2016-10-07 16:58:03 -0700 | [diff] [blame] | 6648 | /* |
| 6649 | * Compaction is already possible, but it takes time to run and there |
| 6650 | * are potentially other callers using the pages just freed. So proceed |
| 6651 | * with reclaim to make a buffer of free pages available to give |
| 6652 | * compaction a reasonable chance of completing and allocating the page. |
| 6653 | * Note that we won't actually reclaim the whole buffer in one attempt |
| 6654 | * as the target watermark in should_continue_reclaim() is lower. But if |
| 6655 | * we are already above the high+gap watermark, don't reclaim at all. |
| 6656 | */ |
| 6657 | watermark = high_wmark_pages(zone) + compact_gap(sc->order); |
| 6658 | |
| 6659 | return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx); |
Mel Gorman | fe4b1b2 | 2012-01-12 17:19:45 -0800 | [diff] [blame] | 6660 | } |
| 6661 | |
Mel Gorman | 69392a4 | 2021-11-05 13:42:32 -0700 | [diff] [blame] | 6662 | static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc) |
| 6663 | { |
Mel Gorman | 66ce520 | 2021-11-05 13:42:49 -0700 | [diff] [blame] | 6664 | /* |
| 6665 | * If reclaim is making progress greater than 12% efficiency then |
| 6666 | * wake all the NOPROGRESS throttled tasks. |
| 6667 | */ |
| 6668 | if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) { |
Mel Gorman | 69392a4 | 2021-11-05 13:42:32 -0700 | [diff] [blame] | 6669 | wait_queue_head_t *wqh; |
| 6670 | |
| 6671 | wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS]; |
| 6672 | if (waitqueue_active(wqh)) |
| 6673 | wake_up(wqh); |
| 6674 | |
| 6675 | return; |
| 6676 | } |
| 6677 | |
| 6678 | /* |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 6679 | * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will |
| 6680 | * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages |
| 6681 | * under writeback and marked for immediate reclaim at the tail of the |
| 6682 | * LRU. |
Mel Gorman | 69392a4 | 2021-11-05 13:42:32 -0700 | [diff] [blame] | 6683 | */ |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 6684 | if (current_is_kswapd() || cgroup_reclaim(sc)) |
Mel Gorman | 69392a4 | 2021-11-05 13:42:32 -0700 | [diff] [blame] | 6685 | return; |
| 6686 | |
| 6687 | /* Throttle if making no progress at high prioities. */ |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 6688 | if (sc->priority == 1 && !sc->nr_reclaimed) |
Mel Gorman | c3f4a9a | 2021-11-05 13:42:42 -0700 | [diff] [blame] | 6689 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS); |
Mel Gorman | 69392a4 | 2021-11-05 13:42:32 -0700 | [diff] [blame] | 6690 | } |
| 6691 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6692 | /* |
| 6693 | * This is the direct reclaim path, for page-allocating processes. We only |
| 6694 | * try to reclaim pages from zones which will satisfy the caller's allocation |
| 6695 | * request. |
| 6696 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6697 | * If a zone is deemed to be full of pinned pages then just give it a light |
| 6698 | * scan then give up on it. |
| 6699 | */ |
Michal Hocko | 0a0337e | 2016-05-20 16:57:00 -0700 | [diff] [blame] | 6700 | static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6701 | { |
Mel Gorman | dd1a239 | 2008-04-28 02:12:17 -0700 | [diff] [blame] | 6702 | struct zoneref *z; |
Mel Gorman | 54a6eb5 | 2008-04-28 02:12:16 -0700 | [diff] [blame] | 6703 | struct zone *zone; |
Andrew Morton | 0608f43 | 2013-09-24 15:27:41 -0700 | [diff] [blame] | 6704 | unsigned long nr_soft_reclaimed; |
| 6705 | unsigned long nr_soft_scanned; |
Weijie Yang | 619d0d76 | 2014-04-07 15:36:59 -0700 | [diff] [blame] | 6706 | gfp_t orig_mask; |
Mel Gorman | 79dafcdc | 2016-07-28 15:45:53 -0700 | [diff] [blame] | 6707 | pg_data_t *last_pgdat = NULL; |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 6708 | pg_data_t *first_pgdat = NULL; |
KAMEZAWA Hiroyuki | 1cfb419 | 2008-02-07 00:14:37 -0800 | [diff] [blame] | 6709 | |
Mel Gorman | cc715d9 | 2012-03-21 16:34:00 -0700 | [diff] [blame] | 6710 | /* |
| 6711 | * If the number of buffer_heads in the machine exceeds the maximum |
| 6712 | * allowed level, force direct reclaim to scan the highmem zone as |
| 6713 | * highmem pages could be pinning lowmem pages storing buffer_heads |
| 6714 | */ |
Weijie Yang | 619d0d76 | 2014-04-07 15:36:59 -0700 | [diff] [blame] | 6715 | orig_mask = sc->gfp_mask; |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 6716 | if (buffer_heads_over_limit) { |
Mel Gorman | cc715d9 | 2012-03-21 16:34:00 -0700 | [diff] [blame] | 6717 | sc->gfp_mask |= __GFP_HIGHMEM; |
Mel Gorman | 4f58833 | 2016-07-28 15:46:38 -0700 | [diff] [blame] | 6718 | sc->reclaim_idx = gfp_zone(sc->gfp_mask); |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 6719 | } |
Mel Gorman | cc715d9 | 2012-03-21 16:34:00 -0700 | [diff] [blame] | 6720 | |
Mel Gorman | d4debc6 | 2010-08-09 17:19:29 -0700 | [diff] [blame] | 6721 | for_each_zone_zonelist_nodemask(zone, z, zonelist, |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 6722 | sc->reclaim_idx, sc->nodemask) { |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 6723 | /* |
KAMEZAWA Hiroyuki | 1cfb419 | 2008-02-07 00:14:37 -0800 | [diff] [blame] | 6724 | * Take care memory controller reclaiming has small influence |
| 6725 | * to global LRU. |
| 6726 | */ |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 6727 | if (!cgroup_reclaim(sc)) { |
Vladimir Davydov | 344736f | 2014-10-20 15:50:30 +0400 | [diff] [blame] | 6728 | if (!cpuset_zone_allowed(zone, |
| 6729 | GFP_KERNEL | __GFP_HARDWALL)) |
KAMEZAWA Hiroyuki | 1cfb419 | 2008-02-07 00:14:37 -0800 | [diff] [blame] | 6730 | continue; |
Vladimir Davydov | 65ec02c | 2014-04-03 14:47:20 -0700 | [diff] [blame] | 6731 | |
Johannes Weiner | 0b06496 | 2014-08-06 16:06:12 -0700 | [diff] [blame] | 6732 | /* |
| 6733 | * If we already have plenty of memory free for |
| 6734 | * compaction in this zone, don't free any more. |
| 6735 | * Even though compaction is invoked for any |
| 6736 | * non-zero order, only frequent costly order |
| 6737 | * reclamation is disruptive enough to become a |
| 6738 | * noticeable problem, like transparent huge |
| 6739 | * page allocations. |
| 6740 | */ |
| 6741 | if (IS_ENABLED(CONFIG_COMPACTION) && |
| 6742 | sc->order > PAGE_ALLOC_COSTLY_ORDER && |
Mel Gorman | 4f58833 | 2016-07-28 15:46:38 -0700 | [diff] [blame] | 6743 | compaction_ready(zone, sc)) { |
Johannes Weiner | 0b06496 | 2014-08-06 16:06:12 -0700 | [diff] [blame] | 6744 | sc->compaction_ready = true; |
| 6745 | continue; |
Rik van Riel | e0887c1 | 2011-10-31 17:09:31 -0700 | [diff] [blame] | 6746 | } |
Johannes Weiner | 0b06496 | 2014-08-06 16:06:12 -0700 | [diff] [blame] | 6747 | |
Andrew Morton | 0608f43 | 2013-09-24 15:27:41 -0700 | [diff] [blame] | 6748 | /* |
Mel Gorman | 79dafcdc | 2016-07-28 15:45:53 -0700 | [diff] [blame] | 6749 | * Shrink each node in the zonelist once. If the |
| 6750 | * zonelist is ordered by zone (not the default) then a |
| 6751 | * node may be shrunk multiple times but in that case |
| 6752 | * the user prefers lower zones being preserved. |
| 6753 | */ |
| 6754 | if (zone->zone_pgdat == last_pgdat) |
| 6755 | continue; |
| 6756 | |
| 6757 | /* |
Andrew Morton | 0608f43 | 2013-09-24 15:27:41 -0700 | [diff] [blame] | 6758 | * This steals pages from memory cgroups over softlimit |
| 6759 | * and returns the number of reclaimed pages and |
| 6760 | * scanned pages. This works for global memory pressure |
| 6761 | * and balancing, not for a memcg's limit. |
| 6762 | */ |
| 6763 | nr_soft_scanned = 0; |
Mel Gorman | ef8f232 | 2016-07-28 15:46:05 -0700 | [diff] [blame] | 6764 | nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat, |
Andrew Morton | 0608f43 | 2013-09-24 15:27:41 -0700 | [diff] [blame] | 6765 | sc->order, sc->gfp_mask, |
| 6766 | &nr_soft_scanned); |
| 6767 | sc->nr_reclaimed += nr_soft_reclaimed; |
| 6768 | sc->nr_scanned += nr_soft_scanned; |
KAMEZAWA Hiroyuki | ac34a1a | 2011-06-27 16:18:12 -0700 | [diff] [blame] | 6769 | /* need some check for avoid more shrink_zone() */ |
KAMEZAWA Hiroyuki | 1cfb419 | 2008-02-07 00:14:37 -0800 | [diff] [blame] | 6770 | } |
Nick Piggin | 408d854 | 2006-09-25 23:31:27 -0700 | [diff] [blame] | 6771 | |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 6772 | if (!first_pgdat) |
| 6773 | first_pgdat = zone->zone_pgdat; |
| 6774 | |
Mel Gorman | 79dafcdc | 2016-07-28 15:45:53 -0700 | [diff] [blame] | 6775 | /* See comment about same check for global reclaim above */ |
| 6776 | if (zone->zone_pgdat == last_pgdat) |
| 6777 | continue; |
| 6778 | last_pgdat = zone->zone_pgdat; |
Mel Gorman | 970a39a | 2016-07-28 15:46:35 -0700 | [diff] [blame] | 6779 | shrink_node(zone->zone_pgdat, sc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6780 | } |
Mel Gorman | e0c2327 | 2011-10-31 17:09:33 -0700 | [diff] [blame] | 6781 | |
Mel Gorman | 8008293 | 2021-12-31 13:10:09 -0800 | [diff] [blame] | 6782 | if (first_pgdat) |
| 6783 | consider_reclaim_throttle(first_pgdat, sc); |
Mel Gorman | 1b4e3f2 | 2021-12-02 15:06:14 +0000 | [diff] [blame] | 6784 | |
Vladimir Davydov | 65ec02c | 2014-04-03 14:47:20 -0700 | [diff] [blame] | 6785 | /* |
Weijie Yang | 619d0d76 | 2014-04-07 15:36:59 -0700 | [diff] [blame] | 6786 | * Restore to original mask to avoid the impact on the caller if we |
| 6787 | * promoted it to __GFP_HIGHMEM. |
| 6788 | */ |
| 6789 | sc->gfp_mask = orig_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6790 | } |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 6791 | |
Johannes Weiner | b910718 | 2019-11-30 17:55:59 -0800 | [diff] [blame] | 6792 | static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 6793 | { |
Johannes Weiner | b910718 | 2019-11-30 17:55:59 -0800 | [diff] [blame] | 6794 | struct lruvec *target_lruvec; |
| 6795 | unsigned long refaults; |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 6796 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 6797 | if (lru_gen_enabled()) |
| 6798 | return; |
| 6799 | |
Johannes Weiner | b910718 | 2019-11-30 17:55:59 -0800 | [diff] [blame] | 6800 | target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); |
Joonsoo Kim | 170b04b7 | 2020-08-11 18:30:43 -0700 | [diff] [blame] | 6801 | refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); |
Yang Yang | e9c2dbc | 2022-08-08 00:56:45 +0000 | [diff] [blame] | 6802 | target_lruvec->refaults[WORKINGSET_ANON] = refaults; |
Joonsoo Kim | 170b04b7 | 2020-08-11 18:30:43 -0700 | [diff] [blame] | 6803 | refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); |
Yang Yang | e9c2dbc | 2022-08-08 00:56:45 +0000 | [diff] [blame] | 6804 | target_lruvec->refaults[WORKINGSET_FILE] = refaults; |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 6805 | } |
| 6806 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6807 | /* |
| 6808 | * This is the main entry point to direct page reclaim. |
| 6809 | * |
| 6810 | * If a full scan of the inactive list fails to free enough memory then we |
| 6811 | * are "out of memory" and something needs to be killed. |
| 6812 | * |
| 6813 | * If the caller is !__GFP_FS then the probability of a failure is reasonably |
| 6814 | * high - the zone may be full of dirty or under-writeback pages, which this |
Jens Axboe | 5b0830c | 2009-09-23 19:37:09 +0200 | [diff] [blame] | 6815 | * caller can't do much about. We kick the writeback threads and take explicit |
| 6816 | * naps in the hope that some of these pages can be written. But if the |
| 6817 | * allocating task holds filesystem locks which prevent writeout this might not |
| 6818 | * work, and the allocation attempt will fail. |
Nishanth Aravamudan | a41f24e | 2008-04-29 00:58:25 -0700 | [diff] [blame] | 6819 | * |
| 6820 | * returns: 0, if no pages reclaimed |
| 6821 | * else, the number of pages reclaimed |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6822 | */ |
Mel Gorman | dac1d27 | 2008-04-28 02:12:12 -0700 | [diff] [blame] | 6823 | static unsigned long do_try_to_free_pages(struct zonelist *zonelist, |
Vladimir Davydov | 3115cd9 | 2014-04-03 14:47:22 -0700 | [diff] [blame] | 6824 | struct scan_control *sc) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6825 | { |
Johannes Weiner | 241994ed | 2015-02-11 15:26:06 -0800 | [diff] [blame] | 6826 | int initial_priority = sc->priority; |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 6827 | pg_data_t *last_pgdat; |
| 6828 | struct zoneref *z; |
| 6829 | struct zone *zone; |
Johannes Weiner | 241994ed | 2015-02-11 15:26:06 -0800 | [diff] [blame] | 6830 | retry: |
Keika Kobayashi | 873b477 | 2008-07-25 01:48:52 -0700 | [diff] [blame] | 6831 | delayacct_freepages_start(); |
| 6832 | |
Johannes Weiner | b5ead35 | 2019-11-30 17:55:40 -0800 | [diff] [blame] | 6833 | if (!cgroup_reclaim(sc)) |
Mel Gorman | 7cc30fc | 2016-07-28 15:46:59 -0700 | [diff] [blame] | 6834 | __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6835 | |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 6836 | do { |
Yosry Ahmed | 73b73ba | 2022-07-14 06:49:18 +0000 | [diff] [blame] | 6837 | if (!sc->proactive) |
| 6838 | vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, |
| 6839 | sc->priority); |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 6840 | sc->nr_scanned = 0; |
Michal Hocko | 0a0337e | 2016-05-20 16:57:00 -0700 | [diff] [blame] | 6841 | shrink_zones(zonelist, sc); |
Mel Gorman | e0c2327 | 2011-10-31 17:09:33 -0700 | [diff] [blame] | 6842 | |
KOSAKI Motohiro | bb21c7c | 2010-06-04 14:15:05 -0700 | [diff] [blame] | 6843 | if (sc->nr_reclaimed >= sc->nr_to_reclaim) |
Johannes Weiner | 0b06496 | 2014-08-06 16:06:12 -0700 | [diff] [blame] | 6844 | break; |
| 6845 | |
| 6846 | if (sc->compaction_ready) |
| 6847 | break; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6848 | |
| 6849 | /* |
Minchan Kim | 0e50ce3 | 2013-02-22 16:35:37 -0800 | [diff] [blame] | 6850 | * If we're getting trouble reclaiming, start doing |
| 6851 | * writepage even in laptop mode. |
| 6852 | */ |
| 6853 | if (sc->priority < DEF_PRIORITY - 2) |
| 6854 | sc->may_writepage = 1; |
Johannes Weiner | 0b06496 | 2014-08-06 16:06:12 -0700 | [diff] [blame] | 6855 | } while (--sc->priority >= 0); |
KOSAKI Motohiro | bb21c7c | 2010-06-04 14:15:05 -0700 | [diff] [blame] | 6856 | |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 6857 | last_pgdat = NULL; |
| 6858 | for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, |
| 6859 | sc->nodemask) { |
| 6860 | if (zone->zone_pgdat == last_pgdat) |
| 6861 | continue; |
| 6862 | last_pgdat = zone->zone_pgdat; |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 6863 | |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 6864 | snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 6865 | |
| 6866 | if (cgroup_reclaim(sc)) { |
| 6867 | struct lruvec *lruvec; |
| 6868 | |
| 6869 | lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, |
| 6870 | zone->zone_pgdat); |
Yosry Ahmed | 1bc545b | 2023-06-21 02:31:01 +0000 | [diff] [blame] | 6871 | clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 6872 | } |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 6873 | } |
| 6874 | |
Keika Kobayashi | 873b477 | 2008-07-25 01:48:52 -0700 | [diff] [blame] | 6875 | delayacct_freepages_end(); |
| 6876 | |
KOSAKI Motohiro | bb21c7c | 2010-06-04 14:15:05 -0700 | [diff] [blame] | 6877 | if (sc->nr_reclaimed) |
| 6878 | return sc->nr_reclaimed; |
| 6879 | |
Mel Gorman | 0cee34f | 2012-01-12 17:19:49 -0800 | [diff] [blame] | 6880 | /* Aborted reclaim to try compaction? don't OOM, then */ |
Johannes Weiner | 0b06496 | 2014-08-06 16:06:12 -0700 | [diff] [blame] | 6881 | if (sc->compaction_ready) |
Mel Gorman | 7335084 | 2012-01-12 17:19:33 -0800 | [diff] [blame] | 6882 | return 1; |
| 6883 | |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 6884 | /* |
| 6885 | * We make inactive:active ratio decisions based on the node's |
| 6886 | * composition of memory, but a restrictive reclaim_idx or a |
| 6887 | * memory.low cgroup setting can exempt large amounts of |
| 6888 | * memory from reclaim. Neither of which are very common, so |
| 6889 | * instead of doing costly eligibility calculations of the |
| 6890 | * entire cgroup subtree up front, we assume the estimates are |
| 6891 | * good, and retry with forcible deactivation if that fails. |
| 6892 | */ |
| 6893 | if (sc->skipped_deactivate) { |
| 6894 | sc->priority = initial_priority; |
| 6895 | sc->force_deactivate = 1; |
| 6896 | sc->skipped_deactivate = 0; |
| 6897 | goto retry; |
| 6898 | } |
| 6899 | |
Johannes Weiner | 241994ed | 2015-02-11 15:26:06 -0800 | [diff] [blame] | 6900 | /* Untapped cgroup reserves? Don't OOM, retry. */ |
Yisheng Xie | d6622f6 | 2017-05-03 14:53:57 -0700 | [diff] [blame] | 6901 | if (sc->memcg_low_skipped) { |
Johannes Weiner | 241994ed | 2015-02-11 15:26:06 -0800 | [diff] [blame] | 6902 | sc->priority = initial_priority; |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 6903 | sc->force_deactivate = 0; |
Yisheng Xie | d6622f6 | 2017-05-03 14:53:57 -0700 | [diff] [blame] | 6904 | sc->memcg_low_reclaim = 1; |
| 6905 | sc->memcg_low_skipped = 0; |
Johannes Weiner | 241994ed | 2015-02-11 15:26:06 -0800 | [diff] [blame] | 6906 | goto retry; |
| 6907 | } |
| 6908 | |
KOSAKI Motohiro | bb21c7c | 2010-06-04 14:15:05 -0700 | [diff] [blame] | 6909 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6910 | } |
| 6911 | |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 6912 | static bool allow_direct_reclaim(pg_data_t *pgdat) |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6913 | { |
| 6914 | struct zone *zone; |
| 6915 | unsigned long pfmemalloc_reserve = 0; |
| 6916 | unsigned long free_pages = 0; |
| 6917 | int i; |
| 6918 | bool wmark_ok; |
| 6919 | |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 6920 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) |
| 6921 | return true; |
| 6922 | |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6923 | for (i = 0; i <= ZONE_NORMAL; i++) { |
| 6924 | zone = &pgdat->node_zones[i]; |
Johannes Weiner | d450abd8 | 2017-05-03 14:51:54 -0700 | [diff] [blame] | 6925 | if (!managed_zone(zone)) |
| 6926 | continue; |
| 6927 | |
| 6928 | if (!zone_reclaimable_pages(zone)) |
Mel Gorman | 675becc | 2014-06-04 16:07:35 -0700 | [diff] [blame] | 6929 | continue; |
| 6930 | |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6931 | pfmemalloc_reserve += min_wmark_pages(zone); |
Marcelo Tosatti | 501b265 | 2023-05-30 11:52:35 -0300 | [diff] [blame] | 6932 | free_pages += zone_page_state_snapshot(zone, NR_FREE_PAGES); |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6933 | } |
| 6934 | |
Mel Gorman | 675becc | 2014-06-04 16:07:35 -0700 | [diff] [blame] | 6935 | /* If there are no reserves (unexpected config) then do not throttle */ |
| 6936 | if (!pfmemalloc_reserve) |
| 6937 | return true; |
| 6938 | |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6939 | wmark_ok = free_pages > pfmemalloc_reserve / 2; |
| 6940 | |
| 6941 | /* kswapd must be awake if processes are being throttled */ |
| 6942 | if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 6943 | if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL) |
| 6944 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL); |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 6945 | |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6946 | wake_up_interruptible(&pgdat->kswapd_wait); |
| 6947 | } |
| 6948 | |
| 6949 | return wmark_ok; |
| 6950 | } |
| 6951 | |
| 6952 | /* |
| 6953 | * Throttle direct reclaimers if backing storage is backed by the network |
| 6954 | * and the PFMEMALLOC reserve for the preferred node is getting dangerously |
| 6955 | * depleted. kswapd will continue to make progress and wake the processes |
Mel Gorman | 50694c2 | 2012-11-26 16:29:48 -0800 | [diff] [blame] | 6956 | * when the low watermark is reached. |
| 6957 | * |
| 6958 | * Returns true if a fatal signal was delivered during throttling. If this |
| 6959 | * happens, the page allocator should not consider triggering the OOM killer. |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6960 | */ |
Mel Gorman | 50694c2 | 2012-11-26 16:29:48 -0800 | [diff] [blame] | 6961 | static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6962 | nodemask_t *nodemask) |
| 6963 | { |
Mel Gorman | 675becc | 2014-06-04 16:07:35 -0700 | [diff] [blame] | 6964 | struct zoneref *z; |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6965 | struct zone *zone; |
Mel Gorman | 675becc | 2014-06-04 16:07:35 -0700 | [diff] [blame] | 6966 | pg_data_t *pgdat = NULL; |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6967 | |
| 6968 | /* |
| 6969 | * Kernel threads should not be throttled as they may be indirectly |
| 6970 | * responsible for cleaning pages necessary for reclaim to make forward |
| 6971 | * progress. kjournald for example may enter direct reclaim while |
| 6972 | * committing a transaction where throttling it could forcing other |
| 6973 | * processes to block on log_wait_commit(). |
| 6974 | */ |
| 6975 | if (current->flags & PF_KTHREAD) |
Mel Gorman | 50694c2 | 2012-11-26 16:29:48 -0800 | [diff] [blame] | 6976 | goto out; |
| 6977 | |
| 6978 | /* |
| 6979 | * If a fatal signal is pending, this process should not throttle. |
| 6980 | * It should return quickly so it can exit and free its memory |
| 6981 | */ |
| 6982 | if (fatal_signal_pending(current)) |
| 6983 | goto out; |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 6984 | |
Mel Gorman | 675becc | 2014-06-04 16:07:35 -0700 | [diff] [blame] | 6985 | /* |
| 6986 | * Check if the pfmemalloc reserves are ok by finding the first node |
| 6987 | * with a usable ZONE_NORMAL or lower zone. The expectation is that |
| 6988 | * GFP_KERNEL will be required for allocating network buffers when |
| 6989 | * swapping over the network so ZONE_HIGHMEM is unusable. |
| 6990 | * |
| 6991 | * Throttling is based on the first usable node and throttled processes |
| 6992 | * wait on a queue until kswapd makes progress and wakes them. There |
| 6993 | * is an affinity then between processes waking up and where reclaim |
| 6994 | * progress has been made assuming the process wakes on the same node. |
| 6995 | * More importantly, processes running on remote nodes will not compete |
| 6996 | * for remote pfmemalloc reserves and processes on different nodes |
| 6997 | * should make reasonable progress. |
| 6998 | */ |
| 6999 | for_each_zone_zonelist_nodemask(zone, z, zonelist, |
Michael S. Tsirkin | 17636fa | 2015-01-26 12:58:41 -0800 | [diff] [blame] | 7000 | gfp_zone(gfp_mask), nodemask) { |
Mel Gorman | 675becc | 2014-06-04 16:07:35 -0700 | [diff] [blame] | 7001 | if (zone_idx(zone) > ZONE_NORMAL) |
| 7002 | continue; |
| 7003 | |
| 7004 | /* Throttle based on the first usable node */ |
| 7005 | pgdat = zone->zone_pgdat; |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7006 | if (allow_direct_reclaim(pgdat)) |
Mel Gorman | 675becc | 2014-06-04 16:07:35 -0700 | [diff] [blame] | 7007 | goto out; |
| 7008 | break; |
| 7009 | } |
| 7010 | |
| 7011 | /* If no zone was usable by the allocation flags then do not throttle */ |
| 7012 | if (!pgdat) |
Mel Gorman | 50694c2 | 2012-11-26 16:29:48 -0800 | [diff] [blame] | 7013 | goto out; |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7014 | |
Mel Gorman | 68243e7 | 2012-07-31 16:44:39 -0700 | [diff] [blame] | 7015 | /* Account for the throttling */ |
| 7016 | count_vm_event(PGSCAN_DIRECT_THROTTLE); |
| 7017 | |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7018 | /* |
| 7019 | * If the caller cannot enter the filesystem, it's possible that it |
| 7020 | * is due to the caller holding an FS lock or performing a journal |
| 7021 | * transaction in the case of a filesystem like ext[3|4]. In this case, |
| 7022 | * it is not safe to block on pfmemalloc_wait as kswapd could be |
| 7023 | * blocked waiting on the same lock. Instead, throttle for up to a |
| 7024 | * second before continuing. |
| 7025 | */ |
Miaohe Lin | 2e786d9 | 2021-09-02 14:59:50 -0700 | [diff] [blame] | 7026 | if (!(gfp_mask & __GFP_FS)) |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7027 | wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7028 | allow_direct_reclaim(pgdat), HZ); |
Miaohe Lin | 2e786d9 | 2021-09-02 14:59:50 -0700 | [diff] [blame] | 7029 | else |
| 7030 | /* Throttle until kswapd wakes the process */ |
| 7031 | wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, |
| 7032 | allow_direct_reclaim(pgdat)); |
Mel Gorman | 50694c2 | 2012-11-26 16:29:48 -0800 | [diff] [blame] | 7033 | |
Mel Gorman | 50694c2 | 2012-11-26 16:29:48 -0800 | [diff] [blame] | 7034 | if (fatal_signal_pending(current)) |
| 7035 | return true; |
| 7036 | |
| 7037 | out: |
| 7038 | return false; |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7039 | } |
| 7040 | |
Mel Gorman | dac1d27 | 2008-04-28 02:12:12 -0700 | [diff] [blame] | 7041 | unsigned long try_to_free_pages(struct zonelist *zonelist, int order, |
KAMEZAWA Hiroyuki | 327c0e9 | 2009-03-31 15:23:31 -0700 | [diff] [blame] | 7042 | gfp_t gfp_mask, nodemask_t *nodemask) |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7043 | { |
Mel Gorman | 33906bc | 2010-08-09 17:19:16 -0700 | [diff] [blame] | 7044 | unsigned long nr_reclaimed; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7045 | struct scan_control sc = { |
KOSAKI Motohiro | 22fba33 | 2009-12-14 17:59:10 -0800 | [diff] [blame] | 7046 | .nr_to_reclaim = SWAP_CLUSTER_MAX, |
Nick Desaulniers | f2f43e5 | 2017-07-06 15:36:50 -0700 | [diff] [blame] | 7047 | .gfp_mask = current_gfp_context(gfp_mask), |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 7048 | .reclaim_idx = gfp_zone(gfp_mask), |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 7049 | .order = order, |
| 7050 | .nodemask = nodemask, |
| 7051 | .priority = DEF_PRIORITY, |
| 7052 | .may_writepage = !laptop_mode, |
Johannes Weiner | a6dc60f8 | 2009-03-31 15:19:30 -0700 | [diff] [blame] | 7053 | .may_unmap = 1, |
KOSAKI Motohiro | 2e2e425 | 2009-04-21 12:24:57 -0700 | [diff] [blame] | 7054 | .may_swap = 1, |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7055 | }; |
| 7056 | |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7057 | /* |
Greg Thelen | bb451fd | 2018-08-17 15:45:19 -0700 | [diff] [blame] | 7058 | * scan_control uses s8 fields for order, priority, and reclaim_idx. |
| 7059 | * Confirm they are large enough for max values. |
| 7060 | */ |
Kirill A. Shutemov | 23baf83 | 2023-03-15 14:31:33 +0300 | [diff] [blame] | 7061 | BUILD_BUG_ON(MAX_ORDER >= S8_MAX); |
Greg Thelen | bb451fd | 2018-08-17 15:45:19 -0700 | [diff] [blame] | 7062 | BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); |
| 7063 | BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); |
| 7064 | |
| 7065 | /* |
Mel Gorman | 50694c2 | 2012-11-26 16:29:48 -0800 | [diff] [blame] | 7066 | * Do not enter reclaim if fatal signal was delivered while throttled. |
| 7067 | * 1 is returned so that the page allocator does not OOM kill at this |
| 7068 | * point. |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7069 | */ |
Nick Desaulniers | f2f43e5 | 2017-07-06 15:36:50 -0700 | [diff] [blame] | 7070 | if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7071 | return 1; |
| 7072 | |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7073 | set_task_reclaim_state(current, &sc.reclaim_state); |
Yafang Shao | 3481c37 | 2019-05-13 17:19:14 -0700 | [diff] [blame] | 7074 | trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); |
Mel Gorman | 33906bc | 2010-08-09 17:19:16 -0700 | [diff] [blame] | 7075 | |
Vladimir Davydov | 3115cd9 | 2014-04-03 14:47:22 -0700 | [diff] [blame] | 7076 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); |
Mel Gorman | 33906bc | 2010-08-09 17:19:16 -0700 | [diff] [blame] | 7077 | |
| 7078 | trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7079 | set_task_reclaim_state(current, NULL); |
Mel Gorman | 33906bc | 2010-08-09 17:19:16 -0700 | [diff] [blame] | 7080 | |
| 7081 | return nr_reclaimed; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7082 | } |
| 7083 | |
Andrew Morton | c255a45 | 2012-07-31 16:43:02 -0700 | [diff] [blame] | 7084 | #ifdef CONFIG_MEMCG |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7085 | |
Michal Hocko | d2e5fb9 | 2019-08-30 16:04:50 -0700 | [diff] [blame] | 7086 | /* Only used by soft limit reclaim. Do not reuse for anything else. */ |
Mel Gorman | a9dd0a8 | 2016-07-28 15:46:02 -0700 | [diff] [blame] | 7087 | unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7088 | gfp_t gfp_mask, bool noswap, |
Mel Gorman | ef8f232 | 2016-07-28 15:46:05 -0700 | [diff] [blame] | 7089 | pg_data_t *pgdat, |
Ying Han | 0ae5e89 | 2011-05-26 16:25:25 -0700 | [diff] [blame] | 7090 | unsigned long *nr_scanned) |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7091 | { |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 7092 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7093 | struct scan_control sc = { |
KOSAKI Motohiro | b8f5c56 | 2010-08-10 18:03:02 -0700 | [diff] [blame] | 7094 | .nr_to_reclaim = SWAP_CLUSTER_MAX, |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 7095 | .target_mem_cgroup = memcg, |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7096 | .may_writepage = !laptop_mode, |
| 7097 | .may_unmap = 1, |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 7098 | .reclaim_idx = MAX_NR_ZONES - 1, |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7099 | .may_swap = !noswap, |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7100 | }; |
Ying Han | 0ae5e89 | 2011-05-26 16:25:25 -0700 | [diff] [blame] | 7101 | |
Michal Hocko | d2e5fb9 | 2019-08-30 16:04:50 -0700 | [diff] [blame] | 7102 | WARN_ON_ONCE(!current->reclaim_state); |
| 7103 | |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7104 | sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | |
| 7105 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); |
KOSAKI Motohiro | bdce6d9 | 2010-08-09 17:19:56 -0700 | [diff] [blame] | 7106 | |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 7107 | trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order, |
Yafang Shao | 3481c37 | 2019-05-13 17:19:14 -0700 | [diff] [blame] | 7108 | sc.gfp_mask); |
KOSAKI Motohiro | bdce6d9 | 2010-08-09 17:19:56 -0700 | [diff] [blame] | 7109 | |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7110 | /* |
| 7111 | * NOTE: Although we can get the priority field, using it |
| 7112 | * here is not a good idea, since it limits the pages we can scan. |
Mel Gorman | a9dd0a8 | 2016-07-28 15:46:02 -0700 | [diff] [blame] | 7113 | * if we don't reclaim here, the shrink_node from balance_pgdat |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7114 | * will pick up pages from other mem cgroup's as well. We hack |
| 7115 | * the priority and make it zero. |
| 7116 | */ |
Johannes Weiner | afaf07a | 2019-11-30 17:55:46 -0800 | [diff] [blame] | 7117 | shrink_lruvec(lruvec, &sc); |
KOSAKI Motohiro | bdce6d9 | 2010-08-09 17:19:56 -0700 | [diff] [blame] | 7118 | |
| 7119 | trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); |
| 7120 | |
Ying Han | 0ae5e89 | 2011-05-26 16:25:25 -0700 | [diff] [blame] | 7121 | *nr_scanned = sc.nr_scanned; |
Yafang Shao | 0308f7c | 2019-07-16 16:26:12 -0700 | [diff] [blame] | 7122 | |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 7123 | return sc.nr_reclaimed; |
| 7124 | } |
| 7125 | |
Johannes Weiner | 72835c8 | 2012-01-12 17:18:32 -0800 | [diff] [blame] | 7126 | unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, |
Johannes Weiner | b70a2a2 | 2014-10-09 15:28:56 -0700 | [diff] [blame] | 7127 | unsigned long nr_pages, |
KOSAKI Motohiro | a7885eb | 2009-01-07 18:08:24 -0800 | [diff] [blame] | 7128 | gfp_t gfp_mask, |
Michal Hocko | 55ab834 | 2022-12-16 10:46:33 +0100 | [diff] [blame] | 7129 | unsigned int reclaim_options) |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7130 | { |
KOSAKI Motohiro | bdce6d9 | 2010-08-09 17:19:56 -0700 | [diff] [blame] | 7131 | unsigned long nr_reclaimed; |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 7132 | unsigned int noreclaim_flag; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7133 | struct scan_control sc = { |
Johannes Weiner | b70a2a2 | 2014-10-09 15:28:56 -0700 | [diff] [blame] | 7134 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), |
Michal Hocko | 7dea19f | 2017-05-03 14:53:15 -0700 | [diff] [blame] | 7135 | .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 7136 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 7137 | .reclaim_idx = MAX_NR_ZONES - 1, |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 7138 | .target_mem_cgroup = memcg, |
| 7139 | .priority = DEF_PRIORITY, |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7140 | .may_writepage = !laptop_mode, |
Johannes Weiner | a6dc60f8 | 2009-03-31 15:19:30 -0700 | [diff] [blame] | 7141 | .may_unmap = 1, |
Yosry Ahmed | 73b73ba | 2022-07-14 06:49:18 +0000 | [diff] [blame] | 7142 | .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP), |
| 7143 | .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE), |
Ying Han | a09ed5e | 2011-05-24 17:12:26 -0700 | [diff] [blame] | 7144 | }; |
Shakeel Butt | fa40d1e | 2019-11-30 17:50:16 -0800 | [diff] [blame] | 7145 | /* |
| 7146 | * Traverse the ZONELIST_FALLBACK zonelist of the current node to put |
| 7147 | * equal pressure on all the nodes. This is based on the assumption that |
| 7148 | * the reclaim does not bail out early. |
| 7149 | */ |
| 7150 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7151 | |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7152 | set_task_reclaim_state(current, &sc.reclaim_state); |
Yafang Shao | 3481c37 | 2019-05-13 17:19:14 -0700 | [diff] [blame] | 7153 | trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask); |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 7154 | noreclaim_flag = memalloc_noreclaim_save(); |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 7155 | |
Vladimir Davydov | 3115cd9 | 2014-04-03 14:47:22 -0700 | [diff] [blame] | 7156 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 7157 | |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 7158 | memalloc_noreclaim_restore(noreclaim_flag); |
KOSAKI Motohiro | bdce6d9 | 2010-08-09 17:19:56 -0700 | [diff] [blame] | 7159 | trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7160 | set_task_reclaim_state(current, NULL); |
KOSAKI Motohiro | bdce6d9 | 2010-08-09 17:19:56 -0700 | [diff] [blame] | 7161 | |
| 7162 | return nr_reclaimed; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 7163 | } |
| 7164 | #endif |
| 7165 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 7166 | static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 7167 | { |
Johannes Weiner | b95a2f2 | 2012-01-12 17:18:06 -0800 | [diff] [blame] | 7168 | struct mem_cgroup *memcg; |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 7169 | struct lruvec *lruvec; |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 7170 | |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 7171 | if (lru_gen_enabled()) { |
| 7172 | lru_gen_age_node(pgdat, sc); |
| 7173 | return; |
| 7174 | } |
| 7175 | |
Dave Hansen | 2f368a9 | 2021-09-02 14:59:23 -0700 | [diff] [blame] | 7176 | if (!can_age_anon_pages(pgdat, sc)) |
Johannes Weiner | b95a2f2 | 2012-01-12 17:18:06 -0800 | [diff] [blame] | 7177 | return; |
| 7178 | |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 7179 | lruvec = mem_cgroup_lruvec(NULL, pgdat); |
| 7180 | if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) |
| 7181 | return; |
| 7182 | |
Johannes Weiner | b95a2f2 | 2012-01-12 17:18:06 -0800 | [diff] [blame] | 7183 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 7184 | do { |
Johannes Weiner | b91ac37 | 2019-11-30 17:56:02 -0800 | [diff] [blame] | 7185 | lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 7186 | shrink_active_list(SWAP_CLUSTER_MAX, lruvec, |
| 7187 | sc, LRU_ACTIVE_ANON); |
Johannes Weiner | b95a2f2 | 2012-01-12 17:18:06 -0800 | [diff] [blame] | 7188 | memcg = mem_cgroup_iter(NULL, memcg, NULL); |
| 7189 | } while (memcg); |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 7190 | } |
| 7191 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7192 | static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx) |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7193 | { |
| 7194 | int i; |
| 7195 | struct zone *zone; |
| 7196 | |
| 7197 | /* |
| 7198 | * Check for watermark boosts top-down as the higher zones |
| 7199 | * are more likely to be boosted. Both watermarks and boosts |
Randy Dunlap | 1eba09c | 2020-08-11 18:33:26 -0700 | [diff] [blame] | 7200 | * should not be checked at the same time as reclaim would |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7201 | * start prematurely when there is no boosting and a lower |
| 7202 | * zone is balanced. |
| 7203 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7204 | for (i = highest_zoneidx; i >= 0; i--) { |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7205 | zone = pgdat->node_zones + i; |
| 7206 | if (!managed_zone(zone)) |
| 7207 | continue; |
| 7208 | |
| 7209 | if (zone->watermark_boost) |
| 7210 | return true; |
| 7211 | } |
| 7212 | |
| 7213 | return false; |
| 7214 | } |
| 7215 | |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7216 | /* |
| 7217 | * Returns true if there is an eligible zone balanced for the request order |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7218 | * and highest_zoneidx |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7219 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7220 | static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) |
Johannes Weiner | 60cefed | 2012-11-29 13:54:23 -0800 | [diff] [blame] | 7221 | { |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7222 | int i; |
| 7223 | unsigned long mark = -1; |
| 7224 | struct zone *zone; |
Johannes Weiner | 60cefed | 2012-11-29 13:54:23 -0800 | [diff] [blame] | 7225 | |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7226 | /* |
| 7227 | * Check watermarks bottom-up as lower zones are more likely to |
| 7228 | * meet watermarks. |
| 7229 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7230 | for (i = 0; i <= highest_zoneidx; i++) { |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7231 | zone = pgdat->node_zones + i; |
Mel Gorman | 6256c6b | 2016-07-28 15:45:56 -0700 | [diff] [blame] | 7232 | |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7233 | if (!managed_zone(zone)) |
| 7234 | continue; |
| 7235 | |
Huang Ying | c574bbe | 2022-03-22 14:46:23 -0700 | [diff] [blame] | 7236 | if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) |
| 7237 | mark = wmark_pages(zone, WMARK_PROMO); |
| 7238 | else |
| 7239 | mark = high_wmark_pages(zone); |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7240 | if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx)) |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7241 | return true; |
| 7242 | } |
| 7243 | |
| 7244 | /* |
Wei Yang | 36c2612 | 2022-04-28 23:16:03 -0700 | [diff] [blame] | 7245 | * If a node has no managed zone within highest_zoneidx, it does not |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7246 | * need balancing by definition. This can happen if a zone-restricted |
| 7247 | * allocation tries to wake a remote kswapd. |
| 7248 | */ |
| 7249 | if (mark == -1) |
| 7250 | return true; |
| 7251 | |
| 7252 | return false; |
Johannes Weiner | 60cefed | 2012-11-29 13:54:23 -0800 | [diff] [blame] | 7253 | } |
| 7254 | |
Mel Gorman | 631b6e0 | 2017-05-03 14:53:41 -0700 | [diff] [blame] | 7255 | /* Clear pgdat state for congested, dirty or under writeback. */ |
| 7256 | static void clear_pgdat_congested(pg_data_t *pgdat) |
| 7257 | { |
Johannes Weiner | 1b05117 | 2019-11-30 17:55:52 -0800 | [diff] [blame] | 7258 | struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); |
| 7259 | |
Yosry Ahmed | 1bc545b | 2023-06-21 02:31:01 +0000 | [diff] [blame] | 7260 | clear_bit(LRUVEC_NODE_CONGESTED, &lruvec->flags); |
| 7261 | clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); |
Mel Gorman | 631b6e0 | 2017-05-03 14:53:41 -0700 | [diff] [blame] | 7262 | clear_bit(PGDAT_DIRTY, &pgdat->flags); |
| 7263 | clear_bit(PGDAT_WRITEBACK, &pgdat->flags); |
| 7264 | } |
| 7265 | |
Mel Gorman | 1741c87 | 2011-01-13 15:46:21 -0800 | [diff] [blame] | 7266 | /* |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7267 | * Prepare kswapd for sleeping. This verifies that there are no processes |
| 7268 | * waiting in throttle_direct_reclaim() and that watermarks have been met. |
| 7269 | * |
| 7270 | * Returns true if kswapd is ready to sleep |
| 7271 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7272 | static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, |
| 7273 | int highest_zoneidx) |
Mel Gorman | f50de2d | 2009-12-14 17:58:53 -0800 | [diff] [blame] | 7274 | { |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7275 | /* |
Vlastimil Babka | 9e5e366 | 2015-01-08 14:32:40 -0800 | [diff] [blame] | 7276 | * The throttled processes are normally woken up in balance_pgdat() as |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7277 | * soon as allow_direct_reclaim() is true. But there is a potential |
Vlastimil Babka | 9e5e366 | 2015-01-08 14:32:40 -0800 | [diff] [blame] | 7278 | * race between when kswapd checks the watermarks and a process gets |
| 7279 | * throttled. There is also a potential race if processes get |
| 7280 | * throttled, kswapd wakes, a large process exits thereby balancing the |
| 7281 | * zones, which causes kswapd to exit balance_pgdat() before reaching |
| 7282 | * the wake up checks. If kswapd is going to sleep, no process should |
| 7283 | * be sleeping on pfmemalloc_wait, so wake them now if necessary. If |
| 7284 | * the wake up is premature, processes will wake kswapd and get |
| 7285 | * throttled again. The difference from wake ups in balance_pgdat() is |
| 7286 | * that here we are under prepare_to_wait(). |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7287 | */ |
Vlastimil Babka | 9e5e366 | 2015-01-08 14:32:40 -0800 | [diff] [blame] | 7288 | if (waitqueue_active(&pgdat->pfmemalloc_wait)) |
| 7289 | wake_up_all(&pgdat->pfmemalloc_wait); |
Mel Gorman | f50de2d | 2009-12-14 17:58:53 -0800 | [diff] [blame] | 7290 | |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7291 | /* Hopeless node, leave it to direct reclaim */ |
| 7292 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) |
| 7293 | return true; |
| 7294 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7295 | if (pgdat_balanced(pgdat, order, highest_zoneidx)) { |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7296 | clear_pgdat_congested(pgdat); |
| 7297 | return true; |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7298 | } |
| 7299 | |
Shantanu Goel | 333b0a4 | 2017-05-03 14:53:38 -0700 | [diff] [blame] | 7300 | return false; |
Mel Gorman | f50de2d | 2009-12-14 17:58:53 -0800 | [diff] [blame] | 7301 | } |
| 7302 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7303 | /* |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7304 | * kswapd shrinks a node of pages that are at or below the highest usable |
| 7305 | * zone that is currently unbalanced. |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7306 | * |
| 7307 | * Returns true if kswapd scanned at least the requested number of pages to |
Mel Gorman | 283aba9 | 2013-07-03 15:01:51 -0700 | [diff] [blame] | 7308 | * reclaim or if the lack of progress was due to pages under writeback. |
| 7309 | * This is used to determine if the scanning priority needs to be raised. |
Mel Gorman | 7548536 | 2013-07-03 15:01:42 -0700 | [diff] [blame] | 7310 | */ |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7311 | static bool kswapd_shrink_node(pg_data_t *pgdat, |
Vlastimil Babka | accf624 | 2016-03-17 14:18:15 -0700 | [diff] [blame] | 7312 | struct scan_control *sc) |
Mel Gorman | 7548536 | 2013-07-03 15:01:42 -0700 | [diff] [blame] | 7313 | { |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7314 | struct zone *zone; |
| 7315 | int z; |
Mel Gorman | 7548536 | 2013-07-03 15:01:42 -0700 | [diff] [blame] | 7316 | |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7317 | /* Reclaim a number of pages proportional to the number of zones */ |
| 7318 | sc->nr_to_reclaim = 0; |
Mel Gorman | 970a39a | 2016-07-28 15:46:35 -0700 | [diff] [blame] | 7319 | for (z = 0; z <= sc->reclaim_idx; z++) { |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7320 | zone = pgdat->node_zones + z; |
Mel Gorman | 6aa303d | 2016-09-01 16:14:55 -0700 | [diff] [blame] | 7321 | if (!managed_zone(zone)) |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7322 | continue; |
Mel Gorman | 7c954f6 | 2013-07-03 15:01:54 -0700 | [diff] [blame] | 7323 | |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7324 | sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); |
Mel Gorman | 7c954f6 | 2013-07-03 15:01:54 -0700 | [diff] [blame] | 7325 | } |
| 7326 | |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7327 | /* |
| 7328 | * Historically care was taken to put equal pressure on all zones but |
| 7329 | * now pressure is applied based on node LRU order. |
| 7330 | */ |
Mel Gorman | 970a39a | 2016-07-28 15:46:35 -0700 | [diff] [blame] | 7331 | shrink_node(pgdat, sc); |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7332 | |
| 7333 | /* |
| 7334 | * Fragmentation may mean that the system cannot be rebalanced for |
| 7335 | * high-order allocations. If twice the allocation size has been |
| 7336 | * reclaimed then recheck watermarks only at order-0 to prevent |
| 7337 | * excessive reclaim. Assume that a process requested a high-order |
| 7338 | * can direct reclaim/compact. |
| 7339 | */ |
Vlastimil Babka | 9861a62 | 2016-10-07 16:57:53 -0700 | [diff] [blame] | 7340 | if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7341 | sc->order = 0; |
| 7342 | |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7343 | return sc->nr_scanned >= sc->nr_to_reclaim; |
Mel Gorman | 7548536 | 2013-07-03 15:01:42 -0700 | [diff] [blame] | 7344 | } |
| 7345 | |
Mel Gorman | c49c2c4 | 2021-06-28 19:42:21 -0700 | [diff] [blame] | 7346 | /* Page allocator PCP high watermark is lowered if reclaim is active. */ |
| 7347 | static inline void |
| 7348 | update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active) |
| 7349 | { |
| 7350 | int i; |
| 7351 | struct zone *zone; |
| 7352 | |
| 7353 | for (i = 0; i <= highest_zoneidx; i++) { |
| 7354 | zone = pgdat->node_zones + i; |
| 7355 | |
| 7356 | if (!managed_zone(zone)) |
| 7357 | continue; |
| 7358 | |
| 7359 | if (active) |
| 7360 | set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); |
| 7361 | else |
| 7362 | clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); |
| 7363 | } |
| 7364 | } |
| 7365 | |
| 7366 | static inline void |
| 7367 | set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) |
| 7368 | { |
| 7369 | update_reclaim_active(pgdat, highest_zoneidx, true); |
| 7370 | } |
| 7371 | |
| 7372 | static inline void |
| 7373 | clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) |
| 7374 | { |
| 7375 | update_reclaim_active(pgdat, highest_zoneidx, false); |
| 7376 | } |
| 7377 | |
Mel Gorman | 7548536 | 2013-07-03 15:01:42 -0700 | [diff] [blame] | 7378 | /* |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7379 | * For kswapd, balance_pgdat() will reclaim pages across a node from zones |
| 7380 | * that are eligible for use by the caller until at least one zone is |
| 7381 | * balanced. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7382 | * |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7383 | * Returns the order kswapd finished reclaiming at. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7384 | * |
| 7385 | * kswapd scans the zones in the highmem->normal->dma direction. It skips |
Mel Gorman | 4185896 | 2009-06-16 15:32:12 -0700 | [diff] [blame] | 7386 | * zones which have free_pages > high_wmark_pages(zone), but once a zone is |
Wei Yang | 8bb4e7a | 2019-03-05 15:46:22 -0800 | [diff] [blame] | 7387 | * found to have free_pages <= high_wmark_pages(zone), any page in that zone |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7388 | * or lower is eligible for reclaim until at least one usable zone is |
| 7389 | * balanced. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7390 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7391 | static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7392 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7393 | int i; |
Andrew Morton | 0608f43 | 2013-09-24 15:27:41 -0700 | [diff] [blame] | 7394 | unsigned long nr_soft_reclaimed; |
| 7395 | unsigned long nr_soft_scanned; |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 7396 | unsigned long pflags; |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7397 | unsigned long nr_boost_reclaim; |
| 7398 | unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; |
| 7399 | bool boosted; |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7400 | struct zone *zone; |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 7401 | struct scan_control sc = { |
| 7402 | .gfp_mask = GFP_KERNEL, |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 7403 | .order = order, |
Johannes Weiner | a6dc60f8 | 2009-03-31 15:19:30 -0700 | [diff] [blame] | 7404 | .may_unmap = 1, |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 7405 | }; |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 7406 | |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7407 | set_task_reclaim_state(current, &sc.reclaim_state); |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 7408 | psi_memstall_enter(&pflags); |
Matthew Wilcox (Oracle) | 4f3eaf4 | 2021-09-02 14:52:58 -0700 | [diff] [blame] | 7409 | __fs_reclaim_acquire(_THIS_IP_); |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 7410 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 7411 | count_vm_event(PAGEOUTRUN); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7412 | |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7413 | /* |
| 7414 | * Account for the reclaim boost. Note that the zone boost is left in |
| 7415 | * place so that parallel allocations that are near the watermark will |
| 7416 | * stall or direct reclaim until kswapd is finished. |
| 7417 | */ |
| 7418 | nr_boost_reclaim = 0; |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7419 | for (i = 0; i <= highest_zoneidx; i++) { |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7420 | zone = pgdat->node_zones + i; |
| 7421 | if (!managed_zone(zone)) |
| 7422 | continue; |
| 7423 | |
| 7424 | nr_boost_reclaim += zone->watermark_boost; |
| 7425 | zone_boosts[i] = zone->watermark_boost; |
| 7426 | } |
| 7427 | boosted = nr_boost_reclaim; |
| 7428 | |
| 7429 | restart: |
Mel Gorman | c49c2c4 | 2021-06-28 19:42:21 -0700 | [diff] [blame] | 7430 | set_reclaim_active(pgdat, highest_zoneidx); |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7431 | sc.priority = DEF_PRIORITY; |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 7432 | do { |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7433 | unsigned long nr_reclaimed = sc.nr_reclaimed; |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7434 | bool raise_priority = true; |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7435 | bool balanced; |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 7436 | bool ret; |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7437 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7438 | sc.reclaim_idx = highest_zoneidx; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7439 | |
Mel Gorman | 86c79f6 | 2016-07-28 15:45:59 -0700 | [diff] [blame] | 7440 | /* |
Mel Gorman | 84c7a77 | 2016-07-28 15:46:44 -0700 | [diff] [blame] | 7441 | * If the number of buffer_heads exceeds the maximum allowed |
| 7442 | * then consider reclaiming from all zones. This has a dual |
| 7443 | * purpose -- on 64-bit systems it is expected that |
| 7444 | * buffer_heads are stripped during active rotation. On 32-bit |
| 7445 | * systems, highmem pages can pin lowmem memory and shrinking |
| 7446 | * buffers can relieve lowmem pressure. Reclaim may still not |
| 7447 | * go ahead if all eligible zones for the original allocation |
| 7448 | * request are balanced to avoid excessive reclaim from kswapd. |
Mel Gorman | 86c79f6 | 2016-07-28 15:45:59 -0700 | [diff] [blame] | 7449 | */ |
| 7450 | if (buffer_heads_over_limit) { |
| 7451 | for (i = MAX_NR_ZONES - 1; i >= 0; i--) { |
| 7452 | zone = pgdat->node_zones + i; |
Mel Gorman | 6aa303d | 2016-09-01 16:14:55 -0700 | [diff] [blame] | 7453 | if (!managed_zone(zone)) |
Mel Gorman | 86c79f6 | 2016-07-28 15:45:59 -0700 | [diff] [blame] | 7454 | continue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7455 | |
Mel Gorman | 970a39a | 2016-07-28 15:46:35 -0700 | [diff] [blame] | 7456 | sc.reclaim_idx = i; |
Andrew Morton | e1dbeda | 2006-12-06 20:32:01 -0800 | [diff] [blame] | 7457 | break; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7458 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7459 | } |
Zlatko Calusic | dafcb73 | 2013-02-22 16:32:34 -0800 | [diff] [blame] | 7460 | |
Mel Gorman | 86c79f6 | 2016-07-28 15:45:59 -0700 | [diff] [blame] | 7461 | /* |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7462 | * If the pgdat is imbalanced then ignore boosting and preserve |
| 7463 | * the watermarks for a later time and restart. Note that the |
| 7464 | * zone watermarks will be still reset at the end of balancing |
| 7465 | * on the grounds that the normal reclaim should be enough to |
| 7466 | * re-evaluate if boosting is required when kswapd next wakes. |
Mel Gorman | 86c79f6 | 2016-07-28 15:45:59 -0700 | [diff] [blame] | 7467 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7468 | balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx); |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7469 | if (!balanced && nr_boost_reclaim) { |
| 7470 | nr_boost_reclaim = 0; |
| 7471 | goto restart; |
| 7472 | } |
| 7473 | |
| 7474 | /* |
| 7475 | * If boosting is not active then only reclaim if there are no |
| 7476 | * eligible zones. Note that sc.reclaim_idx is not used as |
| 7477 | * buffer_heads_over_limit may have adjusted it. |
| 7478 | */ |
| 7479 | if (!nr_boost_reclaim && balanced) |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7480 | goto out; |
Andrew Morton | e1dbeda | 2006-12-06 20:32:01 -0800 | [diff] [blame] | 7481 | |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7482 | /* Limit the priority of boosting to avoid reclaim writeback */ |
| 7483 | if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2) |
| 7484 | raise_priority = false; |
| 7485 | |
| 7486 | /* |
| 7487 | * Do not writeback or swap pages for boosted reclaim. The |
| 7488 | * intent is to relieve pressure not issue sub-optimal IO |
| 7489 | * from reclaim context. If no pages are reclaimed, the |
| 7490 | * reclaim will be aborted. |
| 7491 | */ |
| 7492 | sc.may_writepage = !laptop_mode && !nr_boost_reclaim; |
| 7493 | sc.may_swap = !nr_boost_reclaim; |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7494 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7495 | /* |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 7496 | * Do some background aging, to give pages a chance to be |
| 7497 | * referenced before reclaiming. All pages are rotated |
| 7498 | * regardless of classzone as this is about consistent aging. |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7499 | */ |
Yu Zhao | ac35a49 | 2022-09-18 02:00:03 -0600 | [diff] [blame] | 7500 | kswapd_age_node(pgdat, &sc); |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7501 | |
| 7502 | /* |
Mel Gorman | b7ea3c4 | 2013-07-03 15:01:53 -0700 | [diff] [blame] | 7503 | * If we're getting trouble reclaiming, start doing writepage |
| 7504 | * even in laptop mode. |
| 7505 | */ |
Johannes Weiner | 047d72c | 2017-05-03 14:51:57 -0700 | [diff] [blame] | 7506 | if (sc.priority < DEF_PRIORITY - 2) |
Mel Gorman | b7ea3c4 | 2013-07-03 15:01:53 -0700 | [diff] [blame] | 7507 | sc.may_writepage = 1; |
| 7508 | |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7509 | /* Call soft limit reclaim before calling shrink_node. */ |
| 7510 | sc.nr_scanned = 0; |
| 7511 | nr_soft_scanned = 0; |
Mel Gorman | ef8f232 | 2016-07-28 15:46:05 -0700 | [diff] [blame] | 7512 | nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order, |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7513 | sc.gfp_mask, &nr_soft_scanned); |
| 7514 | sc.nr_reclaimed += nr_soft_reclaimed; |
| 7515 | |
Mel Gorman | b7ea3c4 | 2013-07-03 15:01:53 -0700 | [diff] [blame] | 7516 | /* |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7517 | * There should be no need to raise the scanning priority if |
| 7518 | * enough pages are already being scanned that that high |
| 7519 | * watermark would be met at 100% efficiency. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7520 | */ |
Mel Gorman | 970a39a | 2016-07-28 15:46:35 -0700 | [diff] [blame] | 7521 | if (kswapd_shrink_node(pgdat, &sc)) |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7522 | raise_priority = false; |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7523 | |
| 7524 | /* |
| 7525 | * If the low watermark is met there is no need for processes |
| 7526 | * to be throttled on pfmemalloc_wait as they should not be |
| 7527 | * able to safely make forward progress. Wake them |
| 7528 | */ |
| 7529 | if (waitqueue_active(&pgdat->pfmemalloc_wait) && |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7530 | allow_direct_reclaim(pgdat)) |
Vlastimil Babka | cfc5115 | 2015-02-11 15:25:12 -0800 | [diff] [blame] | 7531 | wake_up_all(&pgdat->pfmemalloc_wait); |
Mel Gorman | 5515061 | 2012-07-31 16:44:35 -0700 | [diff] [blame] | 7532 | |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7533 | /* Check if kswapd should be suspending */ |
Matthew Wilcox (Oracle) | 4f3eaf4 | 2021-09-02 14:52:58 -0700 | [diff] [blame] | 7534 | __fs_reclaim_release(_THIS_IP_); |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 7535 | ret = try_to_freeze(); |
Matthew Wilcox (Oracle) | 4f3eaf4 | 2021-09-02 14:52:58 -0700 | [diff] [blame] | 7536 | __fs_reclaim_acquire(_THIS_IP_); |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 7537 | if (ret || kthread_should_stop()) |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7538 | break; |
| 7539 | |
| 7540 | /* |
| 7541 | * Raise priority if scanning rate is too low or there was no |
| 7542 | * progress in reclaiming pages |
| 7543 | */ |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7544 | nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7545 | nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed); |
| 7546 | |
| 7547 | /* |
| 7548 | * If reclaim made no progress for a boost, stop reclaim as |
| 7549 | * IO cannot be queued and it could be an infinite loop in |
| 7550 | * extreme circumstances. |
| 7551 | */ |
| 7552 | if (nr_boost_reclaim && !nr_reclaimed) |
| 7553 | break; |
| 7554 | |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7555 | if (raise_priority || !nr_reclaimed) |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7556 | sc.priority--; |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7557 | } while (sc.priority >= 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7558 | |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7559 | if (!sc.nr_reclaimed) |
| 7560 | pgdat->kswapd_failures++; |
| 7561 | |
Mel Gorman | b8e83b9 | 2013-07-03 15:01:45 -0700 | [diff] [blame] | 7562 | out: |
Mel Gorman | c49c2c4 | 2021-06-28 19:42:21 -0700 | [diff] [blame] | 7563 | clear_reclaim_active(pgdat, highest_zoneidx); |
| 7564 | |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7565 | /* If reclaim was boosted, account for the reclaim done in this pass */ |
| 7566 | if (boosted) { |
| 7567 | unsigned long flags; |
| 7568 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7569 | for (i = 0; i <= highest_zoneidx; i++) { |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7570 | if (!zone_boosts[i]) |
| 7571 | continue; |
| 7572 | |
| 7573 | /* Increments are under the zone lock */ |
| 7574 | zone = pgdat->node_zones + i; |
| 7575 | spin_lock_irqsave(&zone->lock, flags); |
| 7576 | zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]); |
| 7577 | spin_unlock_irqrestore(&zone->lock, flags); |
| 7578 | } |
| 7579 | |
| 7580 | /* |
| 7581 | * As there is now likely space, wakeup kcompact to defragment |
| 7582 | * pageblocks. |
| 7583 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7584 | wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx); |
Mel Gorman | 1c30844 | 2018-12-28 00:35:52 -0800 | [diff] [blame] | 7585 | } |
| 7586 | |
Johannes Weiner | 2a2e488 | 2017-05-03 14:55:03 -0700 | [diff] [blame] | 7587 | snapshot_refaults(NULL, pgdat); |
Matthew Wilcox (Oracle) | 4f3eaf4 | 2021-09-02 14:52:58 -0700 | [diff] [blame] | 7588 | __fs_reclaim_release(_THIS_IP_); |
Johannes Weiner | eb41468 | 2018-10-26 15:06:27 -0700 | [diff] [blame] | 7589 | psi_memstall_leave(&pflags); |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7590 | set_task_reclaim_state(current, NULL); |
Yafang Shao | e5ca807 | 2019-07-16 16:26:09 -0700 | [diff] [blame] | 7591 | |
Mel Gorman | 0abdee2 | 2011-01-13 15:46:22 -0800 | [diff] [blame] | 7592 | /* |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7593 | * Return the order kswapd stopped reclaiming at as |
| 7594 | * prepare_kswapd_sleep() takes it into account. If another caller |
| 7595 | * entered the allocator slow path while kswapd was awake, order will |
| 7596 | * remain at the higher level. |
Mel Gorman | 0abdee2 | 2011-01-13 15:46:22 -0800 | [diff] [blame] | 7597 | */ |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7598 | return sc.order; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7599 | } |
| 7600 | |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7601 | /* |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7602 | * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to |
| 7603 | * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is |
| 7604 | * not a valid index then either kswapd runs for first time or kswapd couldn't |
| 7605 | * sleep after previous reclaim attempt (node is still unbalanced). In that |
| 7606 | * case return the zone index of the previous kswapd reclaim cycle. |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7607 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7608 | static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, |
| 7609 | enum zone_type prev_highest_zoneidx) |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7610 | { |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7611 | enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7612 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7613 | return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7614 | } |
| 7615 | |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7616 | static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7617 | unsigned int highest_zoneidx) |
KOSAKI Motohiro | f0bc0a6 | 2011-01-13 15:45:50 -0800 | [diff] [blame] | 7618 | { |
| 7619 | long remaining = 0; |
| 7620 | DEFINE_WAIT(wait); |
| 7621 | |
| 7622 | if (freezing(current) || kthread_should_stop()) |
| 7623 | return; |
| 7624 | |
| 7625 | prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); |
| 7626 | |
Shantanu Goel | 333b0a4 | 2017-05-03 14:53:38 -0700 | [diff] [blame] | 7627 | /* |
| 7628 | * Try to sleep for a short interval. Note that kcompactd will only be |
| 7629 | * woken if it is possible to sleep for a short interval. This is |
| 7630 | * deliberate on the assumption that if reclaim cannot keep an |
| 7631 | * eligible zone balanced that it's also unlikely that compaction will |
| 7632 | * succeed. |
| 7633 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7634 | if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { |
Vlastimil Babka | fd901c9 | 2016-04-28 16:18:49 -0700 | [diff] [blame] | 7635 | /* |
| 7636 | * Compaction records what page blocks it recently failed to |
| 7637 | * isolate pages from and skips them in the future scanning. |
| 7638 | * When kswapd is going to sleep, it is reasonable to assume |
| 7639 | * that pages and compaction may succeed so reset the cache. |
| 7640 | */ |
| 7641 | reset_isolation_suitable(pgdat); |
| 7642 | |
| 7643 | /* |
| 7644 | * We have freed the memory, now we should compact it to make |
| 7645 | * allocation of the requested order possible. |
| 7646 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7647 | wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx); |
Vlastimil Babka | fd901c9 | 2016-04-28 16:18:49 -0700 | [diff] [blame] | 7648 | |
KOSAKI Motohiro | f0bc0a6 | 2011-01-13 15:45:50 -0800 | [diff] [blame] | 7649 | remaining = schedule_timeout(HZ/10); |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7650 | |
| 7651 | /* |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7652 | * If woken prematurely then reset kswapd_highest_zoneidx and |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7653 | * order. The values will either be from a wakeup request or |
| 7654 | * the previous request that slept prematurely. |
| 7655 | */ |
| 7656 | if (remaining) { |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7657 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, |
| 7658 | kswapd_highest_zoneidx(pgdat, |
| 7659 | highest_zoneidx)); |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7660 | |
| 7661 | if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) |
| 7662 | WRITE_ONCE(pgdat->kswapd_order, reclaim_order); |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7663 | } |
| 7664 | |
KOSAKI Motohiro | f0bc0a6 | 2011-01-13 15:45:50 -0800 | [diff] [blame] | 7665 | finish_wait(&pgdat->kswapd_wait, &wait); |
| 7666 | prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); |
| 7667 | } |
| 7668 | |
| 7669 | /* |
| 7670 | * After a short sleep, check if it was a premature sleep. If not, then |
| 7671 | * go fully to sleep until explicitly woken up. |
| 7672 | */ |
Mel Gorman | d9f21d4 | 2016-07-28 15:46:41 -0700 | [diff] [blame] | 7673 | if (!remaining && |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7674 | prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { |
KOSAKI Motohiro | f0bc0a6 | 2011-01-13 15:45:50 -0800 | [diff] [blame] | 7675 | trace_mm_vmscan_kswapd_sleep(pgdat->node_id); |
| 7676 | |
| 7677 | /* |
| 7678 | * vmstat counters are not perfectly accurate and the estimated |
| 7679 | * value for counters such as NR_FREE_PAGES can deviate from the |
| 7680 | * true value by nr_online_cpus * threshold. To avoid the zone |
| 7681 | * watermarks being breached while under pressure, we reduce the |
| 7682 | * per-cpu vmstat threshold while kswapd is awake and restore |
| 7683 | * them before going back to sleep. |
| 7684 | */ |
| 7685 | set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold); |
Aaditya Kumar | 1c7e7f6 | 2012-07-17 15:48:07 -0700 | [diff] [blame] | 7686 | |
| 7687 | if (!kthread_should_stop()) |
| 7688 | schedule(); |
| 7689 | |
KOSAKI Motohiro | f0bc0a6 | 2011-01-13 15:45:50 -0800 | [diff] [blame] | 7690 | set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold); |
| 7691 | } else { |
| 7692 | if (remaining) |
| 7693 | count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); |
| 7694 | else |
| 7695 | count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY); |
| 7696 | } |
| 7697 | finish_wait(&pgdat->kswapd_wait, &wait); |
| 7698 | } |
| 7699 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7700 | /* |
| 7701 | * The background pageout daemon, started as a kernel thread |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 7702 | * from the init process. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7703 | * |
| 7704 | * This basically trickles out pages so that we have _some_ |
| 7705 | * free memory available even if there is no other activity |
| 7706 | * that frees anything up. This is needed for things like routing |
| 7707 | * etc, where we otherwise might have all activity going on in |
| 7708 | * asynchronous contexts that cannot page things out. |
| 7709 | * |
| 7710 | * If there are applications that are active memory-allocators |
| 7711 | * (most normal use), this basically shouldn't matter. |
| 7712 | */ |
| 7713 | static int kswapd(void *p) |
| 7714 | { |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7715 | unsigned int alloc_order, reclaim_order; |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7716 | unsigned int highest_zoneidx = MAX_NR_ZONES - 1; |
Zhiyuan Dai | 68d68ff | 2021-05-04 18:40:12 -0700 | [diff] [blame] | 7717 | pg_data_t *pgdat = (pg_data_t *)p; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7718 | struct task_struct *tsk = current; |
Rusty Russell | a70f730 | 2009-03-13 14:49:46 +1030 | [diff] [blame] | 7719 | const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7720 | |
Rusty Russell | 174596a | 2009-01-01 10:12:29 +1030 | [diff] [blame] | 7721 | if (!cpumask_empty(cpumask)) |
Mike Travis | c5f59f0 | 2008-04-04 18:11:10 -0700 | [diff] [blame] | 7722 | set_cpus_allowed_ptr(tsk, cpumask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7723 | |
| 7724 | /* |
| 7725 | * Tell the memory management that we're a "memory allocator", |
| 7726 | * and that if we need more memory we should get access to it |
| 7727 | * regardless (see "__alloc_pages()"). "kswapd" should |
| 7728 | * never get caught in the normal page freeing logic. |
| 7729 | * |
| 7730 | * (Kswapd normally doesn't need memory anyway, but sometimes |
| 7731 | * you need a small amount of memory in order to be able to |
| 7732 | * page out something else, and this flag essentially protects |
| 7733 | * us from recursively trying to free more memory as we're |
| 7734 | * trying to free the first piece of memory in the first place). |
| 7735 | */ |
Hugh Dickins | b698f0a | 2022-03-22 14:45:38 -0700 | [diff] [blame] | 7736 | tsk->flags |= PF_MEMALLOC | PF_KSWAPD; |
Rafael J. Wysocki | 8314418 | 2007-07-17 04:03:35 -0700 | [diff] [blame] | 7737 | set_freezable(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7738 | |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7739 | WRITE_ONCE(pgdat->kswapd_order, 0); |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7740 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); |
Mel Gorman | 8cd7c58 | 2021-11-05 13:42:25 -0700 | [diff] [blame] | 7741 | atomic_set(&pgdat->nr_writeback_throttled, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7742 | for ( ; ; ) { |
Jeff Liu | 6f6313d | 2012-12-11 16:02:48 -0800 | [diff] [blame] | 7743 | bool ret; |
Christoph Lameter | 3e1d1d2 | 2005-06-24 23:13:50 -0700 | [diff] [blame] | 7744 | |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7745 | alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7746 | highest_zoneidx = kswapd_highest_zoneidx(pgdat, |
| 7747 | highest_zoneidx); |
Mel Gorman | e716f2e | 2017-05-03 14:53:45 -0700 | [diff] [blame] | 7748 | |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7749 | kswapd_try_sleep: |
| 7750 | kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7751 | highest_zoneidx); |
Mel Gorman | 215ddd6 | 2011-07-08 15:39:40 -0700 | [diff] [blame] | 7752 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7753 | /* Read the new order and highest_zoneidx */ |
Lukas Bulwahn | 2b47a24 | 2020-12-14 19:12:18 -0800 | [diff] [blame] | 7754 | alloc_order = READ_ONCE(pgdat->kswapd_order); |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7755 | highest_zoneidx = kswapd_highest_zoneidx(pgdat, |
| 7756 | highest_zoneidx); |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7757 | WRITE_ONCE(pgdat->kswapd_order, 0); |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7758 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7759 | |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 7760 | ret = try_to_freeze(); |
| 7761 | if (kthread_should_stop()) |
| 7762 | break; |
| 7763 | |
| 7764 | /* |
| 7765 | * We can speed up thawing tasks if we don't call balance_pgdat |
| 7766 | * after returning from the refrigerator |
| 7767 | */ |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7768 | if (ret) |
| 7769 | continue; |
Mel Gorman | 1d82de6 | 2016-07-28 15:45:43 -0700 | [diff] [blame] | 7770 | |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7771 | /* |
| 7772 | * Reclaim begins at the requested order but if a high-order |
| 7773 | * reclaim fails then kswapd falls back to reclaiming for |
| 7774 | * order-0. If that happens, kswapd will consider sleeping |
| 7775 | * for the order it finished reclaiming at (reclaim_order) |
| 7776 | * but kcompactd is woken to compact for the original |
| 7777 | * request (alloc_order). |
| 7778 | */ |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7779 | trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx, |
Mel Gorman | e5146b1 | 2016-07-28 15:46:47 -0700 | [diff] [blame] | 7780 | alloc_order); |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7781 | reclaim_order = balance_pgdat(pgdat, alloc_order, |
| 7782 | highest_zoneidx); |
Mel Gorman | 38087d9 | 2016-07-28 15:45:49 -0700 | [diff] [blame] | 7783 | if (reclaim_order < alloc_order) |
| 7784 | goto kswapd_try_sleep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7785 | } |
Takamori Yamaguchi | b0a8cc5 | 2012-11-08 15:53:39 -0800 | [diff] [blame] | 7786 | |
Hugh Dickins | b698f0a | 2022-03-22 14:45:38 -0700 | [diff] [blame] | 7787 | tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); |
Johannes Weiner | 71abdc1 | 2014-06-06 14:35:35 -0700 | [diff] [blame] | 7788 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7789 | return 0; |
| 7790 | } |
| 7791 | |
| 7792 | /* |
David Rientjes | 5ecd9d4 | 2018-04-05 16:25:16 -0700 | [diff] [blame] | 7793 | * A zone is low on free memory or too fragmented for high-order memory. If |
| 7794 | * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's |
| 7795 | * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim |
| 7796 | * has failed or is not needed, still wake up kcompactd if only compaction is |
| 7797 | * needed. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7798 | */ |
David Rientjes | 5ecd9d4 | 2018-04-05 16:25:16 -0700 | [diff] [blame] | 7799 | void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7800 | enum zone_type highest_zoneidx) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7801 | { |
| 7802 | pg_data_t *pgdat; |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7803 | enum zone_type curr_idx; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7804 | |
Mel Gorman | 6aa303d | 2016-09-01 16:14:55 -0700 | [diff] [blame] | 7805 | if (!managed_zone(zone)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7806 | return; |
| 7807 | |
David Rientjes | 5ecd9d4 | 2018-04-05 16:25:16 -0700 | [diff] [blame] | 7808 | if (!cpuset_zone_allowed(zone, gfp_flags)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7809 | return; |
Shakeel Butt | dffcac2c | 2019-07-04 15:14:42 -0700 | [diff] [blame] | 7810 | |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7811 | pgdat = zone->zone_pgdat; |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7812 | curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7813 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7814 | if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) |
| 7815 | WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); |
Qian Cai | 5644e1fb | 2020-04-01 21:10:12 -0700 | [diff] [blame] | 7816 | |
| 7817 | if (READ_ONCE(pgdat->kswapd_order) < order) |
| 7818 | WRITE_ONCE(pgdat->kswapd_order, order); |
| 7819 | |
Con Kolivas | 8d0986e | 2005-09-13 01:25:07 -0700 | [diff] [blame] | 7820 | if (!waitqueue_active(&pgdat->kswapd_wait)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7821 | return; |
Mel Gorman | e1a5563 | 2016-07-28 15:46:26 -0700 | [diff] [blame] | 7822 | |
David Rientjes | 5ecd9d4 | 2018-04-05 16:25:16 -0700 | [diff] [blame] | 7823 | /* Hopeless node, leave it to direct reclaim if possible */ |
| 7824 | if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES || |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7825 | (pgdat_balanced(pgdat, order, highest_zoneidx) && |
| 7826 | !pgdat_watermark_boosted(pgdat, highest_zoneidx))) { |
David Rientjes | 5ecd9d4 | 2018-04-05 16:25:16 -0700 | [diff] [blame] | 7827 | /* |
| 7828 | * There may be plenty of free memory available, but it's too |
| 7829 | * fragmented for high-order allocations. Wake up kcompactd |
| 7830 | * and rely on compaction_suitable() to determine if it's |
| 7831 | * needed. If it fails, it will defer subsequent attempts to |
| 7832 | * ratelimit its work. |
| 7833 | */ |
| 7834 | if (!(gfp_flags & __GFP_DIRECT_RECLAIM)) |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7835 | wakeup_kcompactd(pgdat, order, highest_zoneidx); |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7836 | return; |
David Rientjes | 5ecd9d4 | 2018-04-05 16:25:16 -0700 | [diff] [blame] | 7837 | } |
Johannes Weiner | c73322d | 2017-05-03 14:51:51 -0700 | [diff] [blame] | 7838 | |
Joonsoo Kim | 97a225e | 2020-06-03 15:59:01 -0700 | [diff] [blame] | 7839 | trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order, |
David Rientjes | 5ecd9d4 | 2018-04-05 16:25:16 -0700 | [diff] [blame] | 7840 | gfp_flags); |
Con Kolivas | 8d0986e | 2005-09-13 01:25:07 -0700 | [diff] [blame] | 7841 | wake_up_interruptible(&pgdat->kswapd_wait); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7842 | } |
| 7843 | |
Rafael J. Wysocki | c6f37f1 | 2009-05-24 22:16:31 +0200 | [diff] [blame] | 7844 | #ifdef CONFIG_HIBERNATION |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7845 | /* |
KOSAKI Motohiro | 7b51755 | 2009-12-14 17:59:12 -0800 | [diff] [blame] | 7846 | * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of |
Rafael J. Wysocki | d6277db | 2006-06-23 02:03:18 -0700 | [diff] [blame] | 7847 | * freed pages. |
| 7848 | * |
| 7849 | * Rather than trying to age LRUs the aim is to preserve the overall |
| 7850 | * LRU order by reclaiming preferentially |
| 7851 | * inactive > active > active referenced > active mapped |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7852 | */ |
KOSAKI Motohiro | 7b51755 | 2009-12-14 17:59:12 -0800 | [diff] [blame] | 7853 | unsigned long shrink_all_memory(unsigned long nr_to_reclaim) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7854 | { |
Rafael J. Wysocki | d6277db | 2006-06-23 02:03:18 -0700 | [diff] [blame] | 7855 | struct scan_control sc = { |
KOSAKI Motohiro | 7b51755 | 2009-12-14 17:59:12 -0800 | [diff] [blame] | 7856 | .nr_to_reclaim = nr_to_reclaim, |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 7857 | .gfp_mask = GFP_HIGHUSER_MOVABLE, |
Mel Gorman | b2e1875 | 2016-07-28 15:45:37 -0700 | [diff] [blame] | 7858 | .reclaim_idx = MAX_NR_ZONES - 1, |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 7859 | .priority = DEF_PRIORITY, |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 7860 | .may_writepage = 1, |
| 7861 | .may_unmap = 1, |
| 7862 | .may_swap = 1, |
| 7863 | .hibernation_mode = 1, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7864 | }; |
Ying Han | a09ed5e | 2011-05-24 17:12:26 -0700 | [diff] [blame] | 7865 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); |
KOSAKI Motohiro | 7b51755 | 2009-12-14 17:59:12 -0800 | [diff] [blame] | 7866 | unsigned long nr_reclaimed; |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 7867 | unsigned int noreclaim_flag; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7868 | |
Peter Zijlstra | d92a8cf | 2017-03-03 10:13:38 +0100 | [diff] [blame] | 7869 | fs_reclaim_acquire(sc.gfp_mask); |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 7870 | noreclaim_flag = memalloc_noreclaim_save(); |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7871 | set_task_reclaim_state(current, &sc.reclaim_state); |
Andrew Morton | 69e0594 | 2006-03-22 00:08:19 -0800 | [diff] [blame] | 7872 | |
Vladimir Davydov | 3115cd9 | 2014-04-03 14:47:22 -0700 | [diff] [blame] | 7873 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); |
Rafael J. Wysocki | d6277db | 2006-06-23 02:03:18 -0700 | [diff] [blame] | 7874 | |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 7875 | set_task_reclaim_state(current, NULL); |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 7876 | memalloc_noreclaim_restore(noreclaim_flag); |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 7877 | fs_reclaim_release(sc.gfp_mask); |
Rafael J. Wysocki | d6277db | 2006-06-23 02:03:18 -0700 | [diff] [blame] | 7878 | |
KOSAKI Motohiro | 7b51755 | 2009-12-14 17:59:12 -0800 | [diff] [blame] | 7879 | return nr_reclaimed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7880 | } |
Rafael J. Wysocki | c6f37f1 | 2009-05-24 22:16:31 +0200 | [diff] [blame] | 7881 | #endif /* CONFIG_HIBERNATION */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7882 | |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 7883 | /* |
| 7884 | * This kswapd start function will be called by init and node-hot-add. |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 7885 | */ |
Miaohe Lin | e5797dc | 2023-06-06 20:18:13 +0800 | [diff] [blame] | 7886 | void __meminit kswapd_run(int nid) |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 7887 | { |
| 7888 | pg_data_t *pgdat = NODE_DATA(nid); |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 7889 | |
Kefeng Wang | b4a0215 | 2022-08-27 19:19:59 +0800 | [diff] [blame] | 7890 | pgdat_kswapd_lock(pgdat); |
| 7891 | if (!pgdat->kswapd) { |
| 7892 | pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); |
| 7893 | if (IS_ERR(pgdat->kswapd)) { |
| 7894 | /* failure at boot is fatal */ |
| 7895 | BUG_ON(system_state < SYSTEM_RUNNING); |
| 7896 | pr_err("Failed to start kswapd on node %d\n", nid); |
| 7897 | pgdat->kswapd = NULL; |
| 7898 | } |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 7899 | } |
Kefeng Wang | b4a0215 | 2022-08-27 19:19:59 +0800 | [diff] [blame] | 7900 | pgdat_kswapd_unlock(pgdat); |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 7901 | } |
| 7902 | |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 7903 | /* |
Jiang Liu | d8adde1 | 2012-07-11 14:01:52 -0700 | [diff] [blame] | 7904 | * Called by memory hotplug when all memory in a node is offlined. Caller must |
Yun-Ze Li | e8da368 | 2022-06-20 07:15:16 +0000 | [diff] [blame] | 7905 | * be holding mem_hotplug_begin/done(). |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 7906 | */ |
Miaohe Lin | e5797dc | 2023-06-06 20:18:13 +0800 | [diff] [blame] | 7907 | void __meminit kswapd_stop(int nid) |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 7908 | { |
Kefeng Wang | b4a0215 | 2022-08-27 19:19:59 +0800 | [diff] [blame] | 7909 | pg_data_t *pgdat = NODE_DATA(nid); |
| 7910 | struct task_struct *kswapd; |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 7911 | |
Kefeng Wang | b4a0215 | 2022-08-27 19:19:59 +0800 | [diff] [blame] | 7912 | pgdat_kswapd_lock(pgdat); |
| 7913 | kswapd = pgdat->kswapd; |
Jiang Liu | d8adde1 | 2012-07-11 14:01:52 -0700 | [diff] [blame] | 7914 | if (kswapd) { |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 7915 | kthread_stop(kswapd); |
Kefeng Wang | b4a0215 | 2022-08-27 19:19:59 +0800 | [diff] [blame] | 7916 | pgdat->kswapd = NULL; |
Jiang Liu | d8adde1 | 2012-07-11 14:01:52 -0700 | [diff] [blame] | 7917 | } |
Kefeng Wang | b4a0215 | 2022-08-27 19:19:59 +0800 | [diff] [blame] | 7918 | pgdat_kswapd_unlock(pgdat); |
David Rientjes | 8fe23e0 | 2009-12-14 17:58:33 -0800 | [diff] [blame] | 7919 | } |
| 7920 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7921 | static int __init kswapd_init(void) |
| 7922 | { |
Wei Yang | 6b700b5 | 2020-04-01 21:10:09 -0700 | [diff] [blame] | 7923 | int nid; |
Andrew Morton | 69e0594 | 2006-03-22 00:08:19 -0800 | [diff] [blame] | 7924 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7925 | swap_setup(); |
Lai Jiangshan | 48fb2e2 | 2012-12-12 13:51:43 -0800 | [diff] [blame] | 7926 | for_each_node_state(nid, N_MEMORY) |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 7927 | kswapd_run(nid); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7928 | return 0; |
| 7929 | } |
| 7930 | |
| 7931 | module_init(kswapd_init) |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 7932 | |
| 7933 | #ifdef CONFIG_NUMA |
| 7934 | /* |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7935 | * Node reclaim mode |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 7936 | * |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7937 | * If non-zero call node_reclaim when the number of free pages falls below |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 7938 | * the watermarks. |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 7939 | */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7940 | int node_reclaim_mode __read_mostly; |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 7941 | |
Dave Hansen | 5199836 | 2021-02-24 12:09:15 -0800 | [diff] [blame] | 7942 | /* |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7943 | * Priority for NODE_RECLAIM. This determines the fraction of pages |
Christoph Lameter | a92f712 | 2006-02-01 03:05:32 -0800 | [diff] [blame] | 7944 | * of a node considered for each zone_reclaim. 4 scans 1/16th of |
| 7945 | * a zone. |
| 7946 | */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7947 | #define NODE_RECLAIM_PRIORITY 4 |
Christoph Lameter | a92f712 | 2006-02-01 03:05:32 -0800 | [diff] [blame] | 7948 | |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 7949 | /* |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7950 | * Percentage of pages in a zone that must be unmapped for node_reclaim to |
Christoph Lameter | 9614634 | 2006-07-03 00:24:13 -0700 | [diff] [blame] | 7951 | * occur. |
| 7952 | */ |
| 7953 | int sysctl_min_unmapped_ratio = 1; |
| 7954 | |
| 7955 | /* |
Christoph Lameter | 0ff3849 | 2006-09-25 23:31:52 -0700 | [diff] [blame] | 7956 | * If the number of slab pages in a zone grows beyond this percentage then |
| 7957 | * slab reclaim needs to occur. |
| 7958 | */ |
| 7959 | int sysctl_min_slab_ratio = 5; |
| 7960 | |
Mel Gorman | 11fb998 | 2016-07-28 15:46:20 -0700 | [diff] [blame] | 7961 | static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat) |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7962 | { |
Mel Gorman | 11fb998 | 2016-07-28 15:46:20 -0700 | [diff] [blame] | 7963 | unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED); |
| 7964 | unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) + |
| 7965 | node_page_state(pgdat, NR_ACTIVE_FILE); |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7966 | |
| 7967 | /* |
| 7968 | * It's possible for there to be more file mapped pages than |
| 7969 | * accounted for by the pages on the file LRU lists because |
| 7970 | * tmpfs pages accounted for as ANON can also be FILE_MAPPED |
| 7971 | */ |
| 7972 | return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; |
| 7973 | } |
| 7974 | |
| 7975 | /* Work out how many page cache pages we can reclaim in this reclaim_mode */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7976 | static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7977 | { |
Alexandru Moise | d031a15 | 2015-11-05 18:48:08 -0800 | [diff] [blame] | 7978 | unsigned long nr_pagecache_reclaimable; |
| 7979 | unsigned long delta = 0; |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7980 | |
| 7981 | /* |
Zhihui Zhang | 95bbc0c | 2015-06-24 16:56:42 -0700 | [diff] [blame] | 7982 | * If RECLAIM_UNMAP is set, then all file pages are considered |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7983 | * potentially reclaimable. Otherwise, we have to worry about |
Mel Gorman | 11fb998 | 2016-07-28 15:46:20 -0700 | [diff] [blame] | 7984 | * pages like swapcache and node_unmapped_file_pages() provides |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7985 | * a better estimate |
| 7986 | */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7987 | if (node_reclaim_mode & RECLAIM_UNMAP) |
| 7988 | nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES); |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7989 | else |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7990 | nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat); |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7991 | |
| 7992 | /* If we can't clean pages, remove dirty pages from consideration */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 7993 | if (!(node_reclaim_mode & RECLAIM_WRITE)) |
| 7994 | delta += node_page_state(pgdat, NR_FILE_DIRTY); |
Mel Gorman | 90afa5de | 2009-06-16 15:33:20 -0700 | [diff] [blame] | 7995 | |
| 7996 | /* Watch for any possible underflows due to delta */ |
| 7997 | if (unlikely(delta > nr_pagecache_reclaimable)) |
| 7998 | delta = nr_pagecache_reclaimable; |
| 7999 | |
| 8000 | return nr_pagecache_reclaimable - delta; |
| 8001 | } |
| 8002 | |
Christoph Lameter | 0ff3849 | 2006-09-25 23:31:52 -0700 | [diff] [blame] | 8003 | /* |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8004 | * Try to free up some pages from this node through reclaim. |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 8005 | */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8006 | static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 8007 | { |
Christoph Lameter | 7fb2d46d | 2006-03-22 00:08:22 -0800 | [diff] [blame] | 8008 | /* Minimum pages needed in order to stay on node */ |
Andrew Morton | 69e0594 | 2006-03-22 00:08:19 -0800 | [diff] [blame] | 8009 | const unsigned long nr_pages = 1 << order; |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 8010 | struct task_struct *p = current; |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 8011 | unsigned int noreclaim_flag; |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8012 | struct scan_control sc = { |
Andrew Morton | 62b726c | 2013-02-22 16:32:24 -0800 | [diff] [blame] | 8013 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), |
Nick Desaulniers | f2f43e5 | 2017-07-06 15:36:50 -0700 | [diff] [blame] | 8014 | .gfp_mask = current_gfp_context(gfp_mask), |
Johannes Weiner | bd2f619 | 2009-03-31 15:19:38 -0700 | [diff] [blame] | 8015 | .order = order, |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8016 | .priority = NODE_RECLAIM_PRIORITY, |
| 8017 | .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE), |
| 8018 | .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP), |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 8019 | .may_swap = 1, |
Nick Desaulniers | f2f43e5 | 2017-07-06 15:36:50 -0700 | [diff] [blame] | 8020 | .reclaim_idx = gfp_zone(gfp_mask), |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8021 | }; |
Johannes Weiner | 57f2976 | 2021-08-19 19:04:27 -0700 | [diff] [blame] | 8022 | unsigned long pflags; |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 8023 | |
Yafang Shao | 132bb8c | 2019-05-13 17:17:53 -0700 | [diff] [blame] | 8024 | trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order, |
| 8025 | sc.gfp_mask); |
| 8026 | |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 8027 | cond_resched(); |
Johannes Weiner | 57f2976 | 2021-08-19 19:04:27 -0700 | [diff] [blame] | 8028 | psi_memstall_enter(&pflags); |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 8029 | fs_reclaim_acquire(sc.gfp_mask); |
Christoph Lameter | d4f7796 | 2006-02-24 13:04:22 -0800 | [diff] [blame] | 8030 | /* |
Zhihui Zhang | 95bbc0c | 2015-06-24 16:56:42 -0700 | [diff] [blame] | 8031 | * We need to be able to allocate from the reserves for RECLAIM_UNMAP |
Christoph Lameter | d4f7796 | 2006-02-24 13:04:22 -0800 | [diff] [blame] | 8032 | */ |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 8033 | noreclaim_flag = memalloc_noreclaim_save(); |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 8034 | set_task_reclaim_state(p, &sc.reclaim_state); |
Christoph Lameter | c84db23 | 2006-02-01 03:05:29 -0800 | [diff] [blame] | 8035 | |
Miaohe Lin | d8ff6fd | 2022-05-12 20:22:58 -0700 | [diff] [blame] | 8036 | if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || |
| 8037 | node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { |
Christoph Lameter | 0ff3849 | 2006-09-25 23:31:52 -0700 | [diff] [blame] | 8038 | /* |
Andrey Ryabinin | 894befe | 2018-04-10 16:27:51 -0700 | [diff] [blame] | 8039 | * Free memory by calling shrink node with increasing |
Christoph Lameter | 0ff3849 | 2006-09-25 23:31:52 -0700 | [diff] [blame] | 8040 | * priorities until we have enough memory freed. |
| 8041 | */ |
Christoph Lameter | 0ff3849 | 2006-09-25 23:31:52 -0700 | [diff] [blame] | 8042 | do { |
Mel Gorman | 970a39a | 2016-07-28 15:46:35 -0700 | [diff] [blame] | 8043 | shrink_node(pgdat, &sc); |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 8044 | } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); |
Christoph Lameter | 0ff3849 | 2006-09-25 23:31:52 -0700 | [diff] [blame] | 8045 | } |
Christoph Lameter | c84db23 | 2006-02-01 03:05:29 -0800 | [diff] [blame] | 8046 | |
Andrew Morton | 1732d2b01 | 2019-07-16 16:26:15 -0700 | [diff] [blame] | 8047 | set_task_reclaim_state(p, NULL); |
Vlastimil Babka | 499118e | 2017-05-08 15:59:50 -0700 | [diff] [blame] | 8048 | memalloc_noreclaim_restore(noreclaim_flag); |
Omar Sandoval | 9378132 | 2018-06-07 17:07:02 -0700 | [diff] [blame] | 8049 | fs_reclaim_release(sc.gfp_mask); |
Johannes Weiner | 57f2976 | 2021-08-19 19:04:27 -0700 | [diff] [blame] | 8050 | psi_memstall_leave(&pflags); |
Yafang Shao | 132bb8c | 2019-05-13 17:17:53 -0700 | [diff] [blame] | 8051 | |
| 8052 | trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed); |
| 8053 | |
Rik van Riel | a79311c | 2009-01-06 14:40:01 -0800 | [diff] [blame] | 8054 | return sc.nr_reclaimed >= nr_pages; |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 8055 | } |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8056 | |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8057 | int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8058 | { |
David Rientjes | d773ed6 | 2007-10-16 23:26:01 -0700 | [diff] [blame] | 8059 | int ret; |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8060 | |
| 8061 | /* |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8062 | * Node reclaim reclaims unmapped file backed pages and |
Christoph Lameter | 0ff3849 | 2006-09-25 23:31:52 -0700 | [diff] [blame] | 8063 | * slab pages if we are over the defined limits. |
Christoph Lameter | 34aa133 | 2006-06-30 01:55:37 -0700 | [diff] [blame] | 8064 | * |
Christoph Lameter | 9614634 | 2006-07-03 00:24:13 -0700 | [diff] [blame] | 8065 | * A small portion of unmapped file backed pages is needed for |
| 8066 | * file I/O otherwise pages read by file I/O will be immediately |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8067 | * thrown out if the node is overallocated. So we do not reclaim |
| 8068 | * if less than a specified percentage of the node is used by |
Christoph Lameter | 9614634 | 2006-07-03 00:24:13 -0700 | [diff] [blame] | 8069 | * unmapped file backed pages. |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8070 | */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8071 | if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && |
Roman Gushchin | d42f324 | 2020-08-06 23:20:39 -0700 | [diff] [blame] | 8072 | node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= |
| 8073 | pgdat->min_slab_pages) |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8074 | return NODE_RECLAIM_FULL; |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8075 | |
| 8076 | /* |
David Rientjes | d773ed6 | 2007-10-16 23:26:01 -0700 | [diff] [blame] | 8077 | * Do not scan if the allocation should not be delayed. |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8078 | */ |
Mel Gorman | d0164ad | 2015-11-06 16:28:21 -0800 | [diff] [blame] | 8079 | if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8080 | return NODE_RECLAIM_NOSCAN; |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8081 | |
| 8082 | /* |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8083 | * Only run node reclaim on the local node or on nodes that do not |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8084 | * have associated processors. This will favor the local processor |
| 8085 | * over remote processors and spread off node memory allocations |
| 8086 | * as wide as possible. |
| 8087 | */ |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8088 | if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) |
| 8089 | return NODE_RECLAIM_NOSCAN; |
David Rientjes | d773ed6 | 2007-10-16 23:26:01 -0700 | [diff] [blame] | 8090 | |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8091 | if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) |
| 8092 | return NODE_RECLAIM_NOSCAN; |
Mel Gorman | fa5e084 | 2009-06-16 15:33:22 -0700 | [diff] [blame] | 8093 | |
Mel Gorman | a5f5f91 | 2016-07-28 15:46:32 -0700 | [diff] [blame] | 8094 | ret = __node_reclaim(pgdat, gfp_mask, order); |
| 8095 | clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags); |
David Rientjes | d773ed6 | 2007-10-16 23:26:01 -0700 | [diff] [blame] | 8096 | |
Mel Gorman | 24cf72518 | 2009-06-16 15:33:23 -0700 | [diff] [blame] | 8097 | if (!ret) |
| 8098 | count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); |
| 8099 | |
David Rientjes | d773ed6 | 2007-10-16 23:26:01 -0700 | [diff] [blame] | 8100 | return ret; |
Andrew Morton | 179e963 | 2006-03-22 00:08:18 -0800 | [diff] [blame] | 8101 | } |
Christoph Lameter | 9eeff23 | 2006-01-18 17:42:31 -0800 | [diff] [blame] | 8102 | #endif |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 8103 | |
Matthew Wilcox (Oracle) | 77414d1 | 2022-06-04 17:39:09 -0400 | [diff] [blame] | 8104 | /** |
| 8105 | * check_move_unevictable_folios - Move evictable folios to appropriate zone |
| 8106 | * lru list |
| 8107 | * @fbatch: Batch of lru folios to check. |
| 8108 | * |
| 8109 | * Checks folios for evictability, if an evictable folio is in the unevictable |
| 8110 | * lru list, moves it to the appropriate evictable lru list. This function |
| 8111 | * should be only used for lru folios. |
| 8112 | */ |
| 8113 | void check_move_unevictable_folios(struct folio_batch *fbatch) |
| 8114 | { |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 8115 | struct lruvec *lruvec = NULL; |
Hugh Dickins | 2451326 | 2012-01-20 14:34:21 -0800 | [diff] [blame] | 8116 | int pgscanned = 0; |
| 8117 | int pgrescued = 0; |
| 8118 | int i; |
Lee Schermerhorn | 89e004ea | 2008-10-18 20:26:43 -0700 | [diff] [blame] | 8119 | |
Matthew Wilcox (Oracle) | 77414d1 | 2022-06-04 17:39:09 -0400 | [diff] [blame] | 8120 | for (i = 0; i < fbatch->nr; i++) { |
| 8121 | struct folio *folio = fbatch->folios[i]; |
| 8122 | int nr_pages = folio_nr_pages(folio); |
Lee Schermerhorn | af936a1 | 2008-10-18 20:26:53 -0700 | [diff] [blame] | 8123 | |
Hugh Dickins | 8d8869c | 2020-09-18 21:20:12 -0700 | [diff] [blame] | 8124 | pgscanned += nr_pages; |
| 8125 | |
Matthew Wilcox (Oracle) | 77414d1 | 2022-06-04 17:39:09 -0400 | [diff] [blame] | 8126 | /* block memcg migration while the folio moves between lrus */ |
| 8127 | if (!folio_test_clear_lru(folio)) |
Alex Shi | d25b5bd | 2020-12-15 12:34:16 -0800 | [diff] [blame] | 8128 | continue; |
| 8129 | |
Matthew Wilcox (Oracle) | 0de340c | 2021-06-29 22:27:31 -0400 | [diff] [blame] | 8130 | lruvec = folio_lruvec_relock_irq(folio, lruvec); |
Matthew Wilcox (Oracle) | 77414d1 | 2022-06-04 17:39:09 -0400 | [diff] [blame] | 8131 | if (folio_evictable(folio) && folio_test_unevictable(folio)) { |
| 8132 | lruvec_del_folio(lruvec, folio); |
| 8133 | folio_clear_unevictable(folio); |
| 8134 | lruvec_add_folio(lruvec, folio); |
Hugh Dickins | 8d8869c | 2020-09-18 21:20:12 -0700 | [diff] [blame] | 8135 | pgrescued += nr_pages; |
Hugh Dickins | 2451326 | 2012-01-20 14:34:21 -0800 | [diff] [blame] | 8136 | } |
Matthew Wilcox (Oracle) | 77414d1 | 2022-06-04 17:39:09 -0400 | [diff] [blame] | 8137 | folio_set_lru(folio); |
Lee Schermerhorn | 89e004ea | 2008-10-18 20:26:43 -0700 | [diff] [blame] | 8138 | } |
Hugh Dickins | 2451326 | 2012-01-20 14:34:21 -0800 | [diff] [blame] | 8139 | |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 8140 | if (lruvec) { |
Hugh Dickins | 2451326 | 2012-01-20 14:34:21 -0800 | [diff] [blame] | 8141 | __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); |
| 8142 | __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); |
Alex Shi | 6168d0d | 2020-12-15 12:34:29 -0800 | [diff] [blame] | 8143 | unlock_page_lruvec_irq(lruvec); |
Alex Shi | d25b5bd | 2020-12-15 12:34:16 -0800 | [diff] [blame] | 8144 | } else if (pgscanned) { |
| 8145 | count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); |
Hugh Dickins | 2451326 | 2012-01-20 14:34:21 -0800 | [diff] [blame] | 8146 | } |
Hugh Dickins | 85046579 | 2012-01-20 14:34:19 -0800 | [diff] [blame] | 8147 | } |
Matthew Wilcox (Oracle) | 77414d1 | 2022-06-04 17:39:09 -0400 | [diff] [blame] | 8148 | EXPORT_SYMBOL_GPL(check_move_unevictable_folios); |