blob: 2490e727e2dcbc1438a430291977b36bea7887c8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * mm/rmap.c - physical to virtual reverse mappings
3 *
4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 * Released under the General Public License (GPL).
6 *
7 * Simple, low overhead reverse mapping scheme.
8 * Please try to keep this thing as modular as possible.
9 *
10 * Provides methods for unmapping each kind of mapped page:
11 * the anon methods track anonymous pages, and
12 * the file methods track pages belonging to an inode.
13 *
14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001
15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
Hugh Dickins98f32602009-05-21 20:33:58 +010017 * Contributions by Hugh Dickins 2003, 2004
Linus Torvalds1da177e2005-04-16 15:20:36 -070018 */
19
20/*
21 * Lock ordering in mm:
22 *
Jan Kara96087032021-04-12 15:50:21 +020023 * inode->i_rwsem (while writing or truncating, not reading or faulting)
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -070024 * mm->mmap_lock
Jan Kara730633f2021-01-28 19:19:45 +010025 * mapping->invalidate_lock (in filemap_fault)
Matthew Wilcox (Oracle)4dc7d372024-03-26 17:10:31 +000026 * folio_lock
Mike Kravetz8d9bfb22022-09-14 15:18:07 -070027 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below)
Suren Baghdasaryan55fd6fc2023-02-27 09:36:14 -080028 * vma_start_write
29 * mapping->i_mmap_rwsem
30 * anon_vma->rwsem
31 * mm->page_table_lock or pte_lock
32 * swap_lock (in swap_duplicate, swap_info_get)
33 * mmlist_lock (in mmput, drain_mmlist and others)
34 * mapping->private_lock (in block_dirty_folio)
35 * folio_lock_memcg move_lock (in block_dirty_folio)
36 * i_pages lock (widely used)
37 * lruvec->lru_lock (in folio_lruvec_lock_irq)
38 * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
39 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
40 * sb_lock (within inode_lock in fs/fs-writeback.c)
41 * i_pages lock (widely used, in set_page_dirty,
42 * in arch-dependent flush_dcache_mmap_lock,
43 * within bdi.wb->list_lock in __sync_single_inode)
Andi Kleen6a460792009-09-16 11:50:15 +020044 *
Jan Kara96087032021-04-12 15:50:21 +020045 * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon)
Peter Zijlstra9b679320a2011-06-27 16:18:09 -070046 * ->tasklist_lock
Andi Kleen6a460792009-09-16 11:50:15 +020047 * pte map lock
Mike Kravetzc0d03812020-04-01 21:11:05 -070048 *
Mike Kravetz8d9bfb22022-09-14 15:18:07 -070049 * hugetlbfs PageHuge() take locks in this order:
50 * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
51 * vma_lock (hugetlb specific lock for pmd_sharing)
52 * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing)
Matthew Wilcox (Oracle)4dc7d372024-03-26 17:10:31 +000053 * folio_lock
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 */
55
56#include <linux/mm.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010057#include <linux/sched/mm.h>
Ingo Molnar29930022017-02-08 18:51:36 +010058#include <linux/sched/task.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070059#include <linux/pagemap.h>
60#include <linux/swap.h>
61#include <linux/swapops.h>
62#include <linux/slab.h>
63#include <linux/init.h>
Hugh Dickins5ad64682009-12-14 17:59:24 -080064#include <linux/ksm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070065#include <linux/rmap.h>
66#include <linux/rcupdate.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040067#include <linux/export.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080068#include <linux/memcontrol.h>
Andrea Arcangelicddb8a52008-07-28 15:46:29 -070069#include <linux/mmu_notifier.h>
KOSAKI Motohiro64cdd542009-01-06 14:39:16 -080070#include <linux/migrate.h>
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +090071#include <linux/hugetlb.h>
Ben Dooks444f84f2019-10-18 20:20:17 -070072#include <linux/huge_mm.h>
Jan Karaef5d4372012-10-25 13:37:31 -070073#include <linux/backing-dev.h>
Vladimir Davydov33c3fc72015-09-09 15:35:45 -070074#include <linux/page_idle.h>
Jérôme Glissea5430dd2017-09-08 16:12:17 -070075#include <linux/memremap.h>
Christian Borntraegerbce73e42018-07-13 16:58:52 -070076#include <linux/userfaultfd_k.h>
Peter Xu999dad82022-05-12 20:22:53 -070077#include <linux/mm_inline.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
79#include <asm/tlbflush.h>
80
Anshuman Khandual4cc79b32022-03-24 18:10:01 -070081#define CREATE_TRACE_POINTS
Mel Gorman72b252a2015-09-04 15:47:32 -070082#include <trace/events/tlb.h>
Anshuman Khandual4cc79b32022-03-24 18:10:01 -070083#include <trace/events/migrate.h>
Mel Gorman72b252a2015-09-04 15:47:32 -070084
Nick Pigginb291f002008-10-18 20:26:44 -070085#include "internal.h"
86
Adrian Bunkfdd2e5f2008-10-18 20:28:38 -070087static struct kmem_cache *anon_vma_cachep;
Rik van Riel5beb4932010-03-05 13:42:07 -080088static struct kmem_cache *anon_vma_chain_cachep;
Adrian Bunkfdd2e5f2008-10-18 20:28:38 -070089
90static inline struct anon_vma *anon_vma_alloc(void)
91{
Peter Zijlstra01d8b202011-03-22 16:32:49 -070092 struct anon_vma *anon_vma;
93
94 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
95 if (anon_vma) {
96 atomic_set(&anon_vma->refcount, 1);
Jann Horn25552832022-08-31 19:06:00 +020097 anon_vma->num_children = 0;
98 anon_vma->num_active_vmas = 0;
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -080099 anon_vma->parent = anon_vma;
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700100 /*
101 * Initialise the anon_vma root to point to itself. If called
102 * from fork, the root will be reset to the parents anon_vma.
103 */
104 anon_vma->root = anon_vma;
105 }
106
107 return anon_vma;
Adrian Bunkfdd2e5f2008-10-18 20:28:38 -0700108}
109
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700110static inline void anon_vma_free(struct anon_vma *anon_vma)
Adrian Bunkfdd2e5f2008-10-18 20:28:38 -0700111{
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700112 VM_BUG_ON(atomic_read(&anon_vma->refcount));
Peter Zijlstra88c22082011-05-24 17:12:13 -0700113
114 /*
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -0500115 * Synchronize against folio_lock_anon_vma_read() such that
Peter Zijlstra88c22082011-05-24 17:12:13 -0700116 * we can safely hold the lock without the anon_vma getting
117 * freed.
118 *
119 * Relies on the full mb implied by the atomic_dec_and_test() from
120 * put_anon_vma() against the acquire barrier implied by
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -0500121 * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
Peter Zijlstra88c22082011-05-24 17:12:13 -0700122 *
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -0500123 * folio_lock_anon_vma_read() VS put_anon_vma()
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000124 * down_read_trylock() atomic_dec_and_test()
Peter Zijlstra88c22082011-05-24 17:12:13 -0700125 * LOCK MB
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000126 * atomic_read() rwsem_is_locked()
Peter Zijlstra88c22082011-05-24 17:12:13 -0700127 *
128 * LOCK should suffice since the actual taking of the lock must
129 * happen _before_ what follows.
130 */
Hugh Dickins7f39dda2014-06-04 16:05:33 -0700131 might_sleep();
Ingo Molnar5a505082012-12-02 19:56:46 +0000132 if (rwsem_is_locked(&anon_vma->root->rwsem)) {
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000133 anon_vma_lock_write(anon_vma);
Konstantin Khlebnikov08b52702013-02-22 16:34:40 -0800134 anon_vma_unlock_write(anon_vma);
Peter Zijlstra88c22082011-05-24 17:12:13 -0700135 }
136
Adrian Bunkfdd2e5f2008-10-18 20:28:38 -0700137 kmem_cache_free(anon_vma_cachep, anon_vma);
138}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
Linus Torvaldsdd347392011-06-17 19:05:36 -0700140static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
Rik van Riel5beb4932010-03-05 13:42:07 -0800141{
Linus Torvaldsdd347392011-06-17 19:05:36 -0700142 return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
Rik van Riel5beb4932010-03-05 13:42:07 -0800143}
144
Namhyung Kime574b5f2010-10-26 14:22:02 -0700145static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
Rik van Riel5beb4932010-03-05 13:42:07 -0800146{
147 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
148}
149
Kautuk Consul6583a842012-03-21 16:34:01 -0700150static void anon_vma_chain_link(struct vm_area_struct *vma,
151 struct anon_vma_chain *avc,
152 struct anon_vma *anon_vma)
153{
154 avc->vma = vma;
155 avc->anon_vma = anon_vma;
156 list_add(&avc->same_vma, &vma->anon_vma_chain);
Michel Lespinassebf181b92012-10-08 16:31:39 -0700157 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
Kautuk Consul6583a842012-03-21 16:34:01 -0700158}
159
Linus Torvaldsd9d332e2008-10-19 10:32:20 -0700160/**
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800161 * __anon_vma_prepare - attach an anon_vma to a memory region
Linus Torvaldsd9d332e2008-10-19 10:32:20 -0700162 * @vma: the memory region in question
163 *
164 * This makes sure the memory mapping described by 'vma' has
165 * an 'anon_vma' attached to it, so that we can associate the
166 * anonymous pages mapped into it with that anon_vma.
167 *
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800168 * The common case will be that we already have one, which
169 * is handled inline by anon_vma_prepare(). But if
Figo.zhang23a07902010-12-27 15:14:06 +0100170 * not we either need to find an adjacent mapping that we
Linus Torvaldsd9d332e2008-10-19 10:32:20 -0700171 * can re-use the anon_vma from (very common when the only
172 * reason for splitting a vma has been mprotect()), or we
173 * allocate a new one.
174 *
175 * Anon-vma allocations are very subtle, because we may have
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -0500176 * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
Miaohe Linaaf1f992021-02-25 17:17:53 -0800177 * and that may actually touch the rwsem even in the newly
Linus Torvaldsd9d332e2008-10-19 10:32:20 -0700178 * allocated vma (it depends on RCU to make sure that the
179 * anon_vma isn't actually destroyed).
180 *
181 * As a result, we need to do proper anon_vma locking even
182 * for the new allocation. At the same time, we do not want
183 * to do any locking for the common case of already having
184 * an anon_vma.
Linus Torvaldsd9d332e2008-10-19 10:32:20 -0700185 */
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800186int __anon_vma_prepare(struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187{
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800188 struct mm_struct *mm = vma->vm_mm;
189 struct anon_vma *anon_vma, *allocated;
Rik van Riel5beb4932010-03-05 13:42:07 -0800190 struct anon_vma_chain *avc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191
Matthew Wilcox (Oracle)3be51062024-04-26 15:45:00 +0100192 mmap_assert_locked(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 might_sleep();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800195 avc = anon_vma_chain_alloc(GFP_KERNEL);
196 if (!avc)
197 goto out_enomem;
Rik van Riel5beb4932010-03-05 13:42:07 -0800198
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800199 anon_vma = find_mergeable_anon_vma(vma);
200 allocated = NULL;
201 if (!anon_vma) {
202 anon_vma = anon_vma_alloc();
203 if (unlikely(!anon_vma))
204 goto out_enomem_free_avc;
Jann Horn25552832022-08-31 19:06:00 +0200205 anon_vma->num_children++; /* self-parent link for new root */
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800206 allocated = anon_vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 }
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800208
209 anon_vma_lock_write(anon_vma);
210 /* page_table_lock to protect against threads */
211 spin_lock(&mm->page_table_lock);
212 if (likely(!vma->anon_vma)) {
213 vma->anon_vma = anon_vma;
214 anon_vma_chain_link(vma, avc, anon_vma);
Jann Horn25552832022-08-31 19:06:00 +0200215 anon_vma->num_active_vmas++;
Vlastimil Babkad5a187d2016-12-12 16:44:38 -0800216 allocated = NULL;
217 avc = NULL;
218 }
219 spin_unlock(&mm->page_table_lock);
220 anon_vma_unlock_write(anon_vma);
221
222 if (unlikely(allocated))
223 put_anon_vma(allocated);
224 if (unlikely(avc))
225 anon_vma_chain_free(avc);
226
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 return 0;
Rik van Riel5beb4932010-03-05 13:42:07 -0800228
229 out_enomem_free_avc:
230 anon_vma_chain_free(avc);
231 out_enomem:
232 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233}
234
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700235/*
236 * This is a useful helper function for locking the anon_vma root as
237 * we traverse the vma->anon_vma_chain, looping over anon_vma's that
238 * have the same vma.
239 *
240 * Such anon_vma's should have the same root, so you'd expect to see
241 * just a single mutex_lock for the whole traversal.
242 */
243static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
244{
245 struct anon_vma *new_root = anon_vma->root;
246 if (new_root != root) {
247 if (WARN_ON_ONCE(root))
Ingo Molnar5a505082012-12-02 19:56:46 +0000248 up_write(&root->rwsem);
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700249 root = new_root;
Ingo Molnar5a505082012-12-02 19:56:46 +0000250 down_write(&root->rwsem);
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700251 }
252 return root;
253}
254
255static inline void unlock_anon_vma_root(struct anon_vma *root)
256{
257 if (root)
Ingo Molnar5a505082012-12-02 19:56:46 +0000258 up_write(&root->rwsem);
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700259}
260
Rik van Riel5beb4932010-03-05 13:42:07 -0800261/*
262 * Attach the anon_vmas from src to dst.
263 * Returns 0 on success, -ENOMEM on failure.
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800264 *
Liam R. Howlett0503ea82023-01-20 11:26:49 -0500265 * anon_vma_clone() is called by vma_expand(), vma_merge(), __split_vma(),
266 * copy_vma() and anon_vma_fork(). The first four want an exact copy of src,
267 * while the last one, anon_vma_fork(), may try to reuse an existing anon_vma to
268 * prevent endless growth of anon_vma. Since dst->anon_vma is set to NULL before
269 * call, we can identify this case by checking (!dst->anon_vma &&
270 * src->anon_vma).
Wei Yang47b390d2019-11-30 17:50:56 -0800271 *
272 * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find
273 * and reuse existing anon_vma which has no vmas and only one child anon_vma.
274 * This prevents degradation of anon_vma hierarchy to endless linear chain in
275 * case of constantly forking task. On the other hand, an anon_vma with more
276 * than one child isn't reused even if there was no alive vma, thus rmap
277 * walker has a good chance of avoiding scanning the whole hierarchy when it
278 * searches where page is mapped.
Rik van Riel5beb4932010-03-05 13:42:07 -0800279 */
280int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281{
Rik van Riel5beb4932010-03-05 13:42:07 -0800282 struct anon_vma_chain *avc, *pavc;
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700283 struct anon_vma *root = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
Linus Torvalds646d87b2010-04-11 17:15:03 -0700285 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700286 struct anon_vma *anon_vma;
287
Linus Torvaldsdd347392011-06-17 19:05:36 -0700288 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
289 if (unlikely(!avc)) {
290 unlock_anon_vma_root(root);
291 root = NULL;
292 avc = anon_vma_chain_alloc(GFP_KERNEL);
293 if (!avc)
294 goto enomem_failure;
295 }
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700296 anon_vma = pavc->anon_vma;
297 root = lock_anon_vma_root(root, anon_vma);
298 anon_vma_chain_link(dst, avc, anon_vma);
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800299
300 /*
Jann Horn25552832022-08-31 19:06:00 +0200301 * Reuse existing anon_vma if it has no vma and only one
302 * anon_vma child.
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800303 *
Jann Horn25552832022-08-31 19:06:00 +0200304 * Root anon_vma is never reused:
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800305 * it has self-parent reference and at least one child.
306 */
Wei Yang47b390d2019-11-30 17:50:56 -0800307 if (!dst->anon_vma && src->anon_vma &&
Jann Horn25552832022-08-31 19:06:00 +0200308 anon_vma->num_children < 2 &&
309 anon_vma->num_active_vmas == 0)
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800310 dst->anon_vma = anon_vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 }
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800312 if (dst->anon_vma)
Jann Horn25552832022-08-31 19:06:00 +0200313 dst->anon_vma->num_active_vmas++;
Linus Torvaldsbb4aa392011-06-16 20:44:51 -0700314 unlock_anon_vma_root(root);
Rik van Riel5beb4932010-03-05 13:42:07 -0800315 return 0;
316
317 enomem_failure:
Leon Yu3fe89b32015-03-25 15:55:11 -0700318 /*
Ma Wupengd8e454e2022-10-14 09:39:31 +0800319 * dst->anon_vma is dropped here otherwise its num_active_vmas can
320 * be incorrectly decremented in unlink_anon_vmas().
Leon Yu3fe89b32015-03-25 15:55:11 -0700321 * We can safely do this because callers of anon_vma_clone() don't care
322 * about dst->anon_vma if anon_vma_clone() failed.
323 */
324 dst->anon_vma = NULL;
Rik van Riel5beb4932010-03-05 13:42:07 -0800325 unlink_anon_vmas(dst);
326 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327}
328
Rik van Riel5beb4932010-03-05 13:42:07 -0800329/*
330 * Attach vma to its own anon_vma, as well as to the anon_vmas that
331 * the corresponding VMA in the parent process is attached to.
332 * Returns 0 on success, non-zero on failure.
333 */
334int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
Rik van Riel5beb4932010-03-05 13:42:07 -0800336 struct anon_vma_chain *avc;
337 struct anon_vma *anon_vma;
Daniel Forrestc4ea95d2014-12-02 15:59:42 -0800338 int error;
Rik van Riel5beb4932010-03-05 13:42:07 -0800339
340 /* Don't bother if the parent process has no anon_vma here. */
341 if (!pvma->anon_vma)
342 return 0;
343
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800344 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */
345 vma->anon_vma = NULL;
346
Rik van Riel5beb4932010-03-05 13:42:07 -0800347 /*
348 * First, attach the new VMA to the parent VMA's anon_vmas,
349 * so rmap can find non-COWed pages in child processes.
350 */
Daniel Forrestc4ea95d2014-12-02 15:59:42 -0800351 error = anon_vma_clone(vma, pvma);
352 if (error)
353 return error;
Rik van Riel5beb4932010-03-05 13:42:07 -0800354
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800355 /* An existing anon_vma has been reused, all done then. */
356 if (vma->anon_vma)
357 return 0;
358
Rik van Riel5beb4932010-03-05 13:42:07 -0800359 /* Then add our own anon_vma. */
360 anon_vma = anon_vma_alloc();
361 if (!anon_vma)
362 goto out_error;
Jann Horn25552832022-08-31 19:06:00 +0200363 anon_vma->num_active_vmas++;
Linus Torvaldsdd347392011-06-17 19:05:36 -0700364 avc = anon_vma_chain_alloc(GFP_KERNEL);
Rik van Riel5beb4932010-03-05 13:42:07 -0800365 if (!avc)
366 goto out_error_free_anon_vma;
Rik van Riel5c341ee12010-08-09 17:18:39 -0700367
368 /*
Miaohe Linaaf1f992021-02-25 17:17:53 -0800369 * The root anon_vma's rwsem is the lock actually used when we
Rik van Riel5c341ee12010-08-09 17:18:39 -0700370 * lock any of the anon_vmas in this anon_vma tree.
371 */
372 anon_vma->root = pvma->anon_vma->root;
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800373 anon_vma->parent = pvma->anon_vma;
Rik van Riel76545062010-08-09 17:18:41 -0700374 /*
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700375 * With refcounts, an anon_vma can stay around longer than the
376 * process it belongs to. The root anon_vma needs to be pinned until
377 * this anon_vma is freed, because the lock lives in the root.
Rik van Riel76545062010-08-09 17:18:41 -0700378 */
379 get_anon_vma(anon_vma->root);
Rik van Riel5beb4932010-03-05 13:42:07 -0800380 /* Mark this anon_vma as the one where our new (COWed) pages go. */
381 vma->anon_vma = anon_vma;
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000382 anon_vma_lock_write(anon_vma);
Rik van Riel5c341ee12010-08-09 17:18:39 -0700383 anon_vma_chain_link(vma, avc, anon_vma);
Jann Horn25552832022-08-31 19:06:00 +0200384 anon_vma->parent->num_children++;
Konstantin Khlebnikov08b52702013-02-22 16:34:40 -0800385 anon_vma_unlock_write(anon_vma);
Rik van Riel5beb4932010-03-05 13:42:07 -0800386
387 return 0;
388
389 out_error_free_anon_vma:
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700390 put_anon_vma(anon_vma);
Rik van Riel5beb4932010-03-05 13:42:07 -0800391 out_error:
Rik van Riel4946d542010-04-05 12:13:33 -0400392 unlink_anon_vmas(vma);
Rik van Riel5beb4932010-03-05 13:42:07 -0800393 return -ENOMEM;
394}
395
Rik van Riel5beb4932010-03-05 13:42:07 -0800396void unlink_anon_vmas(struct vm_area_struct *vma)
397{
398 struct anon_vma_chain *avc, *next;
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200399 struct anon_vma *root = NULL;
Rik van Riel5beb4932010-03-05 13:42:07 -0800400
Rik van Riel5c341ee12010-08-09 17:18:39 -0700401 /*
402 * Unlink each anon_vma chained to the VMA. This list is ordered
403 * from newest to oldest, ensuring the root anon_vma gets freed last.
404 */
Rik van Riel5beb4932010-03-05 13:42:07 -0800405 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200406 struct anon_vma *anon_vma = avc->anon_vma;
407
408 root = lock_anon_vma_root(root, anon_vma);
Michel Lespinassebf181b92012-10-08 16:31:39 -0700409 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200410
411 /*
412 * Leave empty anon_vmas on the list - we'll need
413 * to free them outside the lock.
414 */
Davidlohr Buesof808c132017-09-08 16:15:08 -0700415 if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
Jann Horn25552832022-08-31 19:06:00 +0200416 anon_vma->parent->num_children--;
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200417 continue;
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -0800418 }
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200419
420 list_del(&avc->same_vma);
421 anon_vma_chain_free(avc);
422 }
Li Xinhaiee8ab192021-02-24 12:04:49 -0800423 if (vma->anon_vma) {
Jann Horn25552832022-08-31 19:06:00 +0200424 vma->anon_vma->num_active_vmas--;
Li Xinhaiee8ab192021-02-24 12:04:49 -0800425
426 /*
427 * vma would still be needed after unlink, and anon_vma will be prepared
428 * when handle fault.
429 */
430 vma->anon_vma = NULL;
431 }
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200432 unlock_anon_vma_root(root);
433
434 /*
435 * Iterate the list once more, it now only contains empty and unlinked
436 * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
Ingo Molnar5a505082012-12-02 19:56:46 +0000437 * needing to write-acquire the anon_vma->root->rwsem.
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200438 */
439 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
440 struct anon_vma *anon_vma = avc->anon_vma;
441
Jann Horn25552832022-08-31 19:06:00 +0200442 VM_WARN_ON(anon_vma->num_children);
443 VM_WARN_ON(anon_vma->num_active_vmas);
Peter Zijlstraeee2acb2011-06-17 13:54:23 +0200444 put_anon_vma(anon_vma);
445
Rik van Riel5beb4932010-03-05 13:42:07 -0800446 list_del(&avc->same_vma);
447 anon_vma_chain_free(avc);
448 }
449}
450
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700451static void anon_vma_ctor(void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452{
Christoph Lametera35afb82007-05-16 22:10:57 -0700453 struct anon_vma *anon_vma = data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454
Ingo Molnar5a505082012-12-02 19:56:46 +0000455 init_rwsem(&anon_vma->rwsem);
Peter Zijlstra83813262011-03-22 16:32:48 -0700456 atomic_set(&anon_vma->refcount, 0);
Davidlohr Buesof808c132017-09-08 16:15:08 -0700457 anon_vma->rb_root = RB_ROOT_CACHED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458}
459
460void __init anon_vma_init(void)
461{
462 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -0800463 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
Vladimir Davydov5d097052016-01-14 15:18:21 -0800464 anon_vma_ctor);
465 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
466 SLAB_PANIC|SLAB_ACCOUNT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467}
468
469/*
Peter Zijlstra6111e4c2011-05-24 17:12:08 -0700470 * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
471 *
David Hildenbrand4d8f7412023-12-20 23:44:56 +0100472 * Since there is no serialization what so ever against folio_remove_rmap_*()
Miaohe Linad8a20c2021-02-25 17:18:06 -0800473 * the best this function can do is return a refcount increased anon_vma
474 * that might have been relevant to this page.
Peter Zijlstra6111e4c2011-05-24 17:12:08 -0700475 *
476 * The page might have been remapped to a different anon_vma or the anon_vma
477 * returned may already be freed (and even reused).
478 *
Peter Zijlstrabc658c92011-05-29 10:33:44 +0200479 * In case it was remapped to a different anon_vma, the new anon_vma will be a
480 * child of the old anon_vma, and the anon_vma lifetime rules will therefore
481 * ensure that any anon_vma obtained from the page will still be valid for as
482 * long as we observe page_mapped() [ hence all those page_mapped() tests ].
483 *
Peter Zijlstra6111e4c2011-05-24 17:12:08 -0700484 * All users of this function must be very careful when walking the anon_vma
485 * chain and verify that the page in question is indeed mapped in it
486 * [ something equivalent to page_mapped_in_vma() ].
487 *
Miles Chen091e4292019-11-30 17:51:23 -0800488 * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
David Hildenbrand4d8f7412023-12-20 23:44:56 +0100489 * folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid
Miles Chen091e4292019-11-30 17:51:23 -0800490 * if there is a mapcount, we can dereference the anon_vma after observing
491 * those.
Andrea Arcangeliadef4402023-12-06 02:36:56 -0800492 *
493 * NOTE: the caller should normally hold folio lock when calling this. If
494 * not, the caller needs to double check the anon_vma didn't change after
495 * taking the anon_vma lock for either read or write (UFFDIO_MOVE can modify it
496 * concurrently without folio lock protection). See folio_lock_anon_vma_read()
497 * which has already covered that, and comment above remap_pages().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 */
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +0100499struct anon_vma *folio_get_anon_vma(struct folio *folio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500{
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700501 struct anon_vma *anon_vma = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 unsigned long anon_mapping;
503
504 rcu_read_lock();
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +0100505 anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
Hugh Dickins3ca7b3c2009-12-14 17:58:57 -0800506 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 goto out;
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +0100508 if (!folio_mapped(folio))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 goto out;
510
511 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700512 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
513 anon_vma = NULL;
514 goto out;
515 }
Hugh Dickinsf18194272010-08-25 23:12:54 -0700516
517 /*
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +0100518 * If this folio is still mapped, then its anon_vma cannot have been
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700519 * freed. But if it has been unmapped, we have no security against the
520 * anon_vma structure being freed and reused (for another anon_vma:
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -0800521 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700522 * above cannot corrupt).
Hugh Dickinsf18194272010-08-25 23:12:54 -0700523 */
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +0100524 if (!folio_mapped(folio)) {
Hugh Dickins7f39dda2014-06-04 16:05:33 -0700525 rcu_read_unlock();
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700526 put_anon_vma(anon_vma);
Hugh Dickins7f39dda2014-06-04 16:05:33 -0700527 return NULL;
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700528 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529out:
530 rcu_read_unlock();
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700531
532 return anon_vma;
533}
534
Peter Zijlstra88c22082011-05-24 17:12:13 -0700535/*
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +0100536 * Similar to folio_get_anon_vma() except it locks the anon_vma.
Peter Zijlstra88c22082011-05-24 17:12:13 -0700537 *
538 * Its a little more complex as it tries to keep the fast path to a single
539 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +0100540 * reference like with folio_get_anon_vma() and then block on the mutex
Minchan Kim6d4675e2022-05-19 14:08:54 -0700541 * on !rwc->try_lock case.
Peter Zijlstra88c22082011-05-24 17:12:13 -0700542 */
Minchan Kim6d4675e2022-05-19 14:08:54 -0700543struct anon_vma *folio_lock_anon_vma_read(struct folio *folio,
544 struct rmap_walk_control *rwc)
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700545{
Peter Zijlstra88c22082011-05-24 17:12:13 -0700546 struct anon_vma *anon_vma = NULL;
Hugh Dickinseee0f252011-05-28 13:20:21 -0700547 struct anon_vma *root_anon_vma;
Peter Zijlstra88c22082011-05-24 17:12:13 -0700548 unsigned long anon_mapping;
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700549
Andrea Arcangeli880a99b2023-12-06 02:36:55 -0800550retry:
Peter Zijlstra88c22082011-05-24 17:12:13 -0700551 rcu_read_lock();
Matthew Wilcox (Oracle)9595d762022-02-01 23:33:08 -0500552 anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
Peter Zijlstra88c22082011-05-24 17:12:13 -0700553 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
554 goto out;
Matthew Wilcox (Oracle)9595d762022-02-01 23:33:08 -0500555 if (!folio_mapped(folio))
Peter Zijlstra88c22082011-05-24 17:12:13 -0700556 goto out;
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700557
Peter Zijlstra88c22082011-05-24 17:12:13 -0700558 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
Jason Low4db0c3c2015-04-15 16:14:08 -0700559 root_anon_vma = READ_ONCE(anon_vma->root);
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000560 if (down_read_trylock(&root_anon_vma->rwsem)) {
Peter Zijlstra88c22082011-05-24 17:12:13 -0700561 /*
Andrea Arcangeli880a99b2023-12-06 02:36:55 -0800562 * folio_move_anon_rmap() might have changed the anon_vma as we
563 * might not hold the folio lock here.
564 */
565 if (unlikely((unsigned long)READ_ONCE(folio->mapping) !=
566 anon_mapping)) {
567 up_read(&root_anon_vma->rwsem);
568 rcu_read_unlock();
569 goto retry;
570 }
571
572 /*
Matthew Wilcox (Oracle)9595d762022-02-01 23:33:08 -0500573 * If the folio is still mapped, then this anon_vma is still
Hugh Dickinseee0f252011-05-28 13:20:21 -0700574 * its anon_vma, and holding the mutex ensures that it will
Peter Zijlstrabc658c92011-05-29 10:33:44 +0200575 * not go away, see anon_vma_free().
Peter Zijlstra88c22082011-05-24 17:12:13 -0700576 */
Matthew Wilcox (Oracle)9595d762022-02-01 23:33:08 -0500577 if (!folio_mapped(folio)) {
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000578 up_read(&root_anon_vma->rwsem);
Peter Zijlstra88c22082011-05-24 17:12:13 -0700579 anon_vma = NULL;
580 }
581 goto out;
582 }
583
Minchan Kim6d4675e2022-05-19 14:08:54 -0700584 if (rwc && rwc->try_lock) {
585 anon_vma = NULL;
586 rwc->contended = true;
587 goto out;
588 }
589
Peter Zijlstra88c22082011-05-24 17:12:13 -0700590 /* trylock failed, we got to sleep */
591 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
592 anon_vma = NULL;
593 goto out;
594 }
595
Matthew Wilcox (Oracle)9595d762022-02-01 23:33:08 -0500596 if (!folio_mapped(folio)) {
Hugh Dickins7f39dda2014-06-04 16:05:33 -0700597 rcu_read_unlock();
Peter Zijlstra88c22082011-05-24 17:12:13 -0700598 put_anon_vma(anon_vma);
Hugh Dickins7f39dda2014-06-04 16:05:33 -0700599 return NULL;
Peter Zijlstra88c22082011-05-24 17:12:13 -0700600 }
601
602 /* we pinned the anon_vma, its safe to sleep */
603 rcu_read_unlock();
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000604 anon_vma_lock_read(anon_vma);
Peter Zijlstra88c22082011-05-24 17:12:13 -0700605
Andrea Arcangeli880a99b2023-12-06 02:36:55 -0800606 /*
607 * folio_move_anon_rmap() might have changed the anon_vma as we might
608 * not hold the folio lock here.
609 */
610 if (unlikely((unsigned long)READ_ONCE(folio->mapping) !=
611 anon_mapping)) {
612 anon_vma_unlock_read(anon_vma);
613 put_anon_vma(anon_vma);
614 anon_vma = NULL;
615 goto retry;
616 }
617
Peter Zijlstra88c22082011-05-24 17:12:13 -0700618 if (atomic_dec_and_test(&anon_vma->refcount)) {
619 /*
620 * Oops, we held the last refcount, release the lock
621 * and bail -- can't simply use put_anon_vma() because
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000622 * we'll deadlock on the anon_vma_lock_write() recursion.
Peter Zijlstra88c22082011-05-24 17:12:13 -0700623 */
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000624 anon_vma_unlock_read(anon_vma);
Peter Zijlstra88c22082011-05-24 17:12:13 -0700625 __put_anon_vma(anon_vma);
626 anon_vma = NULL;
627 }
628
629 return anon_vma;
630
631out:
632 rcu_read_unlock();
Peter Zijlstra746b18d2011-05-24 17:12:10 -0700633 return anon_vma;
Oleg Nesterov34bbd702007-02-28 20:13:49 -0800634}
635
Mel Gorman72b252a2015-09-04 15:47:32 -0700636#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
Mel Gorman72b252a2015-09-04 15:47:32 -0700637/*
638 * Flush TLB entries for recently unmapped pages from remote CPUs. It is
639 * important if a PTE was dirty when it was unmapped that it's flushed
640 * before any IO is initiated on the page to prevent lost writes. Similarly,
641 * it must be flushed before freeing to prevent data leakage.
642 */
643void try_to_unmap_flush(void)
644{
645 struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
Mel Gorman72b252a2015-09-04 15:47:32 -0700646
647 if (!tlb_ubc->flush_required)
648 return;
649
Andy Lutomirskie73ad5f2017-05-22 15:30:03 -0700650 arch_tlbbatch_flush(&tlb_ubc->arch);
Mel Gorman72b252a2015-09-04 15:47:32 -0700651 tlb_ubc->flush_required = false;
Mel Gormand950c942015-09-04 15:47:35 -0700652 tlb_ubc->writable = false;
Mel Gorman72b252a2015-09-04 15:47:32 -0700653}
654
Mel Gormand950c942015-09-04 15:47:35 -0700655/* Flush iff there are potentially writable TLB entries that can race with IO */
656void try_to_unmap_flush_dirty(void)
657{
658 struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
659
660 if (tlb_ubc->writable)
661 try_to_unmap_flush();
662}
663
Huang Ying5ee2fa22022-01-14 14:09:16 -0800664/*
665 * Bits 0-14 of mm->tlb_flush_batched record pending generations.
666 * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations.
667 */
668#define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16
669#define TLB_FLUSH_BATCH_PENDING_MASK \
670 ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1)
671#define TLB_FLUSH_BATCH_PENDING_LARGE \
672 (TLB_FLUSH_BATCH_PENDING_MASK / 2)
673
Barry Songf73419b2023-07-17 21:10:02 +0800674static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
675 unsigned long uaddr)
Mel Gorman72b252a2015-09-04 15:47:32 -0700676{
677 struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
Uros Bizjakbdeb9182023-02-27 22:42:28 +0100678 int batch;
Huang Ying4d4b6d62023-04-24 14:54:08 +0800679 bool writable = pte_dirty(pteval);
680
681 if (!pte_accessible(mm, pteval))
682 return;
Mel Gorman72b252a2015-09-04 15:47:32 -0700683
Barry Songf73419b2023-07-17 21:10:02 +0800684 arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, uaddr);
Mel Gorman72b252a2015-09-04 15:47:32 -0700685 tlb_ubc->flush_required = true;
Mel Gormand950c942015-09-04 15:47:35 -0700686
687 /*
Mel Gorman3ea27712017-08-02 13:31:52 -0700688 * Ensure compiler does not re-order the setting of tlb_flush_batched
689 * before the PTE is cleared.
690 */
691 barrier();
Huang Ying5ee2fa22022-01-14 14:09:16 -0800692 batch = atomic_read(&mm->tlb_flush_batched);
693retry:
694 if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) {
695 /*
696 * Prevent `pending' from catching up with `flushed' because of
697 * overflow. Reset `pending' and `flushed' to be 1 and 0 if
698 * `pending' becomes large.
699 */
Uros Bizjakbdeb9182023-02-27 22:42:28 +0100700 if (!atomic_try_cmpxchg(&mm->tlb_flush_batched, &batch, 1))
Huang Ying5ee2fa22022-01-14 14:09:16 -0800701 goto retry;
Huang Ying5ee2fa22022-01-14 14:09:16 -0800702 } else {
703 atomic_inc(&mm->tlb_flush_batched);
704 }
Mel Gorman3ea27712017-08-02 13:31:52 -0700705
706 /*
Mel Gormand950c942015-09-04 15:47:35 -0700707 * If the PTE was dirty then it's best to assume it's writable. The
708 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
709 * before the page is queued for IO.
710 */
711 if (writable)
712 tlb_ubc->writable = true;
Mel Gorman72b252a2015-09-04 15:47:32 -0700713}
714
715/*
716 * Returns true if the TLB flush should be deferred to the end of a batch of
717 * unmap operations to reduce IPIs.
718 */
719static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
720{
Mel Gorman72b252a2015-09-04 15:47:32 -0700721 if (!(flags & TTU_BATCH_FLUSH))
722 return false;
723
Anshuman Khandual65c8d302023-07-17 21:10:01 +0800724 return arch_tlbbatch_should_defer(mm);
Mel Gorman72b252a2015-09-04 15:47:32 -0700725}
Mel Gorman3ea27712017-08-02 13:31:52 -0700726
727/*
728 * Reclaim unmaps pages under the PTL but do not flush the TLB prior to
729 * releasing the PTL if TLB flushes are batched. It's possible for a parallel
730 * operation such as mprotect or munmap to race between reclaim unmapping
731 * the page and flushing the page. If this race occurs, it potentially allows
732 * access to data via a stale TLB entry. Tracking all mm's that have TLB
733 * batching in flight would be expensive during reclaim so instead track
734 * whether TLB batching occurred in the past and if so then do a flush here
735 * if required. This will cost one additional flush per reclaim cycle paid
736 * by the first operation at risk such as mprotect and mumap.
737 *
738 * This must be called under the PTL so that an access to tlb_flush_batched
739 * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
740 * via the PTL.
741 */
742void flush_tlb_batched_pending(struct mm_struct *mm)
743{
Huang Ying5ee2fa22022-01-14 14:09:16 -0800744 int batch = atomic_read(&mm->tlb_flush_batched);
745 int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK;
746 int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT;
Mel Gorman3ea27712017-08-02 13:31:52 -0700747
Huang Ying5ee2fa22022-01-14 14:09:16 -0800748 if (pending != flushed) {
Yicong Yangdb6c1f62023-07-17 21:10:03 +0800749 arch_flush_tlb_batched_pending(mm);
Mel Gorman3ea27712017-08-02 13:31:52 -0700750 /*
Huang Ying5ee2fa22022-01-14 14:09:16 -0800751 * If the new TLB flushing is pending during flushing, leave
752 * mm->tlb_flush_batched as is, to avoid losing flushing.
Mel Gorman3ea27712017-08-02 13:31:52 -0700753 */
Huang Ying5ee2fa22022-01-14 14:09:16 -0800754 atomic_cmpxchg(&mm->tlb_flush_batched, batch,
755 pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT));
Mel Gorman3ea27712017-08-02 13:31:52 -0700756 }
757}
Mel Gorman72b252a2015-09-04 15:47:32 -0700758#else
Barry Songf73419b2023-07-17 21:10:02 +0800759static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
760 unsigned long uaddr)
Mel Gorman72b252a2015-09-04 15:47:32 -0700761{
762}
763
764static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
765{
766 return false;
767}
768#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
769
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770/*
Huang Shijiebf89c8c2009-10-01 15:44:04 -0700771 * At what user virtual address is page expected in vma?
Naoya Horiguchiab941e02010-05-11 14:06:55 -0700772 * Caller should check the page is actually part of the vma.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 */
774unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
775{
Matthew Wilcox (Oracle)e05b3452022-01-29 11:52:52 -0500776 struct folio *folio = page_folio(page);
Matthew Wilcox (Oracle)412ad5f2024-03-28 22:58:28 +0000777 pgoff_t pgoff;
778
Matthew Wilcox (Oracle)e05b3452022-01-29 11:52:52 -0500779 if (folio_test_anon(folio)) {
780 struct anon_vma *page__anon_vma = folio_anon_vma(folio);
Hugh Dickins4829b902010-10-02 17:46:06 -0700781 /*
782 * Note: swapoff's unuse_vma() is more efficient with this
783 * check, and needs it to match anon_vma when KSM is active.
784 */
785 if (!vma->anon_vma || !page__anon_vma ||
786 vma->anon_vma->root != page__anon_vma->root)
Andrea Arcangeli21d0d442010-08-09 17:19:10 -0700787 return -EFAULT;
Jue Wang31657172021-06-15 18:24:00 -0700788 } else if (!vma->vm_file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 return -EFAULT;
Matthew Wilcox (Oracle)e05b3452022-01-29 11:52:52 -0500790 } else if (vma->vm_file->f_mapping != folio->mapping) {
Jue Wang31657172021-06-15 18:24:00 -0700791 return -EFAULT;
792 }
Hugh Dickins494334e2021-06-15 18:23:56 -0700793
Matthew Wilcox (Oracle)412ad5f2024-03-28 22:58:28 +0000794 /* The !page__anon_vma above handles KSM folios */
795 pgoff = folio->index + folio_page_idx(folio, page);
Matthew Wilcox (Oracle)e0abfbb2024-03-28 22:58:29 +0000796 return vma_address(vma, pgoff, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797}
798
Zach O'Keefe50722802022-07-06 16:59:26 -0700799/*
800 * Returns the actual pmd_t* where we expect 'address' to be mapped from, or
801 * NULL if it doesn't exist. No guarantees / checks on what the pmd_t*
802 * represents.
803 */
Bob Liu62190492012-12-11 16:00:37 -0800804pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
805{
806 pgd_t *pgd;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300807 p4d_t *p4d;
Bob Liu62190492012-12-11 16:00:37 -0800808 pud_t *pud;
809 pmd_t *pmd = NULL;
810
811 pgd = pgd_offset(mm, address);
812 if (!pgd_present(*pgd))
813 goto out;
814
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300815 p4d = p4d_offset(pgd, address);
816 if (!p4d_present(*p4d))
817 goto out;
818
819 pud = pud_offset(p4d, address);
Bob Liu62190492012-12-11 16:00:37 -0800820 if (!pud_present(*pud))
821 goto out;
822
823 pmd = pmd_offset(pud, address);
Bob Liu62190492012-12-11 16:00:37 -0800824out:
825 return pmd;
826}
827
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500828struct folio_referenced_arg {
Joonsoo Kim9f326242014-01-21 15:49:53 -0800829 int mapcount;
830 int referenced;
831 unsigned long vm_flags;
832 struct mem_cgroup *memcg;
833};
Yin Fengwei1acbc3f2023-09-18 15:33:17 +0800834
Nikita Danilov81b40822005-05-01 08:58:36 -0700835/*
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500836 * arg: folio_referenced_arg will be passed
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 */
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -0500838static bool folio_referenced_one(struct folio *folio,
839 struct vm_area_struct *vma, unsigned long address, void *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840{
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500841 struct folio_referenced_arg *pra = arg;
842 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
Vladimir Davydov8749cfe2016-01-15 16:54:45 -0800843 int referenced = 0;
Yin Fengwei1acbc3f2023-09-18 15:33:17 +0800844 unsigned long start = address, ptes = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845
Kirill A. Shutemov8eaeded2017-02-24 14:57:48 -0800846 while (page_vma_mapped_walk(&pvmw)) {
847 address = pvmw.address;
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -0800848
Yin Fengwei1acbc3f2023-09-18 15:33:17 +0800849 if (vma->vm_flags & VM_LOCKED) {
850 if (!folio_test_large(folio) || !pvmw.pte) {
851 /* Restore the mlock which got missed */
852 mlock_vma_folio(folio, vma);
853 page_vma_mapped_walk_done(&pvmw);
854 pra->vm_flags |= VM_LOCKED;
855 return false; /* To break the loop */
856 }
857 /*
858 * For large folio fully mapped to VMA, will
859 * be handled after the pvmw loop.
860 *
861 * For large folio cross VMA boundaries, it's
862 * expected to be picked by page reclaim. But
863 * should skip reference of pages which are in
864 * the range of VM_LOCKED vma. As page reclaim
865 * should just count the reference of pages out
866 * the range of VM_LOCKED vma.
867 */
868 ptes++;
869 pra->mapcount--;
870 continue;
Vladimir Davydov8749cfe2016-01-15 16:54:45 -0800871 }
Kirill A. Shutemov8eaeded2017-02-24 14:57:48 -0800872
873 if (pvmw.pte) {
Ryan Robertsc33c7942023-06-12 16:15:45 +0100874 if (lru_gen_enabled() &&
875 pte_young(ptep_get(pvmw.pte))) {
Yu Zhao018ee472022-09-18 02:00:04 -0600876 lru_gen_look_around(&pvmw);
877 referenced++;
878 }
879
Kirill A. Shutemov8eaeded2017-02-24 14:57:48 -0800880 if (ptep_clear_flush_young_notify(vma, address,
Yu Zhao8788f672022-12-30 14:52:51 -0700881 pvmw.pte))
882 referenced++;
Kirill A. Shutemov8eaeded2017-02-24 14:57:48 -0800883 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
884 if (pmdp_clear_flush_young_notify(vma, address,
885 pvmw.pmd))
886 referenced++;
887 } else {
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500888 /* unexpected pmd-mapped folio? */
Kirill A. Shutemov8eaeded2017-02-24 14:57:48 -0800889 WARN_ON_ONCE(1);
890 }
891
892 pra->mapcount--;
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -0800893 }
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -0800894
Yin Fengwei1acbc3f2023-09-18 15:33:17 +0800895 if ((vma->vm_flags & VM_LOCKED) &&
896 folio_test_large(folio) &&
897 folio_within_vma(folio, vma)) {
898 unsigned long s_align, e_align;
899
900 s_align = ALIGN_DOWN(start, PMD_SIZE);
901 e_align = ALIGN_DOWN(start + folio_size(folio) - 1, PMD_SIZE);
902
903 /* folio doesn't cross page table boundary and fully mapped */
904 if ((s_align == e_align) && (ptes == folio_nr_pages(folio))) {
905 /* Restore the mlock which got missed */
906 mlock_vma_folio(folio, vma);
907 pra->vm_flags |= VM_LOCKED;
908 return false; /* To break the loop */
909 }
910 }
911
Vladimir Davydov33c3fc72015-09-09 15:35:45 -0700912 if (referenced)
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500913 folio_clear_idle(folio);
914 if (folio_test_clear_young(folio))
Vladimir Davydov33c3fc72015-09-09 15:35:45 -0700915 referenced++;
916
Joonsoo Kim9f326242014-01-21 15:49:53 -0800917 if (referenced) {
918 pra->referenced++;
Hugh Dickins47d4f3e2022-02-14 18:42:33 -0800919 pra->vm_flags |= vma->vm_flags & ~VM_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 }
Oleg Nesterov34bbd702007-02-28 20:13:49 -0800921
Joonsoo Kim9f326242014-01-21 15:49:53 -0800922 if (!pra->mapcount)
Minchan Kime4b82222017-05-03 14:54:27 -0700923 return false; /* To break the loop */
Joonsoo Kim9f326242014-01-21 15:49:53 -0800924
Minchan Kime4b82222017-05-03 14:54:27 -0700925 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926}
927
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500928static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929{
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500930 struct folio_referenced_arg *pra = arg;
Joonsoo Kim9f326242014-01-21 15:49:53 -0800931 struct mem_cgroup *memcg = pra->memcg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932
Yu Zhao8788f672022-12-30 14:52:51 -0700933 /*
934 * Ignore references from this mapping if it has no recency. If the
935 * folio has been used in another mapping, we will catch it; if this
936 * other mapping is already gone, the unmap path will have set the
937 * referenced flag or activated the folio in zap_pte_range().
938 */
939 if (!vma_has_recency(vma))
940 return true;
941
942 /*
943 * If we are reclaiming on behalf of a cgroup, skip counting on behalf
944 * of references from different cgroups.
945 */
946 if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
Joonsoo Kim9f326242014-01-21 15:49:53 -0800947 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948
Joonsoo Kim9f326242014-01-21 15:49:53 -0800949 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950}
951
952/**
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500953 * folio_referenced() - Test if the folio was referenced.
954 * @folio: The folio to test.
955 * @is_locked: Caller holds lock on the folio.
Johannes Weiner72835c82012-01-12 17:18:32 -0800956 * @memcg: target memory cgroup
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500957 * @vm_flags: A combination of all the vma->vm_flags which referenced the folio.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 *
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500959 * Quick test_and_clear_referenced for all mappings of a folio,
960 *
Minchan Kim6d4675e2022-05-19 14:08:54 -0700961 * Return: The number of mappings which referenced the folio. Return -1 if
962 * the function bailed out due to rmap lock contention.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 */
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500964int folio_referenced(struct folio *folio, int is_locked,
965 struct mem_cgroup *memcg, unsigned long *vm_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966{
Hao Ge637a9002024-04-28 09:20:48 +0800967 bool we_locked = false;
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500968 struct folio_referenced_arg pra = {
969 .mapcount = folio_mapcount(folio),
Joonsoo Kim9f326242014-01-21 15:49:53 -0800970 .memcg = memcg,
971 };
972 struct rmap_walk_control rwc = {
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500973 .rmap_one = folio_referenced_one,
Joonsoo Kim9f326242014-01-21 15:49:53 -0800974 .arg = (void *)&pra,
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -0500975 .anon_lock = folio_lock_anon_vma_read,
Minchan Kim6d4675e2022-05-19 14:08:54 -0700976 .try_lock = true,
Yu Zhao8788f672022-12-30 14:52:51 -0700977 .invalid_vma = invalid_folio_referenced_vma,
Joonsoo Kim9f326242014-01-21 15:49:53 -0800978 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979
Wu Fengguang6fe6b7e2009-06-16 15:33:05 -0700980 *vm_flags = 0;
Huang Shijie059d8442019-05-13 17:21:07 -0700981 if (!pra.mapcount)
Joonsoo Kim9f326242014-01-21 15:49:53 -0800982 return 0;
983
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500984 if (!folio_raw_mapping(folio))
Joonsoo Kim9f326242014-01-21 15:49:53 -0800985 return 0;
986
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500987 if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) {
988 we_locked = folio_trylock(folio);
Joonsoo Kim9f326242014-01-21 15:49:53 -0800989 if (!we_locked)
990 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 }
Joonsoo Kim9f326242014-01-21 15:49:53 -0800992
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -0500993 rmap_walk(folio, &rwc);
Joonsoo Kim9f326242014-01-21 15:49:53 -0800994 *vm_flags = pra.vm_flags;
995
996 if (we_locked)
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -0500997 folio_unlock(folio);
Joonsoo Kim9f326242014-01-21 15:49:53 -0800998
Minchan Kim6d4675e2022-05-19 14:08:54 -0700999 return rwc.contended ? -1 : pra.referenced;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000}
1001
Muchun Song6a8e0592022-04-28 23:16:10 -07001002static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw)
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001003{
Muchun Song6a8e0592022-04-28 23:16:10 -07001004 int cleaned = 0;
1005 struct vm_area_struct *vma = pvmw->vma;
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08001006 struct mmu_notifier_range range;
Muchun Song6a8e0592022-04-28 23:16:10 -07001007 unsigned long address = pvmw->address;
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001008
Jérôme Glisse369ea822017-08-31 17:17:27 -04001009 /*
1010 * We have to assume the worse case ie pmd for invalidation. Note that
Matthew Wilcox (Oracle)e83c09a2022-01-20 18:20:07 -05001011 * the folio can not be freed from this function.
Jérôme Glisse369ea822017-08-31 17:17:27 -04001012 */
Alistair Popple7d4a8be2023-01-10 13:57:22 +11001013 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0,
1014 vma->vm_mm, address, vma_address_end(pvmw));
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08001015 mmu_notifier_invalidate_range_start(&range);
Jérôme Glisse369ea822017-08-31 17:17:27 -04001016
Muchun Song6a8e0592022-04-28 23:16:10 -07001017 while (page_vma_mapped_walk(pvmw)) {
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001018 int ret = 0;
Jérôme Glisse369ea822017-08-31 17:17:27 -04001019
Muchun Song6a8e0592022-04-28 23:16:10 -07001020 address = pvmw->address;
1021 if (pvmw->pte) {
Muchun Song6a8e0592022-04-28 23:16:10 -07001022 pte_t *pte = pvmw->pte;
Ryan Robertsc33c7942023-06-12 16:15:45 +01001023 pte_t entry = ptep_get(pte);
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001024
Ryan Robertsc33c7942023-06-12 16:15:45 +01001025 if (!pte_dirty(entry) && !pte_write(entry))
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001026 continue;
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001027
Ryan Robertsc33c7942023-06-12 16:15:45 +01001028 flush_cache_page(vma, address, pte_pfn(entry));
Linus Torvalds785373b2017-08-29 09:11:06 -07001029 entry = ptep_clear_flush(vma, address, pte);
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001030 entry = pte_wrprotect(entry);
1031 entry = pte_mkclean(entry);
Linus Torvalds785373b2017-08-29 09:11:06 -07001032 set_pte_at(vma->vm_mm, address, pte, entry);
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001033 ret = 1;
1034 } else {
Matthew Wilcox (Oracle)396bcc52020-04-06 20:04:35 -07001035#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Muchun Song6a8e0592022-04-28 23:16:10 -07001036 pmd_t *pmd = pvmw->pmd;
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001037 pmd_t entry;
1038
1039 if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
1040 continue;
1041
Muchun Song7f9c9b62022-04-28 23:16:09 -07001042 flush_cache_range(vma, address,
1043 address + HPAGE_PMD_SIZE);
Aneesh Kumar K.V024eee02019-05-13 17:19:11 -07001044 entry = pmdp_invalidate(vma, address, pmd);
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001045 entry = pmd_wrprotect(entry);
1046 entry = pmd_mkclean(entry);
Linus Torvalds785373b2017-08-29 09:11:06 -07001047 set_pmd_at(vma->vm_mm, address, pmd, entry);
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001048 ret = 1;
1049#else
Matthew Wilcox (Oracle)e83c09a2022-01-20 18:20:07 -05001050 /* unexpected pmd-mapped folio? */
Kirill A. Shutemovf27176c2017-02-24 14:57:57 -08001051 WARN_ON_ONCE(1);
1052#endif
1053 }
1054
Jérôme Glisse0f108512017-11-15 17:34:07 -08001055 if (ret)
Muchun Song6a8e0592022-04-28 23:16:10 -07001056 cleaned++;
Peter Zijlstrac2fda5f2006-12-22 14:25:52 +01001057 }
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001058
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08001059 mmu_notifier_invalidate_range_end(&range);
Jérôme Glisse369ea822017-08-31 17:17:27 -04001060
Muchun Song6a8e0592022-04-28 23:16:10 -07001061 return cleaned;
1062}
1063
1064static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,
1065 unsigned long address, void *arg)
1066{
1067 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
1068 int *cleaned = arg;
1069
1070 *cleaned += page_vma_mkclean_one(&pvmw);
1071
Minchan Kime4b82222017-05-03 14:54:27 -07001072 return true;
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001073}
1074
Joonsoo Kim9853a402014-01-21 15:49:55 -08001075static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001076{
Joonsoo Kim9853a402014-01-21 15:49:55 -08001077 if (vma->vm_flags & VM_SHARED)
Fengguang Wu871beb82014-01-23 15:53:41 -08001078 return false;
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001079
Fengguang Wu871beb82014-01-23 15:53:41 -08001080 return true;
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001081}
1082
Matthew Wilcox (Oracle)d9c08e22021-02-28 16:02:57 -05001083int folio_mkclean(struct folio *folio)
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001084{
Joonsoo Kim9853a402014-01-21 15:49:55 -08001085 int cleaned = 0;
1086 struct address_space *mapping;
1087 struct rmap_walk_control rwc = {
1088 .arg = (void *)&cleaned,
1089 .rmap_one = page_mkclean_one,
1090 .invalid_vma = invalid_mkclean_vma,
1091 };
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001092
Matthew Wilcox (Oracle)d9c08e22021-02-28 16:02:57 -05001093 BUG_ON(!folio_test_locked(folio));
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001094
Matthew Wilcox (Oracle)d9c08e22021-02-28 16:02:57 -05001095 if (!folio_mapped(folio))
Joonsoo Kim9853a402014-01-21 15:49:55 -08001096 return 0;
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001097
Matthew Wilcox (Oracle)d9c08e22021-02-28 16:02:57 -05001098 mapping = folio_mapping(folio);
Joonsoo Kim9853a402014-01-21 15:49:55 -08001099 if (!mapping)
1100 return 0;
1101
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05001102 rmap_walk(folio, &rwc);
Joonsoo Kim9853a402014-01-21 15:49:55 -08001103
1104 return cleaned;
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001105}
Matthew Wilcox (Oracle)d9c08e22021-02-28 16:02:57 -05001106EXPORT_SYMBOL_GPL(folio_mkclean);
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001107
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108/**
Muchun Song6a8e0592022-04-28 23:16:10 -07001109 * pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of
1110 * [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff)
1111 * within the @vma of shared mappings. And since clean PTEs
1112 * should also be readonly, write protects them too.
1113 * @pfn: start pfn.
1114 * @nr_pages: number of physically contiguous pages srarting with @pfn.
1115 * @pgoff: page offset that the @pfn mapped with.
1116 * @vma: vma that @pfn mapped within.
1117 *
1118 * Returns the number of cleaned PTEs (including PMDs).
1119 */
1120int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
1121 struct vm_area_struct *vma)
1122{
1123 struct page_vma_mapped_walk pvmw = {
1124 .pfn = pfn,
1125 .nr_pages = nr_pages,
1126 .pgoff = pgoff,
1127 .vma = vma,
1128 .flags = PVMW_SYNC,
1129 };
1130
1131 if (invalid_mkclean_vma(vma, NULL))
1132 return 0;
1133
Matthew Wilcox (Oracle)e0abfbb2024-03-28 22:58:29 +00001134 pvmw.address = vma_address(vma, pgoff, nr_pages);
Muchun Song6a8e0592022-04-28 23:16:10 -07001135 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma);
1136
1137 return page_vma_mkclean_one(&pvmw);
1138}
1139
David Hildenbrand96fd7492023-12-20 23:44:37 +01001140static __always_inline unsigned int __folio_add_rmap(struct folio *folio,
1141 struct page *page, int nr_pages, enum rmap_level level,
1142 int *nr_pmdmapped)
1143{
1144 atomic_t *mapped = &folio->_nr_pages_mapped;
David Hildenbrand05c53232024-04-09 21:22:47 +02001145 const int orig_nr_pages = nr_pages;
David Hildenbrand96fd7492023-12-20 23:44:37 +01001146 int first, nr = 0;
1147
1148 __folio_rmap_sanity_checks(folio, page, nr_pages, level);
1149
1150 switch (level) {
1151 case RMAP_LEVEL_PTE:
David Hildenbrand46d62de2024-04-09 21:22:46 +02001152 if (!folio_test_large(folio)) {
1153 nr = atomic_inc_and_test(&page->_mapcount);
1154 break;
1155 }
1156
David Hildenbrand96fd7492023-12-20 23:44:37 +01001157 do {
1158 first = atomic_inc_and_test(&page->_mapcount);
David Hildenbrand46d62de2024-04-09 21:22:46 +02001159 if (first) {
David Hildenbrand96fd7492023-12-20 23:44:37 +01001160 first = atomic_inc_return_relaxed(mapped);
David Hildenbrand46d62de2024-04-09 21:22:46 +02001161 if (first < ENTIRELY_MAPPED)
1162 nr++;
David Hildenbrand96fd7492023-12-20 23:44:37 +01001163 }
David Hildenbrand96fd7492023-12-20 23:44:37 +01001164 } while (page++, --nr_pages > 0);
David Hildenbrand05c53232024-04-09 21:22:47 +02001165 atomic_add(orig_nr_pages, &folio->_large_mapcount);
David Hildenbrand96fd7492023-12-20 23:44:37 +01001166 break;
1167 case RMAP_LEVEL_PMD:
1168 first = atomic_inc_and_test(&folio->_entire_mapcount);
1169 if (first) {
David Hildenbrande78a13f2023-12-20 23:45:03 +01001170 nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped);
1171 if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) {
David Hildenbrand96fd7492023-12-20 23:44:37 +01001172 *nr_pmdmapped = folio_nr_pages(folio);
1173 nr = *nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
1174 /* Raced ahead of a remove and another add? */
1175 if (unlikely(nr < 0))
1176 nr = 0;
1177 } else {
David Hildenbrande78a13f2023-12-20 23:45:03 +01001178 /* Raced ahead of a remove of ENTIRELY_MAPPED */
David Hildenbrand96fd7492023-12-20 23:44:37 +01001179 nr = 0;
1180 }
1181 }
David Hildenbrand05c53232024-04-09 21:22:47 +02001182 atomic_inc(&folio->_large_mapcount);
David Hildenbrand96fd7492023-12-20 23:44:37 +01001183 break;
1184 }
1185 return nr;
1186}
1187
Muchun Song6a8e0592022-04-28 23:16:10 -07001188/**
David Hildenbrand06968622023-10-02 16:29:48 +02001189 * folio_move_anon_rmap - move a folio to our anon_vma
1190 * @folio: The folio to move to our anon_vma
1191 * @vma: The vma the folio belongs to
Rik van Rielc44b6742010-03-05 13:42:09 -08001192 *
David Hildenbrand06968622023-10-02 16:29:48 +02001193 * When a folio belongs exclusively to one process after a COW event,
1194 * that folio can be moved into the anon_vma that belongs to just that
1195 * process, so the rmap code will not search the parent or sibling processes.
Rik van Rielc44b6742010-03-05 13:42:09 -08001196 */
David Hildenbrand06968622023-10-02 16:29:48 +02001197void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma)
Rik van Rielc44b6742010-03-05 13:42:09 -08001198{
Matthew Wilcox (Oracle)595af4c92022-09-02 20:46:45 +01001199 void *anon_vma = vma->anon_vma;
Rik van Rielc44b6742010-03-05 13:42:09 -08001200
Matthew Wilcox (Oracle)595af4c92022-09-02 20:46:45 +01001201 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
Sasha Levin81d1b092014-10-09 15:28:10 -07001202 VM_BUG_ON_VMA(!anon_vma, vma);
Rik van Rielc44b6742010-03-05 13:42:09 -08001203
Matthew Wilcox (Oracle)595af4c92022-09-02 20:46:45 +01001204 anon_vma += PAGE_MAPPING_ANON;
Vladimir Davydov414e2fb2015-06-24 16:56:56 -07001205 /*
1206 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
Matthew Wilcox (Oracle)b3ac0412022-01-21 11:27:31 -05001207 * simultaneously, so a concurrent reader (eg folio_referenced()'s
1208 * folio_test_anon()) will not see one without the other.
Vladimir Davydov414e2fb2015-06-24 16:56:56 -07001209 */
Matthew Wilcox (Oracle)595af4c92022-09-02 20:46:45 +01001210 WRITE_ONCE(folio->mapping, anon_vma);
Rik van Rielc44b6742010-03-05 13:42:09 -08001211}
1212
1213/**
David Hildenbrandc66db8c2023-09-13 14:51:09 +02001214 * __folio_set_anon - set up a new anonymous rmap for a folio
1215 * @folio: The folio to set up the new anonymous rmap for.
1216 * @vma: VM area to add the folio to.
Ryan Robertsc33c7942023-06-12 16:15:45 +01001217 * @address: User virtual address of the mapping
David Hildenbrandc66db8c2023-09-13 14:51:09 +02001218 * @exclusive: Whether the folio is exclusive to the process.
Nick Piggin9617d952006-01-06 00:11:12 -08001219 */
David Hildenbrandc66db8c2023-09-13 14:51:09 +02001220static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma,
1221 unsigned long address, bool exclusive)
Nick Piggin9617d952006-01-06 00:11:12 -08001222{
Rik van Riele8a03fe2010-04-14 17:59:28 -04001223 struct anon_vma *anon_vma = vma->anon_vma;
Nick Piggin9617d952006-01-06 00:11:12 -08001224
Rik van Riele8a03fe2010-04-14 17:59:28 -04001225 BUG_ON(!anon_vma);
Linus Torvaldsea900022010-04-12 12:44:29 -07001226
1227 /*
David Hildenbrandc66db8c2023-09-13 14:51:09 +02001228 * If the folio isn't exclusive to this vma, we must use the _oldest_
1229 * possible anon_vma for the folio mapping!
Linus Torvaldsea900022010-04-12 12:44:29 -07001230 */
Andi Kleen4e1c1972010-09-22 12:43:56 +02001231 if (!exclusive)
Andrea Arcangeli288468c2010-08-09 17:19:09 -07001232 anon_vma = anon_vma->root;
Linus Torvaldsea900022010-04-12 12:44:29 -07001233
Alex Shi16f5e702020-12-15 12:33:42 -08001234 /*
Matthew Wilcox (Oracle)5b4bd902023-01-16 19:29:59 +00001235 * page_idle does a lockless/optimistic rmap scan on folio->mapping.
Alex Shi16f5e702020-12-15 12:33:42 -08001236 * Make sure the compiler doesn't split the stores of anon_vma and
1237 * the PAGE_MAPPING_ANON type identifier, otherwise the rmap code
1238 * could mistake the mapping for a struct address_space and crash.
1239 */
Nick Piggin9617d952006-01-06 00:11:12 -08001240 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
Matthew Wilcox (Oracle)5b4bd902023-01-16 19:29:59 +00001241 WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);
1242 folio->index = linear_page_index(vma, address);
Nick Piggin9617d952006-01-06 00:11:12 -08001243}
1244
1245/**
Randy Dunlap43d8eac2008-03-19 17:00:43 -07001246 * __page_check_anon_rmap - sanity check anonymous rmap addition
Matthew Wilcox (Oracle)dba438b2023-07-06 20:52:51 +01001247 * @folio: The folio containing @page.
1248 * @page: the page to check the mapping of
Nick Pigginc97a9e12007-05-16 22:11:21 -07001249 * @vma: the vm area in which the mapping is added
1250 * @address: the user virtual address mapped
1251 */
Matthew Wilcox (Oracle)dba438b2023-07-06 20:52:51 +01001252static void __page_check_anon_rmap(struct folio *folio, struct page *page,
Nick Pigginc97a9e12007-05-16 22:11:21 -07001253 struct vm_area_struct *vma, unsigned long address)
1254{
Nick Pigginc97a9e12007-05-16 22:11:21 -07001255 /*
1256 * The page's anon-rmap details (mapping and index) are guaranteed to
1257 * be set up correctly at this point.
1258 *
David Hildenbrand84f01692023-12-20 23:44:45 +01001259 * We have exclusion against folio_add_anon_rmap_*() because the caller
Miaohe Lin90aaca82021-02-25 17:17:59 -08001260 * always holds the page locked.
Nick Pigginc97a9e12007-05-16 22:11:21 -07001261 *
Matthew Wilcox (Oracle)cb9089b2023-12-11 16:22:11 +00001262 * We have exclusion against folio_add_new_anon_rmap because those pages
Nick Pigginc97a9e12007-05-16 22:11:21 -07001263 * are initially only visible via the pagetables, and the pte is locked
Matthew Wilcox (Oracle)cb9089b2023-12-11 16:22:11 +00001264 * over the call to folio_add_new_anon_rmap.
Nick Pigginc97a9e12007-05-16 22:11:21 -07001265 */
Matthew Wilcox (Oracle)e05b3452022-01-29 11:52:52 -05001266 VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
1267 folio);
Yang Shi30c46382019-11-30 17:51:26 -08001268 VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
1269 page);
Nick Pigginc97a9e12007-05-16 22:11:21 -07001270}
1271
Yosry Ahmed15c0536f2024-05-06 21:13:33 +00001272static void __folio_mod_stat(struct folio *folio, int nr, int nr_pmdmapped)
1273{
1274 int idx;
1275
1276 if (nr) {
1277 idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED;
1278 __lruvec_stat_mod_folio(folio, idx, nr);
1279 }
1280 if (nr_pmdmapped) {
1281 if (folio_test_anon(folio)) {
1282 idx = NR_ANON_THPS;
1283 __lruvec_stat_mod_folio(folio, idx, nr_pmdmapped);
1284 } else {
1285 /* NR_*_PMDMAPPED are not maintained per-memcg */
1286 idx = folio_test_swapbacked(folio) ?
1287 NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED;
1288 __mod_node_page_state(folio_pgdat(folio), idx,
1289 nr_pmdmapped);
1290 }
1291 }
1292}
1293
David Hildenbrand8bd51302023-12-20 23:44:38 +01001294static __always_inline void __folio_add_anon_rmap(struct folio *folio,
1295 struct page *page, int nr_pages, struct vm_area_struct *vma,
1296 unsigned long address, rmap_t flags, enum rmap_level level)
1297{
1298 int i, nr, nr_pmdmapped = 0;
Hugh Dickinsbe5ef2d2022-11-22 01:42:04 -08001299
Barry Song4c1171f2024-06-18 11:11:37 +12001300 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
1301
David Hildenbrand8bd51302023-12-20 23:44:38 +01001302 nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);
Hugh Dickins5ad64682009-12-14 17:59:24 -08001303
Barry Song4c1171f2024-06-18 11:11:37 +12001304 if (likely(!folio_test_ksm(folio)))
David Hildenbrandc5c54002023-09-13 14:51:10 +02001305 __page_check_anon_rmap(folio, page, vma, address);
David Hildenbrand8bd51302023-12-20 23:44:38 +01001306
Yosry Ahmed15c0536f2024-05-06 21:13:33 +00001307 __folio_mod_stat(folio, nr, nr_pmdmapped);
1308
David Hildenbrand8bd51302023-12-20 23:44:38 +01001309 if (flags & RMAP_EXCLUSIVE) {
1310 switch (level) {
1311 case RMAP_LEVEL_PTE:
1312 for (i = 0; i < nr_pages; i++)
1313 SetPageAnonExclusive(page + i);
1314 break;
1315 case RMAP_LEVEL_PMD:
1316 SetPageAnonExclusive(page);
1317 break;
1318 }
1319 }
1320 for (i = 0; i < nr_pages; i++) {
1321 struct page *cur_page = page + i;
1322
1323 /* While PTE-mapping a THP we have a PMD and a PTE mapping. */
1324 VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||
1325 (folio_test_large(folio) &&
1326 folio_entire_mapcount(folio) > 1)) &&
1327 PageAnonExclusive(cur_page), folio);
1328 }
Hugh Dickinscea86fe2022-02-14 18:26:39 -08001329
Yin Fengwei1acbc3f2023-09-18 15:33:17 +08001330 /*
1331 * For large folio, only mlock it if it's fully mapped to VMA. It's
1332 * not easy to check whether the large folio is fully mapped to VMA
1333 * here. Only mlock normal 4K folio and leave page reclaim to handle
1334 * large folio.
1335 */
1336 if (!folio_test_large(folio))
1337 mlock_vma_folio(folio, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338}
1339
Randy Dunlap43d8eac2008-03-19 17:00:43 -07001340/**
David Hildenbrand8bd51302023-12-20 23:44:38 +01001341 * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
1342 * @folio: The folio to add the mappings to
1343 * @page: The first page to add
1344 * @nr_pages: The number of pages which will be mapped
1345 * @vma: The vm area in which the mappings are added
1346 * @address: The user virtual address of the first page to map
1347 * @flags: The rmap flags
1348 *
1349 * The page range of folio is defined by [first_page, first_page + nr_pages)
1350 *
1351 * The caller needs to hold the page table lock, and the page must be locked in
1352 * the anon_vma case: to serialize mapping,index checking after setting,
1353 * and to ensure that an anon folio is not being upgraded racily to a KSM folio
1354 * (but KSM folios are never downgraded).
1355 */
1356void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
1357 int nr_pages, struct vm_area_struct *vma, unsigned long address,
1358 rmap_t flags)
1359{
1360 __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
1361 RMAP_LEVEL_PTE);
1362}
1363
1364/**
1365 * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
1366 * @folio: The folio to add the mapping to
1367 * @page: The first page to add
1368 * @vma: The vm area in which the mapping is added
1369 * @address: The user virtual address of the first page to map
1370 * @flags: The rmap flags
1371 *
1372 * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
1373 *
1374 * The caller needs to hold the page table lock, and the page must be locked in
1375 * the anon_vma case: to serialize mapping,index checking after setting.
1376 */
1377void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
1378 struct vm_area_struct *vma, unsigned long address, rmap_t flags)
1379{
1380#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1381 __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
1382 RMAP_LEVEL_PMD);
1383#else
1384 WARN_ON_ONCE(true);
1385#endif
1386}
1387
1388/**
Matthew Wilcox (Oracle)4d510f32023-01-11 14:28:54 +00001389 * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
1390 * @folio: The folio to add the mapping to.
Nick Piggin9617d952006-01-06 00:11:12 -08001391 * @vma: the vm area in which the mapping is added
1392 * @address: the user virtual address mapped
Barry Song15bde4a2024-06-18 11:11:35 +12001393 * @flags: The rmap flags
David Hildenbrand40f2bbf2022-05-09 18:20:43 -07001394 *
David Hildenbrand84f01692023-12-20 23:44:45 +01001395 * Like folio_add_anon_rmap_*() but must only be called on *new* folios.
Nick Piggin9617d952006-01-06 00:11:12 -08001396 * This means the inc-and-test can be bypassed.
Barry Song15bde4a2024-06-18 11:11:35 +12001397 * The folio doesn't necessarily need to be locked while it's exclusive
1398 * unless two threads map it concurrently. However, the folio must be
1399 * locked if it's shared.
Matthew Wilcox (Oracle)4d510f32023-01-11 14:28:54 +00001400 *
Barry Song15bde4a2024-06-18 11:11:35 +12001401 * If the folio is pmd-mappable, it is accounted as a THP.
Nick Piggin9617d952006-01-06 00:11:12 -08001402 */
Matthew Wilcox (Oracle)4d510f32023-01-11 14:28:54 +00001403void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
Barry Song15bde4a2024-06-18 11:11:35 +12001404 unsigned long address, rmap_t flags)
Nick Piggin9617d952006-01-06 00:11:12 -08001405{
Barry Song15bde4a2024-06-18 11:11:35 +12001406 const int nr = folio_nr_pages(folio);
1407 const bool exclusive = flags & RMAP_EXCLUSIVE;
Yosry Ahmed15c0536f2024-05-06 21:13:33 +00001408 int nr_pmdmapped = 0;
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001409
David Hildenbranda4ea1862023-12-20 23:44:30 +01001410 VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
Barry Song15bde4a2024-06-18 11:11:35 +12001411 VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio);
Ryan Roberts372cbd42023-12-07 16:12:03 +00001412 VM_BUG_ON_VMA(address < vma->vm_start ||
1413 address + (nr << PAGE_SHIFT) > vma->vm_end, vma);
Barry Song9ae2fea2024-06-18 11:11:36 +12001414
Jason A. Donenfeld9651fce2022-12-08 17:55:04 +01001415 /*
1416 * VM_DROPPABLE mappings don't swap; instead they're just dropped when
1417 * under memory pressure.
1418 */
Linus Torvalds7a3fad32024-07-24 10:29:50 -07001419 if (!folio_test_swapbacked(folio) && !(vma->vm_flags & VM_DROPPABLE))
Barry Song9ae2fea2024-06-18 11:11:36 +12001420 __folio_set_swapbacked(folio);
Barry Song15bde4a2024-06-18 11:11:35 +12001421 __folio_set_anon(folio, vma, address, exclusive);
Hugh Dickinsd8dd5e92022-11-09 18:18:49 -08001422
Ryan Roberts372cbd42023-12-07 16:12:03 +00001423 if (likely(!folio_test_large(folio))) {
Hugh Dickinsd8dd5e92022-11-09 18:18:49 -08001424 /* increment count (starts at -1) */
Matthew Wilcox (Oracle)4d510f32023-01-11 14:28:54 +00001425 atomic_set(&folio->_mapcount, 0);
Barry Song15bde4a2024-06-18 11:11:35 +12001426 if (exclusive)
1427 SetPageAnonExclusive(&folio->page);
Ryan Roberts372cbd42023-12-07 16:12:03 +00001428 } else if (!folio_test_pmd_mappable(folio)) {
1429 int i;
1430
1431 for (i = 0; i < nr; i++) {
1432 struct page *page = folio_page(folio, i);
1433
1434 /* increment count (starts at -1) */
1435 atomic_set(&page->_mapcount, 0);
Barry Song15bde4a2024-06-18 11:11:35 +12001436 if (exclusive)
1437 SetPageAnonExclusive(page);
Ryan Roberts372cbd42023-12-07 16:12:03 +00001438 }
1439
David Hildenbrand05c53232024-04-09 21:22:47 +02001440 /* increment count (starts at -1) */
1441 atomic_set(&folio->_large_mapcount, nr - 1);
Ryan Roberts372cbd42023-12-07 16:12:03 +00001442 atomic_set(&folio->_nr_pages_mapped, nr);
Hugh Dickinsd8dd5e92022-11-09 18:18:49 -08001443 } else {
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08001444 /* increment count (starts at -1) */
Matthew Wilcox (Oracle)4d510f32023-01-11 14:28:54 +00001445 atomic_set(&folio->_entire_mapcount, 0);
David Hildenbrand05c53232024-04-09 21:22:47 +02001446 /* increment count (starts at -1) */
1447 atomic_set(&folio->_large_mapcount, 0);
David Hildenbrande78a13f2023-12-20 23:45:03 +01001448 atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED);
Barry Song15bde4a2024-06-18 11:11:35 +12001449 if (exclusive)
1450 SetPageAnonExclusive(&folio->page);
Yosry Ahmed15c0536f2024-05-06 21:13:33 +00001451 nr_pmdmapped = nr;
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001452 }
Hugh Dickinsd8dd5e92022-11-09 18:18:49 -08001453
Yosry Ahmed15c0536f2024-05-06 21:13:33 +00001454 __folio_mod_stat(folio, nr, nr_pmdmapped);
Nick Piggin9617d952006-01-06 00:11:12 -08001455}
1456
David Hildenbrand68f032082023-12-20 23:44:31 +01001457static __always_inline void __folio_add_file_rmap(struct folio *folio,
1458 struct page *page, int nr_pages, struct vm_area_struct *vma,
1459 enum rmap_level level)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001460{
David Hildenbrand96fd7492023-12-20 23:44:37 +01001461 int nr, nr_pmdmapped = 0;
Kirill A. Shutemovdd78fed2016-07-26 15:25:26 -07001462
David Hildenbrand68f032082023-12-20 23:44:31 +01001463 VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);
Hugh Dickins9bd31552022-11-02 18:53:45 -07001464
David Hildenbrand96fd7492023-12-20 23:44:37 +01001465 nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);
Yosry Ahmed15c0536f2024-05-06 21:13:33 +00001466 __folio_mod_stat(folio, nr, nr_pmdmapped);
Hugh Dickinscea86fe2022-02-14 18:26:39 -08001467
David Hildenbrand84f01692023-12-20 23:44:45 +01001468 /* See comments in folio_add_anon_rmap_*() */
Yin Fengwei1acbc3f2023-09-18 15:33:17 +08001469 if (!folio_test_large(folio))
1470 mlock_vma_folio(folio, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471}
1472
1473/**
David Hildenbrand68f032082023-12-20 23:44:31 +01001474 * folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio
1475 * @folio: The folio to add the mappings to
1476 * @page: The first page to add
1477 * @nr_pages: The number of pages that will be mapped using PTEs
1478 * @vma: The vm area in which the mappings are added
1479 *
1480 * The page range of the folio is defined by [page, page + nr_pages)
1481 *
1482 * The caller needs to hold the page table lock.
1483 */
1484void folio_add_file_rmap_ptes(struct folio *folio, struct page *page,
1485 int nr_pages, struct vm_area_struct *vma)
1486{
1487 __folio_add_file_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);
1488}
1489
1490/**
1491 * folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio
1492 * @folio: The folio to add the mapping to
1493 * @page: The first page to add
1494 * @vma: The vm area in which the mapping is added
1495 *
1496 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1497 *
1498 * The caller needs to hold the page table lock.
1499 */
1500void folio_add_file_rmap_pmd(struct folio *folio, struct page *page,
1501 struct vm_area_struct *vma)
1502{
1503#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1504 __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);
1505#else
1506 WARN_ON_ONCE(true);
1507#endif
1508}
1509
David Hildenbrandb06dc282023-12-20 23:44:47 +01001510static __always_inline void __folio_remove_rmap(struct folio *folio,
1511 struct page *page, int nr_pages, struct vm_area_struct *vma,
1512 enum rmap_level level)
1513{
Matthew Wilcox (Oracle)62beb902023-01-11 14:28:51 +00001514 atomic_t *mapped = &folio->_nr_pages_mapped;
David Hildenbrandb06dc282023-12-20 23:44:47 +01001515 int last, nr = 0, nr_pmdmapped = 0;
Zi Yan7491f3f2024-05-02 09:28:51 -04001516 bool partially_mapped = false;
Hugh Dickins9bd31552022-11-02 18:53:45 -07001517
David Hildenbrandb06dc282023-12-20 23:44:47 +01001518 __folio_rmap_sanity_checks(folio, page, nr_pages, level);
Hugh Dickins9bd31552022-11-02 18:53:45 -07001519
David Hildenbrandb06dc282023-12-20 23:44:47 +01001520 switch (level) {
1521 case RMAP_LEVEL_PTE:
David Hildenbrand46d62de2024-04-09 21:22:46 +02001522 if (!folio_test_large(folio)) {
1523 nr = atomic_add_negative(-1, &page->_mapcount);
1524 break;
1525 }
1526
David Hildenbrand05c53232024-04-09 21:22:47 +02001527 atomic_sub(nr_pages, &folio->_large_mapcount);
David Hildenbrandb06dc282023-12-20 23:44:47 +01001528 do {
1529 last = atomic_add_negative(-1, &page->_mapcount);
David Hildenbrand46d62de2024-04-09 21:22:46 +02001530 if (last) {
David Hildenbrandb06dc282023-12-20 23:44:47 +01001531 last = atomic_dec_return_relaxed(mapped);
David Hildenbrand46d62de2024-04-09 21:22:46 +02001532 if (last < ENTIRELY_MAPPED)
1533 nr++;
David Hildenbrandb06dc282023-12-20 23:44:47 +01001534 }
David Hildenbrandb06dc282023-12-20 23:44:47 +01001535 } while (page++, --nr_pages > 0);
Zi Yan7491f3f2024-05-02 09:28:51 -04001536
1537 partially_mapped = nr && atomic_read(mapped);
David Hildenbrandb06dc282023-12-20 23:44:47 +01001538 break;
1539 case RMAP_LEVEL_PMD:
David Hildenbrand05c53232024-04-09 21:22:47 +02001540 atomic_dec(&folio->_large_mapcount);
Matthew Wilcox (Oracle)62beb902023-01-11 14:28:51 +00001541 last = atomic_add_negative(-1, &folio->_entire_mapcount);
Hugh Dickins9bd31552022-11-02 18:53:45 -07001542 if (last) {
David Hildenbrande78a13f2023-12-20 23:45:03 +01001543 nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped);
1544 if (likely(nr < ENTIRELY_MAPPED)) {
Matthew Wilcox (Oracle)62beb902023-01-11 14:28:51 +00001545 nr_pmdmapped = folio_nr_pages(folio);
Matthew Wilcox (Oracle)eec20422023-01-11 14:28:48 +00001546 nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
Hugh Dickins6287b7d2022-12-04 17:57:07 -08001547 /* Raced ahead of another remove and an add? */
1548 if (unlikely(nr < 0))
1549 nr = 0;
1550 } else {
David Hildenbrande78a13f2023-12-20 23:45:03 +01001551 /* An add of ENTIRELY_MAPPED raced ahead */
Hugh Dickins6287b7d2022-12-04 17:57:07 -08001552 nr = 0;
1553 }
Hugh Dickins9bd31552022-11-02 18:53:45 -07001554 }
Zi Yan7491f3f2024-05-02 09:28:51 -04001555
1556 partially_mapped = nr < nr_pmdmapped;
David Hildenbrandb06dc282023-12-20 23:44:47 +01001557 break;
Hugh Dickinscb67f422022-11-02 18:51:38 -07001558 }
1559
Hugh Dickins9bd31552022-11-02 18:53:45 -07001560 if (nr) {
Hugh Dickins9bd31552022-11-02 18:53:45 -07001561 /*
Ryan Roberts7dc7c5e2023-12-07 16:12:02 +00001562 * Queue anon large folio for deferred split if at least one
Matthew Wilcox (Oracle)62beb902023-01-11 14:28:51 +00001563 * page of the folio is unmapped and at least one page
1564 * is still mapped.
Zi Yan7491f3f2024-05-02 09:28:51 -04001565 *
1566 * Check partially_mapped first to ensure it is a large folio.
Hugh Dickins9bd31552022-11-02 18:53:45 -07001567 */
Zi Yan7491f3f2024-05-02 09:28:51 -04001568 if (folio_test_anon(folio) && partially_mapped &&
1569 list_empty(&folio->_deferred_list))
1570 deferred_split_folio(folio);
Hugh Dickins9bd31552022-11-02 18:53:45 -07001571 }
Yosry Ahmed15c0536f2024-05-06 21:13:33 +00001572 __folio_mod_stat(folio, -nr, -nr_pmdmapped);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08001573
KOSAKI Motohirob904dcf2009-09-21 17:01:28 -07001574 /*
Matthew Wilcox (Oracle)672aa272023-01-16 19:28:26 +00001575 * It would be tidy to reset folio_test_anon mapping when fully
David Hildenbrand84f01692023-12-20 23:44:45 +01001576 * unmapped, but that might overwrite a racing folio_add_anon_rmap_*()
Matthew Wilcox (Oracle)672aa272023-01-16 19:28:26 +00001577 * which increments mapcount after us but sets mapping before us:
1578 * so leave the reset to free_pages_prepare, and remember that
1579 * it's only reliable while mapped.
KOSAKI Motohirob904dcf2009-09-21 17:01:28 -07001580 */
Hugh Dickins9bd31552022-11-02 18:53:45 -07001581
Yin Fengwei1acbc3f2023-09-18 15:33:17 +08001582 munlock_vma_folio(folio, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583}
1584
David Hildenbrandb06dc282023-12-20 23:44:47 +01001585/**
1586 * folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio
1587 * @folio: The folio to remove the mappings from
1588 * @page: The first page to remove
1589 * @nr_pages: The number of pages that will be removed from the mapping
1590 * @vma: The vm area from which the mappings are removed
1591 *
1592 * The page range of the folio is defined by [page, page + nr_pages)
1593 *
1594 * The caller needs to hold the page table lock.
1595 */
1596void folio_remove_rmap_ptes(struct folio *folio, struct page *page,
1597 int nr_pages, struct vm_area_struct *vma)
1598{
1599 __folio_remove_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);
1600}
1601
1602/**
1603 * folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio
1604 * @folio: The folio to remove the mapping from
1605 * @page: The first page to remove
1606 * @vma: The vm area from which the mapping is removed
1607 *
1608 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
1609 *
1610 * The caller needs to hold the page table lock.
1611 */
1612void folio_remove_rmap_pmd(struct folio *folio, struct page *page,
1613 struct vm_area_struct *vma)
1614{
1615#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1616 __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);
1617#else
1618 WARN_ON_ONCE(true);
1619#endif
1620}
1621
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622/*
Joonsoo Kim52629502014-01-21 15:49:50 -08001623 * @arg: enum ttu_flags will be passed to this argument
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624 */
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05001625static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
Joonsoo Kim52629502014-01-21 15:49:50 -08001626 unsigned long address, void *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627{
1628 struct mm_struct *mm = vma->vm_mm;
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001629 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 pte_t pteval;
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001631 struct page *subpage;
David Hildenbrand6c287602022-05-09 18:20:44 -07001632 bool anon_exclusive, ret = true;
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08001633 struct mmu_notifier_range range;
Palmer Dabbelt4708f312020-04-06 20:08:00 -07001634 enum ttu_flags flags = (enum ttu_flags)(long)arg;
Ryan Robertsc33c7942023-06-12 16:15:45 +01001635 unsigned long pfn;
Ryan Roberts935d4f02023-09-22 12:58:03 +01001636 unsigned long hsz = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637
Hugh Dickins732ed552021-06-15 18:23:53 -07001638 /*
1639 * When racing against e.g. zap_pte_range() on another cpu,
David Hildenbrandca1a0742023-12-20 23:44:54 +01001640 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
Yang Shi1fb08ac2021-06-30 18:52:01 -07001641 * try_to_unmap() may return before page_mapped() has become false,
Hugh Dickins732ed552021-06-15 18:23:53 -07001642 * if page table locking is skipped: use TTU_SYNC to wait for that.
1643 */
1644 if (flags & TTU_SYNC)
1645 pvmw.flags = PVMW_SYNC;
1646
Jérôme Glisse369ea822017-08-31 17:17:27 -04001647 /*
Mike Kravetz017b1662018-10-05 15:51:29 -07001648 * For THP, we have to assume the worse case ie pmd for invalidation.
1649 * For hugetlb, it could be much worse if we need to do pud
1650 * invalidation in the case of pmd sharing.
1651 *
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001652 * Note that the folio can not be freed in this function as call of
1653 * try_to_unmap() must hold a reference on the folio.
Jérôme Glisse369ea822017-08-31 17:17:27 -04001654 */
Matthew Wilcox (Oracle)2aff7a42022-02-03 11:40:17 -05001655 range.end = vma_address_end(&pvmw);
Alistair Popple7d4a8be2023-01-10 13:57:22 +11001656 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
Hugh Dickins494334e2021-06-15 18:23:56 -07001657 address, range.end);
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001658 if (folio_test_hugetlb(folio)) {
Mike Kravetz017b1662018-10-05 15:51:29 -07001659 /*
1660 * If sharing is possible, start and end will be adjusted
1661 * accordingly.
1662 */
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08001663 adjust_range_if_pmd_sharing_possible(vma, &range.start,
1664 &range.end);
Ryan Roberts935d4f02023-09-22 12:58:03 +01001665
1666 /* We need the huge page size for set_huge_pte_at() */
1667 hsz = huge_page_size(hstate_vma(vma));
Mike Kravetz017b1662018-10-05 15:51:29 -07001668 }
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08001669 mmu_notifier_invalidate_range_start(&range);
Jérôme Glisse369ea822017-08-31 17:17:27 -04001670
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001671 while (page_vma_mapped_walk(&pvmw)) {
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001672 /*
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001673 * If the folio is in an mlock()d vma, we must not swap it out.
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001674 */
Hugh Dickinsefdb6722021-07-11 20:10:49 -07001675 if (!(flags & TTU_IGNORE_MLOCK) &&
1676 (vma->vm_flags & VM_LOCKED)) {
Hugh Dickinscea86fe2022-02-14 18:26:39 -08001677 /* Restore the mlock which got missed */
Yin Fengwei1acbc3f2023-09-18 15:33:17 +08001678 if (!folio_test_large(folio))
1679 mlock_vma_folio(folio, vma);
Lance Yang26d21b12024-06-14 09:51:36 +08001680 goto walk_abort;
Hugh Dickinsb87537d9e2015-11-05 18:49:33 -08001681 }
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001682
Lance Yang735ecdf2024-06-14 09:51:38 +08001683 if (!pvmw.pte) {
1684 if (unmap_huge_pmd_locked(vma, pvmw.address, pvmw.pmd,
1685 folio))
1686 goto walk_done;
1687
1688 if (flags & TTU_SPLIT_HUGE_PMD) {
1689 /*
1690 * We temporarily have to drop the PTL and
1691 * restart so we can process the PTE-mapped THP.
1692 */
1693 split_huge_pmd_locked(vma, pvmw.address,
1694 pvmw.pmd, false, folio);
1695 flags &= ~TTU_SPLIT_HUGE_PMD;
1696 page_vma_mapped_walk_restart(&pvmw);
1697 continue;
1698 }
Lance Yang29e847d2024-06-14 09:51:37 +08001699 }
1700
1701 /* Unexpected PMD-mapped THP? */
1702 VM_BUG_ON_FOLIO(!pvmw.pte, folio);
1703
Ryan Robertsc33c7942023-06-12 16:15:45 +01001704 pfn = pte_pfn(ptep_get(pvmw.pte));
1705 subpage = folio_page(folio, pfn - folio_pfn(folio));
Linus Torvalds785373b2017-08-29 09:11:06 -07001706 address = pvmw.address;
David Hildenbrand6c287602022-05-09 18:20:44 -07001707 anon_exclusive = folio_test_anon(folio) &&
1708 PageAnonExclusive(subpage);
Linus Torvalds785373b2017-08-29 09:11:06 -07001709
Baolin Wangdfc7ab52022-05-09 18:20:53 -07001710 if (folio_test_hugetlb(folio)) {
Baolin Wang0506c312022-06-20 19:47:15 +08001711 bool anon = folio_test_anon(folio);
1712
Baolin Wang54205e92022-05-09 18:20:53 -07001713 /*
Baolin Wanga00a8752022-05-13 16:48:56 -07001714 * The try_to_unmap() is only passed a hugetlb page
1715 * in the case where the hugetlb page is poisoned.
1716 */
1717 VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage);
1718 /*
Baolin Wang54205e92022-05-09 18:20:53 -07001719 * huge_pmd_unshare may unmap an entire PMD page.
1720 * There is no way of knowing exactly which PMDs may
1721 * be cached for this mm, so we must flush them all.
1722 * start/end were already adjusted above to cover this
1723 * range.
1724 */
1725 flush_cache_range(vma, range.start, range.end);
1726
Baolin Wang0506c312022-06-20 19:47:15 +08001727 /*
1728 * To call huge_pmd_unshare, i_mmap_rwsem must be
1729 * held in write mode. Caller needs to explicitly
1730 * do this outside rmap routines.
Mike Kravetz40549ba2022-09-14 15:18:09 -07001731 *
1732 * We also must hold hugetlb vma_lock in write mode.
1733 * Lock order dictates acquiring vma_lock BEFORE
1734 * i_mmap_rwsem. We can only try lock here and fail
1735 * if unsuccessful.
Baolin Wang0506c312022-06-20 19:47:15 +08001736 */
Mike Kravetz40549ba2022-09-14 15:18:09 -07001737 if (!anon) {
1738 VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
Lance Yang26d21b12024-06-14 09:51:36 +08001739 if (!hugetlb_vma_trylock_write(vma))
1740 goto walk_abort;
Mike Kravetz40549ba2022-09-14 15:18:09 -07001741 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
1742 hugetlb_vma_unlock_write(vma);
1743 flush_tlb_range(vma,
1744 range.start, range.end);
Mike Kravetz40549ba2022-09-14 15:18:09 -07001745 /*
1746 * The ref count of the PMD page was
1747 * dropped which is part of the way map
1748 * counting is done for shared PMDs.
1749 * Return 'true' here. When there is
1750 * no other sharing, huge_pmd_unshare
1751 * returns false and we will unmap the
1752 * actual page and drop map count
1753 * to zero.
1754 */
Lance Yang26d21b12024-06-14 09:51:36 +08001755 goto walk_done;
Mike Kravetz40549ba2022-09-14 15:18:09 -07001756 }
1757 hugetlb_vma_unlock_write(vma);
Mike Kravetz017b1662018-10-05 15:51:29 -07001758 }
Baolin Wanga00a8752022-05-13 16:48:56 -07001759 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
Baolin Wang54205e92022-05-09 18:20:53 -07001760 } else {
Ryan Robertsc33c7942023-06-12 16:15:45 +01001761 flush_cache_page(vma, address, pfn);
David Hildenbrand088b8aa2022-09-01 10:35:59 +02001762 /* Nuke the page table entry. */
1763 if (should_defer_flush(mm, flags)) {
Baolin Wanga00a8752022-05-13 16:48:56 -07001764 /*
1765 * We clear the PTE but do not flush so potentially
1766 * a remote CPU could still be writing to the folio.
1767 * If the entry was previously clean then the
1768 * architecture must guarantee that a clear->dirty
1769 * transition on a cached TLB entry is written through
1770 * and traps if the PTE is unmapped.
1771 */
1772 pteval = ptep_get_and_clear(mm, address, pvmw.pte);
Mel Gorman72b252a2015-09-04 15:47:32 -07001773
Barry Songf73419b2023-07-17 21:10:02 +08001774 set_tlb_ubc_flush_pending(mm, pteval, address);
Baolin Wanga00a8752022-05-13 16:48:56 -07001775 } else {
1776 pteval = ptep_clear_flush(vma, address, pvmw.pte);
1777 }
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001778 }
1779
Peter Xu999dad82022-05-12 20:22:53 -07001780 /*
1781 * Now the pte is cleared. If this pte was uffd-wp armed,
1782 * we may want to replace a none pte with a marker pte if
1783 * it's file-backed, so we don't lose the tracking info.
1784 */
1785 pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval);
1786
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001787 /* Set the dirty flag on the folio now the pte is gone. */
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001788 if (pte_dirty(pteval))
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001789 folio_mark_dirty(folio);
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001790
1791 /* Update high watermark before we lower rss */
1792 update_hiwater_rss(mm);
1793
Naoya Horiguchi6da6b1d2023-02-21 17:59:05 +09001794 if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) {
Punit Agrawal5fd27b82017-07-06 15:39:53 -07001795 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001796 if (folio_test_hugetlb(folio)) {
1797 hugetlb_count_sub(folio_nr_pages(folio), mm);
Ryan Roberts935d4f02023-09-22 12:58:03 +01001798 set_huge_pte_at(mm, address, pvmw.pte, pteval,
1799 hsz);
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001800 } else {
Kefeng Wanga23f5172024-01-11 15:24:28 +00001801 dec_mm_counter(mm, mm_counter(folio));
Linus Torvalds785373b2017-08-29 09:11:06 -07001802 set_pte_at(mm, address, pvmw.pte, pteval);
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001803 }
1804
Christian Borntraegerbce73e42018-07-13 16:58:52 -07001805 } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001806 /*
1807 * The guest indicated that the page content is of no
1808 * interest anymore. Simply discard the pte, vmscan
1809 * will take care of the rest.
Christian Borntraegerbce73e42018-07-13 16:58:52 -07001810 * A future reference will then fault in a new zero
1811 * page. When userfaultfd is active, we must not drop
1812 * this page though, as its main user (postcopy
1813 * migration) will not expect userfaults on already
1814 * copied pages.
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001815 */
Kefeng Wanga23f5172024-01-11 15:24:28 +00001816 dec_mm_counter(mm, mm_counter(folio));
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001817 } else if (folio_test_anon(folio)) {
David Hildenbrandcfeed8f2023-08-21 18:08:46 +02001818 swp_entry_t entry = page_swap_entry(subpage);
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001819 pte_t swp_pte;
1820 /*
1821 * Store the swap location in the pte.
1822 * See handle_pte_fault() ...
1823 */
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001824 if (unlikely(folio_test_swapbacked(folio) !=
1825 folio_test_swapcache(folio))) {
Minchan Kimeb94a872017-05-03 14:52:36 -07001826 WARN_ON_ONCE(1);
Lance Yang26d21b12024-06-14 09:51:36 +08001827 goto walk_abort;
Minchan Kimeb94a872017-05-03 14:52:36 -07001828 }
Minchan Kim854e9ed2016-01-15 16:54:53 -08001829
Shaohua Li802a3a92017-05-03 14:52:32 -07001830 /* MADV_FREE page check */
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001831 if (!folio_test_swapbacked(folio)) {
Mauricio Faria de Oliveira6c8e2a22022-03-24 18:14:09 -07001832 int ref_count, map_count;
1833
1834 /*
1835 * Synchronize with gup_pte_range():
1836 * - clear PTE; barrier; read refcount
1837 * - inc refcount; barrier; read PTE
1838 */
1839 smp_mb();
1840
1841 ref_count = folio_ref_count(folio);
1842 map_count = folio_mapcount(folio);
1843
1844 /*
1845 * Order reads for page refcount and dirty flag
1846 * (see comments in __remove_mapping()).
1847 */
1848 smp_rmb();
1849
1850 /*
1851 * The only page refs must be one from isolation
1852 * plus the rmap(s) (dropped by discard:).
1853 */
1854 if (ref_count == 1 + map_count &&
Jason A. Donenfeld9651fce2022-12-08 17:55:04 +01001855 (!folio_test_dirty(folio) ||
1856 /*
1857 * Unlike MADV_FREE mappings, VM_DROPPABLE
1858 * ones can be dropped even if they've
1859 * been dirtied.
1860 */
1861 (vma->vm_flags & VM_DROPPABLE))) {
Shaohua Li802a3a92017-05-03 14:52:32 -07001862 dec_mm_counter(mm, MM_ANONPAGES);
1863 goto discard;
1864 }
1865
1866 /*
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001867 * If the folio was redirtied, it cannot be
Shaohua Li802a3a92017-05-03 14:52:32 -07001868 * discarded. Remap the page to page table.
1869 */
Linus Torvalds785373b2017-08-29 09:11:06 -07001870 set_pte_at(mm, address, pvmw.pte, pteval);
Jason A. Donenfeld9651fce2022-12-08 17:55:04 +01001871 /*
1872 * Unlike MADV_FREE mappings, VM_DROPPABLE ones
1873 * never get swap backed on failure to drop.
1874 */
1875 if (!(vma->vm_flags & VM_DROPPABLE))
1876 folio_set_swapbacked(folio);
Lance Yang26d21b12024-06-14 09:51:36 +08001877 goto walk_abort;
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001878 }
1879
1880 if (swap_duplicate(entry) < 0) {
Linus Torvalds785373b2017-08-29 09:11:06 -07001881 set_pte_at(mm, address, pvmw.pte, pteval);
Lance Yang26d21b12024-06-14 09:51:36 +08001882 goto walk_abort;
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001883 }
Khalid Azizca827d52018-02-21 10:15:44 -07001884 if (arch_unmap_one(mm, vma, address, pteval) < 0) {
David Hildenbrand322842e2022-05-09 18:20:42 -07001885 swap_free(entry);
Khalid Azizca827d52018-02-21 10:15:44 -07001886 set_pte_at(mm, address, pvmw.pte, pteval);
Lance Yang26d21b12024-06-14 09:51:36 +08001887 goto walk_abort;
Khalid Azizca827d52018-02-21 10:15:44 -07001888 }
David Hildenbrand088b8aa2022-09-01 10:35:59 +02001889
David Hildenbrande3b4b132023-12-20 23:45:02 +01001890 /* See folio_try_share_anon_rmap(): clear PTE first. */
David Hildenbrand6c287602022-05-09 18:20:44 -07001891 if (anon_exclusive &&
David Hildenbrande3b4b132023-12-20 23:45:02 +01001892 folio_try_share_anon_rmap_pte(folio, subpage)) {
David Hildenbrand6c287602022-05-09 18:20:44 -07001893 swap_free(entry);
1894 set_pte_at(mm, address, pvmw.pte, pteval);
Lance Yang26d21b12024-06-14 09:51:36 +08001895 goto walk_abort;
David Hildenbrand6c287602022-05-09 18:20:44 -07001896 }
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001897 if (list_empty(&mm->mmlist)) {
1898 spin_lock(&mmlist_lock);
1899 if (list_empty(&mm->mmlist))
1900 list_add(&mm->mmlist, &init_mm.mmlist);
1901 spin_unlock(&mmlist_lock);
1902 }
Minchan Kim854e9ed2016-01-15 16:54:53 -08001903 dec_mm_counter(mm, MM_ANONPAGES);
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001904 inc_mm_counter(mm, MM_SWAPENTS);
1905 swp_pte = swp_entry_to_pte(entry);
David Hildenbrand1493a192022-05-09 18:20:45 -07001906 if (anon_exclusive)
1907 swp_pte = pte_swp_mkexclusive(swp_pte);
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001908 if (pte_soft_dirty(pteval))
1909 swp_pte = pte_swp_mksoft_dirty(swp_pte);
Peter Xuf45ec5f2020-04-06 20:06:01 -07001910 if (pte_uffd_wp(pteval))
1911 swp_pte = pte_swp_mkuffd_wp(swp_pte);
Linus Torvalds785373b2017-08-29 09:11:06 -07001912 set_pte_at(mm, address, pvmw.pte, swp_pte);
Jérôme Glisse0f108512017-11-15 17:34:07 -08001913 } else {
1914 /*
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001915 * This is a locked file-backed folio,
1916 * so it cannot be removed from the page
1917 * cache and replaced by a new folio before
1918 * mmu_notifier_invalidate_range_end, so no
1919 * concurrent thread might update its page table
1920 * to point at a new folio while a device is
1921 * still using this folio.
Jérôme Glisse0f108512017-11-15 17:34:07 -08001922 *
Mike Rapoportee657282022-06-27 09:00:26 +03001923 * See Documentation/mm/mmu_notifier.rst
Jérôme Glisse0f108512017-11-15 17:34:07 -08001924 */
Kefeng Wang6b27cc6c2024-01-11 15:24:29 +00001925 dec_mm_counter(mm, mm_counter_file(folio));
Jérôme Glisse0f108512017-11-15 17:34:07 -08001926 }
Minchan Kim854e9ed2016-01-15 16:54:53 -08001927discard:
David Hildenbrande1358262023-12-20 23:44:26 +01001928 if (unlikely(folio_test_hugetlb(folio)))
1929 hugetlb_remove_rmap(folio);
1930 else
David Hildenbrandca1a0742023-12-20 23:44:54 +01001931 folio_remove_rmap_pte(folio, subpage, vma);
Hugh Dickinsb7435502022-02-14 18:38:47 -08001932 if (vma->vm_flags & VM_LOCKED)
Lorenzo Stoakes96f97c42023-01-12 12:39:31 +00001933 mlock_drain_local();
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001934 folio_put(folio);
Lance Yang26d21b12024-06-14 09:51:36 +08001935 continue;
1936walk_abort:
1937 ret = false;
1938walk_done:
1939 page_vma_mapped_walk_done(&pvmw);
1940 break;
Kirill A. Shutemovc7ab0d22017-02-24 14:58:01 -08001941 }
Jérôme Glisse369ea822017-08-31 17:17:27 -04001942
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08001943 mmu_notifier_invalidate_range_end(&range);
Jérôme Glisse369ea822017-08-31 17:17:27 -04001944
Linus Torvalds1da177e2005-04-16 15:20:36 -07001945 return ret;
1946}
1947
Joonsoo Kim52629502014-01-21 15:49:50 -08001948static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
1949{
Anshuman Khandual222100e2020-04-01 21:07:52 -07001950 return vma_is_temporary_stack(vma);
Joonsoo Kim52629502014-01-21 15:49:50 -08001951}
1952
Kefeng Wangf3ad0322022-09-27 14:38:26 +08001953static int folio_not_mapped(struct folio *folio)
Joonsoo Kim52629502014-01-21 15:49:50 -08001954{
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05001955 return !folio_mapped(folio);
Kirill A. Shutemov2a52bcb2016-03-17 14:20:04 -07001956}
Joonsoo Kim52629502014-01-21 15:49:50 -08001957
Linus Torvalds1da177e2005-04-16 15:20:36 -07001958/**
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001959 * try_to_unmap - Try to remove all page table mappings to a folio.
1960 * @folio: The folio to unmap.
Andi Kleen14fa31b2009-09-16 11:50:10 +02001961 * @flags: action and flags
Linus Torvalds1da177e2005-04-16 15:20:36 -07001962 *
1963 * Tries to remove all the page table entries which are mapping this
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001964 * folio. It is the caller's responsibility to check if the folio is
1965 * still mapped if needed (use TTU_SYNC to prevent accounting races).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001966 *
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001967 * Context: Caller must hold the folio lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968 */
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05001969void try_to_unmap(struct folio *folio, enum ttu_flags flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001970{
Joonsoo Kim52629502014-01-21 15:49:50 -08001971 struct rmap_walk_control rwc = {
1972 .rmap_one = try_to_unmap_one,
Shaohua Li802a3a92017-05-03 14:52:32 -07001973 .arg = (void *)flags,
Kefeng Wangf3ad0322022-09-27 14:38:26 +08001974 .done = folio_not_mapped,
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05001975 .anon_lock = folio_lock_anon_vma_read,
Joonsoo Kim52629502014-01-21 15:49:50 -08001976 };
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977
Alistair Popplea98a2f02021-06-30 18:54:16 -07001978 if (flags & TTU_RMAP_LOCKED)
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05001979 rmap_walk_locked(folio, &rwc);
Alistair Popplea98a2f02021-06-30 18:54:16 -07001980 else
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05001981 rmap_walk(folio, &rwc);
Alistair Popplea98a2f02021-06-30 18:54:16 -07001982}
1983
1984/*
1985 * @arg: enum ttu_flags will be passed to this argument.
1986 *
1987 * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
Hugh Dickins64b586d2021-07-07 13:06:17 -07001988 * containing migration entries.
Alistair Popplea98a2f02021-06-30 18:54:16 -07001989 */
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05001990static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
Alistair Popplea98a2f02021-06-30 18:54:16 -07001991 unsigned long address, void *arg)
1992{
1993 struct mm_struct *mm = vma->vm_mm;
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05001994 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
Alistair Popplea98a2f02021-06-30 18:54:16 -07001995 pte_t pteval;
1996 struct page *subpage;
David Hildenbrand6c287602022-05-09 18:20:44 -07001997 bool anon_exclusive, ret = true;
Alistair Popplea98a2f02021-06-30 18:54:16 -07001998 struct mmu_notifier_range range;
1999 enum ttu_flags flags = (enum ttu_flags)(long)arg;
Ryan Robertsc33c7942023-06-12 16:15:45 +01002000 unsigned long pfn;
Ryan Roberts935d4f02023-09-22 12:58:03 +01002001 unsigned long hsz = 0;
Alistair Popplea98a2f02021-06-30 18:54:16 -07002002
Alistair Popplea98a2f02021-06-30 18:54:16 -07002003 /*
2004 * When racing against e.g. zap_pte_range() on another cpu,
David Hildenbrandca1a0742023-12-20 23:44:54 +01002005 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
Alistair Popplea98a2f02021-06-30 18:54:16 -07002006 * try_to_migrate() may return before page_mapped() has become false,
2007 * if page table locking is skipped: use TTU_SYNC to wait for that.
2008 */
2009 if (flags & TTU_SYNC)
2010 pvmw.flags = PVMW_SYNC;
2011
2012 /*
2013 * unmap_page() in mm/huge_memory.c is the only user of migration with
2014 * TTU_SPLIT_HUGE_PMD and it wants to freeze.
2015 */
2016 if (flags & TTU_SPLIT_HUGE_PMD)
Matthew Wilcox (Oracle)af28a982022-01-21 10:44:52 -05002017 split_huge_pmd_address(vma, address, true, folio);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002018
2019 /*
2020 * For THP, we have to assume the worse case ie pmd for invalidation.
2021 * For hugetlb, it could be much worse if we need to do pud
2022 * invalidation in the case of pmd sharing.
2023 *
2024 * Note that the page can not be free in this function as call of
2025 * try_to_unmap() must hold a reference on the page.
2026 */
Matthew Wilcox (Oracle)2aff7a42022-02-03 11:40:17 -05002027 range.end = vma_address_end(&pvmw);
Alistair Popple7d4a8be2023-01-10 13:57:22 +11002028 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
Alistair Popplea98a2f02021-06-30 18:54:16 -07002029 address, range.end);
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002030 if (folio_test_hugetlb(folio)) {
Alistair Popplea98a2f02021-06-30 18:54:16 -07002031 /*
2032 * If sharing is possible, start and end will be adjusted
2033 * accordingly.
2034 */
2035 adjust_range_if_pmd_sharing_possible(vma, &range.start,
2036 &range.end);
Ryan Roberts935d4f02023-09-22 12:58:03 +01002037
2038 /* We need the huge page size for set_huge_pte_at() */
2039 hsz = huge_page_size(hstate_vma(vma));
Alistair Popplea98a2f02021-06-30 18:54:16 -07002040 }
2041 mmu_notifier_invalidate_range_start(&range);
2042
2043 while (page_vma_mapped_walk(&pvmw)) {
2044#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
2045 /* PMD-mapped THP migration entry */
2046 if (!pvmw.pte) {
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002047 subpage = folio_page(folio,
2048 pmd_pfn(*pvmw.pmd) - folio_pfn(folio));
2049 VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
2050 !folio_test_pmd_mappable(folio), folio);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002051
David Hildenbrand7f5abe62022-05-09 18:20:44 -07002052 if (set_pmd_migration_entry(&pvmw, subpage)) {
2053 ret = false;
2054 page_vma_mapped_walk_done(&pvmw);
2055 break;
2056 }
Alistair Popplea98a2f02021-06-30 18:54:16 -07002057 continue;
2058 }
2059#endif
2060
2061 /* Unexpected PMD-mapped THP? */
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002062 VM_BUG_ON_FOLIO(!pvmw.pte, folio);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002063
Ryan Robertsc33c7942023-06-12 16:15:45 +01002064 pfn = pte_pfn(ptep_get(pvmw.pte));
2065
David Hildenbrand11182342022-06-23 22:53:32 +02002066 if (folio_is_zone_device(folio)) {
2067 /*
2068 * Our PTE is a non-present device exclusive entry and
2069 * calculating the subpage as for the common case would
2070 * result in an invalid pointer.
2071 *
2072 * Since only PAGE_SIZE pages can currently be
2073 * migrated, just set it to page. This will need to be
2074 * changed when hugepage migrations to device private
2075 * memory are supported.
2076 */
2077 VM_BUG_ON_FOLIO(folio_nr_pages(folio) > 1, folio);
2078 subpage = &folio->page;
2079 } else {
Ryan Robertsc33c7942023-06-12 16:15:45 +01002080 subpage = folio_page(folio, pfn - folio_pfn(folio));
David Hildenbrand11182342022-06-23 22:53:32 +02002081 }
Alistair Popplea98a2f02021-06-30 18:54:16 -07002082 address = pvmw.address;
David Hildenbrand6c287602022-05-09 18:20:44 -07002083 anon_exclusive = folio_test_anon(folio) &&
2084 PageAnonExclusive(subpage);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002085
Baolin Wangdfc7ab52022-05-09 18:20:53 -07002086 if (folio_test_hugetlb(folio)) {
Baolin Wang0506c312022-06-20 19:47:15 +08002087 bool anon = folio_test_anon(folio);
2088
Baolin Wang54205e92022-05-09 18:20:53 -07002089 /*
2090 * huge_pmd_unshare may unmap an entire PMD page.
2091 * There is no way of knowing exactly which PMDs may
2092 * be cached for this mm, so we must flush them all.
2093 * start/end were already adjusted above to cover this
2094 * range.
2095 */
2096 flush_cache_range(vma, range.start, range.end);
2097
Baolin Wang0506c312022-06-20 19:47:15 +08002098 /*
2099 * To call huge_pmd_unshare, i_mmap_rwsem must be
2100 * held in write mode. Caller needs to explicitly
2101 * do this outside rmap routines.
Mike Kravetz40549ba2022-09-14 15:18:09 -07002102 *
2103 * We also must hold hugetlb vma_lock in write mode.
2104 * Lock order dictates acquiring vma_lock BEFORE
2105 * i_mmap_rwsem. We can only try lock here and
2106 * fail if unsuccessful.
Baolin Wang0506c312022-06-20 19:47:15 +08002107 */
Mike Kravetz40549ba2022-09-14 15:18:09 -07002108 if (!anon) {
2109 VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
2110 if (!hugetlb_vma_trylock_write(vma)) {
2111 page_vma_mapped_walk_done(&pvmw);
2112 ret = false;
2113 break;
2114 }
2115 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
2116 hugetlb_vma_unlock_write(vma);
2117 flush_tlb_range(vma,
2118 range.start, range.end);
Baolin Wang0506c312022-06-20 19:47:15 +08002119
Mike Kravetz40549ba2022-09-14 15:18:09 -07002120 /*
2121 * The ref count of the PMD page was
2122 * dropped which is part of the way map
2123 * counting is done for shared PMDs.
2124 * Return 'true' here. When there is
2125 * no other sharing, huge_pmd_unshare
2126 * returns false and we will unmap the
2127 * actual page and drop map count
2128 * to zero.
2129 */
2130 page_vma_mapped_walk_done(&pvmw);
2131 break;
2132 }
2133 hugetlb_vma_unlock_write(vma);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002134 }
Baolin Wang5d4af612022-05-13 16:48:55 -07002135 /* Nuke the hugetlb page table entry */
2136 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
Baolin Wang54205e92022-05-09 18:20:53 -07002137 } else {
Ryan Robertsc33c7942023-06-12 16:15:45 +01002138 flush_cache_page(vma, address, pfn);
Baolin Wang5d4af612022-05-13 16:48:55 -07002139 /* Nuke the page table entry. */
Huang Ying7e12beb2023-02-13 20:34:43 +08002140 if (should_defer_flush(mm, flags)) {
2141 /*
2142 * We clear the PTE but do not flush so potentially
2143 * a remote CPU could still be writing to the folio.
2144 * If the entry was previously clean then the
2145 * architecture must guarantee that a clear->dirty
2146 * transition on a cached TLB entry is written through
2147 * and traps if the PTE is unmapped.
2148 */
2149 pteval = ptep_get_and_clear(mm, address, pvmw.pte);
2150
Barry Songf73419b2023-07-17 21:10:02 +08002151 set_tlb_ubc_flush_pending(mm, pteval, address);
Huang Ying7e12beb2023-02-13 20:34:43 +08002152 } else {
2153 pteval = ptep_clear_flush(vma, address, pvmw.pte);
2154 }
Alistair Popplea98a2f02021-06-30 18:54:16 -07002155 }
2156
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002157 /* Set the dirty flag on the folio now the pte is gone. */
Alistair Popplea98a2f02021-06-30 18:54:16 -07002158 if (pte_dirty(pteval))
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002159 folio_mark_dirty(folio);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002160
2161 /* Update high watermark before we lower rss */
2162 update_hiwater_rss(mm);
2163
Alex Sierraf25cbb72022-07-15 10:05:10 -05002164 if (folio_is_device_private(folio)) {
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002165 unsigned long pfn = folio_pfn(folio);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002166 swp_entry_t entry;
2167 pte_t swp_pte;
2168
David Hildenbrand6c287602022-05-09 18:20:44 -07002169 if (anon_exclusive)
David Hildenbrande3b4b132023-12-20 23:45:02 +01002170 WARN_ON_ONCE(folio_try_share_anon_rmap_pte(folio,
2171 subpage));
David Hildenbrand6c287602022-05-09 18:20:44 -07002172
Alistair Popplea98a2f02021-06-30 18:54:16 -07002173 /*
2174 * Store the pfn of the page in a special migration
2175 * pte. do_swap_page() will wait until the migration
2176 * pte is removed and then restart fault handling.
2177 */
Alistair Popple3d887052021-11-05 13:45:00 -07002178 entry = pte_to_swp_entry(pteval);
2179 if (is_writable_device_private_entry(entry))
2180 entry = make_writable_migration_entry(pfn);
David Hildenbrand6c287602022-05-09 18:20:44 -07002181 else if (anon_exclusive)
2182 entry = make_readable_exclusive_migration_entry(pfn);
Alistair Popple3d887052021-11-05 13:45:00 -07002183 else
2184 entry = make_readable_migration_entry(pfn);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002185 swp_pte = swp_entry_to_pte(entry);
2186
2187 /*
2188 * pteval maps a zone device page and is therefore
2189 * a swap pte.
2190 */
2191 if (pte_swp_soft_dirty(pteval))
2192 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2193 if (pte_swp_uffd_wp(pteval))
2194 swp_pte = pte_swp_mkuffd_wp(swp_pte);
2195 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
Anshuman Khandual4cc79b32022-03-24 18:10:01 -07002196 trace_set_migration_pte(pvmw.address, pte_val(swp_pte),
Matthew Wilcox (Oracle)059ab7be2024-02-15 20:53:05 +00002197 folio_order(folio));
Alistair Popplea98a2f02021-06-30 18:54:16 -07002198 /*
2199 * No need to invalidate here it will synchronize on
2200 * against the special swap migration pte.
Alistair Popplea98a2f02021-06-30 18:54:16 -07002201 */
Matthew Wilcox (Oracle)da358d52022-03-22 14:46:38 -07002202 } else if (PageHWPoison(subpage)) {
Alistair Popplea98a2f02021-06-30 18:54:16 -07002203 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002204 if (folio_test_hugetlb(folio)) {
2205 hugetlb_count_sub(folio_nr_pages(folio), mm);
Ryan Roberts935d4f02023-09-22 12:58:03 +01002206 set_huge_pte_at(mm, address, pvmw.pte, pteval,
2207 hsz);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002208 } else {
Kefeng Wanga23f5172024-01-11 15:24:28 +00002209 dec_mm_counter(mm, mm_counter(folio));
Alistair Popplea98a2f02021-06-30 18:54:16 -07002210 set_pte_at(mm, address, pvmw.pte, pteval);
2211 }
2212
2213 } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
2214 /*
2215 * The guest indicated that the page content is of no
2216 * interest anymore. Simply discard the pte, vmscan
2217 * will take care of the rest.
2218 * A future reference will then fault in a new zero
2219 * page. When userfaultfd is active, we must not drop
2220 * this page though, as its main user (postcopy
2221 * migration) will not expect userfaults on already
2222 * copied pages.
2223 */
Kefeng Wanga23f5172024-01-11 15:24:28 +00002224 dec_mm_counter(mm, mm_counter(folio));
Alistair Popplea98a2f02021-06-30 18:54:16 -07002225 } else {
2226 swp_entry_t entry;
2227 pte_t swp_pte;
2228
2229 if (arch_unmap_one(mm, vma, address, pteval) < 0) {
Baolin Wang5d4af612022-05-13 16:48:55 -07002230 if (folio_test_hugetlb(folio))
Ryan Roberts935d4f02023-09-22 12:58:03 +01002231 set_huge_pte_at(mm, address, pvmw.pte,
2232 pteval, hsz);
Baolin Wang5d4af612022-05-13 16:48:55 -07002233 else
2234 set_pte_at(mm, address, pvmw.pte, pteval);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002235 ret = false;
2236 page_vma_mapped_walk_done(&pvmw);
2237 break;
2238 }
David Hildenbrand6c287602022-05-09 18:20:44 -07002239 VM_BUG_ON_PAGE(pte_write(pteval) && folio_test_anon(folio) &&
2240 !anon_exclusive, subpage);
David Hildenbrand088b8aa2022-09-01 10:35:59 +02002241
David Hildenbrande3b4b132023-12-20 23:45:02 +01002242 /* See folio_try_share_anon_rmap_pte(): clear PTE first. */
David Hildenbrand0c2ec322023-12-20 23:44:29 +01002243 if (folio_test_hugetlb(folio)) {
2244 if (anon_exclusive &&
2245 hugetlb_try_share_anon_rmap(folio)) {
Ryan Roberts935d4f02023-09-22 12:58:03 +01002246 set_huge_pte_at(mm, address, pvmw.pte,
2247 pteval, hsz);
David Hildenbrand0c2ec322023-12-20 23:44:29 +01002248 ret = false;
2249 page_vma_mapped_walk_done(&pvmw);
2250 break;
2251 }
2252 } else if (anon_exclusive &&
David Hildenbrande3b4b132023-12-20 23:45:02 +01002253 folio_try_share_anon_rmap_pte(folio, subpage)) {
David Hildenbrand0c2ec322023-12-20 23:44:29 +01002254 set_pte_at(mm, address, pvmw.pte, pteval);
David Hildenbrand6c287602022-05-09 18:20:44 -07002255 ret = false;
2256 page_vma_mapped_walk_done(&pvmw);
2257 break;
2258 }
Alistair Popplea98a2f02021-06-30 18:54:16 -07002259
2260 /*
2261 * Store the pfn of the page in a special migration
2262 * pte. do_swap_page() will wait until the migration
2263 * pte is removed and then restart fault handling.
2264 */
2265 if (pte_write(pteval))
2266 entry = make_writable_migration_entry(
2267 page_to_pfn(subpage));
David Hildenbrand6c287602022-05-09 18:20:44 -07002268 else if (anon_exclusive)
2269 entry = make_readable_exclusive_migration_entry(
2270 page_to_pfn(subpage));
Alistair Popplea98a2f02021-06-30 18:54:16 -07002271 else
2272 entry = make_readable_migration_entry(
2273 page_to_pfn(subpage));
Peter Xu2e346872022-08-11 12:13:29 -04002274 if (pte_young(pteval))
2275 entry = make_migration_entry_young(entry);
2276 if (pte_dirty(pteval))
2277 entry = make_migration_entry_dirty(entry);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002278 swp_pte = swp_entry_to_pte(entry);
2279 if (pte_soft_dirty(pteval))
2280 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2281 if (pte_uffd_wp(pteval))
2282 swp_pte = pte_swp_mkuffd_wp(swp_pte);
Baolin Wang5d4af612022-05-13 16:48:55 -07002283 if (folio_test_hugetlb(folio))
Ryan Roberts935d4f02023-09-22 12:58:03 +01002284 set_huge_pte_at(mm, address, pvmw.pte, swp_pte,
2285 hsz);
Baolin Wang5d4af612022-05-13 16:48:55 -07002286 else
2287 set_pte_at(mm, address, pvmw.pte, swp_pte);
Anshuman Khandual4cc79b32022-03-24 18:10:01 -07002288 trace_set_migration_pte(address, pte_val(swp_pte),
Matthew Wilcox (Oracle)059ab7be2024-02-15 20:53:05 +00002289 folio_order(folio));
Alistair Popplea98a2f02021-06-30 18:54:16 -07002290 /*
2291 * No need to invalidate here it will synchronize on
2292 * against the special swap migration pte.
2293 */
2294 }
2295
David Hildenbrande1358262023-12-20 23:44:26 +01002296 if (unlikely(folio_test_hugetlb(folio)))
2297 hugetlb_remove_rmap(folio);
2298 else
David Hildenbrandca1a0742023-12-20 23:44:54 +01002299 folio_remove_rmap_pte(folio, subpage, vma);
Hugh Dickinsb7435502022-02-14 18:38:47 -08002300 if (vma->vm_flags & VM_LOCKED)
Lorenzo Stoakes96f97c42023-01-12 12:39:31 +00002301 mlock_drain_local();
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002302 folio_put(folio);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002303 }
2304
2305 mmu_notifier_invalidate_range_end(&range);
2306
2307 return ret;
2308}
2309
2310/**
2311 * try_to_migrate - try to replace all page table mappings with swap entries
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002312 * @folio: the folio to replace page table entries for
Alistair Popplea98a2f02021-06-30 18:54:16 -07002313 * @flags: action and flags
2314 *
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002315 * Tries to remove all the page table entries which are mapping this folio and
2316 * replace them with special swap entries. Caller must hold the folio lock.
Alistair Popplea98a2f02021-06-30 18:54:16 -07002317 */
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002318void try_to_migrate(struct folio *folio, enum ttu_flags flags)
Alistair Popplea98a2f02021-06-30 18:54:16 -07002319{
2320 struct rmap_walk_control rwc = {
2321 .rmap_one = try_to_migrate_one,
2322 .arg = (void *)flags,
Kefeng Wangf3ad0322022-09-27 14:38:26 +08002323 .done = folio_not_mapped,
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002324 .anon_lock = folio_lock_anon_vma_read,
Alistair Popplea98a2f02021-06-30 18:54:16 -07002325 };
2326
2327 /*
2328 * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
Huang Ying7e12beb2023-02-13 20:34:43 +08002329 * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
Alistair Popplea98a2f02021-06-30 18:54:16 -07002330 */
2331 if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
Huang Ying7e12beb2023-02-13 20:34:43 +08002332 TTU_SYNC | TTU_BATCH_FLUSH)))
Alistair Popplea98a2f02021-06-30 18:54:16 -07002333 return;
2334
Alex Sierraf25cbb72022-07-15 10:05:10 -05002335 if (folio_is_zone_device(folio) &&
2336 (!folio_is_device_private(folio) && !folio_is_device_coherent(folio)))
Hugh Dickins6c855fc2021-07-07 13:13:33 -07002337 return;
2338
Joonsoo Kim52629502014-01-21 15:49:50 -08002339 /*
2340 * During exec, a temporary VMA is setup and later moved.
2341 * The VMA is moved under the anon_vma lock but not the
2342 * page tables leading to a race where migration cannot
2343 * find the migration ptes. Rather than increasing the
2344 * locking requirements of exec(), migration skips
2345 * temporary VMAs until after exec() completes.
2346 */
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002347 if (!folio_test_ksm(folio) && folio_test_anon(folio))
Joonsoo Kim52629502014-01-21 15:49:50 -08002348 rwc.invalid_vma = invalid_migration_vma;
2349
Kirill A. Shutemov2a52bcb2016-03-17 14:20:04 -07002350 if (flags & TTU_RMAP_LOCKED)
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002351 rmap_walk_locked(folio, &rwc);
Kirill A. Shutemov2a52bcb2016-03-17 14:20:04 -07002352 else
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002353 rmap_walk(folio, &rwc);
Nick Pigginb291f002008-10-18 20:26:44 -07002354}
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002355
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002356#ifdef CONFIG_DEVICE_PRIVATE
2357struct make_exclusive_args {
2358 struct mm_struct *mm;
2359 unsigned long address;
2360 void *owner;
2361 bool valid;
2362};
2363
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002364static bool page_make_device_exclusive_one(struct folio *folio,
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002365 struct vm_area_struct *vma, unsigned long address, void *priv)
2366{
2367 struct mm_struct *mm = vma->vm_mm;
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002368 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002369 struct make_exclusive_args *args = priv;
2370 pte_t pteval;
2371 struct page *subpage;
2372 bool ret = true;
2373 struct mmu_notifier_range range;
2374 swp_entry_t entry;
2375 pte_t swp_pte;
Ryan Robertsc33c7942023-06-12 16:15:45 +01002376 pte_t ptent;
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002377
Alistair Popple7d4a8be2023-01-10 13:57:22 +11002378 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002379 vma->vm_mm, address, min(vma->vm_end,
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002380 address + folio_size(folio)),
2381 args->owner);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002382 mmu_notifier_invalidate_range_start(&range);
2383
2384 while (page_vma_mapped_walk(&pvmw)) {
2385 /* Unexpected PMD-mapped THP? */
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002386 VM_BUG_ON_FOLIO(!pvmw.pte, folio);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002387
Ryan Robertsc33c7942023-06-12 16:15:45 +01002388 ptent = ptep_get(pvmw.pte);
2389 if (!pte_present(ptent)) {
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002390 ret = false;
2391 page_vma_mapped_walk_done(&pvmw);
2392 break;
2393 }
2394
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002395 subpage = folio_page(folio,
Ryan Robertsc33c7942023-06-12 16:15:45 +01002396 pte_pfn(ptent) - folio_pfn(folio));
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002397 address = pvmw.address;
2398
2399 /* Nuke the page table entry. */
Ryan Robertsc33c7942023-06-12 16:15:45 +01002400 flush_cache_page(vma, address, pte_pfn(ptent));
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002401 pteval = ptep_clear_flush(vma, address, pvmw.pte);
2402
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002403 /* Set the dirty flag on the folio now the pte is gone. */
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002404 if (pte_dirty(pteval))
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002405 folio_mark_dirty(folio);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002406
2407 /*
2408 * Check that our target page is still mapped at the expected
2409 * address.
2410 */
2411 if (args->mm == mm && args->address == address &&
2412 pte_write(pteval))
2413 args->valid = true;
2414
2415 /*
2416 * Store the pfn of the page in a special migration
2417 * pte. do_swap_page() will wait until the migration
2418 * pte is removed and then restart fault handling.
2419 */
2420 if (pte_write(pteval))
2421 entry = make_writable_device_exclusive_entry(
2422 page_to_pfn(subpage));
2423 else
2424 entry = make_readable_device_exclusive_entry(
2425 page_to_pfn(subpage));
2426 swp_pte = swp_entry_to_pte(entry);
2427 if (pte_soft_dirty(pteval))
2428 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2429 if (pte_uffd_wp(pteval))
2430 swp_pte = pte_swp_mkuffd_wp(swp_pte);
2431
2432 set_pte_at(mm, address, pvmw.pte, swp_pte);
2433
2434 /*
2435 * There is a reference on the page for the swap entry which has
2436 * been removed, so shouldn't take another.
2437 */
David Hildenbrandca1a0742023-12-20 23:44:54 +01002438 folio_remove_rmap_pte(folio, subpage, vma);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002439 }
2440
2441 mmu_notifier_invalidate_range_end(&range);
2442
2443 return ret;
2444}
2445
2446/**
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002447 * folio_make_device_exclusive - Mark the folio exclusively owned by a device.
2448 * @folio: The folio to replace page table entries for.
2449 * @mm: The mm_struct where the folio is expected to be mapped.
2450 * @address: Address where the folio is expected to be mapped.
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002451 * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks
2452 *
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002453 * Tries to remove all the page table entries which are mapping this
2454 * folio and replace them with special device exclusive swap entries to
2455 * grant a device exclusive access to the folio.
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002456 *
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002457 * Context: Caller must hold the folio lock.
2458 * Return: false if the page is still mapped, or if it could not be unmapped
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002459 * from the expected address. Otherwise returns true (success).
2460 */
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002461static bool folio_make_device_exclusive(struct folio *folio,
2462 struct mm_struct *mm, unsigned long address, void *owner)
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002463{
2464 struct make_exclusive_args args = {
2465 .mm = mm,
2466 .address = address,
2467 .owner = owner,
2468 .valid = false,
2469 };
2470 struct rmap_walk_control rwc = {
2471 .rmap_one = page_make_device_exclusive_one,
Kefeng Wangf3ad0322022-09-27 14:38:26 +08002472 .done = folio_not_mapped,
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002473 .anon_lock = folio_lock_anon_vma_read,
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002474 .arg = &args,
2475 };
2476
2477 /*
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002478 * Restrict to anonymous folios for now to avoid potential writeback
2479 * issues.
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002480 */
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002481 if (!folio_test_anon(folio))
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002482 return false;
2483
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002484 rmap_walk(folio, &rwc);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002485
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002486 return args.valid && !folio_mapcount(folio);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002487}
2488
2489/**
2490 * make_device_exclusive_range() - Mark a range for exclusive use by a device
Adrian Huangdd062302022-05-09 18:20:54 -07002491 * @mm: mm_struct of associated target process
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002492 * @start: start of the region to mark for exclusive device access
2493 * @end: end address of region
2494 * @pages: returns the pages which were successfully marked for exclusive access
2495 * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering
2496 *
2497 * Returns: number of pages found in the range by GUP. A page is marked for
2498 * exclusive access only if the page pointer is non-NULL.
2499 *
2500 * This function finds ptes mapping page(s) to the given address range, locks
2501 * them and replaces mappings with special swap entries preventing userspace CPU
2502 * access. On fault these entries are replaced with the original mapping after
2503 * calling MMU notifiers.
2504 *
2505 * A driver using this to program access from a device must use a mmu notifier
2506 * critical section to hold a device specific lock during programming. Once
2507 * programming is complete it should drop the page lock and reference after
2508 * which point CPU access to the page will revoke the exclusive access.
2509 */
2510int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
2511 unsigned long end, struct page **pages,
2512 void *owner)
2513{
2514 long npages = (end - start) >> PAGE_SHIFT;
2515 long i;
2516
2517 npages = get_user_pages_remote(mm, start, npages,
2518 FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD,
Lorenzo Stoakesca5e8632023-05-17 20:25:39 +01002519 pages, NULL);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002520 if (npages < 0)
2521 return npages;
2522
2523 for (i = 0; i < npages; i++, start += PAGE_SIZE) {
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002524 struct folio *folio = page_folio(pages[i]);
2525 if (PageTail(pages[i]) || !folio_trylock(folio)) {
2526 folio_put(folio);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002527 pages[i] = NULL;
2528 continue;
2529 }
2530
Matthew Wilcox (Oracle)0d251482022-01-28 16:03:42 -05002531 if (!folio_make_device_exclusive(folio, mm, start, owner)) {
2532 folio_unlock(folio);
2533 folio_put(folio);
Alistair Poppleb756a3b2021-06-30 18:54:25 -07002534 pages[i] = NULL;
2535 }
2536 }
2537
2538 return npages;
2539}
2540EXPORT_SYMBOL_GPL(make_device_exclusive_range);
2541#endif
2542
Peter Zijlstra01d8b202011-03-22 16:32:49 -07002543void __put_anon_vma(struct anon_vma *anon_vma)
Rik van Riel76545062010-08-09 17:18:41 -07002544{
Peter Zijlstra01d8b202011-03-22 16:32:49 -07002545 struct anon_vma *root = anon_vma->root;
Rik van Riel76545062010-08-09 17:18:41 -07002546
Andrey Ryabinin624483f2014-06-06 19:09:30 +04002547 anon_vma_free(anon_vma);
Peter Zijlstra01d8b202011-03-22 16:32:49 -07002548 if (root != anon_vma && atomic_dec_and_test(&root->refcount))
2549 anon_vma_free(root);
Rik van Riel76545062010-08-09 17:18:41 -07002550}
Rik van Riel76545062010-08-09 17:18:41 -07002551
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002552static struct anon_vma *rmap_walk_anon_lock(struct folio *folio,
Minchan Kim6d4675e2022-05-19 14:08:54 -07002553 struct rmap_walk_control *rwc)
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002554{
2555 struct anon_vma *anon_vma;
2556
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002557 if (rwc->anon_lock)
Minchan Kim6d4675e2022-05-19 14:08:54 -07002558 return rwc->anon_lock(folio, rwc);
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002559
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002560 /*
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002561 * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002562 * because that depends on page_mapped(); but not all its usages
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -07002563 * are holding mmap_lock. Users without mmap_lock are required to
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002564 * take a reference count to prevent the anon_vma disappearing
2565 */
Matthew Wilcox (Oracle)e05b3452022-01-29 11:52:52 -05002566 anon_vma = folio_anon_vma(folio);
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002567 if (!anon_vma)
2568 return NULL;
2569
Minchan Kim6d4675e2022-05-19 14:08:54 -07002570 if (anon_vma_trylock_read(anon_vma))
2571 goto out;
2572
2573 if (rwc->try_lock) {
2574 anon_vma = NULL;
2575 rwc->contended = true;
2576 goto out;
2577 }
2578
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002579 anon_vma_lock_read(anon_vma);
Minchan Kim6d4675e2022-05-19 14:08:54 -07002580out:
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002581 return anon_vma;
2582}
2583
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002584/*
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002585 * rmap_walk_anon - do something to anonymous page using the object-based
2586 * rmap method
Kemeng Shi89be82b2023-07-18 17:21:36 +08002587 * @folio: the folio to be handled
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002588 * @rwc: control variable according to each walk type
Kemeng Shi89be82b2023-07-18 17:21:36 +08002589 * @locked: caller holds relevant rmap lock
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002590 *
Kemeng Shi89be82b2023-07-18 17:21:36 +08002591 * Find all the mappings of a folio using the mapping pointer and the vma
2592 * chains contained in the anon_vma struct it points to.
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002593 */
Matthew Wilcox (Oracle)84fbbe22022-01-29 16:16:54 -05002594static void rmap_walk_anon(struct folio *folio,
Minchan Kim6d4675e2022-05-19 14:08:54 -07002595 struct rmap_walk_control *rwc, bool locked)
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002596{
2597 struct anon_vma *anon_vma;
Kirill A. Shutemova8fa41ad2017-02-24 14:57:54 -08002598 pgoff_t pgoff_start, pgoff_end;
Rik van Riel5beb4932010-03-05 13:42:07 -08002599 struct anon_vma_chain *avc;
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002600
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002601 if (locked) {
Matthew Wilcox (Oracle)e05b3452022-01-29 11:52:52 -05002602 anon_vma = folio_anon_vma(folio);
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002603 /* anon_vma disappear under us? */
Matthew Wilcox (Oracle)e05b3452022-01-29 11:52:52 -05002604 VM_BUG_ON_FOLIO(!anon_vma, folio);
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002605 } else {
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002606 anon_vma = rmap_walk_anon_lock(folio, rwc);
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002607 }
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002608 if (!anon_vma)
Minchan Kim1df631a2017-05-03 14:54:23 -07002609 return;
Joonsoo Kimfaecd8d2014-01-21 15:49:46 -08002610
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002611 pgoff_start = folio_pgoff(folio);
2612 pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
Kirill A. Shutemova8fa41ad2017-02-24 14:57:54 -08002613 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
2614 pgoff_start, pgoff_end) {
Rik van Riel5beb4932010-03-05 13:42:07 -08002615 struct vm_area_struct *vma = avc->vma;
Matthew Wilcox (Oracle)e0abfbb2024-03-28 22:58:29 +00002616 unsigned long address = vma_address(vma, pgoff_start,
2617 folio_nr_pages(folio));
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002618
Hugh Dickins494334e2021-06-15 18:23:56 -07002619 VM_BUG_ON_VMA(address == -EFAULT, vma);
Andrea Arcangeliad126952015-11-05 18:49:07 -08002620 cond_resched();
2621
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002622 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2623 continue;
2624
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002625 if (!rwc->rmap_one(folio, vma, address, rwc->arg))
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002626 break;
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002627 if (rwc->done && rwc->done(folio))
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002628 break;
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002629 }
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002630
2631 if (!locked)
2632 anon_vma_unlock_read(anon_vma);
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002633}
2634
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002635/*
2636 * rmap_walk_file - do something to file page using the object-based rmap method
Kemeng Shi89be82b2023-07-18 17:21:36 +08002637 * @folio: the folio to be handled
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002638 * @rwc: control variable according to each walk type
Kemeng Shi89be82b2023-07-18 17:21:36 +08002639 * @locked: caller holds relevant rmap lock
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002640 *
Kemeng Shi89be82b2023-07-18 17:21:36 +08002641 * Find all the mappings of a folio using the mapping pointer and the vma chains
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002642 * contained in the address_space struct it points to.
Joonsoo Kime8351ac2014-01-21 15:49:52 -08002643 */
Matthew Wilcox (Oracle)84fbbe22022-01-29 16:16:54 -05002644static void rmap_walk_file(struct folio *folio,
Minchan Kim6d4675e2022-05-19 14:08:54 -07002645 struct rmap_walk_control *rwc, bool locked)
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002646{
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002647 struct address_space *mapping = folio_mapping(folio);
Kirill A. Shutemova8fa41ad2017-02-24 14:57:54 -08002648 pgoff_t pgoff_start, pgoff_end;
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002649 struct vm_area_struct *vma;
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002650
Joonsoo Kim9f326242014-01-21 15:49:53 -08002651 /*
2652 * The page lock not only makes sure that page->mapping cannot
2653 * suddenly be NULLified by truncation, it makes sure that the
2654 * structure at mapping cannot be freed and reused yet,
Davidlohr Buesoc8c06ef2014-12-12 16:54:24 -08002655 * so we can safely take mapping->i_mmap_rwsem.
Joonsoo Kim9f326242014-01-21 15:49:53 -08002656 */
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002657 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
Joonsoo Kim9f326242014-01-21 15:49:53 -08002658
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002659 if (!mapping)
Minchan Kim1df631a2017-05-03 14:54:23 -07002660 return;
Davidlohr Bueso3dec0ba2014-12-12 16:54:27 -08002661
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002662 pgoff_start = folio_pgoff(folio);
2663 pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
Minchan Kim6d4675e2022-05-19 14:08:54 -07002664 if (!locked) {
2665 if (i_mmap_trylock_read(mapping))
2666 goto lookup;
2667
2668 if (rwc->try_lock) {
2669 rwc->contended = true;
2670 return;
2671 }
2672
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002673 i_mmap_lock_read(mapping);
Minchan Kim6d4675e2022-05-19 14:08:54 -07002674 }
2675lookup:
Kirill A. Shutemova8fa41ad2017-02-24 14:57:54 -08002676 vma_interval_tree_foreach(vma, &mapping->i_mmap,
2677 pgoff_start, pgoff_end) {
Matthew Wilcox (Oracle)e0abfbb2024-03-28 22:58:29 +00002678 unsigned long address = vma_address(vma, pgoff_start,
2679 folio_nr_pages(folio));
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002680
Hugh Dickins494334e2021-06-15 18:23:56 -07002681 VM_BUG_ON_VMA(address == -EFAULT, vma);
Andrea Arcangeliad126952015-11-05 18:49:07 -08002682 cond_resched();
2683
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002684 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2685 continue;
2686
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002687 if (!rwc->rmap_one(folio, vma, address, rwc->arg))
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002688 goto done;
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002689 if (rwc->done && rwc->done(folio))
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002690 goto done;
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002691 }
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002692
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -08002693done:
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002694 if (!locked)
2695 i_mmap_unlock_read(mapping);
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002696}
2697
Minchan Kim6d4675e2022-05-19 14:08:54 -07002698void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc)
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002699{
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002700 if (unlikely(folio_test_ksm(folio)))
2701 rmap_walk_ksm(folio, rwc);
2702 else if (folio_test_anon(folio))
2703 rmap_walk_anon(folio, rwc, false);
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002704 else
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002705 rmap_walk_file(folio, rwc, false);
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002706}
2707
2708/* Like rmap_walk, but caller holds relevant rmap lock */
Minchan Kim6d4675e2022-05-19 14:08:54 -07002709void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002710{
2711 /* no ksm support for now */
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002712 VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
2713 if (folio_test_anon(folio))
2714 rmap_walk_anon(folio, rwc, true);
Kirill A. Shutemovb9773192016-03-17 14:20:01 -07002715 else
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002716 rmap_walk_file(folio, rwc, true);
Hugh Dickinse9995ef2009-12-14 17:59:31 -08002717}
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002718
Naoya Horiguchie3390f62010-06-15 13:18:13 +09002719#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002720/*
Kirill Tkhai451b9512018-12-28 00:39:31 -08002721 * The following two functions are for anonymous (private mapped) hugepages.
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002722 * Unlike common anonymous pages, anonymous hugepages have no accounting code
2723 * and no lru code, because we handle hugepages differently from common pages.
2724 */
David Hildenbrand9d5fafd2023-12-20 23:44:25 +01002725void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
2726 unsigned long address, rmap_t flags)
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002727{
David Hildenbranda4ea1862023-12-20 23:44:30 +01002728 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
David Hildenbrandc5c54002023-09-13 14:51:10 +02002729 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
2730
David Hildenbrand132b1802023-09-13 14:51:12 +02002731 atomic_inc(&folio->_entire_mapcount);
David Hildenbrand05c53232024-04-09 21:22:47 +02002732 atomic_inc(&folio->_large_mapcount);
David Hildenbrandc66db8c2023-09-13 14:51:09 +02002733 if (flags & RMAP_EXCLUSIVE)
David Hildenbrand09c55052023-09-13 14:51:13 +02002734 SetPageAnonExclusive(&folio->page);
David Hildenbrand132b1802023-09-13 14:51:12 +02002735 VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 &&
David Hildenbrand09c55052023-09-13 14:51:13 +02002736 PageAnonExclusive(&folio->page), folio);
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002737}
2738
David Hildenbrand9d5fafd2023-12-20 23:44:25 +01002739void hugetlb_add_new_anon_rmap(struct folio *folio,
2740 struct vm_area_struct *vma, unsigned long address)
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002741{
David Hildenbranda4ea1862023-12-20 23:44:30 +01002742 VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
2743
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002744 BUG_ON(address < vma->vm_start || address >= vma->vm_end);
Hugh Dickinscb67f422022-11-02 18:51:38 -07002745 /* increment count (starts at -1) */
Matthew Wilcox (Oracle)db4e5db2023-01-11 14:28:56 +00002746 atomic_set(&folio->_entire_mapcount, 0);
David Hildenbrand05c53232024-04-09 21:22:47 +02002747 atomic_set(&folio->_large_mapcount, 0);
Matthew Wilcox (Oracle)db4e5db2023-01-11 14:28:56 +00002748 folio_clear_hugetlb_restore_reserve(folio);
David Hildenbrandc66db8c2023-09-13 14:51:09 +02002749 __folio_set_anon(folio, vma, address, true);
2750 SetPageAnonExclusive(&folio->page);
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002751}
Naoya Horiguchie3390f62010-06-15 13:18:13 +09002752#endif /* CONFIG_HUGETLB_PAGE */