blob: 624671aaa60d0e0fdd551d7d14a039ecc9e2af7b [file] [log] [blame]
Thomas Gleixner20c8ccb2019-06-04 10:11:32 +02001// SPDX-License-Identifier: GPL-2.0-only
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08002/*
3 * Copyright (C) 2009 Red Hat, Inc.
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08004 */
5
Andrew Mortonae3a8c12014-06-04 16:06:58 -07006#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08008#include <linux/mm.h>
9#include <linux/sched.h>
Zi Yanfa6c0232021-05-04 18:34:23 -070010#include <linux/sched/mm.h>
Ingo Molnarf7ccbae2017-02-08 18:51:30 +010011#include <linux/sched/coredump.h>
Ingo Molnar6a3827d2017-02-08 18:51:31 +010012#include <linux/sched/numa_balancing.h>
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080013#include <linux/highmem.h>
14#include <linux/hugetlb.h>
15#include <linux/mmu_notifier.h>
16#include <linux/rmap.h>
17#include <linux/swap.h>
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -080018#include <linux/shrinker.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080019#include <linux/mm_inline.h>
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -080020#include <linux/swapops.h>
Matthew Wilcox (Oracle)fb5c2022022-06-28 20:15:29 -040021#include <linux/backing-dev.h>
Matthew Wilcox4897c762015-09-08 14:58:45 -070022#include <linux/dax.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080023#include <linux/khugepaged.h>
Andrea Arcangeli878aee72011-01-13 15:47:10 -080024#include <linux/freezer.h>
Dan Williamsf25748e32016-01-15 16:56:43 -080025#include <linux/pfn_t.h>
Andrea Arcangelia664b2d2011-01-13 15:47:17 -080026#include <linux/mman.h>
Dan Williams3565fce2016-01-15 16:56:55 -080027#include <linux/memremap.h>
Ralf Baechle325adeb2012-10-15 13:44:56 +020028#include <linux/pagemap.h>
Kirill A. Shutemov49071d42016-01-15 16:54:40 -080029#include <linux/debugfs.h>
Mel Gorman4daae3b2012-11-02 11:33:45 +000030#include <linux/migrate.h>
Sasha Levin43b5fbb2013-02-22 16:32:27 -080031#include <linux/hashtable.h>
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -070032#include <linux/userfaultfd_k.h>
Vladimir Davydov33c3fc72015-09-09 15:35:45 -070033#include <linux/page_idle.h>
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -070034#include <linux/shmem_fs.h>
Michal Hocko6b31d592017-08-18 15:16:15 -070035#include <linux/oom.h>
Anshuman Khandual98fa15f2019-03-05 15:42:58 -080036#include <linux/numa.h>
Vlastimil Babkaf7da6772019-08-24 17:54:59 -070037#include <linux/page_owner.h>
Huang Yinga1a3a2f2022-03-22 14:46:27 -070038#include <linux/sched/sysctl.h>
Aneesh Kumar K.V467b1712022-08-18 18:40:41 +053039#include <linux/memory-tiers.h>
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -080040
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080041#include <asm/tlb.h>
42#include <asm/pgalloc.h>
43#include "internal.h"
NeilBrown014bb1d2022-05-09 18:20:47 -070044#include "swap.h"
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080045
Anshuman Khandual283fd6f2022-03-24 18:09:58 -070046#define CREATE_TRACE_POINTS
47#include <trace/events/thp.h>
48
Andrea Arcangeliba761492011-01-13 15:46:58 -080049/*
Michael DeGuzisb14d5952017-05-17 15:19:21 -040050 * By default, transparent hugepage support is disabled in order to avoid
51 * risking an increased memory footprint for applications that are not
52 * guaranteed to benefit from it. When transparent hugepage support is
53 * enabled, it is for all mappings, and khugepaged scans all mappings.
Jianguo Wu8bfa3f92013-11-12 15:07:16 -080054 * Defrag is invoked by khugepaged hugepage allocations and by page faults
55 * for all hugepage allocations.
Andrea Arcangeliba761492011-01-13 15:46:58 -080056 */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080057unsigned long transparent_hugepage_flags __read_mostly =
Andrea Arcangeli13ece882011-01-13 15:47:07 -080058#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
Andrea Arcangeliba761492011-01-13 15:46:58 -080059 (1<<TRANSPARENT_HUGEPAGE_FLAG)|
Andrea Arcangeli13ece882011-01-13 15:47:07 -080060#endif
61#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
62 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
63#endif
Mel Gorman444eb2a42016-03-17 14:19:23 -070064 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
Kirill A. Shutemov79da5402012-12-12 13:51:12 -080065 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
66 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
Andrea Arcangeliba761492011-01-13 15:46:58 -080067
Kirill A. Shutemov9a982252016-01-15 16:54:17 -080068static struct shrinker deferred_split_shrinker;
Andrea Arcangelif0005652011-01-13 15:47:04 -080069
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -080070static atomic_t huge_zero_refcount;
Wang, Yalin56873f42015-02-11 15:24:51 -080071struct page *huge_zero_page __read_mostly;
Hugh Dickins3b77e8c2021-06-15 18:23:49 -070072unsigned long huge_zero_pfn __read_mostly = ~0UL;
Kirill A. Shutemov4a6c1292012-12-12 13:50:47 -080073
Zach O'Keefea7f4e6e2022-07-06 16:59:25 -070074bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags,
75 bool smaps, bool in_pf, bool enforce_sysfs)
Michal Hocko7635d9c2018-12-28 00:38:21 -080076{
Yang Shi9fec5162022-06-16 10:48:37 -070077 if (!vma->vm_mm) /* vdso */
Yang Shic0630662019-07-18 15:57:27 -070078 return false;
Yang Shi9fec5162022-06-16 10:48:37 -070079
Yang Shi7da4e2c2022-06-16 10:48:38 -070080 /*
81 * Explicitly disabled through madvise or prctl, or some
82 * architectures may disable THP for some mappings, for
83 * example, s390 kvm.
84 * */
85 if ((vm_flags & VM_NOHUGEPAGE) ||
86 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
87 return false;
88 /*
89 * If the hardware/firmware marked hugepage support disabled.
90 */
Peter Xu3c556d22023-03-15 13:16:42 -040091 if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED))
Yang Shi9fec5162022-06-16 10:48:37 -070092 return false;
93
Yang Shi7da4e2c2022-06-16 10:48:38 -070094 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */
95 if (vma_is_dax(vma))
96 return in_pf;
97
98 /*
99 * Special VMA and hugetlb VMA.
100 * Must be checked after dax since some dax mappings may have
101 * VM_MIXEDMAP set.
102 */
Yang Shi9fec5162022-06-16 10:48:37 -0700103 if (vm_flags & VM_NO_KHUGEPAGED)
104 return false;
105
Yang Shi7da4e2c2022-06-16 10:48:38 -0700106 /*
107 * Check alignment for file vma and size for both file and anon vma.
108 *
109 * Skip the check for page fault. Huge fault does the check in fault
110 * handlers. And this check is not suitable for huge PUD fault.
111 */
112 if (!in_pf &&
113 !transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE)))
Yang Shi9fec5162022-06-16 10:48:37 -0700114 return false;
115
Yang Shi7da4e2c2022-06-16 10:48:38 -0700116 /*
117 * Enabled via shmem mount options or sysfs settings.
118 * Must be done before hugepage flags check since shmem has its
119 * own flags.
120 */
121 if (!in_pf && shmem_file(vma->vm_file))
David Stevens2cf13382023-01-13 11:30:11 +0900122 return shmem_is_huge(file_inode(vma->vm_file), vma->vm_pgoff,
123 !enforce_sysfs, vma->vm_mm, vm_flags);
Yang Shi9fec5162022-06-16 10:48:37 -0700124
Zach O'Keefea7f4e6e2022-07-06 16:59:25 -0700125 /* Enforce sysfs THP requirements as necessary */
126 if (enforce_sysfs &&
127 (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
128 !hugepage_flags_always())))
Yang Shi9fec5162022-06-16 10:48:37 -0700129 return false;
130
131 /* Only regular file is valid */
Yang Shi7da4e2c2022-06-16 10:48:38 -0700132 if (!in_pf && file_thp_enabled(vma))
Yang Shi78d12c12022-05-19 14:08:49 -0700133 return true;
Michal Hocko7635d9c2018-12-28 00:38:21 -0800134
Yang Shi9fec5162022-06-16 10:48:37 -0700135 if (!vma_is_anonymous(vma))
136 return false;
137
138 if (vma_is_temporary_stack(vma))
139 return false;
140
141 /*
142 * THPeligible bit of smaps should show 1 for proper VMAs even
143 * though anon_vma is not initialized yet.
Yang Shi7da4e2c2022-06-16 10:48:38 -0700144 *
145 * Allow page fault since anon_vma may be not initialized until
146 * the first page fault.
Yang Shi9fec5162022-06-16 10:48:37 -0700147 */
148 if (!vma->anon_vma)
Yang Shi7da4e2c2022-06-16 10:48:38 -0700149 return (smaps || in_pf);
Yang Shi9fec5162022-06-16 10:48:37 -0700150
151 return true;
Michal Hocko7635d9c2018-12-28 00:38:21 -0800152}
153
Miaohe Linaaa97052021-05-04 18:33:55 -0700154static bool get_huge_zero_page(void)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800155{
156 struct page *zero_page;
157retry:
158 if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
Miaohe Linaaa97052021-05-04 18:33:55 -0700159 return true;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800160
161 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
162 HPAGE_PMD_ORDER);
Kirill A. Shutemovd8a8e1f2012-12-12 13:51:09 -0800163 if (!zero_page) {
164 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
Miaohe Linaaa97052021-05-04 18:33:55 -0700165 return false;
Kirill A. Shutemovd8a8e1f2012-12-12 13:51:09 -0800166 }
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800167 preempt_disable();
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700168 if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800169 preempt_enable();
Yu Zhao5ddacbe2014-10-29 14:50:26 -0700170 __free_pages(zero_page, compound_order(zero_page));
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800171 goto retry;
172 }
Hugh Dickins3b77e8c2021-06-15 18:23:49 -0700173 WRITE_ONCE(huge_zero_pfn, page_to_pfn(zero_page));
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800174
175 /* We take additional reference here. It will be put back by shrinker */
176 atomic_set(&huge_zero_refcount, 2);
177 preempt_enable();
Liu Shixinf4981502022-09-09 10:16:53 +0800178 count_vm_event(THP_ZERO_PAGE_ALLOC);
Miaohe Linaaa97052021-05-04 18:33:55 -0700179 return true;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800180}
181
Aaron Lu6fcb52a2016-10-07 17:00:08 -0700182static void put_huge_zero_page(void)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800183{
184 /*
185 * Counter should never go to zero here. Only shrinker can put
186 * last reference.
187 */
188 BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
189}
190
Aaron Lu6fcb52a2016-10-07 17:00:08 -0700191struct page *mm_get_huge_zero_page(struct mm_struct *mm)
192{
193 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
194 return READ_ONCE(huge_zero_page);
195
196 if (!get_huge_zero_page())
197 return NULL;
198
199 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
200 put_huge_zero_page();
201
202 return READ_ONCE(huge_zero_page);
203}
204
205void mm_put_huge_zero_page(struct mm_struct *mm)
206{
207 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
208 put_huge_zero_page();
209}
210
Glauber Costa48896462013-08-28 10:18:15 +1000211static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
212 struct shrink_control *sc)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800213{
Glauber Costa48896462013-08-28 10:18:15 +1000214 /* we can free zero page only if last reference remains */
215 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
216}
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800217
Glauber Costa48896462013-08-28 10:18:15 +1000218static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
219 struct shrink_control *sc)
220{
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800221 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700222 struct page *zero_page = xchg(&huge_zero_page, NULL);
223 BUG_ON(zero_page == NULL);
Hugh Dickins3b77e8c2021-06-15 18:23:49 -0700224 WRITE_ONCE(huge_zero_pfn, ~0UL);
Yu Zhao5ddacbe2014-10-29 14:50:26 -0700225 __free_pages(zero_page, compound_order(zero_page));
Glauber Costa48896462013-08-28 10:18:15 +1000226 return HPAGE_PMD_NR;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800227 }
228
229 return 0;
230}
231
232static struct shrinker huge_zero_page_shrinker = {
Glauber Costa48896462013-08-28 10:18:15 +1000233 .count_objects = shrink_huge_zero_page_count,
234 .scan_objects = shrink_huge_zero_page_scan,
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800235 .seeks = DEFAULT_SEEKS,
236};
237
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800238#ifdef CONFIG_SYSFS
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800239static ssize_t enabled_show(struct kobject *kobj,
240 struct kobj_attribute *attr, char *buf)
241{
Joe Perchesbfb0ffe2020-12-14 19:14:46 -0800242 const char *output;
243
Mel Gorman444eb2a42016-03-17 14:19:23 -0700244 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
Joe Perchesbfb0ffe2020-12-14 19:14:46 -0800245 output = "[always] madvise never";
246 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
247 &transparent_hugepage_flags))
248 output = "always [madvise] never";
Mel Gorman444eb2a42016-03-17 14:19:23 -0700249 else
Joe Perchesbfb0ffe2020-12-14 19:14:46 -0800250 output = "always madvise [never]";
251
252 return sysfs_emit(buf, "%s\n", output);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800253}
Mel Gorman444eb2a42016-03-17 14:19:23 -0700254
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800255static ssize_t enabled_store(struct kobject *kobj,
256 struct kobj_attribute *attr,
257 const char *buf, size_t count)
258{
David Rientjes21440d72017-02-22 15:45:49 -0800259 ssize_t ret = count;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800260
David Rientjesf42f2552020-01-30 22:14:48 -0800261 if (sysfs_streq(buf, "always")) {
David Rientjes21440d72017-02-22 15:45:49 -0800262 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
263 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
David Rientjesf42f2552020-01-30 22:14:48 -0800264 } else if (sysfs_streq(buf, "madvise")) {
David Rientjes21440d72017-02-22 15:45:49 -0800265 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
266 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
David Rientjesf42f2552020-01-30 22:14:48 -0800267 } else if (sysfs_streq(buf, "never")) {
David Rientjes21440d72017-02-22 15:45:49 -0800268 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
269 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
270 } else
271 ret = -EINVAL;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800272
273 if (ret > 0) {
Kirill A. Shutemovb46e7562016-07-26 15:26:24 -0700274 int err = start_stop_khugepaged();
Andrea Arcangeliba761492011-01-13 15:46:58 -0800275 if (err)
276 ret = err;
277 }
Andrea Arcangeliba761492011-01-13 15:46:58 -0800278 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800279}
Miaohe Lin37139bb2022-07-04 21:21:53 +0800280
281static struct kobj_attribute enabled_attr = __ATTR_RW(enabled);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800282
Kirill A. Shutemovb46e7562016-07-26 15:26:24 -0700283ssize_t single_hugepage_flag_show(struct kobject *kobj,
Joe Perchesbfb0ffe2020-12-14 19:14:46 -0800284 struct kobj_attribute *attr, char *buf,
285 enum transparent_hugepage_flag flag)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800286{
Joe Perchesbfb0ffe2020-12-14 19:14:46 -0800287 return sysfs_emit(buf, "%d\n",
288 !!test_bit(flag, &transparent_hugepage_flags));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800289}
Ben Hutchingse27e6152011-04-14 15:22:21 -0700290
Kirill A. Shutemovb46e7562016-07-26 15:26:24 -0700291ssize_t single_hugepage_flag_store(struct kobject *kobj,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800292 struct kobj_attribute *attr,
293 const char *buf, size_t count,
294 enum transparent_hugepage_flag flag)
295{
Ben Hutchingse27e6152011-04-14 15:22:21 -0700296 unsigned long value;
297 int ret;
298
299 ret = kstrtoul(buf, 10, &value);
300 if (ret < 0)
301 return ret;
302 if (value > 1)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800303 return -EINVAL;
304
Ben Hutchingse27e6152011-04-14 15:22:21 -0700305 if (value)
306 set_bit(flag, &transparent_hugepage_flags);
307 else
308 clear_bit(flag, &transparent_hugepage_flags);
309
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800310 return count;
311}
312
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800313static ssize_t defrag_show(struct kobject *kobj,
314 struct kobj_attribute *attr, char *buf)
315{
Joe Perchesbfb0ffe2020-12-14 19:14:46 -0800316 const char *output;
317
318 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
319 &transparent_hugepage_flags))
320 output = "[always] defer defer+madvise madvise never";
321 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
322 &transparent_hugepage_flags))
323 output = "always [defer] defer+madvise madvise never";
324 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
325 &transparent_hugepage_flags))
326 output = "always defer [defer+madvise] madvise never";
327 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
328 &transparent_hugepage_flags))
329 output = "always defer defer+madvise [madvise] never";
330 else
331 output = "always defer defer+madvise madvise [never]";
332
333 return sysfs_emit(buf, "%s\n", output);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800334}
David Rientjes21440d72017-02-22 15:45:49 -0800335
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800336static ssize_t defrag_store(struct kobject *kobj,
337 struct kobj_attribute *attr,
338 const char *buf, size_t count)
339{
David Rientjesf42f2552020-01-30 22:14:48 -0800340 if (sysfs_streq(buf, "always")) {
David Rientjes21440d72017-02-22 15:45:49 -0800341 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
342 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
343 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
344 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
David Rientjesf42f2552020-01-30 22:14:48 -0800345 } else if (sysfs_streq(buf, "defer+madvise")) {
David Rientjes21440d72017-02-22 15:45:49 -0800346 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
347 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
348 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
349 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
David Rientjesf42f2552020-01-30 22:14:48 -0800350 } else if (sysfs_streq(buf, "defer")) {
David Rientjes4fad7fb2017-04-07 16:04:54 -0700351 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
352 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
353 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
354 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
David Rientjesf42f2552020-01-30 22:14:48 -0800355 } else if (sysfs_streq(buf, "madvise")) {
David Rientjes21440d72017-02-22 15:45:49 -0800356 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
357 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
358 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
359 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
David Rientjesf42f2552020-01-30 22:14:48 -0800360 } else if (sysfs_streq(buf, "never")) {
David Rientjes21440d72017-02-22 15:45:49 -0800361 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
362 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
363 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
364 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
365 } else
366 return -EINVAL;
367
368 return count;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800369}
Miaohe Lin37139bb2022-07-04 21:21:53 +0800370static struct kobj_attribute defrag_attr = __ATTR_RW(defrag);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800371
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800372static ssize_t use_zero_page_show(struct kobject *kobj,
Joe Perchesae7a9272020-12-14 19:14:42 -0800373 struct kobj_attribute *attr, char *buf)
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800374{
Kirill A. Shutemovb46e7562016-07-26 15:26:24 -0700375 return single_hugepage_flag_show(kobj, attr, buf,
Joe Perchesae7a9272020-12-14 19:14:42 -0800376 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800377}
378static ssize_t use_zero_page_store(struct kobject *kobj,
379 struct kobj_attribute *attr, const char *buf, size_t count)
380{
Kirill A. Shutemovb46e7562016-07-26 15:26:24 -0700381 return single_hugepage_flag_store(kobj, attr, buf, count,
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800382 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
383}
Miaohe Lin37139bb2022-07-04 21:21:53 +0800384static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page);
Hugh Dickins49920d22016-12-12 16:44:50 -0800385
386static ssize_t hpage_pmd_size_show(struct kobject *kobj,
Joe Perchesae7a9272020-12-14 19:14:42 -0800387 struct kobj_attribute *attr, char *buf)
Hugh Dickins49920d22016-12-12 16:44:50 -0800388{
Joe Perchesae7a9272020-12-14 19:14:42 -0800389 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE);
Hugh Dickins49920d22016-12-12 16:44:50 -0800390}
391static struct kobj_attribute hpage_pmd_size_attr =
392 __ATTR_RO(hpage_pmd_size);
393
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800394static struct attribute *hugepage_attr[] = {
395 &enabled_attr.attr,
396 &defrag_attr.attr,
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800397 &use_zero_page_attr.attr,
Hugh Dickins49920d22016-12-12 16:44:50 -0800398 &hpage_pmd_size_attr.attr,
Matthew Wilcox (Oracle)396bcc52020-04-06 20:04:35 -0700399#ifdef CONFIG_SHMEM
Kirill A. Shutemov5a6e75f2016-07-26 15:26:13 -0700400 &shmem_enabled_attr.attr,
401#endif
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800402 NULL,
403};
404
Arvind Yadav8aa95a22017-09-06 16:22:03 -0700405static const struct attribute_group hugepage_attr_group = {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800406 .attrs = hugepage_attr,
Andrea Arcangeliba761492011-01-13 15:46:58 -0800407};
408
Shaohua Li569e5592012-01-12 17:19:11 -0800409static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
410{
411 int err;
412
413 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
414 if (unlikely(!*hugepage_kobj)) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700415 pr_err("failed to create transparent hugepage kobject\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800416 return -ENOMEM;
417 }
418
419 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
420 if (err) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700421 pr_err("failed to register transparent hugepage group\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800422 goto delete_obj;
423 }
424
425 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
426 if (err) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700427 pr_err("failed to register transparent hugepage group\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800428 goto remove_hp_group;
429 }
430
431 return 0;
432
433remove_hp_group:
434 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
435delete_obj:
436 kobject_put(*hugepage_kobj);
437 return err;
438}
439
440static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
441{
442 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
443 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
444 kobject_put(hugepage_kobj);
445}
446#else
447static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
448{
449 return 0;
450}
451
452static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
453{
454}
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800455#endif /* CONFIG_SYSFS */
456
457static int __init hugepage_init(void)
458{
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800459 int err;
Shaohua Li569e5592012-01-12 17:19:11 -0800460 struct kobject *hugepage_kobj;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800461
Andrea Arcangeli4b7167b2011-01-13 15:47:09 -0800462 if (!has_transparent_hugepage()) {
Peter Xu3c556d22023-03-15 13:16:42 -0400463 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED;
Shaohua Li569e5592012-01-12 17:19:11 -0800464 return -EINVAL;
Andrea Arcangeli4b7167b2011-01-13 15:47:09 -0800465 }
466
Kirill A. Shutemovff20c2e2016-03-01 09:45:14 +0530467 /*
468 * hugepages can't be allocated by the buddy allocator
469 */
Kirill A. Shutemov23baf832023-03-15 14:31:33 +0300470 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_ORDER);
Kirill A. Shutemovff20c2e2016-03-01 09:45:14 +0530471 /*
472 * we use page->mapping and page->index in second tail page
473 * as list_head: assuming THP order >= 2
474 */
475 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
476
Shaohua Li569e5592012-01-12 17:19:11 -0800477 err = hugepage_init_sysfs(&hugepage_kobj);
478 if (err)
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700479 goto err_sysfs;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800480
Kirill A. Shutemovb46e7562016-07-26 15:26:24 -0700481 err = khugepaged_init();
Andrea Arcangeliba761492011-01-13 15:46:58 -0800482 if (err)
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700483 goto err_slab;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800484
Roman Gushchine33c2672022-05-31 20:22:24 -0700485 err = register_shrinker(&huge_zero_page_shrinker, "thp-zero");
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700486 if (err)
487 goto err_hzp_shrinker;
Roman Gushchine33c2672022-05-31 20:22:24 -0700488 err = register_shrinker(&deferred_split_shrinker, "thp-deferred_split");
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800489 if (err)
490 goto err_split_shrinker;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800491
Rik van Riel97562cd2011-01-13 15:47:12 -0800492 /*
493 * By default disable transparent hugepages on smaller systems,
494 * where the extra memory used could hurt more than TLB overhead
495 * is likely to save. The admin can still enable it through /sys.
496 */
Arun KSca79b0c2018-12-28 00:34:29 -0800497 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) {
Rik van Riel97562cd2011-01-13 15:47:12 -0800498 transparent_hugepage_flags = 0;
Kirill A. Shutemov79553da2932015-04-15 16:14:56 -0700499 return 0;
500 }
Rik van Riel97562cd2011-01-13 15:47:12 -0800501
Kirill A. Shutemov79553da2932015-04-15 16:14:56 -0700502 err = start_stop_khugepaged();
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700503 if (err)
504 goto err_khugepaged;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800505
Shaohua Li569e5592012-01-12 17:19:11 -0800506 return 0;
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700507err_khugepaged:
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800508 unregister_shrinker(&deferred_split_shrinker);
509err_split_shrinker:
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700510 unregister_shrinker(&huge_zero_page_shrinker);
511err_hzp_shrinker:
Kirill A. Shutemovb46e7562016-07-26 15:26:24 -0700512 khugepaged_destroy();
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700513err_slab:
Shaohua Li569e5592012-01-12 17:19:11 -0800514 hugepage_exit_sysfs(hugepage_kobj);
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700515err_sysfs:
Andrea Arcangeliba761492011-01-13 15:46:58 -0800516 return err;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800517}
Paul Gortmakera64fb3c2014-01-23 15:53:30 -0800518subsys_initcall(hugepage_init);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800519
520static int __init setup_transparent_hugepage(char *str)
521{
522 int ret = 0;
523 if (!str)
524 goto out;
525 if (!strcmp(str, "always")) {
526 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
527 &transparent_hugepage_flags);
528 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
529 &transparent_hugepage_flags);
530 ret = 1;
531 } else if (!strcmp(str, "madvise")) {
532 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
533 &transparent_hugepage_flags);
534 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
535 &transparent_hugepage_flags);
536 ret = 1;
537 } else if (!strcmp(str, "never")) {
538 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
539 &transparent_hugepage_flags);
540 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
541 &transparent_hugepage_flags);
542 ret = 1;
543 }
544out:
545 if (!ret)
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700546 pr_warn("transparent_hugepage= cannot parse, ignored\n");
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800547 return ret;
548}
549__setup("transparent_hugepage=", setup_transparent_hugepage);
550
Linus Torvaldsf55e1012017-11-29 09:01:01 -0800551pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800552{
Linus Torvaldsf55e1012017-11-29 09:01:01 -0800553 if (likely(vma->vm_flags & VM_WRITE))
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800554 pmd = pmd_mkwrite(pmd);
555 return pmd;
556}
557
Yang Shi87eaceb2019-09-23 15:38:15 -0700558#ifdef CONFIG_MEMCG
Matthew Wilcox (Oracle)f8baa6b2023-01-11 14:29:12 +0000559static inline
560struct deferred_split *get_deferred_split_queue(struct folio *folio)
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800561{
Matthew Wilcox (Oracle)f8baa6b2023-01-11 14:29:12 +0000562 struct mem_cgroup *memcg = folio_memcg(folio);
563 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));
Yang Shi87eaceb2019-09-23 15:38:15 -0700564
565 if (memcg)
566 return &memcg->deferred_split_queue;
567 else
568 return &pgdat->deferred_split_queue;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800569}
Yang Shi87eaceb2019-09-23 15:38:15 -0700570#else
Matthew Wilcox (Oracle)f8baa6b2023-01-11 14:29:12 +0000571static inline
572struct deferred_split *get_deferred_split_queue(struct folio *folio)
Yang Shi87eaceb2019-09-23 15:38:15 -0700573{
Matthew Wilcox (Oracle)f8baa6b2023-01-11 14:29:12 +0000574 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));
Yang Shi87eaceb2019-09-23 15:38:15 -0700575
576 return &pgdat->deferred_split_queue;
577}
578#endif
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800579
580void prep_transhuge_page(struct page *page)
581{
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +0000582 struct folio *folio = (struct folio *)page;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800583
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +0000584 VM_BUG_ON_FOLIO(folio_order(folio) < 2, folio);
585 INIT_LIST_HEAD(&folio->_deferred_list);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800586 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
587}
588
Miaohe Lin562beb72022-03-24 18:13:27 -0700589static inline bool is_transparent_hugepage(struct page *page)
Sean Christopherson005ba372020-01-08 12:24:36 -0800590{
Matthew Wilcox (Oracle)f04029f2023-01-11 14:29:05 +0000591 struct folio *folio;
592
Sean Christopherson005ba372020-01-08 12:24:36 -0800593 if (!PageCompound(page))
Zou Weifa1f68c2020-06-04 16:49:46 -0700594 return false;
Sean Christopherson005ba372020-01-08 12:24:36 -0800595
Matthew Wilcox (Oracle)f04029f2023-01-11 14:29:05 +0000596 folio = page_folio(page);
597 return is_huge_zero_page(&folio->page) ||
598 folio->_folio_dtor == TRANSHUGE_PAGE_DTOR;
Sean Christopherson005ba372020-01-08 12:24:36 -0800599}
Sean Christopherson005ba372020-01-08 12:24:36 -0800600
Kirill A. Shutemov97d3d0f2020-01-13 16:29:10 -0800601static unsigned long __thp_get_unmapped_area(struct file *filp,
602 unsigned long addr, unsigned long len,
Toshi Kani74d2fad2016-10-07 16:59:56 -0700603 loff_t off, unsigned long flags, unsigned long size)
604{
Toshi Kani74d2fad2016-10-07 16:59:56 -0700605 loff_t off_end = off + len;
606 loff_t off_align = round_up(off, size);
Kirill A. Shutemov97d3d0f2020-01-13 16:29:10 -0800607 unsigned long len_pad, ret;
Toshi Kani74d2fad2016-10-07 16:59:56 -0700608
609 if (off_end <= off_align || (off_end - off_align) < size)
610 return 0;
611
612 len_pad = len + size;
613 if (len_pad < len || (off + len_pad) < off)
614 return 0;
615
Kirill A. Shutemov97d3d0f2020-01-13 16:29:10 -0800616 ret = current->mm->get_unmapped_area(filp, addr, len_pad,
Toshi Kani74d2fad2016-10-07 16:59:56 -0700617 off >> PAGE_SHIFT, flags);
Kirill A. Shutemov97d3d0f2020-01-13 16:29:10 -0800618
619 /*
620 * The failure might be due to length padding. The caller will retry
621 * without the padding.
622 */
623 if (IS_ERR_VALUE(ret))
Toshi Kani74d2fad2016-10-07 16:59:56 -0700624 return 0;
625
Kirill A. Shutemov97d3d0f2020-01-13 16:29:10 -0800626 /*
627 * Do not try to align to THP boundary if allocation at the address
628 * hint succeeds.
629 */
630 if (ret == addr)
631 return addr;
632
633 ret += (off - ret) & (size - 1);
634 return ret;
Toshi Kani74d2fad2016-10-07 16:59:56 -0700635}
636
637unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
638 unsigned long len, unsigned long pgoff, unsigned long flags)
639{
Kirill A. Shutemov97d3d0f2020-01-13 16:29:10 -0800640 unsigned long ret;
Toshi Kani74d2fad2016-10-07 16:59:56 -0700641 loff_t off = (loff_t)pgoff << PAGE_SHIFT;
642
Kirill A. Shutemov97d3d0f2020-01-13 16:29:10 -0800643 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);
644 if (ret)
645 return ret;
William Kucharski1854bc62019-09-22 08:43:15 -0400646
Toshi Kani74d2fad2016-10-07 16:59:56 -0700647 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
648}
649EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
650
Souptick Joarder2b740302018-08-23 17:01:36 -0700651static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,
652 struct page *page, gfp_t gfp)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800653{
Jan Kara82b0f8c2016-12-14 15:06:58 -0800654 struct vm_area_struct *vma = vmf->vma;
Kefeng Wangcfe32362023-03-02 19:58:29 +0800655 struct folio *folio = page_folio(page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800656 pgtable_t pgtable;
Jan Kara82b0f8c2016-12-14 15:06:58 -0800657 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
Souptick Joarder2b740302018-08-23 17:01:36 -0700658 vm_fault_t ret = 0;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800659
Kefeng Wangcfe32362023-03-02 19:58:29 +0800660 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
Johannes Weiner00501b52014-08-08 14:19:20 -0700661
Kefeng Wangcfe32362023-03-02 19:58:29 +0800662 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
663 folio_put(folio);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700664 count_vm_event(THP_FAULT_FALLBACK);
David Rientjes85b9f462020-04-06 20:04:28 -0700665 count_vm_event(THP_FAULT_FALLBACK_CHARGE);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700666 return VM_FAULT_FALLBACK;
667 }
Kefeng Wangcfe32362023-03-02 19:58:29 +0800668 folio_throttle_swaprate(folio, gfp);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800669
Joel Fernandes (Google)4cf58922019-01-03 15:28:34 -0800670 pgtable = pte_alloc_one(vma->vm_mm);
Johannes Weiner00501b52014-08-08 14:19:20 -0700671 if (unlikely(!pgtable)) {
Michal Hocko6b31d592017-08-18 15:16:15 -0700672 ret = VM_FAULT_OOM;
673 goto release;
Johannes Weiner00501b52014-08-08 14:19:20 -0700674 }
675
Huang Yingc79b57e2017-09-06 16:25:04 -0700676 clear_huge_page(page, vmf->address, HPAGE_PMD_NR);
Minchan Kim52f37622013-04-29 15:08:15 -0700677 /*
Kefeng Wangcfe32362023-03-02 19:58:29 +0800678 * The memory barrier inside __folio_mark_uptodate makes sure that
Minchan Kim52f37622013-04-29 15:08:15 -0700679 * clear_huge_page writes become visible before the set_pmd_at()
680 * write.
681 */
Kefeng Wangcfe32362023-03-02 19:58:29 +0800682 __folio_mark_uptodate(folio);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800683
Jan Kara82b0f8c2016-12-14 15:06:58 -0800684 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
685 if (unlikely(!pmd_none(*vmf->pmd))) {
Michal Hocko6b31d592017-08-18 15:16:15 -0700686 goto unlock_release;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800687 } else {
688 pmd_t entry;
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700689
Michal Hocko6b31d592017-08-18 15:16:15 -0700690 ret = check_stable_address_space(vma->vm_mm);
691 if (ret)
692 goto unlock_release;
693
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700694 /* Deliver the page fault to userland */
695 if (userfaultfd_missing(vma)) {
Jan Kara82b0f8c2016-12-14 15:06:58 -0800696 spin_unlock(vmf->ptl);
Kefeng Wangcfe32362023-03-02 19:58:29 +0800697 folio_put(folio);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700698 pte_free(vma->vm_mm, pgtable);
Miaohe Lin8fd5eda2021-05-04 18:33:49 -0700699 ret = handle_userfault(vmf, VM_UFFD_MISSING);
700 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
701 return ret;
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700702 }
703
Kirill A. Shutemov31223592013-09-12 15:14:01 -0700704 entry = mk_huge_pmd(page, vma->vm_page_prot);
Linus Torvaldsf55e1012017-11-29 09:01:01 -0800705 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
Kefeng Wangcfe32362023-03-02 19:58:29 +0800706 folio_add_new_anon_rmap(folio, vma, haddr);
707 folio_add_lru_vma(folio, vma);
Jan Kara82b0f8c2016-12-14 15:06:58 -0800708 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
709 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
Bibo Maofca40572021-02-24 12:06:42 -0800710 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700711 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
Kirill A. Shutemovc4812902017-11-15 17:35:37 -0800712 mm_inc_nr_ptes(vma->vm_mm);
Jan Kara82b0f8c2016-12-14 15:06:58 -0800713 spin_unlock(vmf->ptl);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700714 count_vm_event(THP_FAULT_ALLOC);
Johannes Weiner9d82c692020-06-03 16:02:04 -0700715 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800716 }
717
David Rientjesaa2e8782012-05-29 15:06:17 -0700718 return 0;
Michal Hocko6b31d592017-08-18 15:16:15 -0700719unlock_release:
720 spin_unlock(vmf->ptl);
721release:
722 if (pgtable)
723 pte_free(vma->vm_mm, pgtable);
Kefeng Wangcfe32362023-03-02 19:58:29 +0800724 folio_put(folio);
Michal Hocko6b31d592017-08-18 15:16:15 -0700725 return ret;
726
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800727}
728
Mel Gorman444eb2a42016-03-17 14:19:23 -0700729/*
David Rientjes21440d72017-02-22 15:45:49 -0800730 * always: directly stall for all thp allocations
731 * defer: wake kswapd and fail if not immediately available
732 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
733 * fail if not immediately available
734 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
735 * available
736 * never: never stall for any thp allocation
Mel Gorman444eb2a42016-03-17 14:19:23 -0700737 */
Rik van Riel164cc4f2021-02-25 17:16:18 -0800738gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma)
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -0800739{
Rik van Riel164cc4f2021-02-25 17:16:18 -0800740 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE);
Michal Hocko89c83fb2018-11-02 15:48:31 -0700741
David Rientjesac79f782019-09-04 12:54:18 -0700742 /* Always do synchronous compaction */
David Rientjes21440d72017-02-22 15:45:49 -0800743 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
Andrea Arcangelia8282602019-08-13 15:37:53 -0700744 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
David Rientjesac79f782019-09-04 12:54:18 -0700745
746 /* Kick kcompactd and fail quickly */
David Rientjes21440d72017-02-22 15:45:49 -0800747 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
David Rientjes19deb762019-09-04 12:54:20 -0700748 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
David Rientjesac79f782019-09-04 12:54:18 -0700749
750 /* Synchronous compaction if madvised, otherwise kick kcompactd */
David Rientjes21440d72017-02-22 15:45:49 -0800751 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
David Rientjes19deb762019-09-04 12:54:20 -0700752 return GFP_TRANSHUGE_LIGHT |
753 (vma_madvised ? __GFP_DIRECT_RECLAIM :
754 __GFP_KSWAPD_RECLAIM);
David Rientjesac79f782019-09-04 12:54:18 -0700755
756 /* Only do synchronous compaction if madvised */
David Rientjes21440d72017-02-22 15:45:49 -0800757 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
David Rientjes19deb762019-09-04 12:54:20 -0700758 return GFP_TRANSHUGE_LIGHT |
759 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);
David Rientjesac79f782019-09-04 12:54:18 -0700760
David Rientjes19deb762019-09-04 12:54:20 -0700761 return GFP_TRANSHUGE_LIGHT;
Mel Gorman444eb2a42016-03-17 14:19:23 -0700762}
763
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800764/* Caller must hold page table lock. */
Miaohe Lin2efeb8d2021-02-24 12:07:29 -0800765static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800766 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700767 struct page *zero_page)
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800768{
769 pmd_t entry;
Andrew Morton7c414162015-09-08 14:58:43 -0700770 if (!pmd_none(*pmd))
Miaohe Lin2efeb8d2021-02-24 12:07:29 -0800771 return;
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700772 entry = mk_pmd(zero_page, vma->vm_page_prot);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800773 entry = pmd_mkhuge(entry);
Qi Zhengc8bb4162022-08-18 16:27:48 +0800774 pgtable_trans_huge_deposit(mm, pmd, pgtable);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800775 set_pmd_at(mm, haddr, pmd, entry);
Kirill A. Shutemovc4812902017-11-15 17:35:37 -0800776 mm_inc_nr_ptes(mm);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800777}
778
Souptick Joarder2b740302018-08-23 17:01:36 -0700779vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800780{
Jan Kara82b0f8c2016-12-14 15:06:58 -0800781 struct vm_area_struct *vma = vmf->vma;
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -0800782 gfp_t gfp;
Matthew Wilcox (Oracle)cb196ee2022-05-12 20:23:01 -0700783 struct folio *folio;
Jan Kara82b0f8c2016-12-14 15:06:58 -0800784 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800785
Yang Shi43675e62019-07-18 15:57:24 -0700786 if (!transhuge_vma_suitable(vma, haddr))
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700787 return VM_FAULT_FALLBACK;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700788 if (unlikely(anon_vma_prepare(vma)))
789 return VM_FAULT_OOM;
Yang Shi4fa68932022-06-16 10:48:35 -0700790 khugepaged_enter_vma(vma, vma->vm_flags);
Yang Shid2081b22022-05-19 14:08:49 -0700791
Jan Kara82b0f8c2016-12-14 15:06:58 -0800792 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700793 !mm_forbids_zeropage(vma->vm_mm) &&
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700794 transparent_hugepage_use_zero_page()) {
795 pgtable_t pgtable;
796 struct page *zero_page;
Souptick Joarder2b740302018-08-23 17:01:36 -0700797 vm_fault_t ret;
Joel Fernandes (Google)4cf58922019-01-03 15:28:34 -0800798 pgtable = pte_alloc_one(vma->vm_mm);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700799 if (unlikely(!pgtable))
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800800 return VM_FAULT_OOM;
Aaron Lu6fcb52a2016-10-07 17:00:08 -0700801 zero_page = mm_get_huge_zero_page(vma->vm_mm);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700802 if (unlikely(!zero_page)) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700803 pte_free(vma->vm_mm, pgtable);
Andi Kleen81ab4202011-04-14 15:22:06 -0700804 count_vm_event(THP_FAULT_FALLBACK);
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700805 return VM_FAULT_FALLBACK;
Andi Kleen81ab4202011-04-14 15:22:06 -0700806 }
Jan Kara82b0f8c2016-12-14 15:06:58 -0800807 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700808 ret = 0;
Jan Kara82b0f8c2016-12-14 15:06:58 -0800809 if (pmd_none(*vmf->pmd)) {
Michal Hocko6b31d592017-08-18 15:16:15 -0700810 ret = check_stable_address_space(vma->vm_mm);
811 if (ret) {
812 spin_unlock(vmf->ptl);
Gerald Schaeferbfe8cc12020-11-21 22:17:15 -0800813 pte_free(vma->vm_mm, pgtable);
Michal Hocko6b31d592017-08-18 15:16:15 -0700814 } else if (userfaultfd_missing(vma)) {
Jan Kara82b0f8c2016-12-14 15:06:58 -0800815 spin_unlock(vmf->ptl);
Gerald Schaeferbfe8cc12020-11-21 22:17:15 -0800816 pte_free(vma->vm_mm, pgtable);
Jan Kara82b0f8c2016-12-14 15:06:58 -0800817 ret = handle_userfault(vmf, VM_UFFD_MISSING);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700818 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
819 } else {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700820 set_huge_zero_page(pgtable, vma->vm_mm, vma,
Jan Kara82b0f8c2016-12-14 15:06:58 -0800821 haddr, vmf->pmd, zero_page);
Bibo Maofca40572021-02-24 12:06:42 -0800822 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
Jan Kara82b0f8c2016-12-14 15:06:58 -0800823 spin_unlock(vmf->ptl);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700824 }
Gerald Schaeferbfe8cc12020-11-21 22:17:15 -0800825 } else {
Jan Kara82b0f8c2016-12-14 15:06:58 -0800826 spin_unlock(vmf->ptl);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700827 pte_free(vma->vm_mm, pgtable);
Gerald Schaeferbfe8cc12020-11-21 22:17:15 -0800828 }
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700829 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800830 }
Rik van Riel164cc4f2021-02-25 17:16:18 -0800831 gfp = vma_thp_gfp_mask(vma);
Matthew Wilcox (Oracle)cb196ee2022-05-12 20:23:01 -0700832 folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, vma, haddr, true);
833 if (unlikely(!folio)) {
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700834 count_vm_event(THP_FAULT_FALLBACK);
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700835 return VM_FAULT_FALLBACK;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700836 }
Matthew Wilcox (Oracle)cb196ee2022-05-12 20:23:01 -0700837 return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800838}
839
Matthew Wilcoxae18d6d2015-09-08 14:59:14 -0700840static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
Oliver O'Halloran3b6521f2017-05-08 15:59:43 -0700841 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,
842 pgtable_t pgtable)
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700843{
844 struct mm_struct *mm = vma->vm_mm;
845 pmd_t entry;
846 spinlock_t *ptl;
847
848 ptl = pmd_lock(mm, pmd);
Aneesh Kumar K.Vc6f3c5e2019-04-05 18:39:10 -0700849 if (!pmd_none(*pmd)) {
850 if (write) {
851 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {
852 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));
853 goto out_unlock;
854 }
855 entry = pmd_mkyoung(*pmd);
856 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
857 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))
858 update_mmu_cache_pmd(vma, addr, pmd);
859 }
860
861 goto out_unlock;
862 }
863
Dan Williamsf25748e32016-01-15 16:56:43 -0800864 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
865 if (pfn_t_devmap(pfn))
866 entry = pmd_mkdevmap(entry);
Ross Zwisler01871e52016-01-15 16:56:02 -0800867 if (write) {
Linus Torvaldsf55e1012017-11-29 09:01:01 -0800868 entry = pmd_mkyoung(pmd_mkdirty(entry));
869 entry = maybe_pmd_mkwrite(entry, vma);
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700870 }
Oliver O'Halloran3b6521f2017-05-08 15:59:43 -0700871
872 if (pgtable) {
873 pgtable_trans_huge_deposit(mm, pmd, pgtable);
Kirill A. Shutemovc4812902017-11-15 17:35:37 -0800874 mm_inc_nr_ptes(mm);
Aneesh Kumar K.Vc6f3c5e2019-04-05 18:39:10 -0700875 pgtable = NULL;
Oliver O'Halloran3b6521f2017-05-08 15:59:43 -0700876 }
877
Ross Zwisler01871e52016-01-15 16:56:02 -0800878 set_pmd_at(mm, addr, pmd, entry);
879 update_mmu_cache_pmd(vma, addr, pmd);
Aneesh Kumar K.Vc6f3c5e2019-04-05 18:39:10 -0700880
881out_unlock:
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700882 spin_unlock(ptl);
Aneesh Kumar K.Vc6f3c5e2019-04-05 18:39:10 -0700883 if (pgtable)
884 pte_free(mm, pgtable);
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700885}
886
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100887/**
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000888 * vmf_insert_pfn_pmd - insert a pmd size pfn
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100889 * @vmf: Structure describing the fault
890 * @pfn: pfn to insert
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100891 * @write: whether it's a write fault
892 *
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000893 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100894 *
895 * Return: vm_fault_t value.
896 */
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000897vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write)
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700898{
Dan Williamsfce86ff2019-05-13 17:15:33 -0700899 unsigned long addr = vmf->address & PMD_MASK;
900 struct vm_area_struct *vma = vmf->vma;
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000901 pgprot_t pgprot = vma->vm_page_prot;
Oliver O'Halloran3b6521f2017-05-08 15:59:43 -0700902 pgtable_t pgtable = NULL;
Dan Williamsfce86ff2019-05-13 17:15:33 -0700903
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700904 /*
905 * If we had pmd_special, we could avoid all these restrictions,
906 * but we need to be consistent with PTEs and architectures that
907 * can't support a 'special' bit.
908 */
Dave Jiange1fb4a02018-08-17 15:43:40 -0700909 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
910 !pfn_t_devmap(pfn));
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700911 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
912 (VM_PFNMAP|VM_MIXEDMAP));
913 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700914
915 if (addr < vma->vm_start || addr >= vma->vm_end)
916 return VM_FAULT_SIGBUS;
Borislav Petkov308a0472016-10-26 19:43:43 +0200917
Oliver O'Halloran3b6521f2017-05-08 15:59:43 -0700918 if (arch_needs_pgtable_deposit()) {
Joel Fernandes (Google)4cf58922019-01-03 15:28:34 -0800919 pgtable = pte_alloc_one(vma->vm_mm);
Oliver O'Halloran3b6521f2017-05-08 15:59:43 -0700920 if (!pgtable)
921 return VM_FAULT_OOM;
922 }
923
Borislav Petkov308a0472016-10-26 19:43:43 +0200924 track_pfn_insert(vma, &pgprot, pfn);
925
Dan Williamsfce86ff2019-05-13 17:15:33 -0700926 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);
Matthew Wilcoxae18d6d2015-09-08 14:59:14 -0700927 return VM_FAULT_NOPAGE;
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700928}
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000929EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700930
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800931#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
Linus Torvaldsf55e1012017-11-29 09:01:01 -0800932static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800933{
Linus Torvaldsf55e1012017-11-29 09:01:01 -0800934 if (likely(vma->vm_flags & VM_WRITE))
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800935 pud = pud_mkwrite(pud);
936 return pud;
937}
938
939static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000940 pud_t *pud, pfn_t pfn, bool write)
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800941{
942 struct mm_struct *mm = vma->vm_mm;
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000943 pgprot_t prot = vma->vm_page_prot;
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800944 pud_t entry;
945 spinlock_t *ptl;
946
947 ptl = pud_lock(mm, pud);
Aneesh Kumar K.Vc6f3c5e2019-04-05 18:39:10 -0700948 if (!pud_none(*pud)) {
949 if (write) {
950 if (pud_pfn(*pud) != pfn_t_to_pfn(pfn)) {
951 WARN_ON_ONCE(!is_huge_zero_pud(*pud));
952 goto out_unlock;
953 }
954 entry = pud_mkyoung(*pud);
955 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);
956 if (pudp_set_access_flags(vma, addr, pud, entry, 1))
957 update_mmu_cache_pud(vma, addr, pud);
958 }
959 goto out_unlock;
960 }
961
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800962 entry = pud_mkhuge(pfn_t_pud(pfn, prot));
963 if (pfn_t_devmap(pfn))
964 entry = pud_mkdevmap(entry);
965 if (write) {
Linus Torvaldsf55e1012017-11-29 09:01:01 -0800966 entry = pud_mkyoung(pud_mkdirty(entry));
967 entry = maybe_pud_mkwrite(entry, vma);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800968 }
969 set_pud_at(mm, addr, pud, entry);
970 update_mmu_cache_pud(vma, addr, pud);
Aneesh Kumar K.Vc6f3c5e2019-04-05 18:39:10 -0700971
972out_unlock:
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800973 spin_unlock(ptl);
974}
975
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100976/**
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000977 * vmf_insert_pfn_pud - insert a pud size pfn
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100978 * @vmf: Structure describing the fault
979 * @pfn: pfn to insert
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100980 * @write: whether it's a write fault
981 *
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000982 * Insert a pud size pfn. See vmf_insert_pfn() for additional info.
Thomas Hellstrom (VMware)9a9731b2020-03-24 18:48:09 +0100983 *
984 * Return: vm_fault_t value.
985 */
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000986vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write)
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800987{
Dan Williamsfce86ff2019-05-13 17:15:33 -0700988 unsigned long addr = vmf->address & PUD_MASK;
989 struct vm_area_struct *vma = vmf->vma;
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +0000990 pgprot_t pgprot = vma->vm_page_prot;
Dan Williamsfce86ff2019-05-13 17:15:33 -0700991
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800992 /*
993 * If we had pud_special, we could avoid all these restrictions,
994 * but we need to be consistent with PTEs and architectures that
995 * can't support a 'special' bit.
996 */
Dave Jiang62ec0d82018-09-04 15:46:16 -0700997 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
998 !pfn_t_devmap(pfn));
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800999 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1000 (VM_PFNMAP|VM_MIXEDMAP));
1001 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001002
1003 if (addr < vma->vm_start || addr >= vma->vm_end)
1004 return VM_FAULT_SIGBUS;
1005
1006 track_pfn_insert(vma, &pgprot, pfn);
1007
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +00001008 insert_pfn_pud(vma, addr, vmf->pud, pfn, write);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001009 return VM_FAULT_NOPAGE;
1010}
Lorenzo Stoakes7b806d22023-03-12 23:40:14 +00001011EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001012#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1013
Dan Williams3565fce2016-01-15 16:56:55 -08001014static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
Miaohe Lina69e4712022-07-04 21:21:50 +08001015 pmd_t *pmd, bool write)
Dan Williams3565fce2016-01-15 16:56:55 -08001016{
1017 pmd_t _pmd;
1018
Kirill A. Shutemova8f97362017-11-27 06:21:25 +03001019 _pmd = pmd_mkyoung(*pmd);
Miaohe Lina69e4712022-07-04 21:21:50 +08001020 if (write)
Kirill A. Shutemova8f97362017-11-27 06:21:25 +03001021 _pmd = pmd_mkdirty(_pmd);
Dan Williams3565fce2016-01-15 16:56:55 -08001022 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
Miaohe Lina69e4712022-07-04 21:21:50 +08001023 pmd, _pmd, write))
Dan Williams3565fce2016-01-15 16:56:55 -08001024 update_mmu_cache_pmd(vma, addr, pmd);
1025}
1026
1027struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
Keith Buschdf06b372018-10-26 15:10:28 -07001028 pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
Dan Williams3565fce2016-01-15 16:56:55 -08001029{
1030 unsigned long pfn = pmd_pfn(*pmd);
1031 struct mm_struct *mm = vma->vm_mm;
Dan Williams3565fce2016-01-15 16:56:55 -08001032 struct page *page;
Logan Gunthorpe0f089232022-10-21 11:41:08 -06001033 int ret;
Dan Williams3565fce2016-01-15 16:56:55 -08001034
1035 assert_spin_locked(pmd_lockptr(mm, pmd));
1036
Linus Torvaldsf6f37322017-12-15 18:53:22 -08001037 if (flags & FOLL_WRITE && !pmd_write(*pmd))
Dan Williams3565fce2016-01-15 16:56:55 -08001038 return NULL;
1039
1040 if (pmd_present(*pmd) && pmd_devmap(*pmd))
1041 /* pass */;
1042 else
1043 return NULL;
1044
1045 if (flags & FOLL_TOUCH)
Miaohe Lina69e4712022-07-04 21:21:50 +08001046 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);
Dan Williams3565fce2016-01-15 16:56:55 -08001047
1048 /*
1049 * device mapped pages can only be returned if the
1050 * caller will manage the page reference count.
1051 */
John Hubbard3faa52c2020-04-01 21:05:29 -07001052 if (!(flags & (FOLL_GET | FOLL_PIN)))
Dan Williams3565fce2016-01-15 16:56:55 -08001053 return ERR_PTR(-EEXIST);
1054
1055 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
Keith Buschdf06b372018-10-26 15:10:28 -07001056 *pgmap = get_dev_pagemap(pfn, *pgmap);
1057 if (!*pgmap)
Dan Williams3565fce2016-01-15 16:56:55 -08001058 return ERR_PTR(-EFAULT);
1059 page = pfn_to_page(pfn);
Logan Gunthorpe0f089232022-10-21 11:41:08 -06001060 ret = try_grab_page(page, flags);
1061 if (ret)
1062 page = ERR_PTR(ret);
Dan Williams3565fce2016-01-15 16:56:55 -08001063
1064 return page;
1065}
1066
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001067int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1068 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
Peter Xu8f34f1e2021-06-30 18:49:02 -07001069 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001070{
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001071 spinlock_t *dst_ptl, *src_ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001072 struct page *src_page;
1073 pmd_t pmd;
Matthew Wilcox12c9d702016-02-02 16:57:57 -08001074 pgtable_t pgtable = NULL;
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001075 int ret = -ENOMEM;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001076
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001077 /* Skip if can be re-fill on fault */
Peter Xu8f34f1e2021-06-30 18:49:02 -07001078 if (!vma_is_anonymous(dst_vma))
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001079 return 0;
1080
Joel Fernandes (Google)4cf58922019-01-03 15:28:34 -08001081 pgtable = pte_alloc_one(dst_mm);
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001082 if (unlikely(!pgtable))
1083 goto out;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001084
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001085 dst_ptl = pmd_lock(dst_mm, dst_pmd);
1086 src_ptl = pmd_lockptr(src_mm, src_pmd);
1087 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001088
1089 ret = -EAGAIN;
1090 pmd = *src_pmd;
Zi Yan84c3fc42017-09-08 16:11:01 -07001091
1092#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1093 if (unlikely(is_swap_pmd(pmd))) {
1094 swp_entry_t entry = pmd_to_swp_entry(pmd);
1095
1096 VM_BUG_ON(!is_pmd_migration_entry(pmd));
David Hildenbrand6c287602022-05-09 18:20:44 -07001097 if (!is_readable_migration_entry(entry)) {
Alistair Popple4dd845b2021-06-30 18:54:09 -07001098 entry = make_readable_migration_entry(
1099 swp_offset(entry));
Zi Yan84c3fc42017-09-08 16:11:01 -07001100 pmd = swp_entry_to_pmd(entry);
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07001101 if (pmd_swp_soft_dirty(*src_pmd))
1102 pmd = pmd_swp_mksoft_dirty(pmd);
Peter Xu8f34f1e2021-06-30 18:49:02 -07001103 if (pmd_swp_uffd_wp(*src_pmd))
1104 pmd = pmd_swp_mkuffd_wp(pmd);
Zi Yan84c3fc42017-09-08 16:11:01 -07001105 set_pmd_at(src_mm, addr, src_pmd, pmd);
1106 }
Zi Yandd8a67f2017-11-02 15:59:47 -07001107 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
Kirill A. Shutemovaf5b0f62017-11-15 17:35:40 -08001108 mm_inc_nr_ptes(dst_mm);
Zi Yandd8a67f2017-11-02 15:59:47 -07001109 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
Peter Xu8f34f1e2021-06-30 18:49:02 -07001110 if (!userfaultfd_wp(dst_vma))
1111 pmd = pmd_swp_clear_uffd_wp(pmd);
Zi Yan84c3fc42017-09-08 16:11:01 -07001112 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
1113 ret = 0;
1114 goto out_unlock;
1115 }
1116#endif
1117
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001118 if (unlikely(!pmd_trans_huge(pmd))) {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001119 pte_free(dst_mm, pgtable);
1120 goto out_unlock;
1121 }
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001122 /*
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001123 * When page table lock is held, the huge zero pmd should not be
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001124 * under splitting since we don't split the page itself, only pmd to
1125 * a page table.
1126 */
1127 if (is_huge_zero_pmd(pmd)) {
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -08001128 /*
1129 * get_huge_zero_page() will never allocate a new page here,
1130 * since we already have a zero page to copy. It just takes a
1131 * reference.
1132 */
Peter Xu5fc7a5f2021-06-30 18:48:59 -07001133 mm_get_huge_zero_page(dst_mm);
1134 goto out_zero_page;
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001135 }
Mel Gormande466bd2013-12-18 17:08:42 -08001136
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001137 src_page = pmd_page(pmd);
1138 VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
Peter Xud0420352020-09-25 18:26:00 -04001139
David Hildenbrandfb3d8242022-05-09 18:20:43 -07001140 get_page(src_page);
1141 if (unlikely(page_try_dup_anon_rmap(src_page, true, src_vma))) {
1142 /* Page maybe pinned: split and retry the fault on PTEs. */
1143 put_page(src_page);
Peter Xud0420352020-09-25 18:26:00 -04001144 pte_free(dst_mm, pgtable);
1145 spin_unlock(src_ptl);
1146 spin_unlock(dst_ptl);
Peter Xu8f34f1e2021-06-30 18:49:02 -07001147 __split_huge_pmd(src_vma, src_pmd, addr, false, NULL);
Peter Xud0420352020-09-25 18:26:00 -04001148 return -EAGAIN;
1149 }
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001150 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
Peter Xu5fc7a5f2021-06-30 18:48:59 -07001151out_zero_page:
Kirill A. Shutemovc4812902017-11-15 17:35:37 -08001152 mm_inc_nr_ptes(dst_mm);
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001153 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001154 pmdp_set_wrprotect(src_mm, addr, src_pmd);
Peter Xu8f34f1e2021-06-30 18:49:02 -07001155 if (!userfaultfd_wp(dst_vma))
1156 pmd = pmd_clear_uffd_wp(pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001157 pmd = pmd_mkold(pmd_wrprotect(pmd));
1158 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001159
1160 ret = 0;
1161out_unlock:
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001162 spin_unlock(src_ptl);
1163 spin_unlock(dst_ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001164out:
1165 return ret;
1166}
1167
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001168#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1169static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
Miaohe Lin5fe653e2022-07-04 21:21:49 +08001170 pud_t *pud, bool write)
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001171{
1172 pud_t _pud;
1173
Kirill A. Shutemova8f97362017-11-27 06:21:25 +03001174 _pud = pud_mkyoung(*pud);
Miaohe Lin5fe653e2022-07-04 21:21:49 +08001175 if (write)
Kirill A. Shutemova8f97362017-11-27 06:21:25 +03001176 _pud = pud_mkdirty(_pud);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001177 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
Miaohe Lin5fe653e2022-07-04 21:21:49 +08001178 pud, _pud, write))
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001179 update_mmu_cache_pud(vma, addr, pud);
1180}
1181
1182struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
Keith Buschdf06b372018-10-26 15:10:28 -07001183 pud_t *pud, int flags, struct dev_pagemap **pgmap)
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001184{
1185 unsigned long pfn = pud_pfn(*pud);
1186 struct mm_struct *mm = vma->vm_mm;
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001187 struct page *page;
Logan Gunthorpe0f089232022-10-21 11:41:08 -06001188 int ret;
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001189
1190 assert_spin_locked(pud_lockptr(mm, pud));
1191
Linus Torvaldsf6f37322017-12-15 18:53:22 -08001192 if (flags & FOLL_WRITE && !pud_write(*pud))
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001193 return NULL;
1194
1195 if (pud_present(*pud) && pud_devmap(*pud))
1196 /* pass */;
1197 else
1198 return NULL;
1199
1200 if (flags & FOLL_TOUCH)
Miaohe Lin5fe653e2022-07-04 21:21:49 +08001201 touch_pud(vma, addr, pud, flags & FOLL_WRITE);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001202
1203 /*
1204 * device mapped pages can only be returned if the
1205 * caller will manage the page reference count.
John Hubbard3faa52c2020-04-01 21:05:29 -07001206 *
1207 * At least one of FOLL_GET | FOLL_PIN must be set, so assert that here:
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001208 */
John Hubbard3faa52c2020-04-01 21:05:29 -07001209 if (!(flags & (FOLL_GET | FOLL_PIN)))
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001210 return ERR_PTR(-EEXIST);
1211
1212 pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
Keith Buschdf06b372018-10-26 15:10:28 -07001213 *pgmap = get_dev_pagemap(pfn, *pgmap);
1214 if (!*pgmap)
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001215 return ERR_PTR(-EFAULT);
1216 page = pfn_to_page(pfn);
Logan Gunthorpe0f089232022-10-21 11:41:08 -06001217
1218 ret = try_grab_page(page, flags);
1219 if (ret)
1220 page = ERR_PTR(ret);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001221
1222 return page;
1223}
1224
1225int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1226 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1227 struct vm_area_struct *vma)
1228{
1229 spinlock_t *dst_ptl, *src_ptl;
1230 pud_t pud;
1231 int ret;
1232
1233 dst_ptl = pud_lock(dst_mm, dst_pud);
1234 src_ptl = pud_lockptr(src_mm, src_pud);
1235 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1236
1237 ret = -EAGAIN;
1238 pud = *src_pud;
1239 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud)))
1240 goto out_unlock;
1241
1242 /*
1243 * When page table lock is held, the huge zero pud should not be
1244 * under splitting since we don't split the page itself, only pud to
1245 * a page table.
1246 */
1247 if (is_huge_zero_pud(pud)) {
1248 /* No huge zero pud yet */
1249 }
1250
David Hildenbrandfb3d8242022-05-09 18:20:43 -07001251 /*
1252 * TODO: once we support anonymous pages, use page_try_dup_anon_rmap()
1253 * and split if duplicating fails.
1254 */
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001255 pudp_set_wrprotect(src_mm, addr, src_pud);
1256 pud = pud_mkold(pud_wrprotect(pud));
1257 set_pud_at(dst_mm, addr, dst_pud, pud);
1258
1259 ret = 0;
1260out_unlock:
1261 spin_unlock(src_ptl);
1262 spin_unlock(dst_ptl);
1263 return ret;
1264}
1265
1266void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
1267{
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001268 bool write = vmf->flags & FAULT_FLAG_WRITE;
1269
1270 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
1271 if (unlikely(!pud_same(*vmf->pud, orig_pud)))
1272 goto unlock;
1273
Miaohe Lin5fe653e2022-07-04 21:21:49 +08001274 touch_pud(vmf->vma, vmf->address, vmf->pud, write);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001275unlock:
1276 spin_unlock(vmf->ptl);
1277}
1278#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1279
Yang Shi5db4f152021-06-30 18:51:35 -07001280void huge_pmd_set_accessed(struct vm_fault *vmf)
Will Deacona1dd4502012-12-11 16:01:27 -08001281{
Minchan Kim20f664a2017-01-10 16:57:51 -08001282 bool write = vmf->flags & FAULT_FLAG_WRITE;
Will Deacona1dd4502012-12-11 16:01:27 -08001283
Jan Kara82b0f8c2016-12-14 15:06:58 -08001284 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
Miaohe Lina69e4712022-07-04 21:21:50 +08001285 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd)))
Will Deacona1dd4502012-12-11 16:01:27 -08001286 goto unlock;
1287
Miaohe Lina69e4712022-07-04 21:21:50 +08001288 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write);
Will Deacona1dd4502012-12-11 16:01:27 -08001289
1290unlock:
Jan Kara82b0f8c2016-12-14 15:06:58 -08001291 spin_unlock(vmf->ptl);
Will Deacona1dd4502012-12-11 16:01:27 -08001292}
1293
Yang Shi5db4f152021-06-30 18:51:35 -07001294vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001295{
David Hildenbrandc89357e2022-05-09 18:20:45 -07001296 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
Jan Kara82b0f8c2016-12-14 15:06:58 -08001297 struct vm_area_struct *vma = vmf->vma;
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001298 struct folio *folio;
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001299 struct page *page;
Jan Kara82b0f8c2016-12-14 15:06:58 -08001300 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
Yang Shi5db4f152021-06-30 18:51:35 -07001301 pmd_t orig_pmd = vmf->orig_pmd;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001302
Jan Kara82b0f8c2016-12-14 15:06:58 -08001303 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
Sasha Levin81d1b092014-10-09 15:28:10 -07001304 VM_BUG_ON_VMA(!vma->anon_vma, vma);
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001305
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001306 if (is_huge_zero_pmd(orig_pmd))
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001307 goto fallback;
1308
Jan Kara82b0f8c2016-12-14 15:06:58 -08001309 spin_lock(vmf->ptl);
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001310
1311 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
1312 spin_unlock(vmf->ptl);
1313 return 0;
1314 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001315
1316 page = pmd_page(orig_pmd);
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001317 folio = page_folio(page);
Miaohe Linf6004e72021-05-04 18:34:02 -07001318 VM_BUG_ON_PAGE(!PageHead(page), page);
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001319
David Hildenbrand6c287602022-05-09 18:20:44 -07001320 /* Early check when only holding the PT lock. */
1321 if (PageAnonExclusive(page))
1322 goto reuse;
1323
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001324 if (!folio_trylock(folio)) {
1325 folio_get(folio);
Huang Yingba3c4ce2017-09-06 16:22:19 -07001326 spin_unlock(vmf->ptl);
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001327 folio_lock(folio);
Huang Yingba3c4ce2017-09-06 16:22:19 -07001328 spin_lock(vmf->ptl);
1329 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001330 spin_unlock(vmf->ptl);
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001331 folio_unlock(folio);
1332 folio_put(folio);
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001333 return 0;
Huang Yingba3c4ce2017-09-06 16:22:19 -07001334 }
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001335 folio_put(folio);
Huang Yingba3c4ce2017-09-06 16:22:19 -07001336 }
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001337
David Hildenbrand6c287602022-05-09 18:20:44 -07001338 /* Recheck after temporarily dropping the PT lock. */
1339 if (PageAnonExclusive(page)) {
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001340 folio_unlock(folio);
David Hildenbrand6c287602022-05-09 18:20:44 -07001341 goto reuse;
1342 }
1343
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001344 /*
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001345 * See do_wp_page(): we can only reuse the folio exclusively if
1346 * there are no additional references. Note that we always drain
1347 * the LRU pagevecs immediately after adding a THP.
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001348 */
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001349 if (folio_ref_count(folio) >
1350 1 + folio_test_swapcache(folio) * folio_nr_pages(folio))
David Hildenbrand3bff7e32022-03-24 18:13:43 -07001351 goto unlock_fallback;
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001352 if (folio_test_swapcache(folio))
1353 folio_free_swap(folio);
1354 if (folio_ref_count(folio) == 1) {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001355 pmd_t entry;
David Hildenbrand6c54dc62022-05-09 18:20:43 -07001356
1357 page_move_anon_rmap(page, vma);
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001358 folio_unlock(folio);
David Hildenbrand6c287602022-05-09 18:20:44 -07001359reuse:
David Hildenbrandc89357e2022-05-09 18:20:45 -07001360 if (unlikely(unshare)) {
1361 spin_unlock(vmf->ptl);
1362 return 0;
1363 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001364 entry = pmd_mkyoung(orig_pmd);
Linus Torvaldsf55e1012017-11-29 09:01:01 -08001365 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001366 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))
Jan Kara82b0f8c2016-12-14 15:06:58 -08001367 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
Jan Kara82b0f8c2016-12-14 15:06:58 -08001368 spin_unlock(vmf->ptl);
David Hildenbrandcb8d8632022-10-21 12:11:35 +02001369 return 0;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001370 }
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001371
David Hildenbrand3bff7e32022-03-24 18:13:43 -07001372unlock_fallback:
Matthew Wilcox (Oracle)2fad3d12022-09-02 20:46:38 +01001373 folio_unlock(folio);
Jan Kara82b0f8c2016-12-14 15:06:58 -08001374 spin_unlock(vmf->ptl);
Kirill A. Shutemov3917c802020-06-03 16:00:27 -07001375fallback:
1376 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
1377 return VM_FAULT_FALLBACK;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001378}
1379
David Hildenbrandc27f4792022-11-08 18:46:48 +01001380static inline bool can_change_pmd_writable(struct vm_area_struct *vma,
1381 unsigned long addr, pmd_t pmd)
1382{
1383 struct page *page;
1384
1385 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
1386 return false;
1387
1388 /* Don't touch entries that are not even readable (NUMA hinting). */
1389 if (pmd_protnone(pmd))
1390 return false;
1391
1392 /* Do we need write faults for softdirty tracking? */
1393 if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd))
1394 return false;
1395
1396 /* Do we need write faults for uffd-wp tracking? */
1397 if (userfaultfd_huge_pmd_wp(vma, pmd))
1398 return false;
1399
1400 if (!(vma->vm_flags & VM_SHARED)) {
1401 /* See can_change_pte_writable(). */
1402 page = vm_normal_page_pmd(vma, addr, pmd);
1403 return page && PageAnon(page) && PageAnonExclusive(page);
1404 }
1405
1406 /* See can_change_pte_writable(). */
1407 return pmd_dirty(pmd);
1408}
1409
David Hildenbrand5535be32022-08-09 22:56:40 +02001410/* FOLL_FORCE can write to even unwritable PMDs in COW mappings. */
1411static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
1412 struct vm_area_struct *vma,
1413 unsigned int flags)
Keno Fischer8310d482017-01-24 15:17:48 -08001414{
David Hildenbrand5535be32022-08-09 22:56:40 +02001415 /* If the pmd is writable, we can write to the page. */
1416 if (pmd_write(pmd))
1417 return true;
1418
1419 /* Maybe FOLL_FORCE is set to override it? */
1420 if (!(flags & FOLL_FORCE))
1421 return false;
1422
1423 /* But FOLL_FORCE has no effect on shared mappings */
1424 if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED))
1425 return false;
1426
1427 /* ... or read-only private ones */
1428 if (!(vma->vm_flags & VM_MAYWRITE))
1429 return false;
1430
1431 /* ... or already writable ones that just need to take a write fault */
1432 if (vma->vm_flags & VM_WRITE)
1433 return false;
1434
1435 /*
1436 * See can_change_pte_writable(): we broke COW and could map the page
1437 * writable if we have an exclusive anonymous page ...
1438 */
1439 if (!page || !PageAnon(page) || !PageAnonExclusive(page))
1440 return false;
1441
1442 /* ... and a write-fault isn't required for other reasons. */
1443 if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd))
1444 return false;
1445 return !userfaultfd_huge_pmd_wp(vma, pmd);
Keno Fischer8310d482017-01-24 15:17:48 -08001446}
1447
David Rientjesb676b292012-10-08 16:34:03 -07001448struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001449 unsigned long addr,
1450 pmd_t *pmd,
1451 unsigned int flags)
1452{
David Rientjesb676b292012-10-08 16:34:03 -07001453 struct mm_struct *mm = vma->vm_mm;
David Hildenbrand5535be32022-08-09 22:56:40 +02001454 struct page *page;
Logan Gunthorpe0f089232022-10-21 11:41:08 -06001455 int ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001456
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001457 assert_spin_locked(pmd_lockptr(mm, pmd));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001458
David Hildenbrand5535be32022-08-09 22:56:40 +02001459 page = pmd_page(*pmd);
1460 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1461
1462 if ((flags & FOLL_WRITE) &&
1463 !can_follow_write_pmd(*pmd, page, vma, flags))
1464 return NULL;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001465
Kirill A. Shutemov85facf22013-02-04 14:28:42 -08001466 /* Avoid dumping huge zero page */
1467 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1468 return ERR_PTR(-EFAULT);
1469
Mel Gorman2b4847e2013-12-18 17:08:32 -08001470 /* Full NUMA hinting faults to serialise migration in fault paths */
David Hildenbrand474098e2022-08-25 18:46:57 +02001471 if (pmd_protnone(*pmd) && !gup_can_follow_protnone(flags))
David Hildenbrand5535be32022-08-09 22:56:40 +02001472 return NULL;
John Hubbard3faa52c2020-04-01 21:05:29 -07001473
David Hildenbrand84209e82022-11-16 11:26:48 +01001474 if (!pmd_write(*pmd) && gup_must_unshare(vma, flags, page))
David Hildenbranda7f22662022-05-09 18:20:45 -07001475 return ERR_PTR(-EMLINK);
1476
David Hildenbrandb6a26192022-05-09 18:20:45 -07001477 VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
1478 !PageAnonExclusive(page), page);
1479
Logan Gunthorpe0f089232022-10-21 11:41:08 -06001480 ret = try_grab_page(page, flags);
1481 if (ret)
1482 return ERR_PTR(ret);
John Hubbard3faa52c2020-04-01 21:05:29 -07001483
Dan Williams3565fce2016-01-15 16:56:55 -08001484 if (flags & FOLL_TOUCH)
Miaohe Lina69e4712022-07-04 21:21:50 +08001485 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);
John Hubbard3faa52c2020-04-01 21:05:29 -07001486
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001487 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
Dan Williamsca120cf2016-09-03 10:38:03 -07001488 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001489
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001490 return page;
1491}
1492
Mel Gormand10e63f2012-10-25 14:16:31 +02001493/* NUMA hinting page fault entry point for trans huge pmds */
Yang Shi5db4f152021-06-30 18:51:35 -07001494vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
Mel Gormand10e63f2012-10-25 14:16:31 +02001495{
Jan Kara82b0f8c2016-12-14 15:06:58 -08001496 struct vm_area_struct *vma = vmf->vma;
Yang Shic5b5a3d2021-06-30 18:51:42 -07001497 pmd_t oldpmd = vmf->orig_pmd;
1498 pmd_t pmd;
Mel Gormanb32967f2012-11-19 12:35:47 +00001499 struct page *page;
Jan Kara82b0f8c2016-12-14 15:06:58 -08001500 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
Yang Shic5b5a3d2021-06-30 18:51:42 -07001501 int page_nid = NUMA_NO_NODE;
Huang Ying33024532022-07-13 16:39:51 +08001502 int target_nid, last_cpupid = (-1 & LAST_CPUPID_MASK);
David Hildenbrand6a56ccb2022-11-08 18:46:50 +01001503 bool migrated = false, writable = false;
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001504 int flags = 0;
Mel Gormand10e63f2012-10-25 14:16:31 +02001505
Jan Kara82b0f8c2016-12-14 15:06:58 -08001506 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
Yang Shic5b5a3d2021-06-30 18:51:42 -07001507 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {
Jan Kara82b0f8c2016-12-14 15:06:58 -08001508 spin_unlock(vmf->ptl);
Mel Gormande466bd2013-12-18 17:08:42 -08001509 goto out;
1510 }
1511
Yang Shic5b5a3d2021-06-30 18:51:42 -07001512 pmd = pmd_modify(oldpmd, vma->vm_page_prot);
David Hildenbrand6a56ccb2022-11-08 18:46:50 +01001513
1514 /*
1515 * Detect now whether the PMD could be writable; this information
1516 * is only valid while holding the PT lock.
1517 */
1518 writable = pmd_write(pmd);
1519 if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
1520 can_change_pmd_writable(vma, vmf->address, pmd))
1521 writable = true;
1522
Yang Shic5b5a3d2021-06-30 18:51:42 -07001523 page = vm_normal_page_pmd(vma, haddr, pmd);
1524 if (!page)
1525 goto out_map;
1526
1527 /* See similar comment in do_numa_page for explanation */
David Hildenbrand6a56ccb2022-11-08 18:46:50 +01001528 if (!writable)
Yang Shic5b5a3d2021-06-30 18:51:42 -07001529 flags |= TNF_NO_GROUP;
1530
1531 page_nid = page_to_nid(page);
Huang Ying33024532022-07-13 16:39:51 +08001532 /*
1533 * For memory tiering mode, cpupid of slow memory page is used
1534 * to record page access time. So use default value.
1535 */
1536 if (node_is_toptier(page_nid))
1537 last_cpupid = page_cpupid_last(page);
Yang Shic5b5a3d2021-06-30 18:51:42 -07001538 target_nid = numa_migrate_prep(page, vma, haddr, page_nid,
1539 &flags);
1540
1541 if (target_nid == NUMA_NO_NODE) {
1542 put_page(page);
1543 goto out_map;
1544 }
1545
Jan Kara82b0f8c2016-12-14 15:06:58 -08001546 spin_unlock(vmf->ptl);
David Hildenbrand6a56ccb2022-11-08 18:46:50 +01001547 writable = false;
Peter Zijlstra8b1b4362017-06-07 18:05:07 +02001548
Yang Shic5b5a3d2021-06-30 18:51:42 -07001549 migrated = migrate_misplaced_page(page, vma, target_nid);
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001550 if (migrated) {
1551 flags |= TNF_MIGRATED;
Mel Gorman8191acb2013-10-07 11:28:45 +01001552 page_nid = target_nid;
Yang Shic5b5a3d2021-06-30 18:51:42 -07001553 } else {
Mel Gorman074c2382015-03-25 15:55:42 -07001554 flags |= TNF_MIGRATE_FAIL;
Yang Shic5b5a3d2021-06-30 18:51:42 -07001555 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1556 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {
1557 spin_unlock(vmf->ptl);
1558 goto out;
1559 }
1560 goto out_map;
1561 }
Mel Gormanb8916632013-10-07 11:28:44 +01001562
1563out:
Anshuman Khandual98fa15f2019-03-05 15:42:58 -08001564 if (page_nid != NUMA_NO_NODE)
Jan Kara82b0f8c2016-12-14 15:06:58 -08001565 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR,
Aneesh Kumar K.V9a8b3002017-02-24 14:59:56 -08001566 flags);
Mel Gorman8191acb2013-10-07 11:28:45 +01001567
Mel Gormand10e63f2012-10-25 14:16:31 +02001568 return 0;
Yang Shic5b5a3d2021-06-30 18:51:42 -07001569
1570out_map:
1571 /* Restore the PMD */
1572 pmd = pmd_modify(oldpmd, vma->vm_page_prot);
1573 pmd = pmd_mkyoung(pmd);
David Hildenbrand6a56ccb2022-11-08 18:46:50 +01001574 if (writable)
Yang Shic5b5a3d2021-06-30 18:51:42 -07001575 pmd = pmd_mkwrite(pmd);
1576 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
1577 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1578 spin_unlock(vmf->ptl);
1579 goto out;
Mel Gormand10e63f2012-10-25 14:16:31 +02001580}
1581
Huang Ying319904a2016-07-28 15:48:03 -07001582/*
1583 * Return true if we do MADV_FREE successfully on entire pmd page.
1584 * Otherwise, return false.
1585 */
1586bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001587 pmd_t *pmd, unsigned long addr, unsigned long next)
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001588{
1589 spinlock_t *ptl;
1590 pmd_t orig_pmd;
Kefeng Wangfc986a32022-12-07 10:34:30 +08001591 struct folio *folio;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001592 struct mm_struct *mm = tlb->mm;
Huang Ying319904a2016-07-28 15:48:03 -07001593 bool ret = false;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001594
Peter Zijlstraed6a7932018-08-31 14:46:08 +02001595 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
Aneesh Kumar K.V07e32662016-12-12 16:42:40 -08001596
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001597 ptl = pmd_trans_huge_lock(pmd, vma);
1598 if (!ptl)
Linus Torvalds25eedab2016-01-17 18:33:15 -08001599 goto out_unlocked;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001600
1601 orig_pmd = *pmd;
Huang Ying319904a2016-07-28 15:48:03 -07001602 if (is_huge_zero_pmd(orig_pmd))
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001603 goto out;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001604
Zi Yan84c3fc42017-09-08 16:11:01 -07001605 if (unlikely(!pmd_present(orig_pmd))) {
1606 VM_BUG_ON(thp_migration_supported() &&
1607 !is_pmd_migration_entry(orig_pmd));
1608 goto out;
1609 }
1610
Kefeng Wangfc986a32022-12-07 10:34:30 +08001611 folio = pfn_folio(pmd_pfn(orig_pmd));
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001612 /*
Kefeng Wangfc986a32022-12-07 10:34:30 +08001613 * If other processes are mapping this folio, we couldn't discard
1614 * the folio unless they all do MADV_FREE so let's skip the folio.
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001615 */
Kefeng Wangfc986a32022-12-07 10:34:30 +08001616 if (folio_mapcount(folio) != 1)
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001617 goto out;
1618
Kefeng Wangfc986a32022-12-07 10:34:30 +08001619 if (!folio_trylock(folio))
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001620 goto out;
1621
1622 /*
1623 * If user want to discard part-pages of THP, split it so MADV_FREE
1624 * will deactivate only them.
1625 */
1626 if (next - addr != HPAGE_PMD_SIZE) {
Kefeng Wangfc986a32022-12-07 10:34:30 +08001627 folio_get(folio);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001628 spin_unlock(ptl);
Kefeng Wangfc986a32022-12-07 10:34:30 +08001629 split_folio(folio);
1630 folio_unlock(folio);
1631 folio_put(folio);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001632 goto out_unlocked;
1633 }
1634
Kefeng Wangfc986a32022-12-07 10:34:30 +08001635 if (folio_test_dirty(folio))
1636 folio_clear_dirty(folio);
1637 folio_unlock(folio);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001638
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001639 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
Kirill A. Shutemov58ceeb62017-04-13 14:56:26 -07001640 pmdp_invalidate(vma, addr, pmd);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001641 orig_pmd = pmd_mkold(orig_pmd);
1642 orig_pmd = pmd_mkclean(orig_pmd);
1643
1644 set_pmd_at(mm, addr, pmd, orig_pmd);
1645 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1646 }
Shaohua Li802a3a92017-05-03 14:52:32 -07001647
Kefeng Wang6a6fe9e2022-12-09 10:06:18 +08001648 folio_mark_lazyfree(folio);
Huang Ying319904a2016-07-28 15:48:03 -07001649 ret = true;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001650out:
1651 spin_unlock(ptl);
1652out_unlocked:
1653 return ret;
1654}
1655
Aneesh Kumar K.V953c66c2016-12-12 16:44:32 -08001656static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
1657{
1658 pgtable_t pgtable;
1659
1660 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1661 pte_free(mm, pgtable);
Kirill A. Shutemovc4812902017-11-15 17:35:37 -08001662 mm_dec_nr_ptes(mm);
Aneesh Kumar K.V953c66c2016-12-12 16:44:32 -08001663}
1664
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001665int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
Shaohua Lif21760b2012-01-12 17:19:16 -08001666 pmd_t *pmd, unsigned long addr)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001667{
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001668 pmd_t orig_pmd;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001669 spinlock_t *ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001670
Peter Zijlstraed6a7932018-08-31 14:46:08 +02001671 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
Aneesh Kumar K.V07e32662016-12-12 16:42:40 -08001672
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001673 ptl = __pmd_trans_huge_lock(pmd, vma);
1674 if (!ptl)
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001675 return 0;
1676 /*
1677 * For architectures like ppc64 we look at deposited pgtable
1678 * when calling pmdp_huge_get_and_clear. So do the
1679 * pgtable_trans_huge_withdraw after finishing pmdp related
1680 * operations.
1681 */
Aneesh Kumar K.V93a98692020-05-05 12:47:28 +05301682 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,
1683 tlb->fullmm);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001684 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
Thomas Hellstrom (VMware)2484ca92020-03-24 18:47:17 +01001685 if (vma_is_special_huge(vma)) {
Oliver O'Halloran3b6521f2017-05-08 15:59:43 -07001686 if (arch_needs_pgtable_deposit())
1687 zap_deposited_table(tlb->mm, pmd);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001688 spin_unlock(ptl);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001689 } else if (is_huge_zero_pmd(orig_pmd)) {
Oliver O'Halloranc14a6eb2017-05-08 15:59:40 -07001690 zap_deposited_table(tlb->mm, pmd);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001691 spin_unlock(ptl);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001692 } else {
Zi Yan616b8372017-09-08 16:10:57 -07001693 struct page *page = NULL;
1694 int flush_needed = 1;
1695
1696 if (pmd_present(orig_pmd)) {
1697 page = pmd_page(orig_pmd);
Hugh Dickinscea86fe2022-02-14 18:26:39 -08001698 page_remove_rmap(page, vma, true);
Zi Yan616b8372017-09-08 16:10:57 -07001699 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1700 VM_BUG_ON_PAGE(!PageHead(page), page);
1701 } else if (thp_migration_supported()) {
1702 swp_entry_t entry;
1703
1704 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
1705 entry = pmd_to_swp_entry(orig_pmd);
Alistair Poppleaf5cdaf2021-06-30 18:54:06 -07001706 page = pfn_swap_entry_to_page(entry);
Zi Yan616b8372017-09-08 16:10:57 -07001707 flush_needed = 0;
1708 } else
1709 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
1710
Kirill A. Shutemovb5072382016-07-26 15:25:34 -07001711 if (PageAnon(page)) {
Oliver O'Halloranc14a6eb2017-05-08 15:59:40 -07001712 zap_deposited_table(tlb->mm, pmd);
Kirill A. Shutemovb5072382016-07-26 15:25:34 -07001713 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1714 } else {
Aneesh Kumar K.V953c66c2016-12-12 16:44:32 -08001715 if (arch_needs_pgtable_deposit())
1716 zap_deposited_table(tlb->mm, pmd);
Yang Shifadae292018-08-17 15:44:55 -07001717 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR);
Kirill A. Shutemovb5072382016-07-26 15:25:34 -07001718 }
Zi Yan616b8372017-09-08 16:10:57 -07001719
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001720 spin_unlock(ptl);
Zi Yan616b8372017-09-08 16:10:57 -07001721 if (flush_needed)
1722 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001723 }
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001724 return 1;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001725}
1726
Aneesh Kumar K.V1dd38b62016-12-12 16:44:29 -08001727#ifndef pmd_move_must_withdraw
1728static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
1729 spinlock_t *old_pmd_ptl,
1730 struct vm_area_struct *vma)
1731{
1732 /*
1733 * With split pmd lock we also need to move preallocated
1734 * PTE page table if new_pmd is on different PMD page table.
1735 *
1736 * We also don't deposit and withdraw tables for file pages.
1737 */
1738 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
1739}
1740#endif
1741
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07001742static pmd_t move_soft_dirty_pmd(pmd_t pmd)
1743{
1744#ifdef CONFIG_MEM_SOFT_DIRTY
1745 if (unlikely(is_pmd_migration_entry(pmd)))
1746 pmd = pmd_swp_mksoft_dirty(pmd);
1747 else if (pmd_present(pmd))
1748 pmd = pmd_mksoft_dirty(pmd);
1749#endif
1750 return pmd;
1751}
1752
Hugh Dickinsbf8616d2016-05-19 17:12:54 -07001753bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
Wei Yangb8aa9d92020-08-06 23:23:40 -07001754 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001755{
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001756 spinlock_t *old_ptl, *new_ptl;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001757 pmd_t pmd;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001758 struct mm_struct *mm = vma->vm_mm;
Aaron Lu5d190422016-11-10 17:16:33 +08001759 bool force_flush = false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001760
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001761 /*
1762 * The destination pmd shouldn't be established, free_pgtables()
1763 * should have release it.
1764 */
1765 if (WARN_ON(!pmd_none(*new_pmd))) {
1766 VM_BUG_ON(pmd_trans_huge(*new_pmd));
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001767 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001768 }
1769
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001770 /*
1771 * We don't have to worry about the ordering of src and dst
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -07001772 * ptlocks because exclusive mmap_lock prevents deadlock.
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001773 */
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001774 old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
1775 if (old_ptl) {
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001776 new_ptl = pmd_lockptr(mm, new_pmd);
1777 if (new_ptl != old_ptl)
1778 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001779 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
Linus Torvaldseb66ae02018-10-12 15:22:59 -07001780 if (pmd_present(pmd))
Aaron Lua2ce2662016-11-29 13:27:31 +08001781 force_flush = true;
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001782 VM_BUG_ON(!pmd_none(*new_pmd));
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001783
Aneesh Kumar K.V1dd38b62016-12-12 16:44:29 -08001784 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
Aneesh Kumar K.Vb3084f42014-01-13 11:34:24 +05301785 pgtable_t pgtable;
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001786 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1787 pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001788 }
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07001789 pmd = move_soft_dirty_pmd(pmd);
1790 set_pmd_at(mm, new_addr, new_pmd, pmd);
Aaron Lu5d190422016-11-10 17:16:33 +08001791 if (force_flush)
Miaohe Lin7c38f182022-07-04 21:21:46 +08001792 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
Linus Torvaldseb66ae02018-10-12 15:22:59 -07001793 if (new_ptl != old_ptl)
1794 spin_unlock(new_ptl);
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001795 spin_unlock(old_ptl);
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001796 return true;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001797 }
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001798 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001799}
1800
Mel Gormanf123d742013-10-07 11:28:49 +01001801/*
1802 * Returns
1803 * - 0 if PMD could not be locked
Ingo Molnarf0953a12021-05-06 18:06:47 -07001804 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary
Yang Shie346e662021-06-30 18:51:55 -07001805 * or if prot_numa but THP migration is not supported
Ingo Molnarf0953a12021-05-06 18:06:47 -07001806 * - HPAGE_PMD_NR if protections changed and TLB flush necessary
Mel Gormanf123d742013-10-07 11:28:49 +01001807 */
Nadav Amit4a184192022-05-09 18:20:50 -07001808int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1809 pmd_t *pmd, unsigned long addr, pgprot_t newprot,
1810 unsigned long cp_flags)
Johannes Weinercd7548a2011-01-13 15:47:04 -08001811{
1812 struct mm_struct *mm = vma->vm_mm;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001813 spinlock_t *ptl;
Nadav Amitc9fe6652022-05-09 18:20:50 -07001814 pmd_t oldpmd, entry;
Peter Xu58705442020-04-06 20:05:45 -07001815 bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
Peter Xu292924b2020-04-06 20:05:49 -07001816 bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
1817 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
David Hildenbrand6a56ccb2022-11-08 18:46:50 +01001818 int ret = 1;
Johannes Weinercd7548a2011-01-13 15:47:04 -08001819
Nadav Amit4a184192022-05-09 18:20:50 -07001820 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1821
Yang Shie346e662021-06-30 18:51:55 -07001822 if (prot_numa && !thp_migration_supported())
1823 return 1;
1824
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001825 ptl = __pmd_trans_huge_lock(pmd, vma);
Kirill A. Shutemov0a85e51d2017-04-13 14:56:17 -07001826 if (!ptl)
1827 return 0;
Mel Gormane944fd62015-02-12 14:58:35 -08001828
Zi Yan84c3fc42017-09-08 16:11:01 -07001829#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1830 if (is_swap_pmd(*pmd)) {
1831 swp_entry_t entry = pmd_to_swp_entry(*pmd);
David Hildenbrand6c287602022-05-09 18:20:44 -07001832 struct page *page = pfn_swap_entry_to_page(entry);
David Hildenbrand24bf08c2023-04-05 18:02:35 +02001833 pmd_t newpmd;
Zi Yan84c3fc42017-09-08 16:11:01 -07001834
1835 VM_BUG_ON(!is_pmd_migration_entry(*pmd));
Alistair Popple4dd845b2021-06-30 18:54:09 -07001836 if (is_writable_migration_entry(entry)) {
Zi Yan84c3fc42017-09-08 16:11:01 -07001837 /*
1838 * A protection check is difficult so
1839 * just be safe and disable write
1840 */
David Hildenbrand6c287602022-05-09 18:20:44 -07001841 if (PageAnon(page))
1842 entry = make_readable_exclusive_migration_entry(swp_offset(entry));
1843 else
1844 entry = make_readable_migration_entry(swp_offset(entry));
Zi Yan84c3fc42017-09-08 16:11:01 -07001845 newpmd = swp_entry_to_pmd(entry);
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07001846 if (pmd_swp_soft_dirty(*pmd))
1847 newpmd = pmd_swp_mksoft_dirty(newpmd);
David Hildenbrand24bf08c2023-04-05 18:02:35 +02001848 } else {
1849 newpmd = *pmd;
Zi Yan84c3fc42017-09-08 16:11:01 -07001850 }
David Hildenbrand24bf08c2023-04-05 18:02:35 +02001851
1852 if (uffd_wp)
1853 newpmd = pmd_swp_mkuffd_wp(newpmd);
1854 else if (uffd_wp_resolve)
1855 newpmd = pmd_swp_clear_uffd_wp(newpmd);
1856 if (!pmd_same(*pmd, newpmd))
1857 set_pmd_at(mm, addr, pmd, newpmd);
Zi Yan84c3fc42017-09-08 16:11:01 -07001858 goto unlock;
1859 }
1860#endif
1861
Huang Yinga1a3a2f2022-03-22 14:46:27 -07001862 if (prot_numa) {
1863 struct page *page;
Huang Ying33024532022-07-13 16:39:51 +08001864 bool toptier;
Huang Yinga1a3a2f2022-03-22 14:46:27 -07001865 /*
1866 * Avoid trapping faults against the zero page. The read-only
1867 * data is likely to be read-cached on the local CPU and
1868 * local/remote hits to the zero page are not interesting.
1869 */
1870 if (is_huge_zero_pmd(*pmd))
1871 goto unlock;
Johannes Weinercd7548a2011-01-13 15:47:04 -08001872
Huang Yinga1a3a2f2022-03-22 14:46:27 -07001873 if (pmd_protnone(*pmd))
1874 goto unlock;
Kirill A. Shutemov0a85e51d2017-04-13 14:56:17 -07001875
Huang Yinga1a3a2f2022-03-22 14:46:27 -07001876 page = pmd_page(*pmd);
Huang Ying33024532022-07-13 16:39:51 +08001877 toptier = node_is_toptier(page_to_nid(page));
Huang Yinga1a3a2f2022-03-22 14:46:27 -07001878 /*
1879 * Skip scanning top tier node if normal numa
1880 * balancing is disabled
1881 */
1882 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
Huang Ying33024532022-07-13 16:39:51 +08001883 toptier)
Huang Yinga1a3a2f2022-03-22 14:46:27 -07001884 goto unlock;
Huang Ying33024532022-07-13 16:39:51 +08001885
1886 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
1887 !toptier)
1888 xchg_page_access_time(page, jiffies_to_msecs(jiffies));
Huang Yinga1a3a2f2022-03-22 14:46:27 -07001889 }
Kirill A. Shutemovced10802017-04-13 14:56:20 -07001890 /*
Michel Lespinasse3e4e28c2020-06-08 21:33:51 -07001891 * In case prot_numa, we are under mmap_read_lock(mm). It's critical
Kirill A. Shutemovced10802017-04-13 14:56:20 -07001892 * to not clear pmd intermittently to avoid race with MADV_DONTNEED
Michel Lespinasse3e4e28c2020-06-08 21:33:51 -07001893 * which is also under mmap_read_lock(mm):
Kirill A. Shutemovced10802017-04-13 14:56:20 -07001894 *
1895 * CPU0: CPU1:
1896 * change_huge_pmd(prot_numa=1)
1897 * pmdp_huge_get_and_clear_notify()
1898 * madvise_dontneed()
1899 * zap_pmd_range()
1900 * pmd_trans_huge(*pmd) == 0 (without ptl)
1901 * // skip the pmd
1902 * set_pmd_at();
1903 * // pmd is re-established
1904 *
1905 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it
1906 * which may break userspace.
1907 *
Nadav Amit4f831452022-05-09 18:20:50 -07001908 * pmdp_invalidate_ad() is required to make sure we don't miss
Kirill A. Shutemovced10802017-04-13 14:56:20 -07001909 * dirty/young flags set by hardware.
1910 */
Nadav Amit4f831452022-05-09 18:20:50 -07001911 oldpmd = pmdp_invalidate_ad(vma, addr, pmd);
Kirill A. Shutemovced10802017-04-13 14:56:20 -07001912
Nadav Amitc9fe6652022-05-09 18:20:50 -07001913 entry = pmd_modify(oldpmd, newprot);
Peter Xuf1eb1ba2022-12-14 15:15:33 -05001914 if (uffd_wp)
Peter Xu292924b2020-04-06 20:05:49 -07001915 entry = pmd_mkuffd_wp(entry);
Peter Xuf1eb1ba2022-12-14 15:15:33 -05001916 else if (uffd_wp_resolve)
Peter Xu292924b2020-04-06 20:05:49 -07001917 /*
1918 * Leave the write bit to be handled by PF interrupt
1919 * handler, then things like COW could be properly
1920 * handled.
1921 */
1922 entry = pmd_clear_uffd_wp(entry);
David Hildenbrandc27f4792022-11-08 18:46:48 +01001923
1924 /* See change_pte_range(). */
1925 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
1926 can_change_pmd_writable(vma, addr, entry))
1927 entry = pmd_mkwrite(entry);
1928
Kirill A. Shutemov0a85e51d2017-04-13 14:56:17 -07001929 ret = HPAGE_PMD_NR;
1930 set_pmd_at(mm, addr, pmd, entry);
Nadav Amit4a184192022-05-09 18:20:50 -07001931
Nadav Amitc9fe6652022-05-09 18:20:50 -07001932 if (huge_pmd_needs_flush(oldpmd, entry))
1933 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE);
Kirill A. Shutemov0a85e51d2017-04-13 14:56:17 -07001934unlock:
1935 spin_unlock(ptl);
Johannes Weinercd7548a2011-01-13 15:47:04 -08001936 return ret;
1937}
1938
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001939/*
Huang Ying8f19b0c2016-07-26 15:27:04 -07001940 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001941 *
Huang Ying8f19b0c2016-07-26 15:27:04 -07001942 * Note that if it returns page table lock pointer, this routine returns without
1943 * unlocking page table lock. So callers must unlock it.
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001944 */
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001945spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001946{
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001947 spinlock_t *ptl;
1948 ptl = pmd_lock(vma->vm_mm, pmd);
Zi Yan84c3fc42017-09-08 16:11:01 -07001949 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||
1950 pmd_devmap(*pmd)))
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001951 return ptl;
1952 spin_unlock(ptl);
1953 return NULL;
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001954}
1955
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001956/*
Miaohe Lind965e392022-07-04 21:21:48 +08001957 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise.
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001958 *
Miaohe Lind965e392022-07-04 21:21:48 +08001959 * Note that if it returns page table lock pointer, this routine returns without
1960 * unlocking page table lock. So callers must unlock it.
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001961 */
1962spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
1963{
1964 spinlock_t *ptl;
1965
1966 ptl = pud_lock(vma->vm_mm, pud);
1967 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud)))
1968 return ptl;
1969 spin_unlock(ptl);
1970 return NULL;
1971}
1972
1973#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1974int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
1975 pud_t *pud, unsigned long addr)
1976{
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001977 spinlock_t *ptl;
1978
1979 ptl = __pud_trans_huge_lock(pud, vma);
1980 if (!ptl)
1981 return 0;
Miaohe Lin74929072022-07-04 21:21:54 +08001982
Qian Cai70516b92019-03-05 15:50:00 -08001983 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001984 tlb_remove_pud_tlb_entry(tlb, pud, addr);
Thomas Hellstrom (VMware)2484ca92020-03-24 18:47:17 +01001985 if (vma_is_special_huge(vma)) {
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08001986 spin_unlock(ptl);
1987 /* No zero page support yet */
1988 } else {
1989 /* No support for anonymous PUD pages yet */
1990 BUG();
1991 }
1992 return 1;
1993}
1994
1995static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
1996 unsigned long haddr)
1997{
1998 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
1999 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2000 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
2001 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));
2002
Yisheng Xiece9311c2017-03-09 16:17:00 -08002003 count_vm_event(THP_SPLIT_PUD);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08002004
2005 pudp_huge_clear_flush_notify(vma, haddr, pud);
2006}
2007
2008void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
2009 unsigned long address)
2010{
2011 spinlock_t *ptl;
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08002012 struct mmu_notifier_range range;
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08002013
Alistair Popple7d4a8be2023-01-10 13:57:22 +11002014 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
Jérôme Glisse6f4f13e2019-05-13 17:20:49 -07002015 address & HPAGE_PUD_MASK,
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08002016 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
2017 mmu_notifier_invalidate_range_start(&range);
2018 ptl = pud_lock(vma->vm_mm, pud);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08002019 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
2020 goto out;
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08002021 __split_huge_pud_locked(vma, pud, range.start);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08002022
2023out:
2024 spin_unlock(ptl);
Jérôme Glisse4645b9f2017-11-15 17:34:11 -08002025 /*
2026 * No need to double call mmu_notifier->invalidate_range() callback as
2027 * the above pudp_huge_clear_flush_notify() did already call it.
2028 */
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08002029 mmu_notifier_invalidate_range_only_end(&range);
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -08002030}
2031#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
2032
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002033static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2034 unsigned long haddr, pmd_t *pmd)
2035{
2036 struct mm_struct *mm = vma->vm_mm;
2037 pgtable_t pgtable;
David Hildenbrand42b2af22023-03-02 18:54:23 +01002038 pmd_t _pmd, old_pmd;
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002039 int i;
2040
Jérôme Glisse0f108512017-11-15 17:34:07 -08002041 /*
2042 * Leave pmd empty until pte is filled note that it is fine to delay
2043 * notification until mmu_notifier_invalidate_range_end() as we are
2044 * replacing a zero pmd write protected page with a zero pte write
2045 * protected page.
2046 *
Mike Rapoportee657282022-06-27 09:00:26 +03002047 * See Documentation/mm/mmu_notifier.rst
Jérôme Glisse0f108512017-11-15 17:34:07 -08002048 */
David Hildenbrand42b2af22023-03-02 18:54:23 +01002049 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002050
2051 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2052 pmd_populate(mm, &_pmd, pgtable);
2053
2054 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2055 pte_t *pte, entry;
2056 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2057 entry = pte_mkspecial(entry);
David Hildenbrand42b2af22023-03-02 18:54:23 +01002058 if (pmd_uffd_wp(old_pmd))
2059 entry = pte_mkuffd_wp(entry);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002060 pte = pte_offset_map(&_pmd, haddr);
2061 VM_BUG_ON(!pte_none(*pte));
2062 set_pte_at(mm, haddr, pte, entry);
2063 pte_unmap(pte);
2064 }
2065 smp_wmb(); /* make pte visible before pmd */
2066 pmd_populate(mm, pmd, pgtable);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002067}
2068
2069static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002070 unsigned long haddr, bool freeze)
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002071{
2072 struct mm_struct *mm = vma->vm_mm;
2073 struct page *page;
2074 pgtable_t pgtable;
Aneesh Kumar K.V423ac9a2018-01-31 16:18:24 -08002075 pmd_t old_pmd, _pmd;
Peter Xu292924b2020-04-06 20:05:49 -07002076 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false;
Peter Xu0ccf7f12022-08-11 12:13:28 -04002077 bool anon_exclusive = false, dirty = false;
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002078 unsigned long addr;
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002079 int i;
2080
2081 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
2082 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2083 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
Zi Yan84c3fc42017-09-08 16:11:01 -07002084 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)
2085 && !pmd_devmap(*pmd));
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002086
2087 count_vm_event(THP_SPLIT_PMD);
2088
Kirill A. Shutemovd21b9e52016-07-26 15:25:37 -07002089 if (!vma_is_anonymous(vma)) {
Hugh Dickins99fa8a42021-06-15 18:23:45 -07002090 old_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
Aneesh Kumar K.V953c66c2016-12-12 16:44:32 -08002091 /*
2092 * We are going to unmap this huge page. So
2093 * just go ahead and zap it
2094 */
2095 if (arch_needs_pgtable_deposit())
2096 zap_deposited_table(mm, pmd);
Thomas Hellstrom (VMware)2484ca92020-03-24 18:47:17 +01002097 if (vma_is_special_huge(vma))
Kirill A. Shutemovd21b9e52016-07-26 15:25:37 -07002098 return;
Hugh Dickins99fa8a42021-06-15 18:23:45 -07002099 if (unlikely(is_pmd_migration_entry(old_pmd))) {
2100 swp_entry_t entry;
2101
2102 entry = pmd_to_swp_entry(old_pmd);
Alistair Poppleaf5cdaf2021-06-30 18:54:06 -07002103 page = pfn_swap_entry_to_page(entry);
Hugh Dickins99fa8a42021-06-15 18:23:45 -07002104 } else {
2105 page = pmd_page(old_pmd);
2106 if (!PageDirty(page) && pmd_dirty(old_pmd))
2107 set_page_dirty(page);
2108 if (!PageReferenced(page) && pmd_young(old_pmd))
2109 SetPageReferenced(page);
Hugh Dickinscea86fe2022-02-14 18:26:39 -08002110 page_remove_rmap(page, vma, true);
Hugh Dickins99fa8a42021-06-15 18:23:45 -07002111 put_page(page);
2112 }
Yang Shifadae292018-08-17 15:44:55 -07002113 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002114 return;
Hugh Dickins99fa8a42021-06-15 18:23:45 -07002115 }
2116
Hugh Dickins3b77e8c2021-06-15 18:23:49 -07002117 if (is_huge_zero_pmd(*pmd)) {
Jérôme Glisse4645b9f2017-11-15 17:34:11 -08002118 /*
2119 * FIXME: Do we want to invalidate secondary mmu by calling
2120 * mmu_notifier_invalidate_range() see comments below inside
2121 * __split_huge_pmd() ?
2122 *
2123 * We are going from a zero huge page write protected to zero
2124 * small page also write protected so it does not seems useful
2125 * to invalidate secondary mmu at this time.
2126 */
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002127 return __split_huge_zero_page_pmd(vma, haddr, pmd);
2128 }
2129
Aneesh Kumar K.V423ac9a2018-01-31 16:18:24 -08002130 /*
2131 * Up to this point the pmd is present and huge and userland has the
2132 * whole access to the hugepage during the split (which happens in
2133 * place). If we overwrite the pmd with the not-huge version pointing
2134 * to the pte here (which of course we could if all CPUs were bug
2135 * free), userland could trigger a small page size TLB miss on the
2136 * small sized TLB while the hugepage TLB entry is still established in
2137 * the huge TLB. Some CPU doesn't like that.
Alexander A. Klimov42742d92020-08-06 23:26:08 -07002138 * See http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum
2139 * 383 on page 105. Intel should be safe but is also warns that it's
Aneesh Kumar K.V423ac9a2018-01-31 16:18:24 -08002140 * only safe if the permission and cache attributes of the two entries
2141 * loaded in the two TLB is identical (which should be the case here).
2142 * But it is generally safer to never allow small and huge TLB entries
2143 * for the same virtual address to be loaded simultaneously. So instead
2144 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
2145 * current pmd notpresent (atomically because here the pmd_trans_huge
2146 * must remain set at all times on the pmd until the split is complete
2147 * for this pmd), then we flush the SMP TLB and finally we write the
2148 * non-huge version of the pmd entry with pmd_populate.
2149 */
2150 old_pmd = pmdp_invalidate(vma, haddr, pmd);
2151
Aneesh Kumar K.V423ac9a2018-01-31 16:18:24 -08002152 pmd_migration = is_pmd_migration_entry(old_pmd);
Peter Xu2e83ee12018-12-21 14:30:50 -08002153 if (unlikely(pmd_migration)) {
Zi Yan84c3fc42017-09-08 16:11:01 -07002154 swp_entry_t entry;
2155
Aneesh Kumar K.V423ac9a2018-01-31 16:18:24 -08002156 entry = pmd_to_swp_entry(old_pmd);
Alistair Poppleaf5cdaf2021-06-30 18:54:06 -07002157 page = pfn_swap_entry_to_page(entry);
Alistair Popple4dd845b2021-06-30 18:54:09 -07002158 write = is_writable_migration_entry(entry);
David Hildenbrand6c287602022-05-09 18:20:44 -07002159 if (PageAnon(page))
2160 anon_exclusive = is_readable_exclusive_migration_entry(entry);
Peter Xu2e346872022-08-11 12:13:29 -04002161 young = is_migration_entry_young(entry);
2162 dirty = is_migration_entry_dirty(entry);
Peter Xu2e83ee12018-12-21 14:30:50 -08002163 soft_dirty = pmd_swp_soft_dirty(old_pmd);
Peter Xuf45ec5f2020-04-06 20:06:01 -07002164 uffd_wp = pmd_swp_uffd_wp(old_pmd);
Peter Xu2e83ee12018-12-21 14:30:50 -08002165 } else {
Aneesh Kumar K.V423ac9a2018-01-31 16:18:24 -08002166 page = pmd_page(old_pmd);
Peter Xu0ccf7f12022-08-11 12:13:28 -04002167 if (pmd_dirty(old_pmd)) {
2168 dirty = true;
Peter Xu2e83ee12018-12-21 14:30:50 -08002169 SetPageDirty(page);
Peter Xu0ccf7f12022-08-11 12:13:28 -04002170 }
Peter Xu2e83ee12018-12-21 14:30:50 -08002171 write = pmd_write(old_pmd);
2172 young = pmd_young(old_pmd);
2173 soft_dirty = pmd_soft_dirty(old_pmd);
Peter Xu292924b2020-04-06 20:05:49 -07002174 uffd_wp = pmd_uffd_wp(old_pmd);
David Hildenbrand6c287602022-05-09 18:20:44 -07002175
Hugh Dickins9d846042022-03-22 14:46:11 -07002176 VM_BUG_ON_PAGE(!page_count(page), page);
David Hildenbrand6c287602022-05-09 18:20:44 -07002177
2178 /*
2179 * Without "freeze", we'll simply split the PMD, propagating the
2180 * PageAnonExclusive() flag for each PTE by setting it for
2181 * each subpage -- no need to (temporarily) clear.
2182 *
2183 * With "freeze" we want to replace mapped pages by
2184 * migration entries right away. This is only possible if we
2185 * managed to clear PageAnonExclusive() -- see
2186 * set_pmd_migration_entry().
2187 *
2188 * In case we cannot clear PageAnonExclusive(), split the PMD
2189 * only and let try_to_migrate_one() fail later.
David Hildenbrand088b8aa2022-09-01 10:35:59 +02002190 *
2191 * See page_try_share_anon_rmap(): invalidate PMD first.
David Hildenbrand6c287602022-05-09 18:20:44 -07002192 */
2193 anon_exclusive = PageAnon(page) && PageAnonExclusive(page);
2194 if (freeze && anon_exclusive && page_try_share_anon_rmap(page))
2195 freeze = false;
Hugh Dickins96d82de2022-11-22 01:51:50 -08002196 if (!freeze)
2197 page_ref_add(page, HPAGE_PMD_NR - 1);
Peter Xu2e83ee12018-12-21 14:30:50 -08002198 }
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002199
Aneesh Kumar K.V423ac9a2018-01-31 16:18:24 -08002200 /*
2201 * Withdraw the table only after we mark the pmd entry invalid.
2202 * This's critical for some architectures (Power).
2203 */
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002204 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2205 pmd_populate(mm, &_pmd, pgtable);
2206
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002207 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002208 pte_t entry, *pte;
2209 /*
2210 * Note that NUMA hinting access restrictions are not
2211 * transferred to avoid any possibility of altering
2212 * permissions across VMAs.
2213 */
Zi Yan84c3fc42017-09-08 16:11:01 -07002214 if (freeze || pmd_migration) {
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002215 swp_entry_t swp_entry;
Alistair Popple4dd845b2021-06-30 18:54:09 -07002216 if (write)
2217 swp_entry = make_writable_migration_entry(
2218 page_to_pfn(page + i));
David Hildenbrand6c287602022-05-09 18:20:44 -07002219 else if (anon_exclusive)
2220 swp_entry = make_readable_exclusive_migration_entry(
2221 page_to_pfn(page + i));
Alistair Popple4dd845b2021-06-30 18:54:09 -07002222 else
2223 swp_entry = make_readable_migration_entry(
2224 page_to_pfn(page + i));
Peter Xu2e346872022-08-11 12:13:29 -04002225 if (young)
2226 swp_entry = make_migration_entry_young(swp_entry);
2227 if (dirty)
2228 swp_entry = make_migration_entry_dirty(swp_entry);
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002229 entry = swp_entry_to_pte(swp_entry);
Andrea Arcangeli804dd152016-08-25 15:16:57 -07002230 if (soft_dirty)
2231 entry = pte_swp_mksoft_dirty(entry);
Peter Xuf45ec5f2020-04-06 20:06:01 -07002232 if (uffd_wp)
2233 entry = pte_swp_mkuffd_wp(entry);
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002234 } else {
Andrea Arcangeli6d2329f2016-10-07 17:01:22 -07002235 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
David Hildenbrand1462c522023-04-11 16:25:12 +02002236 if (write)
David Hildenbrandf3ebdf02023-04-18 16:21:13 +02002237 entry = pte_mkwrite(entry);
David Hildenbrand6c287602022-05-09 18:20:44 -07002238 if (anon_exclusive)
2239 SetPageAnonExclusive(page + i);
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002240 if (!young)
2241 entry = pte_mkold(entry);
Peter Xue833bc52022-11-25 13:58:57 -05002242 /* NOTE: this may set soft-dirty too on some archs */
2243 if (dirty)
2244 entry = pte_mkdirty(entry);
Andrea Arcangeli804dd152016-08-25 15:16:57 -07002245 if (soft_dirty)
2246 entry = pte_mksoft_dirty(entry);
Peter Xu292924b2020-04-06 20:05:49 -07002247 if (uffd_wp)
2248 entry = pte_mkuffd_wp(entry);
Hugh Dickins96d82de2022-11-22 01:51:50 -08002249 page_add_anon_rmap(page + i, vma, addr, false);
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002250 }
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002251 pte = pte_offset_map(&_pmd, addr);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002252 BUG_ON(!pte_none(*pte));
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002253 set_pte_at(mm, addr, pte, entry);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002254 pte_unmap(pte);
2255 }
2256
Hugh Dickinscb67f422022-11-02 18:51:38 -07002257 if (!pmd_migration)
2258 page_remove_rmap(page, vma, true);
Hugh Dickins96d82de2022-11-22 01:51:50 -08002259 if (freeze)
2260 put_page(page);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002261
2262 smp_wmb(); /* make pte visible before pmd */
2263 pmd_populate(mm, pmd, pgtable);
2264}
2265
2266void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
Matthew Wilcox (Oracle)af28a982022-01-21 10:44:52 -05002267 unsigned long address, bool freeze, struct folio *folio)
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002268{
2269 spinlock_t *ptl;
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08002270 struct mmu_notifier_range range;
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002271
Alistair Popple7d4a8be2023-01-10 13:57:22 +11002272 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
Jérôme Glisse6f4f13e2019-05-13 17:20:49 -07002273 address & HPAGE_PMD_MASK,
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08002274 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
2275 mmu_notifier_invalidate_range_start(&range);
2276 ptl = pmd_lock(vma->vm_mm, pmd);
Naoya Horiguchi33f47512016-07-14 12:07:32 -07002277
2278 /*
Matthew Wilcox (Oracle)af28a982022-01-21 10:44:52 -05002279 * If caller asks to setup a migration entry, we need a folio to check
2280 * pmd against. Otherwise we can end up replacing wrong folio.
Naoya Horiguchi33f47512016-07-14 12:07:32 -07002281 */
Matthew Wilcox (Oracle)af28a982022-01-21 10:44:52 -05002282 VM_BUG_ON(freeze && !folio);
Matthew Wilcox (Oracle)83a84412022-04-06 08:41:39 -04002283 VM_WARN_ON_ONCE(folio && !folio_test_locked(folio));
Naoya Horiguchi33f47512016-07-14 12:07:32 -07002284
David Hildenbrand7f760912022-03-24 18:13:56 -07002285 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd) ||
Matthew Wilcox (Oracle)83a84412022-04-06 08:41:39 -04002286 is_pmd_migration_entry(*pmd)) {
Miaohe Lincea33322022-07-04 21:22:00 +08002287 /*
2288 * It's safe to call pmd_page when folio is set because it's
2289 * guaranteed that pmd is present.
2290 */
Matthew Wilcox (Oracle)83a84412022-04-06 08:41:39 -04002291 if (folio && folio != page_folio(pmd_page(*pmd)))
2292 goto out;
David Hildenbrand7f760912022-03-24 18:13:56 -07002293 __split_huge_pmd_locked(vma, pmd, range.start, freeze);
Matthew Wilcox (Oracle)83a84412022-04-06 08:41:39 -04002294 }
David Hildenbrand7f760912022-03-24 18:13:56 -07002295
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08002296out:
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002297 spin_unlock(ptl);
Jérôme Glisse4645b9f2017-11-15 17:34:11 -08002298 /*
2299 * No need to double call mmu_notifier->invalidate_range() callback.
2300 * They are 3 cases to consider inside __split_huge_pmd_locked():
2301 * 1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious
2302 * 2) __split_huge_zero_page_pmd() read only zero page and any write
2303 * fault will trigger a flush_notify before pointing to a new page
2304 * (it is fine if the secondary mmu keeps pointing to the old zero
2305 * page in the meantime)
2306 * 3) Split a huge pmd into pte pointing to the same page. No need
2307 * to invalidate secondary tlb entry they are all still valid.
2308 * any further changes to individual pte will notify. So no need
2309 * to call mmu_notifier->invalidate_range()
2310 */
Jérôme Glisseac46d4f2018-12-28 00:38:09 -08002311 mmu_notifier_invalidate_range_only_end(&range);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002312}
2313
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07002314void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
Matthew Wilcox (Oracle)af28a982022-01-21 10:44:52 -05002315 bool freeze, struct folio *folio)
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002316{
Zach O'Keefe50722802022-07-06 16:59:26 -07002317 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002318
Zach O'Keefe50722802022-07-06 16:59:26 -07002319 if (!pmd)
Hugh Dickinsf72e7dc2014-06-23 13:22:05 -07002320 return;
2321
Matthew Wilcox (Oracle)af28a982022-01-21 10:44:52 -05002322 __split_huge_pmd(vma, pmd, address, freeze, folio);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002323}
2324
Miaohe Lin71f9e582021-05-04 18:33:52 -07002325static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
2326{
2327 /*
2328 * If the new address isn't hpage aligned and it could previously
2329 * contain an hugepage: check if we need to split an huge pmd.
2330 */
2331 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) &&
2332 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),
2333 ALIGN(address, HPAGE_PMD_SIZE)))
2334 split_huge_pmd_address(vma, address, false, NULL);
2335}
2336
Kirill A. Shutemove1b99962015-09-08 14:58:37 -07002337void vma_adjust_trans_huge(struct vm_area_struct *vma,
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002338 unsigned long start,
2339 unsigned long end,
2340 long adjust_next)
2341{
Miaohe Lin71f9e582021-05-04 18:33:52 -07002342 /* Check if we need to split start first. */
2343 split_huge_pmd_if_needed(vma, start);
2344
2345 /* Check if we need to split end next. */
2346 split_huge_pmd_if_needed(vma, end);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002347
2348 /*
Matthew Wilcox (Oracle)68540502022-09-06 19:49:00 +00002349 * If we're also updating the next vma vm_start,
Miaohe Lin71f9e582021-05-04 18:33:52 -07002350 * check if we need to split it.
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002351 */
2352 if (adjust_next > 0) {
Matthew Wilcox (Oracle)68540502022-09-06 19:49:00 +00002353 struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002354 unsigned long nstart = next->vm_start;
Wei Yangf9d86a62020-10-13 16:53:57 -07002355 nstart += adjust_next;
Miaohe Lin71f9e582021-05-04 18:33:52 -07002356 split_huge_pmd_if_needed(next, nstart);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08002357 }
2358}
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002359
Matthew Wilcox (Oracle)684555a2022-09-02 20:46:49 +01002360static void unmap_folio(struct folio *folio)
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002361{
Alistair Popplea98a2f02021-06-30 18:54:16 -07002362 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
2363 TTU_SYNC;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002364
Matthew Wilcox (Oracle)684555a2022-09-02 20:46:49 +01002365 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002366
Alistair Popplea98a2f02021-06-30 18:54:16 -07002367 /*
2368 * Anon pages need migration entries to preserve them, but file
2369 * pages can simply be left unmapped, then faulted back on demand.
2370 * If that is ever changed (perhaps for mlock), update remap_page().
2371 */
Matthew Wilcox (Oracle)4b8554c2022-01-28 14:29:43 -05002372 if (folio_test_anon(folio))
2373 try_to_migrate(folio, ttu_flags);
Alistair Popplea98a2f02021-06-30 18:54:16 -07002374 else
Matthew Wilcox (Oracle)869f7ee2022-02-15 09:28:49 -05002375 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002376}
2377
Matthew Wilcox (Oracle)4eecb8b2022-01-28 23:32:59 -05002378static void remap_page(struct folio *folio, unsigned long nr)
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002379{
Matthew Wilcox (Oracle)4eecb8b2022-01-28 23:32:59 -05002380 int i = 0;
Hugh Dickinsab02c252021-06-30 18:52:04 -07002381
Matthew Wilcox (Oracle)684555a2022-09-02 20:46:49 +01002382 /* If unmap_folio() uses try_to_migrate() on file, remove this check */
Matthew Wilcox (Oracle)4eecb8b2022-01-28 23:32:59 -05002383 if (!folio_test_anon(folio))
Hugh Dickinsab02c252021-06-30 18:52:04 -07002384 return;
Matthew Wilcox (Oracle)4eecb8b2022-01-28 23:32:59 -05002385 for (;;) {
2386 remove_migration_ptes(folio, folio, true);
2387 i += folio_nr_pages(folio);
2388 if (i >= nr)
2389 break;
2390 folio = folio_next(folio);
Kirill A. Shutemovace71a12017-02-24 14:57:45 -08002391 }
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002392}
2393
Alex Shi94866632020-12-15 12:33:24 -08002394static void lru_add_page_tail(struct page *head, struct page *tail,
Alex Shi88dcb9a2020-12-15 12:33:20 -08002395 struct lruvec *lruvec, struct list_head *list)
2396{
Alex Shi94866632020-12-15 12:33:24 -08002397 VM_BUG_ON_PAGE(!PageHead(head), head);
2398 VM_BUG_ON_PAGE(PageCompound(tail), head);
2399 VM_BUG_ON_PAGE(PageLRU(tail), head);
Alex Shi6168d0d2020-12-15 12:34:29 -08002400 lockdep_assert_held(&lruvec->lru_lock);
Alex Shi88dcb9a2020-12-15 12:33:20 -08002401
Alex Shi6dbb5742020-12-15 12:33:29 -08002402 if (list) {
Alex Shi88dcb9a2020-12-15 12:33:20 -08002403 /* page reclaim is reclaiming a huge page */
Alex Shi6dbb5742020-12-15 12:33:29 -08002404 VM_WARN_ON(PageLRU(head));
Alex Shi94866632020-12-15 12:33:24 -08002405 get_page(tail);
2406 list_add_tail(&tail->lru, list);
Alex Shi88dcb9a2020-12-15 12:33:20 -08002407 } else {
Alex Shi6dbb5742020-12-15 12:33:29 -08002408 /* head is still on lru (and we have it frozen) */
2409 VM_WARN_ON(!PageLRU(head));
Hugh Dickins07ca7602022-02-14 18:29:54 -08002410 if (PageUnevictable(tail))
2411 tail->mlock_count = 0;
2412 else
2413 list_add_tail(&tail->lru, &head->lru);
Alex Shi6dbb5742020-12-15 12:33:29 -08002414 SetPageLRU(tail);
Alex Shi88dcb9a2020-12-15 12:33:20 -08002415 }
2416}
2417
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07002418static void __split_huge_page_tail(struct page *head, int tail,
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002419 struct lruvec *lruvec, struct list_head *list)
2420{
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002421 struct page *page_tail = head + tail;
2422
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07002423 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002424
2425 /*
Konstantin Khlebnikov605ca5e2018-04-05 16:23:28 -07002426 * Clone page flags before unfreezing refcount.
2427 *
2428 * After successful get_page_unless_zero() might follow flags change,
Haitao Shi8958b242020-12-15 20:47:26 -08002429 * for example lock_page() which set PG_waiters.
David Hildenbrand6c287602022-05-09 18:20:44 -07002430 *
2431 * Note that for mapped sub-pages of an anonymous THP,
Matthew Wilcox (Oracle)684555a2022-09-02 20:46:49 +01002432 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in
David Hildenbrand6c287602022-05-09 18:20:44 -07002433 * the migration entry instead from where remap_page() will restore it.
2434 * We can still have PG_anon_exclusive set on effectively unmapped and
2435 * unreferenced sub-pages of an anonymous THP: we can simply drop
2436 * PG_anon_exclusive (-> PG_mappedtodisk) for these here.
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002437 */
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002438 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
2439 page_tail->flags |= (head->flags &
2440 ((1L << PG_referenced) |
2441 (1L << PG_swapbacked) |
Huang Ying38d8b4e2017-07-06 15:37:18 -07002442 (1L << PG_swapcache) |
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002443 (1L << PG_mlocked) |
2444 (1L << PG_uptodate) |
2445 (1L << PG_active) |
Johannes Weiner1899ad12018-10-26 15:06:04 -07002446 (1L << PG_workingset) |
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002447 (1L << PG_locked) |
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08002448 (1L << PG_unevictable) |
Catalin Marinasb0284cd2022-11-03 18:10:34 -07002449#ifdef CONFIG_ARCH_USES_PG_ARCH_X
Catalin Marinas72e6afa2020-07-02 10:19:30 +01002450 (1L << PG_arch_2) |
Peter Collingbourneef6458b2022-11-03 18:10:37 -07002451 (1L << PG_arch_3) |
Catalin Marinas72e6afa2020-07-02 10:19:30 +01002452#endif
Yu Zhaoec1c86b2022-09-18 02:00:02 -06002453 (1L << PG_dirty) |
2454 LRU_GEN_MASK | LRU_REFS_MASK));
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002455
Hugh Dickinscb67f422022-11-02 18:51:38 -07002456 /* ->mapping in first and second tail page is replaced by other uses */
Hugh Dickins173d9d92018-11-30 14:10:16 -08002457 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
2458 page_tail);
2459 page_tail->mapping = head->mapping;
2460 page_tail->index = head->index + tail;
Mel Gorman71e2d662022-10-19 14:41:56 +01002461
2462 /*
2463 * page->private should not be set in tail pages with the exception
2464 * of swap cache pages that store the swp_entry_t in tail pages.
2465 * Fix up and warn once if private is unexpectedly set.
Hugh Dickinscb67f422022-11-02 18:51:38 -07002466 *
Matthew Wilcox (Oracle)94688e82023-01-11 14:28:47 +00002467 * What of 32-bit systems, on which folio->_pincount overlays
Hugh Dickinscb67f422022-11-02 18:51:38 -07002468 * head[1].private? No problem: THP_SWAP is not enabled on 32-bit, and
Matthew Wilcox (Oracle)94688e82023-01-11 14:28:47 +00002469 * pincount must be 0 for folio_ref_freeze() to have succeeded.
Mel Gorman71e2d662022-10-19 14:41:56 +01002470 */
2471 if (!folio_test_swapcache(page_folio(head))) {
Hugh Dickins5aae9262022-10-22 00:51:06 -07002472 VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
Mel Gorman71e2d662022-10-19 14:41:56 +01002473 page_tail->private = 0;
2474 }
Hugh Dickins173d9d92018-11-30 14:10:16 -08002475
Konstantin Khlebnikov605ca5e2018-04-05 16:23:28 -07002476 /* Page flags must be visible before we make the page non-compound. */
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002477 smp_wmb();
2478
Konstantin Khlebnikov605ca5e2018-04-05 16:23:28 -07002479 /*
2480 * Clear PageTail before unfreezing page refcount.
2481 *
2482 * After successful get_page_unless_zero() might follow put_page()
2483 * which needs correct compound_head().
2484 */
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002485 clear_compound_head(page_tail);
2486
Konstantin Khlebnikov605ca5e2018-04-05 16:23:28 -07002487 /* Finally unfreeze refcount. Additional reference from page cache. */
2488 page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) ||
2489 PageSwapCache(head)));
2490
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002491 if (page_is_young(head))
2492 set_page_young(page_tail);
2493 if (page_is_idle(head))
2494 set_page_idle(page_tail);
2495
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002496 page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
Michal Hocko94723aa2018-04-10 16:30:07 -07002497
2498 /*
2499 * always add to the tail because some iterators expect new
2500 * pages to show after the currently processed elements - e.g.
2501 * migrate_pages
2502 */
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002503 lru_add_page_tail(head, page_tail, lruvec, list);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002504}
2505
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002506static void __split_huge_page(struct page *page, struct list_head *list,
Alex Shib6769832020-12-15 12:33:33 -08002507 pgoff_t end)
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002508{
Matthew Wilcox (Oracle)e809c3f2021-06-28 21:59:47 -04002509 struct folio *folio = page_folio(page);
2510 struct page *head = &folio->page;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002511 struct lruvec *lruvec;
Matthew Wilcox (Oracle)41011962019-09-23 15:34:52 -07002512 struct address_space *swap_cache = NULL;
2513 unsigned long offset = 0;
Kirill A. Shutemov8cce5472020-10-15 20:05:36 -07002514 unsigned int nr = thp_nr_pages(head);
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07002515 int i;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002516
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002517 /* complete memcg works before add pages to LRU */
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08002518 split_page_memcg(head, nr);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002519
Matthew Wilcox (Oracle)41011962019-09-23 15:34:52 -07002520 if (PageAnon(head) && PageSwapCache(head)) {
2521 swp_entry_t entry = { .val = page_private(head) };
2522
2523 offset = swp_offset(entry);
2524 swap_cache = swap_address_space(entry);
2525 xa_lock(&swap_cache->i_pages);
2526 }
2527
Ingo Molnarf0953a12021-05-06 18:06:47 -07002528 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
Matthew Wilcox (Oracle)e809c3f2021-06-28 21:59:47 -04002529 lruvec = folio_lruvec_lock(folio);
Alex Shib6769832020-12-15 12:33:33 -08002530
Yang Shieac96c32021-10-28 14:36:11 -07002531 ClearPageHasHWPoisoned(head);
2532
Kirill A. Shutemov8cce5472020-10-15 20:05:36 -07002533 for (i = nr - 1; i >= 1; i--) {
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07002534 __split_huge_page_tail(head, i, lruvec, list);
Hugh Dickinsd144bf62021-09-02 14:54:21 -07002535 /* Some pages can be beyond EOF: drop them from page cache */
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002536 if (head[i].index >= end) {
Matthew Wilcox (Oracle)fb5c2022022-06-28 20:15:29 -04002537 struct folio *tail = page_folio(head + i);
2538
Hugh Dickinsd144bf62021-09-02 14:54:21 -07002539 if (shmem_mapping(head->mapping))
Kirill A. Shutemov800d8c62016-07-26 15:26:18 -07002540 shmem_uncharge(head->mapping->host, 1);
Matthew Wilcox (Oracle)fb5c2022022-06-28 20:15:29 -04002541 else if (folio_test_clear_dirty(tail))
2542 folio_account_cleaned(tail,
2543 inode_to_wb(folio->mapping->host));
2544 __filemap_remove_folio(tail, NULL);
2545 folio_put(tail);
Matthew Wilcox (Oracle)41011962019-09-23 15:34:52 -07002546 } else if (!PageAnon(page)) {
2547 __xa_store(&head->mapping->i_pages, head[i].index,
2548 head + i, 0);
2549 } else if (swap_cache) {
2550 __xa_store(&swap_cache->i_pages, offset + i,
2551 head + i, 0);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002552 }
2553 }
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002554
2555 ClearPageCompound(head);
Alex Shi6168d0d2020-12-15 12:34:29 -08002556 unlock_page_lruvec(lruvec);
Alex Shib6769832020-12-15 12:33:33 -08002557 /* Caller disabled irqs, so they are still disabled here */
Vlastimil Babkaf7da6772019-08-24 17:54:59 -07002558
Kirill A. Shutemov8cce5472020-10-15 20:05:36 -07002559 split_page_owner(head, nr);
Vlastimil Babkaf7da6772019-08-24 17:54:59 -07002560
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002561 /* See comment in __split_huge_page_tail() */
2562 if (PageAnon(head)) {
Matthew Wilcoxaa5dc072017-12-04 10:16:10 -05002563 /* Additional pin to swap cache */
Matthew Wilcox (Oracle)41011962019-09-23 15:34:52 -07002564 if (PageSwapCache(head)) {
Huang Ying38d8b4e2017-07-06 15:37:18 -07002565 page_ref_add(head, 2);
Matthew Wilcox (Oracle)41011962019-09-23 15:34:52 -07002566 xa_unlock(&swap_cache->i_pages);
2567 } else {
Huang Ying38d8b4e2017-07-06 15:37:18 -07002568 page_ref_inc(head);
Matthew Wilcox (Oracle)41011962019-09-23 15:34:52 -07002569 }
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002570 } else {
Matthew Wilcoxaa5dc072017-12-04 10:16:10 -05002571 /* Additional pin to page cache */
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002572 page_ref_add(head, 2);
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07002573 xa_unlock(&head->mapping->i_pages);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002574 }
Alex Shib6769832020-12-15 12:33:33 -08002575 local_irq_enable();
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002576
Matthew Wilcox (Oracle)4eecb8b2022-01-28 23:32:59 -05002577 remap_page(folio, nr);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002578
Huang Yingc4f9c702020-10-15 20:06:07 -07002579 if (PageSwapCache(head)) {
2580 swp_entry_t entry = { .val = page_private(head) };
2581
2582 split_swap_cluster(entry);
2583 }
2584
Kirill A. Shutemov8cce5472020-10-15 20:05:36 -07002585 for (i = 0; i < nr; i++) {
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002586 struct page *subpage = head + i;
2587 if (subpage == page)
2588 continue;
2589 unlock_page(subpage);
2590
2591 /*
2592 * Subpages may be freed if there wasn't any mapping
2593 * like if add_to_swap() is running on a lru page that
2594 * had its mapping zapped. And freeing these pages
2595 * requires taking the lru_lock so we do the put_page
2596 * of the tail pages after the split is complete.
2597 */
Miaohe Lin0b175462022-07-04 21:21:56 +08002598 free_page_and_swap_cache(subpage);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002599 }
2600}
2601
Huang Yingb8f593c2017-07-06 15:37:28 -07002602/* Racy check whether the huge page can be split */
Matthew Wilcox (Oracle)d4b40842022-02-04 14:13:31 -05002603bool can_split_folio(struct folio *folio, int *pextra_pins)
Huang Yingb8f593c2017-07-06 15:37:28 -07002604{
2605 int extra_pins;
2606
Matthew Wilcoxaa5dc072017-12-04 10:16:10 -05002607 /* Additional pins from page cache */
Matthew Wilcox (Oracle)d4b40842022-02-04 14:13:31 -05002608 if (folio_test_anon(folio))
2609 extra_pins = folio_test_swapcache(folio) ?
2610 folio_nr_pages(folio) : 0;
Huang Yingb8f593c2017-07-06 15:37:28 -07002611 else
Matthew Wilcox (Oracle)d4b40842022-02-04 14:13:31 -05002612 extra_pins = folio_nr_pages(folio);
Huang Yingb8f593c2017-07-06 15:37:28 -07002613 if (pextra_pins)
2614 *pextra_pins = extra_pins;
Matthew Wilcox (Oracle)d4b40842022-02-04 14:13:31 -05002615 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 1;
Huang Yingb8f593c2017-07-06 15:37:28 -07002616}
2617
Andrea Arcangeli6d0a07e2016-05-12 15:42:25 -07002618/*
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002619 * This function splits huge page into normal pages. @page can point to any
2620 * subpage of huge page to split. Split doesn't change the position of @page.
2621 *
2622 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2623 * The huge page must be locked.
2624 *
2625 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
2626 *
2627 * Both head page and tail pages will inherit mapping, flags, and so on from
2628 * the hugepage.
2629 *
2630 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
2631 * they are not mapped.
2632 *
2633 * Returns 0 if the hugepage is split successfully.
2634 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2635 * us.
2636 */
2637int split_huge_page_to_list(struct page *page, struct list_head *list)
2638{
Matthew Wilcox (Oracle)4eecb8b2022-01-28 23:32:59 -05002639 struct folio *folio = page_folio(page);
Matthew Wilcox (Oracle)f8baa6b2023-01-11 14:29:12 +00002640 struct deferred_split *ds_queue = get_deferred_split_queue(folio);
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002641 XA_STATE(xas, &folio->mapping->i_pages, folio->index);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002642 struct anon_vma *anon_vma = NULL;
2643 struct address_space *mapping = NULL;
Yang Shi504e0702021-06-15 18:24:07 -07002644 int extra_pins, ret;
Hugh Dickins006d3ff2018-11-30 14:10:21 -08002645 pgoff_t end;
Xu Yu478d1342022-04-28 23:14:43 -07002646 bool is_hzp;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002647
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002648 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2649 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002650
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002651 is_hzp = is_huge_zero_page(&folio->page);
Naoya Horiguchi4737edb2023-04-06 17:20:04 +09002652 if (is_hzp) {
2653 pr_warn_ratelimited("Called split_huge_page for huge zero page\n");
Xu Yu478d1342022-04-28 23:14:43 -07002654 return -EBUSY;
Naoya Horiguchi4737edb2023-04-06 17:20:04 +09002655 }
Xu Yu478d1342022-04-28 23:14:43 -07002656
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002657 if (folio_test_writeback(folio))
Huang Ying59807682017-09-06 16:22:34 -07002658 return -EBUSY;
2659
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002660 if (folio_test_anon(folio)) {
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002661 /*
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -07002662 * The caller does not necessarily hold an mmap_lock that would
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002663 * prevent the anon_vma disappearing so we first we take a
2664 * reference to it and then lock the anon_vma for write. This
Matthew Wilcox (Oracle)2f031c62022-01-29 16:06:53 -05002665 * is similar to folio_lock_anon_vma_read except the write lock
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002666 * is taken to serialise against parallel split or collapse
2667 * operations.
2668 */
Matthew Wilcox (Oracle)29eea9b2022-09-02 20:46:50 +01002669 anon_vma = folio_get_anon_vma(folio);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002670 if (!anon_vma) {
2671 ret = -EBUSY;
2672 goto out;
2673 }
Hugh Dickins006d3ff2018-11-30 14:10:21 -08002674 end = -1;
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002675 mapping = NULL;
2676 anon_vma_lock_write(anon_vma);
2677 } else {
Yin Fengwei6a3edd292022-08-10 14:49:07 +08002678 gfp_t gfp;
2679
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002680 mapping = folio->mapping;
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002681
2682 /* Truncated ? */
2683 if (!mapping) {
2684 ret = -EBUSY;
2685 goto out;
2686 }
2687
Yin Fengwei6a3edd292022-08-10 14:49:07 +08002688 gfp = current_gfp_context(mapping_gfp_mask(mapping) &
2689 GFP_RECLAIM_MASK);
2690
2691 if (folio_test_private(folio) &&
2692 !filemap_release_folio(folio, gfp)) {
2693 ret = -EBUSY;
2694 goto out;
2695 }
2696
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002697 xas_split_alloc(&xas, folio, folio_order(folio), gfp);
Matthew Wilcox (Oracle)6b24ca42020-06-27 22:19:08 -04002698 if (xas_error(&xas)) {
2699 ret = xas_error(&xas);
2700 goto out;
2701 }
2702
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002703 anon_vma = NULL;
2704 i_mmap_lock_read(mapping);
Hugh Dickins006d3ff2018-11-30 14:10:21 -08002705
2706 /*
2707 *__split_huge_page() may need to trim off pages beyond EOF:
2708 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock,
2709 * which cannot be nested inside the page tree lock. So note
2710 * end now: i_size itself may be changed at any moment, but
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002711 * folio lock is good enough to serialize the trimming.
Hugh Dickins006d3ff2018-11-30 14:10:21 -08002712 */
2713 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
Hugh Dickinsd144bf62021-09-02 14:54:21 -07002714 if (shmem_mapping(mapping))
2715 end = shmem_fallocend(mapping->host, end);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002716 }
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002717
2718 /*
Matthew Wilcox (Oracle)684555a2022-09-02 20:46:49 +01002719 * Racy check if we can split the page, before unmap_folio() will
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002720 * split PMDs
2721 */
Matthew Wilcox (Oracle)d4b40842022-02-04 14:13:31 -05002722 if (!can_split_folio(folio, &extra_pins)) {
Baolin Wangfd4a7ac2022-10-24 16:34:22 +08002723 ret = -EAGAIN;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002724 goto out_unlock;
2725 }
2726
Matthew Wilcox (Oracle)684555a2022-09-02 20:46:49 +01002727 unmap_folio(folio);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002728
Alex Shib6769832020-12-15 12:33:33 -08002729 /* block interrupt reentry in xa_lock and spinlock */
2730 local_irq_disable();
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002731 if (mapping) {
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002732 /*
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002733 * Check if the folio is present in page cache.
2734 * We assume all tail are present too, if folio is there.
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002735 */
Matthew Wilcox (Oracle)6b24ca42020-06-27 22:19:08 -04002736 xas_lock(&xas);
2737 xas_reset(&xas);
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002738 if (xas_load(&xas) != folio)
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002739 goto fail;
2740 }
2741
Joonsoo Kim0139aa72016-05-19 17:10:49 -07002742 /* Prevent deferred_split_scan() touching ->_refcount */
Yang Shi364c1ee2019-09-23 15:38:06 -07002743 spin_lock(&ds_queue->split_queue_lock);
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002744 if (folio_ref_freeze(folio, 1 + extra_pins)) {
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002745 if (!list_empty(&folio->_deferred_list)) {
Yang Shi364c1ee2019-09-23 15:38:06 -07002746 ds_queue->split_queue_len--;
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002747 list_del(&folio->_deferred_list);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002748 }
Wei Yangafb97172020-01-30 22:14:35 -08002749 spin_unlock(&ds_queue->split_queue_lock);
Kirill A. Shutemov06d3eff2019-10-18 20:20:30 -07002750 if (mapping) {
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002751 int nr = folio_nr_pages(folio);
Muchun Songbf9ecea2021-02-24 12:03:27 -08002752
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002753 xas_split(&xas, folio, folio_order(folio));
2754 if (folio_test_swapbacked(folio)) {
2755 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS,
Muchun Song57b28472021-02-24 12:03:31 -08002756 -nr);
Marek Szyprowski1ca75542021-10-18 15:16:19 -07002757 } else {
Matthew Wilcox (Oracle)3e9a13d2022-09-02 20:46:48 +01002758 __lruvec_stat_mod_folio(folio, NR_FILE_THPS,
Muchun Songbf9ecea2021-02-24 12:03:27 -08002759 -nr);
Marek Szyprowski1ca75542021-10-18 15:16:19 -07002760 filemap_nr_thps_dec(mapping);
2761 }
Kirill A. Shutemov06d3eff2019-10-18 20:20:30 -07002762 }
2763
Alex Shib6769832020-12-15 12:33:33 -08002764 __split_huge_page(page, list, end);
Huang Yingc4f9c702020-10-15 20:06:07 -07002765 ret = 0;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002766 } else {
Yang Shi364c1ee2019-09-23 15:38:06 -07002767 spin_unlock(&ds_queue->split_queue_lock);
Yang Shi504e0702021-06-15 18:24:07 -07002768fail:
2769 if (mapping)
Matthew Wilcox (Oracle)6b24ca42020-06-27 22:19:08 -04002770 xas_unlock(&xas);
Alex Shib6769832020-12-15 12:33:33 -08002771 local_irq_enable();
Matthew Wilcox (Oracle)4eecb8b2022-01-28 23:32:59 -05002772 remap_page(folio, folio_nr_pages(folio));
Baolin Wangfd4a7ac2022-10-24 16:34:22 +08002773 ret = -EAGAIN;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002774 }
2775
2776out_unlock:
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07002777 if (anon_vma) {
2778 anon_vma_unlock_write(anon_vma);
2779 put_anon_vma(anon_vma);
2780 }
2781 if (mapping)
2782 i_mmap_unlock_read(mapping);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002783out:
Matthew Wilcox (Oracle)69a37a82022-06-08 15:18:34 -04002784 xas_destroy(&xas);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002785 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
2786 return ret;
2787}
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002788
2789void free_transhuge_page(struct page *page)
2790{
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002791 struct folio *folio = (struct folio *)page;
Matthew Wilcox (Oracle)f8baa6b2023-01-11 14:29:12 +00002792 struct deferred_split *ds_queue = get_deferred_split_queue(folio);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002793 unsigned long flags;
2794
Yang Shi364c1ee2019-09-23 15:38:06 -07002795 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002796 if (!list_empty(&folio->_deferred_list)) {
Yang Shi364c1ee2019-09-23 15:38:06 -07002797 ds_queue->split_queue_len--;
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002798 list_del(&folio->_deferred_list);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002799 }
Yang Shi364c1ee2019-09-23 15:38:06 -07002800 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002801 free_compound_page(page);
2802}
2803
Matthew Wilcox (Oracle)f158ed62023-01-11 14:29:13 +00002804void deferred_split_folio(struct folio *folio)
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002805{
Matthew Wilcox (Oracle)f8baa6b2023-01-11 14:29:12 +00002806 struct deferred_split *ds_queue = get_deferred_split_queue(folio);
Yang Shi87eaceb2019-09-23 15:38:15 -07002807#ifdef CONFIG_MEMCG
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002808 struct mem_cgroup *memcg = folio_memcg(folio);
Yang Shi87eaceb2019-09-23 15:38:15 -07002809#endif
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002810 unsigned long flags;
2811
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002812 VM_BUG_ON_FOLIO(folio_order(folio) < 2, folio);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002813
Yang Shi87eaceb2019-09-23 15:38:15 -07002814 /*
2815 * The try_to_unmap() in page reclaim path might reach here too,
2816 * this may cause a race condition to corrupt deferred split queue.
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002817 * And, if page reclaim is already handling the same folio, it is
Yang Shi87eaceb2019-09-23 15:38:15 -07002818 * unnecessary to handle it again in shrinker.
2819 *
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002820 * Check the swapcache flag to determine if the folio is being
2821 * handled by page reclaim since THP swap would add the folio into
Yang Shi87eaceb2019-09-23 15:38:15 -07002822 * swap cache before calling try_to_unmap().
2823 */
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002824 if (folio_test_swapcache(folio))
Yang Shi87eaceb2019-09-23 15:38:15 -07002825 return;
2826
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002827 if (!list_empty(&folio->_deferred_list))
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002828 return;
2829
Yang Shi364c1ee2019-09-23 15:38:06 -07002830 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002831 if (list_empty(&folio->_deferred_list)) {
Kirill A. Shutemovf9719a02016-03-17 14:18:45 -07002832 count_vm_event(THP_DEFERRED_SPLIT_PAGE);
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002833 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
Yang Shi364c1ee2019-09-23 15:38:06 -07002834 ds_queue->split_queue_len++;
Yang Shi87eaceb2019-09-23 15:38:15 -07002835#ifdef CONFIG_MEMCG
2836 if (memcg)
Matthew Wilcox (Oracle)8991de92023-01-11 14:29:11 +00002837 set_shrinker_bit(memcg, folio_nid(folio),
Yang Shi2bfd3632021-05-04 18:36:11 -07002838 deferred_split_shrinker.id);
Yang Shi87eaceb2019-09-23 15:38:15 -07002839#endif
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002840 }
Yang Shi364c1ee2019-09-23 15:38:06 -07002841 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002842}
2843
2844static unsigned long deferred_split_count(struct shrinker *shrink,
2845 struct shrink_control *sc)
2846{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08002847 struct pglist_data *pgdata = NODE_DATA(sc->nid);
Yang Shi364c1ee2019-09-23 15:38:06 -07002848 struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
Yang Shi87eaceb2019-09-23 15:38:15 -07002849
2850#ifdef CONFIG_MEMCG
2851 if (sc->memcg)
2852 ds_queue = &sc->memcg->deferred_split_queue;
2853#endif
Yang Shi364c1ee2019-09-23 15:38:06 -07002854 return READ_ONCE(ds_queue->split_queue_len);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002855}
2856
2857static unsigned long deferred_split_scan(struct shrinker *shrink,
2858 struct shrink_control *sc)
2859{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08002860 struct pglist_data *pgdata = NODE_DATA(sc->nid);
Yang Shi364c1ee2019-09-23 15:38:06 -07002861 struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002862 unsigned long flags;
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002863 LIST_HEAD(list);
2864 struct folio *folio, *next;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002865 int split = 0;
2866
Yang Shi87eaceb2019-09-23 15:38:15 -07002867#ifdef CONFIG_MEMCG
2868 if (sc->memcg)
2869 ds_queue = &sc->memcg->deferred_split_queue;
2870#endif
2871
Yang Shi364c1ee2019-09-23 15:38:06 -07002872 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002873 /* Take pin on all head pages to avoid freeing them under us */
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002874 list_for_each_entry_safe(folio, next, &ds_queue->split_queue,
2875 _deferred_list) {
2876 if (folio_try_get(folio)) {
2877 list_move(&folio->_deferred_list, &list);
Kirill A. Shutemove3ae1952016-02-02 16:57:15 -08002878 } else {
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002879 /* We lost race with folio_put() */
2880 list_del_init(&folio->_deferred_list);
Yang Shi364c1ee2019-09-23 15:38:06 -07002881 ds_queue->split_queue_len--;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002882 }
Kirill A. Shutemove3ae1952016-02-02 16:57:15 -08002883 if (!--sc->nr_to_scan)
2884 break;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002885 }
Yang Shi364c1ee2019-09-23 15:38:06 -07002886 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002887
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002888 list_for_each_entry_safe(folio, next, &list, _deferred_list) {
2889 if (!folio_trylock(folio))
Kirill A. Shutemovfa41b902018-03-22 16:17:31 -07002890 goto next;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002891 /* split_huge_page() removes page from list on success */
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002892 if (!split_folio(folio))
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002893 split++;
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002894 folio_unlock(folio);
Kirill A. Shutemovfa41b902018-03-22 16:17:31 -07002895next:
Matthew Wilcox (Oracle)4375a552023-01-11 14:29:10 +00002896 folio_put(folio);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002897 }
2898
Yang Shi364c1ee2019-09-23 15:38:06 -07002899 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
2900 list_splice_tail(&list, &ds_queue->split_queue);
2901 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002902
Kirill A. Shutemovcb8d68e2016-02-02 16:57:12 -08002903 /*
2904 * Stop shrinker if we didn't split any page, but the queue is empty.
2905 * This can happen if pages were freed under us.
2906 */
Yang Shi364c1ee2019-09-23 15:38:06 -07002907 if (!split && list_empty(&ds_queue->split_queue))
Kirill A. Shutemovcb8d68e2016-02-02 16:57:12 -08002908 return SHRINK_STOP;
2909 return split;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002910}
2911
2912static struct shrinker deferred_split_shrinker = {
2913 .count_objects = deferred_split_count,
2914 .scan_objects = deferred_split_scan,
2915 .seeks = DEFAULT_SEEKS,
Yang Shi87eaceb2019-09-23 15:38:15 -07002916 .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE |
2917 SHRINKER_NONSLAB,
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002918};
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002919
2920#ifdef CONFIG_DEBUG_FS
Zi Yanfa6c0232021-05-04 18:34:23 -07002921static void split_huge_pages_all(void)
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002922{
2923 struct zone *zone;
2924 struct page *page;
Kefeng Wang630e7c52022-12-29 20:25:03 +08002925 struct folio *folio;
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002926 unsigned long pfn, max_zone_pfn;
2927 unsigned long total = 0, split = 0;
2928
Zi Yanfa6c0232021-05-04 18:34:23 -07002929 pr_debug("Split all THPs\n");
Miaohe Lina17206d2022-07-04 21:21:57 +08002930 for_each_zone(zone) {
2931 if (!managed_zone(zone))
2932 continue;
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002933 max_zone_pfn = zone_end_pfn(zone);
2934 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
Miaohe Lina17206d2022-07-04 21:21:57 +08002935 int nr_pages;
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002936
Naoya Horiguchi2b7aa912022-09-08 13:11:50 +09002937 page = pfn_to_online_page(pfn);
Kefeng Wang630e7c52022-12-29 20:25:03 +08002938 if (!page || PageTail(page))
2939 continue;
2940 folio = page_folio(page);
2941 if (!folio_try_get(folio))
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002942 continue;
2943
Kefeng Wang630e7c52022-12-29 20:25:03 +08002944 if (unlikely(page_folio(page) != folio))
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002945 goto next;
2946
Kefeng Wang630e7c52022-12-29 20:25:03 +08002947 if (zone != folio_zone(folio))
2948 goto next;
2949
2950 if (!folio_test_large(folio)
2951 || folio_test_hugetlb(folio)
2952 || !folio_test_lru(folio))
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002953 goto next;
2954
2955 total++;
Kefeng Wang630e7c52022-12-29 20:25:03 +08002956 folio_lock(folio);
2957 nr_pages = folio_nr_pages(folio);
2958 if (!split_folio(folio))
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002959 split++;
Miaohe Lina17206d2022-07-04 21:21:57 +08002960 pfn += nr_pages - 1;
Kefeng Wang630e7c52022-12-29 20:25:03 +08002961 folio_unlock(folio);
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002962next:
Kefeng Wang630e7c52022-12-29 20:25:03 +08002963 folio_put(folio);
Zi Yanfa6c0232021-05-04 18:34:23 -07002964 cond_resched();
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002965 }
2966 }
2967
Zi Yanfa6c0232021-05-04 18:34:23 -07002968 pr_debug("%lu of %lu THP split\n", split, total);
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08002969}
Zi Yanfa6c0232021-05-04 18:34:23 -07002970
2971static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
2972{
2973 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||
2974 is_vm_hugetlb_page(vma);
2975}
2976
2977static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
2978 unsigned long vaddr_end)
2979{
2980 int ret = 0;
2981 struct task_struct *task;
2982 struct mm_struct *mm;
2983 unsigned long total = 0, split = 0;
2984 unsigned long addr;
2985
2986 vaddr_start &= PAGE_MASK;
2987 vaddr_end &= PAGE_MASK;
2988
2989 /* Find the task_struct from pid */
2990 rcu_read_lock();
2991 task = find_task_by_vpid(pid);
2992 if (!task) {
2993 rcu_read_unlock();
2994 ret = -ESRCH;
2995 goto out;
2996 }
2997 get_task_struct(task);
2998 rcu_read_unlock();
2999
3000 /* Find the mm_struct */
3001 mm = get_task_mm(task);
3002 put_task_struct(task);
3003
3004 if (!mm) {
3005 ret = -EINVAL;
3006 goto out;
3007 }
3008
3009 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n",
3010 pid, vaddr_start, vaddr_end);
3011
3012 mmap_read_lock(mm);
3013 /*
3014 * always increase addr by PAGE_SIZE, since we could have a PTE page
3015 * table filled with PTE-mapped THPs, each of which is distinct.
3016 */
3017 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) {
Miaohe Lin74ba2b32022-07-04 21:21:52 +08003018 struct vm_area_struct *vma = vma_lookup(mm, addr);
Zi Yanfa6c0232021-05-04 18:34:23 -07003019 struct page *page;
3020
Miaohe Lin74ba2b32022-07-04 21:21:52 +08003021 if (!vma)
Zi Yanfa6c0232021-05-04 18:34:23 -07003022 break;
3023
3024 /* skip special VMA and hugetlb VMA */
3025 if (vma_not_suitable_for_thp_split(vma)) {
3026 addr = vma->vm_end;
3027 continue;
3028 }
3029
3030 /* FOLL_DUMP to ignore special (like zero) pages */
Miaohe Lin87d27622022-03-22 14:45:29 -07003031 page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);
Zi Yanfa6c0232021-05-04 18:34:23 -07003032
Haiyue Wangf7091ed2022-08-23 21:58:41 +08003033 if (IS_ERR_OR_NULL(page))
Zi Yanfa6c0232021-05-04 18:34:23 -07003034 continue;
3035
3036 if (!is_transparent_hugepage(page))
3037 goto next;
3038
3039 total++;
Matthew Wilcox (Oracle)d4b40842022-02-04 14:13:31 -05003040 if (!can_split_folio(page_folio(page), NULL))
Zi Yanfa6c0232021-05-04 18:34:23 -07003041 goto next;
3042
3043 if (!trylock_page(page))
3044 goto next;
3045
3046 if (!split_huge_page(page))
3047 split++;
3048
3049 unlock_page(page);
3050next:
3051 put_page(page);
3052 cond_resched();
3053 }
3054 mmap_read_unlock(mm);
3055 mmput(mm);
3056
3057 pr_debug("%lu of %lu THP split\n", split, total);
3058
3059out:
3060 return ret;
3061}
3062
Zi Yanfbe37502021-05-04 18:34:26 -07003063static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start,
3064 pgoff_t off_end)
3065{
3066 struct filename *file;
3067 struct file *candidate;
3068 struct address_space *mapping;
3069 int ret = -EINVAL;
3070 pgoff_t index;
3071 int nr_pages = 1;
3072 unsigned long total = 0, split = 0;
3073
3074 file = getname_kernel(file_path);
3075 if (IS_ERR(file))
3076 return ret;
3077
3078 candidate = file_open_name(file, O_RDONLY, 0);
3079 if (IS_ERR(candidate))
3080 goto out;
3081
3082 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n",
3083 file_path, off_start, off_end);
3084
3085 mapping = candidate->f_mapping;
3086
3087 for (index = off_start; index < off_end; index += nr_pages) {
Christoph Hellwig1fb130b2023-03-07 15:34:04 +01003088 struct folio *folio = filemap_get_folio(mapping, index);
Zi Yanfbe37502021-05-04 18:34:26 -07003089
3090 nr_pages = 1;
Christoph Hellwig66dabbb2023-03-07 15:34:10 +01003091 if (IS_ERR(folio))
Zi Yanfbe37502021-05-04 18:34:26 -07003092 continue;
3093
Matthew Wilcox (Oracle)9ee2c082022-10-19 19:33:29 +01003094 if (!folio_test_large(folio))
Zi Yanfbe37502021-05-04 18:34:26 -07003095 goto next;
3096
3097 total++;
Matthew Wilcox (Oracle)9ee2c082022-10-19 19:33:29 +01003098 nr_pages = folio_nr_pages(folio);
Zi Yanfbe37502021-05-04 18:34:26 -07003099
Matthew Wilcox (Oracle)9ee2c082022-10-19 19:33:29 +01003100 if (!folio_trylock(folio))
Zi Yanfbe37502021-05-04 18:34:26 -07003101 goto next;
3102
Matthew Wilcox (Oracle)9ee2c082022-10-19 19:33:29 +01003103 if (!split_folio(folio))
Zi Yanfbe37502021-05-04 18:34:26 -07003104 split++;
3105
Matthew Wilcox (Oracle)9ee2c082022-10-19 19:33:29 +01003106 folio_unlock(folio);
Zi Yanfbe37502021-05-04 18:34:26 -07003107next:
Matthew Wilcox (Oracle)9ee2c082022-10-19 19:33:29 +01003108 folio_put(folio);
Zi Yanfbe37502021-05-04 18:34:26 -07003109 cond_resched();
3110 }
3111
3112 filp_close(candidate, NULL);
3113 ret = 0;
3114
3115 pr_debug("%lu of %lu file-backed THP split\n", split, total);
3116out:
3117 putname(file);
3118 return ret;
3119}
3120
Zi Yanfa6c0232021-05-04 18:34:23 -07003121#define MAX_INPUT_BUF_SZ 255
3122
3123static ssize_t split_huge_pages_write(struct file *file, const char __user *buf,
3124 size_t count, loff_t *ppops)
3125{
3126 static DEFINE_MUTEX(split_debug_mutex);
3127 ssize_t ret;
Zi Yanfbe37502021-05-04 18:34:26 -07003128 /* hold pid, start_vaddr, end_vaddr or file_path, off_start, off_end */
3129 char input_buf[MAX_INPUT_BUF_SZ];
Zi Yanfa6c0232021-05-04 18:34:23 -07003130 int pid;
3131 unsigned long vaddr_start, vaddr_end;
3132
3133 ret = mutex_lock_interruptible(&split_debug_mutex);
3134 if (ret)
3135 return ret;
3136
3137 ret = -EFAULT;
3138
3139 memset(input_buf, 0, MAX_INPUT_BUF_SZ);
3140 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ)))
3141 goto out;
3142
3143 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0';
Zi Yanfbe37502021-05-04 18:34:26 -07003144
3145 if (input_buf[0] == '/') {
3146 char *tok;
3147 char *buf = input_buf;
3148 char file_path[MAX_INPUT_BUF_SZ];
3149 pgoff_t off_start = 0, off_end = 0;
3150 size_t input_len = strlen(input_buf);
3151
3152 tok = strsep(&buf, ",");
3153 if (tok) {
Matthew Wilcox (Oracle)1212e002021-06-30 18:52:11 -07003154 strcpy(file_path, tok);
Zi Yanfbe37502021-05-04 18:34:26 -07003155 } else {
3156 ret = -EINVAL;
3157 goto out;
3158 }
3159
3160 ret = sscanf(buf, "0x%lx,0x%lx", &off_start, &off_end);
3161 if (ret != 2) {
3162 ret = -EINVAL;
3163 goto out;
3164 }
3165 ret = split_huge_pages_in_file(file_path, off_start, off_end);
3166 if (!ret)
3167 ret = input_len;
3168
3169 goto out;
3170 }
3171
Zi Yanfa6c0232021-05-04 18:34:23 -07003172 ret = sscanf(input_buf, "%d,0x%lx,0x%lx", &pid, &vaddr_start, &vaddr_end);
3173 if (ret == 1 && pid == 1) {
3174 split_huge_pages_all();
3175 ret = strlen(input_buf);
3176 goto out;
3177 } else if (ret != 3) {
3178 ret = -EINVAL;
3179 goto out;
3180 }
3181
3182 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end);
3183 if (!ret)
3184 ret = strlen(input_buf);
3185out:
3186 mutex_unlock(&split_debug_mutex);
3187 return ret;
3188
3189}
3190
3191static const struct file_operations split_huge_pages_fops = {
3192 .owner = THIS_MODULE,
3193 .write = split_huge_pages_write,
3194 .llseek = no_llseek,
3195};
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08003196
3197static int __init split_huge_pages_debugfs(void)
3198{
Greg Kroah-Hartmand9f79792019-03-05 15:46:09 -08003199 debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
3200 &split_huge_pages_fops);
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08003201 return 0;
3202}
3203late_initcall(split_huge_pages_debugfs);
3204#endif
Zi Yan616b8372017-09-08 16:10:57 -07003205
3206#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
David Hildenbrand7f5abe62022-05-09 18:20:44 -07003207int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
Zi Yan616b8372017-09-08 16:10:57 -07003208 struct page *page)
3209{
3210 struct vm_area_struct *vma = pvmw->vma;
3211 struct mm_struct *mm = vma->vm_mm;
3212 unsigned long address = pvmw->address;
David Hildenbrand6c287602022-05-09 18:20:44 -07003213 bool anon_exclusive;
Zi Yan616b8372017-09-08 16:10:57 -07003214 pmd_t pmdval;
3215 swp_entry_t entry;
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07003216 pmd_t pmdswp;
Zi Yan616b8372017-09-08 16:10:57 -07003217
3218 if (!(pvmw->pmd && !pvmw->pte))
David Hildenbrand7f5abe62022-05-09 18:20:44 -07003219 return 0;
Zi Yan616b8372017-09-08 16:10:57 -07003220
Zi Yan616b8372017-09-08 16:10:57 -07003221 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
Huang Ying8a8683a2020-03-05 22:28:29 -08003222 pmdval = pmdp_invalidate(vma, address, pvmw->pmd);
David Hildenbrand6c287602022-05-09 18:20:44 -07003223
David Hildenbrand088b8aa2022-09-01 10:35:59 +02003224 /* See page_try_share_anon_rmap(): invalidate PMD first. */
David Hildenbrand6c287602022-05-09 18:20:44 -07003225 anon_exclusive = PageAnon(page) && PageAnonExclusive(page);
3226 if (anon_exclusive && page_try_share_anon_rmap(page)) {
3227 set_pmd_at(mm, address, pvmw->pmd, pmdval);
David Hildenbrand7f5abe62022-05-09 18:20:44 -07003228 return -EBUSY;
David Hildenbrand6c287602022-05-09 18:20:44 -07003229 }
3230
Zi Yan616b8372017-09-08 16:10:57 -07003231 if (pmd_dirty(pmdval))
3232 set_page_dirty(page);
Alistair Popple4dd845b2021-06-30 18:54:09 -07003233 if (pmd_write(pmdval))
3234 entry = make_writable_migration_entry(page_to_pfn(page));
David Hildenbrand6c287602022-05-09 18:20:44 -07003235 else if (anon_exclusive)
3236 entry = make_readable_exclusive_migration_entry(page_to_pfn(page));
Alistair Popple4dd845b2021-06-30 18:54:09 -07003237 else
3238 entry = make_readable_migration_entry(page_to_pfn(page));
Peter Xu2e346872022-08-11 12:13:29 -04003239 if (pmd_young(pmdval))
3240 entry = make_migration_entry_young(entry);
3241 if (pmd_dirty(pmdval))
3242 entry = make_migration_entry_dirty(entry);
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07003243 pmdswp = swp_entry_to_pmd(entry);
3244 if (pmd_soft_dirty(pmdval))
3245 pmdswp = pmd_swp_mksoft_dirty(pmdswp);
David Hildenbrand24bf08c2023-04-05 18:02:35 +02003246 if (pmd_uffd_wp(pmdval))
3247 pmdswp = pmd_swp_mkuffd_wp(pmdswp);
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07003248 set_pmd_at(mm, address, pvmw->pmd, pmdswp);
Hugh Dickinscea86fe2022-02-14 18:26:39 -08003249 page_remove_rmap(page, vma, true);
Zi Yan616b8372017-09-08 16:10:57 -07003250 put_page(page);
Anshuman Khandual283fd6f2022-03-24 18:09:58 -07003251 trace_set_migration_pmd(address, pmd_val(pmdswp));
David Hildenbrand7f5abe62022-05-09 18:20:44 -07003252
3253 return 0;
Zi Yan616b8372017-09-08 16:10:57 -07003254}
3255
3256void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
3257{
3258 struct vm_area_struct *vma = pvmw->vma;
3259 struct mm_struct *mm = vma->vm_mm;
3260 unsigned long address = pvmw->address;
Miaohe Lin4fba8f22022-07-04 21:21:51 +08003261 unsigned long haddr = address & HPAGE_PMD_MASK;
Zi Yan616b8372017-09-08 16:10:57 -07003262 pmd_t pmde;
3263 swp_entry_t entry;
3264
3265 if (!(pvmw->pmd && !pvmw->pte))
3266 return;
3267
3268 entry = pmd_to_swp_entry(*pvmw->pmd);
3269 get_page(new);
Peter Xu2e346872022-08-11 12:13:29 -04003270 pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot));
Naoya Horiguchiab6e3d02017-09-08 16:11:04 -07003271 if (pmd_swp_soft_dirty(*pvmw->pmd))
3272 pmde = pmd_mksoft_dirty(pmde);
David Hildenbrand3c811f72023-04-11 16:25:10 +02003273 if (is_writable_migration_entry(entry))
David Hildenbrandf3ebdf02023-04-18 16:21:13 +02003274 pmde = pmd_mkwrite(pmde);
Peter Xu8f34f1e2021-06-30 18:49:02 -07003275 if (pmd_swp_uffd_wp(*pvmw->pmd))
Peter Xuf1eb1ba2022-12-14 15:15:33 -05003276 pmde = pmd_mkuffd_wp(pmde);
Peter Xu2e346872022-08-11 12:13:29 -04003277 if (!is_migration_entry_young(entry))
3278 pmde = pmd_mkold(pmde);
3279 /* NOTE: this may contain setting soft-dirty on some archs */
3280 if (PageDirty(new) && is_migration_entry_dirty(entry))
3281 pmde = pmd_mkdirty(pmde);
Zi Yan616b8372017-09-08 16:10:57 -07003282
David Hildenbrand6c287602022-05-09 18:20:44 -07003283 if (PageAnon(new)) {
3284 rmap_t rmap_flags = RMAP_COMPOUND;
3285
3286 if (!is_readable_migration_entry(entry))
3287 rmap_flags |= RMAP_EXCLUSIVE;
3288
Miaohe Lin4fba8f22022-07-04 21:21:51 +08003289 page_add_anon_rmap(new, vma, haddr, rmap_flags);
David Hildenbrand6c287602022-05-09 18:20:44 -07003290 } else {
Hugh Dickinscea86fe2022-02-14 18:26:39 -08003291 page_add_file_rmap(new, vma, true);
David Hildenbrand6c287602022-05-09 18:20:44 -07003292 }
3293 VM_BUG_ON(pmd_write(pmde) && PageAnon(new) && !PageAnonExclusive(new));
Miaohe Lin4fba8f22022-07-04 21:21:51 +08003294 set_pmd_at(mm, haddr, pvmw->pmd, pmde);
Muchun Song5cbcf222022-03-22 14:41:53 -07003295
3296 /* No need to invalidate - it was non-present before */
Zi Yan616b8372017-09-08 16:10:57 -07003297 update_mmu_cache_pmd(vma, address, pvmw->pmd);
Anshuman Khandual283fd6f2022-03-24 18:09:58 -07003298 trace_remove_migration_pmd(address, pmd_val(pmde));
Zi Yan616b8372017-09-08 16:10:57 -07003299}
3300#endif