blob: 382e81c33fc437f2e11e49a1f8a7d68e99e5c845 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * mm/mremap.c
4 *
5 * (C) Copyright 1996 Linus Torvalds
6 *
Alan Cox046c6882009-01-05 14:06:29 +00007 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
9 */
10
11#include <linux/mm.h>
Jakub Matěnaca3d76b2022-06-03 16:57:19 +020012#include <linux/mm_inline.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/hugetlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/shm.h>
Hugh Dickins1ff829952009-09-21 17:02:05 -070015#include <linux/ksm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/mman.h>
17#include <linux/swap.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080018#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/fs.h>
Cyrill Gorcunov6dec97d2013-08-27 12:37:18 +040020#include <linux/swapops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/highmem.h>
22#include <linux/security.h>
23#include <linux/syscalls.h>
Andrea Arcangelicddb8a52008-07-28 15:46:29 -070024#include <linux/mmu_notifier.h>
Paul McQuade2581d202014-10-09 15:29:01 -070025#include <linux/uaccess.h>
Pavel Emelyanov72f87652017-02-22 15:42:34 -080026#include <linux/userfaultfd_k.h>
Jakub Matěnaca3d76b2022-06-03 16:57:19 +020027#include <linux/mempolicy.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <asm/cacheflush.h>
Aneesh Kumar K.V3bbda692021-07-07 18:10:18 -070030#include <asm/tlb.h>
Aneesh Kumar K.V0881ace2021-07-07 18:10:12 -070031#include <asm/pgalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Rik van Rielba470de2008-10-18 20:26:50 -070033#include "internal.h"
34
Kalesh Singhc49dd342020-12-14 19:07:30 -080035static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -070036{
37 pgd_t *pgd;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +030038 p4d_t *p4d;
Linus Torvalds1da177e2005-04-16 15:20:36 -070039 pud_t *pud;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
41 pgd = pgd_offset(mm, addr);
42 if (pgd_none_or_clear_bad(pgd))
43 return NULL;
44
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +030045 p4d = p4d_offset(pgd, addr);
46 if (p4d_none_or_clear_bad(p4d))
47 return NULL;
48
49 pud = pud_offset(p4d, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 if (pud_none_or_clear_bad(pud))
51 return NULL;
52
Kalesh Singhc49dd342020-12-14 19:07:30 -080053 return pud;
54}
55
56static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
57{
58 pud_t *pud;
59 pmd_t *pmd;
60
61 pud = get_old_pud(mm, addr);
62 if (!pud)
63 return NULL;
64
Linus Torvalds1da177e2005-04-16 15:20:36 -070065 pmd = pmd_offset(pud, addr);
Andrea Arcangeli37a1c492011-10-31 17:08:30 -070066 if (pmd_none(*pmd))
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 return NULL;
68
Hugh Dickins7be7a542005-10-29 18:16:00 -070069 return pmd;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070}
71
Kalesh Singhc49dd342020-12-14 19:07:30 -080072static pud_t *alloc_new_pud(struct mm_struct *mm, struct vm_area_struct *vma,
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -080073 unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -070074{
75 pgd_t *pgd;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +030076 p4d_t *p4d;
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78 pgd = pgd_offset(mm, addr);
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +030079 p4d = p4d_alloc(mm, pgd, addr);
80 if (!p4d)
81 return NULL;
Kalesh Singhc49dd342020-12-14 19:07:30 -080082
83 return pud_alloc(mm, p4d, addr);
84}
85
86static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
87 unsigned long addr)
88{
89 pud_t *pud;
90 pmd_t *pmd;
91
92 pud = alloc_new_pud(mm, vma, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 if (!pud)
Hugh Dickinsc74df322005-10-29 18:16:23 -070094 return NULL;
Hugh Dickins7be7a542005-10-29 18:16:00 -070095
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 pmd = pmd_alloc(mm, pud, addr);
Hugh Dickins57a8f0c2013-10-16 13:47:09 -070097 if (!pmd)
Hugh Dickinsc74df322005-10-29 18:16:23 -070098 return NULL;
Hugh Dickins7be7a542005-10-29 18:16:00 -070099
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -0800100 VM_BUG_ON(pmd_trans_huge(*pmd));
Hugh Dickinsc74df322005-10-29 18:16:23 -0700101
Hugh Dickins7be7a542005-10-29 18:16:00 -0700102 return pmd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103}
104
Hugh Dickins1d069b72016-05-19 17:12:57 -0700105static void take_rmap_locks(struct vm_area_struct *vma)
106{
107 if (vma->vm_file)
108 i_mmap_lock_write(vma->vm_file->f_mapping);
109 if (vma->anon_vma)
110 anon_vma_lock_write(vma->anon_vma);
111}
112
113static void drop_rmap_locks(struct vm_area_struct *vma)
114{
115 if (vma->anon_vma)
116 anon_vma_unlock_write(vma->anon_vma);
117 if (vma->vm_file)
118 i_mmap_unlock_write(vma->vm_file->f_mapping);
119}
120
Cyrill Gorcunov6dec97d2013-08-27 12:37:18 +0400121static pte_t move_soft_dirty_pte(pte_t pte)
122{
123 /*
124 * Set soft dirty bit so we can notice
125 * in userspace the ptes were moved.
126 */
127#ifdef CONFIG_MEM_SOFT_DIRTY
128 if (pte_present(pte))
129 pte = pte_mksoft_dirty(pte);
130 else if (is_swap_pte(pte))
131 pte = pte_swp_mksoft_dirty(pte);
Cyrill Gorcunov6dec97d2013-08-27 12:37:18 +0400132#endif
133 return pte;
134}
135
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700136static int move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
Hugh Dickins7be7a542005-10-29 18:16:00 -0700137 unsigned long old_addr, unsigned long old_end,
138 struct vm_area_struct *new_vma, pmd_t *new_pmd,
Linus Torvaldseb66ae02018-10-12 15:22:59 -0700139 unsigned long new_addr, bool need_rmap_locks)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141 struct mm_struct *mm = vma->vm_mm;
Hugh Dickins7be7a542005-10-29 18:16:00 -0700142 pte_t *old_pte, *new_pte, pte;
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700143 spinlock_t *old_ptl, *new_ptl;
Aaron Lu5d190422016-11-10 17:16:33 +0800144 bool force_flush = false;
145 unsigned long len = old_end - old_addr;
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700146 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147
Michel Lespinasse38a76012012-10-08 16:31:50 -0700148 /*
Davidlohr Buesoc8c06ef2014-12-12 16:54:24 -0800149 * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma
Michel Lespinasse38a76012012-10-08 16:31:50 -0700150 * locks to ensure that rmap will always observe either the old or the
151 * new ptes. This is the easiest way to avoid races with
152 * truncate_pagecache(), page migration, etc...
153 *
154 * When need_rmap_locks is false, we use other ways to avoid
155 * such races:
156 *
157 * - During exec() shift_arg_pages(), we use a specially tagged vma
Anshuman Khandual222100e2020-04-01 21:07:52 -0700158 * which rmap call sites look for using vma_is_temporary_stack().
Michel Lespinasse38a76012012-10-08 16:31:50 -0700159 *
160 * - During mremap(), new_vma is often known to be placed after vma
161 * in rmap traversal order. This ensures rmap will always observe
162 * either the old pte, or the new pte, or both (the page table locks
163 * serialize access to individual ptes, but only rmap traversal
164 * order guarantees that we won't miss both the old and new ptes).
165 */
Hugh Dickins1d069b72016-05-19 17:12:57 -0700166 if (need_rmap_locks)
167 take_rmap_locks(vma);
Hugh Dickins7be7a542005-10-29 18:16:00 -0700168
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700169 /*
170 * We don't have to worry about the ordering of src and dst
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700171 * pte locks because exclusive mmap_lock prevents deadlock.
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700172 */
Hugh Dickinsc74df322005-10-29 18:16:23 -0700173 old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700174 if (!old_pte) {
175 err = -EAGAIN;
176 goto out;
177 }
178 new_pte = pte_offset_map_nolock(mm, new_pmd, new_addr, &new_ptl);
179 if (!new_pte) {
180 pte_unmap_unlock(old_pte, old_ptl);
181 err = -EAGAIN;
182 goto out;
183 }
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700184 if (new_ptl != old_ptl)
Ingo Molnarf20dc5f2006-07-03 00:25:08 -0700185 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
Mel Gorman3ea27712017-08-02 13:31:52 -0700186 flush_tlb_batched_pending(vma->vm_mm);
Zachary Amsden6606c3e2006-09-30 23:29:33 -0700187 arch_enter_lazy_mmu_mode();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Hugh Dickins7be7a542005-10-29 18:16:00 -0700189 for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
190 new_pte++, new_addr += PAGE_SIZE) {
Ryan Robertsc33c7942023-06-12 16:15:45 +0100191 if (pte_none(ptep_get(old_pte)))
Hugh Dickins7be7a542005-10-29 18:16:00 -0700192 continue;
Aaron Lu5d190422016-11-10 17:16:33 +0800193
Andrea Arcangeli7b6efc22011-10-31 17:08:26 -0700194 pte = ptep_get_and_clear(mm, old_addr, old_pte);
Aaron Lua2ce2662016-11-29 13:27:31 +0800195 /*
Linus Torvaldseb66ae02018-10-12 15:22:59 -0700196 * If we are remapping a valid PTE, make sure
Aaron Lua2ce2662016-11-29 13:27:31 +0800197 * to flush TLB before we drop the PTL for the
Linus Torvaldseb66ae02018-10-12 15:22:59 -0700198 * PTE.
Aaron Lua2ce2662016-11-29 13:27:31 +0800199 *
Linus Torvaldseb66ae02018-10-12 15:22:59 -0700200 * NOTE! Both old and new PTL matter: the old one
201 * for racing with page_mkclean(), the new one to
202 * make sure the physical page stays valid until
203 * the TLB entry for the old mapping has been
204 * flushed.
Aaron Lua2ce2662016-11-29 13:27:31 +0800205 */
Linus Torvaldseb66ae02018-10-12 15:22:59 -0700206 if (pte_present(pte))
Aaron Lua2ce2662016-11-29 13:27:31 +0800207 force_flush = true;
Hugh Dickins7be7a542005-10-29 18:16:00 -0700208 pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
Cyrill Gorcunov6dec97d2013-08-27 12:37:18 +0400209 pte = move_soft_dirty_pte(pte);
210 set_pte_at(mm, new_addr, new_pte, pte);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 }
Hugh Dickins7be7a542005-10-29 18:16:00 -0700212
Zachary Amsden6606c3e2006-09-30 23:29:33 -0700213 arch_leave_lazy_mmu_mode();
Linus Torvaldseb66ae02018-10-12 15:22:59 -0700214 if (force_flush)
215 flush_tlb_range(vma, old_end - len, old_end);
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700216 if (new_ptl != old_ptl)
217 spin_unlock(new_ptl);
Peter Zijlstraece0e2b2010-10-26 14:21:52 -0700218 pte_unmap(new_pte - 1);
Hugh Dickinsc74df322005-10-29 18:16:23 -0700219 pte_unmap_unlock(old_pte - 1, old_ptl);
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700220out:
Hugh Dickins1d069b72016-05-19 17:12:57 -0700221 if (need_rmap_locks)
222 drop_rmap_locks(vma);
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700223 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224}
225
Aneesh Kumar K.V3bbda692021-07-07 18:10:18 -0700226#ifndef arch_supports_page_table_move
227#define arch_supports_page_table_move arch_supports_page_table_move
228static inline bool arch_supports_page_table_move(void)
229{
230 return IS_ENABLED(CONFIG_HAVE_MOVE_PMD) ||
231 IS_ENABLED(CONFIG_HAVE_MOVE_PUD);
232}
233#endif
234
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800235#ifdef CONFIG_HAVE_MOVE_PMD
236static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
Wei Yangb8aa9d92020-08-06 23:23:40 -0700237 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800238{
239 spinlock_t *old_ptl, *new_ptl;
240 struct mm_struct *mm = vma->vm_mm;
241 pmd_t pmd;
242
Aneesh Kumar K.V3bbda692021-07-07 18:10:18 -0700243 if (!arch_supports_page_table_move())
244 return false;
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800245 /*
246 * The destination pmd shouldn't be established, free_pgtables()
Linus Torvaldsf81fdd02020-07-13 11:37:39 -0700247 * should have released it.
248 *
249 * However, there's a case during execve() where we use mremap
250 * to move the initial stack, and in that case the target area
251 * may overlap the source area (always moving down).
252 *
253 * If everything is PMD-aligned, that works fine, as moving
254 * each pmd down will clear the source pmd. But if we first
255 * have a few 4kB-only pages that get moved down, and then
256 * hit the "now the rest is PMD-aligned, let's do everything
257 * one pmd at a time", we will still have the old (now empty
258 * of any 4kB pages, but still there) PMD in the page table
259 * tree.
260 *
261 * Warn on it once - because we really should try to figure
262 * out how to do this better - but then say "I won't move
263 * this pmd".
264 *
265 * One alternative might be to just unmap the target pmd at
266 * this point, and verify that it really is empty. We'll see.
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800267 */
Linus Torvaldsf81fdd02020-07-13 11:37:39 -0700268 if (WARN_ON_ONCE(!pmd_none(*new_pmd)))
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800269 return false;
270
271 /*
272 * We don't have to worry about the ordering of src and dst
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700273 * ptlocks because exclusive mmap_lock prevents deadlock.
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800274 */
275 old_ptl = pmd_lock(vma->vm_mm, old_pmd);
276 new_ptl = pmd_lockptr(mm, new_pmd);
277 if (new_ptl != old_ptl)
278 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
279
280 /* Clear the pmd */
281 pmd = *old_pmd;
282 pmd_clear(old_pmd);
283
284 VM_BUG_ON(!pmd_none(*new_pmd));
285
Aneesh Kumar K.V0881ace2021-07-07 18:10:12 -0700286 pmd_populate(mm, new_pmd, pmd_pgtable(pmd));
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800287 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
288 if (new_ptl != old_ptl)
289 spin_unlock(new_ptl);
290 spin_unlock(old_ptl);
291
292 return true;
293}
Kalesh Singhc49dd342020-12-14 19:07:30 -0800294#else
295static inline bool move_normal_pmd(struct vm_area_struct *vma,
296 unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd,
297 pmd_t *new_pmd)
298{
299 return false;
300}
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800301#endif
302
Aneesh Kumar K.Vd6655df2021-07-07 18:10:09 -0700303#if CONFIG_PGTABLE_LEVELS > 2 && defined(CONFIG_HAVE_MOVE_PUD)
Kalesh Singhc49dd342020-12-14 19:07:30 -0800304static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
305 unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
306{
307 spinlock_t *old_ptl, *new_ptl;
308 struct mm_struct *mm = vma->vm_mm;
309 pud_t pud;
310
Aneesh Kumar K.V3bbda692021-07-07 18:10:18 -0700311 if (!arch_supports_page_table_move())
312 return false;
Kalesh Singhc49dd342020-12-14 19:07:30 -0800313 /*
314 * The destination pud shouldn't be established, free_pgtables()
315 * should have released it.
316 */
317 if (WARN_ON_ONCE(!pud_none(*new_pud)))
318 return false;
319
320 /*
321 * We don't have to worry about the ordering of src and dst
322 * ptlocks because exclusive mmap_lock prevents deadlock.
323 */
324 old_ptl = pud_lock(vma->vm_mm, old_pud);
325 new_ptl = pud_lockptr(mm, new_pud);
326 if (new_ptl != old_ptl)
327 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
328
329 /* Clear the pud */
330 pud = *old_pud;
331 pud_clear(old_pud);
332
333 VM_BUG_ON(!pud_none(*new_pud));
334
Aneesh Kumar K.V0881ace2021-07-07 18:10:12 -0700335 pud_populate(mm, new_pud, pud_pgtable(pud));
Kalesh Singhc49dd342020-12-14 19:07:30 -0800336 flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE);
337 if (new_ptl != old_ptl)
338 spin_unlock(new_ptl);
339 spin_unlock(old_ptl);
340
341 return true;
342}
343#else
344static inline bool move_normal_pud(struct vm_area_struct *vma,
345 unsigned long old_addr, unsigned long new_addr, pud_t *old_pud,
346 pud_t *new_pud)
347{
348 return false;
349}
350#endif
351
Aneesh Kumar K.V54a948a2023-07-25 00:37:52 +0530352#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
Aneesh Kumar K.V7d846db2021-07-07 18:10:06 -0700353static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr,
354 unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
355{
356 spinlock_t *old_ptl, *new_ptl;
357 struct mm_struct *mm = vma->vm_mm;
358 pud_t pud;
359
360 /*
361 * The destination pud shouldn't be established, free_pgtables()
362 * should have released it.
363 */
364 if (WARN_ON_ONCE(!pud_none(*new_pud)))
365 return false;
366
367 /*
368 * We don't have to worry about the ordering of src and dst
369 * ptlocks because exclusive mmap_lock prevents deadlock.
370 */
371 old_ptl = pud_lock(vma->vm_mm, old_pud);
372 new_ptl = pud_lockptr(mm, new_pud);
373 if (new_ptl != old_ptl)
374 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
375
376 /* Clear the pud */
377 pud = *old_pud;
378 pud_clear(old_pud);
379
380 VM_BUG_ON(!pud_none(*new_pud));
381
382 /* Set the new pud */
383 /* mark soft_ditry when we add pud level soft dirty support */
384 set_pud_at(mm, new_addr, new_pud, pud);
385 flush_pud_tlb_range(vma, old_addr, old_addr + HPAGE_PUD_SIZE);
386 if (new_ptl != old_ptl)
387 spin_unlock(new_ptl);
388 spin_unlock(old_ptl);
389
390 return true;
391}
392#else
393static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr,
394 unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
395{
396 WARN_ON_ONCE(1);
397 return false;
398
399}
400#endif
401
Kalesh Singhc49dd342020-12-14 19:07:30 -0800402enum pgt_entry {
403 NORMAL_PMD,
404 HPAGE_PMD,
405 NORMAL_PUD,
Aneesh Kumar K.V7d846db2021-07-07 18:10:06 -0700406 HPAGE_PUD,
Kalesh Singhc49dd342020-12-14 19:07:30 -0800407};
408
409/*
410 * Returns an extent of the corresponding size for the pgt_entry specified if
411 * valid. Else returns a smaller extent bounded by the end of the source and
412 * destination pgt_entry.
413 */
Arnd Bergmanna30a2902021-02-09 13:42:10 -0800414static __always_inline unsigned long get_extent(enum pgt_entry entry,
415 unsigned long old_addr, unsigned long old_end,
416 unsigned long new_addr)
Kalesh Singhc49dd342020-12-14 19:07:30 -0800417{
418 unsigned long next, extent, mask, size;
419
420 switch (entry) {
421 case HPAGE_PMD:
422 case NORMAL_PMD:
423 mask = PMD_MASK;
424 size = PMD_SIZE;
425 break;
Aneesh Kumar K.V7d846db2021-07-07 18:10:06 -0700426 case HPAGE_PUD:
Kalesh Singhc49dd342020-12-14 19:07:30 -0800427 case NORMAL_PUD:
428 mask = PUD_MASK;
429 size = PUD_SIZE;
430 break;
431 default:
432 BUILD_BUG();
433 break;
434 }
435
436 next = (old_addr + size) & mask;
437 /* even if next overflowed, extent below will be ok */
Kalesh Singhe05986e2020-12-29 15:14:40 -0800438 extent = next - old_addr;
439 if (extent > old_end - old_addr)
440 extent = old_end - old_addr;
Kalesh Singhc49dd342020-12-14 19:07:30 -0800441 next = (new_addr + size) & mask;
442 if (extent > next - new_addr)
443 extent = next - new_addr;
444 return extent;
445}
446
447/*
448 * Attempts to speedup the move by moving entry at the level corresponding to
449 * pgt_entry. Returns true if the move was successful, else false.
450 */
451static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma,
452 unsigned long old_addr, unsigned long new_addr,
453 void *old_entry, void *new_entry, bool need_rmap_locks)
454{
455 bool moved = false;
456
457 /* See comment in move_ptes() */
458 if (need_rmap_locks)
459 take_rmap_locks(vma);
460
461 switch (entry) {
462 case NORMAL_PMD:
463 moved = move_normal_pmd(vma, old_addr, new_addr, old_entry,
464 new_entry);
465 break;
466 case NORMAL_PUD:
467 moved = move_normal_pud(vma, old_addr, new_addr, old_entry,
468 new_entry);
469 break;
470 case HPAGE_PMD:
471 moved = IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
472 move_huge_pmd(vma, old_addr, new_addr, old_entry,
473 new_entry);
474 break;
Aneesh Kumar K.V7d846db2021-07-07 18:10:06 -0700475 case HPAGE_PUD:
476 moved = IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
477 move_huge_pud(vma, old_addr, new_addr, old_entry,
478 new_entry);
479 break;
480
Kalesh Singhc49dd342020-12-14 19:07:30 -0800481 default:
482 WARN_ON_ONCE(1);
483 break;
484 }
485
486 if (need_rmap_locks)
487 drop_rmap_locks(vma);
488
489 return moved;
490}
491
Ollie Wildb6a2fea2007-07-19 01:48:16 -0700492unsigned long move_page_tables(struct vm_area_struct *vma,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 unsigned long old_addr, struct vm_area_struct *new_vma,
Michel Lespinasse38a76012012-10-08 16:31:50 -0700494 unsigned long new_addr, unsigned long len,
495 bool need_rmap_locks)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496{
Kalesh Singhc49dd342020-12-14 19:07:30 -0800497 unsigned long extent, old_end;
Jérôme Glisseac46d4f2018-12-28 00:38:09 -0800498 struct mmu_notifier_range range;
Hugh Dickins7be7a542005-10-29 18:16:00 -0700499 pmd_t *old_pmd, *new_pmd;
Aneesh Kumar K.V7d846db2021-07-07 18:10:06 -0700500 pud_t *old_pud, *new_pud;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501
Paolo Bonzini01e67e02022-04-08 13:09:04 -0700502 if (!len)
503 return 0;
504
Hugh Dickins7be7a542005-10-29 18:16:00 -0700505 old_end = old_addr + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506
Mina Almasry550a7d62021-11-05 13:41:40 -0700507 if (is_vm_hugetlb_page(vma))
508 return move_hugetlb_page_tables(vma, new_vma, old_addr,
509 new_addr, len);
510
Baolin Wang3d0b95c2022-05-09 18:20:52 -0700511 flush_cache_range(vma, old_addr, old_end);
Alistair Popple7d4a8be2023-01-10 13:57:22 +1100512 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
Jérôme Glisse6f4f13e2019-05-13 17:20:49 -0700513 old_addr, old_end);
Jérôme Glisseac46d4f2018-12-28 00:38:09 -0800514 mmu_notifier_invalidate_range_start(&range);
Andrea Arcangeli7b6efc22011-10-31 17:08:26 -0700515
Hugh Dickins7be7a542005-10-29 18:16:00 -0700516 for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 cond_resched();
Kalesh Singhc49dd342020-12-14 19:07:30 -0800518 /*
519 * If extent is PUD-sized try to speed up the move by moving at the
520 * PUD level if possible.
521 */
522 extent = get_extent(NORMAL_PUD, old_addr, old_end, new_addr);
Kalesh Singhc49dd342020-12-14 19:07:30 -0800523
Aneesh Kumar K.V7d846db2021-07-07 18:10:06 -0700524 old_pud = get_old_pud(vma->vm_mm, old_addr);
525 if (!old_pud)
526 continue;
527 new_pud = alloc_new_pud(vma->vm_mm, vma, new_addr);
528 if (!new_pud)
529 break;
530 if (pud_trans_huge(*old_pud) || pud_devmap(*old_pud)) {
531 if (extent == HPAGE_PUD_SIZE) {
532 move_pgt_entry(HPAGE_PUD, vma, old_addr, new_addr,
533 old_pud, new_pud, need_rmap_locks);
534 /* We ignore and continue on error? */
Kalesh Singhc49dd342020-12-14 19:07:30 -0800535 continue;
Aneesh Kumar K.V7d846db2021-07-07 18:10:06 -0700536 }
537 } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) {
538
Kalesh Singhc49dd342020-12-14 19:07:30 -0800539 if (move_pgt_entry(NORMAL_PUD, vma, old_addr, new_addr,
Aneesh Kumar K.V97113eb2021-07-07 18:10:15 -0700540 old_pud, new_pud, true))
Kalesh Singhc49dd342020-12-14 19:07:30 -0800541 continue;
542 }
543
544 extent = get_extent(NORMAL_PMD, old_addr, old_end, new_addr);
Hugh Dickins7be7a542005-10-29 18:16:00 -0700545 old_pmd = get_old_pmd(vma->vm_mm, old_addr);
546 if (!old_pmd)
547 continue;
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -0800548 new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
Hugh Dickins7be7a542005-10-29 18:16:00 -0700549 if (!new_pmd)
550 break;
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700551again:
Kalesh Singhc49dd342020-12-14 19:07:30 -0800552 if (is_swap_pmd(*old_pmd) || pmd_trans_huge(*old_pmd) ||
553 pmd_devmap(*old_pmd)) {
554 if (extent == HPAGE_PMD_SIZE &&
555 move_pgt_entry(HPAGE_PMD, vma, old_addr, new_addr,
556 old_pmd, new_pmd, need_rmap_locks))
557 continue;
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -0800558 split_huge_pmd(vma, old_pmd, old_addr);
Kalesh Singhc49dd342020-12-14 19:07:30 -0800559 } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) &&
560 extent == PMD_SIZE) {
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800561 /*
562 * If the extent is PMD-sized, try to speed the move by
563 * moving at the PMD level if possible.
564 */
Kalesh Singhc49dd342020-12-14 19:07:30 -0800565 if (move_pgt_entry(NORMAL_PMD, vma, old_addr, new_addr,
Aneesh Kumar K.V97113eb2021-07-07 18:10:15 -0700566 old_pmd, new_pmd, true))
Joel Fernandes (Google)2c91bd4a2019-01-03 15:28:38 -0800567 continue;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -0700568 }
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700569 if (pmd_none(*old_pmd))
570 continue;
Joel Fernandes (Google)4cf58922019-01-03 15:28:34 -0800571 if (pte_alloc(new_vma->vm_mm, new_pmd))
Andrea Arcangeli37a1c492011-10-31 17:08:30 -0700572 break;
Hugh Dickinsa5be6212023-06-08 18:32:47 -0700573 if (move_ptes(vma, old_pmd, old_addr, old_addr + extent,
574 new_vma, new_pmd, new_addr, need_rmap_locks) < 0)
575 goto again;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 }
Andrea Arcangeli7b6efc22011-10-31 17:08:26 -0700577
Jérôme Glisseac46d4f2018-12-28 00:38:09 -0800578 mmu_notifier_invalidate_range_end(&range);
Hugh Dickins7be7a542005-10-29 18:16:00 -0700579
580 return len + old_addr - old_end; /* how much done */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581}
582
583static unsigned long move_vma(struct vm_area_struct *vma,
584 unsigned long old_addr, unsigned long old_len,
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800585 unsigned long new_len, unsigned long new_addr,
Brian Geffone346b382020-04-01 21:09:17 -0700586 bool *locked, unsigned long flags,
587 struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588{
Dmitry Safonovfdbef612021-11-05 13:39:06 -0700589 long to_account = new_len - old_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 struct mm_struct *mm = vma->vm_mm;
591 struct vm_area_struct *new_vma;
592 unsigned long vm_flags = vma->vm_flags;
593 unsigned long new_pgoff;
594 unsigned long moved_len;
Liam R. Howlett6b73cff2023-01-20 11:26:39 -0500595 unsigned long account_start = 0;
596 unsigned long account_end = 0;
Hugh Dickins365e9c872005-10-29 18:16:18 -0700597 unsigned long hiwater_vm;
Dmitry Safonov73d5e062020-12-14 19:08:21 -0800598 int err = 0;
Michel Lespinasse38a76012012-10-08 16:31:50 -0700599 bool need_rmap_locks;
Liam R. Howlett6b73cff2023-01-20 11:26:39 -0500600 struct vma_iterator vmi;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601
602 /*
603 * We'd prefer to avoid failure later on in do_munmap:
604 * which may split one vma into three before unmapping.
605 */
606 if (mm->map_count >= sysctl_max_map_count - 3)
607 return -ENOMEM;
608
Dmitry Safonovfdbef612021-11-05 13:39:06 -0700609 if (unlikely(flags & MREMAP_DONTUNMAP))
610 to_account = new_len;
611
Dmitry Safonov73d5e062020-12-14 19:08:21 -0800612 if (vma->vm_ops && vma->vm_ops->may_split) {
613 if (vma->vm_start != old_addr)
614 err = vma->vm_ops->may_split(vma, old_addr);
615 if (!err && vma->vm_end != old_addr + old_len)
616 err = vma->vm_ops->may_split(vma, old_addr + old_len);
617 if (err)
618 return err;
619 }
620
Hugh Dickins1ff829952009-09-21 17:02:05 -0700621 /*
622 * Advise KSM to break any KSM pages in the area to be moved:
623 * it would be confusing if they were to turn up at the new
624 * location, where they happen to coincide with different KSM
625 * pages recently unmapped. But leave vma->vm_flags as it was,
626 * so KSM can come around to merge on vma and new_vma afterwards.
627 */
Hugh Dickins7103ad32009-09-21 17:02:28 -0700628 err = ksm_madvise(vma, old_addr, old_addr + old_len,
629 MADV_UNMERGEABLE, &vm_flags);
630 if (err)
631 return err;
Hugh Dickins1ff829952009-09-21 17:02:05 -0700632
Dmitry Safonovfdbef612021-11-05 13:39:06 -0700633 if (vm_flags & VM_ACCOUNT) {
634 if (security_vm_enough_memory_mm(mm, to_account >> PAGE_SHIFT))
Dmitry Safonovad8ee772020-12-14 19:08:09 -0800635 return -ENOMEM;
636 }
637
Suren Baghdasaryand6ac235d2023-02-27 09:36:16 -0800638 vma_start_write(vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
Michel Lespinasse38a76012012-10-08 16:31:50 -0700640 new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
641 &need_rmap_locks);
Dmitry Safonovad8ee772020-12-14 19:08:09 -0800642 if (!new_vma) {
Dmitry Safonovfdbef612021-11-05 13:39:06 -0700643 if (vm_flags & VM_ACCOUNT)
644 vm_unacct_memory(to_account >> PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 return -ENOMEM;
Dmitry Safonovad8ee772020-12-14 19:08:09 -0800646 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647
Michel Lespinasse38a76012012-10-08 16:31:50 -0700648 moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
649 need_rmap_locks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 if (moved_len < old_len) {
Oleg Nesterovdf1eab32015-09-04 15:48:01 -0700651 err = -ENOMEM;
Oleg Nesterov5477e702015-09-04 15:48:04 -0700652 } else if (vma->vm_ops && vma->vm_ops->mremap) {
Brian Geffon14d07112021-04-29 22:57:48 -0700653 err = vma->vm_ops->mremap(new_vma);
Oleg Nesterovdf1eab32015-09-04 15:48:01 -0700654 }
655
656 if (unlikely(err)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 /*
658 * On error, move entries back from new area to old,
659 * which will succeed since page tables still there,
660 * and then proceed to unmap new area instead of old.
661 */
Michel Lespinasse38a76012012-10-08 16:31:50 -0700662 move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
663 true);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 vma = new_vma;
665 old_len = new_len;
666 old_addr = new_addr;
Oleg Nesterovdf1eab32015-09-04 15:48:01 -0700667 new_addr = err;
Laurent Dufour4abad2c2015-06-24 16:56:19 -0700668 } else {
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800669 mremap_userfaultfd_prep(new_vma, uf);
Al Virob2edffd2015-04-06 17:48:54 -0400670 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671
Mina Almasry550a7d62021-11-05 13:41:40 -0700672 if (is_vm_hugetlb_page(vma)) {
673 clear_vma_resv_huge_pages(vma);
674 }
675
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 /* Conceal VM_ACCOUNT so old reservation is not undone */
Dmitry Safonovad8ee772020-12-14 19:08:09 -0800677 if (vm_flags & VM_ACCOUNT && !(flags & MREMAP_DONTUNMAP)) {
Suren Baghdasaryan1c712222023-01-26 11:37:49 -0800678 vm_flags_clear(vma, VM_ACCOUNT);
Liam R. Howlett6b73cff2023-01-20 11:26:39 -0500679 if (vma->vm_start < old_addr)
680 account_start = vma->vm_start;
681 if (vma->vm_end > old_addr + old_len)
682 account_end = vma->vm_end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 }
684
Kirill Korotaev71799062005-05-16 21:53:18 -0700685 /*
Hugh Dickins365e9c872005-10-29 18:16:18 -0700686 * If we failed to move page tables we still do total_vm increment
687 * since do_munmap() will decrement it by old_len == new_len.
688 *
689 * Since total_vm is about to be raised artificially high for a
690 * moment, we need to restore high watermark afterwards: if stats
691 * are taken meanwhile, total_vm and hiwater_vm appear too high.
692 * If this were a serious issue, we'd add a flag to do_munmap().
Kirill Korotaev71799062005-05-16 21:53:18 -0700693 */
Hugh Dickins365e9c872005-10-29 18:16:18 -0700694 hiwater_vm = mm->hiwater_vm;
Konstantin Khlebnikov84638332016-01-14 15:22:07 -0800695 vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT);
Kirill Korotaev71799062005-05-16 21:53:18 -0700696
Toshi Kanid9fe4fa2015-12-22 17:54:23 -0700697 /* Tell pfnmap has moved from this vma */
698 if (unlikely(vma->vm_flags & VM_PFNMAP))
Ma Wupengd155df52023-02-17 10:56:15 +0800699 untrack_pfn_clear(vma);
Toshi Kanid9fe4fa2015-12-22 17:54:23 -0700700
Brian Geffone346b382020-04-01 21:09:17 -0700701 if (unlikely(!err && (flags & MREMAP_DONTUNMAP))) {
Brian Geffone346b382020-04-01 21:09:17 -0700702 /* We always clear VM_LOCKED[ONFAULT] on the old vma */
Suren Baghdasaryane430a952023-01-26 11:37:48 -0800703 vm_flags_clear(vma, VM_LOCKED_MASK);
Brian Geffone346b382020-04-01 21:09:17 -0700704
Li Xinhai1583aa22021-02-24 12:04:53 -0800705 /*
706 * anon_vma links of the old vma is no longer needed after its page
707 * table has been moved.
708 */
709 if (new_vma != vma && vma->vm_start == old_addr &&
710 vma->vm_end == (old_addr + old_len))
711 unlink_anon_vmas(vma);
712
Brian Geffone346b382020-04-01 21:09:17 -0700713 /* Because we won't unmap we don't need to touch locked_vm */
Dmitry Safonovad8ee772020-12-14 19:08:09 -0800714 return new_addr;
Brian Geffone346b382020-04-01 21:09:17 -0700715 }
716
Liam R. Howlett6b73cff2023-01-20 11:26:39 -0500717 vma_iter_init(&vmi, mm, old_addr);
Linus Torvalds3cec5042023-09-16 12:31:42 -0700718 if (do_vmi_munmap(&vmi, mm, old_addr, old_len, uf_unmap, false) < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 /* OOM: unable to split vma, just get accounts right */
Dmitry Safonovad8ee772020-12-14 19:08:09 -0800720 if (vm_flags & VM_ACCOUNT && !(flags & MREMAP_DONTUNMAP))
Chen Wandun5e229282021-09-02 14:56:52 -0700721 vm_acct_memory(old_len >> PAGE_SHIFT);
Liam R. Howlett6b73cff2023-01-20 11:26:39 -0500722 account_start = account_end = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 }
Brian Geffone346b382020-04-01 21:09:17 -0700724
725 if (vm_flags & VM_LOCKED) {
726 mm->locked_vm += new_len >> PAGE_SHIFT;
727 *locked = true;
728 }
Dmitry Safonovad8ee772020-12-14 19:08:09 -0800729
Hugh Dickins365e9c872005-10-29 18:16:18 -0700730 mm->hiwater_vm = hiwater_vm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731
732 /* Restore VM_ACCOUNT if one or two pieces of vma left */
Liam R. Howlett6b73cff2023-01-20 11:26:39 -0500733 if (account_start) {
734 vma = vma_prev(&vmi);
Suren Baghdasaryan1c712222023-01-26 11:37:49 -0800735 vm_flags_set(vma, VM_ACCOUNT);
Liam R. Howlett6b73cff2023-01-20 11:26:39 -0500736 }
737
738 if (account_end) {
739 vma = vma_next(&vmi);
Suren Baghdasaryan1c712222023-01-26 11:37:49 -0800740 vm_flags_set(vma, VM_ACCOUNT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 }
742
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 return new_addr;
744}
745
Al Viro54f5de72009-11-24 07:17:46 -0500746static struct vm_area_struct *vma_to_resize(unsigned long addr,
Dmitry Safonovfdbef612021-11-05 13:39:06 -0700747 unsigned long old_len, unsigned long new_len, unsigned long flags)
Al Viro54f5de72009-11-24 07:17:46 -0500748{
749 struct mm_struct *mm = current->mm;
Liam Howlett5aaf07f2021-06-28 19:39:47 -0700750 struct vm_area_struct *vma;
Oleg Nesterov1d391682015-09-04 15:48:10 -0700751 unsigned long pgoff;
Al Viro54f5de72009-11-24 07:17:46 -0500752
Liam Howlett5aaf07f2021-06-28 19:39:47 -0700753 vma = vma_lookup(mm, addr);
754 if (!vma)
Derek6cd576132015-04-15 16:14:02 -0700755 return ERR_PTR(-EFAULT);
Al Viro54f5de72009-11-24 07:17:46 -0500756
Mike Kravetzdba58d32017-09-06 16:20:55 -0700757 /*
758 * !old_len is a special case where an attempt is made to 'duplicate'
759 * a mapping. This makes no sense for private mappings as it will
760 * instead create a fresh/new mapping unrelated to the original. This
761 * is contrary to the basic idea of mremap which creates new mappings
762 * based on the original. There are no known use cases for this
763 * behavior. As a result, fail such attempts.
764 */
765 if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) {
766 pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current->comm, current->pid);
767 return ERR_PTR(-EINVAL);
768 }
769
Brian Geffona4609382021-04-29 22:57:45 -0700770 if ((flags & MREMAP_DONTUNMAP) &&
771 (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)))
Brian Geffone346b382020-04-01 21:09:17 -0700772 return ERR_PTR(-EINVAL);
773
Al Viro54f5de72009-11-24 07:17:46 -0500774 /* We can't remap across vm area boundaries */
775 if (old_len > vma->vm_end - addr)
Derek6cd576132015-04-15 16:14:02 -0700776 return ERR_PTR(-EFAULT);
Al Viro54f5de72009-11-24 07:17:46 -0500777
Oleg Nesterov1d391682015-09-04 15:48:10 -0700778 if (new_len == old_len)
779 return vma;
Linus Torvalds982134b2011-04-07 07:35:50 -0700780
Oleg Nesterov1d391682015-09-04 15:48:10 -0700781 /* Need to be careful about a growing mapping */
782 pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
783 pgoff += vma->vm_pgoff;
784 if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
785 return ERR_PTR(-EINVAL);
786
787 if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
788 return ERR_PTR(-EFAULT);
Al Viro54f5de72009-11-24 07:17:46 -0500789
Andrew Mortonb0cc5e82023-05-22 13:52:10 -0700790 if (!mlock_future_ok(mm, vma->vm_flags, new_len - old_len))
Miaohe Linf4331952022-04-28 23:16:14 -0700791 return ERR_PTR(-EAGAIN);
Al Viro54f5de72009-11-24 07:17:46 -0500792
Konstantin Khlebnikov84638332016-01-14 15:22:07 -0800793 if (!may_expand_vm(mm, vma->vm_flags,
794 (new_len - old_len) >> PAGE_SHIFT))
Derek6cd576132015-04-15 16:14:02 -0700795 return ERR_PTR(-ENOMEM);
Al Viro54f5de72009-11-24 07:17:46 -0500796
Al Viro54f5de72009-11-24 07:17:46 -0500797 return vma;
Al Viro54f5de72009-11-24 07:17:46 -0500798}
799
Michel Lespinasse81909b82013-02-22 16:32:41 -0800800static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800801 unsigned long new_addr, unsigned long new_len, bool *locked,
Brian Geffone346b382020-04-01 21:09:17 -0700802 unsigned long flags, struct vm_userfaultfd_ctx *uf,
Mike Rapoportb2282372017-08-02 13:31:55 -0700803 struct list_head *uf_unmap_early,
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800804 struct list_head *uf_unmap)
Al Viroecc1a892009-11-24 07:28:07 -0500805{
806 struct mm_struct *mm = current->mm;
807 struct vm_area_struct *vma;
808 unsigned long ret = -EINVAL;
Brian Geffone346b382020-04-01 21:09:17 -0700809 unsigned long map_flags = 0;
Al Viroecc1a892009-11-24 07:28:07 -0500810
Alexander Kuleshovf19cb112015-11-05 18:46:57 -0800811 if (offset_in_page(new_addr))
Al Viroecc1a892009-11-24 07:28:07 -0500812 goto out;
813
814 if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
815 goto out;
816
Oleg Nesterov99432422015-09-04 15:48:13 -0700817 /* Ensure the old/new locations do not overlap */
818 if (addr + old_len > new_addr && new_addr + new_len > addr)
Al Viroecc1a892009-11-24 07:28:07 -0500819 goto out;
820
Oscar Salvadorea2c3f62019-03-05 15:50:14 -0800821 /*
822 * move_vma() need us to stay 4 maps below the threshold, otherwise
823 * it will bail out at the very beginning.
824 * That is a problem if we have already unmaped the regions here
825 * (new_addr, and old_addr), because userspace will not know the
826 * state of the vma's after it gets -ENOMEM.
827 * So, to avoid such scenario we can pre-compute if the whole
828 * operation has high chances to success map-wise.
829 * Worst-scenario case is when both vma's (new_addr and old_addr) get
Ingo Molnarf0953a12021-05-06 18:06:47 -0700830 * split in 3 before unmapping it.
Oscar Salvadorea2c3f62019-03-05 15:50:14 -0800831 * That means 2 more maps (1 for each) to the ones we already hold.
832 * Check whether current map count plus 2 still leads us to 4 maps below
833 * the threshold, otherwise return -ENOMEM here to be more safe.
834 */
835 if ((mm->map_count + 2) >= sysctl_max_map_count - 3)
836 return -ENOMEM;
837
Brian Geffone346b382020-04-01 21:09:17 -0700838 if (flags & MREMAP_FIXED) {
839 ret = do_munmap(mm, new_addr, new_len, uf_unmap_early);
840 if (ret)
841 goto out;
842 }
Al Viroecc1a892009-11-24 07:28:07 -0500843
Miaohe Lin3c9fe8b2022-04-28 23:16:14 -0700844 if (old_len > new_len) {
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800845 ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap);
Miaohe Lin3c9fe8b2022-04-28 23:16:14 -0700846 if (ret)
Al Viroecc1a892009-11-24 07:28:07 -0500847 goto out;
848 old_len = new_len;
849 }
850
Dmitry Safonovfdbef612021-11-05 13:39:06 -0700851 vma = vma_to_resize(addr, old_len, new_len, flags);
Al Viroecc1a892009-11-24 07:28:07 -0500852 if (IS_ERR(vma)) {
853 ret = PTR_ERR(vma);
854 goto out;
855 }
856
Brian Geffone346b382020-04-01 21:09:17 -0700857 /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */
858 if (flags & MREMAP_DONTUNMAP &&
859 !may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) {
860 ret = -ENOMEM;
861 goto out;
862 }
863
864 if (flags & MREMAP_FIXED)
865 map_flags |= MAP_FIXED;
866
Al Viro097eed12009-11-24 08:43:52 -0500867 if (vma->vm_flags & VM_MAYSHARE)
868 map_flags |= MAP_SHARED;
Al Viro9206de92009-12-03 15:23:11 -0500869
Al Viro097eed12009-11-24 08:43:52 -0500870 ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
871 ((addr - vma->vm_start) >> PAGE_SHIFT),
872 map_flags);
Gaowei Puff68dac2019-11-30 17:51:03 -0800873 if (IS_ERR_VALUE(ret))
Dmitry Safonovfdbef612021-11-05 13:39:06 -0700874 goto out;
Al Viro097eed12009-11-24 08:43:52 -0500875
Brian Geffone346b382020-04-01 21:09:17 -0700876 /* We got a new mapping */
877 if (!(flags & MREMAP_FIXED))
878 new_addr = ret;
879
880 ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800881 uf_unmap);
Brian Geffone346b382020-04-01 21:09:17 -0700882
Al Viroecc1a892009-11-24 07:28:07 -0500883out:
884 return ret;
885}
886
Al Viro1a0ef852009-11-24 07:43:18 -0500887static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
888{
Al Virof106af42009-11-24 08:25:18 -0500889 unsigned long end = vma->vm_end + delta;
Liam R. Howlett396a44c2022-09-06 19:49:03 +0000890
Al Viro9206de92009-12-03 15:23:11 -0500891 if (end < vma->vm_end) /* overflow */
Al Viro1a0ef852009-11-24 07:43:18 -0500892 return 0;
Liam R. Howlett396a44c2022-09-06 19:49:03 +0000893 if (find_vma_intersection(vma->vm_mm, vma->vm_end, end))
Al Virof106af42009-11-24 08:25:18 -0500894 return 0;
895 if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
896 0, MAP_FIXED) & ~PAGE_MASK)
897 return 0;
Al Viro1a0ef852009-11-24 07:43:18 -0500898 return 1;
899}
900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901/*
902 * Expand (or shrink) an existing mapping, potentially moving it at the
903 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
904 *
905 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
906 * This option implies MREMAP_MAYMOVE.
907 */
Al Viro63a81db2012-05-30 11:32:04 -0400908SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
909 unsigned long, new_len, unsigned long, flags,
910 unsigned long, new_addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911{
Hugh Dickinsd0de32d2005-10-29 18:16:16 -0700912 struct mm_struct *mm = current->mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 struct vm_area_struct *vma;
914 unsigned long ret = -EINVAL;
Michel Lespinasse81909b82013-02-22 16:32:41 -0800915 bool locked = false;
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800916 struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
Mike Rapoportb2282372017-08-02 13:31:55 -0700917 LIST_HEAD(uf_unmap_early);
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800918 LIST_HEAD(uf_unmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919
Will Deaconb2a84de2020-03-25 11:13:46 +0000920 /*
921 * There is a deliberate asymmetry here: we strip the pointer tag
922 * from the old address but leave the new address alone. This is
923 * for consistency with mmap(), where we prevent the creation of
924 * aliasing mappings in userspace by leaving the tag bits of the
925 * mapping address intact. A non-zero tag will cause the subsequent
926 * range checks to reject the address as invalid.
927 *
Jonathan Corbetc3003402023-06-12 06:12:24 -0600928 * See Documentation/arch/arm64/tagged-address-abi.rst for more
929 * information.
Will Deaconb2a84de2020-03-25 11:13:46 +0000930 */
Andrey Konovalov057d33892019-09-25 16:48:30 -0700931 addr = untagged_addr(addr);
932
Brian Geffone346b382020-04-01 21:09:17 -0700933 if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP))
Rasmus Villemoes9a2458a2013-07-08 15:59:48 -0700934 return ret;
935
936 if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE))
937 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938
Brian Geffone346b382020-04-01 21:09:17 -0700939 /*
940 * MREMAP_DONTUNMAP is always a move and it does not allow resizing
941 * in the process.
942 */
943 if (flags & MREMAP_DONTUNMAP &&
944 (!(flags & MREMAP_MAYMOVE) || old_len != new_len))
945 return ret;
946
947
Alexander Kuleshovf19cb112015-11-05 18:46:57 -0800948 if (offset_in_page(addr))
Rasmus Villemoes9a2458a2013-07-08 15:59:48 -0700949 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950
951 old_len = PAGE_ALIGN(old_len);
952 new_len = PAGE_ALIGN(new_len);
953
954 /*
955 * We allow a zero old-len as a special case
956 * for DOS-emu "duplicate shm area" thing. But
957 * a zero new-len is nonsensical.
958 */
959 if (!new_len)
Rasmus Villemoes9a2458a2013-07-08 15:59:48 -0700960 return ret;
961
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700962 if (mmap_write_lock_killable(current->mm))
Michal Hockodc0ef0d2016-05-23 16:25:27 -0700963 return -EINTR;
Miaohe Lin0e6799d2022-03-22 14:42:41 -0700964 vma = vma_lookup(mm, addr);
965 if (!vma) {
Niels Dossche7d1e64962022-05-09 17:34:28 -0700966 ret = -EFAULT;
Mina Almasry550a7d62021-11-05 13:41:40 -0700967 goto out;
968 }
969
970 if (is_vm_hugetlb_page(vma)) {
971 struct hstate *h __maybe_unused = hstate_vma(vma);
972
973 old_len = ALIGN(old_len, huge_page_size(h));
974 new_len = ALIGN(new_len, huge_page_size(h));
975
976 /* addrs must be huge page aligned */
977 if (addr & ~huge_page_mask(h))
978 goto out;
979 if (new_addr & ~huge_page_mask(h))
980 goto out;
981
982 /*
983 * Don't allow remap expansion, because the underlying hugetlb
984 * reservation is not yet capable to handle split reservation.
985 */
986 if (new_len > old_len)
987 goto out;
988 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989
Brian Geffone346b382020-04-01 21:09:17 -0700990 if (flags & (MREMAP_FIXED | MREMAP_DONTUNMAP)) {
Rasmus Villemoes9a2458a2013-07-08 15:59:48 -0700991 ret = mremap_to(addr, old_len, new_addr, new_len,
Brian Geffone346b382020-04-01 21:09:17 -0700992 &locked, flags, &uf, &uf_unmap_early,
993 &uf_unmap);
Al Viroecc1a892009-11-24 07:28:07 -0500994 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 }
996
997 /*
998 * Always allow a shrinking remap: that just unmaps
999 * the unnecessary pages..
Liam R. Howlett183654c2023-01-20 11:26:13 -05001000 * do_vmi_munmap does all the needed commit accounting, and
Liam R. Howlett408579c2023-06-29 22:28:16 -04001001 * unlocks the mmap_lock if so directed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002 */
1003 if (old_len >= new_len) {
Liam R. Howlett183654c2023-01-20 11:26:13 -05001004 VMA_ITERATOR(vmi, mm, addr + new_len);
Yang Shi85a06832018-10-26 15:08:50 -07001005
Liam R. Howlett408579c2023-06-29 22:28:16 -04001006 if (old_len == new_len) {
1007 ret = addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008 goto out;
Liam R. Howlett11f9a212022-09-06 19:48:52 +00001009 }
1010
Liam R. Howlett408579c2023-06-29 22:28:16 -04001011 ret = do_vmi_munmap(&vmi, mm, addr + new_len, old_len - new_len,
1012 &uf_unmap, true);
1013 if (ret)
1014 goto out;
1015
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 ret = addr;
Liam R. Howlett408579c2023-06-29 22:28:16 -04001017 goto out_unlocked;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 }
1019
1020 /*
Al Viroecc1a892009-11-24 07:28:07 -05001021 * Ok, we need to grow..
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022 */
Dmitry Safonovfdbef612021-11-05 13:39:06 -07001023 vma = vma_to_resize(addr, old_len, new_len, flags);
Al Viro54f5de72009-11-24 07:17:46 -05001024 if (IS_ERR(vma)) {
1025 ret = PTR_ERR(vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 }
1028
1029 /* old_len exactly to the end of the area..
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030 */
Al Viroecc1a892009-11-24 07:28:07 -05001031 if (old_len == vma->vm_end - addr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 /* can we just expand the current mapping? */
Al Viro1a0ef852009-11-24 07:43:18 -05001033 if (vma_expandable(vma, new_len - old_len)) {
Dmitry Safonovfdbef612021-11-05 13:39:06 -07001034 long pages = (new_len - old_len) >> PAGE_SHIFT;
Jakub Matěnaca3d76b2022-06-03 16:57:19 +02001035 unsigned long extension_start = addr + old_len;
1036 unsigned long extension_end = addr + new_len;
Vlastimil Babka6f12be72022-12-16 17:32:27 +01001037 pgoff_t extension_pgoff = vma->vm_pgoff +
1038 ((extension_start - vma->vm_start) >> PAGE_SHIFT);
Liam R. Howletta27a11f2023-01-20 11:26:27 -05001039 VMA_ITERATOR(vmi, mm, extension_start);
Dmitry Safonovfdbef612021-11-05 13:39:06 -07001040
1041 if (vma->vm_flags & VM_ACCOUNT) {
1042 if (security_vm_enough_memory_mm(mm, pages)) {
1043 ret = -ENOMEM;
1044 goto out;
1045 }
1046 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047
Jakub Matěnaca3d76b2022-06-03 16:57:19 +02001048 /*
Vlastimil Babkad014cd72023-01-17 11:19:39 +01001049 * Function vma_merge() is called on the extension we
1050 * are adding to the already existing vma, vma_merge()
1051 * will merge this extension with the already existing
1052 * vma (expand operation itself) and possibly also with
1053 * the next vma if it becomes adjacent to the expanded
1054 * vma and otherwise compatible.
Jakub Matěnaca3d76b2022-06-03 16:57:19 +02001055 */
Vlastimil Babka4bfbe372023-03-09 12:12:58 +01001056 vma = vma_merge(&vmi, mm, vma, extension_start,
1057 extension_end, vma->vm_flags, vma->anon_vma,
1058 vma->vm_file, extension_pgoff, vma_policy(vma),
1059 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
Jakub Matěnaca3d76b2022-06-03 16:57:19 +02001060 if (!vma) {
Dmitry Safonovfdbef612021-11-05 13:39:06 -07001061 vm_unacct_memory(pages);
Rik van Riel5beb4932010-03-05 13:42:07 -08001062 ret = -ENOMEM;
1063 goto out;
1064 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065
Konstantin Khlebnikov84638332016-01-14 15:22:07 -08001066 vm_stat_account(mm, vma->vm_flags, pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 if (vma->vm_flags & VM_LOCKED) {
Hugh Dickinsd0de32d2005-10-29 18:16:16 -07001068 mm->locked_vm += pages;
Michel Lespinasse81909b82013-02-22 16:32:41 -08001069 locked = true;
1070 new_addr = addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071 }
1072 ret = addr;
1073 goto out;
1074 }
1075 }
1076
1077 /*
1078 * We weren't able to just expand or shrink the area,
1079 * we need to create a new one and move it..
1080 */
1081 ret = -ENOMEM;
1082 if (flags & MREMAP_MAYMOVE) {
Al Viroecc1a892009-11-24 07:28:07 -05001083 unsigned long map_flags = 0;
1084 if (vma->vm_flags & VM_MAYSHARE)
1085 map_flags |= MAP_SHARED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086
Al Viroecc1a892009-11-24 07:28:07 -05001087 new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
Al Viro93587412009-11-24 08:45:24 -05001088 vma->vm_pgoff +
1089 ((addr - vma->vm_start) >> PAGE_SHIFT),
1090 map_flags);
Gaowei Puff68dac2019-11-30 17:51:03 -08001091 if (IS_ERR_VALUE(new_addr)) {
Al Viroecc1a892009-11-24 07:28:07 -05001092 ret = new_addr;
1093 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 }
Al Viroecc1a892009-11-24 07:28:07 -05001095
Pavel Emelyanov72f87652017-02-22 15:42:34 -08001096 ret = move_vma(vma, addr, old_len, new_len, new_addr,
Brian Geffone346b382020-04-01 21:09:17 -07001097 &locked, flags, &uf, &uf_unmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098 }
1099out:
Dmitry Safonovfdbef612021-11-05 13:39:06 -07001100 if (offset_in_page(ret))
Zou Weifa1f68c2020-06-04 16:49:46 -07001101 locked = false;
Liam R. Howlett408579c2023-06-29 22:28:16 -04001102 mmap_write_unlock(current->mm);
Michel Lespinasse81909b82013-02-22 16:32:41 -08001103 if (locked && new_len > old_len)
1104 mm_populate(new_addr + old_len, new_len - old_len);
Liam R. Howlett408579c2023-06-29 22:28:16 -04001105out_unlocked:
Mike Rapoportb2282372017-08-02 13:31:55 -07001106 userfaultfd_unmap_complete(mm, &uf_unmap_early);
Brian Geffond15649262020-05-13 17:50:44 -07001107 mremap_userfaultfd_complete(&uf, addr, ret, old_len);
Mike Rapoport897ab3e2017-02-24 14:58:22 -08001108 userfaultfd_unmap_complete(mm, &uf_unmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 return ret;
1110}