blob: 7a2abf0127aef7a9d4879278293d8cab766133e1 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/madvise.c
3 *
4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig
6 */
7
8#include <linux/mman.h>
9#include <linux/pagemap.h>
10#include <linux/syscalls.h>
Prasanna Meda05b74382005-06-21 17:14:37 -070011#include <linux/mempolicy.h>
Andi Kleenafcf9382009-12-16 12:20:00 +010012#include <linux/page-isolation.h>
Pavel Emelyanov05ce7722017-02-22 15:42:40 -080013#include <linux/userfaultfd_k.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/hugetlb.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070015#include <linux/falloc.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040016#include <linux/sched.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070017#include <linux/ksm.h>
Hugh Dickins3f31d072012-05-29 15:06:40 -070018#include <linux/fs.h>
Andy Lutomirski9ab42332012-07-05 16:00:11 -070019#include <linux/file.h>
Shaohua Li1998cc02013-02-22 16:32:31 -080020#include <linux/blkdev.h>
Tejun Heo66114ca2015-05-22 17:13:32 -040021#include <linux/backing-dev.h>
Shaohua Li1998cc02013-02-22 16:32:31 -080022#include <linux/swap.h>
23#include <linux/swapops.h>
Hugh Dickins3a4f8a02017-02-24 14:59:36 -080024#include <linux/shmem_fs.h>
Minchan Kim854e9ed2016-01-15 16:54:53 -080025#include <linux/mmu_notifier.h>
26
27#include <asm/tlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Kirill A. Shutemov23519072017-02-22 15:46:39 -080029#include "internal.h"
30
Linus Torvalds1da177e2005-04-16 15:20:36 -070031/*
Nick Piggin0a27a142007-05-06 14:49:53 -070032 * Any behaviour which results in changes to the vma->vm_flags needs to
33 * take mmap_sem for writing. Others, which simply traverse vmas, need
34 * to only take it for reading.
35 */
36static int madvise_need_mmap_write(int behavior)
37{
38 switch (behavior) {
39 case MADV_REMOVE:
40 case MADV_WILLNEED:
41 case MADV_DONTNEED:
Minchan Kim854e9ed2016-01-15 16:54:53 -080042 case MADV_FREE:
Nick Piggin0a27a142007-05-06 14:49:53 -070043 return 0;
44 default:
45 /* be safe, default to 1. list exceptions explicitly */
46 return 1;
47 }
48}
49
50/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 * We can potentially split a vm area into separate
52 * areas, each area with its own behavior.
53 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -070054static long madvise_behavior(struct vm_area_struct *vma,
Prasanna Meda05b74382005-06-21 17:14:37 -070055 struct vm_area_struct **prev,
56 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -070057{
Vladimir Cernovec9bed92013-09-11 14:20:15 -070058 struct mm_struct *mm = vma->vm_mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 int error = 0;
Prasanna Meda05b74382005-06-21 17:14:37 -070060 pgoff_t pgoff;
Hugh Dickins3866ea92009-09-21 17:01:52 -070061 unsigned long new_flags = vma->vm_flags;
Prasanna Medae798c6e2005-06-21 17:14:36 -070062
63 switch (behavior) {
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080064 case MADV_NORMAL:
65 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
66 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -070067 case MADV_SEQUENTIAL:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080068 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070069 break;
70 case MADV_RANDOM:
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080071 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
Prasanna Medae798c6e2005-06-21 17:14:36 -070072 break;
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080073 case MADV_DONTFORK:
74 new_flags |= VM_DONTCOPY;
75 break;
76 case MADV_DOFORK:
Hugh Dickins3866ea92009-09-21 17:01:52 -070077 if (vma->vm_flags & VM_IO) {
78 error = -EINVAL;
79 goto out;
80 }
Michael S. Tsirkinf8225662006-02-14 13:53:08 -080081 new_flags &= ~VM_DONTCOPY;
Prasanna Medae798c6e2005-06-21 17:14:36 -070082 break;
Jason Baronaccb61f2012-03-23 15:02:51 -070083 case MADV_DONTDUMP:
Konstantin Khlebnikov0103bd162012-10-08 16:28:59 -070084 new_flags |= VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070085 break;
86 case MADV_DODUMP:
Konstantin Khlebnikov0103bd162012-10-08 16:28:59 -070087 if (new_flags & VM_SPECIAL) {
88 error = -EINVAL;
89 goto out;
90 }
91 new_flags &= ~VM_DONTDUMP;
Jason Baronaccb61f2012-03-23 15:02:51 -070092 break;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070093 case MADV_MERGEABLE:
94 case MADV_UNMERGEABLE:
95 error = ksm_madvise(vma, start, end, behavior, &new_flags);
David Rientjesdef5efe2017-02-24 14:58:47 -080096 if (error) {
97 /*
98 * madvise() returns EAGAIN if kernel resources, such as
99 * slab, are temporarily unavailable.
100 */
101 if (error == -ENOMEM)
102 error = -EAGAIN;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700103 goto out;
David Rientjesdef5efe2017-02-24 14:58:47 -0800104 }
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700105 break;
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800106 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -0800107 case MADV_NOHUGEPAGE:
Andrea Arcangeli60ab3242011-01-13 15:47:18 -0800108 error = hugepage_madvise(vma, &new_flags, behavior);
David Rientjesdef5efe2017-02-24 14:58:47 -0800109 if (error) {
110 /*
111 * madvise() returns EAGAIN if kernel resources, such as
112 * slab, are temporarily unavailable.
113 */
114 if (error == -ENOMEM)
115 error = -EAGAIN;
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800116 goto out;
David Rientjesdef5efe2017-02-24 14:58:47 -0800117 }
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800118 break;
Prasanna Medae798c6e2005-06-21 17:14:36 -0700119 }
120
Prasanna Meda05b74382005-06-21 17:14:37 -0700121 if (new_flags == vma->vm_flags) {
122 *prev = vma;
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700123 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700124 }
125
126 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
127 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
Andrea Arcangeli19a809a2015-09-04 15:46:24 -0700128 vma->vm_file, pgoff, vma_policy(vma),
129 vma->vm_userfaultfd_ctx);
Prasanna Meda05b74382005-06-21 17:14:37 -0700130 if (*prev) {
131 vma = *prev;
132 goto success;
133 }
134
135 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
137 if (start != vma->vm_start) {
David Rientjesdef5efe2017-02-24 14:58:47 -0800138 if (unlikely(mm->map_count >= sysctl_max_map_count)) {
139 error = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 goto out;
David Rientjesdef5efe2017-02-24 14:58:47 -0800141 }
142 error = __split_vma(mm, vma, start, 1);
143 if (error) {
144 /*
145 * madvise() returns EAGAIN if kernel resources, such as
146 * slab, are temporarily unavailable.
147 */
148 if (error == -ENOMEM)
149 error = -EAGAIN;
150 goto out;
151 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 }
153
154 if (end != vma->vm_end) {
David Rientjesdef5efe2017-02-24 14:58:47 -0800155 if (unlikely(mm->map_count >= sysctl_max_map_count)) {
156 error = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 goto out;
David Rientjesdef5efe2017-02-24 14:58:47 -0800158 }
159 error = __split_vma(mm, vma, end, 0);
160 if (error) {
161 /*
162 * madvise() returns EAGAIN if kernel resources, such as
163 * slab, are temporarily unavailable.
164 */
165 if (error == -ENOMEM)
166 error = -EAGAIN;
167 goto out;
168 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 }
170
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700171success:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 /*
173 * vm_flags is protected by the mmap_sem held in write mode.
174 */
Prasanna Medae798c6e2005-06-21 17:14:36 -0700175 vma->vm_flags = new_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 return error;
178}
179
Shaohua Li1998cc02013-02-22 16:32:31 -0800180#ifdef CONFIG_SWAP
181static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
182 unsigned long end, struct mm_walk *walk)
183{
184 pte_t *orig_pte;
185 struct vm_area_struct *vma = walk->private;
186 unsigned long index;
187
188 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
189 return 0;
190
191 for (index = start; index != end; index += PAGE_SIZE) {
192 pte_t pte;
193 swp_entry_t entry;
194 struct page *page;
195 spinlock_t *ptl;
196
197 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
198 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
199 pte_unmap_unlock(orig_pte, ptl);
200
Kirill A. Shutemov0661a332015-02-10 14:10:04 -0800201 if (pte_present(pte) || pte_none(pte))
Shaohua Li1998cc02013-02-22 16:32:31 -0800202 continue;
203 entry = pte_to_swp_entry(pte);
204 if (unlikely(non_swap_entry(entry)))
205 continue;
206
207 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
208 vma, index);
209 if (page)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300210 put_page(page);
Shaohua Li1998cc02013-02-22 16:32:31 -0800211 }
212
213 return 0;
214}
215
216static void force_swapin_readahead(struct vm_area_struct *vma,
217 unsigned long start, unsigned long end)
218{
219 struct mm_walk walk = {
220 .mm = vma->vm_mm,
221 .pmd_entry = swapin_walk_pmd_entry,
222 .private = vma,
223 };
224
225 walk_page_range(start, end, &walk);
226
227 lru_add_drain(); /* Push any new pages onto the LRU now */
228}
229
230static void force_shm_swapin_readahead(struct vm_area_struct *vma,
231 unsigned long start, unsigned long end,
232 struct address_space *mapping)
233{
234 pgoff_t index;
235 struct page *page;
236 swp_entry_t swap;
237
238 for (; start < end; start += PAGE_SIZE) {
239 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
240
Johannes Weiner55231e52014-05-22 11:54:17 -0700241 page = find_get_entry(mapping, index);
Shaohua Li1998cc02013-02-22 16:32:31 -0800242 if (!radix_tree_exceptional_entry(page)) {
243 if (page)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300244 put_page(page);
Shaohua Li1998cc02013-02-22 16:32:31 -0800245 continue;
246 }
247 swap = radix_to_swp_entry(page);
248 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
249 NULL, 0);
250 if (page)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300251 put_page(page);
Shaohua Li1998cc02013-02-22 16:32:31 -0800252 }
253
254 lru_add_drain(); /* Push any new pages onto the LRU now */
255}
256#endif /* CONFIG_SWAP */
257
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258/*
259 * Schedule all required I/O operations. Do not wait for completion.
260 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700261static long madvise_willneed(struct vm_area_struct *vma,
262 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 unsigned long start, unsigned long end)
264{
265 struct file *file = vma->vm_file;
266
Shaohua Li1998cc02013-02-22 16:32:31 -0800267#ifdef CONFIG_SWAP
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100268 if (!file) {
Shaohua Li1998cc02013-02-22 16:32:31 -0800269 *prev = vma;
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100270 force_swapin_readahead(vma, start, end);
Shaohua Li1998cc02013-02-22 16:32:31 -0800271 return 0;
272 }
Shaohua Li1998cc02013-02-22 16:32:31 -0800273
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100274 if (shmem_mapping(file->f_mapping)) {
275 *prev = vma;
276 force_shm_swapin_readahead(vma, start, end,
277 file->f_mapping);
278 return 0;
279 }
280#else
Suzuki1bef4002005-10-11 08:29:06 -0700281 if (!file)
282 return -EBADF;
Christoph Hellwig97b713b2015-01-14 10:42:31 +0100283#endif
Suzuki1bef4002005-10-11 08:29:06 -0700284
Matthew Wilcoxe748dcd2015-02-16 15:59:12 -0800285 if (IS_DAX(file_inode(file))) {
Carsten Ottefe77ba62005-06-23 22:05:29 -0700286 /* no bad return value, but ignore advice */
287 return 0;
288 }
289
Prasanna Meda05b74382005-06-21 17:14:37 -0700290 *prev = vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
292 if (end > vma->vm_end)
293 end = vma->vm_end;
294 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
295
Wu Fengguangf7e839d2009-06-16 15:31:20 -0700296 force_page_cache_readahead(file->f_mapping, file, start, end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 return 0;
298}
299
Minchan Kim854e9ed2016-01-15 16:54:53 -0800300static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
301 unsigned long end, struct mm_walk *walk)
302
303{
304 struct mmu_gather *tlb = walk->private;
305 struct mm_struct *mm = tlb->mm;
306 struct vm_area_struct *vma = walk->vma;
307 spinlock_t *ptl;
308 pte_t *orig_pte, *pte, ptent;
309 struct page *page;
Minchan Kim64b42bc2016-01-15 16:55:06 -0800310 int nr_swap = 0;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -0800311 unsigned long next;
Minchan Kim854e9ed2016-01-15 16:54:53 -0800312
Minchan Kimb8d3c4c2016-01-15 16:55:42 -0800313 next = pmd_addr_end(addr, end);
314 if (pmd_trans_huge(*pmd))
315 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next))
316 goto next;
317
Minchan Kim854e9ed2016-01-15 16:54:53 -0800318 if (pmd_trans_unstable(pmd))
319 return 0;
320
Aneesh Kumar K.V07e32662016-12-12 16:42:40 -0800321 tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
Minchan Kim854e9ed2016-01-15 16:54:53 -0800322 orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
323 arch_enter_lazy_mmu_mode();
324 for (; addr != end; pte++, addr += PAGE_SIZE) {
325 ptent = *pte;
326
Minchan Kim64b42bc2016-01-15 16:55:06 -0800327 if (pte_none(ptent))
Minchan Kim854e9ed2016-01-15 16:54:53 -0800328 continue;
Minchan Kim64b42bc2016-01-15 16:55:06 -0800329 /*
330 * If the pte has swp_entry, just clear page table to
331 * prevent swap-in which is more expensive rather than
332 * (page allocation + zeroing).
333 */
334 if (!pte_present(ptent)) {
335 swp_entry_t entry;
336
337 entry = pte_to_swp_entry(ptent);
338 if (non_swap_entry(entry))
339 continue;
340 nr_swap--;
341 free_swap_and_cache(entry);
342 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
343 continue;
344 }
Minchan Kim854e9ed2016-01-15 16:54:53 -0800345
346 page = vm_normal_page(vma, addr, ptent);
347 if (!page)
348 continue;
349
350 /*
351 * If pmd isn't transhuge but the page is THP and
352 * is owned by only this process, split it and
353 * deactivate all pages.
354 */
355 if (PageTransCompound(page)) {
356 if (page_mapcount(page) != 1)
357 goto out;
358 get_page(page);
359 if (!trylock_page(page)) {
360 put_page(page);
361 goto out;
362 }
363 pte_unmap_unlock(orig_pte, ptl);
364 if (split_huge_page(page)) {
365 unlock_page(page);
366 put_page(page);
367 pte_offset_map_lock(mm, pmd, addr, &ptl);
368 goto out;
369 }
370 put_page(page);
371 unlock_page(page);
372 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
373 pte--;
374 addr -= PAGE_SIZE;
375 continue;
376 }
377
378 VM_BUG_ON_PAGE(PageTransCompound(page), page);
379
380 if (PageSwapCache(page) || PageDirty(page)) {
381 if (!trylock_page(page))
382 continue;
383 /*
384 * If page is shared with others, we couldn't clear
385 * PG_dirty of the page.
386 */
387 if (page_mapcount(page) != 1) {
388 unlock_page(page);
389 continue;
390 }
391
392 if (PageSwapCache(page) && !try_to_free_swap(page)) {
393 unlock_page(page);
394 continue;
395 }
396
397 ClearPageDirty(page);
398 unlock_page(page);
399 }
400
401 if (pte_young(ptent) || pte_dirty(ptent)) {
402 /*
403 * Some of architecture(ex, PPC) don't update TLB
404 * with set_pte_at and tlb_remove_tlb_entry so for
405 * the portability, remap the pte with old|clean
406 * after pte clearing.
407 */
408 ptent = ptep_get_and_clear_full(mm, addr, pte,
409 tlb->fullmm);
410
411 ptent = pte_mkold(ptent);
412 ptent = pte_mkclean(ptent);
413 set_pte_at(mm, addr, pte, ptent);
Minchan Kim10853a02016-01-15 16:55:11 -0800414 if (PageActive(page))
415 deactivate_page(page);
Minchan Kim854e9ed2016-01-15 16:54:53 -0800416 tlb_remove_tlb_entry(tlb, pte, addr);
417 }
418 }
419out:
Minchan Kim64b42bc2016-01-15 16:55:06 -0800420 if (nr_swap) {
421 if (current->mm == mm)
422 sync_mm_rss(mm);
423
424 add_mm_counter(mm, MM_SWAPENTS, nr_swap);
425 }
Minchan Kim854e9ed2016-01-15 16:54:53 -0800426 arch_leave_lazy_mmu_mode();
427 pte_unmap_unlock(orig_pte, ptl);
428 cond_resched();
Minchan Kimb8d3c4c2016-01-15 16:55:42 -0800429next:
Minchan Kim854e9ed2016-01-15 16:54:53 -0800430 return 0;
431}
432
433static void madvise_free_page_range(struct mmu_gather *tlb,
434 struct vm_area_struct *vma,
435 unsigned long addr, unsigned long end)
436{
437 struct mm_walk free_walk = {
438 .pmd_entry = madvise_free_pte_range,
439 .mm = vma->vm_mm,
440 .private = tlb,
441 };
442
443 tlb_start_vma(tlb, vma);
444 walk_page_range(addr, end, &free_walk);
445 tlb_end_vma(tlb, vma);
446}
447
448static int madvise_free_single_vma(struct vm_area_struct *vma,
449 unsigned long start_addr, unsigned long end_addr)
450{
451 unsigned long start, end;
452 struct mm_struct *mm = vma->vm_mm;
453 struct mmu_gather tlb;
454
455 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
456 return -EINVAL;
457
458 /* MADV_FREE works for only anon vma at the moment */
459 if (!vma_is_anonymous(vma))
460 return -EINVAL;
461
462 start = max(vma->vm_start, start_addr);
463 if (start >= vma->vm_end)
464 return -EINVAL;
465 end = min(vma->vm_end, end_addr);
466 if (end <= vma->vm_start)
467 return -EINVAL;
468
469 lru_add_drain();
470 tlb_gather_mmu(&tlb, mm, start, end);
471 update_hiwater_rss(mm);
472
473 mmu_notifier_invalidate_range_start(mm, start, end);
474 madvise_free_page_range(&tlb, vma, start, end);
475 mmu_notifier_invalidate_range_end(mm, start, end);
476 tlb_finish_mmu(&tlb, start, end);
477
478 return 0;
479}
480
481static long madvise_free(struct vm_area_struct *vma,
482 struct vm_area_struct **prev,
483 unsigned long start, unsigned long end)
484{
485 *prev = vma;
486 return madvise_free_single_vma(vma, start, end);
487}
488
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489/*
490 * Application no longer needs these pages. If the pages are dirty,
491 * it's OK to just throw them away. The app will be more careful about
492 * data it wants to keep. Be sure to free swap resources too. The
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700493 * zap_page_range call sets things up for shrink_active_list to actually free
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 * these pages later if no one else has touched them in the meantime,
495 * although we could add these pages to a global reuse list for
Fernando Luis Vazquez Cao7e6cbea2008-07-29 22:33:39 -0700496 * shrink_active_list to pick up before reclaiming other pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 *
498 * NB: This interface discards data rather than pushes it out to swap,
499 * as some implementations do. This has performance implications for
500 * applications like large transactional databases which want to discard
501 * pages in anonymous maps after committing to backing store the data
502 * that was kept in them. There is no reason to write this data out to
503 * the swap area if the application is discarding it.
504 *
505 * An interface that causes the system to free clean pages and flush
506 * dirty pages is already available as msync(MS_INVALIDATE).
507 */
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700508static long madvise_dontneed(struct vm_area_struct *vma,
509 struct vm_area_struct **prev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 unsigned long start, unsigned long end)
511{
Prasanna Meda05b74382005-06-21 17:14:37 -0700512 *prev = vma;
Kirill A. Shutemov23519072017-02-22 15:46:39 -0800513 if (!can_madv_dontneed_vma(vma))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 return -EINVAL;
515
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800516 if (!userfaultfd_remove(vma, start, end)) {
517 *prev = NULL; /* mmap_sem has been dropped, prev is stale */
518
519 down_read(&current->mm->mmap_sem);
520 vma = find_vma(current->mm, start);
521 if (!vma)
522 return -ENOMEM;
523 if (start < vma->vm_start) {
524 /*
525 * This "vma" under revalidation is the one
526 * with the lowest vma->vm_start where start
527 * is also < vma->vm_end. If start <
528 * vma->vm_start it means an hole materialized
529 * in the user address space within the
530 * virtual range passed to MADV_DONTNEED.
531 */
532 return -ENOMEM;
533 }
534 if (!can_madv_dontneed_vma(vma))
535 return -EINVAL;
536 if (end > vma->vm_end) {
537 /*
538 * Don't fail if end > vma->vm_end. If the old
539 * vma was splitted while the mmap_sem was
540 * released the effect of the concurrent
541 * operation may not cause MADV_DONTNEED to
542 * have an undefined result. There may be an
543 * adjacent next vma that we'll walk
544 * next. userfaultfd_remove() will generate an
545 * UFFD_EVENT_REMOVE repetition on the
546 * end-vma->vm_end range, but the manager can
547 * handle a repetition fine.
548 */
549 end = vma->vm_end;
550 }
551 VM_WARN_ON(start >= end);
552 }
Kirill A. Shutemovecf13852017-02-22 15:46:37 -0800553 zap_page_range(vma, start, end - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 return 0;
555}
556
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800557/*
558 * Application wants to free up the pages and associated backing store.
559 * This is effectively punching a hole into the middle of a file.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800560 */
561static long madvise_remove(struct vm_area_struct *vma,
Nick Piggin00e9fa22007-03-16 13:38:10 -0800562 struct vm_area_struct **prev,
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800563 unsigned long start, unsigned long end)
564{
Hugh Dickins3f31d072012-05-29 15:06:40 -0700565 loff_t offset;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700566 int error;
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700567 struct file *f;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800568
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700569 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
Nick Piggin00e9fa22007-03-16 13:38:10 -0800570
Mike Kravetz72079ba2015-09-08 15:01:57 -0700571 if (vma->vm_flags & VM_LOCKED)
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800572 return -EINVAL;
573
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700574 f = vma->vm_file;
575
576 if (!f || !f->f_mapping || !f->f_mapping->host) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800577 return -EINVAL;
578 }
579
Hugh Dickins69cf0fa2006-04-17 22:46:32 +0100580 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
581 return -EACCES;
582
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800583 offset = (loff_t)(start - vma->vm_start)
584 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700585
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700586 /*
587 * Filesystem's fallocate may need to take i_mutex. We need to
588 * explicitly grab a reference because the vma (and hence the
589 * vma's reference to the file) can go away as soon as we drop
590 * mmap_sem.
591 */
592 get_file(f);
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800593 if (userfaultfd_remove(vma, start, end)) {
594 /* mmap_sem was not released by userfaultfd_remove() */
595 up_read(&current->mm->mmap_sem);
596 }
Anna Schumaker72c72bd2014-11-07 14:44:25 -0500597 error = vfs_fallocate(f,
Hugh Dickins3f31d072012-05-29 15:06:40 -0700598 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
599 offset, end - start);
Andy Lutomirski9ab42332012-07-05 16:00:11 -0700600 fput(f);
Nick Piggin0a27a142007-05-06 14:49:53 -0700601 down_read(&current->mm->mmap_sem);
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700602 return error;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800603}
604
Andi Kleen9893e492009-09-16 11:50:17 +0200605#ifdef CONFIG_MEMORY_FAILURE
606/*
607 * Error injection support for memory error handling.
608 */
Andi Kleenafcf9382009-12-16 12:20:00 +0100609static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
Andi Kleen9893e492009-09-16 11:50:17 +0200610{
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700611 struct page *p;
Andi Kleen9893e492009-09-16 11:50:17 +0200612 if (!capable(CAP_SYS_ADMIN))
613 return -EPERM;
Wanpeng Li20cb6ca2013-09-30 13:45:21 -0700614 for (; start < end; start += PAGE_SIZE <<
615 compound_order(compound_head(p))) {
Andrew Morton325c4ef2013-09-11 14:23:03 -0700616 int ret;
617
618 ret = get_user_pages_fast(start, 1, 0, &p);
Andi Kleen9893e492009-09-16 11:50:17 +0200619 if (ret != 1)
620 return ret;
Andrew Morton325c4ef2013-09-11 14:23:03 -0700621
Wanpeng Li29b4eed2013-09-11 14:22:59 -0700622 if (PageHWPoison(p)) {
623 put_page(p);
624 continue;
625 }
Andi Kleenafcf9382009-12-16 12:20:00 +0100626 if (bhv == MADV_SOFT_OFFLINE) {
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700627 pr_info("Soft offlining page %#lx at %#lx\n",
Andi Kleenafcf9382009-12-16 12:20:00 +0100628 page_to_pfn(p), start);
629 ret = soft_offline_page(p, MF_COUNT_INCREASED);
630 if (ret)
Wanpeng Li83024232013-09-11 14:23:02 -0700631 return ret;
Andi Kleenafcf9382009-12-16 12:20:00 +0100632 continue;
633 }
Wanpeng Lib194b8c2013-09-11 14:22:57 -0700634 pr_info("Injecting memory failure for page %#lx at %#lx\n",
Andi Kleen9893e492009-09-16 11:50:17 +0200635 page_to_pfn(p), start);
Naoya Horiguchi23a003b2016-03-15 14:56:36 -0700636 ret = memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
637 if (ret)
638 return ret;
Andi Kleen9893e492009-09-16 11:50:17 +0200639 }
Andrew Morton325c4ef2013-09-11 14:23:03 -0700640 return 0;
Andi Kleen9893e492009-09-16 11:50:17 +0200641}
642#endif
643
suzuki165cd402005-07-27 11:43:59 -0700644static long
645madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
646 unsigned long start, unsigned long end, int behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 switch (behavior) {
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800649 case MADV_REMOVE:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700650 return madvise_remove(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 case MADV_WILLNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700652 return madvise_willneed(vma, prev, start, end);
Minchan Kim854e9ed2016-01-15 16:54:53 -0800653 case MADV_FREE:
654 /*
655 * XXX: In this implementation, MADV_FREE works like
656 * MADV_DONTNEED on swapless system or full swap.
657 */
658 if (get_nr_swap_pages() > 0)
659 return madvise_free(vma, prev, start, end);
660 /* passthrough */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 case MADV_DONTNEED:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700662 return madvise_dontneed(vma, prev, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 default:
Hugh Dickins3866ea92009-09-21 17:01:52 -0700664 return madvise_behavior(vma, prev, start, end, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666}
667
Nicholas Krause1ecef9e2015-09-04 15:48:24 -0700668static bool
Nick Piggin75927af2009-06-16 15:32:38 -0700669madvise_behavior_valid(int behavior)
670{
671 switch (behavior) {
672 case MADV_DOFORK:
673 case MADV_DONTFORK:
674 case MADV_NORMAL:
675 case MADV_SEQUENTIAL:
676 case MADV_RANDOM:
677 case MADV_REMOVE:
678 case MADV_WILLNEED:
679 case MADV_DONTNEED:
Minchan Kim854e9ed2016-01-15 16:54:53 -0800680 case MADV_FREE:
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700681#ifdef CONFIG_KSM
682 case MADV_MERGEABLE:
683 case MADV_UNMERGEABLE:
684#endif
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800685#ifdef CONFIG_TRANSPARENT_HUGEPAGE
686 case MADV_HUGEPAGE:
Andrea Arcangelia664b2d2011-01-13 15:47:17 -0800687 case MADV_NOHUGEPAGE:
Andrea Arcangeli0af4e982011-01-13 15:46:55 -0800688#endif
Jason Baronaccb61f2012-03-23 15:02:51 -0700689 case MADV_DONTDUMP:
690 case MADV_DODUMP:
Nicholas Krause1ecef9e2015-09-04 15:48:24 -0700691 return true;
Nick Piggin75927af2009-06-16 15:32:38 -0700692
693 default:
Nicholas Krause1ecef9e2015-09-04 15:48:24 -0700694 return false;
Nick Piggin75927af2009-06-16 15:32:38 -0700695 }
696}
Hugh Dickins3866ea92009-09-21 17:01:52 -0700697
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698/*
699 * The madvise(2) system call.
700 *
701 * Applications can use madvise() to advise the kernel how it should
702 * handle paging I/O in this VM area. The idea is to help the kernel
703 * use appropriate read-ahead and caching techniques. The information
704 * provided is advisory only, and can be safely disregarded by the
705 * kernel without affecting the correct operation of the application.
706 *
707 * behavior values:
708 * MADV_NORMAL - the default behavior is to read clusters. This
709 * results in some read-ahead and read-behind.
710 * MADV_RANDOM - the system should read the minimum amount of data
711 * on any access, since it is unlikely that the appli-
712 * cation will need more than what it asks for.
713 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
714 * once, so they can be aggressively read ahead, and
715 * can be freed soon after they are accessed.
716 * MADV_WILLNEED - the application is notifying the system to read
717 * some pages ahead.
718 * MADV_DONTNEED - the application is finished with the given range,
719 * so the kernel can free resources associated with it.
Naoya Horiguchid7206a72016-03-15 14:56:58 -0700720 * MADV_FREE - the application marks pages in the given range as lazy free,
721 * where actual purges are postponed until memory pressure happens.
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800722 * MADV_REMOVE - the application wants to free up the given range of
723 * pages and associated backing store.
Hugh Dickins3866ea92009-09-21 17:01:52 -0700724 * MADV_DONTFORK - omit this area from child's address space when forking:
725 * typically, to avoid COWing pages pinned by get_user_pages().
726 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
Naoya Horiguchid7206a72016-03-15 14:56:58 -0700727 * MADV_HWPOISON - trigger memory error handler as if the given memory range
728 * were corrupted by unrecoverable hardware memory failure.
729 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700730 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
731 * this area with pages of identical content from other such areas.
732 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
Naoya Horiguchid7206a72016-03-15 14:56:58 -0700733 * MADV_HUGEPAGE - the application wants to back the given range by transparent
734 * huge pages in the future. Existing pages might be coalesced and
735 * new pages might be allocated as THP.
736 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
737 * transparent huge pages so the existing pages will not be
738 * coalesced into THP and new pages will not be allocated as THP.
739 * MADV_DONTDUMP - the application wants to prevent pages in the given range
740 * from being included in its core dump.
741 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 *
743 * return values:
744 * zero - success
745 * -EINVAL - start + len < 0, start is not page-aligned,
746 * "behavior" is not a valid value, or application
747 * is attempting to release locked or shared pages.
748 * -ENOMEM - addresses in the specified range are not currently
749 * mapped, or are outside the AS of the process.
750 * -EIO - an I/O error occurred while paging in data.
751 * -EBADF - map exists, but area maps something that isn't a file.
752 * -EAGAIN - a kernel resource was temporarily unavailable.
753 */
Heiko Carstens3480b252009-01-14 14:14:16 +0100754SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755{
Prasanna Meda05b74382005-06-21 17:14:37 -0700756 unsigned long end, tmp;
Vladimir Cernovec9bed92013-09-11 14:20:15 -0700757 struct vm_area_struct *vma, *prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 int unmapped_error = 0;
759 int error = -EINVAL;
Jason Baronf7977792007-07-15 23:38:21 -0700760 int write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 size_t len;
Shaohua Li1998cc02013-02-22 16:32:31 -0800762 struct blk_plug plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
Andi Kleen9893e492009-09-16 11:50:17 +0200764#ifdef CONFIG_MEMORY_FAILURE
Andi Kleenafcf9382009-12-16 12:20:00 +0100765 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
766 return madvise_hwpoison(behavior, start, start+len_in);
Andi Kleen9893e492009-09-16 11:50:17 +0200767#endif
Nick Piggin75927af2009-06-16 15:32:38 -0700768 if (!madvise_behavior_valid(behavior))
769 return error;
770
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700771 if (start & ~PAGE_MASK)
772 return error;
773 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
774
775 /* Check to see whether len was rounded up from small -ve to zero */
776 if (len_in && !len)
777 return error;
778
779 end = start + len;
780 if (end < start)
781 return error;
782
783 error = 0;
784 if (end == start)
785 return error;
786
Jason Baronf7977792007-07-15 23:38:21 -0700787 write = madvise_need_mmap_write(behavior);
Michal Hockodc0ef0d2016-05-23 16:25:27 -0700788 if (write) {
789 if (down_write_killable(&current->mm->mmap_sem))
790 return -EINTR;
791 } else {
Nick Piggin0a27a142007-05-06 14:49:53 -0700792 down_read(&current->mm->mmap_sem);
Michal Hockodc0ef0d2016-05-23 16:25:27 -0700793 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 /*
796 * If the interval [start,end) covers some unmapped address
797 * ranges, just ignore them, but return -ENOMEM at the end.
Prasanna Meda05b74382005-06-21 17:14:37 -0700798 * - different from the way of handling in mlock etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 */
Prasanna Meda05b74382005-06-21 17:14:37 -0700800 vma = find_vma_prev(current->mm, start, &prev);
Hugh Dickins836d5ff2005-09-03 15:54:53 -0700801 if (vma && start > vma->vm_start)
802 prev = vma;
803
Shaohua Li1998cc02013-02-22 16:32:31 -0800804 blk_start_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 for (;;) {
806 /* Still start < end. */
807 error = -ENOMEM;
808 if (!vma)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700809 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810
Prasanna Meda05b74382005-06-21 17:14:37 -0700811 /* Here start < (end|vma->vm_end). */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 if (start < vma->vm_start) {
813 unmapped_error = -ENOMEM;
814 start = vma->vm_start;
Prasanna Meda05b74382005-06-21 17:14:37 -0700815 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700816 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 }
818
Prasanna Meda05b74382005-06-21 17:14:37 -0700819 /* Here vma->vm_start <= start < (end|vma->vm_end) */
820 tmp = vma->vm_end;
821 if (end < tmp)
822 tmp = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823
Prasanna Meda05b74382005-06-21 17:14:37 -0700824 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
825 error = madvise_vma(vma, &prev, start, tmp, behavior);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 if (error)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700827 goto out;
Prasanna Meda05b74382005-06-21 17:14:37 -0700828 start = tmp;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700829 if (prev && start < prev->vm_end)
Prasanna Meda05b74382005-06-21 17:14:37 -0700830 start = prev->vm_end;
831 error = unmapped_error;
832 if (start >= end)
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700833 goto out;
Hugh Dickins90ed52e2007-03-29 01:20:38 -0700834 if (prev)
835 vma = prev->vm_next;
836 else /* madvise_remove dropped mmap_sem */
837 vma = find_vma(current->mm, start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839out:
Rasmus Villemoes84d96d82013-04-29 15:08:23 -0700840 blk_finish_plug(&plug);
Jason Baronf7977792007-07-15 23:38:21 -0700841 if (write)
Nick Piggin0a27a142007-05-06 14:49:53 -0700842 up_write(&current->mm->mmap_sem);
843 else
844 up_read(&current->mm->mmap_sem);
845
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 return error;
847}