blob: 7296e775e04e291700c53d31c0c03295f0c329f7 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/mm/nommu.c
4 *
5 * Replacement code for mm functions to support CPU's that don't
6 * have any form of memory management unit (thus no virtual memory).
7 *
Stephen Kittdd19d292020-08-12 11:22:30 +02008 * See Documentation/admin-guide/mm/nommu-mmap.rst
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 *
David Howells8feae132009-01-08 12:04:47 +000010 * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
12 * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
13 * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com>
Paul Mundt29c185e2010-12-24 12:08:30 +090014 * Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 */
16
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -070017#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040019#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/mm.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010021#include <linux/sched/mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/mman.h>
23#include <linux/swap.h>
24#include <linux/file.h>
25#include <linux/highmem.h>
26#include <linux/pagemap.h>
27#include <linux/slab.h>
28#include <linux/vmalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/backing-dev.h>
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -070030#include <linux/compiler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#include <linux/mount.h>
32#include <linux/personality.h>
33#include <linux/security.h>
34#include <linux/syscalls.h>
Al Viro120a7952010-10-30 02:54:44 -040035#include <linux/audit.h>
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -070036#include <linux/printk.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080038#include <linux/uaccess.h>
Lorenzo Stoakes4c91c072023-03-22 18:57:04 +000039#include <linux/uio.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <asm/tlb.h>
41#include <asm/tlbflush.h>
Bernd Schmidteb8cdec2009-09-21 17:03:57 -070042#include <asm/mmu_context.h>
David Howells8feae132009-01-08 12:04:47 +000043#include "internal.h"
44
Linus Torvalds1da177e2005-04-16 15:20:36 -070045void *high_memory;
Arnd Bergmann944b6872015-02-05 12:25:12 -080046EXPORT_SYMBOL(high_memory);
Linus Torvalds1da177e2005-04-16 15:20:36 -070047struct page *mem_map;
48unsigned long max_mapnr;
gchen gchen5b8bf302015-03-12 16:26:05 -070049EXPORT_SYMBOL(max_mapnr);
Hugh Dickins4266c972009-09-23 17:05:53 +010050unsigned long highest_memmap_pfn;
David Howellsfc4d5c22009-05-06 16:03:05 -070051int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
Linus Torvalds1da177e2005-04-16 15:20:36 -070052int heap_stack_gap = 0;
53
David Howells33e5d7692009-04-02 16:56:32 -070054atomic_long_t mmap_pages_allocated;
David Howells8feae132009-01-08 12:04:47 +000055
Linus Torvalds1da177e2005-04-16 15:20:36 -070056EXPORT_SYMBOL(mem_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
David Howells8feae132009-01-08 12:04:47 +000058/* list of mapped, potentially shareable regions */
59static struct kmem_cache *vm_region_jar;
60struct rb_root nommu_region_tree = RB_ROOT;
61DECLARE_RWSEM(nommu_region_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +040063const struct vm_operations_struct generic_file_vm_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070064};
65
66/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 * Return the total memory allocated for this pointer, not
68 * just what the caller asked for.
69 *
70 * Doesn't have to be accurate, i.e. may have races.
71 */
72unsigned int kobjsize(const void *objp)
73{
74 struct page *page;
75
Michael Hennerich4016a132008-04-28 02:13:38 -070076 /*
77 * If the object we have should not have ksize performed on it,
78 * return size of 0
79 */
Paul Mundt5a1603b2008-06-12 16:29:55 +090080 if (!objp || !virt_addr_valid(objp))
Paul Mundt6cfd53fc2008-06-05 22:46:08 -070081 return 0;
82
83 page = virt_to_head_page(objp);
Paul Mundt6cfd53fc2008-06-05 22:46:08 -070084
85 /*
86 * If the allocator sets PageSlab, we know the pointer came from
87 * kmalloc().
88 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 if (PageSlab(page))
90 return ksize(objp);
91
Paul Mundt6cfd53fc2008-06-05 22:46:08 -070092 /*
Paul Mundtab2e83e2009-01-08 12:04:48 +000093 * If it's not a compound page, see if we have a matching VMA
94 * region. This test is intentionally done in reverse order,
95 * so if there's no VMA, we still fall through and hand back
96 * PAGE_SIZE for 0-order pages.
97 */
98 if (!PageCompound(page)) {
99 struct vm_area_struct *vma;
100
101 vma = find_vma(current->mm, (unsigned long)objp);
102 if (vma)
103 return vma->vm_end - vma->vm_start;
104 }
105
106 /*
Paul Mundt6cfd53fc2008-06-05 22:46:08 -0700107 * The ksize() function is only guaranteed to work for pointers
Paul Mundt5a1603b2008-06-12 16:29:55 +0900108 * returned by kmalloc(). So handle arbitrary pointers here.
Paul Mundt6cfd53fc2008-06-05 22:46:08 -0700109 */
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -0700110 return page_size(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111}
112
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800113void vfree(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114{
115 kfree(addr);
116}
Paul Mundtb5073172007-07-21 04:37:25 -0700117EXPORT_SYMBOL(vfree);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700119void *__vmalloc_noprof(unsigned long size, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120{
121 /*
Robert P. J. Day85186092007-10-19 23:11:38 +0200122 * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
123 * returns only a logical address.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700125 return kmalloc_noprof(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700127EXPORT_SYMBOL(__vmalloc_noprof);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700129void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
Christoph Hellwig041de932020-06-01 21:52:02 -0700130 unsigned long start, unsigned long end, gfp_t gfp_mask,
131 pgprot_t prot, unsigned long vm_flags, int node,
132 const void *caller)
133{
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700134 return __vmalloc_noprof(size, gfp_mask);
Christoph Hellwig041de932020-06-01 21:52:02 -0700135}
136
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700137void *__vmalloc_node_noprof(unsigned long size, unsigned long align, gfp_t gfp_mask,
Christoph Hellwig2b905942020-06-01 21:51:53 -0700138 int node, const void *caller)
Michal Hockoa7c3e902017-05-08 15:57:09 -0700139{
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700140 return __vmalloc_noprof(size, gfp_mask);
Michal Hockoa7c3e902017-05-08 15:57:09 -0700141}
142
Andrii Nakryikoed817452019-11-23 14:08:35 -0800143static void *__vmalloc_user_flags(unsigned long size, gfp_t flags)
Paul Mundtf905bc42008-02-04 22:29:59 -0800144{
145 void *ret;
146
Christoph Hellwig88dca4c2020-06-01 21:51:40 -0700147 ret = __vmalloc(size, flags);
Paul Mundtf905bc42008-02-04 22:29:59 -0800148 if (ret) {
149 struct vm_area_struct *vma;
150
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700151 mmap_write_lock(current->mm);
Paul Mundtf905bc42008-02-04 22:29:59 -0800152 vma = find_vma(current->mm, (unsigned long)ret);
153 if (vma)
Suren Baghdasaryan1c712222023-01-26 11:37:49 -0800154 vm_flags_set(vma, VM_USERMAP);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700155 mmap_write_unlock(current->mm);
Paul Mundtf905bc42008-02-04 22:29:59 -0800156 }
157
158 return ret;
159}
Andrii Nakryikoed817452019-11-23 14:08:35 -0800160
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700161void *vmalloc_user_noprof(unsigned long size)
Andrii Nakryikoed817452019-11-23 14:08:35 -0800162{
163 return __vmalloc_user_flags(size, GFP_KERNEL | __GFP_ZERO);
164}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700165EXPORT_SYMBOL(vmalloc_user_noprof);
Paul Mundtf905bc42008-02-04 22:29:59 -0800166
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800167struct page *vmalloc_to_page(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168{
169 return virt_to_page(addr);
170}
Paul Mundtb5073172007-07-21 04:37:25 -0700171EXPORT_SYMBOL(vmalloc_to_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800173unsigned long vmalloc_to_pfn(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174{
175 return page_to_pfn(virt_to_page(addr));
176}
Paul Mundtb5073172007-07-21 04:37:25 -0700177EXPORT_SYMBOL(vmalloc_to_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Lorenzo Stoakes4c91c072023-03-22 18:57:04 +0000179long vread_iter(struct iov_iter *iter, const char *addr, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180{
Chen Gang9bde9162013-07-03 15:02:36 -0700181 /* Don't allow overflow */
Lorenzo Stoakes4c91c072023-03-22 18:57:04 +0000182 if ((unsigned long) addr + count < count)
183 count = -(unsigned long) addr;
Chen Gang9bde9162013-07-03 15:02:36 -0700184
Lorenzo Stoakes4c91c072023-03-22 18:57:04 +0000185 return copy_to_iter(addr, count, iter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186}
187
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188/*
Masahiro Yamadae1c05062015-07-07 10:14:59 +0900189 * vmalloc - allocate virtually contiguous memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 *
191 * @size: allocation size
192 *
193 * Allocate enough pages to cover @size from the page level
Masahiro Yamadae1c05062015-07-07 10:14:59 +0900194 * allocator and map them into contiguous kernel virtual space.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 *
Michael Opdenackerc1c88972006-10-03 23:21:02 +0200196 * For tight control over page level allocator and protection flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 * use __vmalloc() instead.
198 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700199void *vmalloc_noprof(unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200{
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700201 return __vmalloc_noprof(size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700203EXPORT_SYMBOL(vmalloc_noprof);
Andrew Mortonf6138882006-02-28 16:59:18 -0800204
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700205void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) __weak __alias(__vmalloc_noprof);
Linus Torvalds0fc74d82022-04-25 01:28:01 -0700206
Dave Younge1ca7782010-10-26 14:22:06 -0700207/*
Masahiro Yamadae1c05062015-07-07 10:14:59 +0900208 * vzalloc - allocate virtually contiguous memory with zero fill
Dave Younge1ca7782010-10-26 14:22:06 -0700209 *
210 * @size: allocation size
211 *
212 * Allocate enough pages to cover @size from the page level
Masahiro Yamadae1c05062015-07-07 10:14:59 +0900213 * allocator and map them into contiguous kernel virtual space.
Dave Younge1ca7782010-10-26 14:22:06 -0700214 * The memory allocated is set to zero.
215 *
216 * For tight control over page level allocator and protection flags
217 * use __vmalloc() instead.
218 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700219void *vzalloc_noprof(unsigned long size)
Dave Younge1ca7782010-10-26 14:22:06 -0700220{
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700221 return __vmalloc_noprof(size, GFP_KERNEL | __GFP_ZERO);
Dave Younge1ca7782010-10-26 14:22:06 -0700222}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700223EXPORT_SYMBOL(vzalloc_noprof);
Dave Younge1ca7782010-10-26 14:22:06 -0700224
225/**
226 * vmalloc_node - allocate memory on a specific node
227 * @size: allocation size
228 * @node: numa node
229 *
230 * Allocate enough pages to cover @size from the page level
231 * allocator and map them into contiguous kernel virtual space.
232 *
233 * For tight control over page level allocator and protection flags
234 * use __vmalloc() instead.
235 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700236void *vmalloc_node_noprof(unsigned long size, int node)
Andrew Mortonf6138882006-02-28 16:59:18 -0800237{
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700238 return vmalloc_noprof(size);
Andrew Mortonf6138882006-02-28 16:59:18 -0800239}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700240EXPORT_SYMBOL(vmalloc_node_noprof);
Dave Younge1ca7782010-10-26 14:22:06 -0700241
242/**
243 * vzalloc_node - allocate memory on a specific node with zero fill
244 * @size: allocation size
245 * @node: numa node
246 *
247 * Allocate enough pages to cover @size from the page level
248 * allocator and map them into contiguous kernel virtual space.
249 * The memory allocated is set to zero.
250 *
251 * For tight control over page level allocator and protection flags
252 * use __vmalloc() instead.
253 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700254void *vzalloc_node_noprof(unsigned long size, int node)
Dave Younge1ca7782010-10-26 14:22:06 -0700255{
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700256 return vzalloc_noprof(size);
Dave Younge1ca7782010-10-26 14:22:06 -0700257}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700258EXPORT_SYMBOL(vzalloc_node_noprof);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
Paul Mundt1af446e2008-08-04 16:01:47 +0900260/**
Paul Mundtb5073172007-07-21 04:37:25 -0700261 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 * @size: allocation size
263 *
264 * Allocate enough 32bit PA addressable pages to cover @size from the
Masahiro Yamadae1c05062015-07-07 10:14:59 +0900265 * page level allocator and map them into contiguous kernel virtual space.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700267void *vmalloc_32_noprof(unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268{
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700269 return __vmalloc_noprof(size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700271EXPORT_SYMBOL(vmalloc_32_noprof);
Paul Mundtb5073172007-07-21 04:37:25 -0700272
273/**
274 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
275 * @size: allocation size
276 *
277 * The resulting memory area is 32bit addressable and zeroed so it can be
278 * mapped to userspace without leaking data.
Paul Mundtf905bc42008-02-04 22:29:59 -0800279 *
280 * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
281 * remap_vmalloc_range() are permissible.
Paul Mundtb5073172007-07-21 04:37:25 -0700282 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700283void *vmalloc_32_user_noprof(unsigned long size)
Paul Mundtb5073172007-07-21 04:37:25 -0700284{
Paul Mundtf905bc42008-02-04 22:29:59 -0800285 /*
286 * We'll have to sort out the ZONE_DMA bits for 64-bit,
287 * but for now this can simply use vmalloc_user() directly.
288 */
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700289 return vmalloc_user_noprof(size);
Paul Mundtb5073172007-07-21 04:37:25 -0700290}
Kent Overstreet88ae5fb2024-03-21 09:36:52 -0700291EXPORT_SYMBOL(vmalloc_32_user_noprof);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292
293void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
294{
295 BUG();
296 return NULL;
297}
Paul Mundtb5073172007-07-21 04:37:25 -0700298EXPORT_SYMBOL(vmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800300void vunmap(const void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301{
302 BUG();
303}
Paul Mundtb5073172007-07-21 04:37:25 -0700304EXPORT_SYMBOL(vunmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305
Christoph Hellwigd4efd792020-06-01 21:51:27 -0700306void *vm_map_ram(struct page **pages, unsigned int count, int node)
Paul Mundteb6434d2009-01-21 17:45:47 +0900307{
308 BUG();
309 return NULL;
310}
311EXPORT_SYMBOL(vm_map_ram);
312
313void vm_unmap_ram(const void *mem, unsigned int count)
314{
315 BUG();
316}
317EXPORT_SYMBOL(vm_unmap_ram);
318
319void vm_unmap_aliases(void)
320{
321}
322EXPORT_SYMBOL_GPL(vm_unmap_aliases);
323
Paul Mundt29c185e2010-12-24 12:08:30 +0900324void free_vm_area(struct vm_struct *area)
325{
326 BUG();
327}
328EXPORT_SYMBOL_GPL(free_vm_area);
329
Paul Mundtb5073172007-07-21 04:37:25 -0700330int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
331 struct page *page)
332{
333 return -EINVAL;
334}
335EXPORT_SYMBOL(vm_insert_page);
336
Jens Axboe62346c62024-03-16 07:21:43 -0600337int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
338 struct page **pages, unsigned long *num)
339{
340 return -EINVAL;
341}
342EXPORT_SYMBOL(vm_insert_pages);
343
Souptick Joardera667d742019-05-13 17:21:56 -0700344int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
345 unsigned long num)
346{
347 return -EINVAL;
348}
349EXPORT_SYMBOL(vm_map_pages);
350
351int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
352 unsigned long num)
353{
354 return -EINVAL;
355}
356EXPORT_SYMBOL(vm_map_pages_zero);
357
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -0700358/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 * sys_brk() for the most part doesn't need the global kernel
360 * lock, except when an application is doing something nasty
361 * like trying to un-brk an area that has already been mapped
362 * to a regular file. in this case, the unmapping will need
363 * to invoke file system routines that need the global lock.
364 */
Heiko Carstens6a6160a2009-01-14 14:14:15 +0100365SYSCALL_DEFINE1(brk, unsigned long, brk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366{
367 struct mm_struct *mm = current->mm;
368
369 if (brk < mm->start_brk || brk > mm->context.end_brk)
370 return mm->brk;
371
372 if (mm->brk == brk)
373 return mm->brk;
374
375 /*
376 * Always allow shrinking brk
377 */
378 if (brk <= mm->brk) {
379 mm->brk = brk;
380 return brk;
381 }
382
383 /*
384 * Ok, looks good - let it rip.
385 */
Christoph Hellwiga75a2df2020-06-07 21:42:49 -0700386 flush_icache_user_range(mm->brk, brk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 return mm->brk = brk;
388}
389
David Howells8feae132009-01-08 12:04:47 +0000390/*
seokhoon.yoon3edf41d2017-02-24 14:56:44 -0800391 * initialise the percpu counter for VM and region record slabs
David Howells8feae132009-01-08 12:04:47 +0000392 */
393void __init mmap_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394{
KOSAKI Motohiro00a62ce2009-04-30 15:08:51 -0700395 int ret;
396
Tejun Heo908c7f12014-09-08 09:51:29 +0900397 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
KOSAKI Motohiro00a62ce2009-04-30 15:08:51 -0700398 VM_BUG_ON(ret);
Vladimir Davydov5d097052016-01-14 15:18:21 -0800399 vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT);
David Howells8feae132009-01-08 12:04:47 +0000400}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
David Howells8feae132009-01-08 12:04:47 +0000402/*
403 * validate the region tree
404 * - the caller must hold the region lock
405 */
406#ifdef CONFIG_DEBUG_NOMMU_REGIONS
407static noinline void validate_nommu_regions(void)
408{
409 struct vm_region *region, *last;
410 struct rb_node *p, *lastp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411
David Howells8feae132009-01-08 12:04:47 +0000412 lastp = rb_first(&nommu_region_tree);
413 if (!lastp)
414 return;
415
416 last = rb_entry(lastp, struct vm_region, vm_rb);
Geliang Tangc9427bc2015-11-05 18:48:38 -0800417 BUG_ON(last->vm_end <= last->vm_start);
418 BUG_ON(last->vm_top < last->vm_end);
David Howells8feae132009-01-08 12:04:47 +0000419
420 while ((p = rb_next(lastp))) {
421 region = rb_entry(p, struct vm_region, vm_rb);
422 last = rb_entry(lastp, struct vm_region, vm_rb);
423
Geliang Tangc9427bc2015-11-05 18:48:38 -0800424 BUG_ON(region->vm_end <= region->vm_start);
425 BUG_ON(region->vm_top < region->vm_end);
426 BUG_ON(region->vm_start < last->vm_top);
David Howells8feae132009-01-08 12:04:47 +0000427
428 lastp = p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429 }
430}
David Howells8feae132009-01-08 12:04:47 +0000431#else
David Howells33e5d7692009-04-02 16:56:32 -0700432static void validate_nommu_regions(void)
433{
434}
David Howells8feae132009-01-08 12:04:47 +0000435#endif
436
437/*
438 * add a region into the global tree
439 */
440static void add_nommu_region(struct vm_region *region)
441{
442 struct vm_region *pregion;
443 struct rb_node **p, *parent;
444
445 validate_nommu_regions();
446
David Howells8feae132009-01-08 12:04:47 +0000447 parent = NULL;
448 p = &nommu_region_tree.rb_node;
449 while (*p) {
450 parent = *p;
451 pregion = rb_entry(parent, struct vm_region, vm_rb);
452 if (region->vm_start < pregion->vm_start)
453 p = &(*p)->rb_left;
454 else if (region->vm_start > pregion->vm_start)
455 p = &(*p)->rb_right;
456 else if (pregion == region)
457 return;
458 else
459 BUG();
460 }
461
462 rb_link_node(&region->vm_rb, parent, p);
463 rb_insert_color(&region->vm_rb, &nommu_region_tree);
464
465 validate_nommu_regions();
466}
467
468/*
469 * delete a region from the global tree
470 */
471static void delete_nommu_region(struct vm_region *region)
472{
473 BUG_ON(!nommu_region_tree.rb_node);
474
475 validate_nommu_regions();
476 rb_erase(&region->vm_rb, &nommu_region_tree);
477 validate_nommu_regions();
478}
479
480/*
481 * free a contiguous series of pages
482 */
483static void free_page_series(unsigned long from, unsigned long to)
484{
485 for (; from < to; from += PAGE_SIZE) {
Linus Walleij93307232022-06-30 10:41:24 +0200486 struct page *page = virt_to_page((void *)from);
David Howells8feae132009-01-08 12:04:47 +0000487
David Howells33e5d7692009-04-02 16:56:32 -0700488 atomic_long_dec(&mmap_pages_allocated);
David Howells8feae132009-01-08 12:04:47 +0000489 put_page(page);
490 }
491}
492
493/*
494 * release a reference to a region
David Howells33e5d7692009-04-02 16:56:32 -0700495 * - the caller must hold the region semaphore for writing, which this releases
Paul Mundtdd8632a2009-01-08 12:04:47 +0000496 * - the region may not have been added to the tree yet, in which case vm_top
David Howells8feae132009-01-08 12:04:47 +0000497 * will equal vm_start
498 */
499static void __put_nommu_region(struct vm_region *region)
500 __releases(nommu_region_sem)
501{
David Howells8feae132009-01-08 12:04:47 +0000502 BUG_ON(!nommu_region_tree.rb_node);
503
David Howells1e2ae592010-01-15 17:01:33 -0800504 if (--region->vm_usage == 0) {
Paul Mundtdd8632a2009-01-08 12:04:47 +0000505 if (region->vm_top > region->vm_start)
David Howells8feae132009-01-08 12:04:47 +0000506 delete_nommu_region(region);
507 up_write(&nommu_region_sem);
508
509 if (region->vm_file)
510 fput(region->vm_file);
511
512 /* IO memory and memory shared directly out of the pagecache
513 * from ramfs/tmpfs mustn't be released here */
Leon Romanovsky22cc8772015-06-24 16:57:47 -0700514 if (region->vm_flags & VM_MAPPED_COPY)
Paul Mundtdd8632a2009-01-08 12:04:47 +0000515 free_page_series(region->vm_start, region->vm_top);
David Howells8feae132009-01-08 12:04:47 +0000516 kmem_cache_free(vm_region_jar, region);
517 } else {
518 up_write(&nommu_region_sem);
519 }
520}
521
522/*
523 * release a reference to a region
524 */
525static void put_nommu_region(struct vm_region *region)
526{
527 down_write(&nommu_region_sem);
528 __put_nommu_region(region);
529}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000531static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm)
David Howells30340972006-09-27 01:50:20 -0700532{
David Howells8feae132009-01-08 12:04:47 +0000533 vma->vm_mm = mm;
534
535 /* add the VMA to the mapping */
536 if (vma->vm_file) {
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000537 struct address_space *mapping = vma->vm_file->f_mapping;
David Howells8feae132009-01-08 12:04:47 +0000538
Davidlohr Bueso83cde9e2014-12-12 16:54:21 -0800539 i_mmap_lock_write(mapping);
David Howells8feae132009-01-08 12:04:47 +0000540 flush_dcache_mmap_lock(mapping);
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700541 vma_interval_tree_insert(vma, &mapping->i_mmap);
David Howells8feae132009-01-08 12:04:47 +0000542 flush_dcache_mmap_unlock(mapping);
Davidlohr Bueso83cde9e2014-12-12 16:54:21 -0800543 i_mmap_unlock_write(mapping);
David Howells8feae132009-01-08 12:04:47 +0000544 }
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000545}
David Howells8feae132009-01-08 12:04:47 +0000546
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000547static void cleanup_vma_from_mm(struct vm_area_struct *vma)
548{
Liam R. Howlett7964cf82022-09-06 19:48:51 +0000549 vma->vm_mm->map_count--;
David Howells8feae132009-01-08 12:04:47 +0000550 /* remove the VMA from the mapping */
551 if (vma->vm_file) {
Liam R. Howlett7964cf82022-09-06 19:48:51 +0000552 struct address_space *mapping;
David Howells8feae132009-01-08 12:04:47 +0000553 mapping = vma->vm_file->f_mapping;
554
Davidlohr Bueso83cde9e2014-12-12 16:54:21 -0800555 i_mmap_lock_write(mapping);
David Howells8feae132009-01-08 12:04:47 +0000556 flush_dcache_mmap_lock(mapping);
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700557 vma_interval_tree_remove(vma, &mapping->i_mmap);
David Howells8feae132009-01-08 12:04:47 +0000558 flush_dcache_mmap_unlock(mapping);
Davidlohr Bueso83cde9e2014-12-12 16:54:21 -0800559 i_mmap_unlock_write(mapping);
David Howells8feae132009-01-08 12:04:47 +0000560 }
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000561}
Liam R. Howlett47d96442023-01-20 11:26:28 -0500562
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000563/*
564 * delete a VMA from its owning mm_struct and address space
565 */
566static int delete_vma_from_mm(struct vm_area_struct *vma)
567{
Liam R. Howlett47d96442023-01-20 11:26:28 -0500568 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_start);
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000569
Liam R. Howlettb5df0922023-07-24 14:31:52 -0400570 vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
571 if (vma_iter_prealloc(&vmi, vma)) {
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000572 pr_warn("Allocation of vma tree for process %d failed\n",
573 current->pid);
574 return -ENOMEM;
575 }
576 cleanup_vma_from_mm(vma);
David Howells8feae132009-01-08 12:04:47 +0000577
578 /* remove from the MM's tree and list */
Liam R. Howlettb5df0922023-07-24 14:31:52 -0400579 vma_iter_clear(&vmi);
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +0000580 return 0;
David Howells8feae132009-01-08 12:04:47 +0000581}
David Howells8feae132009-01-08 12:04:47 +0000582/*
583 * destroy a VMA record
584 */
585static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
586{
David Howells8feae132009-01-08 12:04:47 +0000587 if (vma->vm_ops && vma->vm_ops->close)
588 vma->vm_ops->close(vma);
Konstantin Khlebnikove9714ac2012-10-08 16:28:54 -0700589 if (vma->vm_file)
David Howells8feae132009-01-08 12:04:47 +0000590 fput(vma->vm_file);
David Howells8feae132009-01-08 12:04:47 +0000591 put_nommu_region(vma->vm_region);
Linus Torvalds3928d4f2018-07-21 13:48:51 -0700592 vm_area_free(vma);
David Howells30340972006-09-27 01:50:20 -0700593}
594
Liam R. Howlettabdba2d2022-09-06 19:48:50 +0000595struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
596 unsigned long start_addr,
597 unsigned long end_addr)
598{
599 unsigned long index = start_addr;
600
601 mmap_assert_locked(mm);
602 return mt_find(&mm->mm_mt, &index, end_addr - 1);
603}
604EXPORT_SYMBOL(find_vma_intersection);
605
David Howells30340972006-09-27 01:50:20 -0700606/*
607 * look up the first VMA in which addr resides, NULL if none
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700608 * - should be called with mm->mmap_lock at least held readlocked
David Howells30340972006-09-27 01:50:20 -0700609 */
610struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
611{
Liam R. Howlett47d96442023-01-20 11:26:28 -0500612 VMA_ITERATOR(vmi, mm, addr);
David Howells30340972006-09-27 01:50:20 -0700613
Liam R. Howlett47d96442023-01-20 11:26:28 -0500614 return vma_iter_load(&vmi);
David Howells30340972006-09-27 01:50:20 -0700615}
616EXPORT_SYMBOL(find_vma);
617
618/*
Linus Torvaldsd85a1432023-06-30 18:24:49 -0700619 * At least xtensa ends up having protection faults even with no
620 * MMU.. No stack expansion, at least.
621 */
622struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
623 unsigned long addr, struct pt_regs *regs)
624{
Max Filippov03f88932023-07-01 03:31:55 -0700625 struct vm_area_struct *vma;
626
Linus Torvaldsd85a1432023-06-30 18:24:49 -0700627 mmap_read_lock(mm);
Max Filippov03f88932023-07-01 03:31:55 -0700628 vma = vma_lookup(mm, addr);
629 if (!vma)
630 mmap_read_unlock(mm);
631 return vma;
Linus Torvaldsd85a1432023-06-30 18:24:49 -0700632}
633
634/*
David Howells8feae132009-01-08 12:04:47 +0000635 * expand a stack to a given address
636 * - not supported under NOMMU conditions
637 */
Linus Torvalds8d7071a2023-06-24 13:45:51 -0700638int expand_stack_locked(struct vm_area_struct *vma, unsigned long addr)
Greg Ungerer57c8f632007-07-15 23:38:28 -0700639{
640 return -ENOMEM;
641}
642
Linus Torvalds8d7071a2023-06-24 13:45:51 -0700643struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
644{
645 mmap_read_unlock(mm);
646 return NULL;
647}
648
David Howells930e6522006-09-27 01:50:22 -0700649/*
David Howells6fa5f802006-09-27 01:50:21 -0700650 * look up the first VMA exactly that exactly matches addr
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700651 * - should be called with mm->mmap_lock at least held readlocked
David Howells6fa5f802006-09-27 01:50:21 -0700652 */
David Howells8feae132009-01-08 12:04:47 +0000653static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
654 unsigned long addr,
655 unsigned long len)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656{
657 struct vm_area_struct *vma;
David Howells8feae132009-01-08 12:04:47 +0000658 unsigned long end = addr + len;
Liam R. Howlett47d96442023-01-20 11:26:28 -0500659 VMA_ITERATOR(vmi, mm, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660
Liam R. Howlett47d96442023-01-20 11:26:28 -0500661 vma = vma_iter_load(&vmi);
Liam R. Howlett524e00b2022-09-06 19:48:48 +0000662 if (!vma)
663 return NULL;
664 if (vma->vm_start != addr)
665 return NULL;
666 if (vma->vm_end != end)
667 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668
Liam R. Howlett524e00b2022-09-06 19:48:48 +0000669 return vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670}
671
David Howells30340972006-09-27 01:50:20 -0700672/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 * determine whether a mapping should be permitted and, if so, what sort of
674 * mapping we're capable of supporting
675 */
676static int validate_mmap_request(struct file *file,
677 unsigned long addr,
678 unsigned long len,
679 unsigned long prot,
680 unsigned long flags,
681 unsigned long pgoff,
682 unsigned long *_capabilities)
683{
David Howells8feae132009-01-08 12:04:47 +0000684 unsigned long capabilities, rlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 int ret;
686
687 /* do the simple checks first */
Leon Romanovsky22cc8772015-06-24 16:57:47 -0700688 if (flags & MAP_FIXED)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690
691 if ((flags & MAP_TYPE) != MAP_PRIVATE &&
692 (flags & MAP_TYPE) != MAP_SHARED)
693 return -EINVAL;
694
Mike Frysingerf81cff0d2006-12-06 12:02:59 +1000695 if (!len)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696 return -EINVAL;
697
Mike Frysingerf81cff0d2006-12-06 12:02:59 +1000698 /* Careful about overflows.. */
David Howells8feae132009-01-08 12:04:47 +0000699 rlen = PAGE_ALIGN(len);
700 if (!rlen || rlen > TASK_SIZE)
Mike Frysingerf81cff0d2006-12-06 12:02:59 +1000701 return -ENOMEM;
702
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 /* offset overflow? */
David Howells8feae132009-01-08 12:04:47 +0000704 if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
Mike Frysingerf81cff0d2006-12-06 12:02:59 +1000705 return -EOVERFLOW;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706
707 if (file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 /* files must support mmap */
Al Viro72c2d532013-09-22 16:27:52 -0400709 if (!file->f_op->mmap)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 return -ENODEV;
711
712 /* work out if what we've got could possibly be shared
713 * - we support chardevs that provide their own "memory"
714 * - we support files/blockdevs that are memory backed
715 */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100716 if (file->f_op->mmap_capabilities) {
717 capabilities = file->f_op->mmap_capabilities(file);
718 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 /* no explicit capabilities set, so assume some
720 * defaults */
Al Viro496ad9a2013-01-23 17:07:38 -0500721 switch (file_inode(file)->i_mode & S_IFMT) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 case S_IFREG:
723 case S_IFBLK:
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100724 capabilities = NOMMU_MAP_COPY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 break;
726
727 case S_IFCHR:
728 capabilities =
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100729 NOMMU_MAP_DIRECT |
730 NOMMU_MAP_READ |
731 NOMMU_MAP_WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 break;
733
734 default:
735 return -EINVAL;
736 }
737 }
738
739 /* eliminate any capabilities that we can't support on this
740 * device */
741 if (!file->f_op->get_unmapped_area)
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100742 capabilities &= ~NOMMU_MAP_DIRECT;
Al Viro6e242a12015-03-31 12:35:13 -0400743 if (!(file->f_mode & FMODE_CAN_READ))
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100744 capabilities &= ~NOMMU_MAP_COPY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745
Graff Yang28d7a6a2009-08-18 14:11:17 -0700746 /* The file shall have been opened with read permission. */
747 if (!(file->f_mode & FMODE_READ))
748 return -EACCES;
749
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 if (flags & MAP_SHARED) {
751 /* do checks for writing, appending and locking */
752 if ((prot & PROT_WRITE) &&
753 !(file->f_mode & FMODE_WRITE))
754 return -EACCES;
755
Al Viro496ad9a2013-01-23 17:07:38 -0500756 if (IS_APPEND(file_inode(file)) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 (file->f_mode & FMODE_WRITE))
758 return -EACCES;
759
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100760 if (!(capabilities & NOMMU_MAP_DIRECT))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 return -ENODEV;
762
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 /* we mustn't privatise shared mappings */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100764 capabilities &= ~NOMMU_MAP_COPY;
Choi Gi-yongac714902014-04-07 15:37:36 -0700765 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 /* we're going to read the file into private memory we
767 * allocate */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100768 if (!(capabilities & NOMMU_MAP_COPY))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 return -ENODEV;
770
771 /* we don't permit a private writable mapping to be
772 * shared with the backing device */
773 if (prot & PROT_WRITE)
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100774 capabilities &= ~NOMMU_MAP_DIRECT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 }
776
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100777 if (capabilities & NOMMU_MAP_DIRECT) {
778 if (((prot & PROT_READ) && !(capabilities & NOMMU_MAP_READ)) ||
779 ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
780 ((prot & PROT_EXEC) && !(capabilities & NOMMU_MAP_EXEC))
Bernd Schmidt3c7b2042010-05-25 23:43:00 -0700781 ) {
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100782 capabilities &= ~NOMMU_MAP_DIRECT;
Bernd Schmidt3c7b2042010-05-25 23:43:00 -0700783 if (flags & MAP_SHARED) {
Leon Romanovsky22cc8772015-06-24 16:57:47 -0700784 pr_warn("MAP_SHARED not completely supported on !MMU\n");
Bernd Schmidt3c7b2042010-05-25 23:43:00 -0700785 return -EINVAL;
786 }
787 }
788 }
789
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 /* handle executable mappings and implied executable
791 * mappings */
Eric W. Biederman90f85722015-06-29 14:42:03 -0500792 if (path_noexec(&file->f_path)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793 if (prot & PROT_EXEC)
794 return -EPERM;
Choi Gi-yongac714902014-04-07 15:37:36 -0700795 } else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 /* handle implication of PROT_EXEC by PROT_READ */
797 if (current->personality & READ_IMPLIES_EXEC) {
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100798 if (capabilities & NOMMU_MAP_EXEC)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 prot |= PROT_EXEC;
800 }
Choi Gi-yongac714902014-04-07 15:37:36 -0700801 } else if ((prot & PROT_READ) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 (prot & PROT_EXEC) &&
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100803 !(capabilities & NOMMU_MAP_EXEC)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 ) {
805 /* backing file is not executable, try to copy */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100806 capabilities &= ~NOMMU_MAP_DIRECT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 }
Choi Gi-yongac714902014-04-07 15:37:36 -0700808 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 /* anonymous mappings are always memory backed and can be
810 * privately mapped
811 */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100812 capabilities = NOMMU_MAP_COPY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
814 /* handle PROT_EXEC implication by PROT_READ */
815 if ((prot & PROT_READ) &&
816 (current->personality & READ_IMPLIES_EXEC))
817 prot |= PROT_EXEC;
818 }
819
820 /* allow the security API to have its say */
Al Viroe5467852012-05-30 13:30:51 -0400821 ret = security_mmap_addr(addr);
822 if (ret < 0)
823 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824
825 /* looks okay */
826 *_capabilities = capabilities;
827 return 0;
828}
829
830/*
831 * we've determined that we can make the mapping, now translate what we
832 * now know into VMA flags
833 */
834static unsigned long determine_vm_flags(struct file *file,
835 unsigned long prot,
836 unsigned long flags,
837 unsigned long capabilities)
838{
839 unsigned long vm_flags;
840
Dave Hansene6bfb702016-02-12 13:02:31 -0800841 vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842
David Hildenbrandb6b7a8f2023-01-02 17:08:55 +0100843 if (!file) {
844 /*
845 * MAP_ANONYMOUS. MAP_SHARED is mapped to MAP_PRIVATE, because
846 * there is no fork().
847 */
Bernd Schmidt3c7b2042010-05-25 23:43:00 -0700848 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
David Hildenbrandb6b7a8f2023-01-02 17:08:55 +0100849 } else if (flags & MAP_PRIVATE) {
850 /* MAP_PRIVATE file mapping */
851 if (capabilities & NOMMU_MAP_DIRECT)
852 vm_flags |= (capabilities & NOMMU_VMFLAGS);
853 else
854 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855
David Hildenbrandb6b7a8f2023-01-02 17:08:55 +0100856 if (!(prot & PROT_WRITE) && !current->ptrace)
857 /*
858 * R/O private file mapping which cannot be used to
859 * modify memory, especially also not via active ptrace
860 * (e.g., set breakpoints) or later by upgrading
861 * permissions (no mprotect()). We can try overlaying
862 * the file mapping, which will work e.g., on chardevs,
863 * ramfs/tmpfs/shmfs and romfs/cramf.
864 */
865 vm_flags |= VM_MAYOVERLAY;
866 } else {
867 /* MAP_SHARED file mapping: NOMMU_MAP_DIRECT is set. */
868 vm_flags |= VM_SHARED | VM_MAYSHARE |
869 (capabilities & NOMMU_VMFLAGS);
870 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871
872 return vm_flags;
873}
874
875/*
David Howells8feae132009-01-08 12:04:47 +0000876 * set up a shared mapping on a file (the driver or filesystem provides and
877 * pins the storage)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 */
David Howells8feae132009-01-08 12:04:47 +0000879static int do_mmap_shared_file(struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880{
881 int ret;
882
Miklos Szeredif74ac012017-02-20 16:51:23 +0100883 ret = call_mmap(vma->vm_file, vma);
Paul Mundtdd8632a2009-01-08 12:04:47 +0000884 if (ret == 0) {
885 vma->vm_region->vm_top = vma->vm_region->vm_end;
David Howells645d83c2009-09-24 15:13:10 +0100886 return 0;
Paul Mundtdd8632a2009-01-08 12:04:47 +0000887 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888 if (ret != -ENOSYS)
889 return ret;
890
David Howells3fa30462010-03-23 13:35:21 -0700891 /* getting -ENOSYS indicates that direct mmap isn't possible (as
892 * opposed to tried but failed) so we can only give a suitable error as
893 * it's not possible to make a private copy if MAP_SHARED was given */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 return -ENODEV;
895}
896
897/*
898 * set up a private mapping or an anonymous shared mapping
899 */
David Howells8feae132009-01-08 12:04:47 +0000900static int do_mmap_private(struct vm_area_struct *vma,
901 struct vm_region *region,
David Howells645d83c2009-09-24 15:13:10 +0100902 unsigned long len,
903 unsigned long capabilities)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904{
Joonsoo Kimdbc83582014-12-12 16:55:55 -0800905 unsigned long total, point;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 void *base;
David Howells8feae132009-01-08 12:04:47 +0000907 int ret, order;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
David Hildenbrandb6b7a8f2023-01-02 17:08:55 +0100909 /*
910 * Invoke the file's mapping function so that it can keep track of
911 * shared mappings on devices or memory. VM_MAYOVERLAY will be set if
912 * it may attempt to share, which will make is_nommu_shared_mapping()
913 * happy.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +0100915 if (capabilities & NOMMU_MAP_DIRECT) {
Miklos Szeredif74ac012017-02-20 16:51:23 +0100916 ret = call_mmap(vma->vm_file, vma);
David Hildenbrandfc4f4be2023-01-02 17:08:54 +0100917 /* shouldn't return success if we're not sharing */
918 if (WARN_ON_ONCE(!is_nommu_shared_mapping(vma->vm_flags)))
919 ret = -ENOSYS;
Paul Mundtdd8632a2009-01-08 12:04:47 +0000920 if (ret == 0) {
Paul Mundtdd8632a2009-01-08 12:04:47 +0000921 vma->vm_region->vm_top = vma->vm_region->vm_end;
David Howells645d83c2009-09-24 15:13:10 +0100922 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 }
Paul Mundtdd8632a2009-01-08 12:04:47 +0000924 if (ret != -ENOSYS)
925 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926
927 /* getting an ENOSYS error indicates that direct mmap isn't
928 * possible (as opposed to tried but failed) so we'll try to
929 * make a private copy of the data and map that instead */
930 }
931
David Howells8feae132009-01-08 12:04:47 +0000932
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 /* allocate some memory to hold the mapping
934 * - note that this may not return a page-aligned address if the object
935 * we're allocating is smaller than a page
936 */
Bob Liuf67d9b12011-05-24 17:12:56 -0700937 order = get_order(len);
David Howells8feae132009-01-08 12:04:47 +0000938 total = 1 << order;
Bob Liuf67d9b12011-05-24 17:12:56 -0700939 point = len >> PAGE_SHIFT;
Paul Mundtdd8632a2009-01-08 12:04:47 +0000940
Joonsoo Kimdbc83582014-12-12 16:55:55 -0800941 /* we don't want to allocate a power-of-2 sized page set */
Leon Romanovsky22cc8772015-06-24 16:57:47 -0700942 if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
Joonsoo Kimdbc83582014-12-12 16:55:55 -0800943 total = point;
David Howells8feae132009-01-08 12:04:47 +0000944
Joonsoo Kimda616532015-02-27 15:51:43 -0800945 base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
Joonsoo Kimdbc83582014-12-12 16:55:55 -0800946 if (!base)
947 goto enomem;
David Howells8feae132009-01-08 12:04:47 +0000948
Joonsoo Kimdbc83582014-12-12 16:55:55 -0800949 atomic_long_add(total, &mmap_pages_allocated);
950
Suren Baghdasaryan1c712222023-01-26 11:37:49 -0800951 vm_flags_set(vma, VM_MAPPED_COPY);
952 region->vm_flags = vma->vm_flags;
David Howells8feae132009-01-08 12:04:47 +0000953 region->vm_start = (unsigned long) base;
Bob Liuf67d9b12011-05-24 17:12:56 -0700954 region->vm_end = region->vm_start + len;
Paul Mundtdd8632a2009-01-08 12:04:47 +0000955 region->vm_top = region->vm_start + (total << PAGE_SHIFT);
David Howells8feae132009-01-08 12:04:47 +0000956
957 vma->vm_start = region->vm_start;
958 vma->vm_end = region->vm_start + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959
960 if (vma->vm_file) {
961 /* read the contents of a file into the copy */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 loff_t fpos;
963
964 fpos = vma->vm_pgoff;
965 fpos <<= PAGE_SHIFT;
966
Christoph Hellwigb4bf8022017-09-01 17:39:17 +0200967 ret = kernel_read(vma->vm_file, base, len, &fpos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 if (ret < 0)
969 goto error_free;
970
971 /* clear the last little bit */
Bob Liuf67d9b12011-05-24 17:12:56 -0700972 if (ret < len)
973 memset(base + ret, 0, len - ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974
Kirill A. Shutemovbfd40ea2018-07-26 16:37:35 -0700975 } else {
976 vma_set_anonymous(vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 }
978
979 return 0;
980
981error_free:
Namhyung Kim7223bb42011-05-24 17:11:26 -0700982 free_page_series(region->vm_start, region->vm_top);
David Howells8feae132009-01-08 12:04:47 +0000983 region->vm_start = vma->vm_start = 0;
984 region->vm_end = vma->vm_end = 0;
Paul Mundtdd8632a2009-01-08 12:04:47 +0000985 region->vm_top = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 return ret;
987
988enomem:
Mitchel Humpherysb1de0d12014-06-06 14:38:30 -0700989 pr_err("Allocation of length %lu from process %d (%s) failed\n",
Greg Ungerer05ae6fa2009-01-13 17:30:22 +1000990 len, current->pid, current->comm);
Kefeng Wang1279aa02023-06-30 14:22:53 +0800991 show_mem();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 return -ENOMEM;
993}
994
995/*
996 * handle mapping creation for uClinux
997 */
Oleg Nesterov1fcfd8d2015-09-09 15:39:29 -0700998unsigned long do_mmap(struct file *file,
999 unsigned long addr,
1000 unsigned long len,
1001 unsigned long prot,
1002 unsigned long flags,
Yu-cheng Yu592b5fa2023-06-12 17:10:30 -07001003 vm_flags_t vm_flags,
Oleg Nesterov1fcfd8d2015-09-09 15:39:29 -07001004 unsigned long pgoff,
Mike Rapoport897ab3e2017-02-24 14:58:22 -08001005 unsigned long *populate,
1006 struct list_head *uf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007{
David Howells8feae132009-01-08 12:04:47 +00001008 struct vm_area_struct *vma;
1009 struct vm_region *region;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010 struct rb_node *rb;
Oleg Nesterov1fcfd8d2015-09-09 15:39:29 -07001011 unsigned long capabilities, result;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 int ret;
Liam R. Howlett47d96442023-01-20 11:26:28 -05001013 VMA_ITERATOR(vmi, current->mm, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014
Michel Lespinasse41badc12013-02-22 16:32:47 -08001015 *populate = 0;
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001016
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 /* decide whether we should attempt the mapping, and if so what sort of
1018 * mapping */
1019 ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
1020 &capabilities);
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001021 if (ret < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022 return ret;
1023
David Howells06aab5a32009-09-24 12:33:48 +01001024 /* we ignore the address hint */
1025 addr = 0;
Bob Liuf67d9b12011-05-24 17:12:56 -07001026 len = PAGE_ALIGN(len);
David Howells06aab5a32009-09-24 12:33:48 +01001027
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028 /* we've determined that we can make the mapping, now translate what we
1029 * now know into VMA flags */
Yu-cheng Yu592b5fa2023-06-12 17:10:30 -07001030 vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001032
David Howells8feae132009-01-08 12:04:47 +00001033 /* we're going to need to record the mapping */
1034 region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
1035 if (!region)
1036 goto error_getting_region;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037
Linus Torvalds490fc052018-07-21 15:24:03 -07001038 vma = vm_area_alloc(current->mm);
David Howells8feae132009-01-08 12:04:47 +00001039 if (!vma)
1040 goto error_getting_vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041
David Howells1e2ae592010-01-15 17:01:33 -08001042 region->vm_usage = 1;
David Howells8feae132009-01-08 12:04:47 +00001043 region->vm_flags = vm_flags;
1044 region->vm_pgoff = pgoff;
1045
Suren Baghdasaryan1c712222023-01-26 11:37:49 -08001046 vm_flags_init(vma, vm_flags);
David Howells8feae132009-01-08 12:04:47 +00001047 vma->vm_pgoff = pgoff;
1048
1049 if (file) {
Al Virocb0942b2012-08-27 14:48:26 -04001050 region->vm_file = get_file(file);
1051 vma->vm_file = get_file(file);
David Howells8feae132009-01-08 12:04:47 +00001052 }
1053
1054 down_write(&nommu_region_sem);
1055
1056 /* if we want to share, we need to check for regions created by other
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057 * mmap() calls that overlap with our proposed mapping
David Howells8feae132009-01-08 12:04:47 +00001058 * - we can only share with a superset match on most regular files
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059 * - shared mappings on character devices and memory backed files are
1060 * permitted to overlap inexactly as far as we are concerned for in
1061 * these cases, sharing is handled in the driver or filesystem rather
1062 * than here
1063 */
David Hildenbrandfc4f4be2023-01-02 17:08:54 +01001064 if (is_nommu_shared_mapping(vm_flags)) {
David Howells8feae132009-01-08 12:04:47 +00001065 struct vm_region *pregion;
1066 unsigned long pglen, rpglen, pgend, rpgend, start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067
David Howells8feae132009-01-08 12:04:47 +00001068 pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1069 pgend = pgoff + pglen;
David Howells165b2392007-03-22 00:11:24 -08001070
David Howells8feae132009-01-08 12:04:47 +00001071 for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
1072 pregion = rb_entry(rb, struct vm_region, vm_rb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073
David Hildenbrandfc4f4be2023-01-02 17:08:54 +01001074 if (!is_nommu_shared_mapping(pregion->vm_flags))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 continue;
1076
1077 /* search for overlapping mappings on the same file */
Al Viro496ad9a2013-01-23 17:07:38 -05001078 if (file_inode(pregion->vm_file) !=
1079 file_inode(file))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080 continue;
1081
David Howells8feae132009-01-08 12:04:47 +00001082 if (pregion->vm_pgoff >= pgend)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 continue;
1084
David Howells8feae132009-01-08 12:04:47 +00001085 rpglen = pregion->vm_end - pregion->vm_start;
1086 rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1087 rpgend = pregion->vm_pgoff + rpglen;
1088 if (pgoff >= rpgend)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089 continue;
1090
David Howells8feae132009-01-08 12:04:47 +00001091 /* handle inexactly overlapping matches between
1092 * mappings */
1093 if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
1094 !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1095 /* new mapping is not a subset of the region */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +01001096 if (!(capabilities & NOMMU_MAP_DIRECT))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097 goto sharing_violation;
1098 continue;
1099 }
1100
David Howells8feae132009-01-08 12:04:47 +00001101 /* we've found a region we can share */
David Howells1e2ae592010-01-15 17:01:33 -08001102 pregion->vm_usage++;
David Howells8feae132009-01-08 12:04:47 +00001103 vma->vm_region = pregion;
1104 start = pregion->vm_start;
1105 start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
1106 vma->vm_start = start;
1107 vma->vm_end = start + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001109 if (pregion->vm_flags & VM_MAPPED_COPY)
Suren Baghdasaryan1c712222023-01-26 11:37:49 -08001110 vm_flags_set(vma, VM_MAPPED_COPY);
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001111 else {
David Howells8feae132009-01-08 12:04:47 +00001112 ret = do_mmap_shared_file(vma);
1113 if (ret < 0) {
1114 vma->vm_region = NULL;
1115 vma->vm_start = 0;
1116 vma->vm_end = 0;
David Howells1e2ae592010-01-15 17:01:33 -08001117 pregion->vm_usage--;
David Howells8feae132009-01-08 12:04:47 +00001118 pregion = NULL;
1119 goto error_just_free;
1120 }
1121 }
1122 fput(region->vm_file);
1123 kmem_cache_free(vm_region_jar, region);
1124 region = pregion;
1125 result = start;
1126 goto share;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 }
1128
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129 /* obtain the address at which to make a shared mapping
1130 * - this is the hook for quasi-memory character devices to
1131 * tell us the location of a shared mapping
1132 */
Christoph Hellwigb4caecd2015-01-14 10:42:32 +01001133 if (capabilities & NOMMU_MAP_DIRECT) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 addr = file->f_op->get_unmapped_area(file, addr, len,
1135 pgoff, flags);
Namhyung Kimbb005a52011-05-24 17:11:27 -07001136 if (IS_ERR_VALUE(addr)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 ret = addr;
Namhyung Kimbb005a52011-05-24 17:11:27 -07001138 if (ret != -ENOSYS)
David Howells8feae132009-01-08 12:04:47 +00001139 goto error_just_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140
1141 /* the driver refused to tell us where to site
1142 * the mapping so we'll have to attempt to copy
1143 * it */
Namhyung Kimbb005a52011-05-24 17:11:27 -07001144 ret = -ENODEV;
Christoph Hellwigb4caecd2015-01-14 10:42:32 +01001145 if (!(capabilities & NOMMU_MAP_COPY))
David Howells8feae132009-01-08 12:04:47 +00001146 goto error_just_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147
Christoph Hellwigb4caecd2015-01-14 10:42:32 +01001148 capabilities &= ~NOMMU_MAP_DIRECT;
David Howells8feae132009-01-08 12:04:47 +00001149 } else {
1150 vma->vm_start = region->vm_start = addr;
1151 vma->vm_end = region->vm_end = addr + len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 }
1153 }
1154 }
1155
David Howells8feae132009-01-08 12:04:47 +00001156 vma->vm_region = region;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157
David Howells645d83c2009-09-24 15:13:10 +01001158 /* set up the mapping
Christoph Hellwigb4caecd2015-01-14 10:42:32 +01001159 * - the region is filled in if NOMMU_MAP_DIRECT is still set
David Howells645d83c2009-09-24 15:13:10 +01001160 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 if (file && vma->vm_flags & VM_SHARED)
David Howells8feae132009-01-08 12:04:47 +00001162 ret = do_mmap_shared_file(vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163 else
David Howells645d83c2009-09-24 15:13:10 +01001164 ret = do_mmap_private(vma, region, len, capabilities);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165 if (ret < 0)
David Howells645d83c2009-09-24 15:13:10 +01001166 goto error_just_free;
1167 add_nommu_region(region);
David Howells8feae132009-01-08 12:04:47 +00001168
Jie Zhangea637632009-12-14 18:00:02 -08001169 /* clear anonymous mappings that don't ask for uninitialized data */
Christoph Hellwig0bf5f942019-07-16 16:26:27 -07001170 if (!vma->vm_file &&
1171 (!IS_ENABLED(CONFIG_MMAP_ALLOW_UNINITIALIZED) ||
1172 !(flags & MAP_UNINITIALIZED)))
Jie Zhangea637632009-12-14 18:00:02 -08001173 memset((void *)region->vm_start, 0,
1174 region->vm_end - region->vm_start);
1175
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176 /* okay... we have a mapping; now we have to register it */
David Howells8feae132009-01-08 12:04:47 +00001177 result = vma->vm_start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179 current->mm->total_vm += len >> PAGE_SHIFT;
1180
David Howells8feae132009-01-08 12:04:47 +00001181share:
Liam R. Howlett07f1bc52023-01-20 11:26:29 -05001182 BUG_ON(!vma->vm_region);
Liam R. Howlettb5df0922023-07-24 14:31:52 -04001183 vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
1184 if (vma_iter_prealloc(&vmi, vma))
1185 goto error_just_free;
1186
Liam R. Howlett07f1bc52023-01-20 11:26:29 -05001187 setup_vma_to_mm(vma, current->mm);
1188 current->mm->map_count++;
1189 /* add the VMA to the tree */
1190 vma_iter_store(&vmi, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191
Mike Frysingercfe79c02010-01-06 17:23:23 +00001192 /* we flush the region from the icache only when the first executable
1193 * mapping of it is made */
1194 if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
Christoph Hellwiga75a2df2020-06-07 21:42:49 -07001195 flush_icache_user_range(region->vm_start, region->vm_end);
Mike Frysingercfe79c02010-01-06 17:23:23 +00001196 region->vm_icache_flushed = true;
1197 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198
Mike Frysingercfe79c02010-01-06 17:23:23 +00001199 up_write(&nommu_region_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200
David Howells8feae132009-01-08 12:04:47 +00001201 return result;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202
David Howells8feae132009-01-08 12:04:47 +00001203error_just_free:
1204 up_write(&nommu_region_sem);
1205error:
Liam R. Howlett47d96442023-01-20 11:26:28 -05001206 vma_iter_free(&vmi);
David Howells89a86402009-10-30 13:13:26 +00001207 if (region->vm_file)
1208 fput(region->vm_file);
David Howells8feae132009-01-08 12:04:47 +00001209 kmem_cache_free(vm_region_jar, region);
David Howells89a86402009-10-30 13:13:26 +00001210 if (vma->vm_file)
1211 fput(vma->vm_file);
Linus Torvalds3928d4f2018-07-21 13:48:51 -07001212 vm_area_free(vma);
David Howells8feae132009-01-08 12:04:47 +00001213 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214
David Howells8feae132009-01-08 12:04:47 +00001215sharing_violation:
1216 up_write(&nommu_region_sem);
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001217 pr_warn("Attempt to share mismatched mappings\n");
David Howells8feae132009-01-08 12:04:47 +00001218 ret = -EINVAL;
1219 goto error;
1220
1221error_getting_vma:
1222 kmem_cache_free(vm_region_jar, region);
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001223 pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n",
1224 len, current->pid);
Kefeng Wang1279aa02023-06-30 14:22:53 +08001225 show_mem();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 return -ENOMEM;
1227
David Howells8feae132009-01-08 12:04:47 +00001228error_getting_region:
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001229 pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n",
1230 len, current->pid);
Kefeng Wang1279aa02023-06-30 14:22:53 +08001231 show_mem();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 return -ENOMEM;
1233}
Linus Torvalds6be5ceb2012-04-20 17:13:58 -07001234
Dominik Brodowskia90f5902018-03-11 11:34:46 +01001235unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
1236 unsigned long prot, unsigned long flags,
1237 unsigned long fd, unsigned long pgoff)
Hugh Dickins66f0dc42009-12-30 20:17:34 +00001238{
1239 struct file *file = NULL;
1240 unsigned long retval = -EBADF;
1241
Al Viro120a7952010-10-30 02:54:44 -04001242 audit_mmap_fd(fd, flags);
Hugh Dickins66f0dc42009-12-30 20:17:34 +00001243 if (!(flags & MAP_ANONYMOUS)) {
1244 file = fget(fd);
1245 if (!file)
1246 goto out;
1247 }
1248
Greg Ungererad1ed292012-06-04 14:29:59 +10001249 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
Hugh Dickins66f0dc42009-12-30 20:17:34 +00001250
1251 if (file)
1252 fput(file);
1253out:
1254 return retval;
1255}
1256
Dominik Brodowskia90f5902018-03-11 11:34:46 +01001257SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1258 unsigned long, prot, unsigned long, flags,
1259 unsigned long, fd, unsigned long, pgoff)
1260{
1261 return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
1262}
1263
Christoph Hellwiga4679372010-03-10 15:21:15 -08001264#ifdef __ARCH_WANT_SYS_OLD_MMAP
1265struct mmap_arg_struct {
1266 unsigned long addr;
1267 unsigned long len;
1268 unsigned long prot;
1269 unsigned long flags;
1270 unsigned long fd;
1271 unsigned long offset;
1272};
1273
1274SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1275{
1276 struct mmap_arg_struct a;
1277
1278 if (copy_from_user(&a, arg, sizeof(a)))
1279 return -EFAULT;
Alexander Kuleshov1824cb72015-11-05 18:46:35 -08001280 if (offset_in_page(a.offset))
Christoph Hellwiga4679372010-03-10 15:21:15 -08001281 return -EINVAL;
1282
Dominik Brodowskia90f5902018-03-11 11:34:46 +01001283 return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1284 a.offset >> PAGE_SHIFT);
Christoph Hellwiga4679372010-03-10 15:21:15 -08001285}
1286#endif /* __ARCH_WANT_SYS_OLD_MMAP */
1287
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288/*
David Howells8feae132009-01-08 12:04:47 +00001289 * split a vma into two pieces at address 'addr', a new vma is allocated either
1290 * for the first part or the tail.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 */
Lorenzo Stoakesadb20b02023-10-11 18:04:29 +01001292static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
1293 unsigned long addr, int new_below)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294{
David Howells8feae132009-01-08 12:04:47 +00001295 struct vm_area_struct *new;
1296 struct vm_region *region;
1297 unsigned long npages;
Liam R. Howlett9760ebf2023-01-20 11:26:30 -05001298 struct mm_struct *mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299
David Howells779c1022010-01-15 17:01:34 -08001300 /* we're only permitted to split anonymous regions (these should have
1301 * only a single usage on the region) */
1302 if (vma->vm_file)
David Howells8feae132009-01-08 12:04:47 +00001303 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304
Liam R. Howlett9760ebf2023-01-20 11:26:30 -05001305 mm = vma->vm_mm;
David Howells8feae132009-01-08 12:04:47 +00001306 if (mm->map_count >= sysctl_max_map_count)
1307 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001308
David Howells8feae132009-01-08 12:04:47 +00001309 region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1310 if (!region)
1311 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Linus Torvalds3928d4f2018-07-21 13:48:51 -07001313 new = vm_area_dup(vma);
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001314 if (!new)
1315 goto err_vma_dup;
1316
David Howells8feae132009-01-08 12:04:47 +00001317 /* most fields are the same, copy all, and then fixup */
David Howells8feae132009-01-08 12:04:47 +00001318 *region = *vma->vm_region;
1319 new->vm_region = region;
1320
1321 npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1322
1323 if (new_below) {
Paul Mundtdd8632a2009-01-08 12:04:47 +00001324 region->vm_top = region->vm_end = new->vm_end = addr;
David Howells8feae132009-01-08 12:04:47 +00001325 } else {
1326 region->vm_start = new->vm_start = addr;
1327 region->vm_pgoff = new->vm_pgoff += npages;
1328 }
1329
Liam R. Howlettb5df0922023-07-24 14:31:52 -04001330 vma_iter_config(vmi, new->vm_start, new->vm_end);
1331 if (vma_iter_prealloc(vmi, vma)) {
1332 pr_warn("Allocation of vma tree for process %d failed\n",
1333 current->pid);
1334 goto err_vmi_preallocate;
1335 }
1336
David Howells8feae132009-01-08 12:04:47 +00001337 if (new->vm_ops && new->vm_ops->open)
1338 new->vm_ops->open(new);
1339
David Howells8feae132009-01-08 12:04:47 +00001340 down_write(&nommu_region_sem);
1341 delete_nommu_region(vma->vm_region);
1342 if (new_below) {
1343 vma->vm_region->vm_start = vma->vm_start = addr;
1344 vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1345 } else {
1346 vma->vm_region->vm_end = vma->vm_end = addr;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001347 vma->vm_region->vm_top = addr;
David Howells8feae132009-01-08 12:04:47 +00001348 }
1349 add_nommu_region(vma->vm_region);
1350 add_nommu_region(new->vm_region);
1351 up_write(&nommu_region_sem);
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001352
1353 setup_vma_to_mm(vma, mm);
1354 setup_vma_to_mm(new, mm);
Liam R. Howlett47d96442023-01-20 11:26:28 -05001355 vma_iter_store(vmi, new);
Liam Howlettfd9edbd2023-01-09 20:58:20 +00001356 mm->map_count++;
David Howells8feae132009-01-08 12:04:47 +00001357 return 0;
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001358
Liam R. Howlett47d96442023-01-20 11:26:28 -05001359err_vmi_preallocate:
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001360 vm_area_free(new);
1361err_vma_dup:
1362 kmem_cache_free(vm_region_jar, region);
1363 return -ENOMEM;
David Howells8feae132009-01-08 12:04:47 +00001364}
1365
1366/*
1367 * shrink a VMA by removing the specified chunk from either the beginning or
1368 * the end
1369 */
Liam R. Howlett07f1bc52023-01-20 11:26:29 -05001370static int vmi_shrink_vma(struct vma_iterator *vmi,
David Howells8feae132009-01-08 12:04:47 +00001371 struct vm_area_struct *vma,
1372 unsigned long from, unsigned long to)
1373{
1374 struct vm_region *region;
1375
David Howells8feae132009-01-08 12:04:47 +00001376 /* adjust the VMA's pointers, which may reposition it in the MM's tree
1377 * and list */
Liam R. Howlett07f1bc52023-01-20 11:26:29 -05001378 if (from > vma->vm_start) {
Liam R. Howlettf72cf242023-07-24 14:31:51 -04001379 if (vma_iter_clear_gfp(vmi, from, vma->vm_end, GFP_KERNEL))
1380 return -ENOMEM;
David Howells8feae132009-01-08 12:04:47 +00001381 vma->vm_end = from;
Liam R. Howlett07f1bc52023-01-20 11:26:29 -05001382 } else {
Liam R. Howlettf72cf242023-07-24 14:31:51 -04001383 if (vma_iter_clear_gfp(vmi, vma->vm_start, to, GFP_KERNEL))
1384 return -ENOMEM;
David Howells8feae132009-01-08 12:04:47 +00001385 vma->vm_start = to;
Liam R. Howlett07f1bc52023-01-20 11:26:29 -05001386 }
David Howells8feae132009-01-08 12:04:47 +00001387
1388 /* cut the backing region down to size */
1389 region = vma->vm_region;
David Howells1e2ae592010-01-15 17:01:33 -08001390 BUG_ON(region->vm_usage != 1);
David Howells8feae132009-01-08 12:04:47 +00001391
1392 down_write(&nommu_region_sem);
1393 delete_nommu_region(region);
Paul Mundtdd8632a2009-01-08 12:04:47 +00001394 if (from > region->vm_start) {
1395 to = region->vm_top;
1396 region->vm_top = region->vm_end = from;
1397 } else {
David Howells8feae132009-01-08 12:04:47 +00001398 region->vm_start = to;
Paul Mundtdd8632a2009-01-08 12:04:47 +00001399 }
David Howells8feae132009-01-08 12:04:47 +00001400 add_nommu_region(region);
1401 up_write(&nommu_region_sem);
1402
1403 free_page_series(from, to);
1404 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405}
1406
David Howells30340972006-09-27 01:50:20 -07001407/*
1408 * release a mapping
David Howells8feae132009-01-08 12:04:47 +00001409 * - under NOMMU conditions the chunk to be unmapped must be backed by a single
1410 * VMA, though it need not cover the whole VMA
David Howells30340972006-09-27 01:50:20 -07001411 */
Mike Rapoport897ab3e2017-02-24 14:58:22 -08001412int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413{
Liam R. Howlett47d96442023-01-20 11:26:28 -05001414 VMA_ITERATOR(vmi, mm, start);
David Howells8feae132009-01-08 12:04:47 +00001415 struct vm_area_struct *vma;
Bob Liuf67d9b12011-05-24 17:12:56 -07001416 unsigned long end;
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001417 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418
Bob Liuf67d9b12011-05-24 17:12:56 -07001419 len = PAGE_ALIGN(len);
David Howells8feae132009-01-08 12:04:47 +00001420 if (len == 0)
1421 return -EINVAL;
1422
Bob Liuf67d9b12011-05-24 17:12:56 -07001423 end = start + len;
1424
David Howells8feae132009-01-08 12:04:47 +00001425 /* find the first potentially overlapping VMA */
Liam R. Howlett47d96442023-01-20 11:26:28 -05001426 vma = vma_find(&vmi, end);
David Howells8feae132009-01-08 12:04:47 +00001427 if (!vma) {
Choi Gi-yongac714902014-04-07 15:37:36 -07001428 static int limit;
David Howells33e5d7692009-04-02 16:56:32 -07001429 if (limit < 5) {
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001430 pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n",
1431 current->pid, current->comm,
1432 start, start + len - 1);
David Howells33e5d7692009-04-02 16:56:32 -07001433 limit++;
1434 }
David Howells8feae132009-01-08 12:04:47 +00001435 return -EINVAL;
David Howells30340972006-09-27 01:50:20 -07001436 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001437
David Howells8feae132009-01-08 12:04:47 +00001438 /* we're allowed to split an anonymous VMA but not a file-backed one */
1439 if (vma->vm_file) {
1440 do {
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001441 if (start > vma->vm_start)
David Howells8feae132009-01-08 12:04:47 +00001442 return -EINVAL;
David Howells8feae132009-01-08 12:04:47 +00001443 if (end == vma->vm_end)
1444 goto erase_whole_vma;
Liam R. Howlett47d96442023-01-20 11:26:28 -05001445 vma = vma_find(&vmi, end);
Namhyung Kimd75a3102011-05-24 17:11:25 -07001446 } while (vma);
David Howells8feae132009-01-08 12:04:47 +00001447 return -EINVAL;
1448 } else {
1449 /* the chunk must be a subset of the VMA found */
1450 if (start == vma->vm_start && end == vma->vm_end)
1451 goto erase_whole_vma;
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001452 if (start < vma->vm_start || end > vma->vm_end)
David Howells8feae132009-01-08 12:04:47 +00001453 return -EINVAL;
Alexander Kuleshov1824cb72015-11-05 18:46:35 -08001454 if (offset_in_page(start))
David Howells8feae132009-01-08 12:04:47 +00001455 return -EINVAL;
Alexander Kuleshov1824cb72015-11-05 18:46:35 -08001456 if (end != vma->vm_end && offset_in_page(end))
David Howells8feae132009-01-08 12:04:47 +00001457 return -EINVAL;
David Howells8feae132009-01-08 12:04:47 +00001458 if (start != vma->vm_start && end != vma->vm_end) {
Liam R. Howlett9760ebf2023-01-20 11:26:30 -05001459 ret = split_vma(&vmi, vma, start, 1);
Leon Romanovsky22cc8772015-06-24 16:57:47 -07001460 if (ret < 0)
David Howells8feae132009-01-08 12:04:47 +00001461 return ret;
David Howells8feae132009-01-08 12:04:47 +00001462 }
Liam R. Howlett07f1bc52023-01-20 11:26:29 -05001463 return vmi_shrink_vma(&vmi, vma, start, end);
David Howells8feae132009-01-08 12:04:47 +00001464 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465
David Howells8feae132009-01-08 12:04:47 +00001466erase_whole_vma:
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001467 if (delete_vma_from_mm(vma))
1468 ret = -ENOMEM;
Liam Howlett80be7272023-01-09 20:57:21 +00001469 else
1470 delete_vma(mm, vma);
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001471 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472}
1473
Al Virobfce2812012-04-20 21:57:04 -04001474int vm_munmap(unsigned long addr, size_t len)
David Howells30340972006-09-27 01:50:20 -07001475{
Al Virobfce2812012-04-20 21:57:04 -04001476 struct mm_struct *mm = current->mm;
David Howells30340972006-09-27 01:50:20 -07001477 int ret;
David Howells30340972006-09-27 01:50:20 -07001478
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001479 mmap_write_lock(mm);
Mike Rapoport897ab3e2017-02-24 14:58:22 -08001480 ret = do_munmap(mm, addr, len, NULL);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001481 mmap_write_unlock(mm);
David Howells30340972006-09-27 01:50:20 -07001482 return ret;
1483}
Linus Torvaldsa46ef992012-04-20 16:20:01 -07001484EXPORT_SYMBOL(vm_munmap);
1485
1486SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1487{
Al Virobfce2812012-04-20 21:57:04 -04001488 return vm_munmap(addr, len);
Linus Torvaldsa46ef992012-04-20 16:20:01 -07001489}
David Howells30340972006-09-27 01:50:20 -07001490
1491/*
David Howells8feae132009-01-08 12:04:47 +00001492 * release all the mappings made in a process's VM space
David Howells30340972006-09-27 01:50:20 -07001493 */
David Howells8feae132009-01-08 12:04:47 +00001494void exit_mmap(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495{
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001496 VMA_ITERATOR(vmi, mm, 0);
David Howells8feae132009-01-08 12:04:47 +00001497 struct vm_area_struct *vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498
David Howells8feae132009-01-08 12:04:47 +00001499 if (!mm)
1500 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501
David Howells8feae132009-01-08 12:04:47 +00001502 mm->total_vm = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001504 /*
1505 * Lock the mm to avoid assert complaining even though this is the only
1506 * user of the mm
1507 */
1508 mmap_write_lock(mm);
1509 for_each_vma(vmi, vma) {
1510 cleanup_vma_from_mm(vma);
David Howells8feae132009-01-08 12:04:47 +00001511 delete_vma(mm, vma);
Steven J. Magnani04c34962010-11-24 12:56:54 -08001512 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513 }
Liam R. Howlett524e00b2022-09-06 19:48:48 +00001514 __mt_destroy(&mm->mm_mt);
Matthew Wilcox (Oracle)8220543d2022-09-06 19:49:05 +00001515 mmap_write_unlock(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516}
1517
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518/*
David Howells6fa5f802006-09-27 01:50:21 -07001519 * expand (or shrink) an existing mapping, potentially moving it at the same
1520 * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 *
David Howells6fa5f802006-09-27 01:50:21 -07001522 * under NOMMU conditions, we only permit changing a mapping's size, and only
David Howells8feae132009-01-08 12:04:47 +00001523 * as long as it stays within the region allocated by do_mmap_private() and the
1524 * block is not shareable
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525 *
David Howells6fa5f802006-09-27 01:50:21 -07001526 * MREMAP_FIXED is not supported under NOMMU conditions
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527 */
Al Viro4b377ba2013-03-04 10:47:59 -05001528static unsigned long do_mremap(unsigned long addr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 unsigned long old_len, unsigned long new_len,
1530 unsigned long flags, unsigned long new_addr)
1531{
David Howells6fa5f802006-09-27 01:50:21 -07001532 struct vm_area_struct *vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001533
1534 /* insanity checks first */
Bob Liuf67d9b12011-05-24 17:12:56 -07001535 old_len = PAGE_ALIGN(old_len);
1536 new_len = PAGE_ALIGN(new_len);
David Howells8feae132009-01-08 12:04:47 +00001537 if (old_len == 0 || new_len == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 return (unsigned long) -EINVAL;
1539
Alexander Kuleshov1824cb72015-11-05 18:46:35 -08001540 if (offset_in_page(addr))
David Howells8feae132009-01-08 12:04:47 +00001541 return -EINVAL;
1542
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543 if (flags & MREMAP_FIXED && new_addr != addr)
1544 return (unsigned long) -EINVAL;
1545
David Howells8feae132009-01-08 12:04:47 +00001546 vma = find_vma_exact(current->mm, addr, old_len);
David Howells6fa5f802006-09-27 01:50:21 -07001547 if (!vma)
1548 return (unsigned long) -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
David Howells6fa5f802006-09-27 01:50:21 -07001550 if (vma->vm_end != vma->vm_start + old_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 return (unsigned long) -EFAULT;
1552
David Hildenbrandfc4f4be2023-01-02 17:08:54 +01001553 if (is_nommu_shared_mapping(vma->vm_flags))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554 return (unsigned long) -EPERM;
1555
David Howells8feae132009-01-08 12:04:47 +00001556 if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 return (unsigned long) -ENOMEM;
1558
1559 /* all checks complete - do it */
David Howells6fa5f802006-09-27 01:50:21 -07001560 vma->vm_end = vma->vm_start + new_len;
David Howells6fa5f802006-09-27 01:50:21 -07001561 return vma->vm_start;
1562}
1563
Heiko Carstens6a6160a2009-01-14 14:14:15 +01001564SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
1565 unsigned long, new_len, unsigned long, flags,
1566 unsigned long, new_addr)
David Howells6fa5f802006-09-27 01:50:21 -07001567{
1568 unsigned long ret;
1569
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001570 mmap_write_lock(current->mm);
David Howells6fa5f802006-09-27 01:50:21 -07001571 ret = do_mremap(addr, old_len, new_len, flags, new_addr);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001572 mmap_write_unlock(current->mm);
David Howells6fa5f802006-09-27 01:50:21 -07001573 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574}
1575
Keith Buschdf06b372018-10-26 15:10:28 -07001576struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
1577 unsigned int foll_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578{
1579 return NULL;
1580}
1581
Bob Liu8f3b1322011-07-08 15:39:46 -07001582int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1583 unsigned long pfn, unsigned long size, pgprot_t prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584{
Bob Liu8f3b1322011-07-08 15:39:46 -07001585 if (addr != (pfn << PAGE_SHIFT))
1586 return -EINVAL;
1587
Suren Baghdasaryan1c712222023-01-26 11:37:49 -08001588 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
Greg Ungerer66aa2b42005-09-12 11:18:10 +10001589 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590}
Luke Yang22c4af42006-07-14 00:24:09 -07001591EXPORT_SYMBOL(remap_pfn_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592
Linus Torvalds3c0b9de2013-04-27 13:25:38 -07001593int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
1594{
1595 unsigned long pfn = start >> PAGE_SHIFT;
1596 unsigned long vm_len = vma->vm_end - vma->vm_start;
1597
1598 pfn += vma->vm_pgoff;
1599 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
1600}
1601EXPORT_SYMBOL(vm_iomap_memory);
1602
Paul Mundtf905bc42008-02-04 22:29:59 -08001603int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
1604 unsigned long pgoff)
1605{
1606 unsigned int size = vma->vm_end - vma->vm_start;
1607
1608 if (!(vma->vm_flags & VM_USERMAP))
1609 return -EINVAL;
1610
1611 vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
1612 vma->vm_end = vma->vm_start + size;
1613
1614 return 0;
1615}
1616EXPORT_SYMBOL(remap_vmalloc_range);
1617
Souptick Joarder2bcd6452018-06-07 17:08:00 -07001618vm_fault_t filemap_fault(struct vm_fault *vmf)
David Howellsb0e15192006-01-06 00:11:42 -08001619{
1620 BUG();
Nick Piggind0217ac2007-07-19 01:47:03 -07001621 return 0;
David Howellsb0e15192006-01-06 00:11:42 -08001622}
Paul Mundtb5073172007-07-21 04:37:25 -07001623EXPORT_SYMBOL(filemap_fault);
David Howells0ec76a12006-09-27 01:50:15 -07001624
Geert Uytterhoeven3f98a282021-01-28 11:06:26 +01001625vm_fault_t filemap_map_pages(struct vm_fault *vmf,
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001626 pgoff_t start_pgoff, pgoff_t end_pgoff)
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07001627{
1628 BUG();
Geert Uytterhoeven3f98a282021-01-28 11:06:26 +01001629 return 0;
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07001630}
1631EXPORT_SYMBOL(filemap_map_pages);
1632
Lorenzo Stoakesc43cfa42023-10-03 00:14:51 +01001633static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
1634 void *buf, int len, unsigned int gup_flags)
David Howells0ec76a12006-09-27 01:50:15 -07001635{
David Howells0ec76a12006-09-27 01:50:15 -07001636 struct vm_area_struct *vma;
Lorenzo Stoakes442486e2016-10-13 01:20:18 +01001637 int write = gup_flags & FOLL_WRITE;
David Howells0ec76a12006-09-27 01:50:15 -07001638
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001639 if (mmap_read_lock_killable(mm))
Konstantin Khlebnikov1e426fe2019-07-11 21:00:07 -07001640 return 0;
David Howells0ec76a12006-09-27 01:50:15 -07001641
1642 /* the access must start within one of the target process's mappings */
David Howells0159b142006-09-27 01:50:16 -07001643 vma = find_vma(mm, addr);
1644 if (vma) {
David Howells0ec76a12006-09-27 01:50:15 -07001645 /* don't overrun this mapping */
1646 if (addr + len >= vma->vm_end)
1647 len = vma->vm_end - addr;
1648
1649 /* only read or write mappings where it is permitted */
David Howellsd00c7b992006-09-27 01:50:19 -07001650 if (write && vma->vm_flags & VM_MAYWRITE)
Jie Zhang79597222010-01-06 17:23:28 +00001651 copy_to_user_page(vma, NULL, addr,
1652 (void *) addr, buf, len);
David Howellsd00c7b992006-09-27 01:50:19 -07001653 else if (!write && vma->vm_flags & VM_MAYREAD)
Jie Zhang79597222010-01-06 17:23:28 +00001654 copy_from_user_page(vma, NULL, addr,
1655 buf, (void *) addr, len);
David Howells0ec76a12006-09-27 01:50:15 -07001656 else
1657 len = 0;
1658 } else {
1659 len = 0;
1660 }
1661
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001662 mmap_read_unlock(mm);
Mike Frysingerf55f1992011-03-29 14:05:12 +01001663
1664 return len;
1665}
1666
1667/**
Mike Rapoportb7701a52018-02-06 15:42:13 -08001668 * access_remote_vm - access another process' address space
Mike Frysingerf55f1992011-03-29 14:05:12 +01001669 * @mm: the mm_struct of the target address space
1670 * @addr: start address to access
1671 * @buf: source or destination buffer
1672 * @len: number of bytes to transfer
Lorenzo Stoakes6347e8d2016-10-13 01:20:19 +01001673 * @gup_flags: flags modifying lookup behaviour
Mike Frysingerf55f1992011-03-29 14:05:12 +01001674 *
1675 * The caller must hold a reference on @mm.
1676 */
1677int access_remote_vm(struct mm_struct *mm, unsigned long addr,
Lorenzo Stoakes6347e8d2016-10-13 01:20:19 +01001678 void *buf, int len, unsigned int gup_flags)
Mike Frysingerf55f1992011-03-29 14:05:12 +01001679{
John Hubbardd3f5ffc2020-12-14 19:07:45 -08001680 return __access_remote_vm(mm, addr, buf, len, gup_flags);
Mike Frysingerf55f1992011-03-29 14:05:12 +01001681}
1682
1683/*
1684 * Access another process' address space.
1685 * - source/target buffer must be kernel space
1686 */
Lorenzo Stoakesf307ab62016-10-13 01:20:20 +01001687int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len,
1688 unsigned int gup_flags)
Mike Frysingerf55f1992011-03-29 14:05:12 +01001689{
1690 struct mm_struct *mm;
1691
1692 if (addr + len < addr)
1693 return 0;
1694
1695 mm = get_task_mm(tsk);
1696 if (!mm)
1697 return 0;
1698
John Hubbardd3f5ffc2020-12-14 19:07:45 -08001699 len = __access_remote_vm(mm, addr, buf, len, gup_flags);
Mike Frysingerf55f1992011-03-29 14:05:12 +01001700
David Howells0ec76a12006-09-27 01:50:15 -07001701 mmput(mm);
1702 return len;
1703}
Catalin Marinasfcd35852016-11-01 14:43:25 -07001704EXPORT_SYMBOL_GPL(access_process_vm);
David Howells7e660872010-01-15 17:01:39 -08001705
1706/**
1707 * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1708 * @inode: The inode to check
1709 * @size: The current filesize of the inode
1710 * @newsize: The proposed filesize of the inode
1711 *
1712 * Check the shared mappings on an inode on behalf of a shrinking truncate to
Randy Dunlapc08b3422020-08-11 18:33:11 -07001713 * make sure that any outstanding VMAs aren't broken and then shrink the
1714 * vm_regions that extend beyond so that do_mmap() doesn't
David Howells7e660872010-01-15 17:01:39 -08001715 * automatically grant mappings that are too large.
1716 */
1717int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
1718 size_t newsize)
1719{
1720 struct vm_area_struct *vma;
David Howells7e660872010-01-15 17:01:39 -08001721 struct vm_region *region;
1722 pgoff_t low, high;
1723 size_t r_size, r_top;
1724
1725 low = newsize >> PAGE_SHIFT;
1726 high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1727
1728 down_write(&nommu_region_sem);
Davidlohr Bueso1acf2e02014-12-12 16:54:39 -08001729 i_mmap_lock_read(inode->i_mapping);
David Howells7e660872010-01-15 17:01:39 -08001730
1731 /* search for VMAs that fall within the dead zone */
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001732 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
David Howells7e660872010-01-15 17:01:39 -08001733 /* found one - only interested if it's shared out of the page
1734 * cache */
1735 if (vma->vm_flags & VM_SHARED) {
Davidlohr Bueso1acf2e02014-12-12 16:54:39 -08001736 i_mmap_unlock_read(inode->i_mapping);
David Howells7e660872010-01-15 17:01:39 -08001737 up_write(&nommu_region_sem);
1738 return -ETXTBSY; /* not quite true, but near enough */
1739 }
1740 }
1741
1742 /* reduce any regions that overlap the dead zone - if in existence,
1743 * these will be pointed to by VMAs that don't overlap the dead zone
1744 *
1745 * we don't check for any regions that start beyond the EOF as there
1746 * shouldn't be any
1747 */
Davidlohr Bueso1acf2e02014-12-12 16:54:39 -08001748 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
David Howells7e660872010-01-15 17:01:39 -08001749 if (!(vma->vm_flags & VM_SHARED))
1750 continue;
1751
1752 region = vma->vm_region;
1753 r_size = region->vm_top - region->vm_start;
1754 r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
1755
1756 if (r_top > newsize) {
1757 region->vm_top -= r_top - newsize;
1758 if (region->vm_end > region->vm_top)
1759 region->vm_end = region->vm_top;
1760 }
1761 }
1762
Davidlohr Bueso1acf2e02014-12-12 16:54:39 -08001763 i_mmap_unlock_read(inode->i_mapping);
David Howells7e660872010-01-15 17:01:39 -08001764 up_write(&nommu_region_sem);
1765 return 0;
1766}
Andrew Shewmakerc9b1d092013-04-29 15:08:10 -07001767
1768/*
1769 * Initialise sysctl_user_reserve_kbytes.
1770 *
1771 * This is intended to prevent a user from starting a single memory hogging
1772 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
1773 * mode.
1774 *
1775 * The default value is min(3% of free memory, 128MB)
1776 * 128MB is enough to recover with sshd/login, bash, and top/kill.
1777 */
1778static int __meminit init_user_reserve(void)
1779{
1780 unsigned long free_kbytes;
1781
ZhangPengd5a64742023-08-04 09:25:57 +08001782 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
Andrew Shewmakerc9b1d092013-04-29 15:08:10 -07001783
1784 sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
1785 return 0;
1786}
Paul Gortmakera4bc6fc2015-05-01 20:08:20 -04001787subsys_initcall(init_user_reserve);
Andrew Shewmaker4eeab4f2013-04-29 15:08:11 -07001788
1789/*
1790 * Initialise sysctl_admin_reserve_kbytes.
1791 *
1792 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
1793 * to log in and kill a memory hogging process.
1794 *
1795 * Systems with more than 256MB will reserve 8MB, enough to recover
1796 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
1797 * only reserve 3% of free pages by default.
1798 */
1799static int __meminit init_admin_reserve(void)
1800{
1801 unsigned long free_kbytes;
1802
ZhangPengd5a64742023-08-04 09:25:57 +08001803 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
Andrew Shewmaker4eeab4f2013-04-29 15:08:11 -07001804
1805 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
1806 return 0;
1807}
Paul Gortmakera4bc6fc2015-05-01 20:08:20 -04001808subsys_initcall(init_admin_reserve);