Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/mm/vmalloc.c |
| 3 | * |
| 4 | * Copyright (C) 1993 Linus Torvalds |
| 5 | * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 |
| 6 | * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 |
| 7 | * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 8 | * Numa awareness, Christoph Lameter, SGI, June 2005 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | */ |
| 10 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 11 | #include <linux/vmalloc.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/mm.h> |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/highmem.h> |
Alexey Dobriyan | d43c36d | 2009-10-07 17:09:06 +0400 | [diff] [blame] | 15 | #include <linux/sched.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/slab.h> |
| 17 | #include <linux/spinlock.h> |
| 18 | #include <linux/interrupt.h> |
Alexey Dobriyan | 5f6a6a9 | 2008-10-06 03:50:47 +0400 | [diff] [blame] | 19 | #include <linux/proc_fs.h> |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 20 | #include <linux/seq_file.h> |
Thomas Gleixner | 3ac7fe5 | 2008-04-30 00:55:01 -0700 | [diff] [blame] | 21 | #include <linux/debugobjects.h> |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 22 | #include <linux/kallsyms.h> |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 23 | #include <linux/list.h> |
Chris Wilson | 4da56b9 | 2016-04-04 14:46:42 +0100 | [diff] [blame] | 24 | #include <linux/notifier.h> |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 25 | #include <linux/rbtree.h> |
| 26 | #include <linux/radix-tree.h> |
| 27 | #include <linux/rcupdate.h> |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 28 | #include <linux/pfn.h> |
Catalin Marinas | 89219d3 | 2009-06-11 13:23:19 +0100 | [diff] [blame] | 29 | #include <linux/kmemleak.h> |
Arun Sharma | 60063497 | 2011-07-26 16:09:06 -0700 | [diff] [blame] | 30 | #include <linux/atomic.h> |
Gideon Israel Dsouza | 3b32123 | 2014-04-07 15:37:26 -0700 | [diff] [blame] | 31 | #include <linux/compiler.h> |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 32 | #include <linux/llist.h> |
Toshi Kani | 0f616be | 2015-04-14 15:47:17 -0700 | [diff] [blame] | 33 | #include <linux/bitops.h> |
Gideon Israel Dsouza | 3b32123 | 2014-04-07 15:37:26 -0700 | [diff] [blame] | 34 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | #include <asm/uaccess.h> |
| 36 | #include <asm/tlbflush.h> |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 37 | #include <asm/shmparam.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | |
Mel Gorman | dd56b04 | 2015-11-06 16:28:43 -0800 | [diff] [blame] | 39 | #include "internal.h" |
| 40 | |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 41 | struct vfree_deferred { |
| 42 | struct llist_head list; |
| 43 | struct work_struct wq; |
| 44 | }; |
| 45 | static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); |
| 46 | |
| 47 | static void __vunmap(const void *, int); |
| 48 | |
| 49 | static void free_work(struct work_struct *w) |
| 50 | { |
| 51 | struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); |
| 52 | struct llist_node *llnode = llist_del_all(&p->list); |
| 53 | while (llnode) { |
| 54 | void *p = llnode; |
| 55 | llnode = llist_next(llnode); |
| 56 | __vunmap(p, 1); |
| 57 | } |
| 58 | } |
| 59 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 60 | /*** Page table manipulation functions ***/ |
Adrian Bunk | b221385 | 2006-09-25 23:31:02 -0700 | [diff] [blame] | 61 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end) |
| 63 | { |
| 64 | pte_t *pte; |
| 65 | |
| 66 | pte = pte_offset_kernel(pmd, addr); |
| 67 | do { |
| 68 | pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); |
| 69 | WARN_ON(!pte_none(ptent) && !pte_present(ptent)); |
| 70 | } while (pte++, addr += PAGE_SIZE, addr != end); |
| 71 | } |
| 72 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 73 | static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | { |
| 75 | pmd_t *pmd; |
| 76 | unsigned long next; |
| 77 | |
| 78 | pmd = pmd_offset(pud, addr); |
| 79 | do { |
| 80 | next = pmd_addr_end(addr, end); |
Toshi Kani | b9820d8 | 2015-04-14 15:47:26 -0700 | [diff] [blame] | 81 | if (pmd_clear_huge(pmd)) |
| 82 | continue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 83 | if (pmd_none_or_clear_bad(pmd)) |
| 84 | continue; |
| 85 | vunmap_pte_range(pmd, addr, next); |
| 86 | } while (pmd++, addr = next, addr != end); |
| 87 | } |
| 88 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 89 | static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 90 | { |
| 91 | pud_t *pud; |
| 92 | unsigned long next; |
| 93 | |
| 94 | pud = pud_offset(pgd, addr); |
| 95 | do { |
| 96 | next = pud_addr_end(addr, end); |
Toshi Kani | b9820d8 | 2015-04-14 15:47:26 -0700 | [diff] [blame] | 97 | if (pud_clear_huge(pud)) |
| 98 | continue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | if (pud_none_or_clear_bad(pud)) |
| 100 | continue; |
| 101 | vunmap_pmd_range(pud, addr, next); |
| 102 | } while (pud++, addr = next, addr != end); |
| 103 | } |
| 104 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 105 | static void vunmap_page_range(unsigned long addr, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | { |
| 107 | pgd_t *pgd; |
| 108 | unsigned long next; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | |
| 110 | BUG_ON(addr >= end); |
| 111 | pgd = pgd_offset_k(addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | do { |
| 113 | next = pgd_addr_end(addr, end); |
| 114 | if (pgd_none_or_clear_bad(pgd)) |
| 115 | continue; |
| 116 | vunmap_pud_range(pgd, addr, next); |
| 117 | } while (pgd++, addr = next, addr != end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | } |
| 119 | |
| 120 | static int vmap_pte_range(pmd_t *pmd, unsigned long addr, |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 121 | unsigned long end, pgprot_t prot, struct page **pages, int *nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | { |
| 123 | pte_t *pte; |
| 124 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 125 | /* |
| 126 | * nr is a running index into the array which helps higher level |
| 127 | * callers keep track of where we're up to. |
| 128 | */ |
| 129 | |
Hugh Dickins | 872fec1 | 2005-10-29 18:16:21 -0700 | [diff] [blame] | 130 | pte = pte_alloc_kernel(pmd, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | if (!pte) |
| 132 | return -ENOMEM; |
| 133 | do { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 134 | struct page *page = pages[*nr]; |
| 135 | |
| 136 | if (WARN_ON(!pte_none(*pte))) |
| 137 | return -EBUSY; |
| 138 | if (WARN_ON(!page)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | return -ENOMEM; |
| 140 | set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 141 | (*nr)++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | } while (pte++, addr += PAGE_SIZE, addr != end); |
| 143 | return 0; |
| 144 | } |
| 145 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 146 | static int vmap_pmd_range(pud_t *pud, unsigned long addr, |
| 147 | unsigned long end, pgprot_t prot, struct page **pages, int *nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 148 | { |
| 149 | pmd_t *pmd; |
| 150 | unsigned long next; |
| 151 | |
| 152 | pmd = pmd_alloc(&init_mm, pud, addr); |
| 153 | if (!pmd) |
| 154 | return -ENOMEM; |
| 155 | do { |
| 156 | next = pmd_addr_end(addr, end); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 157 | if (vmap_pte_range(pmd, addr, next, prot, pages, nr)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 158 | return -ENOMEM; |
| 159 | } while (pmd++, addr = next, addr != end); |
| 160 | return 0; |
| 161 | } |
| 162 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 163 | static int vmap_pud_range(pgd_t *pgd, unsigned long addr, |
| 164 | unsigned long end, pgprot_t prot, struct page **pages, int *nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | { |
| 166 | pud_t *pud; |
| 167 | unsigned long next; |
| 168 | |
| 169 | pud = pud_alloc(&init_mm, pgd, addr); |
| 170 | if (!pud) |
| 171 | return -ENOMEM; |
| 172 | do { |
| 173 | next = pud_addr_end(addr, end); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 174 | if (vmap_pmd_range(pud, addr, next, prot, pages, nr)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | return -ENOMEM; |
| 176 | } while (pud++, addr = next, addr != end); |
| 177 | return 0; |
| 178 | } |
| 179 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 180 | /* |
| 181 | * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and |
| 182 | * will have pfns corresponding to the "pages" array. |
| 183 | * |
| 184 | * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] |
| 185 | */ |
Tejun Heo | 8fc4898 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 186 | static int vmap_page_range_noflush(unsigned long start, unsigned long end, |
| 187 | pgprot_t prot, struct page **pages) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | { |
| 189 | pgd_t *pgd; |
| 190 | unsigned long next; |
Adam Lackorzynski | 2e4e27c | 2009-01-04 12:00:46 -0800 | [diff] [blame] | 191 | unsigned long addr = start; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 192 | int err = 0; |
| 193 | int nr = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | |
| 195 | BUG_ON(addr >= end); |
| 196 | pgd = pgd_offset_k(addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | do { |
| 198 | next = pgd_addr_end(addr, end); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 199 | err = vmap_pud_range(pgd, addr, next, prot, pages, &nr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | if (err) |
Figo.zhang | bf88c8c | 2009-09-21 17:01:47 -0700 | [diff] [blame] | 201 | return err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 202 | } while (pgd++, addr = next, addr != end); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 203 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 204 | return nr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | } |
| 206 | |
Tejun Heo | 8fc4898 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 207 | static int vmap_page_range(unsigned long start, unsigned long end, |
| 208 | pgprot_t prot, struct page **pages) |
| 209 | { |
| 210 | int ret; |
| 211 | |
| 212 | ret = vmap_page_range_noflush(start, end, prot, pages); |
| 213 | flush_cache_vmap(start, end); |
| 214 | return ret; |
| 215 | } |
| 216 | |
KAMEZAWA Hiroyuki | 81ac3ad | 2009-09-22 16:45:49 -0700 | [diff] [blame] | 217 | int is_vmalloc_or_module_addr(const void *x) |
Linus Torvalds | 73bdf0a | 2008-10-15 08:35:12 -0700 | [diff] [blame] | 218 | { |
| 219 | /* |
Russell King | ab4f2ee | 2008-11-06 17:11:07 +0000 | [diff] [blame] | 220 | * ARM, x86-64 and sparc64 put modules in a special place, |
Linus Torvalds | 73bdf0a | 2008-10-15 08:35:12 -0700 | [diff] [blame] | 221 | * and fall back on vmalloc() if that fails. Others |
| 222 | * just put it in the vmalloc space. |
| 223 | */ |
| 224 | #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) |
| 225 | unsigned long addr = (unsigned long)x; |
| 226 | if (addr >= MODULES_VADDR && addr < MODULES_END) |
| 227 | return 1; |
| 228 | #endif |
| 229 | return is_vmalloc_addr(x); |
| 230 | } |
| 231 | |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 232 | /* |
malc | add688f | 2014-01-27 17:06:53 -0800 | [diff] [blame] | 233 | * Walk a vmap address to the struct page it maps. |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 234 | */ |
malc | add688f | 2014-01-27 17:06:53 -0800 | [diff] [blame] | 235 | struct page *vmalloc_to_page(const void *vmalloc_addr) |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 236 | { |
| 237 | unsigned long addr = (unsigned long) vmalloc_addr; |
malc | add688f | 2014-01-27 17:06:53 -0800 | [diff] [blame] | 238 | struct page *page = NULL; |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 239 | pgd_t *pgd = pgd_offset_k(addr); |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 240 | |
Ingo Molnar | 7aa413d | 2008-06-19 13:28:11 +0200 | [diff] [blame] | 241 | /* |
| 242 | * XXX we might need to change this if we add VIRTUAL_BUG_ON for |
| 243 | * architectures that do not vmalloc module space |
| 244 | */ |
Linus Torvalds | 73bdf0a | 2008-10-15 08:35:12 -0700 | [diff] [blame] | 245 | VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); |
Jiri Slaby | 59ea746 | 2008-06-12 13:56:40 +0200 | [diff] [blame] | 246 | |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 247 | if (!pgd_none(*pgd)) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 248 | pud_t *pud = pud_offset(pgd, addr); |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 249 | if (!pud_none(*pud)) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 250 | pmd_t *pmd = pmd_offset(pud, addr); |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 251 | if (!pmd_none(*pmd)) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 252 | pte_t *ptep, pte; |
| 253 | |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 254 | ptep = pte_offset_map(pmd, addr); |
| 255 | pte = *ptep; |
| 256 | if (pte_present(pte)) |
malc | add688f | 2014-01-27 17:06:53 -0800 | [diff] [blame] | 257 | page = pte_page(pte); |
Christoph Lameter | 48667e7 | 2008-02-04 22:28:31 -0800 | [diff] [blame] | 258 | pte_unmap(ptep); |
| 259 | } |
| 260 | } |
| 261 | } |
malc | add688f | 2014-01-27 17:06:53 -0800 | [diff] [blame] | 262 | return page; |
Jianyu Zhan | ece86e22 | 2014-01-21 15:49:12 -0800 | [diff] [blame] | 263 | } |
| 264 | EXPORT_SYMBOL(vmalloc_to_page); |
| 265 | |
malc | add688f | 2014-01-27 17:06:53 -0800 | [diff] [blame] | 266 | /* |
| 267 | * Map a vmalloc()-space virtual address to the physical page frame number. |
| 268 | */ |
| 269 | unsigned long vmalloc_to_pfn(const void *vmalloc_addr) |
| 270 | { |
| 271 | return page_to_pfn(vmalloc_to_page(vmalloc_addr)); |
| 272 | } |
| 273 | EXPORT_SYMBOL(vmalloc_to_pfn); |
| 274 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 275 | |
| 276 | /*** Global kva allocator ***/ |
| 277 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 278 | #define VM_VM_AREA 0x04 |
| 279 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 280 | static DEFINE_SPINLOCK(vmap_area_lock); |
Joonsoo Kim | f1c4069 | 2013-04-29 15:07:37 -0700 | [diff] [blame] | 281 | /* Export for kexec only */ |
| 282 | LIST_HEAD(vmap_area_list); |
Chris Wilson | 80c4bd7 | 2016-05-20 16:57:38 -0700 | [diff] [blame] | 283 | static LLIST_HEAD(vmap_purge_list); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 284 | static struct rb_root vmap_area_root = RB_ROOT; |
| 285 | |
| 286 | /* The vmap cache globals are protected by vmap_area_lock */ |
| 287 | static struct rb_node *free_vmap_cache; |
| 288 | static unsigned long cached_hole_size; |
| 289 | static unsigned long cached_vstart; |
| 290 | static unsigned long cached_align; |
| 291 | |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 292 | static unsigned long vmap_area_pcpu_hole; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 293 | |
| 294 | static struct vmap_area *__find_vmap_area(unsigned long addr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 295 | { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 296 | struct rb_node *n = vmap_area_root.rb_node; |
| 297 | |
| 298 | while (n) { |
| 299 | struct vmap_area *va; |
| 300 | |
| 301 | va = rb_entry(n, struct vmap_area, rb_node); |
| 302 | if (addr < va->va_start) |
| 303 | n = n->rb_left; |
HATAYAMA Daisuke | cef2ac3 | 2013-07-03 15:02:17 -0700 | [diff] [blame] | 304 | else if (addr >= va->va_end) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 305 | n = n->rb_right; |
| 306 | else |
| 307 | return va; |
| 308 | } |
| 309 | |
| 310 | return NULL; |
| 311 | } |
| 312 | |
| 313 | static void __insert_vmap_area(struct vmap_area *va) |
| 314 | { |
| 315 | struct rb_node **p = &vmap_area_root.rb_node; |
| 316 | struct rb_node *parent = NULL; |
| 317 | struct rb_node *tmp; |
| 318 | |
| 319 | while (*p) { |
Namhyung Kim | 170168d | 2010-10-26 14:22:02 -0700 | [diff] [blame] | 320 | struct vmap_area *tmp_va; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 321 | |
| 322 | parent = *p; |
Namhyung Kim | 170168d | 2010-10-26 14:22:02 -0700 | [diff] [blame] | 323 | tmp_va = rb_entry(parent, struct vmap_area, rb_node); |
| 324 | if (va->va_start < tmp_va->va_end) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 325 | p = &(*p)->rb_left; |
Namhyung Kim | 170168d | 2010-10-26 14:22:02 -0700 | [diff] [blame] | 326 | else if (va->va_end > tmp_va->va_start) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 327 | p = &(*p)->rb_right; |
| 328 | else |
| 329 | BUG(); |
| 330 | } |
| 331 | |
| 332 | rb_link_node(&va->rb_node, parent, p); |
| 333 | rb_insert_color(&va->rb_node, &vmap_area_root); |
| 334 | |
Joonsoo Kim | 4341fa4 | 2013-04-29 15:07:39 -0700 | [diff] [blame] | 335 | /* address-sort this list */ |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 336 | tmp = rb_prev(&va->rb_node); |
| 337 | if (tmp) { |
| 338 | struct vmap_area *prev; |
| 339 | prev = rb_entry(tmp, struct vmap_area, rb_node); |
| 340 | list_add_rcu(&va->list, &prev->list); |
| 341 | } else |
| 342 | list_add_rcu(&va->list, &vmap_area_list); |
| 343 | } |
| 344 | |
| 345 | static void purge_vmap_area_lazy(void); |
| 346 | |
Chris Wilson | 4da56b9 | 2016-04-04 14:46:42 +0100 | [diff] [blame] | 347 | static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); |
| 348 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 349 | /* |
| 350 | * Allocate a region of KVA of the specified size and alignment, within the |
| 351 | * vstart and vend. |
| 352 | */ |
| 353 | static struct vmap_area *alloc_vmap_area(unsigned long size, |
| 354 | unsigned long align, |
| 355 | unsigned long vstart, unsigned long vend, |
| 356 | int node, gfp_t gfp_mask) |
| 357 | { |
| 358 | struct vmap_area *va; |
| 359 | struct rb_node *n; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 360 | unsigned long addr; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 361 | int purged = 0; |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 362 | struct vmap_area *first; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 363 | |
Nick Piggin | 7766970 | 2009-02-27 14:03:03 -0800 | [diff] [blame] | 364 | BUG_ON(!size); |
Alexander Kuleshov | 891c49a | 2015-11-05 18:46:51 -0800 | [diff] [blame] | 365 | BUG_ON(offset_in_page(size)); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 366 | BUG_ON(!is_power_of_2(align)); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 367 | |
Chris Wilson | 4da56b9 | 2016-04-04 14:46:42 +0100 | [diff] [blame] | 368 | might_sleep_if(gfpflags_allow_blocking(gfp_mask)); |
| 369 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 370 | va = kmalloc_node(sizeof(struct vmap_area), |
| 371 | gfp_mask & GFP_RECLAIM_MASK, node); |
| 372 | if (unlikely(!va)) |
| 373 | return ERR_PTR(-ENOMEM); |
| 374 | |
Catalin Marinas | 7f88f88 | 2013-11-12 15:07:45 -0800 | [diff] [blame] | 375 | /* |
| 376 | * Only scan the relevant parts containing pointers to other objects |
| 377 | * to avoid false negatives. |
| 378 | */ |
| 379 | kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK); |
| 380 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 381 | retry: |
| 382 | spin_lock(&vmap_area_lock); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 383 | /* |
| 384 | * Invalidate cache if we have more permissive parameters. |
| 385 | * cached_hole_size notes the largest hole noticed _below_ |
| 386 | * the vmap_area cached in free_vmap_cache: if size fits |
| 387 | * into that hole, we want to scan from vstart to reuse |
| 388 | * the hole instead of allocating above free_vmap_cache. |
| 389 | * Note that __free_vmap_area may update free_vmap_cache |
| 390 | * without updating cached_hole_size or cached_align. |
| 391 | */ |
| 392 | if (!free_vmap_cache || |
| 393 | size < cached_hole_size || |
| 394 | vstart < cached_vstart || |
| 395 | align < cached_align) { |
| 396 | nocache: |
| 397 | cached_hole_size = 0; |
| 398 | free_vmap_cache = NULL; |
| 399 | } |
| 400 | /* record if we encounter less permissive parameters */ |
| 401 | cached_vstart = vstart; |
| 402 | cached_align = align; |
Nick Piggin | 7766970 | 2009-02-27 14:03:03 -0800 | [diff] [blame] | 403 | |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 404 | /* find starting point for our search */ |
| 405 | if (free_vmap_cache) { |
| 406 | first = rb_entry(free_vmap_cache, struct vmap_area, rb_node); |
Johannes Weiner | 248ac0e | 2011-05-24 17:11:43 -0700 | [diff] [blame] | 407 | addr = ALIGN(first->va_end, align); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 408 | if (addr < vstart) |
| 409 | goto nocache; |
Zhang Yanfei | bcb615a | 2013-07-08 16:00:19 -0700 | [diff] [blame] | 410 | if (addr + size < addr) |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 411 | goto overflow; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 412 | |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 413 | } else { |
| 414 | addr = ALIGN(vstart, align); |
Zhang Yanfei | bcb615a | 2013-07-08 16:00:19 -0700 | [diff] [blame] | 415 | if (addr + size < addr) |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 416 | goto overflow; |
| 417 | |
| 418 | n = vmap_area_root.rb_node; |
| 419 | first = NULL; |
| 420 | |
| 421 | while (n) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 422 | struct vmap_area *tmp; |
| 423 | tmp = rb_entry(n, struct vmap_area, rb_node); |
| 424 | if (tmp->va_end >= addr) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 425 | first = tmp; |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 426 | if (tmp->va_start <= addr) |
| 427 | break; |
| 428 | n = n->rb_left; |
| 429 | } else |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 430 | n = n->rb_right; |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 431 | } |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 432 | |
| 433 | if (!first) |
| 434 | goto found; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 435 | } |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 436 | |
| 437 | /* from the starting point, walk areas until a suitable hole is found */ |
Johannes Weiner | 248ac0e | 2011-05-24 17:11:43 -0700 | [diff] [blame] | 438 | while (addr + size > first->va_start && addr + size <= vend) { |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 439 | if (addr + cached_hole_size < first->va_start) |
| 440 | cached_hole_size = first->va_start - addr; |
Johannes Weiner | 248ac0e | 2011-05-24 17:11:43 -0700 | [diff] [blame] | 441 | addr = ALIGN(first->va_end, align); |
Zhang Yanfei | bcb615a | 2013-07-08 16:00:19 -0700 | [diff] [blame] | 442 | if (addr + size < addr) |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 443 | goto overflow; |
| 444 | |
Hong zhi guo | 92ca922 | 2012-07-31 16:41:35 -0700 | [diff] [blame] | 445 | if (list_is_last(&first->list, &vmap_area_list)) |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 446 | goto found; |
Hong zhi guo | 92ca922 | 2012-07-31 16:41:35 -0700 | [diff] [blame] | 447 | |
Geliang Tang | 6219c2a | 2016-01-14 15:19:08 -0800 | [diff] [blame] | 448 | first = list_next_entry(first, list); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 449 | } |
| 450 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 451 | found: |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 452 | if (addr + size > vend) |
| 453 | goto overflow; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 454 | |
| 455 | va->va_start = addr; |
| 456 | va->va_end = addr + size; |
| 457 | va->flags = 0; |
| 458 | __insert_vmap_area(va); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 459 | free_vmap_cache = &va->rb_node; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 460 | spin_unlock(&vmap_area_lock); |
| 461 | |
Wang Xiaoqiang | 61e1655 | 2016-01-15 16:57:19 -0800 | [diff] [blame] | 462 | BUG_ON(!IS_ALIGNED(va->va_start, align)); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 463 | BUG_ON(va->va_start < vstart); |
| 464 | BUG_ON(va->va_end > vend); |
| 465 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 466 | return va; |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 467 | |
| 468 | overflow: |
| 469 | spin_unlock(&vmap_area_lock); |
| 470 | if (!purged) { |
| 471 | purge_vmap_area_lazy(); |
| 472 | purged = 1; |
| 473 | goto retry; |
| 474 | } |
Chris Wilson | 4da56b9 | 2016-04-04 14:46:42 +0100 | [diff] [blame] | 475 | |
| 476 | if (gfpflags_allow_blocking(gfp_mask)) { |
| 477 | unsigned long freed = 0; |
| 478 | blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); |
| 479 | if (freed > 0) { |
| 480 | purged = 0; |
| 481 | goto retry; |
| 482 | } |
| 483 | } |
| 484 | |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 485 | if (printk_ratelimit()) |
Joe Perches | 756a025 | 2016-03-17 14:19:47 -0700 | [diff] [blame] | 486 | pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", |
| 487 | size); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 488 | kfree(va); |
| 489 | return ERR_PTR(-EBUSY); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 490 | } |
| 491 | |
Chris Wilson | 4da56b9 | 2016-04-04 14:46:42 +0100 | [diff] [blame] | 492 | int register_vmap_purge_notifier(struct notifier_block *nb) |
| 493 | { |
| 494 | return blocking_notifier_chain_register(&vmap_notify_list, nb); |
| 495 | } |
| 496 | EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); |
| 497 | |
| 498 | int unregister_vmap_purge_notifier(struct notifier_block *nb) |
| 499 | { |
| 500 | return blocking_notifier_chain_unregister(&vmap_notify_list, nb); |
| 501 | } |
| 502 | EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); |
| 503 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 504 | static void __free_vmap_area(struct vmap_area *va) |
| 505 | { |
| 506 | BUG_ON(RB_EMPTY_NODE(&va->rb_node)); |
Nick Piggin | 8969960 | 2011-03-22 16:30:36 -0700 | [diff] [blame] | 507 | |
| 508 | if (free_vmap_cache) { |
| 509 | if (va->va_end < cached_vstart) { |
| 510 | free_vmap_cache = NULL; |
| 511 | } else { |
| 512 | struct vmap_area *cache; |
| 513 | cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node); |
| 514 | if (va->va_start <= cache->va_start) { |
| 515 | free_vmap_cache = rb_prev(&va->rb_node); |
| 516 | /* |
| 517 | * We don't try to update cached_hole_size or |
| 518 | * cached_align, but it won't go very wrong. |
| 519 | */ |
| 520 | } |
| 521 | } |
| 522 | } |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 523 | rb_erase(&va->rb_node, &vmap_area_root); |
| 524 | RB_CLEAR_NODE(&va->rb_node); |
| 525 | list_del_rcu(&va->list); |
| 526 | |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 527 | /* |
| 528 | * Track the highest possible candidate for pcpu area |
| 529 | * allocation. Areas outside of vmalloc area can be returned |
| 530 | * here too, consider only end addresses which fall inside |
| 531 | * vmalloc area proper. |
| 532 | */ |
| 533 | if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END) |
| 534 | vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end); |
| 535 | |
Lai Jiangshan | 14769de | 2011-03-18 12:12:19 +0800 | [diff] [blame] | 536 | kfree_rcu(va, rcu_head); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 537 | } |
| 538 | |
| 539 | /* |
| 540 | * Free a region of KVA allocated by alloc_vmap_area |
| 541 | */ |
| 542 | static void free_vmap_area(struct vmap_area *va) |
| 543 | { |
| 544 | spin_lock(&vmap_area_lock); |
| 545 | __free_vmap_area(va); |
| 546 | spin_unlock(&vmap_area_lock); |
| 547 | } |
| 548 | |
| 549 | /* |
| 550 | * Clear the pagetable entries of a given vmap_area |
| 551 | */ |
| 552 | static void unmap_vmap_area(struct vmap_area *va) |
| 553 | { |
| 554 | vunmap_page_range(va->va_start, va->va_end); |
| 555 | } |
| 556 | |
Nick Piggin | cd52858 | 2009-01-06 14:39:20 -0800 | [diff] [blame] | 557 | static void vmap_debug_free_range(unsigned long start, unsigned long end) |
| 558 | { |
| 559 | /* |
Joonsoo Kim | f48d97f | 2016-03-17 14:17:49 -0700 | [diff] [blame] | 560 | * Unmap page tables and force a TLB flush immediately if pagealloc |
| 561 | * debugging is enabled. This catches use after free bugs similarly to |
| 562 | * those in linear kernel virtual address space after a page has been |
| 563 | * freed. |
Nick Piggin | cd52858 | 2009-01-06 14:39:20 -0800 | [diff] [blame] | 564 | * |
Joonsoo Kim | f48d97f | 2016-03-17 14:17:49 -0700 | [diff] [blame] | 565 | * All the lazy freeing logic is still retained, in order to minimise |
| 566 | * intrusiveness of this debugging feature. |
Nick Piggin | cd52858 | 2009-01-06 14:39:20 -0800 | [diff] [blame] | 567 | * |
Joonsoo Kim | f48d97f | 2016-03-17 14:17:49 -0700 | [diff] [blame] | 568 | * This is going to be *slow* (linear kernel virtual address debugging |
| 569 | * doesn't do a broadcast TLB flush so it is a lot faster). |
Nick Piggin | cd52858 | 2009-01-06 14:39:20 -0800 | [diff] [blame] | 570 | */ |
Joonsoo Kim | f48d97f | 2016-03-17 14:17:49 -0700 | [diff] [blame] | 571 | if (debug_pagealloc_enabled()) { |
| 572 | vunmap_page_range(start, end); |
| 573 | flush_tlb_kernel_range(start, end); |
| 574 | } |
Nick Piggin | cd52858 | 2009-01-06 14:39:20 -0800 | [diff] [blame] | 575 | } |
| 576 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 577 | /* |
| 578 | * lazy_max_pages is the maximum amount of virtual address space we gather up |
| 579 | * before attempting to purge with a TLB flush. |
| 580 | * |
| 581 | * There is a tradeoff here: a larger number will cover more kernel page tables |
| 582 | * and take slightly longer to purge, but it will linearly reduce the number of |
| 583 | * global TLB flushes that must be performed. It would seem natural to scale |
| 584 | * this number up linearly with the number of CPUs (because vmapping activity |
| 585 | * could also scale linearly with the number of CPUs), however it is likely |
| 586 | * that in practice, workloads might be constrained in other ways that mean |
| 587 | * vmap activity will not scale linearly with CPUs. Also, I want to be |
| 588 | * conservative and not introduce a big latency on huge systems, so go with |
| 589 | * a less aggressive log scale. It will still be an improvement over the old |
| 590 | * code, and it will be simple to change the scale factor if we find that it |
| 591 | * becomes a problem on bigger systems. |
| 592 | */ |
| 593 | static unsigned long lazy_max_pages(void) |
| 594 | { |
| 595 | unsigned int log; |
| 596 | |
| 597 | log = fls(num_online_cpus()); |
| 598 | |
| 599 | return log * (32UL * 1024 * 1024 / PAGE_SIZE); |
| 600 | } |
| 601 | |
| 602 | static atomic_t vmap_lazy_nr = ATOMIC_INIT(0); |
| 603 | |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 604 | /* for per-CPU blocks */ |
| 605 | static void purge_fragmented_blocks_allcpus(void); |
| 606 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 607 | /* |
Cliff Wickman | 3ee48b6 | 2010-09-16 11:44:02 -0500 | [diff] [blame] | 608 | * called before a call to iounmap() if the caller wants vm_area_struct's |
| 609 | * immediately freed. |
| 610 | */ |
| 611 | void set_iounmap_nonlazy(void) |
| 612 | { |
| 613 | atomic_set(&vmap_lazy_nr, lazy_max_pages()+1); |
| 614 | } |
| 615 | |
| 616 | /* |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 617 | * Purges all lazily-freed vmap areas. |
| 618 | * |
| 619 | * If sync is 0 then don't purge if there is already a purge in progress. |
| 620 | * If force_flush is 1, then flush kernel TLBs between *start and *end even |
| 621 | * if we found no lazy vmap areas to unmap (callers can use this to optimise |
| 622 | * their own TLB flushing). |
| 623 | * Returns with *start = min(*start, lowest purged address) |
| 624 | * *end = max(*end, highest purged address) |
| 625 | */ |
| 626 | static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, |
| 627 | int sync, int force_flush) |
| 628 | { |
Andrew Morton | 46666d8 | 2009-01-15 13:51:15 -0800 | [diff] [blame] | 629 | static DEFINE_SPINLOCK(purge_lock); |
Chris Wilson | 80c4bd7 | 2016-05-20 16:57:38 -0700 | [diff] [blame] | 630 | struct llist_node *valist; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 631 | struct vmap_area *va; |
Vegard Nossum | cbb7667 | 2009-02-27 14:03:04 -0800 | [diff] [blame] | 632 | struct vmap_area *n_va; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 633 | int nr = 0; |
| 634 | |
| 635 | /* |
| 636 | * If sync is 0 but force_flush is 1, we'll go sync anyway but callers |
| 637 | * should not expect such behaviour. This just simplifies locking for |
| 638 | * the case that isn't actually used at the moment anyway. |
| 639 | */ |
| 640 | if (!sync && !force_flush) { |
Andrew Morton | 46666d8 | 2009-01-15 13:51:15 -0800 | [diff] [blame] | 641 | if (!spin_trylock(&purge_lock)) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 642 | return; |
| 643 | } else |
Andrew Morton | 46666d8 | 2009-01-15 13:51:15 -0800 | [diff] [blame] | 644 | spin_lock(&purge_lock); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 645 | |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 646 | if (sync) |
| 647 | purge_fragmented_blocks_allcpus(); |
| 648 | |
Chris Wilson | 80c4bd7 | 2016-05-20 16:57:38 -0700 | [diff] [blame] | 649 | valist = llist_del_all(&vmap_purge_list); |
| 650 | llist_for_each_entry(va, valist, purge_list) { |
| 651 | if (va->va_start < *start) |
| 652 | *start = va->va_start; |
| 653 | if (va->va_end > *end) |
| 654 | *end = va->va_end; |
| 655 | nr += (va->va_end - va->va_start) >> PAGE_SHIFT; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 656 | } |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 657 | |
Yongseok Koh | 88f5004 | 2010-01-19 17:33:49 +0900 | [diff] [blame] | 658 | if (nr) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 659 | atomic_sub(nr, &vmap_lazy_nr); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 660 | |
| 661 | if (nr || force_flush) |
| 662 | flush_tlb_kernel_range(*start, *end); |
| 663 | |
| 664 | if (nr) { |
| 665 | spin_lock(&vmap_area_lock); |
Chris Wilson | 80c4bd7 | 2016-05-20 16:57:38 -0700 | [diff] [blame] | 666 | llist_for_each_entry_safe(va, n_va, valist, purge_list) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 667 | __free_vmap_area(va); |
| 668 | spin_unlock(&vmap_area_lock); |
| 669 | } |
Andrew Morton | 46666d8 | 2009-01-15 13:51:15 -0800 | [diff] [blame] | 670 | spin_unlock(&purge_lock); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 671 | } |
| 672 | |
| 673 | /* |
Nick Piggin | 496850e | 2008-11-19 15:36:33 -0800 | [diff] [blame] | 674 | * Kick off a purge of the outstanding lazy areas. Don't bother if somebody |
| 675 | * is already purging. |
| 676 | */ |
| 677 | static void try_purge_vmap_area_lazy(void) |
| 678 | { |
| 679 | unsigned long start = ULONG_MAX, end = 0; |
| 680 | |
| 681 | __purge_vmap_area_lazy(&start, &end, 0, 0); |
| 682 | } |
| 683 | |
| 684 | /* |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 685 | * Kick off a purge of the outstanding lazy areas. |
| 686 | */ |
| 687 | static void purge_vmap_area_lazy(void) |
| 688 | { |
| 689 | unsigned long start = ULONG_MAX, end = 0; |
| 690 | |
Nick Piggin | 496850e | 2008-11-19 15:36:33 -0800 | [diff] [blame] | 691 | __purge_vmap_area_lazy(&start, &end, 1, 0); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 692 | } |
| 693 | |
| 694 | /* |
Jeremy Fitzhardinge | 64141da | 2010-12-02 14:31:18 -0800 | [diff] [blame] | 695 | * Free a vmap area, caller ensuring that the area has been unmapped |
| 696 | * and flush_cache_vunmap had been called for the correct range |
| 697 | * previously. |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 698 | */ |
Jeremy Fitzhardinge | 64141da | 2010-12-02 14:31:18 -0800 | [diff] [blame] | 699 | static void free_vmap_area_noflush(struct vmap_area *va) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 700 | { |
Chris Wilson | 80c4bd7 | 2016-05-20 16:57:38 -0700 | [diff] [blame] | 701 | int nr_lazy; |
| 702 | |
| 703 | nr_lazy = atomic_add_return((va->va_end - va->va_start) >> PAGE_SHIFT, |
| 704 | &vmap_lazy_nr); |
| 705 | |
| 706 | /* After this point, we may free va at any time */ |
| 707 | llist_add(&va->purge_list, &vmap_purge_list); |
| 708 | |
| 709 | if (unlikely(nr_lazy > lazy_max_pages())) |
Nick Piggin | 496850e | 2008-11-19 15:36:33 -0800 | [diff] [blame] | 710 | try_purge_vmap_area_lazy(); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 711 | } |
| 712 | |
Nick Piggin | b29acbd | 2008-12-01 13:13:47 -0800 | [diff] [blame] | 713 | /* |
Jeremy Fitzhardinge | 64141da | 2010-12-02 14:31:18 -0800 | [diff] [blame] | 714 | * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been |
| 715 | * called for the correct range previously. |
| 716 | */ |
| 717 | static void free_unmap_vmap_area_noflush(struct vmap_area *va) |
| 718 | { |
| 719 | unmap_vmap_area(va); |
| 720 | free_vmap_area_noflush(va); |
| 721 | } |
| 722 | |
| 723 | /* |
Nick Piggin | b29acbd | 2008-12-01 13:13:47 -0800 | [diff] [blame] | 724 | * Free and unmap a vmap area |
| 725 | */ |
| 726 | static void free_unmap_vmap_area(struct vmap_area *va) |
| 727 | { |
| 728 | flush_cache_vunmap(va->va_start, va->va_end); |
| 729 | free_unmap_vmap_area_noflush(va); |
| 730 | } |
| 731 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 732 | static struct vmap_area *find_vmap_area(unsigned long addr) |
| 733 | { |
| 734 | struct vmap_area *va; |
| 735 | |
| 736 | spin_lock(&vmap_area_lock); |
| 737 | va = __find_vmap_area(addr); |
| 738 | spin_unlock(&vmap_area_lock); |
| 739 | |
| 740 | return va; |
| 741 | } |
| 742 | |
| 743 | static void free_unmap_vmap_area_addr(unsigned long addr) |
| 744 | { |
| 745 | struct vmap_area *va; |
| 746 | |
| 747 | va = find_vmap_area(addr); |
| 748 | BUG_ON(!va); |
| 749 | free_unmap_vmap_area(va); |
| 750 | } |
| 751 | |
| 752 | |
| 753 | /*** Per cpu kva allocator ***/ |
| 754 | |
| 755 | /* |
| 756 | * vmap space is limited especially on 32 bit architectures. Ensure there is |
| 757 | * room for at least 16 percpu vmap blocks per CPU. |
| 758 | */ |
| 759 | /* |
| 760 | * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able |
| 761 | * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess |
| 762 | * instead (we just need a rough idea) |
| 763 | */ |
| 764 | #if BITS_PER_LONG == 32 |
| 765 | #define VMALLOC_SPACE (128UL*1024*1024) |
| 766 | #else |
| 767 | #define VMALLOC_SPACE (128UL*1024*1024*1024) |
| 768 | #endif |
| 769 | |
| 770 | #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) |
| 771 | #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ |
| 772 | #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ |
| 773 | #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) |
| 774 | #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ |
| 775 | #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ |
Clemens Ladisch | f982f915 | 2011-06-21 22:09:50 +0200 | [diff] [blame] | 776 | #define VMAP_BBMAP_BITS \ |
| 777 | VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ |
| 778 | VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ |
| 779 | VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 780 | |
| 781 | #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) |
| 782 | |
Jeremy Fitzhardinge | 9b46333 | 2008-10-28 19:22:34 +1100 | [diff] [blame] | 783 | static bool vmap_initialized __read_mostly = false; |
| 784 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 785 | struct vmap_block_queue { |
| 786 | spinlock_t lock; |
| 787 | struct list_head free; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 788 | }; |
| 789 | |
| 790 | struct vmap_block { |
| 791 | spinlock_t lock; |
| 792 | struct vmap_area *va; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 793 | unsigned long free, dirty; |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 794 | unsigned long dirty_min, dirty_max; /*< dirty range */ |
Nick Piggin | de56042 | 2010-02-01 22:24:18 +1100 | [diff] [blame] | 795 | struct list_head free_list; |
| 796 | struct rcu_head rcu_head; |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 797 | struct list_head purge; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 798 | }; |
| 799 | |
| 800 | /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ |
| 801 | static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); |
| 802 | |
| 803 | /* |
| 804 | * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block |
| 805 | * in the free path. Could get rid of this if we change the API to return a |
| 806 | * "cookie" from alloc, to be passed to free. But no big deal yet. |
| 807 | */ |
| 808 | static DEFINE_SPINLOCK(vmap_block_tree_lock); |
| 809 | static RADIX_TREE(vmap_block_tree, GFP_ATOMIC); |
| 810 | |
| 811 | /* |
| 812 | * We should probably have a fallback mechanism to allocate virtual memory |
| 813 | * out of partially filled vmap blocks. However vmap block sizing should be |
| 814 | * fairly reasonable according to the vmalloc size, so it shouldn't be a |
| 815 | * big problem. |
| 816 | */ |
| 817 | |
| 818 | static unsigned long addr_to_vb_idx(unsigned long addr) |
| 819 | { |
| 820 | addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); |
| 821 | addr /= VMAP_BLOCK_SIZE; |
| 822 | return addr; |
| 823 | } |
| 824 | |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 825 | static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) |
| 826 | { |
| 827 | unsigned long addr; |
| 828 | |
| 829 | addr = va_start + (pages_off << PAGE_SHIFT); |
| 830 | BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); |
| 831 | return (void *)addr; |
| 832 | } |
| 833 | |
| 834 | /** |
| 835 | * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this |
| 836 | * block. Of course pages number can't exceed VMAP_BBMAP_BITS |
| 837 | * @order: how many 2^order pages should be occupied in newly allocated block |
| 838 | * @gfp_mask: flags for the page level allocator |
| 839 | * |
| 840 | * Returns: virtual address in a newly allocated block or ERR_PTR(-errno) |
| 841 | */ |
| 842 | static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 843 | { |
| 844 | struct vmap_block_queue *vbq; |
| 845 | struct vmap_block *vb; |
| 846 | struct vmap_area *va; |
| 847 | unsigned long vb_idx; |
| 848 | int node, err; |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 849 | void *vaddr; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 850 | |
| 851 | node = numa_node_id(); |
| 852 | |
| 853 | vb = kmalloc_node(sizeof(struct vmap_block), |
| 854 | gfp_mask & GFP_RECLAIM_MASK, node); |
| 855 | if (unlikely(!vb)) |
| 856 | return ERR_PTR(-ENOMEM); |
| 857 | |
| 858 | va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, |
| 859 | VMALLOC_START, VMALLOC_END, |
| 860 | node, gfp_mask); |
Tobias Klauser | ddf9c6d4 | 2011-01-13 15:46:15 -0800 | [diff] [blame] | 861 | if (IS_ERR(va)) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 862 | kfree(vb); |
Julia Lawall | e7d8634 | 2010-08-09 17:18:28 -0700 | [diff] [blame] | 863 | return ERR_CAST(va); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 864 | } |
| 865 | |
| 866 | err = radix_tree_preload(gfp_mask); |
| 867 | if (unlikely(err)) { |
| 868 | kfree(vb); |
| 869 | free_vmap_area(va); |
| 870 | return ERR_PTR(err); |
| 871 | } |
| 872 | |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 873 | vaddr = vmap_block_vaddr(va->va_start, 0); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 874 | spin_lock_init(&vb->lock); |
| 875 | vb->va = va; |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 876 | /* At least something should be left free */ |
| 877 | BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); |
| 878 | vb->free = VMAP_BBMAP_BITS - (1UL << order); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 879 | vb->dirty = 0; |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 880 | vb->dirty_min = VMAP_BBMAP_BITS; |
| 881 | vb->dirty_max = 0; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 882 | INIT_LIST_HEAD(&vb->free_list); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 883 | |
| 884 | vb_idx = addr_to_vb_idx(va->va_start); |
| 885 | spin_lock(&vmap_block_tree_lock); |
| 886 | err = radix_tree_insert(&vmap_block_tree, vb_idx, vb); |
| 887 | spin_unlock(&vmap_block_tree_lock); |
| 888 | BUG_ON(err); |
| 889 | radix_tree_preload_end(); |
| 890 | |
| 891 | vbq = &get_cpu_var(vmap_block_queue); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 892 | spin_lock(&vbq->lock); |
Roman Pen | 68ac546 | 2015-04-15 16:13:48 -0700 | [diff] [blame] | 893 | list_add_tail_rcu(&vb->free_list, &vbq->free); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 894 | spin_unlock(&vbq->lock); |
Tejun Heo | 3f04ba8 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 895 | put_cpu_var(vmap_block_queue); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 896 | |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 897 | return vaddr; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 898 | } |
| 899 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 900 | static void free_vmap_block(struct vmap_block *vb) |
| 901 | { |
| 902 | struct vmap_block *tmp; |
| 903 | unsigned long vb_idx; |
| 904 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 905 | vb_idx = addr_to_vb_idx(vb->va->va_start); |
| 906 | spin_lock(&vmap_block_tree_lock); |
| 907 | tmp = radix_tree_delete(&vmap_block_tree, vb_idx); |
| 908 | spin_unlock(&vmap_block_tree_lock); |
| 909 | BUG_ON(tmp != vb); |
| 910 | |
Jeremy Fitzhardinge | 64141da | 2010-12-02 14:31:18 -0800 | [diff] [blame] | 911 | free_vmap_area_noflush(vb->va); |
Lai Jiangshan | 22a3c7d | 2011-03-18 12:13:08 +0800 | [diff] [blame] | 912 | kfree_rcu(vb, rcu_head); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 913 | } |
| 914 | |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 915 | static void purge_fragmented_blocks(int cpu) |
| 916 | { |
| 917 | LIST_HEAD(purge); |
| 918 | struct vmap_block *vb; |
| 919 | struct vmap_block *n_vb; |
| 920 | struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); |
| 921 | |
| 922 | rcu_read_lock(); |
| 923 | list_for_each_entry_rcu(vb, &vbq->free, free_list) { |
| 924 | |
| 925 | if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS)) |
| 926 | continue; |
| 927 | |
| 928 | spin_lock(&vb->lock); |
| 929 | if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) { |
| 930 | vb->free = 0; /* prevent further allocs after releasing lock */ |
| 931 | vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */ |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 932 | vb->dirty_min = 0; |
| 933 | vb->dirty_max = VMAP_BBMAP_BITS; |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 934 | spin_lock(&vbq->lock); |
| 935 | list_del_rcu(&vb->free_list); |
| 936 | spin_unlock(&vbq->lock); |
| 937 | spin_unlock(&vb->lock); |
| 938 | list_add_tail(&vb->purge, &purge); |
| 939 | } else |
| 940 | spin_unlock(&vb->lock); |
| 941 | } |
| 942 | rcu_read_unlock(); |
| 943 | |
| 944 | list_for_each_entry_safe(vb, n_vb, &purge, purge) { |
| 945 | list_del(&vb->purge); |
| 946 | free_vmap_block(vb); |
| 947 | } |
| 948 | } |
| 949 | |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 950 | static void purge_fragmented_blocks_allcpus(void) |
| 951 | { |
| 952 | int cpu; |
| 953 | |
| 954 | for_each_possible_cpu(cpu) |
| 955 | purge_fragmented_blocks(cpu); |
| 956 | } |
| 957 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 958 | static void *vb_alloc(unsigned long size, gfp_t gfp_mask) |
| 959 | { |
| 960 | struct vmap_block_queue *vbq; |
| 961 | struct vmap_block *vb; |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 962 | void *vaddr = NULL; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 963 | unsigned int order; |
| 964 | |
Alexander Kuleshov | 891c49a | 2015-11-05 18:46:51 -0800 | [diff] [blame] | 965 | BUG_ON(offset_in_page(size)); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 966 | BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); |
Jan Kara | aa91c4d | 2012-07-31 16:41:37 -0700 | [diff] [blame] | 967 | if (WARN_ON(size == 0)) { |
| 968 | /* |
| 969 | * Allocating 0 bytes isn't what caller wants since |
| 970 | * get_order(0) returns funny result. Just warn and terminate |
| 971 | * early. |
| 972 | */ |
| 973 | return NULL; |
| 974 | } |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 975 | order = get_order(size); |
| 976 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 977 | rcu_read_lock(); |
| 978 | vbq = &get_cpu_var(vmap_block_queue); |
| 979 | list_for_each_entry_rcu(vb, &vbq->free, free_list) { |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 980 | unsigned long pages_off; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 981 | |
| 982 | spin_lock(&vb->lock); |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 983 | if (vb->free < (1UL << order)) { |
| 984 | spin_unlock(&vb->lock); |
| 985 | continue; |
| 986 | } |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 987 | |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 988 | pages_off = VMAP_BBMAP_BITS - vb->free; |
| 989 | vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 990 | vb->free -= 1UL << order; |
| 991 | if (vb->free == 0) { |
| 992 | spin_lock(&vbq->lock); |
| 993 | list_del_rcu(&vb->free_list); |
| 994 | spin_unlock(&vbq->lock); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 995 | } |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 996 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 997 | spin_unlock(&vb->lock); |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 998 | break; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 999 | } |
Nick Piggin | 02b709d | 2010-02-01 22:25:57 +1100 | [diff] [blame] | 1000 | |
Tejun Heo | 3f04ba8 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 1001 | put_cpu_var(vmap_block_queue); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1002 | rcu_read_unlock(); |
| 1003 | |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 1004 | /* Allocate new block if nothing was found */ |
| 1005 | if (!vaddr) |
| 1006 | vaddr = new_vmap_block(order, gfp_mask); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1007 | |
Roman Pen | cf725ce | 2015-04-15 16:13:52 -0700 | [diff] [blame] | 1008 | return vaddr; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1009 | } |
| 1010 | |
| 1011 | static void vb_free(const void *addr, unsigned long size) |
| 1012 | { |
| 1013 | unsigned long offset; |
| 1014 | unsigned long vb_idx; |
| 1015 | unsigned int order; |
| 1016 | struct vmap_block *vb; |
| 1017 | |
Alexander Kuleshov | 891c49a | 2015-11-05 18:46:51 -0800 | [diff] [blame] | 1018 | BUG_ON(offset_in_page(size)); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1019 | BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); |
Nick Piggin | b29acbd | 2008-12-01 13:13:47 -0800 | [diff] [blame] | 1020 | |
| 1021 | flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size); |
| 1022 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1023 | order = get_order(size); |
| 1024 | |
| 1025 | offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1); |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 1026 | offset >>= PAGE_SHIFT; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1027 | |
| 1028 | vb_idx = addr_to_vb_idx((unsigned long)addr); |
| 1029 | rcu_read_lock(); |
| 1030 | vb = radix_tree_lookup(&vmap_block_tree, vb_idx); |
| 1031 | rcu_read_unlock(); |
| 1032 | BUG_ON(!vb); |
| 1033 | |
Jeremy Fitzhardinge | 64141da | 2010-12-02 14:31:18 -0800 | [diff] [blame] | 1034 | vunmap_page_range((unsigned long)addr, (unsigned long)addr + size); |
| 1035 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1036 | spin_lock(&vb->lock); |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 1037 | |
| 1038 | /* Expand dirty range */ |
| 1039 | vb->dirty_min = min(vb->dirty_min, offset); |
| 1040 | vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); |
MinChan Kim | d086817 | 2009-03-31 15:19:26 -0700 | [diff] [blame] | 1041 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1042 | vb->dirty += 1UL << order; |
| 1043 | if (vb->dirty == VMAP_BBMAP_BITS) { |
Nick Piggin | de56042 | 2010-02-01 22:24:18 +1100 | [diff] [blame] | 1044 | BUG_ON(vb->free); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1045 | spin_unlock(&vb->lock); |
| 1046 | free_vmap_block(vb); |
| 1047 | } else |
| 1048 | spin_unlock(&vb->lock); |
| 1049 | } |
| 1050 | |
| 1051 | /** |
| 1052 | * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer |
| 1053 | * |
| 1054 | * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily |
| 1055 | * to amortize TLB flushing overheads. What this means is that any page you |
| 1056 | * have now, may, in a former life, have been mapped into kernel virtual |
| 1057 | * address by the vmap layer and so there might be some CPUs with TLB entries |
| 1058 | * still referencing that page (additional to the regular 1:1 kernel mapping). |
| 1059 | * |
| 1060 | * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can |
| 1061 | * be sure that none of the pages we have control over will have any aliases |
| 1062 | * from the vmap layer. |
| 1063 | */ |
| 1064 | void vm_unmap_aliases(void) |
| 1065 | { |
| 1066 | unsigned long start = ULONG_MAX, end = 0; |
| 1067 | int cpu; |
| 1068 | int flush = 0; |
| 1069 | |
Jeremy Fitzhardinge | 9b46333 | 2008-10-28 19:22:34 +1100 | [diff] [blame] | 1070 | if (unlikely(!vmap_initialized)) |
| 1071 | return; |
| 1072 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1073 | for_each_possible_cpu(cpu) { |
| 1074 | struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); |
| 1075 | struct vmap_block *vb; |
| 1076 | |
| 1077 | rcu_read_lock(); |
| 1078 | list_for_each_entry_rcu(vb, &vbq->free, free_list) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1079 | spin_lock(&vb->lock); |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 1080 | if (vb->dirty) { |
| 1081 | unsigned long va_start = vb->va->va_start; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1082 | unsigned long s, e; |
Joonsoo Kim | b136be5e | 2013-09-11 14:21:40 -0700 | [diff] [blame] | 1083 | |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 1084 | s = va_start + (vb->dirty_min << PAGE_SHIFT); |
| 1085 | e = va_start + (vb->dirty_max << PAGE_SHIFT); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1086 | |
Roman Pen | 7d61bfe8 | 2015-04-15 16:13:55 -0700 | [diff] [blame] | 1087 | start = min(s, start); |
| 1088 | end = max(e, end); |
| 1089 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1090 | flush = 1; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1091 | } |
| 1092 | spin_unlock(&vb->lock); |
| 1093 | } |
| 1094 | rcu_read_unlock(); |
| 1095 | } |
| 1096 | |
| 1097 | __purge_vmap_area_lazy(&start, &end, 1, flush); |
| 1098 | } |
| 1099 | EXPORT_SYMBOL_GPL(vm_unmap_aliases); |
| 1100 | |
| 1101 | /** |
| 1102 | * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram |
| 1103 | * @mem: the pointer returned by vm_map_ram |
| 1104 | * @count: the count passed to that vm_map_ram call (cannot unmap partial) |
| 1105 | */ |
| 1106 | void vm_unmap_ram(const void *mem, unsigned int count) |
| 1107 | { |
Guillermo Julián Moreno | 65ee03c | 2016-06-03 14:55:33 -0700 | [diff] [blame] | 1108 | unsigned long size = (unsigned long)count << PAGE_SHIFT; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1109 | unsigned long addr = (unsigned long)mem; |
| 1110 | |
| 1111 | BUG_ON(!addr); |
| 1112 | BUG_ON(addr < VMALLOC_START); |
| 1113 | BUG_ON(addr > VMALLOC_END); |
Shawn Lin | a1c0b1a | 2016-03-17 14:20:37 -0700 | [diff] [blame] | 1114 | BUG_ON(!PAGE_ALIGNED(addr)); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1115 | |
| 1116 | debug_check_no_locks_freed(mem, size); |
Nick Piggin | cd52858 | 2009-01-06 14:39:20 -0800 | [diff] [blame] | 1117 | vmap_debug_free_range(addr, addr+size); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1118 | |
| 1119 | if (likely(count <= VMAP_MAX_ALLOC)) |
| 1120 | vb_free(mem, size); |
| 1121 | else |
| 1122 | free_unmap_vmap_area_addr(addr); |
| 1123 | } |
| 1124 | EXPORT_SYMBOL(vm_unmap_ram); |
| 1125 | |
| 1126 | /** |
| 1127 | * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) |
| 1128 | * @pages: an array of pointers to the pages to be mapped |
| 1129 | * @count: number of pages |
| 1130 | * @node: prefer to allocate data structures on this node |
| 1131 | * @prot: memory protection to use. PAGE_KERNEL for regular RAM |
Randy Dunlap | e99c97a | 2008-10-29 14:01:09 -0700 | [diff] [blame] | 1132 | * |
Gioh Kim | 3643763 | 2014-04-07 15:37:37 -0700 | [diff] [blame] | 1133 | * If you use this function for less than VMAP_MAX_ALLOC pages, it could be |
| 1134 | * faster than vmap so it's good. But if you mix long-life and short-life |
| 1135 | * objects with vm_map_ram(), it could consume lots of address space through |
| 1136 | * fragmentation (especially on a 32bit machine). You could see failures in |
| 1137 | * the end. Please use this function for short-lived objects. |
| 1138 | * |
Randy Dunlap | e99c97a | 2008-10-29 14:01:09 -0700 | [diff] [blame] | 1139 | * Returns: a pointer to the address that has been mapped, or %NULL on failure |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1140 | */ |
| 1141 | void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) |
| 1142 | { |
Guillermo Julián Moreno | 65ee03c | 2016-06-03 14:55:33 -0700 | [diff] [blame] | 1143 | unsigned long size = (unsigned long)count << PAGE_SHIFT; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1144 | unsigned long addr; |
| 1145 | void *mem; |
| 1146 | |
| 1147 | if (likely(count <= VMAP_MAX_ALLOC)) { |
| 1148 | mem = vb_alloc(size, GFP_KERNEL); |
| 1149 | if (IS_ERR(mem)) |
| 1150 | return NULL; |
| 1151 | addr = (unsigned long)mem; |
| 1152 | } else { |
| 1153 | struct vmap_area *va; |
| 1154 | va = alloc_vmap_area(size, PAGE_SIZE, |
| 1155 | VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); |
| 1156 | if (IS_ERR(va)) |
| 1157 | return NULL; |
| 1158 | |
| 1159 | addr = va->va_start; |
| 1160 | mem = (void *)addr; |
| 1161 | } |
| 1162 | if (vmap_page_range(addr, addr + size, prot, pages) < 0) { |
| 1163 | vm_unmap_ram(mem, count); |
| 1164 | return NULL; |
| 1165 | } |
| 1166 | return mem; |
| 1167 | } |
| 1168 | EXPORT_SYMBOL(vm_map_ram); |
| 1169 | |
Joonsoo Kim | 4341fa4 | 2013-04-29 15:07:39 -0700 | [diff] [blame] | 1170 | static struct vm_struct *vmlist __initdata; |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1171 | /** |
Nicolas Pitre | be9b733 | 2011-08-25 00:24:21 -0400 | [diff] [blame] | 1172 | * vm_area_add_early - add vmap area early during boot |
| 1173 | * @vm: vm_struct to add |
| 1174 | * |
| 1175 | * This function is used to add fixed kernel vm area to vmlist before |
| 1176 | * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags |
| 1177 | * should contain proper values and the other fields should be zero. |
| 1178 | * |
| 1179 | * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. |
| 1180 | */ |
| 1181 | void __init vm_area_add_early(struct vm_struct *vm) |
| 1182 | { |
| 1183 | struct vm_struct *tmp, **p; |
| 1184 | |
| 1185 | BUG_ON(vmap_initialized); |
| 1186 | for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { |
| 1187 | if (tmp->addr >= vm->addr) { |
| 1188 | BUG_ON(tmp->addr < vm->addr + vm->size); |
| 1189 | break; |
| 1190 | } else |
| 1191 | BUG_ON(tmp->addr + tmp->size > vm->addr); |
| 1192 | } |
| 1193 | vm->next = *p; |
| 1194 | *p = vm; |
| 1195 | } |
| 1196 | |
| 1197 | /** |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1198 | * vm_area_register_early - register vmap area early during boot |
| 1199 | * @vm: vm_struct to register |
Tejun Heo | c0c0a29 | 2009-02-24 11:57:21 +0900 | [diff] [blame] | 1200 | * @align: requested alignment |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1201 | * |
| 1202 | * This function is used to register kernel vm area before |
| 1203 | * vmalloc_init() is called. @vm->size and @vm->flags should contain |
| 1204 | * proper values on entry and other fields should be zero. On return, |
| 1205 | * vm->addr contains the allocated address. |
| 1206 | * |
| 1207 | * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. |
| 1208 | */ |
Tejun Heo | c0c0a29 | 2009-02-24 11:57:21 +0900 | [diff] [blame] | 1209 | void __init vm_area_register_early(struct vm_struct *vm, size_t align) |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1210 | { |
| 1211 | static size_t vm_init_off __initdata; |
Tejun Heo | c0c0a29 | 2009-02-24 11:57:21 +0900 | [diff] [blame] | 1212 | unsigned long addr; |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1213 | |
Tejun Heo | c0c0a29 | 2009-02-24 11:57:21 +0900 | [diff] [blame] | 1214 | addr = ALIGN(VMALLOC_START + vm_init_off, align); |
| 1215 | vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START; |
| 1216 | |
| 1217 | vm->addr = (void *)addr; |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1218 | |
Nicolas Pitre | be9b733 | 2011-08-25 00:24:21 -0400 | [diff] [blame] | 1219 | vm_area_add_early(vm); |
Tejun Heo | f0aa661 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1220 | } |
| 1221 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1222 | void __init vmalloc_init(void) |
| 1223 | { |
Ivan Kokshaysky | 822c18f | 2009-01-15 13:50:48 -0800 | [diff] [blame] | 1224 | struct vmap_area *va; |
| 1225 | struct vm_struct *tmp; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1226 | int i; |
| 1227 | |
| 1228 | for_each_possible_cpu(i) { |
| 1229 | struct vmap_block_queue *vbq; |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1230 | struct vfree_deferred *p; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1231 | |
| 1232 | vbq = &per_cpu(vmap_block_queue, i); |
| 1233 | spin_lock_init(&vbq->lock); |
| 1234 | INIT_LIST_HEAD(&vbq->free); |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1235 | p = &per_cpu(vfree_deferred, i); |
| 1236 | init_llist_head(&p->list); |
| 1237 | INIT_WORK(&p->wq, free_work); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1238 | } |
Jeremy Fitzhardinge | 9b46333 | 2008-10-28 19:22:34 +1100 | [diff] [blame] | 1239 | |
Ivan Kokshaysky | 822c18f | 2009-01-15 13:50:48 -0800 | [diff] [blame] | 1240 | /* Import existing vmlist entries. */ |
| 1241 | for (tmp = vmlist; tmp; tmp = tmp->next) { |
Pekka Enberg | 43ebdac | 2009-05-25 15:01:35 +0300 | [diff] [blame] | 1242 | va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT); |
KyongHo | dbda591 | 2012-05-29 15:06:49 -0700 | [diff] [blame] | 1243 | va->flags = VM_VM_AREA; |
Ivan Kokshaysky | 822c18f | 2009-01-15 13:50:48 -0800 | [diff] [blame] | 1244 | va->va_start = (unsigned long)tmp->addr; |
| 1245 | va->va_end = va->va_start + tmp->size; |
KyongHo | dbda591 | 2012-05-29 15:06:49 -0700 | [diff] [blame] | 1246 | va->vm = tmp; |
Ivan Kokshaysky | 822c18f | 2009-01-15 13:50:48 -0800 | [diff] [blame] | 1247 | __insert_vmap_area(va); |
| 1248 | } |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 1249 | |
| 1250 | vmap_area_pcpu_hole = VMALLOC_END; |
| 1251 | |
Jeremy Fitzhardinge | 9b46333 | 2008-10-28 19:22:34 +1100 | [diff] [blame] | 1252 | vmap_initialized = true; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1253 | } |
| 1254 | |
Tejun Heo | 8fc4898 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1255 | /** |
| 1256 | * map_kernel_range_noflush - map kernel VM area with the specified pages |
| 1257 | * @addr: start of the VM area to map |
| 1258 | * @size: size of the VM area to map |
| 1259 | * @prot: page protection flags to use |
| 1260 | * @pages: pages to map |
| 1261 | * |
| 1262 | * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size |
| 1263 | * specify should have been allocated using get_vm_area() and its |
| 1264 | * friends. |
| 1265 | * |
| 1266 | * NOTE: |
| 1267 | * This function does NOT do any cache flushing. The caller is |
| 1268 | * responsible for calling flush_cache_vmap() on to-be-mapped areas |
| 1269 | * before calling this function. |
| 1270 | * |
| 1271 | * RETURNS: |
| 1272 | * The number of pages mapped on success, -errno on failure. |
| 1273 | */ |
| 1274 | int map_kernel_range_noflush(unsigned long addr, unsigned long size, |
| 1275 | pgprot_t prot, struct page **pages) |
| 1276 | { |
| 1277 | return vmap_page_range_noflush(addr, addr + size, prot, pages); |
| 1278 | } |
| 1279 | |
| 1280 | /** |
| 1281 | * unmap_kernel_range_noflush - unmap kernel VM area |
| 1282 | * @addr: start of the VM area to unmap |
| 1283 | * @size: size of the VM area to unmap |
| 1284 | * |
| 1285 | * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size |
| 1286 | * specify should have been allocated using get_vm_area() and its |
| 1287 | * friends. |
| 1288 | * |
| 1289 | * NOTE: |
| 1290 | * This function does NOT do any cache flushing. The caller is |
| 1291 | * responsible for calling flush_cache_vunmap() on to-be-mapped areas |
| 1292 | * before calling this function and flush_tlb_kernel_range() after. |
| 1293 | */ |
| 1294 | void unmap_kernel_range_noflush(unsigned long addr, unsigned long size) |
| 1295 | { |
| 1296 | vunmap_page_range(addr, addr + size); |
| 1297 | } |
Huang Ying | 81e88fd | 2011-01-12 14:44:55 +0800 | [diff] [blame] | 1298 | EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush); |
Tejun Heo | 8fc4898 | 2009-02-20 16:29:08 +0900 | [diff] [blame] | 1299 | |
| 1300 | /** |
| 1301 | * unmap_kernel_range - unmap kernel VM area and flush cache and TLB |
| 1302 | * @addr: start of the VM area to unmap |
| 1303 | * @size: size of the VM area to unmap |
| 1304 | * |
| 1305 | * Similar to unmap_kernel_range_noflush() but flushes vcache before |
| 1306 | * the unmapping and tlb after. |
| 1307 | */ |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1308 | void unmap_kernel_range(unsigned long addr, unsigned long size) |
| 1309 | { |
| 1310 | unsigned long end = addr + size; |
Tejun Heo | f6fcba7 | 2009-02-20 15:38:48 -0800 | [diff] [blame] | 1311 | |
| 1312 | flush_cache_vunmap(addr, end); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1313 | vunmap_page_range(addr, end); |
| 1314 | flush_tlb_kernel_range(addr, end); |
| 1315 | } |
Minchan Kim | 93ef6d6c | 2014-06-04 16:11:09 -0700 | [diff] [blame] | 1316 | EXPORT_SYMBOL_GPL(unmap_kernel_range); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1317 | |
WANG Chao | f6f8ed4 | 2014-08-06 16:06:58 -0700 | [diff] [blame] | 1318 | int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1319 | { |
| 1320 | unsigned long addr = (unsigned long)area->addr; |
Wanpeng Li | 762216a | 2013-09-11 14:22:42 -0700 | [diff] [blame] | 1321 | unsigned long end = addr + get_vm_area_size(area); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1322 | int err; |
| 1323 | |
WANG Chao | f6f8ed4 | 2014-08-06 16:06:58 -0700 | [diff] [blame] | 1324 | err = vmap_page_range(addr, end, prot, pages); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1325 | |
WANG Chao | f6f8ed4 | 2014-08-06 16:06:58 -0700 | [diff] [blame] | 1326 | return err > 0 ? 0 : err; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1327 | } |
| 1328 | EXPORT_SYMBOL_GPL(map_vm_area); |
| 1329 | |
Mitsuo Hayasaka | f5252e0 | 2011-10-31 17:08:13 -0700 | [diff] [blame] | 1330 | static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, |
Marek Szyprowski | 5e6cafc | 2012-04-13 12:32:09 +0200 | [diff] [blame] | 1331 | unsigned long flags, const void *caller) |
Tejun Heo | cf88c79 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 1332 | { |
Joonsoo Kim | c69480a | 2013-04-29 15:07:30 -0700 | [diff] [blame] | 1333 | spin_lock(&vmap_area_lock); |
Tejun Heo | cf88c79 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 1334 | vm->flags = flags; |
| 1335 | vm->addr = (void *)va->va_start; |
| 1336 | vm->size = va->va_end - va->va_start; |
| 1337 | vm->caller = caller; |
Minchan Kim | db1aeca | 2012-01-10 15:08:39 -0800 | [diff] [blame] | 1338 | va->vm = vm; |
Tejun Heo | cf88c79 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 1339 | va->flags |= VM_VM_AREA; |
Joonsoo Kim | c69480a | 2013-04-29 15:07:30 -0700 | [diff] [blame] | 1340 | spin_unlock(&vmap_area_lock); |
Mitsuo Hayasaka | f5252e0 | 2011-10-31 17:08:13 -0700 | [diff] [blame] | 1341 | } |
Tejun Heo | cf88c79 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 1342 | |
Zhang Yanfei | 20fc02b | 2013-07-08 15:59:58 -0700 | [diff] [blame] | 1343 | static void clear_vm_uninitialized_flag(struct vm_struct *vm) |
Mitsuo Hayasaka | f5252e0 | 2011-10-31 17:08:13 -0700 | [diff] [blame] | 1344 | { |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 1345 | /* |
Zhang Yanfei | 20fc02b | 2013-07-08 15:59:58 -0700 | [diff] [blame] | 1346 | * Before removing VM_UNINITIALIZED, |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 1347 | * we should make sure that vm has proper values. |
| 1348 | * Pair with smp_rmb() in show_numa_info(). |
| 1349 | */ |
| 1350 | smp_wmb(); |
Zhang Yanfei | 20fc02b | 2013-07-08 15:59:58 -0700 | [diff] [blame] | 1351 | vm->flags &= ~VM_UNINITIALIZED; |
Tejun Heo | cf88c79 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 1352 | } |
| 1353 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1354 | static struct vm_struct *__get_vm_area_node(unsigned long size, |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1355 | unsigned long align, unsigned long flags, unsigned long start, |
Marek Szyprowski | 5e6cafc | 2012-04-13 12:32:09 +0200 | [diff] [blame] | 1356 | unsigned long end, int node, gfp_t gfp_mask, const void *caller) |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1357 | { |
Kautuk Consul | 0006526 | 2011-12-19 17:12:04 -0800 | [diff] [blame] | 1358 | struct vmap_area *va; |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1359 | struct vm_struct *area; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1360 | |
Giridhar Pemmasani | 52fd24c | 2006-10-28 10:38:34 -0700 | [diff] [blame] | 1361 | BUG_ON(in_interrupt()); |
Zhang Yanfei | 0f2d4a8 | 2013-07-03 15:04:50 -0700 | [diff] [blame] | 1362 | if (flags & VM_IOREMAP) |
Toshi Kani | 0f616be | 2015-04-14 15:47:17 -0700 | [diff] [blame] | 1363 | align = 1ul << clamp_t(int, fls_long(size), |
| 1364 | PAGE_SHIFT, IOREMAP_MAX_ORDER); |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1365 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1366 | size = PAGE_ALIGN(size); |
OGAWA Hirofumi | 31be830 | 2006-11-16 01:19:29 -0800 | [diff] [blame] | 1367 | if (unlikely(!size)) |
| 1368 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1369 | |
Tejun Heo | cf88c79 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 1370 | area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1371 | if (unlikely(!area)) |
| 1372 | return NULL; |
| 1373 | |
Andrey Ryabinin | 71394fe | 2015-02-13 14:40:03 -0800 | [diff] [blame] | 1374 | if (!(flags & VM_NO_GUARD)) |
| 1375 | size += PAGE_SIZE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1376 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1377 | va = alloc_vmap_area(size, align, start, end, node, gfp_mask); |
| 1378 | if (IS_ERR(va)) { |
| 1379 | kfree(area); |
| 1380 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1381 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1382 | |
Zhang Yanfei | d82b1d8 | 2013-07-03 15:04:47 -0700 | [diff] [blame] | 1383 | setup_vmalloc_vm(area, va, flags, caller); |
Mitsuo Hayasaka | f5252e0 | 2011-10-31 17:08:13 -0700 | [diff] [blame] | 1384 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1385 | return area; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1386 | } |
| 1387 | |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1388 | struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, |
| 1389 | unsigned long start, unsigned long end) |
| 1390 | { |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1391 | return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, |
| 1392 | GFP_KERNEL, __builtin_return_address(0)); |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1393 | } |
Rusty Russell | 5992b6d | 2007-07-19 01:49:21 -0700 | [diff] [blame] | 1394 | EXPORT_SYMBOL_GPL(__get_vm_area); |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1395 | |
Benjamin Herrenschmidt | c296861 | 2009-02-18 14:48:12 -0800 | [diff] [blame] | 1396 | struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, |
| 1397 | unsigned long start, unsigned long end, |
Marek Szyprowski | 5e6cafc | 2012-04-13 12:32:09 +0200 | [diff] [blame] | 1398 | const void *caller) |
Benjamin Herrenschmidt | c296861 | 2009-02-18 14:48:12 -0800 | [diff] [blame] | 1399 | { |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1400 | return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, |
| 1401 | GFP_KERNEL, caller); |
Benjamin Herrenschmidt | c296861 | 2009-02-18 14:48:12 -0800 | [diff] [blame] | 1402 | } |
| 1403 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1404 | /** |
Simon Arlott | 183ff22 | 2007-10-20 01:27:18 +0200 | [diff] [blame] | 1405 | * get_vm_area - reserve a contiguous kernel virtual area |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1406 | * @size: size of the area |
| 1407 | * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC |
| 1408 | * |
| 1409 | * Search an area of @size in the kernel virtual mapping area, |
| 1410 | * and reserved it for out purposes. Returns the area descriptor |
| 1411 | * on success or %NULL on failure. |
| 1412 | */ |
| 1413 | struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) |
| 1414 | { |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1415 | return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1416 | NUMA_NO_NODE, GFP_KERNEL, |
| 1417 | __builtin_return_address(0)); |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 1418 | } |
| 1419 | |
| 1420 | struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, |
Marek Szyprowski | 5e6cafc | 2012-04-13 12:32:09 +0200 | [diff] [blame] | 1421 | const void *caller) |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 1422 | { |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1423 | return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1424 | NUMA_NO_NODE, GFP_KERNEL, caller); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1425 | } |
| 1426 | |
Marek Szyprowski | e9da6e9 | 2012-07-30 09:11:33 +0200 | [diff] [blame] | 1427 | /** |
| 1428 | * find_vm_area - find a continuous kernel virtual area |
| 1429 | * @addr: base address |
| 1430 | * |
| 1431 | * Search for the kernel VM area starting at @addr, and return it. |
| 1432 | * It is up to the caller to do all required locking to keep the returned |
| 1433 | * pointer valid. |
| 1434 | */ |
| 1435 | struct vm_struct *find_vm_area(const void *addr) |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1436 | { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1437 | struct vmap_area *va; |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1438 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1439 | va = find_vmap_area((unsigned long)addr); |
| 1440 | if (va && va->flags & VM_VM_AREA) |
Minchan Kim | db1aeca | 2012-01-10 15:08:39 -0800 | [diff] [blame] | 1441 | return va->vm; |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1442 | |
Andi Kleen | 7856dfe | 2005-05-20 14:27:57 -0700 | [diff] [blame] | 1443 | return NULL; |
Andi Kleen | 7856dfe | 2005-05-20 14:27:57 -0700 | [diff] [blame] | 1444 | } |
| 1445 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1446 | /** |
Simon Arlott | 183ff22 | 2007-10-20 01:27:18 +0200 | [diff] [blame] | 1447 | * remove_vm_area - find and remove a continuous kernel virtual area |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1448 | * @addr: base address |
| 1449 | * |
| 1450 | * Search for the kernel VM area starting at @addr, and remove it. |
| 1451 | * This function returns the found VM area, but using it is NOT safe |
Andi Kleen | 7856dfe | 2005-05-20 14:27:57 -0700 | [diff] [blame] | 1452 | * on SMP machines, except for its size or flags. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1453 | */ |
Christoph Lameter | b3bdda0 | 2008-02-04 22:28:32 -0800 | [diff] [blame] | 1454 | struct vm_struct *remove_vm_area(const void *addr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1455 | { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1456 | struct vmap_area *va; |
| 1457 | |
| 1458 | va = find_vmap_area((unsigned long)addr); |
| 1459 | if (va && va->flags & VM_VM_AREA) { |
Minchan Kim | db1aeca | 2012-01-10 15:08:39 -0800 | [diff] [blame] | 1460 | struct vm_struct *vm = va->vm; |
Mitsuo Hayasaka | f5252e0 | 2011-10-31 17:08:13 -0700 | [diff] [blame] | 1461 | |
Joonsoo Kim | c69480a | 2013-04-29 15:07:30 -0700 | [diff] [blame] | 1462 | spin_lock(&vmap_area_lock); |
| 1463 | va->vm = NULL; |
| 1464 | va->flags &= ~VM_VM_AREA; |
| 1465 | spin_unlock(&vmap_area_lock); |
| 1466 | |
KAMEZAWA Hiroyuki | dd32c27 | 2009-09-21 17:02:32 -0700 | [diff] [blame] | 1467 | vmap_debug_free_range(va->va_start, va->va_end); |
Andrey Ryabinin | a5af5aa | 2015-03-12 16:26:11 -0700 | [diff] [blame] | 1468 | kasan_free_shadow(vm); |
KAMEZAWA Hiroyuki | dd32c27 | 2009-09-21 17:02:32 -0700 | [diff] [blame] | 1469 | free_unmap_vmap_area(va); |
KAMEZAWA Hiroyuki | dd32c27 | 2009-09-21 17:02:32 -0700 | [diff] [blame] | 1470 | |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1471 | return vm; |
| 1472 | } |
| 1473 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1474 | } |
| 1475 | |
Christoph Lameter | b3bdda0 | 2008-02-04 22:28:32 -0800 | [diff] [blame] | 1476 | static void __vunmap(const void *addr, int deallocate_pages) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1477 | { |
| 1478 | struct vm_struct *area; |
| 1479 | |
| 1480 | if (!addr) |
| 1481 | return; |
| 1482 | |
HATAYAMA Daisuke | e69e9d4a | 2013-07-03 15:02:18 -0700 | [diff] [blame] | 1483 | if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", |
Dan Carpenter | ab15d9b | 2013-07-08 15:59:53 -0700 | [diff] [blame] | 1484 | addr)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1485 | return; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1486 | |
| 1487 | area = remove_vm_area(addr); |
| 1488 | if (unlikely(!area)) { |
Arjan van de Ven | 4c8573e | 2008-07-25 19:45:37 -0700 | [diff] [blame] | 1489 | WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1490 | addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1491 | return; |
| 1492 | } |
| 1493 | |
Jerome Marchand | 7511c3e | 2015-11-20 15:57:02 -0800 | [diff] [blame] | 1494 | debug_check_no_locks_freed(addr, get_vm_area_size(area)); |
| 1495 | debug_check_no_obj_freed(addr, get_vm_area_size(area)); |
Ingo Molnar | 9a11b49a | 2006-07-03 00:24:33 -0700 | [diff] [blame] | 1496 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1497 | if (deallocate_pages) { |
| 1498 | int i; |
| 1499 | |
| 1500 | for (i = 0; i < area->nr_pages; i++) { |
Christoph Lameter | bf53d6f | 2008-02-04 22:28:34 -0800 | [diff] [blame] | 1501 | struct page *page = area->pages[i]; |
| 1502 | |
| 1503 | BUG_ON(!page); |
Vladimir Davydov | 37f08dd | 2016-01-14 15:18:18 -0800 | [diff] [blame] | 1504 | __free_kmem_pages(page, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1505 | } |
| 1506 | |
David Rientjes | 244d63e | 2016-01-14 15:19:35 -0800 | [diff] [blame] | 1507 | kvfree(area->pages); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1508 | } |
| 1509 | |
| 1510 | kfree(area); |
| 1511 | return; |
| 1512 | } |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1513 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1514 | /** |
| 1515 | * vfree - release memory allocated by vmalloc() |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1516 | * @addr: memory base address |
| 1517 | * |
Simon Arlott | 183ff22 | 2007-10-20 01:27:18 +0200 | [diff] [blame] | 1518 | * Free the virtually continuous memory area starting at @addr, as |
Pekka Enberg | 80e93ef | 2005-09-09 13:10:16 -0700 | [diff] [blame] | 1519 | * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is |
| 1520 | * NULL, no operation is performed. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1521 | * |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1522 | * Must not be called in NMI context (strictly speaking, only if we don't |
| 1523 | * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling |
| 1524 | * conventions for vfree() arch-depenedent would be a really bad idea) |
Andrew Morton | c9fcee5 | 2013-05-07 16:18:18 -0700 | [diff] [blame] | 1525 | * |
| 1526 | * NOTE: assumes that the object at *addr has a size >= sizeof(llist_node) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1527 | */ |
Christoph Lameter | b3bdda0 | 2008-02-04 22:28:32 -0800 | [diff] [blame] | 1528 | void vfree(const void *addr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1529 | { |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1530 | BUG_ON(in_nmi()); |
Catalin Marinas | 89219d3 | 2009-06-11 13:23:19 +0100 | [diff] [blame] | 1531 | |
| 1532 | kmemleak_free(addr); |
| 1533 | |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1534 | if (!addr) |
| 1535 | return; |
| 1536 | if (unlikely(in_interrupt())) { |
Christoph Lameter | 7c8e018 | 2014-06-04 16:07:56 -0700 | [diff] [blame] | 1537 | struct vfree_deferred *p = this_cpu_ptr(&vfree_deferred); |
Oleg Nesterov | 59d3132 | 2013-07-08 16:00:08 -0700 | [diff] [blame] | 1538 | if (llist_add((struct llist_node *)addr, &p->list)) |
| 1539 | schedule_work(&p->wq); |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1540 | } else |
| 1541 | __vunmap(addr, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1542 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1543 | EXPORT_SYMBOL(vfree); |
| 1544 | |
| 1545 | /** |
| 1546 | * vunmap - release virtual mapping obtained by vmap() |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1547 | * @addr: memory base address |
| 1548 | * |
| 1549 | * Free the virtually contiguous memory area starting at @addr, |
| 1550 | * which was created from the page array passed to vmap(). |
| 1551 | * |
Pekka Enberg | 80e93ef | 2005-09-09 13:10:16 -0700 | [diff] [blame] | 1552 | * Must not be called in interrupt context. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1553 | */ |
Christoph Lameter | b3bdda0 | 2008-02-04 22:28:32 -0800 | [diff] [blame] | 1554 | void vunmap(const void *addr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1555 | { |
| 1556 | BUG_ON(in_interrupt()); |
Peter Zijlstra | 34754b6 | 2009-02-25 16:04:03 +0100 | [diff] [blame] | 1557 | might_sleep(); |
Al Viro | 32fcfd4 | 2013-03-10 20:14:08 -0400 | [diff] [blame] | 1558 | if (addr) |
| 1559 | __vunmap(addr, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1560 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1561 | EXPORT_SYMBOL(vunmap); |
| 1562 | |
| 1563 | /** |
| 1564 | * vmap - map an array of pages into virtually contiguous space |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1565 | * @pages: array of page pointers |
| 1566 | * @count: number of pages to map |
| 1567 | * @flags: vm_area->flags |
| 1568 | * @prot: page protection for the mapping |
| 1569 | * |
| 1570 | * Maps @count pages from @pages into contiguous kernel virtual |
| 1571 | * space. |
| 1572 | */ |
| 1573 | void *vmap(struct page **pages, unsigned int count, |
| 1574 | unsigned long flags, pgprot_t prot) |
| 1575 | { |
| 1576 | struct vm_struct *area; |
Guillermo Julián Moreno | 65ee03c | 2016-06-03 14:55:33 -0700 | [diff] [blame] | 1577 | unsigned long size; /* In bytes */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1578 | |
Peter Zijlstra | 34754b6 | 2009-02-25 16:04:03 +0100 | [diff] [blame] | 1579 | might_sleep(); |
| 1580 | |
Jan Beulich | 4481374 | 2009-09-21 17:03:05 -0700 | [diff] [blame] | 1581 | if (count > totalram_pages) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1582 | return NULL; |
| 1583 | |
Guillermo Julián Moreno | 65ee03c | 2016-06-03 14:55:33 -0700 | [diff] [blame] | 1584 | size = (unsigned long)count << PAGE_SHIFT; |
| 1585 | area = get_vm_area_caller(size, flags, __builtin_return_address(0)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1586 | if (!area) |
| 1587 | return NULL; |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 1588 | |
WANG Chao | f6f8ed4 | 2014-08-06 16:06:58 -0700 | [diff] [blame] | 1589 | if (map_vm_area(area, prot, pages)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1590 | vunmap(area->addr); |
| 1591 | return NULL; |
| 1592 | } |
| 1593 | |
| 1594 | return area->addr; |
| 1595 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1596 | EXPORT_SYMBOL(vmap); |
| 1597 | |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1598 | static void *__vmalloc_node(unsigned long size, unsigned long align, |
| 1599 | gfp_t gfp_mask, pgprot_t prot, |
Marek Szyprowski | 5e6cafc | 2012-04-13 12:32:09 +0200 | [diff] [blame] | 1600 | int node, const void *caller); |
Adrian Bunk | e31d9eb | 2008-02-04 22:29:09 -0800 | [diff] [blame] | 1601 | static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, |
Wanpeng Li | 3722e13 | 2013-11-12 15:07:29 -0800 | [diff] [blame] | 1602 | pgprot_t prot, int node) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1603 | { |
Dave Hansen | 22943ab | 2011-05-24 17:12:18 -0700 | [diff] [blame] | 1604 | const int order = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1605 | struct page **pages; |
| 1606 | unsigned int nr_pages, array_size, i; |
David Rientjes | 930f036 | 2014-08-06 16:06:28 -0700 | [diff] [blame] | 1607 | const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; |
| 1608 | const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1609 | |
Wanpeng Li | 762216a | 2013-09-11 14:22:42 -0700 | [diff] [blame] | 1610 | nr_pages = get_vm_area_size(area) >> PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1611 | array_size = (nr_pages * sizeof(struct page *)); |
| 1612 | |
| 1613 | area->nr_pages = nr_pages; |
| 1614 | /* Please note that the recursion is strictly bounded. */ |
Jan Kiszka | 8757d5f | 2006-07-14 00:23:56 -0700 | [diff] [blame] | 1615 | if (array_size > PAGE_SIZE) { |
Jan Beulich | 976d6df | 2009-12-14 17:58:39 -0800 | [diff] [blame] | 1616 | pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM, |
Wanpeng Li | 3722e13 | 2013-11-12 15:07:29 -0800 | [diff] [blame] | 1617 | PAGE_KERNEL, node, area->caller); |
Andrew Morton | 286e1ea | 2006-10-17 00:09:57 -0700 | [diff] [blame] | 1618 | } else { |
Jan Beulich | 976d6df | 2009-12-14 17:58:39 -0800 | [diff] [blame] | 1619 | pages = kmalloc_node(array_size, nested_gfp, node); |
Andrew Morton | 286e1ea | 2006-10-17 00:09:57 -0700 | [diff] [blame] | 1620 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1621 | area->pages = pages; |
| 1622 | if (!area->pages) { |
| 1623 | remove_vm_area(area->addr); |
| 1624 | kfree(area); |
| 1625 | return NULL; |
| 1626 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1627 | |
| 1628 | for (i = 0; i < area->nr_pages; i++) { |
Christoph Lameter | bf53d6f | 2008-02-04 22:28:34 -0800 | [diff] [blame] | 1629 | struct page *page; |
| 1630 | |
Jianguo Wu | 4b90951 | 2013-11-12 15:07:11 -0800 | [diff] [blame] | 1631 | if (node == NUMA_NO_NODE) |
Vladimir Davydov | 37f08dd | 2016-01-14 15:18:18 -0800 | [diff] [blame] | 1632 | page = alloc_kmem_pages(alloc_mask, order); |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1633 | else |
Vladimir Davydov | 37f08dd | 2016-01-14 15:18:18 -0800 | [diff] [blame] | 1634 | page = alloc_kmem_pages_node(node, alloc_mask, order); |
Christoph Lameter | bf53d6f | 2008-02-04 22:28:34 -0800 | [diff] [blame] | 1635 | |
| 1636 | if (unlikely(!page)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1637 | /* Successfully allocated i pages, free them in __vunmap() */ |
| 1638 | area->nr_pages = i; |
| 1639 | goto fail; |
| 1640 | } |
Christoph Lameter | bf53d6f | 2008-02-04 22:28:34 -0800 | [diff] [blame] | 1641 | area->pages[i] = page; |
Mel Gorman | d0164ad | 2015-11-06 16:28:21 -0800 | [diff] [blame] | 1642 | if (gfpflags_allow_blocking(gfp_mask)) |
Eric Dumazet | 660654f | 2014-08-06 16:06:25 -0700 | [diff] [blame] | 1643 | cond_resched(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1644 | } |
| 1645 | |
WANG Chao | f6f8ed4 | 2014-08-06 16:06:58 -0700 | [diff] [blame] | 1646 | if (map_vm_area(area, prot, pages)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1647 | goto fail; |
| 1648 | return area->addr; |
| 1649 | |
| 1650 | fail: |
Joe Perches | 3ee9a4f | 2011-10-31 17:08:35 -0700 | [diff] [blame] | 1651 | warn_alloc_failed(gfp_mask, order, |
| 1652 | "vmalloc: allocation failure, allocated %ld of %ld bytes\n", |
Dave Hansen | 22943ab | 2011-05-24 17:12:18 -0700 | [diff] [blame] | 1653 | (area->nr_pages*PAGE_SIZE), area->size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1654 | vfree(area->addr); |
| 1655 | return NULL; |
| 1656 | } |
| 1657 | |
David Rientjes | d0a2126 | 2011-01-13 15:46:02 -0800 | [diff] [blame] | 1658 | /** |
| 1659 | * __vmalloc_node_range - allocate virtually contiguous memory |
| 1660 | * @size: allocation size |
| 1661 | * @align: desired alignment |
| 1662 | * @start: vm area range start |
| 1663 | * @end: vm area range end |
| 1664 | * @gfp_mask: flags for the page level allocator |
| 1665 | * @prot: protection mask for the allocated pages |
Andrey Ryabinin | cb9e3c2 | 2015-02-13 14:40:07 -0800 | [diff] [blame] | 1666 | * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1667 | * @node: node to use for allocation or NUMA_NO_NODE |
David Rientjes | d0a2126 | 2011-01-13 15:46:02 -0800 | [diff] [blame] | 1668 | * @caller: caller's return address |
| 1669 | * |
| 1670 | * Allocate enough pages to cover @size from the page level |
| 1671 | * allocator with @gfp_mask flags. Map them into contiguous |
| 1672 | * kernel virtual space, using a pagetable protection of @prot. |
| 1673 | */ |
| 1674 | void *__vmalloc_node_range(unsigned long size, unsigned long align, |
| 1675 | unsigned long start, unsigned long end, gfp_t gfp_mask, |
Andrey Ryabinin | cb9e3c2 | 2015-02-13 14:40:07 -0800 | [diff] [blame] | 1676 | pgprot_t prot, unsigned long vm_flags, int node, |
| 1677 | const void *caller) |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1678 | { |
David Rientjes | d0a2126 | 2011-01-13 15:46:02 -0800 | [diff] [blame] | 1679 | struct vm_struct *area; |
| 1680 | void *addr; |
| 1681 | unsigned long real_size = size; |
| 1682 | |
| 1683 | size = PAGE_ALIGN(size); |
| 1684 | if (!size || (size >> PAGE_SHIFT) > totalram_pages) |
Joe Perches | de7d2b5 | 2011-10-31 17:08:48 -0700 | [diff] [blame] | 1685 | goto fail; |
David Rientjes | d0a2126 | 2011-01-13 15:46:02 -0800 | [diff] [blame] | 1686 | |
Andrey Ryabinin | cb9e3c2 | 2015-02-13 14:40:07 -0800 | [diff] [blame] | 1687 | area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED | |
| 1688 | vm_flags, start, end, node, gfp_mask, caller); |
David Rientjes | d0a2126 | 2011-01-13 15:46:02 -0800 | [diff] [blame] | 1689 | if (!area) |
Joe Perches | de7d2b5 | 2011-10-31 17:08:48 -0700 | [diff] [blame] | 1690 | goto fail; |
David Rientjes | d0a2126 | 2011-01-13 15:46:02 -0800 | [diff] [blame] | 1691 | |
Wanpeng Li | 3722e13 | 2013-11-12 15:07:29 -0800 | [diff] [blame] | 1692 | addr = __vmalloc_area_node(area, gfp_mask, prot, node); |
Mel Gorman | 1368edf | 2011-12-08 14:34:30 -0800 | [diff] [blame] | 1693 | if (!addr) |
Wanpeng Li | b82225f3 | 2013-11-12 15:07:33 -0800 | [diff] [blame] | 1694 | return NULL; |
Catalin Marinas | 89219d3 | 2009-06-11 13:23:19 +0100 | [diff] [blame] | 1695 | |
| 1696 | /* |
Zhang Yanfei | 20fc02b | 2013-07-08 15:59:58 -0700 | [diff] [blame] | 1697 | * In this function, newly allocated vm_struct has VM_UNINITIALIZED |
| 1698 | * flag. It means that vm_struct is not fully initialized. |
Joonsoo Kim | 4341fa4 | 2013-04-29 15:07:39 -0700 | [diff] [blame] | 1699 | * Now, it is fully initialized, so remove this flag here. |
Mitsuo Hayasaka | f5252e0 | 2011-10-31 17:08:13 -0700 | [diff] [blame] | 1700 | */ |
Zhang Yanfei | 20fc02b | 2013-07-08 15:59:58 -0700 | [diff] [blame] | 1701 | clear_vm_uninitialized_flag(area); |
Mitsuo Hayasaka | f5252e0 | 2011-10-31 17:08:13 -0700 | [diff] [blame] | 1702 | |
| 1703 | /* |
Catalin Marinas | 7f88f88 | 2013-11-12 15:07:45 -0800 | [diff] [blame] | 1704 | * A ref_count = 2 is needed because vm_struct allocated in |
| 1705 | * __get_vm_area_node() contains a reference to the virtual address of |
| 1706 | * the vmalloc'ed block. |
Catalin Marinas | 89219d3 | 2009-06-11 13:23:19 +0100 | [diff] [blame] | 1707 | */ |
Catalin Marinas | 7f88f88 | 2013-11-12 15:07:45 -0800 | [diff] [blame] | 1708 | kmemleak_alloc(addr, real_size, 2, gfp_mask); |
Catalin Marinas | 89219d3 | 2009-06-11 13:23:19 +0100 | [diff] [blame] | 1709 | |
| 1710 | return addr; |
Joe Perches | de7d2b5 | 2011-10-31 17:08:48 -0700 | [diff] [blame] | 1711 | |
| 1712 | fail: |
| 1713 | warn_alloc_failed(gfp_mask, 0, |
| 1714 | "vmalloc: allocation failure: %lu bytes\n", |
| 1715 | real_size); |
| 1716 | return NULL; |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1717 | } |
| 1718 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1719 | /** |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1720 | * __vmalloc_node - allocate virtually contiguous memory |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1721 | * @size: allocation size |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1722 | * @align: desired alignment |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1723 | * @gfp_mask: flags for the page level allocator |
| 1724 | * @prot: protection mask for the allocated pages |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1725 | * @node: node to use for allocation or NUMA_NO_NODE |
Randy Dunlap | c85d194 | 2008-05-01 04:34:48 -0700 | [diff] [blame] | 1726 | * @caller: caller's return address |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1727 | * |
| 1728 | * Allocate enough pages to cover @size from the page level |
| 1729 | * allocator with @gfp_mask flags. Map them into contiguous |
| 1730 | * kernel virtual space, using a pagetable protection of @prot. |
| 1731 | */ |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1732 | static void *__vmalloc_node(unsigned long size, unsigned long align, |
| 1733 | gfp_t gfp_mask, pgprot_t prot, |
Marek Szyprowski | 5e6cafc | 2012-04-13 12:32:09 +0200 | [diff] [blame] | 1734 | int node, const void *caller) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1735 | { |
David Rientjes | d0a2126 | 2011-01-13 15:46:02 -0800 | [diff] [blame] | 1736 | return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, |
Andrey Ryabinin | cb9e3c2 | 2015-02-13 14:40:07 -0800 | [diff] [blame] | 1737 | gfp_mask, prot, 0, node, caller); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1738 | } |
| 1739 | |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1740 | void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) |
| 1741 | { |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1742 | return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE, |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 1743 | __builtin_return_address(0)); |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1744 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1745 | EXPORT_SYMBOL(__vmalloc); |
| 1746 | |
Dave Young | e1ca778 | 2010-10-26 14:22:06 -0700 | [diff] [blame] | 1747 | static inline void *__vmalloc_node_flags(unsigned long size, |
| 1748 | int node, gfp_t flags) |
| 1749 | { |
| 1750 | return __vmalloc_node(size, 1, flags, PAGE_KERNEL, |
| 1751 | node, __builtin_return_address(0)); |
| 1752 | } |
| 1753 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1754 | /** |
| 1755 | * vmalloc - allocate virtually contiguous memory |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1756 | * @size: allocation size |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1757 | * Allocate enough pages to cover @size from the page level |
| 1758 | * allocator and map them into contiguous kernel virtual space. |
| 1759 | * |
Michael Opdenacker | c1c8897 | 2006-10-03 23:21:02 +0200 | [diff] [blame] | 1760 | * For tight control over page level allocator and protection flags |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1761 | * use __vmalloc() instead. |
| 1762 | */ |
| 1763 | void *vmalloc(unsigned long size) |
| 1764 | { |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1765 | return __vmalloc_node_flags(size, NUMA_NO_NODE, |
| 1766 | GFP_KERNEL | __GFP_HIGHMEM); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1767 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1768 | EXPORT_SYMBOL(vmalloc); |
| 1769 | |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1770 | /** |
Dave Young | e1ca778 | 2010-10-26 14:22:06 -0700 | [diff] [blame] | 1771 | * vzalloc - allocate virtually contiguous memory with zero fill |
| 1772 | * @size: allocation size |
| 1773 | * Allocate enough pages to cover @size from the page level |
| 1774 | * allocator and map them into contiguous kernel virtual space. |
| 1775 | * The memory allocated is set to zero. |
| 1776 | * |
| 1777 | * For tight control over page level allocator and protection flags |
| 1778 | * use __vmalloc() instead. |
| 1779 | */ |
| 1780 | void *vzalloc(unsigned long size) |
| 1781 | { |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1782 | return __vmalloc_node_flags(size, NUMA_NO_NODE, |
Dave Young | e1ca778 | 2010-10-26 14:22:06 -0700 | [diff] [blame] | 1783 | GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); |
| 1784 | } |
| 1785 | EXPORT_SYMBOL(vzalloc); |
| 1786 | |
| 1787 | /** |
Rolf Eike Beer | ead0408 | 2006-09-27 01:50:13 -0700 | [diff] [blame] | 1788 | * vmalloc_user - allocate zeroed virtually contiguous memory for userspace |
| 1789 | * @size: allocation size |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1790 | * |
Rolf Eike Beer | ead0408 | 2006-09-27 01:50:13 -0700 | [diff] [blame] | 1791 | * The resulting memory area is zeroed so it can be mapped to userspace |
| 1792 | * without leaking data. |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1793 | */ |
| 1794 | void *vmalloc_user(unsigned long size) |
| 1795 | { |
| 1796 | struct vm_struct *area; |
| 1797 | void *ret; |
| 1798 | |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1799 | ret = __vmalloc_node(size, SHMLBA, |
| 1800 | GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1801 | PAGE_KERNEL, NUMA_NO_NODE, |
| 1802 | __builtin_return_address(0)); |
Eric Dumazet | 2b4ac44 | 2006-11-10 12:27:48 -0800 | [diff] [blame] | 1803 | if (ret) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1804 | area = find_vm_area(ret); |
Eric Dumazet | 2b4ac44 | 2006-11-10 12:27:48 -0800 | [diff] [blame] | 1805 | area->flags |= VM_USERMAP; |
Eric Dumazet | 2b4ac44 | 2006-11-10 12:27:48 -0800 | [diff] [blame] | 1806 | } |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1807 | return ret; |
| 1808 | } |
| 1809 | EXPORT_SYMBOL(vmalloc_user); |
| 1810 | |
| 1811 | /** |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1812 | * vmalloc_node - allocate memory on a specific node |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1813 | * @size: allocation size |
Randy Dunlap | d44e078 | 2005-11-07 01:01:10 -0800 | [diff] [blame] | 1814 | * @node: numa node |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1815 | * |
| 1816 | * Allocate enough pages to cover @size from the page level |
| 1817 | * allocator and map them into contiguous kernel virtual space. |
| 1818 | * |
Michael Opdenacker | c1c8897 | 2006-10-03 23:21:02 +0200 | [diff] [blame] | 1819 | * For tight control over page level allocator and protection flags |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1820 | * use __vmalloc() instead. |
| 1821 | */ |
| 1822 | void *vmalloc_node(unsigned long size, int node) |
| 1823 | { |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1824 | return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 1825 | node, __builtin_return_address(0)); |
Christoph Lameter | 930fc45 | 2005-10-29 18:15:41 -0700 | [diff] [blame] | 1826 | } |
| 1827 | EXPORT_SYMBOL(vmalloc_node); |
| 1828 | |
Dave Young | e1ca778 | 2010-10-26 14:22:06 -0700 | [diff] [blame] | 1829 | /** |
| 1830 | * vzalloc_node - allocate memory on a specific node with zero fill |
| 1831 | * @size: allocation size |
| 1832 | * @node: numa node |
| 1833 | * |
| 1834 | * Allocate enough pages to cover @size from the page level |
| 1835 | * allocator and map them into contiguous kernel virtual space. |
| 1836 | * The memory allocated is set to zero. |
| 1837 | * |
| 1838 | * For tight control over page level allocator and protection flags |
| 1839 | * use __vmalloc_node() instead. |
| 1840 | */ |
| 1841 | void *vzalloc_node(unsigned long size, int node) |
| 1842 | { |
| 1843 | return __vmalloc_node_flags(size, node, |
| 1844 | GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); |
| 1845 | } |
| 1846 | EXPORT_SYMBOL(vzalloc_node); |
| 1847 | |
Pavel Pisa | 4dc3b16 | 2005-05-01 08:59:25 -0700 | [diff] [blame] | 1848 | #ifndef PAGE_KERNEL_EXEC |
| 1849 | # define PAGE_KERNEL_EXEC PAGE_KERNEL |
| 1850 | #endif |
| 1851 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1852 | /** |
| 1853 | * vmalloc_exec - allocate virtually contiguous, executable memory |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1854 | * @size: allocation size |
| 1855 | * |
| 1856 | * Kernel-internal function to allocate enough pages to cover @size |
| 1857 | * the page level allocator and map them into contiguous and |
| 1858 | * executable kernel virtual space. |
| 1859 | * |
Michael Opdenacker | c1c8897 | 2006-10-03 23:21:02 +0200 | [diff] [blame] | 1860 | * For tight control over page level allocator and protection flags |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1861 | * use __vmalloc() instead. |
| 1862 | */ |
| 1863 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1864 | void *vmalloc_exec(unsigned long size) |
| 1865 | { |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1866 | return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC, |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1867 | NUMA_NO_NODE, __builtin_return_address(0)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1868 | } |
| 1869 | |
Andi Kleen | 0d08e0d | 2007-05-02 19:27:12 +0200 | [diff] [blame] | 1870 | #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) |
Benjamin Herrenschmidt | 7ac674f | 2007-07-19 01:49:10 -0700 | [diff] [blame] | 1871 | #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL |
Andi Kleen | 0d08e0d | 2007-05-02 19:27:12 +0200 | [diff] [blame] | 1872 | #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) |
Benjamin Herrenschmidt | 7ac674f | 2007-07-19 01:49:10 -0700 | [diff] [blame] | 1873 | #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL |
Andi Kleen | 0d08e0d | 2007-05-02 19:27:12 +0200 | [diff] [blame] | 1874 | #else |
| 1875 | #define GFP_VMALLOC32 GFP_KERNEL |
| 1876 | #endif |
| 1877 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1878 | /** |
| 1879 | * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1880 | * @size: allocation size |
| 1881 | * |
| 1882 | * Allocate enough 32bit PA addressable pages to cover @size from the |
| 1883 | * page level allocator and map them into contiguous kernel virtual space. |
| 1884 | */ |
| 1885 | void *vmalloc_32(unsigned long size) |
| 1886 | { |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1887 | return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL, |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1888 | NUMA_NO_NODE, __builtin_return_address(0)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1889 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1890 | EXPORT_SYMBOL(vmalloc_32); |
| 1891 | |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1892 | /** |
Rolf Eike Beer | ead0408 | 2006-09-27 01:50:13 -0700 | [diff] [blame] | 1893 | * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1894 | * @size: allocation size |
Rolf Eike Beer | ead0408 | 2006-09-27 01:50:13 -0700 | [diff] [blame] | 1895 | * |
| 1896 | * The resulting memory area is 32bit addressable and zeroed so it can be |
| 1897 | * mapped to userspace without leaking data. |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1898 | */ |
| 1899 | void *vmalloc_32_user(unsigned long size) |
| 1900 | { |
| 1901 | struct vm_struct *area; |
| 1902 | void *ret; |
| 1903 | |
David Miller | 2dca699 | 2009-09-21 12:22:34 -0700 | [diff] [blame] | 1904 | ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, |
David Rientjes | 00ef2d2 | 2013-02-22 16:35:36 -0800 | [diff] [blame] | 1905 | NUMA_NO_NODE, __builtin_return_address(0)); |
Eric Dumazet | 2b4ac44 | 2006-11-10 12:27:48 -0800 | [diff] [blame] | 1906 | if (ret) { |
Nick Piggin | db64fe0 | 2008-10-18 20:27:03 -0700 | [diff] [blame] | 1907 | area = find_vm_area(ret); |
Eric Dumazet | 2b4ac44 | 2006-11-10 12:27:48 -0800 | [diff] [blame] | 1908 | area->flags |= VM_USERMAP; |
Eric Dumazet | 2b4ac44 | 2006-11-10 12:27:48 -0800 | [diff] [blame] | 1909 | } |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 1910 | return ret; |
| 1911 | } |
| 1912 | EXPORT_SYMBOL(vmalloc_32_user); |
| 1913 | |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 1914 | /* |
| 1915 | * small helper routine , copy contents to buf from addr. |
| 1916 | * If the page is not present, fill zero. |
| 1917 | */ |
| 1918 | |
| 1919 | static int aligned_vread(char *buf, char *addr, unsigned long count) |
| 1920 | { |
| 1921 | struct page *p; |
| 1922 | int copied = 0; |
| 1923 | |
| 1924 | while (count) { |
| 1925 | unsigned long offset, length; |
| 1926 | |
Alexander Kuleshov | 891c49a | 2015-11-05 18:46:51 -0800 | [diff] [blame] | 1927 | offset = offset_in_page(addr); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 1928 | length = PAGE_SIZE - offset; |
| 1929 | if (length > count) |
| 1930 | length = count; |
| 1931 | p = vmalloc_to_page(addr); |
| 1932 | /* |
| 1933 | * To do safe access to this _mapped_ area, we need |
| 1934 | * lock. But adding lock here means that we need to add |
| 1935 | * overhead of vmalloc()/vfree() calles for this _debug_ |
| 1936 | * interface, rarely used. Instead of that, we'll use |
| 1937 | * kmap() and get small overhead in this access function. |
| 1938 | */ |
| 1939 | if (p) { |
| 1940 | /* |
| 1941 | * we can expect USER0 is not used (see vread/vwrite's |
| 1942 | * function description) |
| 1943 | */ |
Cong Wang | 9b04c5f | 2011-11-25 23:14:39 +0800 | [diff] [blame] | 1944 | void *map = kmap_atomic(p); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 1945 | memcpy(buf, map + offset, length); |
Cong Wang | 9b04c5f | 2011-11-25 23:14:39 +0800 | [diff] [blame] | 1946 | kunmap_atomic(map); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 1947 | } else |
| 1948 | memset(buf, 0, length); |
| 1949 | |
| 1950 | addr += length; |
| 1951 | buf += length; |
| 1952 | copied += length; |
| 1953 | count -= length; |
| 1954 | } |
| 1955 | return copied; |
| 1956 | } |
| 1957 | |
| 1958 | static int aligned_vwrite(char *buf, char *addr, unsigned long count) |
| 1959 | { |
| 1960 | struct page *p; |
| 1961 | int copied = 0; |
| 1962 | |
| 1963 | while (count) { |
| 1964 | unsigned long offset, length; |
| 1965 | |
Alexander Kuleshov | 891c49a | 2015-11-05 18:46:51 -0800 | [diff] [blame] | 1966 | offset = offset_in_page(addr); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 1967 | length = PAGE_SIZE - offset; |
| 1968 | if (length > count) |
| 1969 | length = count; |
| 1970 | p = vmalloc_to_page(addr); |
| 1971 | /* |
| 1972 | * To do safe access to this _mapped_ area, we need |
| 1973 | * lock. But adding lock here means that we need to add |
| 1974 | * overhead of vmalloc()/vfree() calles for this _debug_ |
| 1975 | * interface, rarely used. Instead of that, we'll use |
| 1976 | * kmap() and get small overhead in this access function. |
| 1977 | */ |
| 1978 | if (p) { |
| 1979 | /* |
| 1980 | * we can expect USER0 is not used (see vread/vwrite's |
| 1981 | * function description) |
| 1982 | */ |
Cong Wang | 9b04c5f | 2011-11-25 23:14:39 +0800 | [diff] [blame] | 1983 | void *map = kmap_atomic(p); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 1984 | memcpy(map + offset, buf, length); |
Cong Wang | 9b04c5f | 2011-11-25 23:14:39 +0800 | [diff] [blame] | 1985 | kunmap_atomic(map); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 1986 | } |
| 1987 | addr += length; |
| 1988 | buf += length; |
| 1989 | copied += length; |
| 1990 | count -= length; |
| 1991 | } |
| 1992 | return copied; |
| 1993 | } |
| 1994 | |
| 1995 | /** |
| 1996 | * vread() - read vmalloc area in a safe way. |
| 1997 | * @buf: buffer for reading data |
| 1998 | * @addr: vm address. |
| 1999 | * @count: number of bytes to be read. |
| 2000 | * |
| 2001 | * Returns # of bytes which addr and buf should be increased. |
| 2002 | * (same number to @count). Returns 0 if [addr...addr+count) doesn't |
| 2003 | * includes any intersect with alive vmalloc area. |
| 2004 | * |
| 2005 | * This function checks that addr is a valid vmalloc'ed area, and |
| 2006 | * copy data from that area to a given buffer. If the given memory range |
| 2007 | * of [addr...addr+count) includes some valid address, data is copied to |
| 2008 | * proper area of @buf. If there are memory holes, they'll be zero-filled. |
| 2009 | * IOREMAP area is treated as memory hole and no copy is done. |
| 2010 | * |
| 2011 | * If [addr...addr+count) doesn't includes any intersects with alive |
Cong Wang | a8e5202 | 2012-06-23 11:30:16 +0800 | [diff] [blame] | 2012 | * vm_struct area, returns 0. @buf should be kernel's buffer. |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2013 | * |
| 2014 | * Note: In usual ops, vread() is never necessary because the caller |
| 2015 | * should know vmalloc() area is valid and can use memcpy(). |
| 2016 | * This is for routines which have to access vmalloc area without |
| 2017 | * any informaion, as /dev/kmem. |
| 2018 | * |
| 2019 | */ |
| 2020 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2021 | long vread(char *buf, char *addr, unsigned long count) |
| 2022 | { |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2023 | struct vmap_area *va; |
| 2024 | struct vm_struct *vm; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2025 | char *vaddr, *buf_start = buf; |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2026 | unsigned long buflen = count; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2027 | unsigned long n; |
| 2028 | |
| 2029 | /* Don't allow overflow */ |
| 2030 | if ((unsigned long) addr + count < count) |
| 2031 | count = -(unsigned long) addr; |
| 2032 | |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2033 | spin_lock(&vmap_area_lock); |
| 2034 | list_for_each_entry(va, &vmap_area_list, list) { |
| 2035 | if (!count) |
| 2036 | break; |
| 2037 | |
| 2038 | if (!(va->flags & VM_VM_AREA)) |
| 2039 | continue; |
| 2040 | |
| 2041 | vm = va->vm; |
| 2042 | vaddr = (char *) vm->addr; |
Wanpeng Li | 762216a | 2013-09-11 14:22:42 -0700 | [diff] [blame] | 2043 | if (addr >= vaddr + get_vm_area_size(vm)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2044 | continue; |
| 2045 | while (addr < vaddr) { |
| 2046 | if (count == 0) |
| 2047 | goto finished; |
| 2048 | *buf = '\0'; |
| 2049 | buf++; |
| 2050 | addr++; |
| 2051 | count--; |
| 2052 | } |
Wanpeng Li | 762216a | 2013-09-11 14:22:42 -0700 | [diff] [blame] | 2053 | n = vaddr + get_vm_area_size(vm) - addr; |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2054 | if (n > count) |
| 2055 | n = count; |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2056 | if (!(vm->flags & VM_IOREMAP)) |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2057 | aligned_vread(buf, addr, n); |
| 2058 | else /* IOREMAP area is treated as memory hole */ |
| 2059 | memset(buf, 0, n); |
| 2060 | buf += n; |
| 2061 | addr += n; |
| 2062 | count -= n; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2063 | } |
| 2064 | finished: |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2065 | spin_unlock(&vmap_area_lock); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2066 | |
| 2067 | if (buf == buf_start) |
| 2068 | return 0; |
| 2069 | /* zero-fill memory holes */ |
| 2070 | if (buf != buf_start + buflen) |
| 2071 | memset(buf, 0, buflen - (buf - buf_start)); |
| 2072 | |
| 2073 | return buflen; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2074 | } |
| 2075 | |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2076 | /** |
| 2077 | * vwrite() - write vmalloc area in a safe way. |
| 2078 | * @buf: buffer for source data |
| 2079 | * @addr: vm address. |
| 2080 | * @count: number of bytes to be read. |
| 2081 | * |
| 2082 | * Returns # of bytes which addr and buf should be incresed. |
| 2083 | * (same number to @count). |
| 2084 | * If [addr...addr+count) doesn't includes any intersect with valid |
| 2085 | * vmalloc area, returns 0. |
| 2086 | * |
| 2087 | * This function checks that addr is a valid vmalloc'ed area, and |
| 2088 | * copy data from a buffer to the given addr. If specified range of |
| 2089 | * [addr...addr+count) includes some valid address, data is copied from |
| 2090 | * proper area of @buf. If there are memory holes, no copy to hole. |
| 2091 | * IOREMAP area is treated as memory hole and no copy is done. |
| 2092 | * |
| 2093 | * If [addr...addr+count) doesn't includes any intersects with alive |
Cong Wang | a8e5202 | 2012-06-23 11:30:16 +0800 | [diff] [blame] | 2094 | * vm_struct area, returns 0. @buf should be kernel's buffer. |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2095 | * |
| 2096 | * Note: In usual ops, vwrite() is never necessary because the caller |
| 2097 | * should know vmalloc() area is valid and can use memcpy(). |
| 2098 | * This is for routines which have to access vmalloc area without |
| 2099 | * any informaion, as /dev/kmem. |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2100 | */ |
| 2101 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2102 | long vwrite(char *buf, char *addr, unsigned long count) |
| 2103 | { |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2104 | struct vmap_area *va; |
| 2105 | struct vm_struct *vm; |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2106 | char *vaddr; |
| 2107 | unsigned long n, buflen; |
| 2108 | int copied = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2109 | |
| 2110 | /* Don't allow overflow */ |
| 2111 | if ((unsigned long) addr + count < count) |
| 2112 | count = -(unsigned long) addr; |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2113 | buflen = count; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2114 | |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2115 | spin_lock(&vmap_area_lock); |
| 2116 | list_for_each_entry(va, &vmap_area_list, list) { |
| 2117 | if (!count) |
| 2118 | break; |
| 2119 | |
| 2120 | if (!(va->flags & VM_VM_AREA)) |
| 2121 | continue; |
| 2122 | |
| 2123 | vm = va->vm; |
| 2124 | vaddr = (char *) vm->addr; |
Wanpeng Li | 762216a | 2013-09-11 14:22:42 -0700 | [diff] [blame] | 2125 | if (addr >= vaddr + get_vm_area_size(vm)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2126 | continue; |
| 2127 | while (addr < vaddr) { |
| 2128 | if (count == 0) |
| 2129 | goto finished; |
| 2130 | buf++; |
| 2131 | addr++; |
| 2132 | count--; |
| 2133 | } |
Wanpeng Li | 762216a | 2013-09-11 14:22:42 -0700 | [diff] [blame] | 2134 | n = vaddr + get_vm_area_size(vm) - addr; |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2135 | if (n > count) |
| 2136 | n = count; |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2137 | if (!(vm->flags & VM_IOREMAP)) { |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2138 | aligned_vwrite(buf, addr, n); |
| 2139 | copied++; |
| 2140 | } |
| 2141 | buf += n; |
| 2142 | addr += n; |
| 2143 | count -= n; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2144 | } |
| 2145 | finished: |
Joonsoo Kim | e81ce85 | 2013-04-29 15:07:32 -0700 | [diff] [blame] | 2146 | spin_unlock(&vmap_area_lock); |
KAMEZAWA Hiroyuki | d0107eb | 2009-09-21 17:02:34 -0700 | [diff] [blame] | 2147 | if (!copied) |
| 2148 | return 0; |
| 2149 | return buflen; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2150 | } |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 2151 | |
| 2152 | /** |
HATAYAMA Daisuke | e69e9d4a | 2013-07-03 15:02:18 -0700 | [diff] [blame] | 2153 | * remap_vmalloc_range_partial - map vmalloc pages to userspace |
| 2154 | * @vma: vma to cover |
| 2155 | * @uaddr: target user address to start at |
| 2156 | * @kaddr: virtual address of vmalloc kernel memory |
| 2157 | * @size: size of map area |
| 2158 | * |
| 2159 | * Returns: 0 for success, -Exxx on failure |
| 2160 | * |
| 2161 | * This function checks that @kaddr is a valid vmalloc'ed area, |
| 2162 | * and that it is big enough to cover the range starting at |
| 2163 | * @uaddr in @vma. Will return failure if that criteria isn't |
| 2164 | * met. |
| 2165 | * |
| 2166 | * Similar to remap_pfn_range() (see mm/memory.c) |
| 2167 | */ |
| 2168 | int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, |
| 2169 | void *kaddr, unsigned long size) |
| 2170 | { |
| 2171 | struct vm_struct *area; |
| 2172 | |
| 2173 | size = PAGE_ALIGN(size); |
| 2174 | |
| 2175 | if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) |
| 2176 | return -EINVAL; |
| 2177 | |
| 2178 | area = find_vm_area(kaddr); |
| 2179 | if (!area) |
| 2180 | return -EINVAL; |
| 2181 | |
| 2182 | if (!(area->flags & VM_USERMAP)) |
| 2183 | return -EINVAL; |
| 2184 | |
| 2185 | if (kaddr + size > area->addr + area->size) |
| 2186 | return -EINVAL; |
| 2187 | |
| 2188 | do { |
| 2189 | struct page *page = vmalloc_to_page(kaddr); |
| 2190 | int ret; |
| 2191 | |
| 2192 | ret = vm_insert_page(vma, uaddr, page); |
| 2193 | if (ret) |
| 2194 | return ret; |
| 2195 | |
| 2196 | uaddr += PAGE_SIZE; |
| 2197 | kaddr += PAGE_SIZE; |
| 2198 | size -= PAGE_SIZE; |
| 2199 | } while (size > 0); |
| 2200 | |
| 2201 | vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; |
| 2202 | |
| 2203 | return 0; |
| 2204 | } |
| 2205 | EXPORT_SYMBOL(remap_vmalloc_range_partial); |
| 2206 | |
| 2207 | /** |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 2208 | * remap_vmalloc_range - map vmalloc pages to userspace |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 2209 | * @vma: vma to cover (map full range of vma) |
| 2210 | * @addr: vmalloc memory |
| 2211 | * @pgoff: number of pages into addr before first page to map |
Randy Dunlap | 7682486 | 2008-03-19 17:00:40 -0700 | [diff] [blame] | 2212 | * |
| 2213 | * Returns: 0 for success, -Exxx on failure |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 2214 | * |
| 2215 | * This function checks that addr is a valid vmalloc'ed area, and |
| 2216 | * that it is big enough to cover the vma. Will return failure if |
| 2217 | * that criteria isn't met. |
| 2218 | * |
Robert P. J. Day | 72fd4a3 | 2007-02-10 01:45:59 -0800 | [diff] [blame] | 2219 | * Similar to remap_pfn_range() (see mm/memory.c) |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 2220 | */ |
| 2221 | int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, |
| 2222 | unsigned long pgoff) |
| 2223 | { |
HATAYAMA Daisuke | e69e9d4a | 2013-07-03 15:02:18 -0700 | [diff] [blame] | 2224 | return remap_vmalloc_range_partial(vma, vma->vm_start, |
| 2225 | addr + (pgoff << PAGE_SHIFT), |
| 2226 | vma->vm_end - vma->vm_start); |
Nick Piggin | 8334231 | 2006-06-23 02:03:20 -0700 | [diff] [blame] | 2227 | } |
| 2228 | EXPORT_SYMBOL(remap_vmalloc_range); |
| 2229 | |
Christoph Hellwig | 1eeb66a | 2007-05-08 00:27:03 -0700 | [diff] [blame] | 2230 | /* |
| 2231 | * Implement a stub for vmalloc_sync_all() if the architecture chose not to |
| 2232 | * have one. |
| 2233 | */ |
Gideon Israel Dsouza | 3b32123 | 2014-04-07 15:37:26 -0700 | [diff] [blame] | 2234 | void __weak vmalloc_sync_all(void) |
Christoph Hellwig | 1eeb66a | 2007-05-08 00:27:03 -0700 | [diff] [blame] | 2235 | { |
| 2236 | } |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2237 | |
| 2238 | |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 2239 | static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data) |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2240 | { |
David Vrabel | cd12909 | 2011-09-29 16:53:32 +0100 | [diff] [blame] | 2241 | pte_t ***p = data; |
| 2242 | |
| 2243 | if (p) { |
| 2244 | *(*p) = pte; |
| 2245 | (*p)++; |
| 2246 | } |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2247 | return 0; |
| 2248 | } |
| 2249 | |
| 2250 | /** |
| 2251 | * alloc_vm_area - allocate a range of kernel address space |
| 2252 | * @size: size of the area |
David Vrabel | cd12909 | 2011-09-29 16:53:32 +0100 | [diff] [blame] | 2253 | * @ptes: returns the PTEs for the address space |
Randy Dunlap | 7682486 | 2008-03-19 17:00:40 -0700 | [diff] [blame] | 2254 | * |
| 2255 | * Returns: NULL on failure, vm_struct on success |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2256 | * |
| 2257 | * This function reserves a range of kernel address space, and |
| 2258 | * allocates pagetables to map that range. No actual mappings |
David Vrabel | cd12909 | 2011-09-29 16:53:32 +0100 | [diff] [blame] | 2259 | * are created. |
| 2260 | * |
| 2261 | * If @ptes is non-NULL, pointers to the PTEs (in init_mm) |
| 2262 | * allocated for the VM area are returned. |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2263 | */ |
David Vrabel | cd12909 | 2011-09-29 16:53:32 +0100 | [diff] [blame] | 2264 | struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2265 | { |
| 2266 | struct vm_struct *area; |
| 2267 | |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 2268 | area = get_vm_area_caller(size, VM_IOREMAP, |
| 2269 | __builtin_return_address(0)); |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2270 | if (area == NULL) |
| 2271 | return NULL; |
| 2272 | |
| 2273 | /* |
| 2274 | * This ensures that page tables are constructed for this region |
| 2275 | * of kernel virtual address space and mapped into init_mm. |
| 2276 | */ |
| 2277 | if (apply_to_page_range(&init_mm, (unsigned long)area->addr, |
David Vrabel | cd12909 | 2011-09-29 16:53:32 +0100 | [diff] [blame] | 2278 | size, f, ptes ? &ptes : NULL)) { |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2279 | free_vm_area(area); |
| 2280 | return NULL; |
| 2281 | } |
| 2282 | |
Jeremy Fitzhardinge | 5f4352f | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 2283 | return area; |
| 2284 | } |
| 2285 | EXPORT_SYMBOL_GPL(alloc_vm_area); |
| 2286 | |
| 2287 | void free_vm_area(struct vm_struct *area) |
| 2288 | { |
| 2289 | struct vm_struct *ret; |
| 2290 | ret = remove_vm_area(area->addr); |
| 2291 | BUG_ON(ret != area); |
| 2292 | kfree(area); |
| 2293 | } |
| 2294 | EXPORT_SYMBOL_GPL(free_vm_area); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2295 | |
Tejun Heo | 4f8b02b | 2010-09-03 18:22:47 +0200 | [diff] [blame] | 2296 | #ifdef CONFIG_SMP |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2297 | static struct vmap_area *node_to_va(struct rb_node *n) |
| 2298 | { |
| 2299 | return n ? rb_entry(n, struct vmap_area, rb_node) : NULL; |
| 2300 | } |
| 2301 | |
| 2302 | /** |
| 2303 | * pvm_find_next_prev - find the next and prev vmap_area surrounding @end |
| 2304 | * @end: target address |
| 2305 | * @pnext: out arg for the next vmap_area |
| 2306 | * @pprev: out arg for the previous vmap_area |
| 2307 | * |
| 2308 | * Returns: %true if either or both of next and prev are found, |
| 2309 | * %false if no vmap_area exists |
| 2310 | * |
| 2311 | * Find vmap_areas end addresses of which enclose @end. ie. if not |
| 2312 | * NULL, *pnext->va_end > @end and *pprev->va_end <= @end. |
| 2313 | */ |
| 2314 | static bool pvm_find_next_prev(unsigned long end, |
| 2315 | struct vmap_area **pnext, |
| 2316 | struct vmap_area **pprev) |
| 2317 | { |
| 2318 | struct rb_node *n = vmap_area_root.rb_node; |
| 2319 | struct vmap_area *va = NULL; |
| 2320 | |
| 2321 | while (n) { |
| 2322 | va = rb_entry(n, struct vmap_area, rb_node); |
| 2323 | if (end < va->va_end) |
| 2324 | n = n->rb_left; |
| 2325 | else if (end > va->va_end) |
| 2326 | n = n->rb_right; |
| 2327 | else |
| 2328 | break; |
| 2329 | } |
| 2330 | |
| 2331 | if (!va) |
| 2332 | return false; |
| 2333 | |
| 2334 | if (va->va_end > end) { |
| 2335 | *pnext = va; |
| 2336 | *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); |
| 2337 | } else { |
| 2338 | *pprev = va; |
| 2339 | *pnext = node_to_va(rb_next(&(*pprev)->rb_node)); |
| 2340 | } |
| 2341 | return true; |
| 2342 | } |
| 2343 | |
| 2344 | /** |
| 2345 | * pvm_determine_end - find the highest aligned address between two vmap_areas |
| 2346 | * @pnext: in/out arg for the next vmap_area |
| 2347 | * @pprev: in/out arg for the previous vmap_area |
| 2348 | * @align: alignment |
| 2349 | * |
| 2350 | * Returns: determined end address |
| 2351 | * |
| 2352 | * Find the highest aligned address between *@pnext and *@pprev below |
| 2353 | * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned |
| 2354 | * down address is between the end addresses of the two vmap_areas. |
| 2355 | * |
| 2356 | * Please note that the address returned by this function may fall |
| 2357 | * inside *@pnext vmap_area. The caller is responsible for checking |
| 2358 | * that. |
| 2359 | */ |
| 2360 | static unsigned long pvm_determine_end(struct vmap_area **pnext, |
| 2361 | struct vmap_area **pprev, |
| 2362 | unsigned long align) |
| 2363 | { |
| 2364 | const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); |
| 2365 | unsigned long addr; |
| 2366 | |
| 2367 | if (*pnext) |
| 2368 | addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end); |
| 2369 | else |
| 2370 | addr = vmalloc_end; |
| 2371 | |
| 2372 | while (*pprev && (*pprev)->va_end > addr) { |
| 2373 | *pnext = *pprev; |
| 2374 | *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); |
| 2375 | } |
| 2376 | |
| 2377 | return addr; |
| 2378 | } |
| 2379 | |
| 2380 | /** |
| 2381 | * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator |
| 2382 | * @offsets: array containing offset of each area |
| 2383 | * @sizes: array containing size of each area |
| 2384 | * @nr_vms: the number of areas to allocate |
| 2385 | * @align: alignment, all entries in @offsets and @sizes must be aligned to this |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2386 | * |
| 2387 | * Returns: kmalloc'd vm_struct pointer array pointing to allocated |
| 2388 | * vm_structs on success, %NULL on failure |
| 2389 | * |
| 2390 | * Percpu allocator wants to use congruent vm areas so that it can |
| 2391 | * maintain the offsets among percpu areas. This function allocates |
David Rientjes | ec3f64f | 2011-01-13 15:46:01 -0800 | [diff] [blame] | 2392 | * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to |
| 2393 | * be scattered pretty far, distance between two areas easily going up |
| 2394 | * to gigabytes. To avoid interacting with regular vmallocs, these |
| 2395 | * areas are allocated from top. |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2396 | * |
| 2397 | * Despite its complicated look, this allocator is rather simple. It |
| 2398 | * does everything top-down and scans areas from the end looking for |
| 2399 | * matching slot. While scanning, if any of the areas overlaps with |
| 2400 | * existing vmap_area, the base address is pulled down to fit the |
| 2401 | * area. Scanning is repeated till all the areas fit and then all |
| 2402 | * necessary data structres are inserted and the result is returned. |
| 2403 | */ |
| 2404 | struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, |
| 2405 | const size_t *sizes, int nr_vms, |
David Rientjes | ec3f64f | 2011-01-13 15:46:01 -0800 | [diff] [blame] | 2406 | size_t align) |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2407 | { |
| 2408 | const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); |
| 2409 | const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); |
| 2410 | struct vmap_area **vas, *prev, *next; |
| 2411 | struct vm_struct **vms; |
| 2412 | int area, area2, last_area, term_area; |
| 2413 | unsigned long base, start, end, last_end; |
| 2414 | bool purged = false; |
| 2415 | |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2416 | /* verify parameters and allocate data structures */ |
Alexander Kuleshov | 891c49a | 2015-11-05 18:46:51 -0800 | [diff] [blame] | 2417 | BUG_ON(offset_in_page(align) || !is_power_of_2(align)); |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2418 | for (last_area = 0, area = 0; area < nr_vms; area++) { |
| 2419 | start = offsets[area]; |
| 2420 | end = start + sizes[area]; |
| 2421 | |
| 2422 | /* is everything aligned properly? */ |
| 2423 | BUG_ON(!IS_ALIGNED(offsets[area], align)); |
| 2424 | BUG_ON(!IS_ALIGNED(sizes[area], align)); |
| 2425 | |
| 2426 | /* detect the area with the highest address */ |
| 2427 | if (start > offsets[last_area]) |
| 2428 | last_area = area; |
| 2429 | |
| 2430 | for (area2 = 0; area2 < nr_vms; area2++) { |
| 2431 | unsigned long start2 = offsets[area2]; |
| 2432 | unsigned long end2 = start2 + sizes[area2]; |
| 2433 | |
| 2434 | if (area2 == area) |
| 2435 | continue; |
| 2436 | |
| 2437 | BUG_ON(start2 >= start && start2 < end); |
| 2438 | BUG_ON(end2 <= end && end2 > start); |
| 2439 | } |
| 2440 | } |
| 2441 | last_end = offsets[last_area] + sizes[last_area]; |
| 2442 | |
| 2443 | if (vmalloc_end - vmalloc_start < last_end) { |
| 2444 | WARN_ON(true); |
| 2445 | return NULL; |
| 2446 | } |
| 2447 | |
Thomas Meyer | 4d67d86 | 2012-05-29 15:06:21 -0700 | [diff] [blame] | 2448 | vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); |
| 2449 | vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2450 | if (!vas || !vms) |
Kautuk Consul | f1db7af | 2012-01-12 17:20:08 -0800 | [diff] [blame] | 2451 | goto err_free2; |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2452 | |
| 2453 | for (area = 0; area < nr_vms; area++) { |
David Rientjes | ec3f64f | 2011-01-13 15:46:01 -0800 | [diff] [blame] | 2454 | vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL); |
| 2455 | vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2456 | if (!vas[area] || !vms[area]) |
| 2457 | goto err_free; |
| 2458 | } |
| 2459 | retry: |
| 2460 | spin_lock(&vmap_area_lock); |
| 2461 | |
| 2462 | /* start scanning - we scan from the top, begin with the last area */ |
| 2463 | area = term_area = last_area; |
| 2464 | start = offsets[area]; |
| 2465 | end = start + sizes[area]; |
| 2466 | |
| 2467 | if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) { |
| 2468 | base = vmalloc_end - last_end; |
| 2469 | goto found; |
| 2470 | } |
| 2471 | base = pvm_determine_end(&next, &prev, align) - end; |
| 2472 | |
| 2473 | while (true) { |
| 2474 | BUG_ON(next && next->va_end <= base + end); |
| 2475 | BUG_ON(prev && prev->va_end > base + end); |
| 2476 | |
| 2477 | /* |
| 2478 | * base might have underflowed, add last_end before |
| 2479 | * comparing. |
| 2480 | */ |
| 2481 | if (base + last_end < vmalloc_start + last_end) { |
| 2482 | spin_unlock(&vmap_area_lock); |
| 2483 | if (!purged) { |
| 2484 | purge_vmap_area_lazy(); |
| 2485 | purged = true; |
| 2486 | goto retry; |
| 2487 | } |
| 2488 | goto err_free; |
| 2489 | } |
| 2490 | |
| 2491 | /* |
| 2492 | * If next overlaps, move base downwards so that it's |
| 2493 | * right below next and then recheck. |
| 2494 | */ |
| 2495 | if (next && next->va_start < base + end) { |
| 2496 | base = pvm_determine_end(&next, &prev, align) - end; |
| 2497 | term_area = area; |
| 2498 | continue; |
| 2499 | } |
| 2500 | |
| 2501 | /* |
| 2502 | * If prev overlaps, shift down next and prev and move |
| 2503 | * base so that it's right below new next and then |
| 2504 | * recheck. |
| 2505 | */ |
| 2506 | if (prev && prev->va_end > base + start) { |
| 2507 | next = prev; |
| 2508 | prev = node_to_va(rb_prev(&next->rb_node)); |
| 2509 | base = pvm_determine_end(&next, &prev, align) - end; |
| 2510 | term_area = area; |
| 2511 | continue; |
| 2512 | } |
| 2513 | |
| 2514 | /* |
| 2515 | * This area fits, move on to the previous one. If |
| 2516 | * the previous one is the terminal one, we're done. |
| 2517 | */ |
| 2518 | area = (area + nr_vms - 1) % nr_vms; |
| 2519 | if (area == term_area) |
| 2520 | break; |
| 2521 | start = offsets[area]; |
| 2522 | end = start + sizes[area]; |
| 2523 | pvm_find_next_prev(base + end, &next, &prev); |
| 2524 | } |
| 2525 | found: |
| 2526 | /* we've found a fitting base, insert all va's */ |
| 2527 | for (area = 0; area < nr_vms; area++) { |
| 2528 | struct vmap_area *va = vas[area]; |
| 2529 | |
| 2530 | va->va_start = base + offsets[area]; |
| 2531 | va->va_end = va->va_start + sizes[area]; |
| 2532 | __insert_vmap_area(va); |
| 2533 | } |
| 2534 | |
| 2535 | vmap_area_pcpu_hole = base + offsets[last_area]; |
| 2536 | |
| 2537 | spin_unlock(&vmap_area_lock); |
| 2538 | |
| 2539 | /* insert all vm's */ |
| 2540 | for (area = 0; area < nr_vms; area++) |
Zhang Yanfei | 3645cb4 | 2013-07-03 15:04:48 -0700 | [diff] [blame] | 2541 | setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC, |
| 2542 | pcpu_get_vm_areas); |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2543 | |
| 2544 | kfree(vas); |
| 2545 | return vms; |
| 2546 | |
| 2547 | err_free: |
| 2548 | for (area = 0; area < nr_vms; area++) { |
Kautuk Consul | f1db7af | 2012-01-12 17:20:08 -0800 | [diff] [blame] | 2549 | kfree(vas[area]); |
| 2550 | kfree(vms[area]); |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2551 | } |
Kautuk Consul | f1db7af | 2012-01-12 17:20:08 -0800 | [diff] [blame] | 2552 | err_free2: |
Tejun Heo | ca23e40 | 2009-08-14 15:00:52 +0900 | [diff] [blame] | 2553 | kfree(vas); |
| 2554 | kfree(vms); |
| 2555 | return NULL; |
| 2556 | } |
| 2557 | |
| 2558 | /** |
| 2559 | * pcpu_free_vm_areas - free vmalloc areas for percpu allocator |
| 2560 | * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() |
| 2561 | * @nr_vms: the number of allocated areas |
| 2562 | * |
| 2563 | * Free vm_structs and the array allocated by pcpu_get_vm_areas(). |
| 2564 | */ |
| 2565 | void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) |
| 2566 | { |
| 2567 | int i; |
| 2568 | |
| 2569 | for (i = 0; i < nr_vms; i++) |
| 2570 | free_vm_area(vms[i]); |
| 2571 | kfree(vms); |
| 2572 | } |
Tejun Heo | 4f8b02b | 2010-09-03 18:22:47 +0200 | [diff] [blame] | 2573 | #endif /* CONFIG_SMP */ |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2574 | |
| 2575 | #ifdef CONFIG_PROC_FS |
| 2576 | static void *s_start(struct seq_file *m, loff_t *pos) |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2577 | __acquires(&vmap_area_lock) |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2578 | { |
| 2579 | loff_t n = *pos; |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2580 | struct vmap_area *va; |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2581 | |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2582 | spin_lock(&vmap_area_lock); |
Geliang Tang | 6219c2a | 2016-01-14 15:19:08 -0800 | [diff] [blame] | 2583 | va = list_first_entry(&vmap_area_list, typeof(*va), list); |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2584 | while (n > 0 && &va->list != &vmap_area_list) { |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2585 | n--; |
Geliang Tang | 6219c2a | 2016-01-14 15:19:08 -0800 | [diff] [blame] | 2586 | va = list_next_entry(va, list); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2587 | } |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2588 | if (!n && &va->list != &vmap_area_list) |
| 2589 | return va; |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2590 | |
| 2591 | return NULL; |
| 2592 | |
| 2593 | } |
| 2594 | |
| 2595 | static void *s_next(struct seq_file *m, void *p, loff_t *pos) |
| 2596 | { |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2597 | struct vmap_area *va = p, *next; |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2598 | |
| 2599 | ++*pos; |
Geliang Tang | 6219c2a | 2016-01-14 15:19:08 -0800 | [diff] [blame] | 2600 | next = list_next_entry(va, list); |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2601 | if (&next->list != &vmap_area_list) |
| 2602 | return next; |
| 2603 | |
| 2604 | return NULL; |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2605 | } |
| 2606 | |
| 2607 | static void s_stop(struct seq_file *m, void *p) |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2608 | __releases(&vmap_area_lock) |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2609 | { |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2610 | spin_unlock(&vmap_area_lock); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2611 | } |
| 2612 | |
Eric Dumazet | a47a126 | 2008-07-23 21:27:38 -0700 | [diff] [blame] | 2613 | static void show_numa_info(struct seq_file *m, struct vm_struct *v) |
| 2614 | { |
Kirill A. Shutemov | e5adfff | 2012-12-11 16:00:29 -0800 | [diff] [blame] | 2615 | if (IS_ENABLED(CONFIG_NUMA)) { |
Eric Dumazet | a47a126 | 2008-07-23 21:27:38 -0700 | [diff] [blame] | 2616 | unsigned int nr, *counters = m->private; |
| 2617 | |
| 2618 | if (!counters) |
| 2619 | return; |
| 2620 | |
Wanpeng Li | af12346 | 2013-11-12 15:07:32 -0800 | [diff] [blame] | 2621 | if (v->flags & VM_UNINITIALIZED) |
| 2622 | return; |
Dmitry Vyukov | 7e5b528 | 2014-12-12 16:56:30 -0800 | [diff] [blame] | 2623 | /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ |
| 2624 | smp_rmb(); |
Wanpeng Li | af12346 | 2013-11-12 15:07:32 -0800 | [diff] [blame] | 2625 | |
Eric Dumazet | a47a126 | 2008-07-23 21:27:38 -0700 | [diff] [blame] | 2626 | memset(counters, 0, nr_node_ids * sizeof(unsigned int)); |
| 2627 | |
| 2628 | for (nr = 0; nr < v->nr_pages; nr++) |
| 2629 | counters[page_to_nid(v->pages[nr])]++; |
| 2630 | |
| 2631 | for_each_node_state(nr, N_HIGH_MEMORY) |
| 2632 | if (counters[nr]) |
| 2633 | seq_printf(m, " N%u=%u", nr, counters[nr]); |
| 2634 | } |
| 2635 | } |
| 2636 | |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2637 | static int s_show(struct seq_file *m, void *p) |
| 2638 | { |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2639 | struct vmap_area *va = p; |
| 2640 | struct vm_struct *v; |
| 2641 | |
Wanpeng Li | c2ce8c1 | 2013-11-12 15:07:31 -0800 | [diff] [blame] | 2642 | /* |
| 2643 | * s_show can encounter race with remove_vm_area, !VM_VM_AREA on |
| 2644 | * behalf of vmap area is being tear down or vm_map_ram allocation. |
| 2645 | */ |
| 2646 | if (!(va->flags & VM_VM_AREA)) |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2647 | return 0; |
| 2648 | |
Joonsoo Kim | d4033af | 2013-04-29 15:07:35 -0700 | [diff] [blame] | 2649 | v = va->vm; |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2650 | |
Kees Cook | 45ec169 | 2012-10-08 16:34:09 -0700 | [diff] [blame] | 2651 | seq_printf(m, "0x%pK-0x%pK %7ld", |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2652 | v->addr, v->addr + v->size, v->size); |
| 2653 | |
Joe Perches | 62c70bc | 2011-01-13 15:45:52 -0800 | [diff] [blame] | 2654 | if (v->caller) |
| 2655 | seq_printf(m, " %pS", v->caller); |
Christoph Lameter | 2301696 | 2008-04-28 02:12:42 -0700 | [diff] [blame] | 2656 | |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2657 | if (v->nr_pages) |
| 2658 | seq_printf(m, " pages=%d", v->nr_pages); |
| 2659 | |
| 2660 | if (v->phys_addr) |
Kenji Kaneshige | ffa71f3 | 2010-06-18 12:22:40 +0900 | [diff] [blame] | 2661 | seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2662 | |
| 2663 | if (v->flags & VM_IOREMAP) |
Fabian Frederick | f4527c9 | 2014-06-04 16:08:09 -0700 | [diff] [blame] | 2664 | seq_puts(m, " ioremap"); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2665 | |
| 2666 | if (v->flags & VM_ALLOC) |
Fabian Frederick | f4527c9 | 2014-06-04 16:08:09 -0700 | [diff] [blame] | 2667 | seq_puts(m, " vmalloc"); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2668 | |
| 2669 | if (v->flags & VM_MAP) |
Fabian Frederick | f4527c9 | 2014-06-04 16:08:09 -0700 | [diff] [blame] | 2670 | seq_puts(m, " vmap"); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2671 | |
| 2672 | if (v->flags & VM_USERMAP) |
Fabian Frederick | f4527c9 | 2014-06-04 16:08:09 -0700 | [diff] [blame] | 2673 | seq_puts(m, " user"); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2674 | |
David Rientjes | 244d63e | 2016-01-14 15:19:35 -0800 | [diff] [blame] | 2675 | if (is_vmalloc_addr(v->pages)) |
Fabian Frederick | f4527c9 | 2014-06-04 16:08:09 -0700 | [diff] [blame] | 2676 | seq_puts(m, " vpages"); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2677 | |
Eric Dumazet | a47a126 | 2008-07-23 21:27:38 -0700 | [diff] [blame] | 2678 | show_numa_info(m, v); |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2679 | seq_putc(m, '\n'); |
| 2680 | return 0; |
| 2681 | } |
| 2682 | |
Alexey Dobriyan | 5f6a6a9 | 2008-10-06 03:50:47 +0400 | [diff] [blame] | 2683 | static const struct seq_operations vmalloc_op = { |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2684 | .start = s_start, |
| 2685 | .next = s_next, |
| 2686 | .stop = s_stop, |
| 2687 | .show = s_show, |
| 2688 | }; |
Alexey Dobriyan | 5f6a6a9 | 2008-10-06 03:50:47 +0400 | [diff] [blame] | 2689 | |
| 2690 | static int vmalloc_open(struct inode *inode, struct file *file) |
| 2691 | { |
Rob Jones | 703394c | 2014-10-09 15:28:01 -0700 | [diff] [blame] | 2692 | if (IS_ENABLED(CONFIG_NUMA)) |
| 2693 | return seq_open_private(file, &vmalloc_op, |
| 2694 | nr_node_ids * sizeof(unsigned int)); |
| 2695 | else |
| 2696 | return seq_open(file, &vmalloc_op); |
Alexey Dobriyan | 5f6a6a9 | 2008-10-06 03:50:47 +0400 | [diff] [blame] | 2697 | } |
| 2698 | |
| 2699 | static const struct file_operations proc_vmalloc_operations = { |
| 2700 | .open = vmalloc_open, |
| 2701 | .read = seq_read, |
| 2702 | .llseek = seq_lseek, |
| 2703 | .release = seq_release_private, |
| 2704 | }; |
| 2705 | |
| 2706 | static int __init proc_vmalloc_init(void) |
| 2707 | { |
| 2708 | proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations); |
| 2709 | return 0; |
| 2710 | } |
| 2711 | module_init(proc_vmalloc_init); |
Joonsoo Kim | db3808c | 2013-04-29 15:07:28 -0700 | [diff] [blame] | 2712 | |
Christoph Lameter | a10aa57 | 2008-04-28 02:12:40 -0700 | [diff] [blame] | 2713 | #endif |
| 2714 | |