Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Copyright (C) 1994 - 2001, 2003 by Ralf Baechle |
| 7 | * Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc. |
| 8 | */ |
| 9 | #ifndef _ASM_PGALLOC_H |
| 10 | #define _ASM_PGALLOC_H |
| 11 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/highmem.h> |
| 13 | #include <linux/mm.h> |
Alexey Dobriyan | e8edc6e | 2007-05-21 01:22:52 +0400 | [diff] [blame] | 14 | #include <linux/sched.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | |
Mike Rapoport | 1355c31 | 2020-08-06 23:22:39 -0700 | [diff] [blame] | 16 | #define __HAVE_ARCH_PMD_ALLOC_ONE |
Mike Rapoport | d9e8b92 | 2020-08-06 23:22:44 -0700 | [diff] [blame] | 17 | #define __HAVE_ARCH_PUD_ALLOC_ONE |
Yaliang Wang | 2bc5bab9 | 2022-03-10 19:31:16 +0800 | [diff] [blame] | 18 | #define __HAVE_ARCH_PGD_FREE |
Mike Rapoport | 1355c31 | 2020-08-06 23:22:39 -0700 | [diff] [blame] | 19 | #include <asm-generic/pgalloc.h> |
Mike Rapoport | b7902ce | 2019-07-11 20:58:14 -0700 | [diff] [blame] | 20 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, |
| 22 | pte_t *pte) |
| 23 | { |
| 24 | set_pmd(pmd, __pmd((unsigned long)pte)); |
| 25 | } |
| 26 | |
| 27 | static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 28 | pgtable_t pte) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | { |
| 30 | set_pmd(pmd, __pmd((unsigned long)page_address(pte))); |
| 31 | } |
| 32 | |
| 33 | /* |
Ralf Baechle | c6e8b58 | 2005-02-10 12:19:59 +0000 | [diff] [blame] | 34 | * Initialize a new pmd table with invalid pointers. |
| 35 | */ |
Feiyang Chen | 22c4e80 | 2022-10-27 20:52:50 +0800 | [diff] [blame] | 36 | extern void pmd_init(void *addr); |
Ralf Baechle | c6e8b58 | 2005-02-10 12:19:59 +0000 | [diff] [blame] | 37 | |
David Daney | 325f8a0 | 2009-12-04 13:52:36 -0800 | [diff] [blame] | 38 | #ifndef __PAGETABLE_PMD_FOLDED |
Ralf Baechle | c6e8b58 | 2005-02-10 12:19:59 +0000 | [diff] [blame] | 39 | |
| 40 | static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) |
| 41 | { |
| 42 | set_pud(pud, __pud((unsigned long)pmd)); |
| 43 | } |
| 44 | #endif |
| 45 | |
| 46 | /* |
Feiyang Chen | 22c4e80 | 2022-10-27 20:52:50 +0800 | [diff] [blame] | 47 | * Initialize a new pgd table with invalid pointers. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | */ |
Feiyang Chen | 22c4e80 | 2022-10-27 20:52:50 +0800 | [diff] [blame] | 49 | extern void pgd_init(void *addr); |
James Hogan | 814f91b | 2017-02-02 01:21:35 +0000 | [diff] [blame] | 50 | extern pgd_t *pgd_alloc(struct mm_struct *mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 51 | |
Yaliang Wang | 2bc5bab9 | 2022-03-10 19:31:16 +0800 | [diff] [blame] | 52 | static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) |
| 53 | { |
Mike Rapoport | bb5af4f | 2022-07-03 17:11:55 +0300 | [diff] [blame] | 54 | free_pages((unsigned long)pgd, PGD_TABLE_ORDER); |
Yaliang Wang | 2bc5bab9 | 2022-03-10 19:31:16 +0800 | [diff] [blame] | 55 | } |
| 56 | |
Benjamin Herrenschmidt | 9e1b32c | 2009-07-22 15:44:28 +1000 | [diff] [blame] | 57 | #define __pte_free_tlb(tlb,pte,address) \ |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 58 | do { \ |
Mark Rutland | b4ed71f | 2019-09-25 16:49:46 -0700 | [diff] [blame] | 59 | pgtable_pte_page_dtor(pte); \ |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 60 | tlb_remove_page((tlb), pte); \ |
| 61 | } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | |
David Daney | 325f8a0 | 2009-12-04 13:52:36 -0800 | [diff] [blame] | 63 | #ifndef __PAGETABLE_PMD_FOLDED |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) |
| 66 | { |
Huang Pei | 6aa3246 | 2021-07-21 17:30:45 +0800 | [diff] [blame] | 67 | pmd_t *pmd; |
Huang Pei | ed914d4 | 2021-06-11 15:09:46 +0800 | [diff] [blame] | 68 | struct page *pg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | |
Matthew Wilcox (Oracle) | c94b14b | 2022-07-03 17:11:52 +0300 | [diff] [blame] | 70 | pg = alloc_pages(GFP_KERNEL_ACCOUNT, PMD_TABLE_ORDER); |
Huang Pei | 6aa3246 | 2021-07-21 17:30:45 +0800 | [diff] [blame] | 71 | if (!pg) |
| 72 | return NULL; |
| 73 | |
| 74 | if (!pgtable_pmd_page_ctor(pg)) { |
Matthew Wilcox (Oracle) | c94b14b | 2022-07-03 17:11:52 +0300 | [diff] [blame] | 75 | __free_pages(pg, PMD_TABLE_ORDER); |
Huang Pei | 6aa3246 | 2021-07-21 17:30:45 +0800 | [diff] [blame] | 76 | return NULL; |
Huang Pei | ed914d4 | 2021-06-11 15:09:46 +0800 | [diff] [blame] | 77 | } |
Huang Pei | 6aa3246 | 2021-07-21 17:30:45 +0800 | [diff] [blame] | 78 | |
| 79 | pmd = (pmd_t *)page_address(pg); |
Feiyang Chen | 22c4e80 | 2022-10-27 20:52:50 +0800 | [diff] [blame] | 80 | pmd_init(pmd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 81 | return pmd; |
| 82 | } |
| 83 | |
Benjamin Herrenschmidt | 9e1b32c | 2009-07-22 15:44:28 +1000 | [diff] [blame] | 84 | #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 85 | |
| 86 | #endif |
| 87 | |
Alex Belits | 3377e22 | 2017-02-16 17:27:34 -0800 | [diff] [blame] | 88 | #ifndef __PAGETABLE_PUD_FOLDED |
| 89 | |
| 90 | static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) |
| 91 | { |
| 92 | pud_t *pud; |
| 93 | |
Mike Rapoport | 8e20a4d | 2022-07-03 17:11:53 +0300 | [diff] [blame] | 94 | pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_TABLE_ORDER); |
Alex Belits | 3377e22 | 2017-02-16 17:27:34 -0800 | [diff] [blame] | 95 | if (pud) |
Feiyang Chen | 22c4e80 | 2022-10-27 20:52:50 +0800 | [diff] [blame] | 96 | pud_init(pud); |
Alex Belits | 3377e22 | 2017-02-16 17:27:34 -0800 | [diff] [blame] | 97 | return pud; |
| 98 | } |
| 99 | |
Mike Rapoport | 2bee1b5 | 2019-11-21 18:21:33 +0200 | [diff] [blame] | 100 | static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) |
Alex Belits | 3377e22 | 2017-02-16 17:27:34 -0800 | [diff] [blame] | 101 | { |
Mike Rapoport | 2bee1b5 | 2019-11-21 18:21:33 +0200 | [diff] [blame] | 102 | set_p4d(p4d, __p4d((unsigned long)pud)); |
Alex Belits | 3377e22 | 2017-02-16 17:27:34 -0800 | [diff] [blame] | 103 | } |
| 104 | |
| 105 | #define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) |
| 106 | |
| 107 | #endif /* __PAGETABLE_PUD_FOLDED */ |
| 108 | |
Atsushi Nemoto | 6a1e552 | 2007-02-19 01:27:34 +0900 | [diff] [blame] | 109 | extern void pagetable_init(void); |
| 110 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 111 | #endif /* _ASM_PGALLOC_H */ |