Thomas Gleixner | 2874c5f | 2019-05-27 08:55:01 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 2 | /* |
| 3 | * PowerPC version |
| 4 | * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) |
| 5 | * |
| 6 | * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) |
| 7 | * and Cort Dougan (PReP) (cort@cs.nmt.edu) |
| 8 | * Copyright (C) 1996 Paul Mackerras |
| 9 | * |
| 10 | * Derived from "arch/i386/mm/init.c" |
| 11 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds |
| 12 | * |
| 13 | * Dave Engebretsen <engebret@us.ibm.com> |
| 14 | * Rework for PPC64 port. |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 15 | */ |
| 16 | |
| 17 | #undef DEBUG |
| 18 | |
| 19 | #include <linux/string.h> |
Mike Rapoport | ca5999f | 2020-06-08 21:32:38 -0700 | [diff] [blame] | 20 | #include <linux/pgtable.h> |
Mike Rapoport | 65fddcf | 2020-06-08 21:32:42 -0700 | [diff] [blame] | 21 | #include <asm/pgalloc.h> |
Christophe Leroy | 69795ca | 2019-04-18 16:51:18 +1000 | [diff] [blame] | 22 | #include <asm/kup.h> |
Christophe Leroy | 6754862 | 2021-10-19 09:29:19 +0200 | [diff] [blame] | 23 | #include <asm/smp.h> |
Christophe Leroy | 69795ca | 2019-04-18 16:51:18 +1000 | [diff] [blame] | 24 | |
Jason Yan | 4ed47db | 2019-09-20 17:45:36 +0800 | [diff] [blame] | 25 | phys_addr_t memstart_addr __ro_after_init = (phys_addr_t)~0ull; |
| 26 | EXPORT_SYMBOL_GPL(memstart_addr); |
| 27 | phys_addr_t kernstart_addr __ro_after_init; |
| 28 | EXPORT_SYMBOL_GPL(kernstart_addr); |
Jason Yan | 39f4b7b | 2019-09-20 17:45:37 +0800 | [diff] [blame] | 29 | unsigned long kernstart_virt_addr __ro_after_init = KERNELBASE; |
| 30 | EXPORT_SYMBOL_GPL(kernstart_virt_addr); |
Jason Yan | 4ed47db | 2019-09-20 17:45:36 +0800 | [diff] [blame] | 31 | |
Aneesh Kumar K.V | 61130e2 | 2020-12-02 10:08:54 +0530 | [diff] [blame] | 32 | bool disable_kuep = !IS_ENABLED(CONFIG_PPC_KUEP); |
| 33 | bool disable_kuap = !IS_ENABLED(CONFIG_PPC_KUAP); |
Hari Bathini | 353d7a8 | 2024-07-01 18:30:21 +0530 | [diff] [blame] | 34 | #ifdef CONFIG_KFENCE |
| 35 | bool __ro_after_init kfence_disabled; |
| 36 | #endif |
Christophe Leroy | 0fb1c25 | 2019-04-18 16:51:19 +1000 | [diff] [blame] | 37 | |
| 38 | static int __init parse_nosmep(char *p) |
| 39 | { |
Christophe Leroy | 6754862 | 2021-10-19 09:29:19 +0200 | [diff] [blame] | 40 | if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64)) |
| 41 | return 0; |
| 42 | |
Christophe Leroy | 0fb1c25 | 2019-04-18 16:51:19 +1000 | [diff] [blame] | 43 | disable_kuep = true; |
| 44 | pr_warn("Disabling Kernel Userspace Execution Prevention\n"); |
| 45 | return 0; |
| 46 | } |
| 47 | early_param("nosmep", parse_nosmep); |
| 48 | |
Christophe Leroy | de78a9c | 2019-04-18 16:51:20 +1000 | [diff] [blame] | 49 | static int __init parse_nosmap(char *p) |
| 50 | { |
| 51 | disable_kuap = true; |
| 52 | pr_warn("Disabling Kernel Userspace Access Protection\n"); |
| 53 | return 0; |
| 54 | } |
| 55 | early_param("nosmap", parse_nosmap); |
| 56 | |
Christophe Leroy | 6754862 | 2021-10-19 09:29:19 +0200 | [diff] [blame] | 57 | void __weak setup_kuep(bool disabled) |
| 58 | { |
| 59 | if (!IS_ENABLED(CONFIG_PPC_KUEP) || disabled) |
| 60 | return; |
| 61 | |
| 62 | if (smp_processor_id() != boot_cpuid) |
| 63 | return; |
| 64 | |
| 65 | pr_info("Activating Kernel Userspace Execution Prevention\n"); |
| 66 | } |
| 67 | |
Christophe Leroy | 6c1fa60 | 2021-10-19 09:29:12 +0200 | [diff] [blame] | 68 | void setup_kup(void) |
| 69 | { |
| 70 | setup_kuap(disable_kuap); |
| 71 | setup_kuep(disable_kuep); |
| 72 | } |
| 73 | |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 74 | #define CTOR(shift) static void ctor_##shift(void *addr) \ |
| 75 | { \ |
Christophe Leroy | e7a9af8 | 2024-08-08 11:32:47 +0200 | [diff] [blame] | 76 | memset(addr, 0, sizeof(pgd_t) << (shift)); \ |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 77 | } |
| 78 | |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 79 | CTOR(0); CTOR(1); CTOR(2); CTOR(3); CTOR(4); CTOR(5); CTOR(6); CTOR(7); |
| 80 | CTOR(8); CTOR(9); CTOR(10); CTOR(11); CTOR(12); CTOR(13); CTOR(14); CTOR(15); |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 81 | |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 82 | static inline void (*ctor(int shift))(void *) |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 83 | { |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 84 | BUILD_BUG_ON(MAX_PGTABLE_INDEX_SIZE != 15); |
| 85 | |
| 86 | switch (shift) { |
| 87 | case 0: return ctor_0; |
| 88 | case 1: return ctor_1; |
| 89 | case 2: return ctor_2; |
| 90 | case 3: return ctor_3; |
| 91 | case 4: return ctor_4; |
| 92 | case 5: return ctor_5; |
| 93 | case 6: return ctor_6; |
| 94 | case 7: return ctor_7; |
| 95 | case 8: return ctor_8; |
| 96 | case 9: return ctor_9; |
| 97 | case 10: return ctor_10; |
| 98 | case 11: return ctor_11; |
| 99 | case 12: return ctor_12; |
| 100 | case 13: return ctor_13; |
| 101 | case 14: return ctor_14; |
| 102 | case 15: return ctor_15; |
| 103 | } |
| 104 | return NULL; |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 105 | } |
| 106 | |
Christophe Leroy | 129dd32 | 2018-11-29 14:07:03 +0000 | [diff] [blame] | 107 | struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE + 1]; |
Paul Mackerras | ba9b399 | 2017-01-30 21:21:38 +1100 | [diff] [blame] | 108 | EXPORT_SYMBOL_GPL(pgtable_cache); /* used by kvm_hv module */ |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 109 | |
| 110 | /* |
| 111 | * Create a kmem_cache() for pagetables. This is not used for PTE |
| 112 | * pages - they're linked to struct page, come from the normal free |
| 113 | * pages pool and have a different entry size (see real_pte_t) to |
| 114 | * everything else. Caches created by this function are used for all |
| 115 | * the higher level pagetables, and for hugepage pagetables. |
| 116 | */ |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 117 | void pgtable_cache_add(unsigned int shift) |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 118 | { |
| 119 | char *name; |
Christophe Leroy | e7a9af8 | 2024-08-08 11:32:47 +0200 | [diff] [blame] | 120 | unsigned long table_size = sizeof(pgd_t) << shift; |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 121 | unsigned long align = table_size; |
| 122 | |
| 123 | /* When batching pgtable pointers for RCU freeing, we store |
| 124 | * the index size in the low bits. Table alignment must be |
| 125 | * big enough to fit it. |
Christophe Leroy | 0c22e4b | 2024-07-02 15:51:34 +0200 | [diff] [blame] | 126 | */ |
| 127 | unsigned long minalign = MAX_PGTABLE_INDEX_SIZE + 1; |
Kunwu Chan | f46c8a7 | 2023-12-04 10:32:23 +0800 | [diff] [blame] | 128 | struct kmem_cache *new = NULL; |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 129 | |
| 130 | /* It would be nice if this was a BUILD_BUG_ON(), but at the |
| 131 | * moment, gcc doesn't seem to recognize is_power_of_2 as a |
| 132 | * constant expression, so so much for that. */ |
| 133 | BUG_ON(!is_power_of_2(minalign)); |
Christophe Leroy | 129dd32 | 2018-11-29 14:07:03 +0000 | [diff] [blame] | 134 | BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 135 | |
| 136 | if (PGT_CACHE(shift)) |
| 137 | return; /* Already have a cache of this size */ |
| 138 | |
| 139 | align = max_t(unsigned long, align, minalign); |
| 140 | name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift); |
Kunwu Chan | f46c8a7 | 2023-12-04 10:32:23 +0800 | [diff] [blame] | 141 | if (name) |
| 142 | new = kmem_cache_create(name, table_size, align, 0, ctor(shift)); |
Nicholas Piggin | bf5ca68 | 2017-01-04 01:55:17 +1000 | [diff] [blame] | 143 | if (!new) |
| 144 | panic("Could not allocate pgtable cache for order %d", shift); |
| 145 | |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 146 | kfree(name); |
Christophe Leroy | 129dd32 | 2018-11-29 14:07:03 +0000 | [diff] [blame] | 147 | pgtable_cache[shift] = new; |
Nicholas Piggin | bf5ca68 | 2017-01-04 01:55:17 +1000 | [diff] [blame] | 148 | |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 149 | pr_debug("Allocated pgtable cache for order %d\n", shift); |
| 150 | } |
Paul Mackerras | ba9b399 | 2017-01-30 21:21:38 +1100 | [diff] [blame] | 151 | EXPORT_SYMBOL_GPL(pgtable_cache_add); /* used by kvm_hv module */ |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 152 | |
| 153 | void pgtable_cache_init(void) |
| 154 | { |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 155 | pgtable_cache_add(PGD_INDEX_SIZE); |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 156 | |
Christophe Leroy | 32bff4b | 2018-11-29 14:07:09 +0000 | [diff] [blame] | 157 | if (PMD_CACHE_INDEX) |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 158 | pgtable_cache_add(PMD_CACHE_INDEX); |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 159 | /* |
| 160 | * In all current configs, when the PUD index exists it's the |
| 161 | * same size as either the pgd or pmd index except with THP enabled |
| 162 | * on book3s 64 |
| 163 | */ |
Christophe Leroy | 32bff4b | 2018-11-29 14:07:09 +0000 | [diff] [blame] | 164 | if (PUD_CACHE_INDEX) |
Christophe Leroy | 1e03c7e2 | 2018-11-29 14:07:07 +0000 | [diff] [blame] | 165 | pgtable_cache_add(PUD_CACHE_INDEX); |
Christophe Leroy | 9b081e1 | 2016-12-07 08:47:24 +0100 | [diff] [blame] | 166 | } |