Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 1 | /* |
| 2 | * MMU context allocation for 64-bit kernels. |
| 3 | * |
| 4 | * Copyright (C) 2004 Anton Blanchard, IBM Corp. <anton@samba.org> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | */ |
| 12 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 13 | #include <linux/sched.h> |
| 14 | #include <linux/kernel.h> |
| 15 | #include <linux/errno.h> |
| 16 | #include <linux/string.h> |
| 17 | #include <linux/types.h> |
| 18 | #include <linux/mm.h> |
| 19 | #include <linux/spinlock.h> |
| 20 | #include <linux/idr.h> |
Paul Gortmaker | 4b16f8e | 2011-07-22 18:24:23 -0400 | [diff] [blame] | 21 | #include <linux/export.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 22 | #include <linux/gfp.h> |
Tseng-Hui (Frank) Lin | 851d2e2 | 2011-05-02 20:43:04 +0000 | [diff] [blame] | 23 | #include <linux/slab.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 24 | |
| 25 | #include <asm/mmu_context.h> |
Aneesh Kumar K.V | 5c1f6ee | 2013-04-28 09:37:33 +0000 | [diff] [blame] | 26 | #include <asm/pgalloc.h> |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 27 | |
| 28 | static DEFINE_SPINLOCK(mmu_context_lock); |
Anton Blanchard | 7317ac8 | 2010-02-07 12:30:12 +0000 | [diff] [blame] | 29 | static DEFINE_IDA(mmu_context_ida); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 30 | |
Michael Ellerman | c1ff840 | 2017-03-29 22:10:45 +1100 | [diff] [blame] | 31 | static int alloc_context_id(int min_id, int max_id) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 32 | { |
Michael Ellerman | c1ff840 | 2017-03-29 22:10:45 +1100 | [diff] [blame] | 33 | int index, err; |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 34 | |
| 35 | again: |
Anton Blanchard | 7317ac8 | 2010-02-07 12:30:12 +0000 | [diff] [blame] | 36 | if (!ida_pre_get(&mmu_context_ida, GFP_KERNEL)) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 37 | return -ENOMEM; |
| 38 | |
| 39 | spin_lock(&mmu_context_lock); |
Michael Ellerman | c1ff840 | 2017-03-29 22:10:45 +1100 | [diff] [blame] | 40 | err = ida_get_new_above(&mmu_context_ida, min_id, &index); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 41 | spin_unlock(&mmu_context_lock); |
| 42 | |
| 43 | if (err == -EAGAIN) |
| 44 | goto again; |
| 45 | else if (err) |
| 46 | return err; |
| 47 | |
Michael Ellerman | c1ff840 | 2017-03-29 22:10:45 +1100 | [diff] [blame] | 48 | if (index > max_id) { |
Sonny Rao | f86c9747 | 2006-06-27 08:46:09 -0400 | [diff] [blame] | 49 | spin_lock(&mmu_context_lock); |
Anton Blanchard | 7317ac8 | 2010-02-07 12:30:12 +0000 | [diff] [blame] | 50 | ida_remove(&mmu_context_ida, index); |
Sonny Rao | f86c9747 | 2006-06-27 08:46:09 -0400 | [diff] [blame] | 51 | spin_unlock(&mmu_context_lock); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 52 | return -ENOMEM; |
| 53 | } |
| 54 | |
Alexander Graf | e85a471 | 2009-11-02 12:02:30 +0000 | [diff] [blame] | 55 | return index; |
| 56 | } |
Michael Ellerman | a336f2f | 2017-03-29 22:00:46 +1100 | [diff] [blame] | 57 | |
Aneesh Kumar K.V | 82228e3 | 2017-03-22 09:07:00 +0530 | [diff] [blame] | 58 | void hash__reserve_context_id(int id) |
| 59 | { |
| 60 | int rc, result = 0; |
| 61 | |
| 62 | do { |
| 63 | if (!ida_pre_get(&mmu_context_ida, GFP_KERNEL)) |
| 64 | break; |
| 65 | |
| 66 | spin_lock(&mmu_context_lock); |
| 67 | rc = ida_get_new_above(&mmu_context_ida, id, &result); |
| 68 | spin_unlock(&mmu_context_lock); |
| 69 | } while (rc == -EAGAIN); |
| 70 | |
| 71 | WARN(result != id, "mmu: Failed to reserve context id %d (rc %d)\n", id, result); |
| 72 | } |
| 73 | |
Michael Ellerman | a336f2f | 2017-03-29 22:00:46 +1100 | [diff] [blame] | 74 | int hash__alloc_context_id(void) |
| 75 | { |
Aneesh Kumar K.V | e6f81a9 | 2017-03-29 17:21:53 +1100 | [diff] [blame] | 76 | unsigned long max; |
| 77 | |
| 78 | if (mmu_has_feature(MMU_FTR_68_BIT_VA)) |
| 79 | max = MAX_USER_CONTEXT; |
| 80 | else |
| 81 | max = MAX_USER_CONTEXT_65BIT_VA; |
| 82 | |
| 83 | return alloc_context_id(MIN_USER_CONTEXT, max); |
Michael Ellerman | a336f2f | 2017-03-29 22:00:46 +1100 | [diff] [blame] | 84 | } |
| 85 | EXPORT_SYMBOL_GPL(hash__alloc_context_id); |
| 86 | |
Michael Ellerman | 760573c | 2017-03-29 22:36:56 +1100 | [diff] [blame] | 87 | static int hash__init_new_context(struct mm_struct *mm) |
Alexander Graf | e85a471 | 2009-11-02 12:02:30 +0000 | [diff] [blame] | 88 | { |
| 89 | int index; |
| 90 | |
Michael Ellerman | c1ff840 | 2017-03-29 22:10:45 +1100 | [diff] [blame] | 91 | index = hash__alloc_context_id(); |
Alexander Graf | e85a471 | 2009-11-02 12:02:30 +0000 | [diff] [blame] | 92 | if (index < 0) |
| 93 | return index; |
| 94 | |
Michael Ellerman | 760573c | 2017-03-29 22:36:56 +1100 | [diff] [blame] | 95 | /* |
Nicholas Piggin | effc1b2 | 2017-11-10 04:27:37 +1100 | [diff] [blame] | 96 | * In the case of exec, use the default limit, |
| 97 | * otherwise inherit it from the mm we are duplicating. |
Aneesh Kumar K.V | 957b778 | 2017-03-22 09:06:58 +0530 | [diff] [blame] | 98 | */ |
Nicholas Piggin | 4722476 | 2017-11-10 04:27:40 +1100 | [diff] [blame] | 99 | if (!mm->context.slb_addr_limit) |
| 100 | mm->context.slb_addr_limit = DEFAULT_MAP_WINDOW_USER64; |
Aneesh Kumar K.V | 957b778 | 2017-03-22 09:06:58 +0530 | [diff] [blame] | 101 | |
| 102 | /* |
Michael Ellerman | 760573c | 2017-03-29 22:36:56 +1100 | [diff] [blame] | 103 | * The old code would re-promote on fork, we don't do that when using |
| 104 | * slices as it could cause problem promoting slices that have been |
| 105 | * forced down to 4K. |
| 106 | * |
| 107 | * For book3s we have MMU_NO_CONTEXT set to be ~0. Hence check |
| 108 | * explicitly against context.id == 0. This ensures that we properly |
| 109 | * initialize context slice details for newly allocated mm's (which will |
| 110 | * have id == 0) and don't alter context slice inherited via fork (which |
| 111 | * will have id != 0). |
| 112 | * |
| 113 | * We should not be calling init_new_context() on init_mm. Hence a |
| 114 | * check against 0 is OK. |
| 115 | */ |
| 116 | if (mm->context.id == 0) |
| 117 | slice_set_user_psize(mm, mmu_virtual_psize); |
Aneesh Kumar K.V | 7e381c0 | 2016-04-29 23:26:02 +1000 | [diff] [blame] | 118 | |
Michael Ellerman | 760573c | 2017-03-29 22:36:56 +1100 | [diff] [blame] | 119 | subpage_prot_init_new_context(mm); |
| 120 | |
| 121 | return index; |
| 122 | } |
| 123 | |
| 124 | static int radix__init_new_context(struct mm_struct *mm) |
| 125 | { |
| 126 | unsigned long rts_field; |
Benjamin Herrenschmidt | a25bd72 | 2017-07-24 14:26:06 +1000 | [diff] [blame] | 127 | int index, max_id; |
Michael Ellerman | 760573c | 2017-03-29 22:36:56 +1100 | [diff] [blame] | 128 | |
Benjamin Herrenschmidt | a25bd72 | 2017-07-24 14:26:06 +1000 | [diff] [blame] | 129 | max_id = (1 << mmu_pid_bits) - 1; |
| 130 | index = alloc_context_id(mmu_base_pid, max_id); |
Michael Ellerman | 760573c | 2017-03-29 22:36:56 +1100 | [diff] [blame] | 131 | if (index < 0) |
| 132 | return index; |
| 133 | |
| 134 | /* |
| 135 | * set the process table entry, |
| 136 | */ |
| 137 | rts_field = radix__get_tree_size(); |
| 138 | process_tb[index].prtb0 = cpu_to_be64(rts_field | __pa(mm->pgd) | RADIX_PGD_INDEX_SIZE); |
| 139 | |
Benjamin Herrenschmidt | 3a6a047 | 2017-07-07 16:12:16 -0500 | [diff] [blame] | 140 | /* |
| 141 | * Order the above store with subsequent update of the PID |
| 142 | * register (at which point HW can start loading/caching |
| 143 | * the entry) and the corresponding load by the MMU from |
| 144 | * the L2 cache. |
| 145 | */ |
| 146 | asm volatile("ptesync;isync" : : : "memory"); |
| 147 | |
Alistair Popple | 1ab66d1 | 2017-04-03 19:51:44 +1000 | [diff] [blame] | 148 | mm->context.npu_context = NULL; |
| 149 | |
Michael Ellerman | 760573c | 2017-03-29 22:36:56 +1100 | [diff] [blame] | 150 | return index; |
| 151 | } |
| 152 | |
| 153 | int init_new_context(struct task_struct *tsk, struct mm_struct *mm) |
| 154 | { |
| 155 | int index; |
| 156 | |
| 157 | if (radix_enabled()) |
| 158 | index = radix__init_new_context(mm); |
| 159 | else |
| 160 | index = hash__init_new_context(mm); |
| 161 | |
| 162 | if (index < 0) |
| 163 | return index; |
| 164 | |
Stephen Rothwell | 9dfe5c53 | 2007-08-15 16:33:55 +1000 | [diff] [blame] | 165 | mm->context.id = index; |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 166 | |
Aneesh Kumar K.V | 5c1f6ee | 2013-04-28 09:37:33 +0000 | [diff] [blame] | 167 | #ifdef CONFIG_PPC_64K_PAGES |
| 168 | mm->context.pte_frag = NULL; |
| 169 | #endif |
Alexey Kardashevskiy | 15b244a | 2015-06-05 16:35:24 +1000 | [diff] [blame] | 170 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
Alexey Kardashevskiy | 88f54a3 | 2016-11-30 17:51:59 +1100 | [diff] [blame] | 171 | mm_iommu_init(mm); |
Alexey Kardashevskiy | 15b244a | 2015-06-05 16:35:24 +1000 | [diff] [blame] | 172 | #endif |
Benjamin Herrenschmidt | a619e59 | 2017-07-24 14:28:02 +1000 | [diff] [blame] | 173 | atomic_set(&mm->context.active_cpus, 0); |
| 174 | |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 175 | return 0; |
| 176 | } |
| 177 | |
Alexander Graf | e85a471 | 2009-11-02 12:02:30 +0000 | [diff] [blame] | 178 | void __destroy_context(int context_id) |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 179 | { |
| 180 | spin_lock(&mmu_context_lock); |
Anton Blanchard | 7317ac8 | 2010-02-07 12:30:12 +0000 | [diff] [blame] | 181 | ida_remove(&mmu_context_ida, context_id); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 182 | spin_unlock(&mmu_context_lock); |
Alexander Graf | e85a471 | 2009-11-02 12:02:30 +0000 | [diff] [blame] | 183 | } |
| 184 | EXPORT_SYMBOL_GPL(__destroy_context); |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 185 | |
Aneesh Kumar K.V | 5c1f6ee | 2013-04-28 09:37:33 +0000 | [diff] [blame] | 186 | #ifdef CONFIG_PPC_64K_PAGES |
| 187 | static void destroy_pagetable_page(struct mm_struct *mm) |
| 188 | { |
| 189 | int count; |
| 190 | void *pte_frag; |
| 191 | struct page *page; |
| 192 | |
| 193 | pte_frag = mm->context.pte_frag; |
| 194 | if (!pte_frag) |
| 195 | return; |
| 196 | |
| 197 | page = virt_to_page(pte_frag); |
| 198 | /* drop all the pending references */ |
| 199 | count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT; |
| 200 | /* We allow PTE_FRAG_NR fragments from a PTE page */ |
Joonsoo Kim | fe896d1 | 2016-03-17 14:19:26 -0700 | [diff] [blame] | 201 | if (page_ref_sub_and_test(page, PTE_FRAG_NR - count)) { |
Aneesh Kumar K.V | 5c1f6ee | 2013-04-28 09:37:33 +0000 | [diff] [blame] | 202 | pgtable_page_dtor(page); |
Mel Gorman | 2d4894b | 2017-11-15 17:37:59 -0800 | [diff] [blame] | 203 | free_unref_page(page); |
Aneesh Kumar K.V | 5c1f6ee | 2013-04-28 09:37:33 +0000 | [diff] [blame] | 204 | } |
| 205 | } |
| 206 | |
| 207 | #else |
| 208 | static inline void destroy_pagetable_page(struct mm_struct *mm) |
| 209 | { |
| 210 | return; |
| 211 | } |
| 212 | #endif |
| 213 | |
Alexander Graf | e85a471 | 2009-11-02 12:02:30 +0000 | [diff] [blame] | 214 | void destroy_context(struct mm_struct *mm) |
| 215 | { |
Alexey Kardashevskiy | 15b244a | 2015-06-05 16:35:24 +1000 | [diff] [blame] | 216 | #ifdef CONFIG_SPAPR_TCE_IOMMU |
Alexey Kardashevskiy | 4b6fad7 | 2016-11-30 17:52:05 +1100 | [diff] [blame] | 217 | WARN_ON_ONCE(!list_empty(&mm->context.iommu_group_mem_list)); |
Alexey Kardashevskiy | 15b244a | 2015-06-05 16:35:24 +1000 | [diff] [blame] | 218 | #endif |
Nicholas Piggin | 30b49ec | 2017-10-24 23:06:54 +1000 | [diff] [blame] | 219 | if (radix_enabled()) |
| 220 | WARN_ON(process_tb[mm->context.id].prtb0 != 0); |
| 221 | else |
| 222 | subpage_prot_free(mm); |
| 223 | destroy_pagetable_page(mm); |
| 224 | __destroy_context(mm->context.id); |
| 225 | mm->context.id = MMU_NO_CONTEXT; |
| 226 | } |
| 227 | |
| 228 | void arch_exit_mmap(struct mm_struct *mm) |
| 229 | { |
Benjamin Herrenschmidt | c6bb0b8 | 2017-07-08 07:45:32 -0500 | [diff] [blame] | 230 | if (radix_enabled()) { |
| 231 | /* |
| 232 | * Radix doesn't have a valid bit in the process table |
| 233 | * entries. However we know that at least P9 implementation |
| 234 | * will avoid caching an entry with an invalid RTS field, |
| 235 | * and 0 is invalid. So this will do. |
Nicholas Piggin | 30b49ec | 2017-10-24 23:06:54 +1000 | [diff] [blame] | 236 | * |
| 237 | * This runs before the "fullmm" tlb flush in exit_mmap, |
| 238 | * which does a RIC=2 tlbie to clear the process table |
| 239 | * entry. See the "fullmm" comments in tlb-radix.c. |
| 240 | * |
| 241 | * No barrier required here after the store because |
| 242 | * this process will do the invalidate, which starts with |
| 243 | * ptesync. |
Benjamin Herrenschmidt | c6bb0b8 | 2017-07-08 07:45:32 -0500 | [diff] [blame] | 244 | */ |
| 245 | process_tb[mm->context.id].prtb0 = 0; |
Nicholas Piggin | 30b49ec | 2017-10-24 23:06:54 +1000 | [diff] [blame] | 246 | } |
Paul Mackerras | 14cf11a | 2005-09-26 16:04:21 +1000 | [diff] [blame] | 247 | } |
Aneesh Kumar K.V | 7e381c0 | 2016-04-29 23:26:02 +1000 | [diff] [blame] | 248 | |
| 249 | #ifdef CONFIG_PPC_RADIX_MMU |
| 250 | void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) |
| 251 | { |
Benjamin Herrenschmidt | 74e27c6 | 2017-06-25 15:08:46 -0500 | [diff] [blame] | 252 | |
| 253 | if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { |
| 254 | isync(); |
| 255 | mtspr(SPRN_PID, next->context.id); |
| 256 | isync(); |
| 257 | asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); |
| 258 | } else { |
| 259 | mtspr(SPRN_PID, next->context.id); |
| 260 | isync(); |
| 261 | } |
Aneesh Kumar K.V | 7e381c0 | 2016-04-29 23:26:02 +1000 | [diff] [blame] | 262 | } |
| 263 | #endif |