Thomas Gleixner | d94d71c | 2019-05-29 07:12:40 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 2 | /* |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 3 | * |
| 4 | * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
| 5 | * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com> |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 6 | * Copyright 2016 Alexey Kardashevskiy, IBM Corporation <aik@au1.ibm.com> |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 7 | */ |
| 8 | |
| 9 | #include <linux/types.h> |
| 10 | #include <linux/string.h> |
| 11 | #include <linux/kvm.h> |
| 12 | #include <linux/kvm_host.h> |
| 13 | #include <linux/highmem.h> |
| 14 | #include <linux/gfp.h> |
| 15 | #include <linux/slab.h> |
Ingo Molnar | 3f07c01 | 2017-02-08 18:51:30 +0100 | [diff] [blame] | 16 | #include <linux/sched/signal.h> |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 17 | #include <linux/hugetlb.h> |
| 18 | #include <linux/list.h> |
| 19 | #include <linux/anon_inodes.h> |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 20 | #include <linux/iommu.h> |
| 21 | #include <linux/file.h> |
Daniel Jordan | 79eb597 | 2019-07-16 16:30:54 -0700 | [diff] [blame] | 22 | #include <linux/mm.h> |
Kent Overstreet | 1e2f2d3 | 2023-12-15 15:51:54 -0500 | [diff] [blame] | 23 | #include <linux/rcupdate_wait.h> |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 24 | |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 25 | #include <asm/kvm_ppc.h> |
| 26 | #include <asm/kvm_book3s.h> |
Aneesh Kumar K.V | f64e8084 | 2016-03-01 12:59:20 +0530 | [diff] [blame] | 27 | #include <asm/book3s/64/mmu-hash.h> |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 28 | #include <asm/hvcall.h> |
| 29 | #include <asm/synch.h> |
| 30 | #include <asm/ppc-opcode.h> |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 31 | #include <asm/udbg.h> |
Alexey Kardashevskiy | 462ee11 | 2016-02-15 12:55:07 +1100 | [diff] [blame] | 32 | #include <asm/iommu.h> |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 33 | #include <asm/tce.h> |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 34 | #include <asm/mmu_context.h> |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 35 | |
Alexey Kardashevskiy | cad32d9 | 2022-05-06 15:37:55 +1000 | [diff] [blame] | 36 | static struct kvmppc_spapr_tce_table *kvmppc_find_table(struct kvm *kvm, |
| 37 | unsigned long liobn) |
| 38 | { |
| 39 | struct kvmppc_spapr_tce_table *stt; |
| 40 | |
| 41 | list_for_each_entry_lockless(stt, &kvm->arch.spapr_tce_tables, list) |
| 42 | if (stt->liobn == liobn) |
| 43 | return stt; |
| 44 | |
| 45 | return NULL; |
| 46 | } |
| 47 | |
Alexey Kardashevskiy | fe26e52 | 2016-03-01 17:54:38 +1100 | [diff] [blame] | 48 | static unsigned long kvmppc_tce_pages(unsigned long iommu_pages) |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 49 | { |
Alexey Kardashevskiy | fe26e52 | 2016-03-01 17:54:38 +1100 | [diff] [blame] | 50 | return ALIGN(iommu_pages * sizeof(u64), PAGE_SIZE) / PAGE_SIZE; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 51 | } |
| 52 | |
Alexey Kardashevskiy | f862698 | 2016-02-15 12:55:06 +1100 | [diff] [blame] | 53 | static unsigned long kvmppc_stt_pages(unsigned long tce_pages) |
| 54 | { |
| 55 | unsigned long stt_bytes = sizeof(struct kvmppc_spapr_tce_table) + |
| 56 | (tce_pages * sizeof(struct page *)); |
| 57 | |
| 58 | return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE; |
| 59 | } |
| 60 | |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 61 | static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head) |
| 62 | { |
| 63 | struct kvmppc_spapr_tce_iommu_table *stit = container_of(head, |
| 64 | struct kvmppc_spapr_tce_iommu_table, rcu); |
| 65 | |
| 66 | iommu_tce_table_put(stit->tbl); |
| 67 | |
| 68 | kfree(stit); |
| 69 | } |
| 70 | |
| 71 | static void kvm_spapr_tce_liobn_put(struct kref *kref) |
| 72 | { |
| 73 | struct kvmppc_spapr_tce_iommu_table *stit = container_of(kref, |
| 74 | struct kvmppc_spapr_tce_iommu_table, kref); |
| 75 | |
| 76 | list_del_rcu(&stit->next); |
| 77 | |
| 78 | call_rcu(&stit->rcu, kvm_spapr_tce_iommu_table_free); |
| 79 | } |
| 80 | |
Benjamin Gray | 419d5d1 | 2023-10-11 16:37:04 +1100 | [diff] [blame] | 81 | void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, |
| 82 | struct iommu_group *grp) |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 83 | { |
| 84 | int i; |
| 85 | struct kvmppc_spapr_tce_table *stt; |
| 86 | struct kvmppc_spapr_tce_iommu_table *stit, *tmp; |
| 87 | struct iommu_table_group *table_group = NULL; |
| 88 | |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 89 | rcu_read_lock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 90 | list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) { |
| 91 | |
| 92 | table_group = iommu_group_get_iommudata(grp); |
| 93 | if (WARN_ON(!table_group)) |
| 94 | continue; |
| 95 | |
| 96 | list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) { |
| 97 | for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { |
| 98 | if (table_group->tables[i] != stit->tbl) |
| 99 | continue; |
| 100 | |
| 101 | kref_put(&stit->kref, kvm_spapr_tce_liobn_put); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 102 | } |
| 103 | } |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 104 | cond_resched_rcu(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 105 | } |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 106 | rcu_read_unlock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 107 | } |
| 108 | |
Benjamin Gray | 419d5d1 | 2023-10-11 16:37:04 +1100 | [diff] [blame] | 109 | long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, |
| 110 | struct iommu_group *grp) |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 111 | { |
| 112 | struct kvmppc_spapr_tce_table *stt = NULL; |
| 113 | bool found = false; |
| 114 | struct iommu_table *tbl = NULL; |
| 115 | struct iommu_table_group *table_group; |
| 116 | long i; |
| 117 | struct kvmppc_spapr_tce_iommu_table *stit; |
| 118 | struct fd f; |
| 119 | |
| 120 | f = fdget(tablefd); |
| 121 | if (!f.file) |
| 122 | return -EBADF; |
| 123 | |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 124 | rcu_read_lock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 125 | list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) { |
| 126 | if (stt == f.file->private_data) { |
| 127 | found = true; |
| 128 | break; |
| 129 | } |
| 130 | } |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 131 | rcu_read_unlock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 132 | |
| 133 | fdput(f); |
| 134 | |
| 135 | if (!found) |
| 136 | return -EINVAL; |
| 137 | |
| 138 | table_group = iommu_group_get_iommudata(grp); |
| 139 | if (WARN_ON(!table_group)) |
| 140 | return -EFAULT; |
| 141 | |
| 142 | for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { |
| 143 | struct iommu_table *tbltmp = table_group->tables[i]; |
| 144 | |
| 145 | if (!tbltmp) |
| 146 | continue; |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 147 | /* Make sure hardware table parameters are compatible */ |
| 148 | if ((tbltmp->it_page_shift <= stt->page_shift) && |
| 149 | (tbltmp->it_offset << tbltmp->it_page_shift == |
| 150 | stt->offset << stt->page_shift) && |
Alexey Kardashevskiy | 76346cd | 2018-06-20 18:42:58 +1000 | [diff] [blame] | 151 | (tbltmp->it_size << tbltmp->it_page_shift >= |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 152 | stt->size << stt->page_shift)) { |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 153 | /* |
| 154 | * Reference the table to avoid races with |
| 155 | * add/remove DMA windows. |
| 156 | */ |
| 157 | tbl = iommu_tce_table_get(tbltmp); |
| 158 | break; |
| 159 | } |
| 160 | } |
| 161 | if (!tbl) |
| 162 | return -EINVAL; |
| 163 | |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 164 | rcu_read_lock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 165 | list_for_each_entry_rcu(stit, &stt->iommu_tables, next) { |
| 166 | if (tbl != stit->tbl) |
| 167 | continue; |
| 168 | |
| 169 | if (!kref_get_unless_zero(&stit->kref)) { |
| 170 | /* stit is being destroyed */ |
| 171 | iommu_tce_table_put(tbl); |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 172 | rcu_read_unlock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 173 | return -ENOTTY; |
| 174 | } |
| 175 | /* |
| 176 | * The table is already known to this KVM, we just increased |
| 177 | * its KVM reference counter and can return. |
| 178 | */ |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 179 | rcu_read_unlock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 180 | return 0; |
| 181 | } |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 182 | rcu_read_unlock(); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 183 | |
| 184 | stit = kzalloc(sizeof(*stit), GFP_KERNEL); |
| 185 | if (!stit) { |
| 186 | iommu_tce_table_put(tbl); |
| 187 | return -ENOMEM; |
| 188 | } |
| 189 | |
| 190 | stit->tbl = tbl; |
| 191 | kref_init(&stit->kref); |
| 192 | |
| 193 | list_add_rcu(&stit->next, &stt->iommu_tables); |
| 194 | |
| 195 | return 0; |
| 196 | } |
| 197 | |
Alexey Kardashevskiy | 366baf2 | 2016-02-15 12:55:05 +1100 | [diff] [blame] | 198 | static void release_spapr_tce_table(struct rcu_head *head) |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 199 | { |
Alexey Kardashevskiy | 366baf2 | 2016-02-15 12:55:05 +1100 | [diff] [blame] | 200 | struct kvmppc_spapr_tce_table *stt = container_of(head, |
| 201 | struct kvmppc_spapr_tce_table, rcu); |
Alexey Kardashevskiy | fe26e52 | 2016-03-01 17:54:38 +1100 | [diff] [blame] | 202 | unsigned long i, npages = kvmppc_tce_pages(stt->size); |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 203 | |
Alexey Kardashevskiy | f862698 | 2016-02-15 12:55:06 +1100 | [diff] [blame] | 204 | for (i = 0; i < npages; i++) |
Alexey Kardashevskiy | e1a1ef8 | 2019-03-29 16:43:26 +1100 | [diff] [blame] | 205 | if (stt->pages[i]) |
| 206 | __free_page(stt->pages[i]); |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 207 | |
Alexey Kardashevskiy | 366baf2 | 2016-02-15 12:55:05 +1100 | [diff] [blame] | 208 | kfree(stt); |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 209 | } |
| 210 | |
Alexey Kardashevskiy | e1a1ef8 | 2019-03-29 16:43:26 +1100 | [diff] [blame] | 211 | static struct page *kvm_spapr_get_tce_page(struct kvmppc_spapr_tce_table *stt, |
| 212 | unsigned long sttpage) |
| 213 | { |
| 214 | struct page *page = stt->pages[sttpage]; |
| 215 | |
| 216 | if (page) |
| 217 | return page; |
| 218 | |
| 219 | mutex_lock(&stt->alloc_lock); |
| 220 | page = stt->pages[sttpage]; |
| 221 | if (!page) { |
| 222 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
| 223 | WARN_ON_ONCE(!page); |
| 224 | if (page) |
| 225 | stt->pages[sttpage] = page; |
| 226 | } |
| 227 | mutex_unlock(&stt->alloc_lock); |
| 228 | |
| 229 | return page; |
| 230 | } |
| 231 | |
Souptick Joarder | 16d5c39 | 2018-05-10 23:57:19 +0530 | [diff] [blame] | 232 | static vm_fault_t kvm_spapr_tce_fault(struct vm_fault *vmf) |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 233 | { |
Dave Jiang | 11bac80 | 2017-02-24 14:56:41 -0800 | [diff] [blame] | 234 | struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 235 | struct page *page; |
| 236 | |
Alexey Kardashevskiy | fe26e52 | 2016-03-01 17:54:38 +1100 | [diff] [blame] | 237 | if (vmf->pgoff >= kvmppc_tce_pages(stt->size)) |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 238 | return VM_FAULT_SIGBUS; |
| 239 | |
Alexey Kardashevskiy | e1a1ef8 | 2019-03-29 16:43:26 +1100 | [diff] [blame] | 240 | page = kvm_spapr_get_tce_page(stt, vmf->pgoff); |
| 241 | if (!page) |
| 242 | return VM_FAULT_OOM; |
| 243 | |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 244 | get_page(page); |
| 245 | vmf->page = page; |
| 246 | return 0; |
| 247 | } |
| 248 | |
| 249 | static const struct vm_operations_struct kvm_spapr_tce_vm_ops = { |
| 250 | .fault = kvm_spapr_tce_fault, |
| 251 | }; |
| 252 | |
| 253 | static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma) |
| 254 | { |
| 255 | vma->vm_ops = &kvm_spapr_tce_vm_ops; |
| 256 | return 0; |
| 257 | } |
| 258 | |
| 259 | static int kvm_spapr_tce_release(struct inode *inode, struct file *filp) |
| 260 | { |
| 261 | struct kvmppc_spapr_tce_table *stt = filp->private_data; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 262 | struct kvmppc_spapr_tce_iommu_table *stit, *tmp; |
Paul Mackerras | edd0360 | 2017-08-28 14:31:24 +1000 | [diff] [blame] | 263 | struct kvm *kvm = stt->kvm; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 264 | |
Paul Mackerras | edd0360 | 2017-08-28 14:31:24 +1000 | [diff] [blame] | 265 | mutex_lock(&kvm->lock); |
Alexey Kardashevskiy | 366baf2 | 2016-02-15 12:55:05 +1100 | [diff] [blame] | 266 | list_del_rcu(&stt->list); |
Paul Mackerras | edd0360 | 2017-08-28 14:31:24 +1000 | [diff] [blame] | 267 | mutex_unlock(&kvm->lock); |
Alexey Kardashevskiy | 366baf2 | 2016-02-15 12:55:05 +1100 | [diff] [blame] | 268 | |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 269 | list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) { |
| 270 | WARN_ON(!kref_read(&stit->kref)); |
| 271 | while (1) { |
| 272 | if (kref_put(&stit->kref, kvm_spapr_tce_liobn_put)) |
| 273 | break; |
| 274 | } |
| 275 | } |
| 276 | |
Leonardo Bras | 8a9c892 | 2019-11-26 19:36:30 -0300 | [diff] [blame] | 277 | account_locked_vm(kvm->mm, |
| 278 | kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false); |
| 279 | |
Alexey Kardashevskiy | 366baf2 | 2016-02-15 12:55:05 +1100 | [diff] [blame] | 280 | kvm_put_kvm(stt->kvm); |
| 281 | |
| 282 | call_rcu(&stt->rcu, release_spapr_tce_table); |
| 283 | |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 284 | return 0; |
| 285 | } |
| 286 | |
Al Viro | 75ef9de | 2013-04-04 19:09:41 -0400 | [diff] [blame] | 287 | static const struct file_operations kvm_spapr_tce_fops = { |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 288 | .mmap = kvm_spapr_tce_mmap, |
| 289 | .release = kvm_spapr_tce_release, |
| 290 | }; |
| 291 | |
Thomas Huth | 67c48662e | 2023-02-08 15:01:00 +0100 | [diff] [blame] | 292 | int kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, |
| 293 | struct kvm_create_spapr_tce_64 *args) |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 294 | { |
| 295 | struct kvmppc_spapr_tce_table *stt = NULL; |
Paul Mackerras | 47c5310 | 2017-08-24 19:14:47 +1000 | [diff] [blame] | 296 | struct kvmppc_spapr_tce_table *siter; |
Leonardo Bras | 8a9c892 | 2019-11-26 19:36:30 -0300 | [diff] [blame] | 297 | struct mm_struct *mm = kvm->mm; |
Deming Wang | 6fa1efe | 2022-07-03 13:29:32 -0400 | [diff] [blame] | 298 | unsigned long npages; |
Jing Xiangfeng | eb17355 | 2020-09-19 15:12:30 +0800 | [diff] [blame] | 299 | int ret; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 300 | |
Alexey Kardashevskiy | e45719a | 2018-05-14 20:00:29 +1000 | [diff] [blame] | 301 | if (!args->size || args->page_shift < 12 || args->page_shift > 34 || |
| 302 | (args->offset + args->size > (ULLONG_MAX >> args->page_shift))) |
Alexey Kardashevskiy | 58ded42 | 2016-03-01 17:54:40 +1100 | [diff] [blame] | 303 | return -EINVAL; |
| 304 | |
Deming Wang | 6fa1efe | 2022-07-03 13:29:32 -0400 | [diff] [blame] | 305 | npages = kvmppc_tce_pages(args->size); |
Leonardo Bras | 8a9c892 | 2019-11-26 19:36:30 -0300 | [diff] [blame] | 306 | ret = account_locked_vm(mm, kvmppc_stt_pages(npages), true); |
Paul Mackerras | 47c5310 | 2017-08-24 19:14:47 +1000 | [diff] [blame] | 307 | if (ret) |
| 308 | return ret; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 309 | |
Wei Yongjun | 5982f08 | 2017-02-08 16:20:01 +0000 | [diff] [blame] | 310 | ret = -ENOMEM; |
Alexey Kardashevskiy | 4dee21e | 2022-06-28 18:02:28 +1000 | [diff] [blame] | 311 | stt = kzalloc(struct_size(stt, pages, npages), GFP_KERNEL | __GFP_NOWARN); |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 312 | if (!stt) |
Paul Mackerras | 47c5310 | 2017-08-24 19:14:47 +1000 | [diff] [blame] | 313 | goto fail_acct; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 314 | |
| 315 | stt->liobn = args->liobn; |
Alexey Kardashevskiy | 58ded42 | 2016-03-01 17:54:40 +1100 | [diff] [blame] | 316 | stt->page_shift = args->page_shift; |
| 317 | stt->offset = args->offset; |
Deming Wang | 6fa1efe | 2022-07-03 13:29:32 -0400 | [diff] [blame] | 318 | stt->size = args->size; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 319 | stt->kvm = kvm; |
Alexey Kardashevskiy | e1a1ef8 | 2019-03-29 16:43:26 +1100 | [diff] [blame] | 320 | mutex_init(&stt->alloc_lock); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 321 | INIT_LIST_HEAD_RCU(&stt->iommu_tables); |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 322 | |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 323 | mutex_lock(&kvm->lock); |
Paul Mackerras | 47c5310 | 2017-08-24 19:14:47 +1000 | [diff] [blame] | 324 | |
| 325 | /* Check this LIOBN hasn't been previously allocated */ |
| 326 | ret = 0; |
| 327 | list_for_each_entry(siter, &kvm->arch.spapr_tce_tables, list) { |
| 328 | if (siter->liobn == args->liobn) { |
| 329 | ret = -EBUSY; |
| 330 | break; |
| 331 | } |
| 332 | } |
| 333 | |
Alexey Kardashevskiy | 716cb11 | 2019-02-21 14:44:14 +1100 | [diff] [blame] | 334 | kvm_get_kvm(kvm); |
Paul Mackerras | edd0360 | 2017-08-28 14:31:24 +1000 | [diff] [blame] | 335 | if (!ret) |
| 336 | ret = anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops, |
| 337 | stt, O_RDWR | O_CLOEXEC); |
| 338 | |
Alexey Kardashevskiy | 716cb11 | 2019-02-21 14:44:14 +1100 | [diff] [blame] | 339 | if (ret >= 0) |
Paul Mackerras | 47c5310 | 2017-08-24 19:14:47 +1000 | [diff] [blame] | 340 | list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables); |
Alexey Kardashevskiy | 716cb11 | 2019-02-21 14:44:14 +1100 | [diff] [blame] | 341 | else |
Sean Christopherson | 149487b | 2019-10-21 15:58:42 -0700 | [diff] [blame] | 342 | kvm_put_kvm_no_destroy(kvm); |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 343 | |
| 344 | mutex_unlock(&kvm->lock); |
| 345 | |
Paul Mackerras | edd0360 | 2017-08-28 14:31:24 +1000 | [diff] [blame] | 346 | if (ret >= 0) |
| 347 | return ret; |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 348 | |
Paul Mackerras | 47c5310 | 2017-08-24 19:14:47 +1000 | [diff] [blame] | 349 | kfree(stt); |
| 350 | fail_acct: |
Leonardo Bras | 8a9c892 | 2019-11-26 19:36:30 -0300 | [diff] [blame] | 351 | account_locked_vm(mm, kvmppc_stt_pages(npages), false); |
Benjamin Herrenschmidt | f31e65e | 2012-03-15 21:58:34 +0000 | [diff] [blame] | 352 | return ret; |
| 353 | } |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 354 | |
Alexey Kardashevskiy | 2001825 | 2019-03-29 16:42:20 +1100 | [diff] [blame] | 355 | static long kvmppc_tce_to_ua(struct kvm *kvm, unsigned long tce, |
| 356 | unsigned long *ua) |
| 357 | { |
| 358 | unsigned long gfn = tce >> PAGE_SHIFT; |
| 359 | struct kvm_memory_slot *memslot; |
| 360 | |
David Matlack | 0f22af9 | 2021-08-04 22:28:39 +0000 | [diff] [blame] | 361 | memslot = __gfn_to_memslot(kvm_memslots(kvm), gfn); |
Alexey Kardashevskiy | 2001825 | 2019-03-29 16:42:20 +1100 | [diff] [blame] | 362 | if (!memslot) |
| 363 | return -EINVAL; |
| 364 | |
| 365 | *ua = __gfn_to_hva_memslot(memslot, gfn) | |
| 366 | (tce & ~(PAGE_MASK | TCE_PCI_READ | TCE_PCI_WRITE)); |
| 367 | |
| 368 | return 0; |
| 369 | } |
| 370 | |
Alexey Kardashevskiy | 42de7b9 | 2018-09-10 18:29:10 +1000 | [diff] [blame] | 371 | static long kvmppc_tce_validate(struct kvmppc_spapr_tce_table *stt, |
| 372 | unsigned long tce) |
| 373 | { |
| 374 | unsigned long gpa = tce & ~(TCE_PCI_READ | TCE_PCI_WRITE); |
| 375 | enum dma_data_direction dir = iommu_tce_direction(tce); |
| 376 | struct kvmppc_spapr_tce_iommu_table *stit; |
| 377 | unsigned long ua = 0; |
| 378 | |
| 379 | /* Allow userspace to poison TCE table */ |
| 380 | if (dir == DMA_NONE) |
| 381 | return H_SUCCESS; |
| 382 | |
| 383 | if (iommu_tce_check_gpa(stt->page_shift, gpa)) |
| 384 | return H_TOO_HARD; |
| 385 | |
Alexey Kardashevskiy | 2001825 | 2019-03-29 16:42:20 +1100 | [diff] [blame] | 386 | if (kvmppc_tce_to_ua(stt->kvm, tce, &ua)) |
Alexey Kardashevskiy | 42de7b9 | 2018-09-10 18:29:10 +1000 | [diff] [blame] | 387 | return H_TOO_HARD; |
| 388 | |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 389 | rcu_read_lock(); |
Alexey Kardashevskiy | 42de7b9 | 2018-09-10 18:29:10 +1000 | [diff] [blame] | 390 | list_for_each_entry_rcu(stit, &stt->iommu_tables, next) { |
| 391 | unsigned long hpa = 0; |
| 392 | struct mm_iommu_table_group_mem_t *mem; |
| 393 | long shift = stit->tbl->it_page_shift; |
| 394 | |
| 395 | mem = mm_iommu_lookup(stt->kvm->mm, ua, 1ULL << shift); |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 396 | if (!mem || mm_iommu_ua_to_hpa(mem, ua, shift, &hpa)) { |
| 397 | rcu_read_unlock(); |
Alexey Kardashevskiy | 42de7b9 | 2018-09-10 18:29:10 +1000 | [diff] [blame] | 398 | return H_TOO_HARD; |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 399 | } |
Alexey Kardashevskiy | 42de7b9 | 2018-09-10 18:29:10 +1000 | [diff] [blame] | 400 | } |
Qian Cai | ab8b65b | 2020-05-10 01:18:34 -0400 | [diff] [blame] | 401 | rcu_read_unlock(); |
Alexey Kardashevskiy | 42de7b9 | 2018-09-10 18:29:10 +1000 | [diff] [blame] | 402 | |
| 403 | return H_SUCCESS; |
| 404 | } |
| 405 | |
Alexey Kardashevskiy | e1a1ef8 | 2019-03-29 16:43:26 +1100 | [diff] [blame] | 406 | /* |
| 407 | * Handles TCE requests for emulated devices. |
| 408 | * Puts guest TCE values to the table and expects user space to convert them. |
| 409 | * Cannot fail so kvmppc_tce_validate must be called before it. |
| 410 | */ |
| 411 | static void kvmppc_tce_put(struct kvmppc_spapr_tce_table *stt, |
| 412 | unsigned long idx, unsigned long tce) |
| 413 | { |
| 414 | struct page *page; |
| 415 | u64 *tbl; |
| 416 | unsigned long sttpage; |
| 417 | |
| 418 | idx -= stt->offset; |
| 419 | sttpage = idx / TCES_PER_PAGE; |
| 420 | page = stt->pages[sttpage]; |
| 421 | |
| 422 | if (!page) { |
| 423 | /* We allow any TCE, not just with read|write permissions */ |
| 424 | if (!tce) |
| 425 | return; |
| 426 | |
| 427 | page = kvm_spapr_get_tce_page(stt, sttpage); |
| 428 | if (!page) |
| 429 | return; |
| 430 | } |
| 431 | tbl = page_to_virt(page); |
| 432 | |
| 433 | tbl[idx % TCES_PER_PAGE] = tce; |
| 434 | } |
| 435 | |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 436 | static void kvmppc_clear_tce(struct mm_struct *mm, struct kvmppc_spapr_tce_table *stt, |
| 437 | struct iommu_table *tbl, unsigned long entry) |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 438 | { |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 439 | unsigned long i; |
| 440 | unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift); |
| 441 | unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 442 | |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 443 | for (i = 0; i < subpages; ++i) { |
| 444 | unsigned long hpa = 0; |
| 445 | enum dma_data_direction dir = DMA_NONE; |
| 446 | |
| 447 | iommu_tce_xchg_no_kill(mm, tbl, io_entry + i, &hpa, &dir); |
| 448 | } |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 449 | } |
| 450 | |
| 451 | static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm, |
| 452 | struct iommu_table *tbl, unsigned long entry) |
| 453 | { |
| 454 | struct mm_iommu_table_group_mem_t *mem = NULL; |
| 455 | const unsigned long pgsize = 1ULL << tbl->it_page_shift; |
Alexey Kardashevskiy | 6e301a8 | 2018-10-15 21:08:41 +1100 | [diff] [blame] | 456 | __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 457 | |
| 458 | if (!pua) |
Alexey Kardashevskiy | 6e301a8 | 2018-10-15 21:08:41 +1100 | [diff] [blame] | 459 | return H_SUCCESS; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 460 | |
Alexey Kardashevskiy | 00a5c58 | 2018-07-04 16:13:46 +1000 | [diff] [blame] | 461 | mem = mm_iommu_lookup(kvm->mm, be64_to_cpu(*pua), pgsize); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 462 | if (!mem) |
| 463 | return H_TOO_HARD; |
| 464 | |
| 465 | mm_iommu_mapped_dec(mem); |
| 466 | |
Alexey Kardashevskiy | 00a5c58 | 2018-07-04 16:13:46 +1000 | [diff] [blame] | 467 | *pua = cpu_to_be64(0); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 468 | |
| 469 | return H_SUCCESS; |
| 470 | } |
| 471 | |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 472 | static long kvmppc_tce_iommu_do_unmap(struct kvm *kvm, |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 473 | struct iommu_table *tbl, unsigned long entry) |
| 474 | { |
| 475 | enum dma_data_direction dir = DMA_NONE; |
| 476 | unsigned long hpa = 0; |
| 477 | long ret; |
| 478 | |
Alexey Kardashevskiy | 01b7d12 | 2019-08-29 18:52:49 +1000 | [diff] [blame] | 479 | if (WARN_ON_ONCE(iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, |
| 480 | &dir))) |
Alexey Kardashevskiy | f7960e2 | 2018-09-10 18:29:09 +1000 | [diff] [blame] | 481 | return H_TOO_HARD; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 482 | |
| 483 | if (dir == DMA_NONE) |
| 484 | return H_SUCCESS; |
| 485 | |
| 486 | ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry); |
| 487 | if (ret != H_SUCCESS) |
Alexey Kardashevskiy | 01b7d12 | 2019-08-29 18:52:49 +1000 | [diff] [blame] | 488 | iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 489 | |
| 490 | return ret; |
| 491 | } |
| 492 | |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 493 | static long kvmppc_tce_iommu_unmap(struct kvm *kvm, |
| 494 | struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl, |
| 495 | unsigned long entry) |
| 496 | { |
| 497 | unsigned long i, ret = H_SUCCESS; |
| 498 | unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift); |
| 499 | unsigned long io_entry = entry * subpages; |
| 500 | |
| 501 | for (i = 0; i < subpages; ++i) { |
| 502 | ret = kvmppc_tce_iommu_do_unmap(kvm, tbl, io_entry + i); |
| 503 | if (ret != H_SUCCESS) |
| 504 | break; |
| 505 | } |
| 506 | |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 507 | iommu_tce_kill(tbl, io_entry, subpages); |
| 508 | |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 509 | return ret; |
| 510 | } |
| 511 | |
Wang Wensheng | cf59eb1 | 2020-09-21 11:22:11 +0000 | [diff] [blame] | 512 | static long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 513 | unsigned long entry, unsigned long ua, |
| 514 | enum dma_data_direction dir) |
| 515 | { |
| 516 | long ret; |
Alexey Kardashevskiy | 00a5c58 | 2018-07-04 16:13:46 +1000 | [diff] [blame] | 517 | unsigned long hpa; |
| 518 | __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 519 | struct mm_iommu_table_group_mem_t *mem; |
| 520 | |
| 521 | if (!pua) |
| 522 | /* it_userspace allocation might be delayed */ |
| 523 | return H_TOO_HARD; |
| 524 | |
| 525 | mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift); |
| 526 | if (!mem) |
| 527 | /* This only handles v2 IOMMU type, v1 is handled via ioctl() */ |
| 528 | return H_TOO_HARD; |
| 529 | |
Alexey Kardashevskiy | 76fa497 | 2018-07-17 17:19:13 +1000 | [diff] [blame] | 530 | if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, tbl->it_page_shift, &hpa))) |
Alexey Kardashevskiy | f7960e2 | 2018-09-10 18:29:09 +1000 | [diff] [blame] | 531 | return H_TOO_HARD; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 532 | |
| 533 | if (mm_iommu_mapped_inc(mem)) |
Alexey Kardashevskiy | f7960e2 | 2018-09-10 18:29:09 +1000 | [diff] [blame] | 534 | return H_TOO_HARD; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 535 | |
Alexey Kardashevskiy | 01b7d12 | 2019-08-29 18:52:49 +1000 | [diff] [blame] | 536 | ret = iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 537 | if (WARN_ON_ONCE(ret)) { |
| 538 | mm_iommu_mapped_dec(mem); |
Alexey Kardashevskiy | f7960e2 | 2018-09-10 18:29:09 +1000 | [diff] [blame] | 539 | return H_TOO_HARD; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 540 | } |
| 541 | |
| 542 | if (dir != DMA_NONE) |
| 543 | kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry); |
| 544 | |
Alexey Kardashevskiy | 00a5c58 | 2018-07-04 16:13:46 +1000 | [diff] [blame] | 545 | *pua = cpu_to_be64(ua); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 546 | |
| 547 | return 0; |
| 548 | } |
| 549 | |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 550 | static long kvmppc_tce_iommu_map(struct kvm *kvm, |
| 551 | struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl, |
| 552 | unsigned long entry, unsigned long ua, |
| 553 | enum dma_data_direction dir) |
| 554 | { |
| 555 | unsigned long i, pgoff, ret = H_SUCCESS; |
| 556 | unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift); |
| 557 | unsigned long io_entry = entry * subpages; |
| 558 | |
| 559 | for (i = 0, pgoff = 0; i < subpages; |
| 560 | ++i, pgoff += IOMMU_PAGE_SIZE(tbl)) { |
| 561 | |
| 562 | ret = kvmppc_tce_iommu_do_map(kvm, tbl, |
| 563 | io_entry + i, ua + pgoff, dir); |
| 564 | if (ret != H_SUCCESS) |
| 565 | break; |
| 566 | } |
| 567 | |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 568 | iommu_tce_kill(tbl, io_entry, subpages); |
| 569 | |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 570 | return ret; |
| 571 | } |
| 572 | |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 573 | long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn, |
| 574 | unsigned long ioba, unsigned long tce) |
| 575 | { |
Alexey Kardashevskiy | 503bfcbe | 2017-03-22 15:21:53 +1100 | [diff] [blame] | 576 | struct kvmppc_spapr_tce_table *stt; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 577 | long ret, idx; |
| 578 | struct kvmppc_spapr_tce_iommu_table *stit; |
| 579 | unsigned long entry, ua = 0; |
| 580 | enum dma_data_direction dir; |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 581 | |
| 582 | /* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */ |
| 583 | /* liobn, ioba, tce); */ |
| 584 | |
Alexey Kardashevskiy | 503bfcbe | 2017-03-22 15:21:53 +1100 | [diff] [blame] | 585 | stt = kvmppc_find_table(vcpu->kvm, liobn); |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 586 | if (!stt) |
| 587 | return H_TOO_HARD; |
| 588 | |
| 589 | ret = kvmppc_ioba_validate(stt, ioba, 1); |
| 590 | if (ret != H_SUCCESS) |
| 591 | return ret; |
| 592 | |
Alexey Kardashevskiy | 345077c8 | 2019-03-29 16:41:13 +1100 | [diff] [blame] | 593 | idx = srcu_read_lock(&vcpu->kvm->srcu); |
| 594 | |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 595 | ret = kvmppc_tce_validate(stt, tce); |
| 596 | if (ret != H_SUCCESS) |
Alexey Kardashevskiy | 345077c8 | 2019-03-29 16:41:13 +1100 | [diff] [blame] | 597 | goto unlock_exit; |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 598 | |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 599 | dir = iommu_tce_direction(tce); |
Alexey Kardashevskiy | 8f6a9f0 | 2017-10-11 16:00:34 +1100 | [diff] [blame] | 600 | |
Alexey Kardashevskiy | 2001825 | 2019-03-29 16:42:20 +1100 | [diff] [blame] | 601 | if ((dir != DMA_NONE) && kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) { |
Alexey Kardashevskiy | 8f6a9f0 | 2017-10-11 16:00:34 +1100 | [diff] [blame] | 602 | ret = H_PARAMETER; |
| 603 | goto unlock_exit; |
| 604 | } |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 605 | |
| 606 | entry = ioba >> stt->page_shift; |
| 607 | |
| 608 | list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { |
Alexey Kardashevskiy | 8f6a9f0 | 2017-10-11 16:00:34 +1100 | [diff] [blame] | 609 | if (dir == DMA_NONE) |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 610 | ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt, |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 611 | stit->tbl, entry); |
Alexey Kardashevskiy | 8f6a9f0 | 2017-10-11 16:00:34 +1100 | [diff] [blame] | 612 | else |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 613 | ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl, |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 614 | entry, ua, dir); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 615 | |
Alexey Kardashevskiy | 01b7d12 | 2019-08-29 18:52:49 +1000 | [diff] [blame] | 616 | |
Alexey Kardashevskiy | 2691f0f | 2018-09-10 18:29:11 +1000 | [diff] [blame] | 617 | if (ret != H_SUCCESS) { |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 618 | kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry); |
Alexey Kardashevskiy | 8f6a9f0 | 2017-10-11 16:00:34 +1100 | [diff] [blame] | 619 | goto unlock_exit; |
Alexey Kardashevskiy | 2691f0f | 2018-09-10 18:29:11 +1000 | [diff] [blame] | 620 | } |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 621 | } |
| 622 | |
| 623 | kvmppc_tce_put(stt, entry, tce); |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 624 | |
Alexey Kardashevskiy | 8f6a9f0 | 2017-10-11 16:00:34 +1100 | [diff] [blame] | 625 | unlock_exit: |
| 626 | srcu_read_unlock(&vcpu->kvm->srcu, idx); |
| 627 | |
| 628 | return ret; |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 629 | } |
| 630 | EXPORT_SYMBOL_GPL(kvmppc_h_put_tce); |
| 631 | |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 632 | long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu, |
| 633 | unsigned long liobn, unsigned long ioba, |
| 634 | unsigned long tce_list, unsigned long npages) |
| 635 | { |
| 636 | struct kvmppc_spapr_tce_table *stt; |
| 637 | long i, ret = H_SUCCESS, idx; |
| 638 | unsigned long entry, ua = 0; |
Daniel Axtens | f875051 | 2016-07-12 10:54:48 +1000 | [diff] [blame] | 639 | u64 __user *tces; |
| 640 | u64 tce; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 641 | struct kvmppc_spapr_tce_iommu_table *stit; |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 642 | |
Alexey Kardashevskiy | 503bfcbe | 2017-03-22 15:21:53 +1100 | [diff] [blame] | 643 | stt = kvmppc_find_table(vcpu->kvm, liobn); |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 644 | if (!stt) |
| 645 | return H_TOO_HARD; |
| 646 | |
Alexey Kardashevskiy | fe26e52 | 2016-03-01 17:54:38 +1100 | [diff] [blame] | 647 | entry = ioba >> stt->page_shift; |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 648 | /* |
| 649 | * SPAPR spec says that the maximum size of the list is 512 TCEs |
| 650 | * so the whole table fits in 4K page |
| 651 | */ |
| 652 | if (npages > 512) |
| 653 | return H_PARAMETER; |
| 654 | |
| 655 | if (tce_list & (SZ_4K - 1)) |
| 656 | return H_PARAMETER; |
| 657 | |
| 658 | ret = kvmppc_ioba_validate(stt, ioba, npages); |
| 659 | if (ret != H_SUCCESS) |
| 660 | return ret; |
| 661 | |
| 662 | idx = srcu_read_lock(&vcpu->kvm->srcu); |
Alexey Kardashevskiy | 2001825 | 2019-03-29 16:42:20 +1100 | [diff] [blame] | 663 | if (kvmppc_tce_to_ua(vcpu->kvm, tce_list, &ua)) { |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 664 | ret = H_TOO_HARD; |
| 665 | goto unlock_exit; |
| 666 | } |
| 667 | tces = (u64 __user *) ua; |
| 668 | |
| 669 | for (i = 0; i < npages; ++i) { |
| 670 | if (get_user(tce, tces + i)) { |
| 671 | ret = H_TOO_HARD; |
| 672 | goto unlock_exit; |
| 673 | } |
| 674 | tce = be64_to_cpu(tce); |
| 675 | |
| 676 | ret = kvmppc_tce_validate(stt, tce); |
| 677 | if (ret != H_SUCCESS) |
| 678 | goto unlock_exit; |
Alexey Kardashevskiy | e199ad2 | 2018-09-10 18:29:08 +1000 | [diff] [blame] | 679 | } |
| 680 | |
| 681 | for (i = 0; i < npages; ++i) { |
| 682 | /* |
| 683 | * This looks unsafe, because we validate, then regrab |
| 684 | * the TCE from userspace which could have been changed by |
| 685 | * another thread. |
| 686 | * |
| 687 | * But it actually is safe, because the relevant checks will be |
| 688 | * re-executed in the following code. If userspace tries to |
| 689 | * change this dodgily it will result in a messier failure mode |
| 690 | * but won't threaten the host. |
| 691 | */ |
| 692 | if (get_user(tce, tces + i)) { |
| 693 | ret = H_TOO_HARD; |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 694 | goto unlock_exit; |
Alexey Kardashevskiy | e199ad2 | 2018-09-10 18:29:08 +1000 | [diff] [blame] | 695 | } |
| 696 | tce = be64_to_cpu(tce); |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 697 | |
Alexey Kardashevskiy | 4f91659 | 2019-08-26 14:55:20 +1000 | [diff] [blame] | 698 | if (kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) { |
| 699 | ret = H_PARAMETER; |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 700 | goto unlock_exit; |
Alexey Kardashevskiy | 4f91659 | 2019-08-26 14:55:20 +1000 | [diff] [blame] | 701 | } |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 702 | |
| 703 | list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 704 | ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 705 | stit->tbl, entry + i, ua, |
| 706 | iommu_tce_direction(tce)); |
| 707 | |
Alexey Kardashevskiy | 2691f0f | 2018-09-10 18:29:11 +1000 | [diff] [blame] | 708 | if (ret != H_SUCCESS) { |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 709 | kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, |
| 710 | entry + i); |
| 711 | goto unlock_exit; |
Alexey Kardashevskiy | 2691f0f | 2018-09-10 18:29:11 +1000 | [diff] [blame] | 712 | } |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 713 | } |
| 714 | |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 715 | kvmppc_tce_put(stt, entry + i, tce); |
| 716 | } |
| 717 | |
| 718 | unlock_exit: |
| 719 | srcu_read_unlock(&vcpu->kvm->srcu, idx); |
| 720 | |
| 721 | return ret; |
| 722 | } |
| 723 | EXPORT_SYMBOL_GPL(kvmppc_h_put_tce_indirect); |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 724 | |
| 725 | long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu, |
| 726 | unsigned long liobn, unsigned long ioba, |
| 727 | unsigned long tce_value, unsigned long npages) |
| 728 | { |
| 729 | struct kvmppc_spapr_tce_table *stt; |
| 730 | long i, ret; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 731 | struct kvmppc_spapr_tce_iommu_table *stit; |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 732 | |
Alexey Kardashevskiy | 503bfcbe | 2017-03-22 15:21:53 +1100 | [diff] [blame] | 733 | stt = kvmppc_find_table(vcpu->kvm, liobn); |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 734 | if (!stt) |
| 735 | return H_TOO_HARD; |
| 736 | |
| 737 | ret = kvmppc_ioba_validate(stt, ioba, npages); |
| 738 | if (ret != H_SUCCESS) |
| 739 | return ret; |
| 740 | |
| 741 | /* Check permission bits only to allow userspace poison TCE for debug */ |
| 742 | if (tce_value & (TCE_PCI_WRITE | TCE_PCI_READ)) |
| 743 | return H_PARAMETER; |
| 744 | |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 745 | list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { |
Alexey Kardashevskiy | c6b6166 | 2018-05-14 20:00:27 +1000 | [diff] [blame] | 746 | unsigned long entry = ioba >> stt->page_shift; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 747 | |
| 748 | for (i = 0; i < npages; ++i) { |
Alexey Kardashevskiy | ca1fc48 | 2018-05-14 20:00:28 +1000 | [diff] [blame] | 749 | ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt, |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 750 | stit->tbl, entry + i); |
| 751 | |
| 752 | if (ret == H_SUCCESS) |
| 753 | continue; |
| 754 | |
| 755 | if (ret == H_TOO_HARD) |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 756 | return ret; |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 757 | |
| 758 | WARN_ON_ONCE(1); |
Alexey Kardashevskiy | 26a62b7 | 2022-04-20 15:08:40 +1000 | [diff] [blame] | 759 | kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i); |
Alexey Kardashevskiy | 121f80b | 2017-03-22 15:21:56 +1100 | [diff] [blame] | 760 | } |
| 761 | } |
| 762 | |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 763 | for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift)) |
| 764 | kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value); |
| 765 | |
Alexey Kardashevskiy | 01b7d12 | 2019-08-29 18:52:49 +1000 | [diff] [blame] | 766 | return ret; |
Alexey Kardashevskiy | 31217db | 2016-03-18 13:50:42 +1100 | [diff] [blame] | 767 | } |
| 768 | EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce); |
Alexey Kardashevskiy | cad32d9 | 2022-05-06 15:37:55 +1000 | [diff] [blame] | 769 | |
| 770 | long kvmppc_h_get_tce(struct kvm_vcpu *vcpu, unsigned long liobn, |
| 771 | unsigned long ioba) |
| 772 | { |
| 773 | struct kvmppc_spapr_tce_table *stt; |
| 774 | long ret; |
| 775 | unsigned long idx; |
| 776 | struct page *page; |
| 777 | u64 *tbl; |
| 778 | |
| 779 | stt = kvmppc_find_table(vcpu->kvm, liobn); |
| 780 | if (!stt) |
| 781 | return H_TOO_HARD; |
| 782 | |
| 783 | ret = kvmppc_ioba_validate(stt, ioba, 1); |
| 784 | if (ret != H_SUCCESS) |
| 785 | return ret; |
| 786 | |
| 787 | idx = (ioba >> stt->page_shift) - stt->offset; |
| 788 | page = stt->pages[idx / TCES_PER_PAGE]; |
| 789 | if (!page) { |
Jordan Niethe | 0e85b7d | 2023-09-14 13:05:50 +1000 | [diff] [blame] | 790 | kvmppc_set_gpr(vcpu, 4, 0); |
Alexey Kardashevskiy | cad32d9 | 2022-05-06 15:37:55 +1000 | [diff] [blame] | 791 | return H_SUCCESS; |
| 792 | } |
| 793 | tbl = (u64 *)page_address(page); |
| 794 | |
Jordan Niethe | 0e85b7d | 2023-09-14 13:05:50 +1000 | [diff] [blame] | 795 | kvmppc_set_gpr(vcpu, 4, tbl[idx % TCES_PER_PAGE]); |
Alexey Kardashevskiy | cad32d9 | 2022-05-06 15:37:55 +1000 | [diff] [blame] | 796 | |
| 797 | return H_SUCCESS; |
| 798 | } |
| 799 | EXPORT_SYMBOL_GPL(kvmppc_h_get_tce); |