blob: fe888b5dcc006281133c2df808449da9834fbe5d [file] [log] [blame]
Thomas Gleixnerd2912cb2019-06-04 10:11:33 +02001// SPDX-License-Identifier: GPL-2.0-only
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10002/*
3 * VFIO: IOMMU DMA mapping support for TCE on POWER
4 *
5 * Copyright (C) 2013 IBM Corp. All rights reserved.
6 * Author: Alexey Kardashevskiy <aik@ozlabs.ru>
7 *
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10008 * Derived from original vfio_iommu_type1.c:
9 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
10 * Author: Alex Williamson <alex.williamson@redhat.com>
11 */
12
13#include <linux/module.h>
14#include <linux/pci.h>
15#include <linux/slab.h>
16#include <linux/uaccess.h>
17#include <linux/err.h>
18#include <linux/vfio.h>
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +100019#include <linux/vmalloc.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010020#include <linux/sched/mm.h>
Ingo Molnar3f07c012017-02-08 18:51:30 +010021#include <linux/sched/signal.h>
Daniel Jordan79eb5972019-07-16 16:30:54 -070022#include <linux/mm.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010023
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +100024#include <asm/iommu.h>
25#include <asm/tce.h>
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +100026#include <asm/mmu_context.h>
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +100027
28#define DRIVER_VERSION "0.1"
29#define DRIVER_AUTHOR "aik@ozlabs.ru"
30#define DRIVER_DESC "VFIO IOMMU SPAPR TCE"
31
32static void tce_iommu_detach_group(void *iommu_data,
33 struct iommu_group *iommu_group);
34
35/*
36 * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
37 *
38 * This code handles mapping and unmapping of user data buffers
39 * into DMA'ble space using the IOMMU
40 */
41
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +100042struct tce_iommu_group {
43 struct list_head next;
44 struct iommu_group *grp;
45};
46
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +100047/*
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +110048 * A container needs to remember which preregistered region it has
49 * referenced to do proper cleanup at the userspace process exit.
50 */
51struct tce_iommu_prereg {
52 struct list_head next;
53 struct mm_iommu_table_group_mem_t *mem;
54};
55
56/*
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +100057 * The container descriptor supports only a single group per container.
58 * Required by the API as the container is not supplied with the IOMMU group
59 * at the moment of initialization.
60 */
61struct tce_container {
62 struct mutex lock;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +100063 bool enabled;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +100064 bool v2;
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +110065 bool def_window_pending;
Alexey Kardashevskiy2d270df2015-06-05 16:35:01 +100066 unsigned long locked_pages;
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +110067 struct mm_struct *mm;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +100068 struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
69 struct list_head group_list;
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +110070 struct list_head prereg_list;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +100071};
72
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +110073static long tce_iommu_mm_set(struct tce_container *container)
74{
75 if (container->mm) {
76 if (container->mm == current->mm)
77 return 0;
78 return -EPERM;
79 }
80 BUG_ON(!current->mm);
81 container->mm = current->mm;
Julia Lawall7a49de92019-12-29 16:42:57 +010082 mmgrab(container->mm);
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +110083
84 return 0;
85}
86
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +110087static long tce_iommu_prereg_free(struct tce_container *container,
88 struct tce_iommu_prereg *tcemem)
89{
90 long ret;
91
92 ret = mm_iommu_put(container->mm, tcemem->mem);
93 if (ret)
94 return ret;
95
96 list_del(&tcemem->next);
97 kfree(tcemem);
98
99 return 0;
100}
101
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000102static long tce_iommu_unregister_pages(struct tce_container *container,
103 __u64 vaddr, __u64 size)
104{
105 struct mm_iommu_table_group_mem_t *mem;
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100106 struct tce_iommu_prereg *tcemem;
107 bool found = false;
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100108 long ret;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000109
110 if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
111 return -EINVAL;
112
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100113 mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000114 if (!mem)
115 return -ENOENT;
116
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100117 list_for_each_entry(tcemem, &container->prereg_list, next) {
118 if (tcemem->mem == mem) {
119 found = true;
120 break;
121 }
122 }
123
124 if (!found)
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100125 ret = -ENOENT;
126 else
127 ret = tce_iommu_prereg_free(container, tcemem);
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100128
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100129 mm_iommu_put(container->mm, mem);
130
131 return ret;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000132}
133
134static long tce_iommu_register_pages(struct tce_container *container,
135 __u64 vaddr, __u64 size)
136{
137 long ret = 0;
138 struct mm_iommu_table_group_mem_t *mem = NULL;
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100139 struct tce_iommu_prereg *tcemem;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000140 unsigned long entries = size >> PAGE_SHIFT;
141
142 if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
143 ((vaddr + size) < vaddr))
144 return -EINVAL;
145
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100146 mem = mm_iommu_get(container->mm, vaddr, entries);
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100147 if (mem) {
148 list_for_each_entry(tcemem, &container->prereg_list, next) {
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100149 if (tcemem->mem == mem) {
150 ret = -EBUSY;
151 goto put_exit;
152 }
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100153 }
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100154 } else {
155 ret = mm_iommu_new(container->mm, vaddr, entries, &mem);
156 if (ret)
157 return ret;
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100158 }
159
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100160 tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
Alexey Kardashevskiy3393af22017-03-27 14:23:40 +1100161 if (!tcemem) {
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100162 ret = -ENOMEM;
163 goto put_exit;
Alexey Kardashevskiy3393af22017-03-27 14:23:40 +1100164 }
165
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100166 tcemem->mem = mem;
167 list_add(&tcemem->next, &container->prereg_list);
168
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000169 container->enabled = true;
170
171 return 0;
Alexey Kardashevskiye0bf78b2018-12-19 19:52:14 +1100172
173put_exit:
174 mm_iommu_put(container->mm, mem);
175 return ret;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000176}
177
Alexey Kardashevskiyc10c21e2018-12-19 19:52:15 +1100178static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,
Matthew Wilcox (Oracle)94ad9332019-09-23 15:34:28 -0700179 unsigned int it_page_shift)
Alexey Kardashevskiye432bc72015-06-05 16:34:59 +1000180{
Alexey Kardashevskiyc10c21e2018-12-19 19:52:15 +1100181 struct page *page;
182 unsigned long size = 0;
183
Matthew Wilcox (Oracle)94ad9332019-09-23 15:34:28 -0700184 if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))
185 return size == (1UL << it_page_shift);
Alexey Kardashevskiyc10c21e2018-12-19 19:52:15 +1100186
187 page = pfn_to_page(hpa >> PAGE_SHIFT);
Alexey Kardashevskiye432bc72015-06-05 16:34:59 +1000188 /*
189 * Check that the TCE table granularity is not bigger than the size of
190 * a page we just found. Otherwise the hardware can get access to
191 * a bigger memory chunk that it should.
192 */
Matthew Wilcox (Oracle)94ad9332019-09-23 15:34:28 -0700193 return page_shift(compound_head(page)) >= it_page_shift;
Alexey Kardashevskiye432bc72015-06-05 16:34:59 +1000194}
195
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000196static inline bool tce_groups_attached(struct tce_container *container)
197{
198 return !list_empty(&container->group_list);
199}
200
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000201static long tce_iommu_find_table(struct tce_container *container,
202 phys_addr_t ioba, struct iommu_table **ptbl)
203{
204 long i;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000205
206 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000207 struct iommu_table *tbl = container->tables[i];
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000208
209 if (tbl) {
210 unsigned long entry = ioba >> tbl->it_page_shift;
211 unsigned long start = tbl->it_offset;
212 unsigned long end = start + tbl->it_size;
213
214 if ((start <= entry) && (entry < end)) {
215 *ptbl = tbl;
216 return i;
217 }
218 }
219 }
220
221 return -1;
222}
223
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +1000224static int tce_iommu_find_free_table(struct tce_container *container)
225{
226 int i;
227
228 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
229 if (!container->tables[i])
230 return i;
231 }
232
233 return -ENOSPC;
234}
235
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000236static int tce_iommu_enable(struct tce_container *container)
237{
238 int ret = 0;
Alexey Kardashevskiy2d270df2015-06-05 16:35:01 +1000239 unsigned long locked;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000240 struct iommu_table_group *table_group;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000241 struct tce_iommu_group *tcegrp;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000242
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000243 if (container->enabled)
244 return -EBUSY;
245
246 /*
247 * When userspace pages are mapped into the IOMMU, they are effectively
248 * locked memory, so, theoretically, we need to update the accounting
249 * of locked pages on each map and unmap. For powerpc, the map unmap
250 * paths can be very hot, though, and the accounting would kill
251 * performance, especially since it would be difficult to impossible
252 * to handle the accounting in real mode only.
253 *
254 * To address that, rather than precisely accounting every page, we
255 * instead account for a worst case on locked memory when the iommu is
256 * enabled and disabled. The worst case upper bound on locked memory
257 * is the size of the whole iommu window, which is usually relatively
258 * small (compared to total memory sizes) on POWER hardware.
259 *
260 * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
261 * that would effectively kill the guest at random points, much better
262 * enforcing the limit based on the max that the guest can map.
Alexey Kardashevskiy2d270df2015-06-05 16:35:01 +1000263 *
264 * Unfortunately at the moment it counts whole tables, no matter how
265 * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups
266 * each with 2GB DMA window, 8GB will be counted here. The reason for
267 * this is that we cannot tell here the amount of RAM used by the guest
268 * as this information is only available from KVM and VFIO is
269 * KVM agnostic.
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +1000270 *
271 * So we do not allow enabling a container without a group attached
272 * as there is no way to know how much we should increment
273 * the locked_vm counter.
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000274 */
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000275 if (!tce_groups_attached(container))
276 return -ENODEV;
277
278 tcegrp = list_first_entry(&container->group_list,
279 struct tce_iommu_group, next);
280 table_group = iommu_group_get_iommudata(tcegrp->grp);
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000281 if (!table_group)
282 return -ENODEV;
283
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +1000284 if (!table_group->tce32_size)
285 return -EPERM;
286
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100287 ret = tce_iommu_mm_set(container);
288 if (ret)
289 return ret;
290
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +1000291 locked = table_group->tce32_size >> PAGE_SHIFT;
Daniel Jordan79eb5972019-07-16 16:30:54 -0700292 ret = account_locked_vm(container->mm, locked, true);
Alexey Kardashevskiy2d270df2015-06-05 16:35:01 +1000293 if (ret)
294 return ret;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000295
Alexey Kardashevskiy2d270df2015-06-05 16:35:01 +1000296 container->locked_pages = locked;
297
298 container->enabled = true;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000299
300 return ret;
301}
302
303static void tce_iommu_disable(struct tce_container *container)
304{
305 if (!container->enabled)
306 return;
307
308 container->enabled = false;
309
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100310 BUG_ON(!container->mm);
Daniel Jordan79eb5972019-07-16 16:30:54 -0700311 account_locked_vm(container->mm, container->locked_pages, false);
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000312}
313
314static void *tce_iommu_open(unsigned long arg)
315{
316 struct tce_container *container;
317
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000318 if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000319 pr_err("tce_vfio: Wrong IOMMU type\n");
320 return ERR_PTR(-EINVAL);
321 }
322
323 container = kzalloc(sizeof(*container), GFP_KERNEL);
324 if (!container)
325 return ERR_PTR(-ENOMEM);
326
327 mutex_init(&container->lock);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000328 INIT_LIST_HEAD_RCU(&container->group_list);
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100329 INIT_LIST_HEAD_RCU(&container->prereg_list);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000330
331 container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000332
333 return container;
334}
335
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000336static int tce_iommu_clear(struct tce_container *container,
337 struct iommu_table *tbl,
338 unsigned long entry, unsigned long pages);
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100339static void tce_iommu_free_table(struct tce_container *container,
340 struct iommu_table *tbl);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000341
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000342static void tce_iommu_release(void *iommu_data)
343{
344 struct tce_container *container = iommu_data;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000345 struct tce_iommu_group *tcegrp;
Alexey Kardashevskiy517ad4a2018-10-02 13:22:31 +1000346 struct tce_iommu_prereg *tcemem, *tmtmp;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000347 long i;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000348
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000349 while (tce_groups_attached(container)) {
350 tcegrp = list_first_entry(&container->group_list,
351 struct tce_iommu_group, next);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000352 tce_iommu_detach_group(iommu_data, tcegrp->grp);
353 }
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000354
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000355 /*
356 * If VFIO created a table, it was not disposed
357 * by tce_iommu_detach_group() so do it now.
358 */
359 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
360 struct iommu_table *tbl = container->tables[i];
361
362 if (!tbl)
363 continue;
364
365 tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100366 tce_iommu_free_table(container, tbl);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000367 }
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000368
Alexey Kardashevskiy517ad4a2018-10-02 13:22:31 +1000369 list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)
370 WARN_ON(tce_iommu_prereg_free(container, tcemem));
Alexey Kardashevskiy4b6fad72016-11-30 17:52:05 +1100371
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000372 tce_iommu_disable(container);
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100373 if (container->mm)
374 mmdrop(container->mm);
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000375 mutex_destroy(&container->lock);
376
377 kfree(container);
378}
379
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000380static void tce_iommu_unuse_page(struct tce_container *container,
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000381 unsigned long hpa)
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000382{
383 struct page *page;
384
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000385 page = pfn_to_page(hpa >> PAGE_SHIFT);
John Hubbard9d532f22020-07-27 13:43:38 -0600386 unpin_user_page(page);
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000387}
388
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100389static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
Alexey Kardashevskiy1463edc2018-07-17 17:19:12 +1000390 unsigned long tce, unsigned long shift,
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000391 unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
392{
393 long ret = 0;
394 struct mm_iommu_table_group_mem_t *mem;
395
Alexey Kardashevskiy1463edc2018-07-17 17:19:12 +1000396 mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000397 if (!mem)
398 return -EINVAL;
399
Alexey Kardashevskiy76fa4972018-07-17 17:19:13 +1000400 ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000401 if (ret)
402 return -EINVAL;
403
404 *pmem = mem;
405
406 return 0;
407}
408
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100409static void tce_iommu_unuse_page_v2(struct tce_container *container,
410 struct iommu_table *tbl, unsigned long entry)
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000411{
412 struct mm_iommu_table_group_mem_t *mem = NULL;
413 int ret;
414 unsigned long hpa = 0;
Alexey Kardashevskiy6e301a82018-10-15 21:08:41 +1100415 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000416
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100417 if (!pua)
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000418 return;
419
Alexey Kardashevskiy00a5c582018-07-04 16:13:46 +1000420 ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
Michael Ellermanb3124ec2018-08-13 15:59:06 +1000421 tbl->it_page_shift, &hpa, &mem);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000422 if (ret)
Alexey Kardashevskiy00a5c582018-07-04 16:13:46 +1000423 pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
424 __func__, be64_to_cpu(*pua), entry, ret);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000425 if (mem)
426 mm_iommu_mapped_dec(mem);
427
Alexey Kardashevskiy00a5c582018-07-04 16:13:46 +1000428 *pua = cpu_to_be64(0);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000429}
430
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000431static int tce_iommu_clear(struct tce_container *container,
432 struct iommu_table *tbl,
433 unsigned long entry, unsigned long pages)
434{
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000435 unsigned long oldhpa;
436 long ret;
437 enum dma_data_direction direction;
Alexey Kardashevskiy650ab1e2019-08-29 18:52:50 +1000438 unsigned long lastentry = entry + pages, firstentry = entry;
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000439
Alexey Kardashevskiy6e301a82018-10-15 21:08:41 +1100440 for ( ; entry < lastentry; ++entry) {
441 if (tbl->it_indirect_levels && tbl->it_userspace) {
442 /*
443 * For multilevel tables, we can take a shortcut here
444 * and skip some TCEs as we know that the userspace
445 * addresses cache is a mirror of the real TCE table
446 * and if it is missing some indirect levels, then
447 * the hardware table does not have them allocated
448 * either and therefore does not require updating.
449 */
450 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
451 entry);
452 if (!pua) {
453 /* align to level_size which is power of two */
454 entry |= tbl->it_level_size - 1;
455 continue;
456 }
457 }
458
Alexey Kardashevskiy5c2fefd2017-10-02 12:39:11 -0600459 cond_resched();
460
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000461 direction = DMA_NONE;
462 oldhpa = 0;
Alexey Kardashevskiy650ab1e2019-08-29 18:52:50 +1000463 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
Alexey Kardashevskiyc10c21e2018-12-19 19:52:15 +1100464 &direction);
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000465 if (ret)
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000466 continue;
467
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000468 if (direction == DMA_NONE)
469 continue;
470
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000471 if (container->v2) {
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100472 tce_iommu_unuse_page_v2(container, tbl, entry);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000473 continue;
474 }
475
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000476 tce_iommu_unuse_page(container, oldhpa);
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000477 }
478
Alexey Kardashevskiy650ab1e2019-08-29 18:52:50 +1000479 iommu_tce_kill(tbl, firstentry, pages);
480
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000481 return 0;
482}
483
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000484static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
485{
486 struct page *page = NULL;
487 enum dma_data_direction direction = iommu_tce_direction(tce);
488
John Hubbard9d532f22020-07-27 13:43:38 -0600489 if (pin_user_pages_fast(tce & PAGE_MASK, 1,
Ira Weiny73b01402019-05-13 17:17:11 -0700490 direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
491 &page) != 1)
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000492 return -EFAULT;
493
494 *hpa = __pa((unsigned long) page_address(page));
495
496 return 0;
497}
498
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000499static long tce_iommu_build(struct tce_container *container,
500 struct iommu_table *tbl,
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000501 unsigned long entry, unsigned long tce, unsigned long pages,
502 enum dma_data_direction direction)
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000503{
504 long i, ret = 0;
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000505 unsigned long hpa;
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000506 enum dma_data_direction dirtmp;
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000507
508 for (i = 0; i < pages; ++i) {
509 unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
510
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000511 ret = tce_iommu_use_page(tce, &hpa);
512 if (ret)
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000513 break;
Alexey Kardashevskiye432bc72015-06-05 16:34:59 +1000514
Alexey Kardashevskiyc10c21e2018-12-19 19:52:15 +1100515 if (!tce_page_is_contained(container->mm, hpa,
516 tbl->it_page_shift)) {
Alexey Kardashevskiye432bc72015-06-05 16:34:59 +1000517 ret = -EPERM;
518 break;
519 }
520
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000521 hpa |= offset;
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000522 dirtmp = direction;
Alexey Kardashevskiy650ab1e2019-08-29 18:52:50 +1000523 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
524 &hpa, &dirtmp);
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000525 if (ret) {
Alexey Kardashevskiy649354b2015-06-05 16:35:03 +1000526 tce_iommu_unuse_page(container, hpa);
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000527 pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
528 __func__, entry << tbl->it_page_shift,
529 tce, ret);
530 break;
531 }
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000532
533 if (dirtmp != DMA_NONE)
534 tce_iommu_unuse_page(container, hpa);
535
Alexey Kardashevskiy00663d42015-06-05 16:35:00 +1000536 tce += IOMMU_PAGE_SIZE(tbl);
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000537 }
538
539 if (ret)
540 tce_iommu_clear(container, tbl, entry, i);
Alexey Kardashevskiy650ab1e2019-08-29 18:52:50 +1000541 else
542 iommu_tce_kill(tbl, entry, pages);
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000543
544 return ret;
545}
546
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000547static long tce_iommu_build_v2(struct tce_container *container,
548 struct iommu_table *tbl,
549 unsigned long entry, unsigned long tce, unsigned long pages,
550 enum dma_data_direction direction)
551{
552 long i, ret = 0;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000553 unsigned long hpa;
554 enum dma_data_direction dirtmp;
555
556 for (i = 0; i < pages; ++i) {
557 struct mm_iommu_table_group_mem_t *mem = NULL;
Alexey Kardashevskiy00a5c582018-07-04 16:13:46 +1000558 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000559
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100560 ret = tce_iommu_prereg_ua_to_hpa(container,
Alexey Kardashevskiy1463edc2018-07-17 17:19:12 +1000561 tce, tbl->it_page_shift, &hpa, &mem);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000562 if (ret)
563 break;
564
Alexey Kardashevskiyc10c21e2018-12-19 19:52:15 +1100565 if (!tce_page_is_contained(container->mm, hpa,
566 tbl->it_page_shift)) {
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000567 ret = -EPERM;
568 break;
569 }
570
571 /* Preserve offset within IOMMU page */
572 hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
573 dirtmp = direction;
574
575 /* The registered region is being unregistered */
576 if (mm_iommu_mapped_inc(mem))
577 break;
578
Alexey Kardashevskiy650ab1e2019-08-29 18:52:50 +1000579 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
580 &hpa, &dirtmp);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000581 if (ret) {
582 /* dirtmp cannot be DMA_NONE here */
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100583 tce_iommu_unuse_page_v2(container, tbl, entry + i);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000584 pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
585 __func__, entry << tbl->it_page_shift,
586 tce, ret);
587 break;
588 }
589
590 if (dirtmp != DMA_NONE)
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100591 tce_iommu_unuse_page_v2(container, tbl, entry + i);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000592
Alexey Kardashevskiy00a5c582018-07-04 16:13:46 +1000593 *pua = cpu_to_be64(tce);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000594
595 tce += IOMMU_PAGE_SIZE(tbl);
596 }
597
598 if (ret)
599 tce_iommu_clear(container, tbl, entry, i);
Alexey Kardashevskiy650ab1e2019-08-29 18:52:50 +1000600 else
601 iommu_tce_kill(tbl, entry, pages);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000602
603 return ret;
604}
605
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +1000606static long tce_iommu_create_table(struct tce_container *container,
607 struct iommu_table_group *table_group,
608 int num,
609 __u32 page_shift,
610 __u64 window_size,
611 __u32 levels,
612 struct iommu_table **ptbl)
613{
614 long ret, table_size;
615
616 table_size = table_group->ops->get_table_size(page_shift, window_size,
617 levels);
618 if (!table_size)
619 return -EINVAL;
620
Daniel Jordan79eb5972019-07-16 16:30:54 -0700621 ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +1000622 if (ret)
623 return ret;
624
625 ret = table_group->ops->create_table(table_group, num,
626 page_shift, window_size, levels, ptbl);
627
628 WARN_ON(!ret && !(*ptbl)->it_ops->free);
Alexey Kardashevskiya68bd122018-07-04 16:13:49 +1000629 WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +1000630
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +1000631 return ret;
632}
633
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100634static void tce_iommu_free_table(struct tce_container *container,
635 struct iommu_table *tbl)
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +1000636{
637 unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
638
Alexey Kardashevskiye5afdf92017-03-22 15:21:50 +1100639 iommu_tce_table_put(tbl);
Daniel Jordan79eb5972019-07-16 16:30:54 -0700640 account_locked_vm(container->mm, pages, false);
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +1000641}
642
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +1000643static long tce_iommu_create_window(struct tce_container *container,
644 __u32 page_shift, __u64 window_size, __u32 levels,
645 __u64 *start_addr)
646{
647 struct tce_iommu_group *tcegrp;
648 struct iommu_table_group *table_group;
649 struct iommu_table *tbl = NULL;
650 long ret, num;
651
652 num = tce_iommu_find_free_table(container);
653 if (num < 0)
654 return num;
655
656 /* Get the first group for ops::create_table */
657 tcegrp = list_first_entry(&container->group_list,
658 struct tce_iommu_group, next);
659 table_group = iommu_group_get_iommudata(tcegrp->grp);
660 if (!table_group)
661 return -EFAULT;
662
663 if (!(table_group->pgsizes & (1ULL << page_shift)))
664 return -EINVAL;
665
666 if (!table_group->ops->set_window || !table_group->ops->unset_window ||
667 !table_group->ops->get_table_size ||
668 !table_group->ops->create_table)
669 return -EPERM;
670
671 /* Create TCE table */
672 ret = tce_iommu_create_table(container, table_group, num,
673 page_shift, window_size, levels, &tbl);
674 if (ret)
675 return ret;
676
677 BUG_ON(!tbl->it_ops->free);
678
679 /*
680 * Program the table to every group.
681 * Groups have been tested for compatibility at the attach time.
682 */
683 list_for_each_entry(tcegrp, &container->group_list, next) {
684 table_group = iommu_group_get_iommudata(tcegrp->grp);
685
686 ret = table_group->ops->set_window(table_group, num, tbl);
687 if (ret)
688 goto unset_exit;
689 }
690
691 container->tables[num] = tbl;
692
693 /* Return start address assigned by platform in create_table() */
694 *start_addr = tbl->it_offset << tbl->it_page_shift;
695
696 return 0;
697
698unset_exit:
699 list_for_each_entry(tcegrp, &container->group_list, next) {
700 table_group = iommu_group_get_iommudata(tcegrp->grp);
701 table_group->ops->unset_window(table_group, num);
702 }
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100703 tce_iommu_free_table(container, tbl);
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +1000704
705 return ret;
706}
707
708static long tce_iommu_remove_window(struct tce_container *container,
709 __u64 start_addr)
710{
711 struct iommu_table_group *table_group = NULL;
712 struct iommu_table *tbl;
713 struct tce_iommu_group *tcegrp;
714 int num;
715
716 num = tce_iommu_find_table(container, start_addr, &tbl);
717 if (num < 0)
718 return -EINVAL;
719
720 BUG_ON(!tbl->it_size);
721
722 /* Detach groups from IOMMUs */
723 list_for_each_entry(tcegrp, &container->group_list, next) {
724 table_group = iommu_group_get_iommudata(tcegrp->grp);
725
726 /*
727 * SPAPR TCE IOMMU exposes the default DMA window to
728 * the guest via dma32_window_start/size of
729 * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow
730 * the userspace to remove this window, some do not so
731 * here we check for the platform capability.
732 */
733 if (!table_group->ops || !table_group->ops->unset_window)
734 return -EPERM;
735
736 table_group->ops->unset_window(table_group, num);
737 }
738
739 /* Free table */
740 tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100741 tce_iommu_free_table(container, tbl);
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +1000742 container->tables[num] = NULL;
743
744 return 0;
745}
746
Alexey Kardashevskiy6f01cc62016-11-30 17:52:02 +1100747static long tce_iommu_create_default_window(struct tce_container *container)
748{
749 long ret;
750 __u64 start_addr = 0;
751 struct tce_iommu_group *tcegrp;
752 struct iommu_table_group *table_group;
753
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +1100754 if (!container->def_window_pending)
755 return 0;
756
Alexey Kardashevskiy6f01cc62016-11-30 17:52:02 +1100757 if (!tce_groups_attached(container))
758 return -ENODEV;
759
760 tcegrp = list_first_entry(&container->group_list,
761 struct tce_iommu_group, next);
762 table_group = iommu_group_get_iommudata(tcegrp->grp);
763 if (!table_group)
764 return -ENODEV;
765
766 ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
767 table_group->tce32_size, 1, &start_addr);
768 WARN_ON_ONCE(!ret && start_addr);
769
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +1100770 if (!ret)
771 container->def_window_pending = false;
772
Alexey Kardashevskiy6f01cc62016-11-30 17:52:02 +1100773 return ret;
774}
775
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000776static long tce_iommu_ioctl(void *iommu_data,
777 unsigned int cmd, unsigned long arg)
778{
779 struct tce_container *container = iommu_data;
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +1000780 unsigned long minsz, ddwsz;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000781 long ret;
782
783 switch (cmd) {
784 case VFIO_CHECK_EXTENSION:
Gavin Shan1b69be52014-06-10 11:41:57 +1000785 switch (arg) {
786 case VFIO_SPAPR_TCE_IOMMU:
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000787 case VFIO_SPAPR_TCE_v2_IOMMU:
Gavin Shan1b69be52014-06-10 11:41:57 +1000788 ret = 1;
789 break;
790 default:
791 ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg);
792 break;
793 }
794
795 return (ret < 0) ? 0 : ret;
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100796 }
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000797
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100798 /*
799 * Sanity check to prevent one userspace from manipulating
800 * another userspace mm.
801 */
802 BUG_ON(!container);
803 if (container->mm && container->mm != current->mm)
804 return -EPERM;
805
806 switch (cmd) {
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000807 case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
808 struct vfio_iommu_spapr_tce_info info;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000809 struct tce_iommu_group *tcegrp;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000810 struct iommu_table_group *table_group;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000811
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000812 if (!tce_groups_attached(container))
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000813 return -ENXIO;
814
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000815 tcegrp = list_first_entry(&container->group_list,
816 struct tce_iommu_group, next);
817 table_group = iommu_group_get_iommudata(tcegrp->grp);
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000818
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +1000819 if (!table_group)
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000820 return -ENXIO;
821
822 minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
823 dma32_window_size);
824
825 if (copy_from_user(&info, (void __user *)arg, minsz))
826 return -EFAULT;
827
828 if (info.argsz < minsz)
829 return -EINVAL;
830
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +1000831 info.dma32_window_start = table_group->tce32_start;
832 info.dma32_window_size = table_group->tce32_size;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000833 info.flags = 0;
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +1000834 memset(&info.ddw, 0, sizeof(info.ddw));
835
836 if (table_group->max_dynamic_windows_supported &&
837 container->v2) {
838 info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;
839 info.ddw.pgsizes = table_group->pgsizes;
840 info.ddw.max_dynamic_windows_supported =
841 table_group->max_dynamic_windows_supported;
842 info.ddw.levels = table_group->max_levels;
843 }
844
845 ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);
846
847 if (info.argsz >= ddwsz)
848 minsz = ddwsz;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000849
850 if (copy_to_user((void __user *)arg, &info, minsz))
851 return -EFAULT;
852
853 return 0;
854 }
855 case VFIO_IOMMU_MAP_DMA: {
856 struct vfio_iommu_type1_dma_map param;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000857 struct iommu_table *tbl = NULL;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000858 long num;
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000859 enum dma_data_direction direction;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000860
Alexey Kardashevskiy3c56e822015-06-05 16:35:02 +1000861 if (!container->enabled)
862 return -EPERM;
863
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000864 minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
865
866 if (copy_from_user(&param, (void __user *)arg, minsz))
867 return -EFAULT;
868
869 if (param.argsz < minsz)
870 return -EINVAL;
871
872 if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
873 VFIO_DMA_MAP_FLAG_WRITE))
874 return -EINVAL;
875
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +1100876 ret = tce_iommu_create_default_window(container);
877 if (ret)
878 return ret;
879
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000880 num = tce_iommu_find_table(container, param.iova, &tbl);
881 if (num < 0)
882 return -ENXIO;
883
Alexey Kardashevskiy00663d42015-06-05 16:35:00 +1000884 if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
885 (param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000886 return -EINVAL;
887
888 /* iova is checked by the IOMMU API */
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000889 if (param.flags & VFIO_DMA_MAP_FLAG_READ) {
890 if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
891 direction = DMA_BIDIRECTIONAL;
892 else
893 direction = DMA_TO_DEVICE;
894 } else {
895 if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
896 direction = DMA_FROM_DEVICE;
897 else
898 return -EINVAL;
899 }
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000900
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +1000901 ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000902 if (ret)
903 return ret;
904
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000905 if (container->v2)
906 ret = tce_iommu_build_v2(container, tbl,
907 param.iova >> tbl->it_page_shift,
908 param.vaddr,
909 param.size >> tbl->it_page_shift,
910 direction);
911 else
912 ret = tce_iommu_build(container, tbl,
913 param.iova >> tbl->it_page_shift,
914 param.vaddr,
915 param.size >> tbl->it_page_shift,
916 direction);
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000917
918 iommu_flush_tce(tbl);
919
920 return ret;
921 }
922 case VFIO_IOMMU_UNMAP_DMA: {
923 struct vfio_iommu_type1_dma_unmap param;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000924 struct iommu_table *tbl = NULL;
925 long num;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000926
Alexey Kardashevskiy3c56e822015-06-05 16:35:02 +1000927 if (!container->enabled)
928 return -EPERM;
929
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000930 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
931 size);
932
933 if (copy_from_user(&param, (void __user *)arg, minsz))
934 return -EFAULT;
935
936 if (param.argsz < minsz)
937 return -EINVAL;
938
939 /* No flag is supported now */
940 if (param.flags)
941 return -EINVAL;
942
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +1100943 ret = tce_iommu_create_default_window(container);
944 if (ret)
945 return ret;
946
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +1000947 num = tce_iommu_find_table(container, param.iova, &tbl);
948 if (num < 0)
949 return -ENXIO;
950
Alexey Kardashevskiy00663d42015-06-05 16:35:00 +1000951 if (param.size & ~IOMMU_PAGE_MASK(tbl))
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000952 return -EINVAL;
953
954 ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
Alexey Kardashevskiy00663d42015-06-05 16:35:00 +1000955 param.size >> tbl->it_page_shift);
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000956 if (ret)
957 return ret;
958
Alexey Kardashevskiy9b14a1f2015-06-05 16:34:58 +1000959 ret = tce_iommu_clear(container, tbl,
Alexey Kardashevskiy00663d42015-06-05 16:35:00 +1000960 param.iova >> tbl->it_page_shift,
961 param.size >> tbl->it_page_shift);
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +1000962 iommu_flush_tce(tbl);
963
964 return ret;
965 }
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000966 case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {
967 struct vfio_iommu_spapr_register_memory param;
968
969 if (!container->v2)
970 break;
971
972 minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
973 size);
974
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +1100975 ret = tce_iommu_mm_set(container);
976 if (ret)
977 return ret;
978
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +1000979 if (copy_from_user(&param, (void __user *)arg, minsz))
980 return -EFAULT;
981
982 if (param.argsz < minsz)
983 return -EINVAL;
984
985 /* No flag is supported now */
986 if (param.flags)
987 return -EINVAL;
988
989 mutex_lock(&container->lock);
990 ret = tce_iommu_register_pages(container, param.vaddr,
991 param.size);
992 mutex_unlock(&container->lock);
993
994 return ret;
995 }
996 case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {
997 struct vfio_iommu_spapr_register_memory param;
998
999 if (!container->v2)
1000 break;
1001
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +11001002 if (!container->mm)
1003 return -EPERM;
1004
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001005 minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1006 size);
1007
1008 if (copy_from_user(&param, (void __user *)arg, minsz))
1009 return -EFAULT;
1010
1011 if (param.argsz < minsz)
1012 return -EINVAL;
1013
1014 /* No flag is supported now */
1015 if (param.flags)
1016 return -EINVAL;
1017
1018 mutex_lock(&container->lock);
1019 ret = tce_iommu_unregister_pages(container, param.vaddr,
1020 param.size);
1021 mutex_unlock(&container->lock);
1022
1023 return ret;
1024 }
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001025 case VFIO_IOMMU_ENABLE:
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001026 if (container->v2)
1027 break;
1028
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001029 mutex_lock(&container->lock);
1030 ret = tce_iommu_enable(container);
1031 mutex_unlock(&container->lock);
1032 return ret;
1033
1034
1035 case VFIO_IOMMU_DISABLE:
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001036 if (container->v2)
1037 break;
1038
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001039 mutex_lock(&container->lock);
1040 tce_iommu_disable(container);
1041 mutex_unlock(&container->lock);
1042 return 0;
Gavin Shan1b69be52014-06-10 11:41:57 +10001043
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001044 case VFIO_EEH_PE_OP: {
1045 struct tce_iommu_group *tcegrp;
1046
1047 ret = 0;
1048 list_for_each_entry(tcegrp, &container->group_list, next) {
1049 ret = vfio_spapr_iommu_eeh_ioctl(tcegrp->grp,
1050 cmd, arg);
1051 if (ret)
1052 return ret;
1053 }
1054 return ret;
1055 }
1056
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +10001057 case VFIO_IOMMU_SPAPR_TCE_CREATE: {
1058 struct vfio_iommu_spapr_tce_create create;
1059
1060 if (!container->v2)
1061 break;
1062
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +11001063 ret = tce_iommu_mm_set(container);
1064 if (ret)
1065 return ret;
1066
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +10001067 if (!tce_groups_attached(container))
1068 return -ENXIO;
1069
1070 minsz = offsetofend(struct vfio_iommu_spapr_tce_create,
1071 start_addr);
1072
1073 if (copy_from_user(&create, (void __user *)arg, minsz))
1074 return -EFAULT;
1075
1076 if (create.argsz < minsz)
1077 return -EINVAL;
1078
1079 if (create.flags)
1080 return -EINVAL;
1081
1082 mutex_lock(&container->lock);
1083
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +11001084 ret = tce_iommu_create_default_window(container);
Alexey Kardashevskiy2da64d22017-02-01 14:26:16 +11001085 if (!ret)
1086 ret = tce_iommu_create_window(container,
1087 create.page_shift,
1088 create.window_size, create.levels,
1089 &create.start_addr);
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +10001090
1091 mutex_unlock(&container->lock);
1092
1093 if (!ret && copy_to_user((void __user *)arg, &create, minsz))
1094 ret = -EFAULT;
1095
1096 return ret;
1097 }
1098 case VFIO_IOMMU_SPAPR_TCE_REMOVE: {
1099 struct vfio_iommu_spapr_tce_remove remove;
1100
1101 if (!container->v2)
1102 break;
1103
Alexey Kardashevskiybc82d122016-11-30 17:52:04 +11001104 ret = tce_iommu_mm_set(container);
1105 if (ret)
1106 return ret;
1107
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +10001108 if (!tce_groups_attached(container))
1109 return -ENXIO;
1110
1111 minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,
1112 start_addr);
1113
1114 if (copy_from_user(&remove, (void __user *)arg, minsz))
1115 return -EFAULT;
1116
1117 if (remove.argsz < minsz)
1118 return -EINVAL;
1119
1120 if (remove.flags)
1121 return -EINVAL;
1122
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +11001123 if (container->def_window_pending && !remove.start_addr) {
1124 container->def_window_pending = false;
1125 return 0;
1126 }
1127
Alexey Kardashevskiye633bc82015-06-05 16:35:26 +10001128 mutex_lock(&container->lock);
1129
1130 ret = tce_iommu_remove_window(container, remove.start_addr);
1131
1132 mutex_unlock(&container->lock);
1133
1134 return ret;
1135 }
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001136 }
1137
1138 return -ENOTTY;
1139}
1140
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001141static void tce_iommu_release_ownership(struct tce_container *container,
1142 struct iommu_table_group *table_group)
1143{
1144 int i;
1145
1146 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001147 struct iommu_table *tbl = container->tables[i];
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001148
1149 if (!tbl)
1150 continue;
1151
1152 tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
1153 if (tbl->it_map)
1154 iommu_release_ownership(tbl);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001155
1156 container->tables[i] = NULL;
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001157 }
1158}
1159
1160static int tce_iommu_take_ownership(struct tce_container *container,
1161 struct iommu_table_group *table_group)
1162{
1163 int i, j, rc = 0;
1164
1165 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1166 struct iommu_table *tbl = table_group->tables[i];
1167
1168 if (!tbl || !tbl->it_map)
1169 continue;
1170
Alexey Kardashevskiy39701e52016-11-30 17:52:01 +11001171 rc = iommu_take_ownership(tbl);
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001172 if (rc) {
1173 for (j = 0; j < i; ++j)
1174 iommu_release_ownership(
1175 table_group->tables[j]);
1176
1177 return rc;
1178 }
1179 }
1180
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001181 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1182 container->tables[i] = table_group->tables[i];
1183
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001184 return 0;
1185}
1186
1187static void tce_iommu_release_ownership_ddw(struct tce_container *container,
1188 struct iommu_table_group *table_group)
1189{
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10001190 long i;
1191
1192 if (!table_group->ops->unset_window) {
1193 WARN_ON_ONCE(1);
1194 return;
1195 }
1196
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001197 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
Alexey Kardashevskiya3906852019-02-11 18:49:17 +11001198 if (container->tables[i])
1199 table_group->ops->unset_window(table_group, i);
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10001200
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001201 table_group->ops->release_ownership(table_group);
1202}
1203
1204static long tce_iommu_take_ownership_ddw(struct tce_container *container,
1205 struct iommu_table_group *table_group)
1206{
Alexey Kardashevskiy930a42d2017-02-07 17:26:57 +11001207 long i, ret = 0;
1208
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10001209 if (!table_group->ops->create_table || !table_group->ops->set_window ||
1210 !table_group->ops->release_ownership) {
1211 WARN_ON_ONCE(1);
1212 return -EFAULT;
1213 }
1214
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001215 table_group->ops->take_ownership(table_group);
1216
Alexey Kardashevskiy930a42d2017-02-07 17:26:57 +11001217 /* Set all windows to the new group */
1218 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1219 struct iommu_table *tbl = container->tables[i];
1220
1221 if (!tbl)
1222 continue;
1223
1224 ret = table_group->ops->set_window(table_group, i, tbl);
1225 if (ret)
1226 goto release_exit;
1227 }
1228
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001229 return 0;
Alexey Kardashevskiy930a42d2017-02-07 17:26:57 +11001230
1231release_exit:
1232 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1233 table_group->ops->unset_window(table_group, i);
1234
1235 table_group->ops->release_ownership(table_group);
1236
1237 return ret;
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001238}
1239
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001240static int tce_iommu_attach_group(void *iommu_data,
1241 struct iommu_group *iommu_group)
1242{
Alexey Kardashevskiy78becab2019-08-19 11:51:17 +10001243 int ret = 0;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001244 struct tce_container *container = iommu_data;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001245 struct iommu_table_group *table_group;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001246 struct tce_iommu_group *tcegrp = NULL;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001247
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001248 mutex_lock(&container->lock);
1249
1250 /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
1251 iommu_group_id(iommu_group), iommu_group); */
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001252 table_group = iommu_group_get_iommudata(iommu_group);
Greg Kurzbd00fdf2017-01-24 17:50:26 +01001253 if (!table_group) {
1254 ret = -ENODEV;
1255 goto unlock_exit;
1256 }
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001257
1258 if (tce_groups_attached(container) && (!table_group->ops ||
1259 !table_group->ops->take_ownership ||
1260 !table_group->ops->release_ownership)) {
1261 ret = -EBUSY;
1262 goto unlock_exit;
1263 }
1264
1265 /* Check if new group has the same iommu_ops (i.e. compatible) */
1266 list_for_each_entry(tcegrp, &container->group_list, next) {
1267 struct iommu_table_group *table_group_tmp;
1268
1269 if (tcegrp->grp == iommu_group) {
1270 pr_warn("tce_vfio: Group %d is already attached\n",
1271 iommu_group_id(iommu_group));
1272 ret = -EBUSY;
1273 goto unlock_exit;
1274 }
1275 table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
Alexey Kardashevskiy54de2852016-04-29 18:55:15 +10001276 if (table_group_tmp->ops->create_table !=
1277 table_group->ops->create_table) {
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001278 pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
1279 iommu_group_id(iommu_group),
1280 iommu_group_id(tcegrp->grp));
1281 ret = -EPERM;
1282 goto unlock_exit;
1283 }
1284 }
1285
1286 tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
1287 if (!tcegrp) {
1288 ret = -ENOMEM;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001289 goto unlock_exit;
1290 }
1291
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001292 if (!table_group->ops || !table_group->ops->take_ownership ||
Alexey Kardashevskiy6f01cc62016-11-30 17:52:02 +11001293 !table_group->ops->release_ownership) {
Alexey Kardashevskiy1282ba72017-03-24 17:44:06 +11001294 if (container->v2) {
1295 ret = -EPERM;
Alexey Kardashevskiy78becab2019-08-19 11:51:17 +10001296 goto free_exit;
Alexey Kardashevskiy1282ba72017-03-24 17:44:06 +11001297 }
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001298 ret = tce_iommu_take_ownership(container, table_group);
Alexey Kardashevskiy6f01cc62016-11-30 17:52:02 +11001299 } else {
Alexey Kardashevskiy1282ba72017-03-24 17:44:06 +11001300 if (!container->v2) {
1301 ret = -EPERM;
Alexey Kardashevskiy78becab2019-08-19 11:51:17 +10001302 goto free_exit;
Alexey Kardashevskiy1282ba72017-03-24 17:44:06 +11001303 }
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001304 ret = tce_iommu_take_ownership_ddw(container, table_group);
Alexey Kardashevskiy6f01cc62016-11-30 17:52:02 +11001305 if (!tce_groups_attached(container) && !container->tables[0])
Alexey Kardashevskiyd9c72892016-11-30 17:52:03 +11001306 container->def_window_pending = true;
Alexey Kardashevskiy6f01cc62016-11-30 17:52:02 +11001307 }
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001308
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001309 if (!ret) {
1310 tcegrp->grp = iommu_group;
1311 list_add(&tcegrp->next, &container->group_list);
1312 }
Alexey Kardashevskiy22af4852015-06-05 16:35:04 +10001313
Alexey Kardashevskiy78becab2019-08-19 11:51:17 +10001314free_exit:
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001315 if (ret && tcegrp)
1316 kfree(tcegrp);
1317
Alexey Kardashevskiy78becab2019-08-19 11:51:17 +10001318unlock_exit:
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001319 mutex_unlock(&container->lock);
1320
1321 return ret;
1322}
1323
1324static void tce_iommu_detach_group(void *iommu_data,
1325 struct iommu_group *iommu_group)
1326{
1327 struct tce_container *container = iommu_data;
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001328 struct iommu_table_group *table_group;
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001329 bool found = false;
1330 struct tce_iommu_group *tcegrp;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001331
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001332 mutex_lock(&container->lock);
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001333
1334 list_for_each_entry(tcegrp, &container->group_list, next) {
1335 if (tcegrp->grp == iommu_group) {
1336 found = true;
1337 break;
1338 }
1339 }
1340
1341 if (!found) {
1342 pr_warn("tce_vfio: detaching unattached group #%u\n",
1343 iommu_group_id(iommu_group));
Alexey Kardashevskiy22af4852015-06-05 16:35:04 +10001344 goto unlock_exit;
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001345 }
Alexey Kardashevskiy22af4852015-06-05 16:35:04 +10001346
Alexey Kardashevskiy2157e7b2015-06-05 16:35:25 +10001347 list_del(&tcegrp->next);
1348 kfree(tcegrp);
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001349
1350 table_group = iommu_group_get_iommudata(iommu_group);
1351 BUG_ON(!table_group);
1352
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10001353 if (!table_group->ops || !table_group->ops->release_ownership)
1354 tce_iommu_release_ownership(container, table_group);
1355 else
1356 tce_iommu_release_ownership_ddw(container, table_group);
Alexey Kardashevskiy22af4852015-06-05 16:35:04 +10001357
1358unlock_exit:
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001359 mutex_unlock(&container->lock);
1360}
1361
Wang Haie39dd512019-04-03 12:36:21 -06001362static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001363 .name = "iommu-vfio-powerpc",
1364 .owner = THIS_MODULE,
1365 .open = tce_iommu_open,
1366 .release = tce_iommu_release,
1367 .ioctl = tce_iommu_ioctl,
1368 .attach_group = tce_iommu_attach_group,
1369 .detach_group = tce_iommu_detach_group,
1370};
1371
1372static int __init tce_iommu_init(void)
1373{
1374 return vfio_register_iommu_driver(&tce_iommu_driver_ops);
1375}
1376
1377static void __exit tce_iommu_cleanup(void)
1378{
1379 vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
1380}
1381
1382module_init(tce_iommu_init);
1383module_exit(tce_iommu_cleanup);
1384
1385MODULE_VERSION(DRIVER_VERSION);
1386MODULE_LICENSE("GPL v2");
1387MODULE_AUTHOR(DRIVER_AUTHOR);
1388MODULE_DESCRIPTION(DRIVER_DESC);
1389