Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Gerd Hoffmann | 913965c | 2018-09-11 15:42:04 +0200 | [diff] [blame] | 2 | #include <linux/cred.h> |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 3 | #include <linux/device.h> |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 4 | #include <linux/dma-buf.h> |
Dmitry Osipenko | aa3f998 | 2022-11-10 23:13:46 +0300 | [diff] [blame] | 5 | #include <linux/dma-resv.h> |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 6 | #include <linux/highmem.h> |
Gerd Hoffmann | 913965c | 2018-09-11 15:42:04 +0200 | [diff] [blame] | 7 | #include <linux/init.h> |
| 8 | #include <linux/kernel.h> |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 9 | #include <linux/memfd.h> |
Gerd Hoffmann | 913965c | 2018-09-11 15:42:04 +0200 | [diff] [blame] | 10 | #include <linux/miscdevice.h> |
| 11 | #include <linux/module.h> |
| 12 | #include <linux/shmem_fs.h> |
| 13 | #include <linux/slab.h> |
| 14 | #include <linux/udmabuf.h> |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 15 | #include <linux/hugetlb.h> |
Lukasz Wiecaszek | 7ae2e68 | 2022-11-17 18:18:09 +0100 | [diff] [blame] | 16 | #include <linux/vmalloc.h> |
| 17 | #include <linux/iosys-map.h> |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 18 | |
Dongwon Kim | 2e717a5 | 2021-06-11 14:21:07 -0700 | [diff] [blame] | 19 | static int list_limit = 1024; |
| 20 | module_param(list_limit, int, 0644); |
| 21 | MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024."); |
| 22 | |
| 23 | static int size_limit_mb = 64; |
| 24 | module_param(size_limit_mb, int, 0644); |
| 25 | MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64."); |
Gerd Hoffmann | dc4716d | 2018-09-11 15:42:10 +0200 | [diff] [blame] | 26 | |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 27 | struct udmabuf { |
Gerd Hoffmann | b35f57c | 2018-09-11 15:42:06 +0200 | [diff] [blame] | 28 | pgoff_t pagecount; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 29 | struct page **pages; |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 30 | struct sg_table *sg; |
Gurchetan Singh | c1bbed6 | 2019-12-02 17:36:25 -0800 | [diff] [blame] | 31 | struct miscdevice *device; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 32 | }; |
| 33 | |
Souptick Joarder | 300133d | 2019-01-03 15:26:34 -0800 | [diff] [blame] | 34 | static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 35 | { |
| 36 | struct vm_area_struct *vma = vmf->vma; |
| 37 | struct udmabuf *ubuf = vma->vm_private_data; |
Gerd Hoffmann | 05b252c | 2022-06-20 09:15:47 +0200 | [diff] [blame] | 38 | pgoff_t pgoff = vmf->pgoff; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 39 | |
Gerd Hoffmann | 05b252c | 2022-06-20 09:15:47 +0200 | [diff] [blame] | 40 | if (pgoff >= ubuf->pagecount) |
| 41 | return VM_FAULT_SIGBUS; |
| 42 | vmf->page = ubuf->pages[pgoff]; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 43 | get_page(vmf->page); |
| 44 | return 0; |
| 45 | } |
| 46 | |
| 47 | static const struct vm_operations_struct udmabuf_vm_ops = { |
| 48 | .fault = udmabuf_vm_fault, |
| 49 | }; |
| 50 | |
| 51 | static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) |
| 52 | { |
| 53 | struct udmabuf *ubuf = buf->priv; |
| 54 | |
Dmitry Osipenko | aa3f998 | 2022-11-10 23:13:46 +0300 | [diff] [blame] | 55 | dma_resv_assert_held(buf->resv); |
| 56 | |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 57 | if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) |
| 58 | return -EINVAL; |
| 59 | |
| 60 | vma->vm_ops = &udmabuf_vm_ops; |
| 61 | vma->vm_private_data = ubuf; |
| 62 | return 0; |
| 63 | } |
| 64 | |
Lukasz Wiecaszek | 7ae2e68 | 2022-11-17 18:18:09 +0100 | [diff] [blame] | 65 | static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) |
| 66 | { |
| 67 | struct udmabuf *ubuf = buf->priv; |
| 68 | void *vaddr; |
| 69 | |
| 70 | dma_resv_assert_held(buf->resv); |
| 71 | |
| 72 | vaddr = vm_map_ram(ubuf->pages, ubuf->pagecount, -1); |
| 73 | if (!vaddr) |
| 74 | return -EINVAL; |
| 75 | |
| 76 | iosys_map_set_vaddr(map, vaddr); |
| 77 | return 0; |
| 78 | } |
| 79 | |
| 80 | static void vunmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) |
| 81 | { |
| 82 | struct udmabuf *ubuf = buf->priv; |
| 83 | |
| 84 | dma_resv_assert_held(buf->resv); |
| 85 | |
| 86 | vm_unmap_ram(map->vaddr, ubuf->pagecount); |
| 87 | } |
| 88 | |
Gurchetan Singh | 17a7ce2 | 2019-12-02 17:36:26 -0800 | [diff] [blame] | 89 | static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, |
| 90 | enum dma_data_direction direction) |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 91 | { |
Gurchetan Singh | 17a7ce2 | 2019-12-02 17:36:26 -0800 | [diff] [blame] | 92 | struct udmabuf *ubuf = buf->priv; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 93 | struct sg_table *sg; |
Gerd Hoffmann | a3e722d | 2018-09-11 15:42:05 +0200 | [diff] [blame] | 94 | int ret; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 95 | |
| 96 | sg = kzalloc(sizeof(*sg), GFP_KERNEL); |
| 97 | if (!sg) |
Gerd Hoffmann | a3e722d | 2018-09-11 15:42:05 +0200 | [diff] [blame] | 98 | return ERR_PTR(-ENOMEM); |
| 99 | ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount, |
| 100 | 0, ubuf->pagecount << PAGE_SHIFT, |
| 101 | GFP_KERNEL); |
| 102 | if (ret < 0) |
| 103 | goto err; |
Marek Szyprowski | 62296b39 | 2020-04-06 16:41:45 +0200 | [diff] [blame] | 104 | ret = dma_map_sgtable(dev, sg, direction, 0); |
| 105 | if (ret < 0) |
Gerd Hoffmann | a3e722d | 2018-09-11 15:42:05 +0200 | [diff] [blame] | 106 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 107 | return sg; |
| 108 | |
Gerd Hoffmann | a3e722d | 2018-09-11 15:42:05 +0200 | [diff] [blame] | 109 | err: |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 110 | sg_free_table(sg); |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 111 | kfree(sg); |
Gerd Hoffmann | a3e722d | 2018-09-11 15:42:05 +0200 | [diff] [blame] | 112 | return ERR_PTR(ret); |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 113 | } |
| 114 | |
Gurchetan Singh | 17a7ce2 | 2019-12-02 17:36:26 -0800 | [diff] [blame] | 115 | static void put_sg_table(struct device *dev, struct sg_table *sg, |
| 116 | enum dma_data_direction direction) |
| 117 | { |
Marek Szyprowski | 62296b39 | 2020-04-06 16:41:45 +0200 | [diff] [blame] | 118 | dma_unmap_sgtable(dev, sg, direction, 0); |
Gurchetan Singh | 17a7ce2 | 2019-12-02 17:36:26 -0800 | [diff] [blame] | 119 | sg_free_table(sg); |
| 120 | kfree(sg); |
| 121 | } |
| 122 | |
| 123 | static struct sg_table *map_udmabuf(struct dma_buf_attachment *at, |
| 124 | enum dma_data_direction direction) |
| 125 | { |
| 126 | return get_sg_table(at->dev, at->dmabuf, direction); |
| 127 | } |
| 128 | |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 129 | static void unmap_udmabuf(struct dma_buf_attachment *at, |
| 130 | struct sg_table *sg, |
| 131 | enum dma_data_direction direction) |
| 132 | { |
Gurchetan Singh | 17a7ce2 | 2019-12-02 17:36:26 -0800 | [diff] [blame] | 133 | return put_sg_table(at->dev, sg, direction); |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 134 | } |
| 135 | |
| 136 | static void release_udmabuf(struct dma_buf *buf) |
| 137 | { |
| 138 | struct udmabuf *ubuf = buf->priv; |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 139 | struct device *dev = ubuf->device->this_device; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 140 | pgoff_t pg; |
| 141 | |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 142 | if (ubuf->sg) |
| 143 | put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL); |
| 144 | |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 145 | for (pg = 0; pg < ubuf->pagecount; pg++) |
| 146 | put_page(ubuf->pages[pg]); |
| 147 | kfree(ubuf->pages); |
| 148 | kfree(ubuf); |
| 149 | } |
| 150 | |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 151 | static int begin_cpu_udmabuf(struct dma_buf *buf, |
| 152 | enum dma_data_direction direction) |
| 153 | { |
| 154 | struct udmabuf *ubuf = buf->priv; |
| 155 | struct device *dev = ubuf->device->this_device; |
Vivek Kasireddy | d9c04a1 | 2022-08-24 23:35:22 -0700 | [diff] [blame] | 156 | int ret = 0; |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 157 | |
| 158 | if (!ubuf->sg) { |
| 159 | ubuf->sg = get_sg_table(dev, buf, direction); |
Vivek Kasireddy | d9c04a1 | 2022-08-24 23:35:22 -0700 | [diff] [blame] | 160 | if (IS_ERR(ubuf->sg)) { |
| 161 | ret = PTR_ERR(ubuf->sg); |
| 162 | ubuf->sg = NULL; |
| 163 | } |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 164 | } else { |
Gurchetan Singh | 1ffe095 | 2019-12-17 15:02:28 -0800 | [diff] [blame] | 165 | dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents, |
| 166 | direction); |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 167 | } |
| 168 | |
Vivek Kasireddy | d9c04a1 | 2022-08-24 23:35:22 -0700 | [diff] [blame] | 169 | return ret; |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 170 | } |
| 171 | |
| 172 | static int end_cpu_udmabuf(struct dma_buf *buf, |
| 173 | enum dma_data_direction direction) |
| 174 | { |
| 175 | struct udmabuf *ubuf = buf->priv; |
| 176 | struct device *dev = ubuf->device->this_device; |
| 177 | |
| 178 | if (!ubuf->sg) |
| 179 | return -EINVAL; |
| 180 | |
Gurchetan Singh | 1ffe095 | 2019-12-17 15:02:28 -0800 | [diff] [blame] | 181 | dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction); |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 182 | return 0; |
| 183 | } |
| 184 | |
Gerd Hoffmann | a348528 | 2018-09-11 15:42:07 +0200 | [diff] [blame] | 185 | static const struct dma_buf_ops udmabuf_ops = { |
Gurchetan Singh | bc7a71d | 2019-12-02 17:36:24 -0800 | [diff] [blame] | 186 | .cache_sgt_mapping = true, |
| 187 | .map_dma_buf = map_udmabuf, |
| 188 | .unmap_dma_buf = unmap_udmabuf, |
| 189 | .release = release_udmabuf, |
| 190 | .mmap = mmap_udmabuf, |
Lukasz Wiecaszek | 7ae2e68 | 2022-11-17 18:18:09 +0100 | [diff] [blame] | 191 | .vmap = vmap_udmabuf, |
| 192 | .vunmap = vunmap_udmabuf, |
Gurchetan Singh | 284562e | 2019-12-02 17:36:27 -0800 | [diff] [blame] | 193 | .begin_cpu_access = begin_cpu_udmabuf, |
| 194 | .end_cpu_access = end_cpu_udmabuf, |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 195 | }; |
| 196 | |
| 197 | #define SEALS_WANTED (F_SEAL_SHRINK) |
| 198 | #define SEALS_DENIED (F_SEAL_WRITE) |
| 199 | |
Gurchetan Singh | c1bbed6 | 2019-12-02 17:36:25 -0800 | [diff] [blame] | 200 | static long udmabuf_create(struct miscdevice *device, |
| 201 | struct udmabuf_create_list *head, |
| 202 | struct udmabuf_create_item *list) |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 203 | { |
| 204 | DEFINE_DMA_BUF_EXPORT_INFO(exp_info); |
| 205 | struct file *memfd = NULL; |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 206 | struct address_space *mapping = NULL; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 207 | struct udmabuf *ubuf; |
| 208 | struct dma_buf *buf; |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 209 | pgoff_t pgoff, pgcnt, pgidx, pgbuf = 0, pglimit; |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 210 | struct page *page, *hpage = NULL; |
| 211 | pgoff_t subpgoff, maxsubpgs; |
| 212 | struct hstate *hpstate; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 213 | int seals, ret = -EINVAL; |
| 214 | u32 i, flags; |
| 215 | |
Gerd Hoffmann | 33f3542 | 2018-09-11 15:42:15 +0200 | [diff] [blame] | 216 | ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL); |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 217 | if (!ubuf) |
| 218 | return -ENOMEM; |
| 219 | |
Gerd Hoffmann | dc4716d | 2018-09-11 15:42:10 +0200 | [diff] [blame] | 220 | pglimit = (size_limit_mb * 1024 * 1024) >> PAGE_SHIFT; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 221 | for (i = 0; i < head->count; i++) { |
| 222 | if (!IS_ALIGNED(list[i].offset, PAGE_SIZE)) |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 223 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 224 | if (!IS_ALIGNED(list[i].size, PAGE_SIZE)) |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 225 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 226 | ubuf->pagecount += list[i].size >> PAGE_SHIFT; |
Gerd Hoffmann | dc4716d | 2018-09-11 15:42:10 +0200 | [diff] [blame] | 227 | if (ubuf->pagecount > pglimit) |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 228 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 229 | } |
Pavel Skripkin | 2b6dd60 | 2021-12-30 17:26:49 +0300 | [diff] [blame] | 230 | |
| 231 | if (!ubuf->pagecount) |
| 232 | goto err; |
| 233 | |
Gerd Hoffmann | 33f3542 | 2018-09-11 15:42:15 +0200 | [diff] [blame] | 234 | ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->pages), |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 235 | GFP_KERNEL); |
| 236 | if (!ubuf->pages) { |
| 237 | ret = -ENOMEM; |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 238 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 239 | } |
| 240 | |
| 241 | pgbuf = 0; |
| 242 | for (i = 0; i < head->count; i++) { |
Gerd Hoffmann | 7a1c67d | 2018-09-11 15:42:12 +0200 | [diff] [blame] | 243 | ret = -EBADFD; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 244 | memfd = fget(list[i].memfd); |
| 245 | if (!memfd) |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 246 | goto err; |
Al Viro | ff58105 | 2022-08-20 13:10:13 -0400 | [diff] [blame] | 247 | mapping = memfd->f_mapping; |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 248 | if (!shmem_mapping(mapping) && !is_file_hugepages(memfd)) |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 249 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 250 | seals = memfd_fcntl(memfd, F_GET_SEALS, 0); |
Gerd Hoffmann | 7a1c67d | 2018-09-11 15:42:12 +0200 | [diff] [blame] | 251 | if (seals == -EINVAL) |
| 252 | goto err; |
| 253 | ret = -EINVAL; |
| 254 | if ((seals & SEALS_WANTED) != SEALS_WANTED || |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 255 | (seals & SEALS_DENIED) != 0) |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 256 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 257 | pgoff = list[i].offset >> PAGE_SHIFT; |
| 258 | pgcnt = list[i].size >> PAGE_SHIFT; |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 259 | if (is_file_hugepages(memfd)) { |
| 260 | hpstate = hstate_file(memfd); |
| 261 | pgoff = list[i].offset >> huge_page_shift(hpstate); |
| 262 | subpgoff = (list[i].offset & |
| 263 | ~huge_page_mask(hpstate)) >> PAGE_SHIFT; |
| 264 | maxsubpgs = huge_page_size(hpstate) >> PAGE_SHIFT; |
| 265 | } |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 266 | for (pgidx = 0; pgidx < pgcnt; pgidx++) { |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 267 | if (is_file_hugepages(memfd)) { |
| 268 | if (!hpage) { |
| 269 | hpage = find_get_page_flags(mapping, pgoff, |
| 270 | FGP_ACCESSED); |
Pavel Skripkin | b9770b0 | 2021-08-11 20:50:52 +0300 | [diff] [blame] | 271 | if (!hpage) { |
| 272 | ret = -EINVAL; |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 273 | goto err; |
| 274 | } |
| 275 | } |
| 276 | page = hpage + subpgoff; |
| 277 | get_page(page); |
| 278 | subpgoff++; |
| 279 | if (subpgoff == maxsubpgs) { |
| 280 | put_page(hpage); |
| 281 | hpage = NULL; |
| 282 | subpgoff = 0; |
| 283 | pgoff++; |
| 284 | } |
| 285 | } else { |
| 286 | page = shmem_read_mapping_page(mapping, |
| 287 | pgoff + pgidx); |
| 288 | if (IS_ERR(page)) { |
| 289 | ret = PTR_ERR(page); |
| 290 | goto err; |
| 291 | } |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 292 | } |
| 293 | ubuf->pages[pgbuf++] = page; |
| 294 | } |
| 295 | fput(memfd); |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 296 | memfd = NULL; |
Vivek Kasireddy | 16c243e | 2021-06-09 11:29:15 -0700 | [diff] [blame] | 297 | if (hpage) { |
| 298 | put_page(hpage); |
| 299 | hpage = NULL; |
| 300 | } |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 301 | } |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 302 | |
| 303 | exp_info.ops = &udmabuf_ops; |
| 304 | exp_info.size = ubuf->pagecount << PAGE_SHIFT; |
| 305 | exp_info.priv = ubuf; |
Gerd Hoffmann | 5c074ee | 2018-11-14 13:20:29 +0100 | [diff] [blame] | 306 | exp_info.flags = O_RDWR; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 307 | |
Gurchetan Singh | c1bbed6 | 2019-12-02 17:36:25 -0800 | [diff] [blame] | 308 | ubuf->device = device; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 309 | buf = dma_buf_export(&exp_info); |
| 310 | if (IS_ERR(buf)) { |
| 311 | ret = PTR_ERR(buf); |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 312 | goto err; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 313 | } |
| 314 | |
| 315 | flags = 0; |
| 316 | if (head->flags & UDMABUF_FLAGS_CLOEXEC) |
| 317 | flags |= O_CLOEXEC; |
| 318 | return dma_buf_fd(buf, flags); |
| 319 | |
Gerd Hoffmann | 0d17455 | 2018-09-11 15:42:11 +0200 | [diff] [blame] | 320 | err: |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 321 | while (pgbuf > 0) |
| 322 | put_page(ubuf->pages[--pgbuf]); |
Gustavo A. R. Silva | 683a0e6 | 2018-09-04 14:07:49 -0500 | [diff] [blame] | 323 | if (memfd) |
| 324 | fput(memfd); |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 325 | kfree(ubuf->pages); |
| 326 | kfree(ubuf); |
| 327 | return ret; |
| 328 | } |
| 329 | |
| 330 | static long udmabuf_ioctl_create(struct file *filp, unsigned long arg) |
| 331 | { |
| 332 | struct udmabuf_create create; |
| 333 | struct udmabuf_create_list head; |
| 334 | struct udmabuf_create_item list; |
| 335 | |
| 336 | if (copy_from_user(&create, (void __user *)arg, |
Gerd Hoffmann | 33f3542 | 2018-09-11 15:42:15 +0200 | [diff] [blame] | 337 | sizeof(create))) |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 338 | return -EFAULT; |
| 339 | |
| 340 | head.flags = create.flags; |
| 341 | head.count = 1; |
| 342 | list.memfd = create.memfd; |
| 343 | list.offset = create.offset; |
| 344 | list.size = create.size; |
| 345 | |
Gurchetan Singh | c1bbed6 | 2019-12-02 17:36:25 -0800 | [diff] [blame] | 346 | return udmabuf_create(filp->private_data, &head, &list); |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 347 | } |
| 348 | |
| 349 | static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg) |
| 350 | { |
| 351 | struct udmabuf_create_list head; |
| 352 | struct udmabuf_create_item *list; |
| 353 | int ret = -EINVAL; |
| 354 | u32 lsize; |
| 355 | |
| 356 | if (copy_from_user(&head, (void __user *)arg, sizeof(head))) |
| 357 | return -EFAULT; |
Gerd Hoffmann | dc4716d | 2018-09-11 15:42:10 +0200 | [diff] [blame] | 358 | if (head.count > list_limit) |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 359 | return -EINVAL; |
| 360 | lsize = sizeof(struct udmabuf_create_item) * head.count; |
| 361 | list = memdup_user((void __user *)(arg + sizeof(head)), lsize); |
| 362 | if (IS_ERR(list)) |
| 363 | return PTR_ERR(list); |
| 364 | |
Gurchetan Singh | c1bbed6 | 2019-12-02 17:36:25 -0800 | [diff] [blame] | 365 | ret = udmabuf_create(filp->private_data, &head, list); |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 366 | kfree(list); |
| 367 | return ret; |
| 368 | } |
| 369 | |
| 370 | static long udmabuf_ioctl(struct file *filp, unsigned int ioctl, |
| 371 | unsigned long arg) |
| 372 | { |
| 373 | long ret; |
| 374 | |
| 375 | switch (ioctl) { |
| 376 | case UDMABUF_CREATE: |
| 377 | ret = udmabuf_ioctl_create(filp, arg); |
| 378 | break; |
| 379 | case UDMABUF_CREATE_LIST: |
| 380 | ret = udmabuf_ioctl_create_list(filp, arg); |
| 381 | break; |
| 382 | default: |
Gerd Hoffmann | 52499d9c | 2018-09-11 15:42:13 +0200 | [diff] [blame] | 383 | ret = -ENOTTY; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 384 | break; |
| 385 | } |
| 386 | return ret; |
| 387 | } |
| 388 | |
| 389 | static const struct file_operations udmabuf_fops = { |
| 390 | .owner = THIS_MODULE, |
| 391 | .unlocked_ioctl = udmabuf_ioctl, |
Kristian H. Kristensen | d4a197f | 2020-09-03 18:16:52 +0000 | [diff] [blame] | 392 | #ifdef CONFIG_COMPAT |
| 393 | .compat_ioctl = udmabuf_ioctl, |
| 394 | #endif |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 395 | }; |
| 396 | |
| 397 | static struct miscdevice udmabuf_misc = { |
| 398 | .minor = MISC_DYNAMIC_MINOR, |
| 399 | .name = "udmabuf", |
| 400 | .fops = &udmabuf_fops, |
| 401 | }; |
| 402 | |
| 403 | static int __init udmabuf_dev_init(void) |
| 404 | { |
Vivek Kasireddy | 9e9fa6a | 2022-05-20 13:52:35 -0700 | [diff] [blame] | 405 | int ret; |
| 406 | |
| 407 | ret = misc_register(&udmabuf_misc); |
| 408 | if (ret < 0) { |
| 409 | pr_err("Could not initialize udmabuf device\n"); |
| 410 | return ret; |
| 411 | } |
| 412 | |
| 413 | ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device, |
| 414 | DMA_BIT_MASK(64)); |
| 415 | if (ret < 0) { |
| 416 | pr_err("Could not setup DMA mask for udmabuf device\n"); |
| 417 | misc_deregister(&udmabuf_misc); |
| 418 | return ret; |
| 419 | } |
| 420 | |
| 421 | return 0; |
Gerd Hoffmann | fbb0de7 | 2018-08-27 11:34:44 +0200 | [diff] [blame] | 422 | } |
| 423 | |
| 424 | static void __exit udmabuf_dev_exit(void) |
| 425 | { |
| 426 | misc_deregister(&udmabuf_misc); |
| 427 | } |
| 428 | |
| 429 | module_init(udmabuf_dev_init) |
| 430 | module_exit(udmabuf_dev_exit) |
| 431 | |
| 432 | MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>"); |