Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * An async IO implementation for Linux |
| 3 | * Written by Benjamin LaHaise <bcrl@kvack.org> |
| 4 | * |
| 5 | * Implements an efficient asynchronous io interface. |
| 6 | * |
| 7 | * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved. |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 8 | * Copyright 2018 Christoph Hellwig. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | * |
| 10 | * See ../COPYING for licensing terms. |
| 11 | */ |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 12 | #define pr_fmt(fmt) "%s: " fmt, __func__ |
| 13 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <linux/kernel.h> |
| 15 | #include <linux/init.h> |
| 16 | #include <linux/errno.h> |
| 17 | #include <linux/time.h> |
| 18 | #include <linux/aio_abi.h> |
Paul Gortmaker | 630d9c4 | 2011-11-16 23:57:37 -0500 | [diff] [blame] | 19 | #include <linux/export.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <linux/syscalls.h> |
Jens Axboe | b9d128f | 2009-10-29 13:59:26 +0100 | [diff] [blame] | 21 | #include <linux/backing-dev.h> |
Christoph Hellwig | 9018ccc | 2018-07-24 11:36:37 +0200 | [diff] [blame] | 22 | #include <linux/refcount.h> |
Badari Pulavarty | 027445c | 2006-09-30 23:28:46 -0700 | [diff] [blame] | 23 | #include <linux/uio.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | |
Ingo Molnar | 174cd4b | 2017-02-02 19:15:33 +0100 | [diff] [blame] | 25 | #include <linux/sched/signal.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #include <linux/fs.h> |
| 27 | #include <linux/file.h> |
| 28 | #include <linux/mm.h> |
| 29 | #include <linux/mman.h> |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 30 | #include <linux/percpu.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | #include <linux/slab.h> |
| 32 | #include <linux/timer.h> |
| 33 | #include <linux/aio.h> |
| 34 | #include <linux/highmem.h> |
| 35 | #include <linux/workqueue.h> |
| 36 | #include <linux/security.h> |
Davide Libenzi | 9c3060b | 2007-05-10 22:23:21 -0700 | [diff] [blame] | 37 | #include <linux/eventfd.h> |
Jeff Moyer | cfb1e33 | 2009-10-02 18:57:36 -0400 | [diff] [blame] | 38 | #include <linux/blkdev.h> |
Jeff Moyer | 9d85cba7 | 2010-05-26 14:44:26 -0700 | [diff] [blame] | 39 | #include <linux/compat.h> |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 40 | #include <linux/migrate.h> |
| 41 | #include <linux/ramfs.h> |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 42 | #include <linux/percpu-refcount.h> |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 43 | #include <linux/mount.h> |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 44 | #include <linux/pseudo_fs.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | |
Linus Torvalds | 7c0f6ba | 2016-12-24 11:46:01 -0800 | [diff] [blame] | 46 | #include <linux/uaccess.h> |
Jeff Moyer | a538e3f | 2018-12-11 12:37:49 -0500 | [diff] [blame] | 47 | #include <linux/nospec.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | |
Al Viro | 68d70d0 | 2013-06-19 15:26:04 +0400 | [diff] [blame] | 49 | #include "internal.h" |
| 50 | |
Christoph Hellwig | f3a2752 | 2018-03-30 11:19:25 +0200 | [diff] [blame] | 51 | #define KIOCB_KEY 0 |
| 52 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 53 | #define AIO_RING_MAGIC 0xa10a10a1 |
| 54 | #define AIO_RING_COMPAT_FEATURES 1 |
| 55 | #define AIO_RING_INCOMPAT_FEATURES 0 |
| 56 | struct aio_ring { |
| 57 | unsigned id; /* kernel internal index number */ |
| 58 | unsigned nr; /* number of io_events */ |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 59 | unsigned head; /* Written to by userland or under ring_lock |
| 60 | * mutex by aio_read_events_ring(). */ |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 61 | unsigned tail; |
| 62 | |
| 63 | unsigned magic; |
| 64 | unsigned compat_features; |
| 65 | unsigned incompat_features; |
| 66 | unsigned header_length; /* size of aio_ring */ |
| 67 | |
| 68 | |
Gustavo A. R. Silva | 241cb28 | 2020-05-28 09:35:11 -0500 | [diff] [blame] | 69 | struct io_event io_events[]; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 70 | }; /* 128 bytes + ring size */ |
| 71 | |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 72 | /* |
| 73 | * Plugging is meant to work with larger batches of IOs. If we don't |
| 74 | * have more than the below, then don't bother setting up a plug. |
| 75 | */ |
| 76 | #define AIO_PLUG_THRESHOLD 2 |
| 77 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 78 | #define AIO_RING_PAGES 8 |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 79 | |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 80 | struct kioctx_table { |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 81 | struct rcu_head rcu; |
| 82 | unsigned nr; |
| 83 | struct kioctx __rcu *table[]; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 84 | }; |
| 85 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 86 | struct kioctx_cpu { |
| 87 | unsigned reqs_available; |
| 88 | }; |
| 89 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 90 | struct ctx_rq_wait { |
| 91 | struct completion comp; |
| 92 | atomic_t count; |
| 93 | }; |
| 94 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 95 | struct kioctx { |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 96 | struct percpu_ref users; |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 97 | atomic_t dead; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 98 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 99 | struct percpu_ref reqs; |
| 100 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 101 | unsigned long user_id; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 102 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 103 | struct __percpu kioctx_cpu *cpu; |
| 104 | |
| 105 | /* |
| 106 | * For percpu reqs_available, number of slots we move to/from global |
| 107 | * counter at a time: |
| 108 | */ |
| 109 | unsigned req_batch; |
Kent Overstreet | 3e845ce | 2013-05-07 16:18:51 -0700 | [diff] [blame] | 110 | /* |
| 111 | * This is what userspace passed to io_setup(), it's not used for |
| 112 | * anything but counting against the global max_reqs quota. |
| 113 | * |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 114 | * The real limit is nr_events - 1, which will be larger (see |
Kent Overstreet | 3e845ce | 2013-05-07 16:18:51 -0700 | [diff] [blame] | 115 | * aio_setup_ring()) |
| 116 | */ |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 117 | unsigned max_reqs; |
| 118 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 119 | /* Size of ringbuffer, in units of struct io_event */ |
| 120 | unsigned nr_events; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 121 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 122 | unsigned long mmap_base; |
| 123 | unsigned long mmap_size; |
| 124 | |
| 125 | struct page **ring_pages; |
| 126 | long nr_pages; |
| 127 | |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 128 | struct rcu_work free_rwork; /* see free_ioctx() */ |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 129 | |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 130 | /* |
| 131 | * signals when all in-flight requests are done |
| 132 | */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 133 | struct ctx_rq_wait *rq_wait; |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 134 | |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 135 | struct { |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 136 | /* |
| 137 | * This counts the number of available slots in the ringbuffer, |
| 138 | * so we avoid overflowing it: it's decremented (if positive) |
| 139 | * when allocating a kiocb and incremented when the resulting |
| 140 | * io_event is pulled off the ringbuffer. |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 141 | * |
| 142 | * We batch accesses to it with a percpu version. |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 143 | */ |
| 144 | atomic_t reqs_available; |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 145 | } ____cacheline_aligned_in_smp; |
| 146 | |
| 147 | struct { |
| 148 | spinlock_t ctx_lock; |
| 149 | struct list_head active_reqs; /* used for cancellation */ |
| 150 | } ____cacheline_aligned_in_smp; |
| 151 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 152 | struct { |
| 153 | struct mutex ring_lock; |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 154 | wait_queue_head_t wait; |
| 155 | } ____cacheline_aligned_in_smp; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 156 | |
| 157 | struct { |
| 158 | unsigned tail; |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 159 | unsigned completed_events; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 160 | spinlock_t completion_lock; |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 161 | } ____cacheline_aligned_in_smp; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 162 | |
| 163 | struct page *internal_pages[AIO_RING_PAGES]; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 164 | struct file *aio_ring_file; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 165 | |
| 166 | unsigned id; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 167 | }; |
| 168 | |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 169 | /* |
| 170 | * First field must be the file pointer in all the |
| 171 | * iocb unions! See also 'struct kiocb' in <linux/fs.h> |
| 172 | */ |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 173 | struct fsync_iocb { |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 174 | struct file *file; |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 175 | struct work_struct work; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 176 | bool datasync; |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 177 | struct cred *creds; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 178 | }; |
| 179 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 180 | struct poll_iocb { |
| 181 | struct file *file; |
| 182 | struct wait_queue_head *head; |
| 183 | __poll_t events; |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 184 | bool done; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 185 | bool cancelled; |
| 186 | struct wait_queue_entry wait; |
| 187 | struct work_struct work; |
| 188 | }; |
| 189 | |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 190 | /* |
| 191 | * NOTE! Each of the iocb union members has the file pointer |
| 192 | * as the first entry in their struct definition. So you can |
| 193 | * access the file pointer through any of the sub-structs, |
| 194 | * or directly as just 'ki_filp' in this struct. |
| 195 | */ |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 196 | struct aio_kiocb { |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 197 | union { |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 198 | struct file *ki_filp; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 199 | struct kiocb rw; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 200 | struct fsync_iocb fsync; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 201 | struct poll_iocb poll; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 202 | }; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 203 | |
| 204 | struct kioctx *ki_ctx; |
| 205 | kiocb_cancel_fn *ki_cancel; |
| 206 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 207 | struct io_event ki_res; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 208 | |
| 209 | struct list_head ki_list; /* the aio core uses this |
| 210 | * for cancellation */ |
Christoph Hellwig | 9018ccc | 2018-07-24 11:36:37 +0200 | [diff] [blame] | 211 | refcount_t ki_refcnt; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 212 | |
| 213 | /* |
| 214 | * If the aio_resfd field of the userspace iocb is not zero, |
| 215 | * this is the underlying eventfd context to deliver events to. |
| 216 | */ |
| 217 | struct eventfd_ctx *ki_eventfd; |
| 218 | }; |
| 219 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | /*------ sysctl variables----*/ |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 221 | static DEFINE_SPINLOCK(aio_nr_lock); |
| 222 | unsigned long aio_nr; /* current system wide number of aio requests */ |
| 223 | unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | /*----end sysctl variables---*/ |
| 225 | |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 226 | static struct kmem_cache *kiocb_cachep; |
| 227 | static struct kmem_cache *kioctx_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 228 | |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 229 | static struct vfsmount *aio_mnt; |
| 230 | |
| 231 | static const struct file_operations aio_ring_fops; |
| 232 | static const struct address_space_operations aio_ctx_aops; |
| 233 | |
| 234 | static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages) |
| 235 | { |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 236 | struct file *file; |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 237 | struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb); |
Dan Carpenter | 7f62656 | 2013-11-13 10:49:40 +0300 | [diff] [blame] | 238 | if (IS_ERR(inode)) |
| 239 | return ERR_CAST(inode); |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 240 | |
| 241 | inode->i_mapping->a_ops = &aio_ctx_aops; |
| 242 | inode->i_mapping->private_data = ctx; |
| 243 | inode->i_size = PAGE_SIZE * nr_pages; |
| 244 | |
Al Viro | d93aa9d | 2018-06-09 09:40:05 -0400 | [diff] [blame] | 245 | file = alloc_file_pseudo(inode, aio_mnt, "[aio]", |
| 246 | O_RDWR, &aio_ring_fops); |
Al Viro | c9c554f | 2018-07-11 14:19:04 -0400 | [diff] [blame] | 247 | if (IS_ERR(file)) |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 248 | iput(inode); |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 249 | return file; |
| 250 | } |
| 251 | |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 252 | static int aio_init_fs_context(struct fs_context *fc) |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 253 | { |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 254 | if (!init_pseudo(fc, AIO_RING_MAGIC)) |
| 255 | return -ENOMEM; |
| 256 | fc->s_iflags |= SB_I_NOEXEC; |
| 257 | return 0; |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 258 | } |
| 259 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 260 | /* aio_setup |
| 261 | * Creates the slab caches used by the aio routines, panic on |
| 262 | * failure as this is done early during the boot sequence. |
| 263 | */ |
| 264 | static int __init aio_setup(void) |
| 265 | { |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 266 | static struct file_system_type aio_fs = { |
| 267 | .name = "aio", |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 268 | .init_fs_context = aio_init_fs_context, |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 269 | .kill_sb = kill_anon_super, |
| 270 | }; |
| 271 | aio_mnt = kern_mount(&aio_fs); |
| 272 | if (IS_ERR(aio_mnt)) |
| 273 | panic("Failed to create aio fs mount."); |
| 274 | |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 275 | kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); |
Christoph Lameter | 0a31bd5 | 2007-05-06 14:49:57 -0700 | [diff] [blame] | 276 | kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 277 | return 0; |
| 278 | } |
H Hartley Sweeten | 385773e | 2009-09-22 16:43:53 -0700 | [diff] [blame] | 279 | __initcall(aio_setup); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 281 | static void put_aio_ring_file(struct kioctx *ctx) |
| 282 | { |
| 283 | struct file *aio_ring_file = ctx->aio_ring_file; |
Rasmus Villemoes | de04e76 | 2016-09-15 00:25:03 +0200 | [diff] [blame] | 284 | struct address_space *i_mapping; |
| 285 | |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 286 | if (aio_ring_file) { |
Al Viro | 4506309 | 2016-12-04 18:24:56 -0500 | [diff] [blame] | 287 | truncate_setsize(file_inode(aio_ring_file), 0); |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 288 | |
| 289 | /* Prevent further access to the kioctx from migratepages */ |
Al Viro | 4506309 | 2016-12-04 18:24:56 -0500 | [diff] [blame] | 290 | i_mapping = aio_ring_file->f_mapping; |
Rasmus Villemoes | de04e76 | 2016-09-15 00:25:03 +0200 | [diff] [blame] | 291 | spin_lock(&i_mapping->private_lock); |
| 292 | i_mapping->private_data = NULL; |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 293 | ctx->aio_ring_file = NULL; |
Rasmus Villemoes | de04e76 | 2016-09-15 00:25:03 +0200 | [diff] [blame] | 294 | spin_unlock(&i_mapping->private_lock); |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 295 | |
| 296 | fput(aio_ring_file); |
| 297 | } |
| 298 | } |
| 299 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 300 | static void aio_free_ring(struct kioctx *ctx) |
| 301 | { |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 302 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 303 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 304 | /* Disconnect the kiotx from the ring file. This prevents future |
| 305 | * accesses to the kioctx from page migration. |
| 306 | */ |
| 307 | put_aio_ring_file(ctx); |
| 308 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 309 | for (i = 0; i < ctx->nr_pages; i++) { |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 310 | struct page *page; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 311 | pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i, |
| 312 | page_count(ctx->ring_pages[i])); |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 313 | page = ctx->ring_pages[i]; |
| 314 | if (!page) |
| 315 | continue; |
| 316 | ctx->ring_pages[i] = NULL; |
| 317 | put_page(page); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 318 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 319 | |
Sasha Levin | ddb8c45 | 2013-11-19 17:33:03 -0500 | [diff] [blame] | 320 | if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) { |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 321 | kfree(ctx->ring_pages); |
Sasha Levin | ddb8c45 | 2013-11-19 17:33:03 -0500 | [diff] [blame] | 322 | ctx->ring_pages = NULL; |
| 323 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 324 | } |
| 325 | |
Dmitry Safonov | cd544fd | 2020-12-14 19:08:13 -0800 | [diff] [blame] | 326 | static int aio_ring_mremap(struct vm_area_struct *vma, unsigned long flags) |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 327 | { |
Oleg Nesterov | 5477e70 | 2015-09-04 15:48:04 -0700 | [diff] [blame] | 328 | struct file *file = vma->vm_file; |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 329 | struct mm_struct *mm = vma->vm_mm; |
| 330 | struct kioctx_table *table; |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 331 | int i, res = -EINVAL; |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 332 | |
Dmitry Safonov | cd544fd | 2020-12-14 19:08:13 -0800 | [diff] [blame] | 333 | if (flags & MREMAP_DONTUNMAP) |
| 334 | return -EINVAL; |
| 335 | |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 336 | spin_lock(&mm->ioctx_lock); |
| 337 | rcu_read_lock(); |
| 338 | table = rcu_dereference(mm->ioctx_table); |
| 339 | for (i = 0; i < table->nr; i++) { |
| 340 | struct kioctx *ctx; |
| 341 | |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 342 | ctx = rcu_dereference(table->table[i]); |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 343 | if (ctx && ctx->aio_ring_file == file) { |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 344 | if (!atomic_read(&ctx->dead)) { |
| 345 | ctx->user_id = ctx->mmap_base = vma->vm_start; |
| 346 | res = 0; |
| 347 | } |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 348 | break; |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | rcu_read_unlock(); |
| 353 | spin_unlock(&mm->ioctx_lock); |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 354 | return res; |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 355 | } |
| 356 | |
Oleg Nesterov | 5477e70 | 2015-09-04 15:48:04 -0700 | [diff] [blame] | 357 | static const struct vm_operations_struct aio_ring_vm_ops = { |
| 358 | .mremap = aio_ring_mremap, |
| 359 | #if IS_ENABLED(CONFIG_MMU) |
| 360 | .fault = filemap_fault, |
| 361 | .map_pages = filemap_map_pages, |
| 362 | .page_mkwrite = filemap_page_mkwrite, |
| 363 | #endif |
| 364 | }; |
| 365 | |
| 366 | static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma) |
| 367 | { |
| 368 | vma->vm_flags |= VM_DONTEXPAND; |
| 369 | vma->vm_ops = &aio_ring_vm_ops; |
| 370 | return 0; |
| 371 | } |
| 372 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 373 | static const struct file_operations aio_ring_fops = { |
| 374 | .mmap = aio_ring_mmap, |
| 375 | }; |
| 376 | |
Benjamin LaHaise | 0c45355 | 2013-07-17 09:34:24 -0400 | [diff] [blame] | 377 | #if IS_ENABLED(CONFIG_MIGRATION) |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 378 | static int aio_migratepage(struct address_space *mapping, struct page *new, |
| 379 | struct page *old, enum migrate_mode mode) |
| 380 | { |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 381 | struct kioctx *ctx; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 382 | unsigned long flags; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 383 | pgoff_t idx; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 384 | int rc; |
| 385 | |
Jérôme Glisse | 2916ecc | 2017-09-08 16:12:06 -0700 | [diff] [blame] | 386 | /* |
| 387 | * We cannot support the _NO_COPY case here, because copy needs to |
| 388 | * happen under the ctx->completion_lock. That does not work with the |
| 389 | * migration workflow of MIGRATE_SYNC_NO_COPY. |
| 390 | */ |
| 391 | if (mode == MIGRATE_SYNC_NO_COPY) |
| 392 | return -EINVAL; |
| 393 | |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 394 | rc = 0; |
| 395 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 396 | /* mapping->private_lock here protects against the kioctx teardown. */ |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 397 | spin_lock(&mapping->private_lock); |
| 398 | ctx = mapping->private_data; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 399 | if (!ctx) { |
| 400 | rc = -EINVAL; |
| 401 | goto out; |
| 402 | } |
| 403 | |
| 404 | /* The ring_lock mutex. The prevents aio_read_events() from writing |
| 405 | * to the ring's head, and prevents page migration from mucking in |
| 406 | * a partially initialized kiotx. |
| 407 | */ |
| 408 | if (!mutex_trylock(&ctx->ring_lock)) { |
| 409 | rc = -EAGAIN; |
| 410 | goto out; |
| 411 | } |
| 412 | |
| 413 | idx = old->index; |
| 414 | if (idx < (pgoff_t)ctx->nr_pages) { |
| 415 | /* Make sure the old page hasn't already been changed */ |
| 416 | if (ctx->ring_pages[idx] != old) |
| 417 | rc = -EAGAIN; |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 418 | } else |
| 419 | rc = -EINVAL; |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 420 | |
| 421 | if (rc != 0) |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 422 | goto out_unlock; |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 423 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 424 | /* Writeback must be complete */ |
| 425 | BUG_ON(PageWriteback(old)); |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 426 | get_page(new); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 427 | |
Keith Busch | 3710969 | 2019-07-18 15:58:46 -0700 | [diff] [blame] | 428 | rc = migrate_page_move_mapping(mapping, new, old, 1); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 429 | if (rc != MIGRATEPAGE_SUCCESS) { |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 430 | put_page(new); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 431 | goto out_unlock; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 432 | } |
| 433 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 434 | /* Take completion_lock to prevent other writes to the ring buffer |
| 435 | * while the old page is copied to the new. This prevents new |
| 436 | * events from being lost. |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 437 | */ |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 438 | spin_lock_irqsave(&ctx->completion_lock, flags); |
| 439 | migrate_page_copy(new, old); |
| 440 | BUG_ON(ctx->ring_pages[idx] != old); |
| 441 | ctx->ring_pages[idx] = new; |
| 442 | spin_unlock_irqrestore(&ctx->completion_lock, flags); |
| 443 | |
| 444 | /* The old page is no longer accessible. */ |
| 445 | put_page(old); |
| 446 | |
| 447 | out_unlock: |
| 448 | mutex_unlock(&ctx->ring_lock); |
| 449 | out: |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 450 | spin_unlock(&mapping->private_lock); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 451 | return rc; |
| 452 | } |
Benjamin LaHaise | 0c45355 | 2013-07-17 09:34:24 -0400 | [diff] [blame] | 453 | #endif |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 454 | |
| 455 | static const struct address_space_operations aio_ctx_aops = { |
Gu Zheng | 835f252 | 2014-11-06 17:46:21 +0800 | [diff] [blame] | 456 | .set_page_dirty = __set_page_dirty_no_writeback, |
Benjamin LaHaise | 0c45355 | 2013-07-17 09:34:24 -0400 | [diff] [blame] | 457 | #if IS_ENABLED(CONFIG_MIGRATION) |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 458 | .migratepage = aio_migratepage, |
Benjamin LaHaise | 0c45355 | 2013-07-17 09:34:24 -0400 | [diff] [blame] | 459 | #endif |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 460 | }; |
| 461 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 462 | static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 463 | { |
| 464 | struct aio_ring *ring; |
Zach Brown | 41003a7 | 2013-05-07 16:18:25 -0700 | [diff] [blame] | 465 | struct mm_struct *mm = current->mm; |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 466 | unsigned long size, unused; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 467 | int nr_pages; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 468 | int i; |
| 469 | struct file *file; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 470 | |
| 471 | /* Compensate for the ring buffer's head/tail overlap entry */ |
| 472 | nr_events += 2; /* 1 is required, 2 for good luck */ |
| 473 | |
| 474 | size = sizeof(struct aio_ring); |
| 475 | size += sizeof(struct io_event) * nr_events; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 476 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 477 | nr_pages = PFN_UP(size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 478 | if (nr_pages < 0) |
| 479 | return -EINVAL; |
| 480 | |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 481 | file = aio_private_file(ctx, nr_pages); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 482 | if (IS_ERR(file)) { |
| 483 | ctx->aio_ring_file = NULL; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 484 | return -ENOMEM; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 485 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 486 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 487 | ctx->aio_ring_file = file; |
| 488 | nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) |
| 489 | / sizeof(struct io_event); |
| 490 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 491 | ctx->ring_pages = ctx->internal_pages; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 492 | if (nr_pages > AIO_RING_PAGES) { |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 493 | ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *), |
| 494 | GFP_KERNEL); |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 495 | if (!ctx->ring_pages) { |
| 496 | put_aio_ring_file(ctx); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 497 | return -ENOMEM; |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 498 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 499 | } |
| 500 | |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 501 | for (i = 0; i < nr_pages; i++) { |
| 502 | struct page *page; |
Al Viro | 4506309 | 2016-12-04 18:24:56 -0500 | [diff] [blame] | 503 | page = find_or_create_page(file->f_mapping, |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 504 | i, GFP_HIGHUSER | __GFP_ZERO); |
| 505 | if (!page) |
| 506 | break; |
| 507 | pr_debug("pid(%d) page[%d]->count=%d\n", |
| 508 | current->pid, i, page_count(page)); |
| 509 | SetPageUptodate(page); |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 510 | unlock_page(page); |
| 511 | |
| 512 | ctx->ring_pages[i] = page; |
| 513 | } |
| 514 | ctx->nr_pages = i; |
| 515 | |
| 516 | if (unlikely(i != nr_pages)) { |
| 517 | aio_free_ring(ctx); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 518 | return -ENOMEM; |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 519 | } |
| 520 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 521 | ctx->mmap_size = nr_pages * PAGE_SIZE; |
| 522 | pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 523 | |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 524 | if (mmap_write_lock_killable(mm)) { |
Michal Hocko | 013373e | 2016-05-23 16:25:59 -0700 | [diff] [blame] | 525 | ctx->mmap_size = 0; |
| 526 | aio_free_ring(ctx); |
| 527 | return -EINTR; |
| 528 | } |
| 529 | |
Peter Collingbourne | 45e5530 | 2020-08-06 23:23:37 -0700 | [diff] [blame] | 530 | ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size, |
| 531 | PROT_READ | PROT_WRITE, |
| 532 | MAP_SHARED, 0, &unused, NULL); |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 533 | mmap_write_unlock(mm); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 534 | if (IS_ERR((void *)ctx->mmap_base)) { |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 535 | ctx->mmap_size = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 536 | aio_free_ring(ctx); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 537 | return -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 538 | } |
| 539 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 540 | pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base); |
Benjamin LaHaise | d6c355c | 2013-09-09 11:57:59 -0400 | [diff] [blame] | 541 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 542 | ctx->user_id = ctx->mmap_base; |
| 543 | ctx->nr_events = nr_events; /* trusted copy */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 544 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 545 | ring = kmap_atomic(ctx->ring_pages[0]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 546 | ring->nr = nr_events; /* user copy */ |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 547 | ring->id = ~0U; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 548 | ring->head = ring->tail = 0; |
| 549 | ring->magic = AIO_RING_MAGIC; |
| 550 | ring->compat_features = AIO_RING_COMPAT_FEATURES; |
| 551 | ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; |
| 552 | ring->header_length = sizeof(struct aio_ring); |
Cong Wang | e8e3c3d | 2011-11-25 23:14:27 +0800 | [diff] [blame] | 553 | kunmap_atomic(ring); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 554 | flush_dcache_page(ctx->ring_pages[0]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 555 | |
| 556 | return 0; |
| 557 | } |
| 558 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 559 | #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event)) |
| 560 | #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event)) |
| 561 | #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE) |
| 562 | |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 563 | void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel) |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 564 | { |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 565 | struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw); |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 566 | struct kioctx *ctx = req->ki_ctx; |
| 567 | unsigned long flags; |
| 568 | |
Christoph Hellwig | 75321b5 | 2018-04-09 14:57:56 +0200 | [diff] [blame] | 569 | if (WARN_ON_ONCE(!list_empty(&req->ki_list))) |
| 570 | return; |
| 571 | |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 572 | spin_lock_irqsave(&ctx->ctx_lock, flags); |
Christoph Hellwig | 75321b5 | 2018-04-09 14:57:56 +0200 | [diff] [blame] | 573 | list_add_tail(&req->ki_list, &ctx->active_reqs); |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 574 | req->ki_cancel = cancel; |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 575 | spin_unlock_irqrestore(&ctx->ctx_lock, flags); |
| 576 | } |
| 577 | EXPORT_SYMBOL(kiocb_set_cancel_fn); |
| 578 | |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 579 | /* |
| 580 | * free_ioctx() should be RCU delayed to synchronize against the RCU |
| 581 | * protected lookup_ioctx() and also needs process context to call |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 582 | * aio_free_ring(). Use rcu_work. |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 583 | */ |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 584 | static void free_ioctx(struct work_struct *work) |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 585 | { |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 586 | struct kioctx *ctx = container_of(to_rcu_work(work), struct kioctx, |
| 587 | free_rwork); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 588 | pr_debug("freeing %p\n", ctx); |
| 589 | |
| 590 | aio_free_ring(ctx); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 591 | free_percpu(ctx->cpu); |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 592 | percpu_ref_exit(&ctx->reqs); |
| 593 | percpu_ref_exit(&ctx->users); |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 594 | kmem_cache_free(kioctx_cachep, ctx); |
| 595 | } |
| 596 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 597 | static void free_ioctx_reqs(struct percpu_ref *ref) |
| 598 | { |
| 599 | struct kioctx *ctx = container_of(ref, struct kioctx, reqs); |
| 600 | |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 601 | /* At this point we know that there are no any in-flight requests */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 602 | if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) |
| 603 | complete(&ctx->rq_wait->comp); |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 604 | |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 605 | /* Synchronize against RCU protected table->table[] dereferences */ |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 606 | INIT_RCU_WORK(&ctx->free_rwork, free_ioctx); |
| 607 | queue_rcu_work(system_wq, &ctx->free_rwork); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 608 | } |
| 609 | |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 610 | /* |
| 611 | * When this function runs, the kioctx has been removed from the "hash table" |
| 612 | * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - |
| 613 | * now it's safe to cancel any that need to be. |
| 614 | */ |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 615 | static void free_ioctx_users(struct percpu_ref *ref) |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 616 | { |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 617 | struct kioctx *ctx = container_of(ref, struct kioctx, users); |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 618 | struct aio_kiocb *req; |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 619 | |
| 620 | spin_lock_irq(&ctx->ctx_lock); |
| 621 | |
| 622 | while (!list_empty(&ctx->active_reqs)) { |
| 623 | req = list_first_entry(&ctx->active_reqs, |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 624 | struct aio_kiocb, ki_list); |
Christoph Hellwig | 888933f | 2018-05-23 14:11:02 +0200 | [diff] [blame] | 625 | req->ki_cancel(&req->rw); |
Al Viro | 4faa9996 | 2018-05-23 22:53:22 -0400 | [diff] [blame] | 626 | list_del_init(&req->ki_list); |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 627 | } |
| 628 | |
| 629 | spin_unlock_irq(&ctx->ctx_lock); |
| 630 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 631 | percpu_ref_kill(&ctx->reqs); |
| 632 | percpu_ref_put(&ctx->reqs); |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 633 | } |
| 634 | |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 635 | static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) |
| 636 | { |
| 637 | unsigned i, new_nr; |
| 638 | struct kioctx_table *table, *old; |
| 639 | struct aio_ring *ring; |
| 640 | |
| 641 | spin_lock(&mm->ioctx_lock); |
Oleg Nesterov | 855ef0d | 2014-04-30 16:16:36 +0200 | [diff] [blame] | 642 | table = rcu_dereference_raw(mm->ioctx_table); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 643 | |
| 644 | while (1) { |
| 645 | if (table) |
| 646 | for (i = 0; i < table->nr; i++) |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 647 | if (!rcu_access_pointer(table->table[i])) { |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 648 | ctx->id = i; |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 649 | rcu_assign_pointer(table->table[i], ctx); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 650 | spin_unlock(&mm->ioctx_lock); |
| 651 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 652 | /* While kioctx setup is in progress, |
| 653 | * we are protected from page migration |
| 654 | * changes ring_pages by ->ring_lock. |
| 655 | */ |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 656 | ring = kmap_atomic(ctx->ring_pages[0]); |
| 657 | ring->id = ctx->id; |
| 658 | kunmap_atomic(ring); |
| 659 | return 0; |
| 660 | } |
| 661 | |
| 662 | new_nr = (table ? table->nr : 1) * 4; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 663 | spin_unlock(&mm->ioctx_lock); |
| 664 | |
| 665 | table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) * |
| 666 | new_nr, GFP_KERNEL); |
| 667 | if (!table) |
| 668 | return -ENOMEM; |
| 669 | |
| 670 | table->nr = new_nr; |
| 671 | |
| 672 | spin_lock(&mm->ioctx_lock); |
Oleg Nesterov | 855ef0d | 2014-04-30 16:16:36 +0200 | [diff] [blame] | 673 | old = rcu_dereference_raw(mm->ioctx_table); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 674 | |
| 675 | if (!old) { |
| 676 | rcu_assign_pointer(mm->ioctx_table, table); |
| 677 | } else if (table->nr > old->nr) { |
| 678 | memcpy(table->table, old->table, |
| 679 | old->nr * sizeof(struct kioctx *)); |
| 680 | |
| 681 | rcu_assign_pointer(mm->ioctx_table, table); |
| 682 | kfree_rcu(old, rcu); |
| 683 | } else { |
| 684 | kfree(table); |
| 685 | table = old; |
| 686 | } |
| 687 | } |
| 688 | } |
| 689 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 690 | static void aio_nr_sub(unsigned nr) |
| 691 | { |
| 692 | spin_lock(&aio_nr_lock); |
| 693 | if (WARN_ON(aio_nr - nr > aio_nr)) |
| 694 | aio_nr = 0; |
| 695 | else |
| 696 | aio_nr -= nr; |
| 697 | spin_unlock(&aio_nr_lock); |
| 698 | } |
| 699 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 700 | /* ioctx_alloc |
| 701 | * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed. |
| 702 | */ |
| 703 | static struct kioctx *ioctx_alloc(unsigned nr_events) |
| 704 | { |
Zach Brown | 41003a7 | 2013-05-07 16:18:25 -0700 | [diff] [blame] | 705 | struct mm_struct *mm = current->mm; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 706 | struct kioctx *ctx; |
Al Viro | e23754f | 2012-03-06 14:33:22 -0500 | [diff] [blame] | 707 | int err = -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 708 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 709 | /* |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 710 | * Store the original nr_events -- what userspace passed to io_setup(), |
| 711 | * for counting against the global limit -- before it changes. |
| 712 | */ |
| 713 | unsigned int max_reqs = nr_events; |
| 714 | |
| 715 | /* |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 716 | * We keep track of the number of available ringbuffer slots, to prevent |
| 717 | * overflow (reqs_available), and we also use percpu counters for this. |
| 718 | * |
| 719 | * So since up to half the slots might be on other cpu's percpu counters |
| 720 | * and unavailable, double nr_events so userspace sees what they |
| 721 | * expected: additionally, we move req_batch slots to/from percpu |
| 722 | * counters at a time, so make sure that isn't 0: |
| 723 | */ |
| 724 | nr_events = max(nr_events, num_possible_cpus() * 4); |
| 725 | nr_events *= 2; |
| 726 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 727 | /* Prevent overflows */ |
Al Viro | 08397ac | 2015-03-31 11:43:52 -0400 | [diff] [blame] | 728 | if (nr_events > (0x10000000U / sizeof(struct io_event))) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 729 | pr_debug("ENOMEM: nr_events too high\n"); |
| 730 | return ERR_PTR(-EINVAL); |
| 731 | } |
| 732 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 733 | if (!nr_events || (unsigned long)max_reqs > aio_max_nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 734 | return ERR_PTR(-EAGAIN); |
| 735 | |
Robert P. J. Day | c376222 | 2007-02-10 01:45:03 -0800 | [diff] [blame] | 736 | ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 737 | if (!ctx) |
| 738 | return ERR_PTR(-ENOMEM); |
| 739 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 740 | ctx->max_reqs = max_reqs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 741 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 742 | spin_lock_init(&ctx->ctx_lock); |
| 743 | spin_lock_init(&ctx->completion_lock); |
| 744 | mutex_init(&ctx->ring_lock); |
| 745 | /* Protect against page migration throughout kiotx setup by keeping |
| 746 | * the ring_lock mutex held until setup is complete. */ |
| 747 | mutex_lock(&ctx->ring_lock); |
| 748 | init_waitqueue_head(&ctx->wait); |
| 749 | |
| 750 | INIT_LIST_HEAD(&ctx->active_reqs); |
| 751 | |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 752 | if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL)) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 753 | goto err; |
| 754 | |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 755 | if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL)) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 756 | goto err; |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 757 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 758 | ctx->cpu = alloc_percpu(struct kioctx_cpu); |
| 759 | if (!ctx->cpu) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 760 | goto err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 761 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 762 | err = aio_setup_ring(ctx, nr_events); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 763 | if (err < 0) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 764 | goto err; |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 765 | |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 766 | atomic_set(&ctx->reqs_available, ctx->nr_events - 1); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 767 | ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4); |
Benjamin LaHaise | 6878ea7 | 2013-07-31 10:34:18 -0400 | [diff] [blame] | 768 | if (ctx->req_batch < 1) |
| 769 | ctx->req_batch = 1; |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 770 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 771 | /* limit the number of system wide aios */ |
Al Viro | 9fa1cb3 | 2012-03-10 23:14:05 -0500 | [diff] [blame] | 772 | spin_lock(&aio_nr_lock); |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 773 | if (aio_nr + ctx->max_reqs > aio_max_nr || |
| 774 | aio_nr + ctx->max_reqs < aio_nr) { |
Al Viro | 9fa1cb3 | 2012-03-10 23:14:05 -0500 | [diff] [blame] | 775 | spin_unlock(&aio_nr_lock); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 776 | err = -EAGAIN; |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 777 | goto err_ctx; |
Al Viro | 2dd542b7 | 2012-03-10 23:10:35 -0500 | [diff] [blame] | 778 | } |
| 779 | aio_nr += ctx->max_reqs; |
Al Viro | 9fa1cb3 | 2012-03-10 23:14:05 -0500 | [diff] [blame] | 780 | spin_unlock(&aio_nr_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 781 | |
Benjamin LaHaise | 1881686 | 2013-12-21 15:49:28 -0500 | [diff] [blame] | 782 | percpu_ref_get(&ctx->users); /* io_setup() will drop this ref */ |
| 783 | percpu_ref_get(&ctx->reqs); /* free_ioctx_users() will drop this */ |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 784 | |
Benjamin LaHaise | da90382 | 2013-08-05 13:21:43 -0400 | [diff] [blame] | 785 | err = ioctx_add_table(ctx, mm); |
| 786 | if (err) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 787 | goto err_cleanup; |
Benjamin LaHaise | da90382 | 2013-08-05 13:21:43 -0400 | [diff] [blame] | 788 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 789 | /* Release the ring_lock mutex now that all setup is complete. */ |
| 790 | mutex_unlock(&ctx->ring_lock); |
| 791 | |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 792 | pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n", |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 793 | ctx, ctx->user_id, mm, ctx->nr_events); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 794 | return ctx; |
| 795 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 796 | err_cleanup: |
| 797 | aio_nr_sub(ctx->max_reqs); |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 798 | err_ctx: |
Al Viro | deeb852 | 2015-04-06 17:57:44 -0400 | [diff] [blame] | 799 | atomic_set(&ctx->dead, 1); |
| 800 | if (ctx->mmap_size) |
| 801 | vm_munmap(ctx->mmap_base, ctx->mmap_size); |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 802 | aio_free_ring(ctx); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 803 | err: |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 804 | mutex_unlock(&ctx->ring_lock); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 805 | free_percpu(ctx->cpu); |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 806 | percpu_ref_exit(&ctx->reqs); |
| 807 | percpu_ref_exit(&ctx->users); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 808 | kmem_cache_free(kioctx_cachep, ctx); |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 809 | pr_debug("error allocating ioctx %d\n", err); |
Al Viro | e23754f | 2012-03-06 14:33:22 -0500 | [diff] [blame] | 810 | return ERR_PTR(err); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 811 | } |
| 812 | |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 813 | /* kill_ioctx |
| 814 | * Cancels all outstanding aio requests on an aio context. Used |
| 815 | * when the processes owning a context have all exited to encourage |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 816 | * the rapid destruction of the kioctx. |
| 817 | */ |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 818 | static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx, |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 819 | struct ctx_rq_wait *wait) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 820 | { |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 821 | struct kioctx_table *table; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 822 | |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 823 | spin_lock(&mm->ioctx_lock); |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 824 | if (atomic_xchg(&ctx->dead, 1)) { |
| 825 | spin_unlock(&mm->ioctx_lock); |
| 826 | return -EINVAL; |
| 827 | } |
| 828 | |
Oleg Nesterov | 855ef0d | 2014-04-30 16:16:36 +0200 | [diff] [blame] | 829 | table = rcu_dereference_raw(mm->ioctx_table); |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 830 | WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id])); |
| 831 | RCU_INIT_POINTER(table->table[ctx->id], NULL); |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 832 | spin_unlock(&mm->ioctx_lock); |
Kent Overstreet | 4fcc712 | 2013-06-12 14:04:59 -0700 | [diff] [blame] | 833 | |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 834 | /* free_ioctx_reqs() will do the necessary RCU synchronization */ |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 835 | wake_up_all(&ctx->wait); |
Kent Overstreet | 4fcc712 | 2013-06-12 14:04:59 -0700 | [diff] [blame] | 836 | |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 837 | /* |
| 838 | * It'd be more correct to do this in free_ioctx(), after all |
| 839 | * the outstanding kiocbs have finished - but by then io_destroy |
| 840 | * has already returned, so io_setup() could potentially return |
| 841 | * -EAGAIN with no ioctxs actually in use (as far as userspace |
| 842 | * could tell). |
| 843 | */ |
| 844 | aio_nr_sub(ctx->max_reqs); |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 845 | |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 846 | if (ctx->mmap_size) |
| 847 | vm_munmap(ctx->mmap_base, ctx->mmap_size); |
| 848 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 849 | ctx->rq_wait = wait; |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 850 | percpu_ref_kill(&ctx->users); |
| 851 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 852 | } |
| 853 | |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 854 | /* |
| 855 | * exit_aio: called when the last user of mm goes away. At this point, there is |
| 856 | * no way for any new requests to be submited or any of the io_* syscalls to be |
| 857 | * called on the context. |
| 858 | * |
| 859 | * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on |
| 860 | * them. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 861 | */ |
Harvey Harrison | fc9b52c | 2008-02-08 04:19:52 -0800 | [diff] [blame] | 862 | void exit_aio(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 863 | { |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 864 | struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table); |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 865 | struct ctx_rq_wait wait; |
| 866 | int i, skipped; |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 867 | |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 868 | if (!table) |
| 869 | return; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 870 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 871 | atomic_set(&wait.count, table->nr); |
| 872 | init_completion(&wait.comp); |
| 873 | |
| 874 | skipped = 0; |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 875 | for (i = 0; i < table->nr; ++i) { |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 876 | struct kioctx *ctx = |
| 877 | rcu_dereference_protected(table->table[i], true); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 878 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 879 | if (!ctx) { |
| 880 | skipped++; |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 881 | continue; |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 882 | } |
| 883 | |
Al Viro | 936af15 | 2012-04-20 21:49:41 -0400 | [diff] [blame] | 884 | /* |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 885 | * We don't need to bother with munmap() here - exit_mmap(mm) |
| 886 | * is coming and it'll unmap everything. And we simply can't, |
| 887 | * this is not necessarily our ->mm. |
| 888 | * Since kill_ioctx() uses non-zero ->mmap_size as indicator |
| 889 | * that it needs to unmap the area, just set it to 0. |
Al Viro | 936af15 | 2012-04-20 21:49:41 -0400 | [diff] [blame] | 890 | */ |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 891 | ctx->mmap_size = 0; |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 892 | kill_ioctx(mm, ctx, &wait); |
| 893 | } |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 894 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 895 | if (!atomic_sub_and_test(skipped, &wait.count)) { |
Gu Zheng | 6098b45 | 2014-09-03 17:45:44 +0800 | [diff] [blame] | 896 | /* Wait until all IO for the context are done. */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 897 | wait_for_completion(&wait.comp); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 898 | } |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 899 | |
| 900 | RCU_INIT_POINTER(mm->ioctx_table, NULL); |
| 901 | kfree(table); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 902 | } |
| 903 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 904 | static void put_reqs_available(struct kioctx *ctx, unsigned nr) |
| 905 | { |
| 906 | struct kioctx_cpu *kcpu; |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 907 | unsigned long flags; |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 908 | |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 909 | local_irq_save(flags); |
Benjamin LaHaise | be6fb45 | 2014-07-22 09:56:56 -0400 | [diff] [blame] | 910 | kcpu = this_cpu_ptr(ctx->cpu); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 911 | kcpu->reqs_available += nr; |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 912 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 913 | while (kcpu->reqs_available >= ctx->req_batch * 2) { |
| 914 | kcpu->reqs_available -= ctx->req_batch; |
| 915 | atomic_add(ctx->req_batch, &ctx->reqs_available); |
| 916 | } |
| 917 | |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 918 | local_irq_restore(flags); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 919 | } |
| 920 | |
Christoph Hellwig | 432c799 | 2018-11-19 15:57:42 -0700 | [diff] [blame] | 921 | static bool __get_reqs_available(struct kioctx *ctx) |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 922 | { |
| 923 | struct kioctx_cpu *kcpu; |
| 924 | bool ret = false; |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 925 | unsigned long flags; |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 926 | |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 927 | local_irq_save(flags); |
Benjamin LaHaise | be6fb45 | 2014-07-22 09:56:56 -0400 | [diff] [blame] | 928 | kcpu = this_cpu_ptr(ctx->cpu); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 929 | if (!kcpu->reqs_available) { |
| 930 | int old, avail = atomic_read(&ctx->reqs_available); |
| 931 | |
| 932 | do { |
| 933 | if (avail < ctx->req_batch) |
| 934 | goto out; |
| 935 | |
| 936 | old = avail; |
| 937 | avail = atomic_cmpxchg(&ctx->reqs_available, |
| 938 | avail, avail - ctx->req_batch); |
| 939 | } while (avail != old); |
| 940 | |
| 941 | kcpu->reqs_available += ctx->req_batch; |
| 942 | } |
| 943 | |
| 944 | ret = true; |
| 945 | kcpu->reqs_available--; |
| 946 | out: |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 947 | local_irq_restore(flags); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 948 | return ret; |
| 949 | } |
| 950 | |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 951 | /* refill_reqs_available |
| 952 | * Updates the reqs_available reference counts used for tracking the |
| 953 | * number of free slots in the completion ring. This can be called |
| 954 | * from aio_complete() (to optimistically update reqs_available) or |
| 955 | * from aio_get_req() (the we're out of events case). It must be |
| 956 | * called holding ctx->completion_lock. |
| 957 | */ |
| 958 | static void refill_reqs_available(struct kioctx *ctx, unsigned head, |
| 959 | unsigned tail) |
| 960 | { |
| 961 | unsigned events_in_ring, completed; |
| 962 | |
| 963 | /* Clamp head since userland can write to it. */ |
| 964 | head %= ctx->nr_events; |
| 965 | if (head <= tail) |
| 966 | events_in_ring = tail - head; |
| 967 | else |
| 968 | events_in_ring = ctx->nr_events - (head - tail); |
| 969 | |
| 970 | completed = ctx->completed_events; |
| 971 | if (events_in_ring < completed) |
| 972 | completed -= events_in_ring; |
| 973 | else |
| 974 | completed = 0; |
| 975 | |
| 976 | if (!completed) |
| 977 | return; |
| 978 | |
| 979 | ctx->completed_events -= completed; |
| 980 | put_reqs_available(ctx, completed); |
| 981 | } |
| 982 | |
| 983 | /* user_refill_reqs_available |
| 984 | * Called to refill reqs_available when aio_get_req() encounters an |
| 985 | * out of space in the completion ring. |
| 986 | */ |
| 987 | static void user_refill_reqs_available(struct kioctx *ctx) |
| 988 | { |
| 989 | spin_lock_irq(&ctx->completion_lock); |
| 990 | if (ctx->completed_events) { |
| 991 | struct aio_ring *ring; |
| 992 | unsigned head; |
| 993 | |
| 994 | /* Access of ring->head may race with aio_read_events_ring() |
| 995 | * here, but that's okay since whether we read the old version |
| 996 | * or the new version, and either will be valid. The important |
| 997 | * part is that head cannot pass tail since we prevent |
| 998 | * aio_complete() from updating tail by holding |
| 999 | * ctx->completion_lock. Even if head is invalid, the check |
| 1000 | * against ctx->completed_events below will make sure we do the |
| 1001 | * safe/right thing. |
| 1002 | */ |
| 1003 | ring = kmap_atomic(ctx->ring_pages[0]); |
| 1004 | head = ring->head; |
| 1005 | kunmap_atomic(ring); |
| 1006 | |
| 1007 | refill_reqs_available(ctx, head, ctx->tail); |
| 1008 | } |
| 1009 | |
| 1010 | spin_unlock_irq(&ctx->completion_lock); |
| 1011 | } |
| 1012 | |
Christoph Hellwig | 432c799 | 2018-11-19 15:57:42 -0700 | [diff] [blame] | 1013 | static bool get_reqs_available(struct kioctx *ctx) |
| 1014 | { |
| 1015 | if (__get_reqs_available(ctx)) |
| 1016 | return true; |
| 1017 | user_refill_reqs_available(ctx); |
| 1018 | return __get_reqs_available(ctx); |
| 1019 | } |
| 1020 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1021 | /* aio_get_req |
Kent Overstreet | 57282d8 | 2013-05-13 13:42:52 -0700 | [diff] [blame] | 1022 | * Allocate a slot for an aio request. |
| 1023 | * Returns NULL if no requests are free. |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1024 | * |
| 1025 | * The refcount is initialized to 2 - one for the async op completion, |
| 1026 | * one for the synchronous code that does this. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1027 | */ |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 1028 | static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1029 | { |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 1030 | struct aio_kiocb *req; |
Kent Overstreet | a1c8eae | 2013-05-07 16:18:53 -0700 | [diff] [blame] | 1031 | |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1032 | req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1033 | if (unlikely(!req)) |
Christoph Hellwig | 432c799 | 2018-11-19 15:57:42 -0700 | [diff] [blame] | 1034 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1035 | |
Al Viro | fa0ca2a | 2019-03-06 18:21:08 -0500 | [diff] [blame] | 1036 | if (unlikely(!get_reqs_available(ctx))) { |
Wei Yongjun | 6af1c84 | 2019-04-04 08:44:05 +0000 | [diff] [blame] | 1037 | kmem_cache_free(kiocb_cachep, req); |
Al Viro | fa0ca2a | 2019-03-06 18:21:08 -0500 | [diff] [blame] | 1038 | return NULL; |
| 1039 | } |
| 1040 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 1041 | percpu_ref_get(&ctx->reqs); |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1042 | req->ki_ctx = ctx; |
Christoph Hellwig | 75321b5 | 2018-04-09 14:57:56 +0200 | [diff] [blame] | 1043 | INIT_LIST_HEAD(&req->ki_list); |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1044 | refcount_set(&req->ki_refcnt, 2); |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1045 | req->ki_eventfd = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1046 | return req; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1047 | } |
| 1048 | |
Adrian Bunk | d5470b5 | 2008-04-29 00:58:57 -0700 | [diff] [blame] | 1049 | static struct kioctx *lookup_ioctx(unsigned long ctx_id) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1050 | { |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1051 | struct aio_ring __user *ring = (void __user *)ctx_id; |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1052 | struct mm_struct *mm = current->mm; |
Jeff Moyer | 65c2449 | 2009-03-18 17:04:21 -0700 | [diff] [blame] | 1053 | struct kioctx *ctx, *ret = NULL; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1054 | struct kioctx_table *table; |
| 1055 | unsigned id; |
| 1056 | |
| 1057 | if (get_user(id, &ring->id)) |
| 1058 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1059 | |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1060 | rcu_read_lock(); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1061 | table = rcu_dereference(mm->ioctx_table); |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1062 | |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1063 | if (!table || id >= table->nr) |
| 1064 | goto out; |
| 1065 | |
Jeff Moyer | a538e3f | 2018-12-11 12:37:49 -0500 | [diff] [blame] | 1066 | id = array_index_nospec(id, table->nr); |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 1067 | ctx = rcu_dereference(table->table[id]); |
Benjamin LaHaise | f30d704 | 2013-08-07 18:23:48 -0400 | [diff] [blame] | 1068 | if (ctx && ctx->user_id == ctx_id) { |
Al Viro | baf1056 | 2018-05-20 16:46:23 -0400 | [diff] [blame] | 1069 | if (percpu_ref_tryget_live(&ctx->users)) |
| 1070 | ret = ctx; |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1071 | } |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1072 | out: |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1073 | rcu_read_unlock(); |
Jeff Moyer | 65c2449 | 2009-03-18 17:04:21 -0700 | [diff] [blame] | 1074 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1075 | } |
| 1076 | |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1077 | static inline void iocb_destroy(struct aio_kiocb *iocb) |
| 1078 | { |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1079 | if (iocb->ki_eventfd) |
| 1080 | eventfd_ctx_put(iocb->ki_eventfd); |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1081 | if (iocb->ki_filp) |
| 1082 | fput(iocb->ki_filp); |
| 1083 | percpu_ref_put(&iocb->ki_ctx->reqs); |
| 1084 | kmem_cache_free(kiocb_cachep, iocb); |
| 1085 | } |
| 1086 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1087 | /* aio_complete |
| 1088 | * Called when the io request on the given iocb is complete. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1089 | */ |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1090 | static void aio_complete(struct aio_kiocb *iocb) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1091 | { |
| 1092 | struct kioctx *ctx = iocb->ki_ctx; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1093 | struct aio_ring *ring; |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1094 | struct io_event *ev_page, *event; |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 1095 | unsigned tail, pos, head; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1096 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1097 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1098 | /* |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 1099 | * Add a completion event to the ring buffer. Must be done holding |
Tang Chen | 4b30f07 | 2013-07-03 15:09:16 -0700 | [diff] [blame] | 1100 | * ctx->completion_lock to prevent other code from messing with the tail |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 1101 | * pointer since we might be called from irq context. |
| 1102 | */ |
| 1103 | spin_lock_irqsave(&ctx->completion_lock, flags); |
| 1104 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1105 | tail = ctx->tail; |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1106 | pos = tail + AIO_EVENTS_OFFSET; |
| 1107 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1108 | if (++tail >= ctx->nr_events) |
Ken Chen | 4bf69b2 | 2005-05-01 08:59:15 -0700 | [diff] [blame] | 1109 | tail = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1110 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1111 | ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1112 | event = ev_page + pos % AIO_EVENTS_PER_PAGE; |
| 1113 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 1114 | *event = iocb->ki_res; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1115 | |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1116 | kunmap_atomic(ev_page); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1117 | flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1118 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 1119 | pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb, |
| 1120 | (void __user *)(unsigned long)iocb->ki_res.obj, |
| 1121 | iocb->ki_res.data, iocb->ki_res.res, iocb->ki_res.res2); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1122 | |
| 1123 | /* after flagging the request as done, we |
| 1124 | * must never even look at it again |
| 1125 | */ |
| 1126 | smp_wmb(); /* make event visible before updating tail */ |
| 1127 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1128 | ctx->tail = tail; |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1129 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1130 | ring = kmap_atomic(ctx->ring_pages[0]); |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 1131 | head = ring->head; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1132 | ring->tail = tail; |
Cong Wang | e8e3c3d | 2011-11-25 23:14:27 +0800 | [diff] [blame] | 1133 | kunmap_atomic(ring); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1134 | flush_dcache_page(ctx->ring_pages[0]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1135 | |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 1136 | ctx->completed_events++; |
| 1137 | if (ctx->completed_events > 1) |
| 1138 | refill_reqs_available(ctx, head, tail); |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 1139 | spin_unlock_irqrestore(&ctx->completion_lock, flags); |
| 1140 | |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1141 | pr_debug("added to ring %p at [%u]\n", iocb, tail); |
Davide Libenzi | 8d1c98b | 2008-04-10 21:29:19 -0700 | [diff] [blame] | 1142 | |
| 1143 | /* |
| 1144 | * Check if the user asked us to deliver the result through an |
| 1145 | * eventfd. The eventfd_signal() function is safe to be called |
| 1146 | * from IRQ context. |
| 1147 | */ |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1148 | if (iocb->ki_eventfd) |
Davide Libenzi | 8d1c98b | 2008-04-10 21:29:19 -0700 | [diff] [blame] | 1149 | eventfd_signal(iocb->ki_eventfd, 1); |
| 1150 | |
Quentin Barnes | 6cb2a21 | 2008-03-19 17:00:39 -0700 | [diff] [blame] | 1151 | /* |
| 1152 | * We have to order our ring_info tail store above and test |
| 1153 | * of the wait list below outside the wait lock. This is |
| 1154 | * like in wake_up_bit() where clearing a bit has to be |
| 1155 | * ordered with the unlocked test. |
| 1156 | */ |
| 1157 | smp_mb(); |
| 1158 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1159 | if (waitqueue_active(&ctx->wait)) |
| 1160 | wake_up(&ctx->wait); |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1161 | } |
| 1162 | |
| 1163 | static inline void iocb_put(struct aio_kiocb *iocb) |
| 1164 | { |
| 1165 | if (refcount_dec_and_test(&iocb->ki_refcnt)) { |
| 1166 | aio_complete(iocb); |
| 1167 | iocb_destroy(iocb); |
| 1168 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1169 | } |
| 1170 | |
Gu Zheng | 2be4e7d | 2014-07-23 18:03:53 +0800 | [diff] [blame] | 1171 | /* aio_read_events_ring |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1172 | * Pull an event off of the ioctx's event ring. Returns the number of |
| 1173 | * events fetched |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1174 | */ |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1175 | static long aio_read_events_ring(struct kioctx *ctx, |
| 1176 | struct io_event __user *event, long nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1177 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1178 | struct aio_ring *ring; |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1179 | unsigned head, tail, pos; |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1180 | long ret = 0; |
| 1181 | int copy_ret; |
| 1182 | |
Dave Chinner | 9c9ce76 | 2015-02-03 19:29:05 -0500 | [diff] [blame] | 1183 | /* |
| 1184 | * The mutex can block and wake us up and that will cause |
| 1185 | * wait_event_interruptible_hrtimeout() to schedule without sleeping |
| 1186 | * and repeat. This should be rare enough that it doesn't cause |
| 1187 | * peformance issues. See the comment in read_events() for more detail. |
| 1188 | */ |
| 1189 | sched_annotate_sleep(); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1190 | mutex_lock(&ctx->ring_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1191 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 1192 | /* Access to ->ring_pages here is protected by ctx->ring_lock. */ |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1193 | ring = kmap_atomic(ctx->ring_pages[0]); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1194 | head = ring->head; |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1195 | tail = ring->tail; |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1196 | kunmap_atomic(ring); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1197 | |
Jeff Moyer | 2ff396be | 2014-09-02 13:17:00 -0400 | [diff] [blame] | 1198 | /* |
| 1199 | * Ensure that once we've read the current tail pointer, that |
| 1200 | * we also see the events that were stored up to the tail. |
| 1201 | */ |
| 1202 | smp_rmb(); |
| 1203 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1204 | pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1205 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1206 | if (head == tail) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1207 | goto out; |
| 1208 | |
Benjamin LaHaise | edfbbf3 | 2014-06-24 13:32:51 -0400 | [diff] [blame] | 1209 | head %= ctx->nr_events; |
| 1210 | tail %= ctx->nr_events; |
| 1211 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1212 | while (ret < nr) { |
| 1213 | long avail; |
| 1214 | struct io_event *ev; |
| 1215 | struct page *page; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1216 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1217 | avail = (head <= tail ? tail : ctx->nr_events) - head; |
| 1218 | if (head == tail) |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1219 | break; |
| 1220 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1221 | pos = head + AIO_EVENTS_OFFSET; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1222 | page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]; |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1223 | pos %= AIO_EVENTS_PER_PAGE; |
| 1224 | |
Al Viro | d2988bd4 | 2018-05-26 19:13:10 -0400 | [diff] [blame] | 1225 | avail = min(avail, nr - ret); |
| 1226 | avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); |
| 1227 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1228 | ev = kmap(page); |
| 1229 | copy_ret = copy_to_user(event + ret, ev + pos, |
| 1230 | sizeof(*ev) * avail); |
| 1231 | kunmap(page); |
| 1232 | |
| 1233 | if (unlikely(copy_ret)) { |
| 1234 | ret = -EFAULT; |
| 1235 | goto out; |
| 1236 | } |
| 1237 | |
| 1238 | ret += avail; |
| 1239 | head += avail; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1240 | head %= ctx->nr_events; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1241 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1242 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1243 | ring = kmap_atomic(ctx->ring_pages[0]); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1244 | ring->head = head; |
Zhao Hongjiang | 91d80a8 | 2013-04-26 11:03:53 +0800 | [diff] [blame] | 1245 | kunmap_atomic(ring); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1246 | flush_dcache_page(ctx->ring_pages[0]); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1247 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1248 | pr_debug("%li h%u t%u\n", ret, head, tail); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1249 | out: |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1250 | mutex_unlock(&ctx->ring_lock); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1251 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1252 | return ret; |
| 1253 | } |
| 1254 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1255 | static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr, |
| 1256 | struct io_event __user *event, long *i) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1257 | { |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1258 | long ret = aio_read_events_ring(ctx, event + *i, nr - *i); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1259 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1260 | if (ret > 0) |
| 1261 | *i += ret; |
| 1262 | |
| 1263 | if (unlikely(atomic_read(&ctx->dead))) |
| 1264 | ret = -EINVAL; |
| 1265 | |
| 1266 | if (!*i) |
| 1267 | *i = ret; |
| 1268 | |
| 1269 | return ret < 0 || *i >= min_nr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1270 | } |
| 1271 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1272 | static long read_events(struct kioctx *ctx, long min_nr, long nr, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1273 | struct io_event __user *event, |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 1274 | ktime_t until) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1275 | { |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1276 | long ret = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1277 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1278 | /* |
| 1279 | * Note that aio_read_events() is being called as the conditional - i.e. |
| 1280 | * we're calling it after prepare_to_wait() has set task state to |
| 1281 | * TASK_INTERRUPTIBLE. |
| 1282 | * |
| 1283 | * But aio_read_events() can block, and if it blocks it's going to flip |
| 1284 | * the task state back to TASK_RUNNING. |
| 1285 | * |
| 1286 | * This should be ok, provided it doesn't flip the state back to |
| 1287 | * TASK_RUNNING and return 0 too much - that causes us to spin. That |
| 1288 | * will only happen if the mutex_lock() call blocks, and we then find |
| 1289 | * the ringbuffer empty. So in practice we should be ok, but it's |
| 1290 | * something to be aware of when touching this code. |
| 1291 | */ |
Thomas Gleixner | 2456e85 | 2016-12-25 11:38:40 +0100 | [diff] [blame] | 1292 | if (until == 0) |
Fam Zheng | 5f785de | 2014-11-06 20:44:36 +0800 | [diff] [blame] | 1293 | aio_read_events(ctx, min_nr, nr, event, &ret); |
| 1294 | else |
| 1295 | wait_event_interruptible_hrtimeout(ctx->wait, |
| 1296 | aio_read_events(ctx, min_nr, nr, event, &ret), |
| 1297 | until); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1298 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1299 | } |
| 1300 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1301 | /* sys_io_setup: |
| 1302 | * Create an aio_context capable of receiving at least nr_events. |
| 1303 | * ctxp must not point to an aio_context that already exists, and |
| 1304 | * must be initialized to 0 prior to the call. On successful |
| 1305 | * creation of the aio_context, *ctxp is filled in with the resulting |
| 1306 | * handle. May fail with -EINVAL if *ctxp is not initialized, |
| 1307 | * if the specified nr_events exceeds internal limits. May fail |
| 1308 | * with -EAGAIN if the specified nr_events exceeds the user's limit |
| 1309 | * of available events. May fail with -ENOMEM if insufficient kernel |
| 1310 | * resources are available. May fail with -EFAULT if an invalid |
| 1311 | * pointer is passed for ctxp. Will fail with -ENOSYS if not |
| 1312 | * implemented. |
| 1313 | */ |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 1314 | SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1315 | { |
| 1316 | struct kioctx *ioctx = NULL; |
| 1317 | unsigned long ctx; |
| 1318 | long ret; |
| 1319 | |
| 1320 | ret = get_user(ctx, ctxp); |
| 1321 | if (unlikely(ret)) |
| 1322 | goto out; |
| 1323 | |
| 1324 | ret = -EINVAL; |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 1325 | if (unlikely(ctx || nr_events == 0)) { |
Kinglong Mee | acd88d4 | 2015-02-04 21:15:59 +0800 | [diff] [blame] | 1326 | pr_debug("EINVAL: ctx %lu nr_events %u\n", |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 1327 | ctx, nr_events); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1328 | goto out; |
| 1329 | } |
| 1330 | |
| 1331 | ioctx = ioctx_alloc(nr_events); |
| 1332 | ret = PTR_ERR(ioctx); |
| 1333 | if (!IS_ERR(ioctx)) { |
| 1334 | ret = put_user(ioctx->user_id, ctxp); |
Al Viro | a2e1859 | 2012-03-20 16:27:57 -0400 | [diff] [blame] | 1335 | if (ret) |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1336 | kill_ioctx(current->mm, ioctx, NULL); |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 1337 | percpu_ref_put(&ioctx->users); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1338 | } |
| 1339 | |
| 1340 | out: |
| 1341 | return ret; |
| 1342 | } |
| 1343 | |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 1344 | #ifdef CONFIG_COMPAT |
| 1345 | COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p) |
| 1346 | { |
| 1347 | struct kioctx *ioctx = NULL; |
| 1348 | unsigned long ctx; |
| 1349 | long ret; |
| 1350 | |
| 1351 | ret = get_user(ctx, ctx32p); |
| 1352 | if (unlikely(ret)) |
| 1353 | goto out; |
| 1354 | |
| 1355 | ret = -EINVAL; |
| 1356 | if (unlikely(ctx || nr_events == 0)) { |
| 1357 | pr_debug("EINVAL: ctx %lu nr_events %u\n", |
| 1358 | ctx, nr_events); |
| 1359 | goto out; |
| 1360 | } |
| 1361 | |
| 1362 | ioctx = ioctx_alloc(nr_events); |
| 1363 | ret = PTR_ERR(ioctx); |
| 1364 | if (!IS_ERR(ioctx)) { |
| 1365 | /* truncating is ok because it's a user address */ |
| 1366 | ret = put_user((u32)ioctx->user_id, ctx32p); |
| 1367 | if (ret) |
| 1368 | kill_ioctx(current->mm, ioctx, NULL); |
| 1369 | percpu_ref_put(&ioctx->users); |
| 1370 | } |
| 1371 | |
| 1372 | out: |
| 1373 | return ret; |
| 1374 | } |
| 1375 | #endif |
| 1376 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1377 | /* sys_io_destroy: |
| 1378 | * Destroy the aio_context specified. May cancel any outstanding |
| 1379 | * AIOs and block on completion. Will fail with -ENOSYS if not |
Satoru Takeuchi | 642b512 | 2010-08-05 11:23:11 -0700 | [diff] [blame] | 1380 | * implemented. May fail with -EINVAL if the context pointed to |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1381 | * is invalid. |
| 1382 | */ |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 1383 | SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1384 | { |
| 1385 | struct kioctx *ioctx = lookup_ioctx(ctx); |
| 1386 | if (likely(NULL != ioctx)) { |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1387 | struct ctx_rq_wait wait; |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 1388 | int ret; |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1389 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1390 | init_completion(&wait.comp); |
| 1391 | atomic_set(&wait.count, 1); |
| 1392 | |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1393 | /* Pass requests_done to kill_ioctx() where it can be set |
| 1394 | * in a thread-safe way. If we try to set it here then we have |
| 1395 | * a race condition if two io_destroy() called simultaneously. |
| 1396 | */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1397 | ret = kill_ioctx(current->mm, ioctx, &wait); |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 1398 | percpu_ref_put(&ioctx->users); |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1399 | |
| 1400 | /* Wait until all IO for the context are done. Otherwise kernel |
| 1401 | * keep using user-space buffers even if user thinks the context |
| 1402 | * is destroyed. |
| 1403 | */ |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 1404 | if (!ret) |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1405 | wait_for_completion(&wait.comp); |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1406 | |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 1407 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1408 | } |
Kinglong Mee | acd88d4 | 2015-02-04 21:15:59 +0800 | [diff] [blame] | 1409 | pr_debug("EINVAL: invalid context id\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1410 | return -EINVAL; |
| 1411 | } |
| 1412 | |
Al Viro | 3c96c7f | 2018-05-28 13:37:43 -0400 | [diff] [blame] | 1413 | static void aio_remove_iocb(struct aio_kiocb *iocb) |
| 1414 | { |
| 1415 | struct kioctx *ctx = iocb->ki_ctx; |
| 1416 | unsigned long flags; |
| 1417 | |
| 1418 | spin_lock_irqsave(&ctx->ctx_lock, flags); |
| 1419 | list_del(&iocb->ki_list); |
| 1420 | spin_unlock_irqrestore(&ctx->ctx_lock, flags); |
| 1421 | } |
| 1422 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1423 | static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) |
| 1424 | { |
| 1425 | struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); |
| 1426 | |
Al Viro | 3c96c7f | 2018-05-28 13:37:43 -0400 | [diff] [blame] | 1427 | if (!list_empty_careful(&iocb->ki_list)) |
| 1428 | aio_remove_iocb(iocb); |
| 1429 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1430 | if (kiocb->ki_flags & IOCB_WRITE) { |
| 1431 | struct inode *inode = file_inode(kiocb->ki_filp); |
| 1432 | |
| 1433 | /* |
| 1434 | * Tell lockdep we inherited freeze protection from submission |
| 1435 | * thread. |
| 1436 | */ |
| 1437 | if (S_ISREG(inode->i_mode)) |
| 1438 | __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE); |
| 1439 | file_end_write(kiocb->ki_filp); |
| 1440 | } |
| 1441 | |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1442 | iocb->ki_res.res = res; |
| 1443 | iocb->ki_res.res2 = res2; |
| 1444 | iocb_put(iocb); |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1445 | } |
| 1446 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1447 | static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1448 | { |
| 1449 | int ret; |
| 1450 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1451 | req->ki_complete = aio_complete_rw; |
Mike Marshall | ec51f8e | 2019-02-05 14:13:35 -0500 | [diff] [blame] | 1452 | req->private = NULL; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1453 | req->ki_pos = iocb->aio_offset; |
| 1454 | req->ki_flags = iocb_flags(req->ki_filp); |
| 1455 | if (iocb->aio_flags & IOCB_FLAG_RESFD) |
| 1456 | req->ki_flags |= IOCB_EVENTFD; |
Adam Manzanares | fc28724 | 2018-05-22 10:52:18 -0700 | [diff] [blame] | 1457 | req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp)); |
Adam Manzanares | d9a08a9 | 2018-05-22 10:52:19 -0700 | [diff] [blame] | 1458 | if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { |
| 1459 | /* |
| 1460 | * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then |
| 1461 | * aio_reqprio is interpreted as an I/O scheduling |
| 1462 | * class and priority. |
| 1463 | */ |
| 1464 | ret = ioprio_check_cap(iocb->aio_reqprio); |
| 1465 | if (ret) { |
Adam Manzanares | 9a6d9a6 | 2018-06-04 10:59:57 -0700 | [diff] [blame] | 1466 | pr_debug("aio ioprio check cap error: %d\n", ret); |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1467 | return ret; |
Adam Manzanares | d9a08a9 | 2018-05-22 10:52:19 -0700 | [diff] [blame] | 1468 | } |
| 1469 | |
| 1470 | req->ki_ioprio = iocb->aio_reqprio; |
| 1471 | } else |
Damien Le Moal | 76dc891 | 2018-11-20 10:52:36 +0900 | [diff] [blame] | 1472 | req->ki_ioprio = get_current_ioprio(); |
Adam Manzanares | d9a08a9 | 2018-05-22 10:52:19 -0700 | [diff] [blame] | 1473 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1474 | ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); |
| 1475 | if (unlikely(ret)) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1476 | return ret; |
Christoph Hellwig | 154989e | 2018-11-22 16:44:07 +0100 | [diff] [blame] | 1477 | |
| 1478 | req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ |
| 1479 | return 0; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1480 | } |
| 1481 | |
Jens Axboe | 87e5e6d | 2019-05-14 16:02:22 -0600 | [diff] [blame] | 1482 | static ssize_t aio_setup_rw(int rw, const struct iocb *iocb, |
| 1483 | struct iovec **iovec, bool vectored, bool compat, |
| 1484 | struct iov_iter *iter) |
Badari Pulavarty | eed4e51 | 2006-09-30 23:28:49 -0700 | [diff] [blame] | 1485 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1486 | void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf; |
| 1487 | size_t len = iocb->aio_nbytes; |
| 1488 | |
| 1489 | if (!vectored) { |
| 1490 | ssize_t ret = import_single_range(rw, buf, len, *iovec, iter); |
| 1491 | *iovec = NULL; |
| 1492 | return ret; |
| 1493 | } |
Christoph Hellwig | 89cd35c | 2020-09-25 06:51:41 +0200 | [diff] [blame] | 1494 | |
| 1495 | return __import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter, compat); |
Badari Pulavarty | eed4e51 | 2006-09-30 23:28:49 -0700 | [diff] [blame] | 1496 | } |
| 1497 | |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1498 | static inline void aio_rw_done(struct kiocb *req, ssize_t ret) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1499 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1500 | switch (ret) { |
| 1501 | case -EIOCBQUEUED: |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1502 | break; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1503 | case -ERESTARTSYS: |
| 1504 | case -ERESTARTNOINTR: |
| 1505 | case -ERESTARTNOHAND: |
| 1506 | case -ERESTART_RESTARTBLOCK: |
Kent Overstreet | 41ef4eb | 2013-05-07 16:19:11 -0700 | [diff] [blame] | 1507 | /* |
| 1508 | * There's no easy way to restart the syscall since other AIO's |
| 1509 | * may be already running. Just fail this IO with EINTR. |
| 1510 | */ |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1511 | ret = -EINTR; |
Gustavo A. R. Silva | df561f66 | 2020-08-23 17:36:59 -0500 | [diff] [blame] | 1512 | fallthrough; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1513 | default: |
Jens Axboe | bc9bff6 | 2018-11-06 14:27:13 -0700 | [diff] [blame] | 1514 | req->ki_complete(req, ret, 0); |
Kent Overstreet | 41ef4eb | 2013-05-07 16:19:11 -0700 | [diff] [blame] | 1515 | } |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1516 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1517 | |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1518 | static int aio_read(struct kiocb *req, const struct iocb *iocb, |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1519 | bool vectored, bool compat) |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1520 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1521 | struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; |
| 1522 | struct iov_iter iter; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1523 | struct file *file; |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1524 | int ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1525 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1526 | ret = aio_prep_rw(req, iocb); |
| 1527 | if (ret) |
| 1528 | return ret; |
| 1529 | file = req->ki_filp; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1530 | if (unlikely(!(file->f_mode & FMODE_READ))) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1531 | return -EBADF; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1532 | ret = -EINVAL; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1533 | if (unlikely(!file->f_op->read_iter)) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1534 | return -EINVAL; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1535 | |
| 1536 | ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter); |
Jens Axboe | 87e5e6d | 2019-05-14 16:02:22 -0600 | [diff] [blame] | 1537 | if (ret < 0) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1538 | return ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1539 | ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); |
| 1540 | if (!ret) |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1541 | aio_rw_done(req, call_read_iter(file, req, &iter)); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1542 | kfree(iovec); |
| 1543 | return ret; |
| 1544 | } |
| 1545 | |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1546 | static int aio_write(struct kiocb *req, const struct iocb *iocb, |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1547 | bool vectored, bool compat) |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1548 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1549 | struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; |
| 1550 | struct iov_iter iter; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1551 | struct file *file; |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1552 | int ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1553 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1554 | ret = aio_prep_rw(req, iocb); |
| 1555 | if (ret) |
| 1556 | return ret; |
| 1557 | file = req->ki_filp; |
| 1558 | |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1559 | if (unlikely(!(file->f_mode & FMODE_WRITE))) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1560 | return -EBADF; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1561 | if (unlikely(!file->f_op->write_iter)) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1562 | return -EINVAL; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1563 | |
| 1564 | ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter); |
Jens Axboe | 87e5e6d | 2019-05-14 16:02:22 -0600 | [diff] [blame] | 1565 | if (ret < 0) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1566 | return ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1567 | ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); |
| 1568 | if (!ret) { |
Jan Kara | 70fe2f4 | 2016-10-30 11:42:04 -0500 | [diff] [blame] | 1569 | /* |
Christoph Hellwig | 92ce472 | 2018-04-06 09:28:17 +0200 | [diff] [blame] | 1570 | * Open-code file_start_write here to grab freeze protection, |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1571 | * which will be released by another thread in |
| 1572 | * aio_complete_rw(). Fool lockdep by telling it the lock got |
| 1573 | * released so that it doesn't complain about the held lock when |
| 1574 | * we return to userspace. |
Jan Kara | 70fe2f4 | 2016-10-30 11:42:04 -0500 | [diff] [blame] | 1575 | */ |
Christoph Hellwig | 92ce472 | 2018-04-06 09:28:17 +0200 | [diff] [blame] | 1576 | if (S_ISREG(file_inode(file)->i_mode)) { |
Darrick J. Wong | 8a3c84b | 2020-11-10 16:50:21 -0800 | [diff] [blame] | 1577 | sb_start_write(file_inode(file)->i_sb); |
Shaohua Li | a12f1ae | 2016-12-13 12:09:56 -0800 | [diff] [blame] | 1578 | __sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE); |
Christoph Hellwig | 92ce472 | 2018-04-06 09:28:17 +0200 | [diff] [blame] | 1579 | } |
| 1580 | req->ki_flags |= IOCB_WRITE; |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1581 | aio_rw_done(req, call_write_iter(file, req, &iter)); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1582 | } |
| 1583 | kfree(iovec); |
| 1584 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1585 | } |
| 1586 | |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1587 | static void aio_fsync_work(struct work_struct *work) |
| 1588 | { |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1589 | struct aio_kiocb *iocb = container_of(work, struct aio_kiocb, fsync.work); |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 1590 | const struct cred *old_cred = override_creds(iocb->fsync.creds); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1591 | |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1592 | iocb->ki_res.res = vfs_fsync(iocb->fsync.file, iocb->fsync.datasync); |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 1593 | revert_creds(old_cred); |
| 1594 | put_cred(iocb->fsync.creds); |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1595 | iocb_put(iocb); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1596 | } |
| 1597 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1598 | static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, |
| 1599 | bool datasync) |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1600 | { |
| 1601 | if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || |
| 1602 | iocb->aio_rw_flags)) |
| 1603 | return -EINVAL; |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 1604 | |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1605 | if (unlikely(!req->file->f_op->fsync)) |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1606 | return -EINVAL; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1607 | |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 1608 | req->creds = prepare_creds(); |
| 1609 | if (!req->creds) |
| 1610 | return -ENOMEM; |
| 1611 | |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1612 | req->datasync = datasync; |
| 1613 | INIT_WORK(&req->work, aio_fsync_work); |
| 1614 | schedule_work(&req->work); |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1615 | return 0; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1616 | } |
| 1617 | |
Jens Axboe | 01d7a35 | 2020-02-03 10:33:42 -0700 | [diff] [blame] | 1618 | static void aio_poll_put_work(struct work_struct *work) |
| 1619 | { |
| 1620 | struct poll_iocb *req = container_of(work, struct poll_iocb, work); |
| 1621 | struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); |
| 1622 | |
| 1623 | iocb_put(iocb); |
| 1624 | } |
| 1625 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1626 | static void aio_poll_complete_work(struct work_struct *work) |
| 1627 | { |
| 1628 | struct poll_iocb *req = container_of(work, struct poll_iocb, work); |
| 1629 | struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); |
| 1630 | struct poll_table_struct pt = { ._key = req->events }; |
| 1631 | struct kioctx *ctx = iocb->ki_ctx; |
| 1632 | __poll_t mask = 0; |
| 1633 | |
| 1634 | if (!READ_ONCE(req->cancelled)) |
| 1635 | mask = vfs_poll(req->file, &pt) & req->events; |
| 1636 | |
| 1637 | /* |
| 1638 | * Note that ->ki_cancel callers also delete iocb from active_reqs after |
| 1639 | * calling ->ki_cancel. We need the ctx_lock roundtrip here to |
| 1640 | * synchronize with them. In the cancellation case the list_del_init |
| 1641 | * itself is not actually needed, but harmless so we keep it in to |
| 1642 | * avoid further branches in the fast path. |
| 1643 | */ |
| 1644 | spin_lock_irq(&ctx->ctx_lock); |
| 1645 | if (!mask && !READ_ONCE(req->cancelled)) { |
| 1646 | add_wait_queue(req->head, &req->wait); |
| 1647 | spin_unlock_irq(&ctx->ctx_lock); |
| 1648 | return; |
| 1649 | } |
| 1650 | list_del_init(&iocb->ki_list); |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1651 | iocb->ki_res.res = mangle_poll(mask); |
| 1652 | req->done = true; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1653 | spin_unlock_irq(&ctx->ctx_lock); |
| 1654 | |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1655 | iocb_put(iocb); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1656 | } |
| 1657 | |
| 1658 | /* assumes we are called with irqs disabled */ |
| 1659 | static int aio_poll_cancel(struct kiocb *iocb) |
| 1660 | { |
| 1661 | struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw); |
| 1662 | struct poll_iocb *req = &aiocb->poll; |
| 1663 | |
| 1664 | spin_lock(&req->head->lock); |
| 1665 | WRITE_ONCE(req->cancelled, true); |
| 1666 | if (!list_empty(&req->wait.entry)) { |
| 1667 | list_del_init(&req->wait.entry); |
| 1668 | schedule_work(&aiocb->poll.work); |
| 1669 | } |
| 1670 | spin_unlock(&req->head->lock); |
| 1671 | |
| 1672 | return 0; |
| 1673 | } |
| 1674 | |
| 1675 | static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, |
| 1676 | void *key) |
| 1677 | { |
| 1678 | struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); |
Christoph Hellwig | e8693bc | 2018-07-16 12:25:17 +0200 | [diff] [blame] | 1679 | struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1680 | __poll_t mask = key_to_poll(key); |
Bart Van Assche | d3d6a18 | 2019-02-08 16:59:49 -0800 | [diff] [blame] | 1681 | unsigned long flags; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1682 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1683 | /* for instances that support it check for an event match first: */ |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1684 | if (mask && !(mask & req->events)) |
| 1685 | return 0; |
Christoph Hellwig | e8693bc | 2018-07-16 12:25:17 +0200 | [diff] [blame] | 1686 | |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1687 | list_del_init(&req->wait.entry); |
| 1688 | |
| 1689 | if (mask && spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) { |
Jens Axboe | 01d7a35 | 2020-02-03 10:33:42 -0700 | [diff] [blame] | 1690 | struct kioctx *ctx = iocb->ki_ctx; |
| 1691 | |
Bart Van Assche | d3d6a18 | 2019-02-08 16:59:49 -0800 | [diff] [blame] | 1692 | /* |
| 1693 | * Try to complete the iocb inline if we can. Use |
| 1694 | * irqsave/irqrestore because not all filesystems (e.g. fuse) |
| 1695 | * call this function with IRQs disabled and because IRQs |
| 1696 | * have to be disabled before ctx_lock is obtained. |
| 1697 | */ |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1698 | list_del(&iocb->ki_list); |
| 1699 | iocb->ki_res.res = mangle_poll(mask); |
| 1700 | req->done = true; |
Jens Axboe | 01d7a35 | 2020-02-03 10:33:42 -0700 | [diff] [blame] | 1701 | if (iocb->ki_eventfd && eventfd_signal_count()) { |
| 1702 | iocb = NULL; |
| 1703 | INIT_WORK(&req->work, aio_poll_put_work); |
| 1704 | schedule_work(&req->work); |
| 1705 | } |
| 1706 | spin_unlock_irqrestore(&ctx->ctx_lock, flags); |
| 1707 | if (iocb) |
| 1708 | iocb_put(iocb); |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1709 | } else { |
| 1710 | schedule_work(&req->work); |
Christoph Hellwig | e8693bc | 2018-07-16 12:25:17 +0200 | [diff] [blame] | 1711 | } |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1712 | return 1; |
| 1713 | } |
| 1714 | |
| 1715 | struct aio_poll_table { |
| 1716 | struct poll_table_struct pt; |
| 1717 | struct aio_kiocb *iocb; |
| 1718 | int error; |
| 1719 | }; |
| 1720 | |
| 1721 | static void |
| 1722 | aio_poll_queue_proc(struct file *file, struct wait_queue_head *head, |
| 1723 | struct poll_table_struct *p) |
| 1724 | { |
| 1725 | struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt); |
| 1726 | |
| 1727 | /* multiple wait queues per file are not supported */ |
| 1728 | if (unlikely(pt->iocb->poll.head)) { |
| 1729 | pt->error = -EINVAL; |
| 1730 | return; |
| 1731 | } |
| 1732 | |
| 1733 | pt->error = 0; |
| 1734 | pt->iocb->poll.head = head; |
| 1735 | add_wait_queue(head, &pt->iocb->poll.wait); |
| 1736 | } |
| 1737 | |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1738 | static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1739 | { |
| 1740 | struct kioctx *ctx = aiocb->ki_ctx; |
| 1741 | struct poll_iocb *req = &aiocb->poll; |
| 1742 | struct aio_poll_table apt; |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1743 | bool cancel = false; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1744 | __poll_t mask; |
| 1745 | |
| 1746 | /* reject any unknown events outside the normal event mask. */ |
| 1747 | if ((u16)iocb->aio_buf != iocb->aio_buf) |
| 1748 | return -EINVAL; |
| 1749 | /* reject fields that are not defined for poll */ |
| 1750 | if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags) |
| 1751 | return -EINVAL; |
| 1752 | |
| 1753 | INIT_WORK(&req->work, aio_poll_complete_work); |
| 1754 | req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1755 | |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1756 | req->head = NULL; |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1757 | req->done = false; |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1758 | req->cancelled = false; |
| 1759 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1760 | apt.pt._qproc = aio_poll_queue_proc; |
| 1761 | apt.pt._key = req->events; |
| 1762 | apt.iocb = aiocb; |
| 1763 | apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */ |
| 1764 | |
| 1765 | /* initialized the list so that we can do list_empty checks */ |
| 1766 | INIT_LIST_HEAD(&req->wait.entry); |
| 1767 | init_waitqueue_func_entry(&req->wait, aio_poll_wake); |
| 1768 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1769 | mask = vfs_poll(req->file, &apt.pt) & req->events; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1770 | spin_lock_irq(&ctx->ctx_lock); |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1771 | if (likely(req->head)) { |
| 1772 | spin_lock(&req->head->lock); |
| 1773 | if (unlikely(list_empty(&req->wait.entry))) { |
| 1774 | if (apt.error) |
| 1775 | cancel = true; |
| 1776 | apt.error = 0; |
| 1777 | mask = 0; |
| 1778 | } |
| 1779 | if (mask || apt.error) { |
| 1780 | list_del_init(&req->wait.entry); |
| 1781 | } else if (cancel) { |
| 1782 | WRITE_ONCE(req->cancelled, true); |
| 1783 | } else if (!req->done) { /* actually waiting for an event */ |
| 1784 | list_add_tail(&aiocb->ki_list, &ctx->active_reqs); |
| 1785 | aiocb->ki_cancel = aio_poll_cancel; |
| 1786 | } |
| 1787 | spin_unlock(&req->head->lock); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1788 | } |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1789 | if (mask) { /* no async, we'd stolen it */ |
| 1790 | aiocb->ki_res.res = mangle_poll(mask); |
| 1791 | apt.error = 0; |
| 1792 | } |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1793 | spin_unlock_irq(&ctx->ctx_lock); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1794 | if (mask) |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1795 | iocb_put(aiocb); |
| 1796 | return apt.error; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1797 | } |
| 1798 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1799 | static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb, |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1800 | struct iocb __user *user_iocb, struct aio_kiocb *req, |
| 1801 | bool compat) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1802 | { |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1803 | req->ki_filp = fget(iocb->aio_fildes); |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1804 | if (unlikely(!req->ki_filp)) |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1805 | return -EBADF; |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1806 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1807 | if (iocb->aio_flags & IOCB_FLAG_RESFD) { |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1808 | struct eventfd_ctx *eventfd; |
Davide Libenzi | 9c3060b | 2007-05-10 22:23:21 -0700 | [diff] [blame] | 1809 | /* |
| 1810 | * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an |
| 1811 | * instance of the file* now. The file descriptor must be |
| 1812 | * an eventfd() fd, and will be signaled for each completed |
| 1813 | * event using the eventfd_signal() function. |
| 1814 | */ |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1815 | eventfd = eventfd_ctx_fdget(iocb->aio_resfd); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1816 | if (IS_ERR(eventfd)) |
Dan Carpenter | 18bfb9c | 2019-04-03 09:22:35 +0300 | [diff] [blame] | 1817 | return PTR_ERR(eventfd); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1818 | |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1819 | req->ki_eventfd = eventfd; |
Goldwyn Rodrigues | 9830f4b | 2017-06-20 07:05:42 -0500 | [diff] [blame] | 1820 | } |
| 1821 | |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1822 | if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) { |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 1823 | pr_debug("EFAULT: aio_key\n"); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1824 | return -EFAULT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1825 | } |
| 1826 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 1827 | req->ki_res.obj = (u64)(unsigned long)user_iocb; |
| 1828 | req->ki_res.data = iocb->aio_data; |
| 1829 | req->ki_res.res = 0; |
| 1830 | req->ki_res.res2 = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1831 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1832 | switch (iocb->aio_lio_opcode) { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1833 | case IOCB_CMD_PREAD: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1834 | return aio_read(&req->rw, iocb, false, compat); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1835 | case IOCB_CMD_PWRITE: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1836 | return aio_write(&req->rw, iocb, false, compat); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1837 | case IOCB_CMD_PREADV: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1838 | return aio_read(&req->rw, iocb, true, compat); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1839 | case IOCB_CMD_PWRITEV: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1840 | return aio_write(&req->rw, iocb, true, compat); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1841 | case IOCB_CMD_FSYNC: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1842 | return aio_fsync(&req->fsync, iocb, false); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1843 | case IOCB_CMD_FDSYNC: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1844 | return aio_fsync(&req->fsync, iocb, true); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1845 | case IOCB_CMD_POLL: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1846 | return aio_poll(req, iocb); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1847 | default: |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1848 | pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1849 | return -EINVAL; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1850 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1851 | } |
| 1852 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1853 | static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, |
| 1854 | bool compat) |
| 1855 | { |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1856 | struct aio_kiocb *req; |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1857 | struct iocb iocb; |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1858 | int err; |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1859 | |
| 1860 | if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb)))) |
| 1861 | return -EFAULT; |
| 1862 | |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1863 | /* enforce forwards compatibility on users */ |
| 1864 | if (unlikely(iocb.aio_reserved2)) { |
| 1865 | pr_debug("EINVAL: reserve field set\n"); |
| 1866 | return -EINVAL; |
| 1867 | } |
| 1868 | |
| 1869 | /* prevent overflows */ |
| 1870 | if (unlikely( |
| 1871 | (iocb.aio_buf != (unsigned long)iocb.aio_buf) || |
| 1872 | (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) || |
| 1873 | ((ssize_t)iocb.aio_nbytes < 0) |
| 1874 | )) { |
| 1875 | pr_debug("EINVAL: overflow check\n"); |
| 1876 | return -EINVAL; |
| 1877 | } |
| 1878 | |
| 1879 | req = aio_get_req(ctx); |
| 1880 | if (unlikely(!req)) |
| 1881 | return -EAGAIN; |
| 1882 | |
| 1883 | err = __io_submit_one(ctx, &iocb, user_iocb, req, compat); |
| 1884 | |
| 1885 | /* Done with the synchronous reference */ |
| 1886 | iocb_put(req); |
| 1887 | |
| 1888 | /* |
| 1889 | * If err is 0, we'd either done aio_complete() ourselves or have |
| 1890 | * arranged for that to be done asynchronously. Anything non-zero |
| 1891 | * means that we need to destroy req ourselves. |
| 1892 | */ |
| 1893 | if (unlikely(err)) { |
| 1894 | iocb_destroy(req); |
| 1895 | put_reqs_available(ctx, 1); |
| 1896 | } |
| 1897 | return err; |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1898 | } |
| 1899 | |
Jeff Moyer | 9d85cba7 | 2010-05-26 14:44:26 -0700 | [diff] [blame] | 1900 | /* sys_io_submit: |
| 1901 | * Queue the nr iocbs pointed to by iocbpp for processing. Returns |
| 1902 | * the number of iocbs queued. May return -EINVAL if the aio_context |
| 1903 | * specified by ctx_id is invalid, if nr is < 0, if the iocb at |
| 1904 | * *iocbpp[0] is not properly initialized, if the operation specified |
| 1905 | * is invalid for the file descriptor in the iocb. May fail with |
| 1906 | * -EFAULT if any of the data structures point to invalid data. May |
| 1907 | * fail with -EBADF if the file descriptor specified in the first |
| 1908 | * iocb is invalid. May fail with -EAGAIN if insufficient resources |
| 1909 | * are available to queue any iocbs. Will return 0 if nr is 0. Will |
| 1910 | * fail with -ENOSYS if not implemented. |
| 1911 | */ |
| 1912 | SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, |
| 1913 | struct iocb __user * __user *, iocbpp) |
| 1914 | { |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1915 | struct kioctx *ctx; |
| 1916 | long ret = 0; |
| 1917 | int i = 0; |
| 1918 | struct blk_plug plug; |
| 1919 | |
| 1920 | if (unlikely(nr < 0)) |
| 1921 | return -EINVAL; |
| 1922 | |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1923 | ctx = lookup_ioctx(ctx_id); |
| 1924 | if (unlikely(!ctx)) { |
| 1925 | pr_debug("EINVAL: invalid context id\n"); |
| 1926 | return -EINVAL; |
| 1927 | } |
| 1928 | |
Al Viro | 1da9277 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1929 | if (nr > ctx->nr_events) |
| 1930 | nr = ctx->nr_events; |
| 1931 | |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 1932 | if (nr > AIO_PLUG_THRESHOLD) |
| 1933 | blk_start_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1934 | for (i = 0; i < nr; i++) { |
| 1935 | struct iocb __user *user_iocb; |
| 1936 | |
| 1937 | if (unlikely(get_user(user_iocb, iocbpp + i))) { |
| 1938 | ret = -EFAULT; |
| 1939 | break; |
| 1940 | } |
| 1941 | |
| 1942 | ret = io_submit_one(ctx, user_iocb, false); |
| 1943 | if (ret) |
| 1944 | break; |
| 1945 | } |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 1946 | if (nr > AIO_PLUG_THRESHOLD) |
| 1947 | blk_finish_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1948 | |
| 1949 | percpu_ref_put(&ctx->users); |
| 1950 | return i ? i : ret; |
Jeff Moyer | 9d85cba7 | 2010-05-26 14:44:26 -0700 | [diff] [blame] | 1951 | } |
| 1952 | |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 1953 | #ifdef CONFIG_COMPAT |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 1954 | COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id, |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1955 | int, nr, compat_uptr_t __user *, iocbpp) |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 1956 | { |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1957 | struct kioctx *ctx; |
| 1958 | long ret = 0; |
| 1959 | int i = 0; |
| 1960 | struct blk_plug plug; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 1961 | |
| 1962 | if (unlikely(nr < 0)) |
| 1963 | return -EINVAL; |
| 1964 | |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1965 | ctx = lookup_ioctx(ctx_id); |
| 1966 | if (unlikely(!ctx)) { |
| 1967 | pr_debug("EINVAL: invalid context id\n"); |
| 1968 | return -EINVAL; |
| 1969 | } |
| 1970 | |
Al Viro | 1da9277 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1971 | if (nr > ctx->nr_events) |
| 1972 | nr = ctx->nr_events; |
| 1973 | |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 1974 | if (nr > AIO_PLUG_THRESHOLD) |
| 1975 | blk_start_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1976 | for (i = 0; i < nr; i++) { |
| 1977 | compat_uptr_t user_iocb; |
| 1978 | |
| 1979 | if (unlikely(get_user(user_iocb, iocbpp + i))) { |
| 1980 | ret = -EFAULT; |
| 1981 | break; |
| 1982 | } |
| 1983 | |
| 1984 | ret = io_submit_one(ctx, compat_ptr(user_iocb), true); |
| 1985 | if (ret) |
| 1986 | break; |
| 1987 | } |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 1988 | if (nr > AIO_PLUG_THRESHOLD) |
| 1989 | blk_finish_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 1990 | |
| 1991 | percpu_ref_put(&ctx->users); |
| 1992 | return i ? i : ret; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 1993 | } |
| 1994 | #endif |
| 1995 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1996 | /* sys_io_cancel: |
| 1997 | * Attempts to cancel an iocb previously passed to io_submit. If |
| 1998 | * the operation is successfully cancelled, the resulting event is |
| 1999 | * copied into the memory pointed to by result without being placed |
| 2000 | * into the completion queue and 0 is returned. May fail with |
| 2001 | * -EFAULT if any of the data structures pointed to are invalid. |
| 2002 | * May fail with -EINVAL if aio_context specified by ctx_id is |
| 2003 | * invalid. May fail with -EAGAIN if the iocb specified was not |
| 2004 | * cancelled. Will fail with -ENOSYS if not implemented. |
| 2005 | */ |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 2006 | SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, |
| 2007 | struct io_event __user *, result) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2008 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2009 | struct kioctx *ctx; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 2010 | struct aio_kiocb *kiocb; |
Christoph Hellwig | 888933f | 2018-05-23 14:11:02 +0200 | [diff] [blame] | 2011 | int ret = -EINVAL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2012 | u32 key; |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 2013 | u64 obj = (u64)(unsigned long)iocb; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2014 | |
Christoph Hellwig | f3a2752 | 2018-03-30 11:19:25 +0200 | [diff] [blame] | 2015 | if (unlikely(get_user(key, &iocb->aio_key))) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2016 | return -EFAULT; |
Christoph Hellwig | f3a2752 | 2018-03-30 11:19:25 +0200 | [diff] [blame] | 2017 | if (unlikely(key != KIOCB_KEY)) |
| 2018 | return -EINVAL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2019 | |
| 2020 | ctx = lookup_ioctx(ctx_id); |
| 2021 | if (unlikely(!ctx)) |
| 2022 | return -EINVAL; |
| 2023 | |
| 2024 | spin_lock_irq(&ctx->ctx_lock); |
Al Viro | 833f415 | 2019-03-11 19:00:36 -0400 | [diff] [blame] | 2025 | /* TODO: use a hash or array, this sucks. */ |
| 2026 | list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) { |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 2027 | if (kiocb->ki_res.obj == obj) { |
Al Viro | 833f415 | 2019-03-11 19:00:36 -0400 | [diff] [blame] | 2028 | ret = kiocb->ki_cancel(&kiocb->rw); |
| 2029 | list_del_init(&kiocb->ki_list); |
| 2030 | break; |
| 2031 | } |
Christoph Hellwig | 888933f | 2018-05-23 14:11:02 +0200 | [diff] [blame] | 2032 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2033 | spin_unlock_irq(&ctx->ctx_lock); |
| 2034 | |
Kent Overstreet | 906b973 | 2013-05-07 16:18:31 -0700 | [diff] [blame] | 2035 | if (!ret) { |
Kent Overstreet | bec68faa | 2013-05-13 14:45:08 -0700 | [diff] [blame] | 2036 | /* |
| 2037 | * The result argument is no longer used - the io_event is |
| 2038 | * always delivered via the ring buffer. -EINPROGRESS indicates |
| 2039 | * cancellation is progress: |
Kent Overstreet | 906b973 | 2013-05-07 16:18:31 -0700 | [diff] [blame] | 2040 | */ |
Kent Overstreet | bec68faa | 2013-05-13 14:45:08 -0700 | [diff] [blame] | 2041 | ret = -EINPROGRESS; |
Kent Overstreet | 906b973 | 2013-05-07 16:18:31 -0700 | [diff] [blame] | 2042 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2043 | |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 2044 | percpu_ref_put(&ctx->users); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2045 | |
| 2046 | return ret; |
| 2047 | } |
| 2048 | |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2049 | static long do_io_getevents(aio_context_t ctx_id, |
| 2050 | long min_nr, |
| 2051 | long nr, |
| 2052 | struct io_event __user *events, |
| 2053 | struct timespec64 *ts) |
| 2054 | { |
| 2055 | ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX; |
| 2056 | struct kioctx *ioctx = lookup_ioctx(ctx_id); |
| 2057 | long ret = -EINVAL; |
| 2058 | |
| 2059 | if (likely(ioctx)) { |
| 2060 | if (likely(min_nr <= nr && min_nr >= 0)) |
| 2061 | ret = read_events(ioctx, min_nr, nr, events, until); |
| 2062 | percpu_ref_put(&ioctx->users); |
| 2063 | } |
| 2064 | |
| 2065 | return ret; |
| 2066 | } |
| 2067 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2068 | /* io_getevents: |
| 2069 | * Attempts to read at least min_nr events and up to nr events from |
Satoru Takeuchi | 642b512 | 2010-08-05 11:23:11 -0700 | [diff] [blame] | 2070 | * the completion queue for the aio_context specified by ctx_id. If |
| 2071 | * it succeeds, the number of read events is returned. May fail with |
| 2072 | * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is |
| 2073 | * out of range, if timeout is out of range. May fail with -EFAULT |
| 2074 | * if any of the memory specified is invalid. May return 0 or |
| 2075 | * < min_nr if the timeout specified by timeout has elapsed |
| 2076 | * before sufficient events are available, where timeout == NULL |
| 2077 | * specifies an infinite timeout. Note that the timeout pointed to by |
Jeff Moyer | 6900807 | 2013-05-24 15:55:24 -0700 | [diff] [blame] | 2078 | * timeout is relative. Will fail with -ENOSYS if not implemented. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2079 | */ |
Arnd Bergmann | 3ca47e9 | 2019-04-23 17:43:50 +0200 | [diff] [blame] | 2080 | #ifdef CONFIG_64BIT |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2081 | |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 2082 | SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, |
| 2083 | long, min_nr, |
| 2084 | long, nr, |
| 2085 | struct io_event __user *, events, |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2086 | struct __kernel_timespec __user *, timeout) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2087 | { |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2088 | struct timespec64 ts; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2089 | int ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2090 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2091 | if (timeout && unlikely(get_timespec64(&ts, timeout))) |
| 2092 | return -EFAULT; |
| 2093 | |
| 2094 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); |
| 2095 | if (!ret && signal_pending(current)) |
| 2096 | ret = -EINTR; |
| 2097 | return ret; |
| 2098 | } |
| 2099 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2100 | #endif |
| 2101 | |
Christoph Hellwig | 9ba546c | 2018-07-11 15:48:46 +0200 | [diff] [blame] | 2102 | struct __aio_sigset { |
| 2103 | const sigset_t __user *sigmask; |
| 2104 | size_t sigsetsize; |
| 2105 | }; |
| 2106 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2107 | SYSCALL_DEFINE6(io_pgetevents, |
| 2108 | aio_context_t, ctx_id, |
| 2109 | long, min_nr, |
| 2110 | long, nr, |
| 2111 | struct io_event __user *, events, |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2112 | struct __kernel_timespec __user *, timeout, |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2113 | const struct __aio_sigset __user *, usig) |
| 2114 | { |
| 2115 | struct __aio_sigset ksig = { NULL, }; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2116 | struct timespec64 ts; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2117 | bool interrupted; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2118 | int ret; |
| 2119 | |
| 2120 | if (timeout && unlikely(get_timespec64(&ts, timeout))) |
| 2121 | return -EFAULT; |
| 2122 | |
| 2123 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2124 | return -EFAULT; |
| 2125 | |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2126 | ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2127 | if (ret) |
| 2128 | return ret; |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2129 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2130 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2131 | |
| 2132 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2133 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2134 | if (interrupted && !ret) |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2135 | ret = -ERESTARTNOHAND; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2136 | |
| 2137 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2138 | } |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2139 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2140 | #if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT) |
| 2141 | |
| 2142 | SYSCALL_DEFINE6(io_pgetevents_time32, |
| 2143 | aio_context_t, ctx_id, |
| 2144 | long, min_nr, |
| 2145 | long, nr, |
| 2146 | struct io_event __user *, events, |
| 2147 | struct old_timespec32 __user *, timeout, |
| 2148 | const struct __aio_sigset __user *, usig) |
| 2149 | { |
| 2150 | struct __aio_sigset ksig = { NULL, }; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2151 | struct timespec64 ts; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2152 | bool interrupted; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2153 | int ret; |
| 2154 | |
| 2155 | if (timeout && unlikely(get_old_timespec32(&ts, timeout))) |
| 2156 | return -EFAULT; |
| 2157 | |
| 2158 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2159 | return -EFAULT; |
| 2160 | |
Deepa Dinamani | ded653c | 2018-09-19 21:41:04 -0700 | [diff] [blame] | 2161 | |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2162 | ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); |
Deepa Dinamani | ded653c | 2018-09-19 21:41:04 -0700 | [diff] [blame] | 2163 | if (ret) |
| 2164 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2165 | |
| 2166 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2167 | |
| 2168 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2169 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2170 | if (interrupted && !ret) |
Deepa Dinamani | 854a6ed | 2018-09-19 21:41:05 -0700 | [diff] [blame] | 2171 | ret = -ERESTARTNOHAND; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2172 | |
| 2173 | return ret; |
| 2174 | } |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2175 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2176 | #endif |
| 2177 | |
| 2178 | #if defined(CONFIG_COMPAT_32BIT_TIME) |
| 2179 | |
Arnd Bergmann | 8dabe72 | 2019-01-07 00:33:08 +0100 | [diff] [blame] | 2180 | SYSCALL_DEFINE5(io_getevents_time32, __u32, ctx_id, |
| 2181 | __s32, min_nr, |
| 2182 | __s32, nr, |
| 2183 | struct io_event __user *, events, |
| 2184 | struct old_timespec32 __user *, timeout) |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2185 | { |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2186 | struct timespec64 t; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2187 | int ret; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2188 | |
Arnd Bergmann | 9afc5ee | 2018-07-13 12:52:28 +0200 | [diff] [blame] | 2189 | if (timeout && get_old_timespec32(&t, timeout)) |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2190 | return -EFAULT; |
| 2191 | |
| 2192 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); |
| 2193 | if (!ret && signal_pending(current)) |
| 2194 | ret = -EINTR; |
| 2195 | return ret; |
| 2196 | } |
| 2197 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2198 | #endif |
| 2199 | |
| 2200 | #ifdef CONFIG_COMPAT |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2201 | |
| 2202 | struct __compat_aio_sigset { |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2203 | compat_uptr_t sigmask; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2204 | compat_size_t sigsetsize; |
| 2205 | }; |
| 2206 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2207 | #if defined(CONFIG_COMPAT_32BIT_TIME) |
| 2208 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2209 | COMPAT_SYSCALL_DEFINE6(io_pgetevents, |
| 2210 | compat_aio_context_t, ctx_id, |
| 2211 | compat_long_t, min_nr, |
| 2212 | compat_long_t, nr, |
| 2213 | struct io_event __user *, events, |
Arnd Bergmann | 9afc5ee | 2018-07-13 12:52:28 +0200 | [diff] [blame] | 2214 | struct old_timespec32 __user *, timeout, |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2215 | const struct __compat_aio_sigset __user *, usig) |
| 2216 | { |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2217 | struct __compat_aio_sigset ksig = { 0, }; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2218 | struct timespec64 t; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2219 | bool interrupted; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2220 | int ret; |
| 2221 | |
Arnd Bergmann | 9afc5ee | 2018-07-13 12:52:28 +0200 | [diff] [blame] | 2222 | if (timeout && get_old_timespec32(&t, timeout)) |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2223 | return -EFAULT; |
| 2224 | |
| 2225 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2226 | return -EFAULT; |
| 2227 | |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2228 | ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize); |
Deepa Dinamani | ded653c | 2018-09-19 21:41:04 -0700 | [diff] [blame] | 2229 | if (ret) |
| 2230 | return ret; |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2231 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2232 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2233 | |
| 2234 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2235 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2236 | if (interrupted && !ret) |
Deepa Dinamani | 854a6ed | 2018-09-19 21:41:05 -0700 | [diff] [blame] | 2237 | ret = -ERESTARTNOHAND; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2238 | |
| 2239 | return ret; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2240 | } |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2241 | |
| 2242 | #endif |
| 2243 | |
| 2244 | COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64, |
| 2245 | compat_aio_context_t, ctx_id, |
| 2246 | compat_long_t, min_nr, |
| 2247 | compat_long_t, nr, |
| 2248 | struct io_event __user *, events, |
| 2249 | struct __kernel_timespec __user *, timeout, |
| 2250 | const struct __compat_aio_sigset __user *, usig) |
| 2251 | { |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2252 | struct __compat_aio_sigset ksig = { 0, }; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2253 | struct timespec64 t; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2254 | bool interrupted; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2255 | int ret; |
| 2256 | |
| 2257 | if (timeout && get_timespec64(&t, timeout)) |
| 2258 | return -EFAULT; |
| 2259 | |
| 2260 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2261 | return -EFAULT; |
| 2262 | |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2263 | ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize); |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2264 | if (ret) |
| 2265 | return ret; |
| 2266 | |
| 2267 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2268 | |
| 2269 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2270 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2271 | if (interrupted && !ret) |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2272 | ret = -ERESTARTNOHAND; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2273 | |
| 2274 | return ret; |
| 2275 | } |
| 2276 | #endif |