Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * An async IO implementation for Linux |
| 3 | * Written by Benjamin LaHaise <bcrl@kvack.org> |
| 4 | * |
| 5 | * Implements an efficient asynchronous io interface. |
| 6 | * |
| 7 | * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved. |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 8 | * Copyright 2018 Christoph Hellwig. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | * |
| 10 | * See ../COPYING for licensing terms. |
| 11 | */ |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 12 | #define pr_fmt(fmt) "%s: " fmt, __func__ |
| 13 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <linux/kernel.h> |
| 15 | #include <linux/init.h> |
| 16 | #include <linux/errno.h> |
| 17 | #include <linux/time.h> |
| 18 | #include <linux/aio_abi.h> |
Paul Gortmaker | 630d9c4 | 2011-11-16 23:57:37 -0500 | [diff] [blame] | 19 | #include <linux/export.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <linux/syscalls.h> |
Jens Axboe | b9d128f | 2009-10-29 13:59:26 +0100 | [diff] [blame] | 21 | #include <linux/backing-dev.h> |
Christoph Hellwig | 9018ccc | 2018-07-24 11:36:37 +0200 | [diff] [blame] | 22 | #include <linux/refcount.h> |
Badari Pulavarty | 027445c | 2006-09-30 23:28:46 -0700 | [diff] [blame] | 23 | #include <linux/uio.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | |
Ingo Molnar | 174cd4b | 2017-02-02 19:15:33 +0100 | [diff] [blame] | 25 | #include <linux/sched/signal.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #include <linux/fs.h> |
| 27 | #include <linux/file.h> |
| 28 | #include <linux/mm.h> |
| 29 | #include <linux/mman.h> |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 30 | #include <linux/percpu.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | #include <linux/slab.h> |
| 32 | #include <linux/timer.h> |
| 33 | #include <linux/aio.h> |
| 34 | #include <linux/highmem.h> |
| 35 | #include <linux/workqueue.h> |
| 36 | #include <linux/security.h> |
Davide Libenzi | 9c3060b | 2007-05-10 22:23:21 -0700 | [diff] [blame] | 37 | #include <linux/eventfd.h> |
Jeff Moyer | cfb1e33 | 2009-10-02 18:57:36 -0400 | [diff] [blame] | 38 | #include <linux/blkdev.h> |
Jeff Moyer | 9d85cba7 | 2010-05-26 14:44:26 -0700 | [diff] [blame] | 39 | #include <linux/compat.h> |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 40 | #include <linux/migrate.h> |
| 41 | #include <linux/ramfs.h> |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 42 | #include <linux/percpu-refcount.h> |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 43 | #include <linux/mount.h> |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 44 | #include <linux/pseudo_fs.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | |
Linus Torvalds | 7c0f6ba | 2016-12-24 11:46:01 -0800 | [diff] [blame] | 46 | #include <linux/uaccess.h> |
Jeff Moyer | a538e3f | 2018-12-11 12:37:49 -0500 | [diff] [blame] | 47 | #include <linux/nospec.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | |
Al Viro | 68d70d0 | 2013-06-19 15:26:04 +0400 | [diff] [blame] | 49 | #include "internal.h" |
| 50 | |
Christoph Hellwig | f3a2752 | 2018-03-30 11:19:25 +0200 | [diff] [blame] | 51 | #define KIOCB_KEY 0 |
| 52 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 53 | #define AIO_RING_MAGIC 0xa10a10a1 |
| 54 | #define AIO_RING_COMPAT_FEATURES 1 |
| 55 | #define AIO_RING_INCOMPAT_FEATURES 0 |
| 56 | struct aio_ring { |
| 57 | unsigned id; /* kernel internal index number */ |
| 58 | unsigned nr; /* number of io_events */ |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 59 | unsigned head; /* Written to by userland or under ring_lock |
| 60 | * mutex by aio_read_events_ring(). */ |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 61 | unsigned tail; |
| 62 | |
| 63 | unsigned magic; |
| 64 | unsigned compat_features; |
| 65 | unsigned incompat_features; |
| 66 | unsigned header_length; /* size of aio_ring */ |
| 67 | |
| 68 | |
Gustavo A. R. Silva | 241cb28 | 2020-05-28 09:35:11 -0500 | [diff] [blame] | 69 | struct io_event io_events[]; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 70 | }; /* 128 bytes + ring size */ |
| 71 | |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 72 | /* |
| 73 | * Plugging is meant to work with larger batches of IOs. If we don't |
| 74 | * have more than the below, then don't bother setting up a plug. |
| 75 | */ |
| 76 | #define AIO_PLUG_THRESHOLD 2 |
| 77 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 78 | #define AIO_RING_PAGES 8 |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 79 | |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 80 | struct kioctx_table { |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 81 | struct rcu_head rcu; |
| 82 | unsigned nr; |
Kees Cook | db7fcc8 | 2023-09-15 13:14:14 -0700 | [diff] [blame] | 83 | struct kioctx __rcu *table[] __counted_by(nr); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 84 | }; |
| 85 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 86 | struct kioctx_cpu { |
| 87 | unsigned reqs_available; |
| 88 | }; |
| 89 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 90 | struct ctx_rq_wait { |
| 91 | struct completion comp; |
| 92 | atomic_t count; |
| 93 | }; |
| 94 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 95 | struct kioctx { |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 96 | struct percpu_ref users; |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 97 | atomic_t dead; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 98 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 99 | struct percpu_ref reqs; |
| 100 | |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 101 | unsigned long user_id; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 102 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 103 | struct __percpu kioctx_cpu *cpu; |
| 104 | |
| 105 | /* |
| 106 | * For percpu reqs_available, number of slots we move to/from global |
| 107 | * counter at a time: |
| 108 | */ |
| 109 | unsigned req_batch; |
Kent Overstreet | 3e845ce | 2013-05-07 16:18:51 -0700 | [diff] [blame] | 110 | /* |
| 111 | * This is what userspace passed to io_setup(), it's not used for |
| 112 | * anything but counting against the global max_reqs quota. |
| 113 | * |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 114 | * The real limit is nr_events - 1, which will be larger (see |
Kent Overstreet | 3e845ce | 2013-05-07 16:18:51 -0700 | [diff] [blame] | 115 | * aio_setup_ring()) |
| 116 | */ |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 117 | unsigned max_reqs; |
| 118 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 119 | /* Size of ringbuffer, in units of struct io_event */ |
| 120 | unsigned nr_events; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 121 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 122 | unsigned long mmap_base; |
| 123 | unsigned long mmap_size; |
| 124 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 125 | struct folio **ring_folios; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 126 | long nr_pages; |
| 127 | |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 128 | struct rcu_work free_rwork; /* see free_ioctx() */ |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 129 | |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 130 | /* |
| 131 | * signals when all in-flight requests are done |
| 132 | */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 133 | struct ctx_rq_wait *rq_wait; |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 134 | |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 135 | struct { |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 136 | /* |
| 137 | * This counts the number of available slots in the ringbuffer, |
| 138 | * so we avoid overflowing it: it's decremented (if positive) |
| 139 | * when allocating a kiocb and incremented when the resulting |
| 140 | * io_event is pulled off the ringbuffer. |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 141 | * |
| 142 | * We batch accesses to it with a percpu version. |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 143 | */ |
| 144 | atomic_t reqs_available; |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 145 | } ____cacheline_aligned_in_smp; |
| 146 | |
| 147 | struct { |
| 148 | spinlock_t ctx_lock; |
| 149 | struct list_head active_reqs; /* used for cancellation */ |
| 150 | } ____cacheline_aligned_in_smp; |
| 151 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 152 | struct { |
| 153 | struct mutex ring_lock; |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 154 | wait_queue_head_t wait; |
| 155 | } ____cacheline_aligned_in_smp; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 156 | |
| 157 | struct { |
| 158 | unsigned tail; |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 159 | unsigned completed_events; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 160 | spinlock_t completion_lock; |
Kent Overstreet | 4e23bca | 2013-05-07 16:18:56 -0700 | [diff] [blame] | 161 | } ____cacheline_aligned_in_smp; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 162 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 163 | struct folio *internal_folios[AIO_RING_PAGES]; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 164 | struct file *aio_ring_file; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 165 | |
| 166 | unsigned id; |
Kent Overstreet | 4e179bc | 2013-05-07 16:18:33 -0700 | [diff] [blame] | 167 | }; |
| 168 | |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 169 | /* |
| 170 | * First field must be the file pointer in all the |
| 171 | * iocb unions! See also 'struct kiocb' in <linux/fs.h> |
| 172 | */ |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 173 | struct fsync_iocb { |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 174 | struct file *file; |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 175 | struct work_struct work; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 176 | bool datasync; |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 177 | struct cred *creds; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 178 | }; |
| 179 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 180 | struct poll_iocb { |
| 181 | struct file *file; |
| 182 | struct wait_queue_head *head; |
| 183 | __poll_t events; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 184 | bool cancelled; |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 185 | bool work_scheduled; |
| 186 | bool work_need_resched; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 187 | struct wait_queue_entry wait; |
| 188 | struct work_struct work; |
| 189 | }; |
| 190 | |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 191 | /* |
| 192 | * NOTE! Each of the iocb union members has the file pointer |
| 193 | * as the first entry in their struct definition. So you can |
| 194 | * access the file pointer through any of the sub-structs, |
| 195 | * or directly as just 'ki_filp' in this struct. |
| 196 | */ |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 197 | struct aio_kiocb { |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 198 | union { |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 199 | struct file *ki_filp; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 200 | struct kiocb rw; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 201 | struct fsync_iocb fsync; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 202 | struct poll_iocb poll; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 203 | }; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 204 | |
| 205 | struct kioctx *ki_ctx; |
| 206 | kiocb_cancel_fn *ki_cancel; |
| 207 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 208 | struct io_event ki_res; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 209 | |
| 210 | struct list_head ki_list; /* the aio core uses this |
| 211 | * for cancellation */ |
Christoph Hellwig | 9018ccc | 2018-07-24 11:36:37 +0200 | [diff] [blame] | 212 | refcount_t ki_refcnt; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 213 | |
| 214 | /* |
| 215 | * If the aio_resfd field of the userspace iocb is not zero, |
| 216 | * this is the underlying eventfd context to deliver events to. |
| 217 | */ |
| 218 | struct eventfd_ctx *ki_eventfd; |
| 219 | }; |
| 220 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 221 | /*------ sysctl variables----*/ |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 222 | static DEFINE_SPINLOCK(aio_nr_lock); |
Xiaoming Ni | 86b12b6 | 2022-01-21 22:11:24 -0800 | [diff] [blame] | 223 | static unsigned long aio_nr; /* current system wide number of aio requests */ |
| 224 | static unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | /*----end sysctl variables---*/ |
Xiaoming Ni | 86b12b6 | 2022-01-21 22:11:24 -0800 | [diff] [blame] | 226 | #ifdef CONFIG_SYSCTL |
| 227 | static struct ctl_table aio_sysctls[] = { |
| 228 | { |
| 229 | .procname = "aio-nr", |
| 230 | .data = &aio_nr, |
| 231 | .maxlen = sizeof(aio_nr), |
| 232 | .mode = 0444, |
| 233 | .proc_handler = proc_doulongvec_minmax, |
| 234 | }, |
| 235 | { |
| 236 | .procname = "aio-max-nr", |
| 237 | .data = &aio_max_nr, |
| 238 | .maxlen = sizeof(aio_max_nr), |
| 239 | .mode = 0644, |
| 240 | .proc_handler = proc_doulongvec_minmax, |
| 241 | }, |
Xiaoming Ni | 86b12b6 | 2022-01-21 22:11:24 -0800 | [diff] [blame] | 242 | }; |
| 243 | |
| 244 | static void __init aio_sysctl_init(void) |
| 245 | { |
| 246 | register_sysctl_init("fs", aio_sysctls); |
| 247 | } |
| 248 | #else |
| 249 | #define aio_sysctl_init() do { } while (0) |
| 250 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 251 | |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 252 | static struct kmem_cache *kiocb_cachep; |
| 253 | static struct kmem_cache *kioctx_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 254 | |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 255 | static struct vfsmount *aio_mnt; |
| 256 | |
| 257 | static const struct file_operations aio_ring_fops; |
| 258 | static const struct address_space_operations aio_ctx_aops; |
| 259 | |
| 260 | static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages) |
| 261 | { |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 262 | struct file *file; |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 263 | struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb); |
Dan Carpenter | 7f62656 | 2013-11-13 10:49:40 +0300 | [diff] [blame] | 264 | if (IS_ERR(inode)) |
| 265 | return ERR_CAST(inode); |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 266 | |
| 267 | inode->i_mapping->a_ops = &aio_ctx_aops; |
Matthew Wilcox (Oracle) | 600f111 | 2023-11-17 21:58:23 +0000 | [diff] [blame] | 268 | inode->i_mapping->i_private_data = ctx; |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 269 | inode->i_size = PAGE_SIZE * nr_pages; |
| 270 | |
Al Viro | d93aa9d | 2018-06-09 09:40:05 -0400 | [diff] [blame] | 271 | file = alloc_file_pseudo(inode, aio_mnt, "[aio]", |
| 272 | O_RDWR, &aio_ring_fops); |
Al Viro | c9c554f | 2018-07-11 14:19:04 -0400 | [diff] [blame] | 273 | if (IS_ERR(file)) |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 274 | iput(inode); |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 275 | return file; |
| 276 | } |
| 277 | |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 278 | static int aio_init_fs_context(struct fs_context *fc) |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 279 | { |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 280 | if (!init_pseudo(fc, AIO_RING_MAGIC)) |
| 281 | return -ENOMEM; |
| 282 | fc->s_iflags |= SB_I_NOEXEC; |
| 283 | return 0; |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 284 | } |
| 285 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 286 | /* aio_setup |
| 287 | * Creates the slab caches used by the aio routines, panic on |
| 288 | * failure as this is done early during the boot sequence. |
| 289 | */ |
| 290 | static int __init aio_setup(void) |
| 291 | { |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 292 | static struct file_system_type aio_fs = { |
| 293 | .name = "aio", |
David Howells | 52db59d | 2019-03-25 16:38:23 +0000 | [diff] [blame] | 294 | .init_fs_context = aio_init_fs_context, |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 295 | .kill_sb = kill_anon_super, |
| 296 | }; |
| 297 | aio_mnt = kern_mount(&aio_fs); |
| 298 | if (IS_ERR(aio_mnt)) |
| 299 | panic("Failed to create aio fs mount."); |
| 300 | |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 301 | kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); |
Christoph Lameter | 0a31bd5 | 2007-05-06 14:49:57 -0700 | [diff] [blame] | 302 | kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); |
Xiaoming Ni | 86b12b6 | 2022-01-21 22:11:24 -0800 | [diff] [blame] | 303 | aio_sysctl_init(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 304 | return 0; |
| 305 | } |
H Hartley Sweeten | 385773e | 2009-09-22 16:43:53 -0700 | [diff] [blame] | 306 | __initcall(aio_setup); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 307 | |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 308 | static void put_aio_ring_file(struct kioctx *ctx) |
| 309 | { |
| 310 | struct file *aio_ring_file = ctx->aio_ring_file; |
Rasmus Villemoes | de04e76 | 2016-09-15 00:25:03 +0200 | [diff] [blame] | 311 | struct address_space *i_mapping; |
| 312 | |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 313 | if (aio_ring_file) { |
Al Viro | 4506309 | 2016-12-04 18:24:56 -0500 | [diff] [blame] | 314 | truncate_setsize(file_inode(aio_ring_file), 0); |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 315 | |
| 316 | /* Prevent further access to the kioctx from migratepages */ |
Al Viro | 4506309 | 2016-12-04 18:24:56 -0500 | [diff] [blame] | 317 | i_mapping = aio_ring_file->f_mapping; |
Matthew Wilcox (Oracle) | 600f111 | 2023-11-17 21:58:23 +0000 | [diff] [blame] | 318 | spin_lock(&i_mapping->i_private_lock); |
| 319 | i_mapping->i_private_data = NULL; |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 320 | ctx->aio_ring_file = NULL; |
Matthew Wilcox (Oracle) | 600f111 | 2023-11-17 21:58:23 +0000 | [diff] [blame] | 321 | spin_unlock(&i_mapping->i_private_lock); |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 322 | |
| 323 | fput(aio_ring_file); |
| 324 | } |
| 325 | } |
| 326 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 327 | static void aio_free_ring(struct kioctx *ctx) |
| 328 | { |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 329 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 330 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 331 | /* Disconnect the kiotx from the ring file. This prevents future |
| 332 | * accesses to the kioctx from page migration. |
| 333 | */ |
| 334 | put_aio_ring_file(ctx); |
| 335 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 336 | for (i = 0; i < ctx->nr_pages; i++) { |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 337 | struct folio *folio = ctx->ring_folios[i]; |
Kefeng Wang | 5ea0a35 | 2024-03-21 21:16:39 +0800 | [diff] [blame] | 338 | |
| 339 | if (!folio) |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 340 | continue; |
Kefeng Wang | 5ea0a35 | 2024-03-21 21:16:39 +0800 | [diff] [blame] | 341 | |
| 342 | pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, |
| 343 | folio_ref_count(folio)); |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 344 | ctx->ring_folios[i] = NULL; |
Kefeng Wang | 5ea0a35 | 2024-03-21 21:16:39 +0800 | [diff] [blame] | 345 | folio_put(folio); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 346 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 347 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 348 | if (ctx->ring_folios && ctx->ring_folios != ctx->internal_folios) { |
| 349 | kfree(ctx->ring_folios); |
| 350 | ctx->ring_folios = NULL; |
Sasha Levin | ddb8c45 | 2013-11-19 17:33:03 -0500 | [diff] [blame] | 351 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 352 | } |
| 353 | |
Brian Geffon | 14d0711 | 2021-04-29 22:57:48 -0700 | [diff] [blame] | 354 | static int aio_ring_mremap(struct vm_area_struct *vma) |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 355 | { |
Oleg Nesterov | 5477e70 | 2015-09-04 15:48:04 -0700 | [diff] [blame] | 356 | struct file *file = vma->vm_file; |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 357 | struct mm_struct *mm = vma->vm_mm; |
| 358 | struct kioctx_table *table; |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 359 | int i, res = -EINVAL; |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 360 | |
| 361 | spin_lock(&mm->ioctx_lock); |
| 362 | rcu_read_lock(); |
| 363 | table = rcu_dereference(mm->ioctx_table); |
Seth Jenkins | 81e9d6f | 2023-01-31 12:25:55 -0500 | [diff] [blame] | 364 | if (!table) |
| 365 | goto out_unlock; |
| 366 | |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 367 | for (i = 0; i < table->nr; i++) { |
| 368 | struct kioctx *ctx; |
| 369 | |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 370 | ctx = rcu_dereference(table->table[i]); |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 371 | if (ctx && ctx->aio_ring_file == file) { |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 372 | if (!atomic_read(&ctx->dead)) { |
| 373 | ctx->user_id = ctx->mmap_base = vma->vm_start; |
| 374 | res = 0; |
| 375 | } |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 376 | break; |
| 377 | } |
| 378 | } |
| 379 | |
Seth Jenkins | 81e9d6f | 2023-01-31 12:25:55 -0500 | [diff] [blame] | 380 | out_unlock: |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 381 | rcu_read_unlock(); |
| 382 | spin_unlock(&mm->ioctx_lock); |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 383 | return res; |
Pavel Emelyanov | e4a0d3e | 2014-09-18 19:56:17 +0400 | [diff] [blame] | 384 | } |
| 385 | |
Oleg Nesterov | 5477e70 | 2015-09-04 15:48:04 -0700 | [diff] [blame] | 386 | static const struct vm_operations_struct aio_ring_vm_ops = { |
| 387 | .mremap = aio_ring_mremap, |
| 388 | #if IS_ENABLED(CONFIG_MMU) |
| 389 | .fault = filemap_fault, |
| 390 | .map_pages = filemap_map_pages, |
| 391 | .page_mkwrite = filemap_page_mkwrite, |
| 392 | #endif |
| 393 | }; |
| 394 | |
| 395 | static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma) |
| 396 | { |
Suren Baghdasaryan | 1c71222 | 2023-01-26 11:37:49 -0800 | [diff] [blame] | 397 | vm_flags_set(vma, VM_DONTEXPAND); |
Oleg Nesterov | 5477e70 | 2015-09-04 15:48:04 -0700 | [diff] [blame] | 398 | vma->vm_ops = &aio_ring_vm_ops; |
| 399 | return 0; |
| 400 | } |
| 401 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 402 | static const struct file_operations aio_ring_fops = { |
| 403 | .mmap = aio_ring_mmap, |
| 404 | }; |
| 405 | |
Benjamin LaHaise | 0c45355 | 2013-07-17 09:34:24 -0400 | [diff] [blame] | 406 | #if IS_ENABLED(CONFIG_MIGRATION) |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 407 | static int aio_migrate_folio(struct address_space *mapping, struct folio *dst, |
| 408 | struct folio *src, enum migrate_mode mode) |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 409 | { |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 410 | struct kioctx *ctx; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 411 | unsigned long flags; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 412 | pgoff_t idx; |
Kefeng Wang | 9066328 | 2024-05-24 13:28:43 +0800 | [diff] [blame] | 413 | int rc = 0; |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 414 | |
Matthew Wilcox (Oracle) | 600f111 | 2023-11-17 21:58:23 +0000 | [diff] [blame] | 415 | /* mapping->i_private_lock here protects against the kioctx teardown. */ |
| 416 | spin_lock(&mapping->i_private_lock); |
| 417 | ctx = mapping->i_private_data; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 418 | if (!ctx) { |
| 419 | rc = -EINVAL; |
| 420 | goto out; |
| 421 | } |
| 422 | |
| 423 | /* The ring_lock mutex. The prevents aio_read_events() from writing |
| 424 | * to the ring's head, and prevents page migration from mucking in |
| 425 | * a partially initialized kiotx. |
| 426 | */ |
| 427 | if (!mutex_trylock(&ctx->ring_lock)) { |
| 428 | rc = -EAGAIN; |
| 429 | goto out; |
| 430 | } |
| 431 | |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 432 | idx = src->index; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 433 | if (idx < (pgoff_t)ctx->nr_pages) { |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 434 | /* Make sure the old folio hasn't already been changed */ |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 435 | if (ctx->ring_folios[idx] != src) |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 436 | rc = -EAGAIN; |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 437 | } else |
| 438 | rc = -EINVAL; |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 439 | |
| 440 | if (rc != 0) |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 441 | goto out_unlock; |
Benjamin LaHaise | 8e321fe | 2013-12-21 17:56:08 -0500 | [diff] [blame] | 442 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 443 | /* Writeback must be complete */ |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 444 | BUG_ON(folio_test_writeback(src)); |
| 445 | folio_get(dst); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 446 | |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 447 | rc = folio_migrate_mapping(mapping, dst, src, 1); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 448 | if (rc != MIGRATEPAGE_SUCCESS) { |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 449 | folio_put(dst); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 450 | goto out_unlock; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 451 | } |
| 452 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 453 | /* Take completion_lock to prevent other writes to the ring buffer |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 454 | * while the old folio is copied to the new. This prevents new |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 455 | * events from being lost. |
Benjamin LaHaise | 5e9ae2e | 2013-09-26 20:34:51 -0400 | [diff] [blame] | 456 | */ |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 457 | spin_lock_irqsave(&ctx->completion_lock, flags); |
Kefeng Wang | 3f59493 | 2024-06-26 16:53:28 +0800 | [diff] [blame] | 458 | folio_copy(dst, src); |
| 459 | folio_migrate_flags(dst, src); |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 460 | BUG_ON(ctx->ring_folios[idx] != src); |
| 461 | ctx->ring_folios[idx] = dst; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 462 | spin_unlock_irqrestore(&ctx->completion_lock, flags); |
| 463 | |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 464 | /* The old folio is no longer accessible. */ |
| 465 | folio_put(src); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 466 | |
| 467 | out_unlock: |
| 468 | mutex_unlock(&ctx->ring_lock); |
| 469 | out: |
Matthew Wilcox (Oracle) | 600f111 | 2023-11-17 21:58:23 +0000 | [diff] [blame] | 470 | spin_unlock(&mapping->i_private_lock); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 471 | return rc; |
| 472 | } |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 473 | #else |
| 474 | #define aio_migrate_folio NULL |
Benjamin LaHaise | 0c45355 | 2013-07-17 09:34:24 -0400 | [diff] [blame] | 475 | #endif |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 476 | |
| 477 | static const struct address_space_operations aio_ctx_aops = { |
Matthew Wilcox (Oracle) | 46de8b97 | 2022-02-09 20:22:13 +0000 | [diff] [blame] | 478 | .dirty_folio = noop_dirty_folio, |
Matthew Wilcox (Oracle) | 3648951 | 2022-06-06 10:47:21 -0400 | [diff] [blame] | 479 | .migrate_folio = aio_migrate_folio, |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 480 | }; |
| 481 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 482 | static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 483 | { |
| 484 | struct aio_ring *ring; |
Zach Brown | 41003a7 | 2013-05-07 16:18:25 -0700 | [diff] [blame] | 485 | struct mm_struct *mm = current->mm; |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 486 | unsigned long size, unused; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 487 | int nr_pages; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 488 | int i; |
| 489 | struct file *file; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 490 | |
| 491 | /* Compensate for the ring buffer's head/tail overlap entry */ |
| 492 | nr_events += 2; /* 1 is required, 2 for good luck */ |
| 493 | |
| 494 | size = sizeof(struct aio_ring); |
| 495 | size += sizeof(struct io_event) * nr_events; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 496 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 497 | nr_pages = PFN_UP(size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 498 | if (nr_pages < 0) |
| 499 | return -EINVAL; |
| 500 | |
Benjamin LaHaise | 71ad749 | 2013-09-17 10:18:25 -0400 | [diff] [blame] | 501 | file = aio_private_file(ctx, nr_pages); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 502 | if (IS_ERR(file)) { |
| 503 | ctx->aio_ring_file = NULL; |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 504 | return -ENOMEM; |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 505 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 506 | |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 507 | ctx->aio_ring_file = file; |
| 508 | nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) |
| 509 | / sizeof(struct io_event); |
| 510 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 511 | ctx->ring_folios = ctx->internal_folios; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 512 | if (nr_pages > AIO_RING_PAGES) { |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 513 | ctx->ring_folios = kcalloc(nr_pages, sizeof(struct folio *), |
| 514 | GFP_KERNEL); |
| 515 | if (!ctx->ring_folios) { |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 516 | put_aio_ring_file(ctx); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 517 | return -ENOMEM; |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 518 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 519 | } |
| 520 | |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 521 | for (i = 0; i < nr_pages; i++) { |
Kefeng Wang | 75a07b5 | 2024-03-21 21:16:38 +0800 | [diff] [blame] | 522 | struct folio *folio; |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 523 | |
Kefeng Wang | 75a07b5 | 2024-03-21 21:16:38 +0800 | [diff] [blame] | 524 | folio = __filemap_get_folio(file->f_mapping, i, |
| 525 | FGP_LOCK | FGP_ACCESSED | FGP_CREAT, |
| 526 | GFP_USER | __GFP_ZERO); |
| 527 | if (IS_ERR(folio)) |
| 528 | break; |
| 529 | |
| 530 | pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, |
| 531 | folio_ref_count(folio)); |
| 532 | folio_end_read(folio, true); |
| 533 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 534 | ctx->ring_folios[i] = folio; |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 535 | } |
| 536 | ctx->nr_pages = i; |
| 537 | |
| 538 | if (unlikely(i != nr_pages)) { |
| 539 | aio_free_ring(ctx); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 540 | return -ENOMEM; |
Linus Torvalds | 3dc9acb | 2013-12-20 05:11:12 +0900 | [diff] [blame] | 541 | } |
| 542 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 543 | ctx->mmap_size = nr_pages * PAGE_SIZE; |
| 544 | pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size); |
Gu Zheng | 36bc08c | 2013-07-16 17:56:16 +0800 | [diff] [blame] | 545 | |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 546 | if (mmap_write_lock_killable(mm)) { |
Michal Hocko | 013373e | 2016-05-23 16:25:59 -0700 | [diff] [blame] | 547 | ctx->mmap_size = 0; |
| 548 | aio_free_ring(ctx); |
| 549 | return -EINTR; |
| 550 | } |
| 551 | |
Peter Collingbourne | 45e5530 | 2020-08-06 23:23:37 -0700 | [diff] [blame] | 552 | ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size, |
| 553 | PROT_READ | PROT_WRITE, |
Yu-cheng Yu | 592b5fa | 2023-06-12 17:10:30 -0700 | [diff] [blame] | 554 | MAP_SHARED, 0, 0, &unused, NULL); |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 555 | mmap_write_unlock(mm); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 556 | if (IS_ERR((void *)ctx->mmap_base)) { |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 557 | ctx->mmap_size = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 558 | aio_free_ring(ctx); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 559 | return -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 560 | } |
| 561 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 562 | pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base); |
Benjamin LaHaise | d6c355c | 2013-09-09 11:57:59 -0400 | [diff] [blame] | 563 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 564 | ctx->user_id = ctx->mmap_base; |
| 565 | ctx->nr_events = nr_events; /* trusted copy */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 566 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 567 | ring = folio_address(ctx->ring_folios[0]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 568 | ring->nr = nr_events; /* user copy */ |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 569 | ring->id = ~0U; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 570 | ring->head = ring->tail = 0; |
| 571 | ring->magic = AIO_RING_MAGIC; |
| 572 | ring->compat_features = AIO_RING_COMPAT_FEATURES; |
| 573 | ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; |
| 574 | ring->header_length = sizeof(struct aio_ring); |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 575 | flush_dcache_folio(ctx->ring_folios[0]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 576 | |
| 577 | return 0; |
| 578 | } |
| 579 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 580 | #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event)) |
| 581 | #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event)) |
| 582 | #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE) |
| 583 | |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 584 | void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel) |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 585 | { |
Bart Van Assche | 961ebd1 | 2024-03-04 15:57:15 -0800 | [diff] [blame] | 586 | struct aio_kiocb *req; |
| 587 | struct kioctx *ctx; |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 588 | unsigned long flags; |
| 589 | |
Bart Van Assche | b820de7 | 2024-02-15 12:47:38 -0800 | [diff] [blame] | 590 | /* |
| 591 | * kiocb didn't come from aio or is neither a read nor a write, hence |
| 592 | * ignore it. |
| 593 | */ |
| 594 | if (!(iocb->ki_flags & IOCB_AIO_RW)) |
| 595 | return; |
| 596 | |
Bart Van Assche | 961ebd1 | 2024-03-04 15:57:15 -0800 | [diff] [blame] | 597 | req = container_of(iocb, struct aio_kiocb, rw); |
| 598 | |
Christoph Hellwig | 75321b5 | 2018-04-09 14:57:56 +0200 | [diff] [blame] | 599 | if (WARN_ON_ONCE(!list_empty(&req->ki_list))) |
| 600 | return; |
| 601 | |
Bart Van Assche | 961ebd1 | 2024-03-04 15:57:15 -0800 | [diff] [blame] | 602 | ctx = req->ki_ctx; |
| 603 | |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 604 | spin_lock_irqsave(&ctx->ctx_lock, flags); |
Christoph Hellwig | 75321b5 | 2018-04-09 14:57:56 +0200 | [diff] [blame] | 605 | list_add_tail(&req->ki_list, &ctx->active_reqs); |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 606 | req->ki_cancel = cancel; |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 607 | spin_unlock_irqrestore(&ctx->ctx_lock, flags); |
| 608 | } |
| 609 | EXPORT_SYMBOL(kiocb_set_cancel_fn); |
| 610 | |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 611 | /* |
| 612 | * free_ioctx() should be RCU delayed to synchronize against the RCU |
| 613 | * protected lookup_ioctx() and also needs process context to call |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 614 | * aio_free_ring(). Use rcu_work. |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 615 | */ |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 616 | static void free_ioctx(struct work_struct *work) |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 617 | { |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 618 | struct kioctx *ctx = container_of(to_rcu_work(work), struct kioctx, |
| 619 | free_rwork); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 620 | pr_debug("freeing %p\n", ctx); |
| 621 | |
| 622 | aio_free_ring(ctx); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 623 | free_percpu(ctx->cpu); |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 624 | percpu_ref_exit(&ctx->reqs); |
| 625 | percpu_ref_exit(&ctx->users); |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 626 | kmem_cache_free(kioctx_cachep, ctx); |
| 627 | } |
| 628 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 629 | static void free_ioctx_reqs(struct percpu_ref *ref) |
| 630 | { |
| 631 | struct kioctx *ctx = container_of(ref, struct kioctx, reqs); |
| 632 | |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 633 | /* At this point we know that there are no any in-flight requests */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 634 | if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) |
| 635 | complete(&ctx->rq_wait->comp); |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 636 | |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 637 | /* Synchronize against RCU protected table->table[] dereferences */ |
Tejun Heo | f729863 | 2018-03-14 12:45:15 -0700 | [diff] [blame] | 638 | INIT_RCU_WORK(&ctx->free_rwork, free_ioctx); |
| 639 | queue_rcu_work(system_wq, &ctx->free_rwork); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 640 | } |
| 641 | |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 642 | /* |
| 643 | * When this function runs, the kioctx has been removed from the "hash table" |
| 644 | * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - |
| 645 | * now it's safe to cancel any that need to be. |
| 646 | */ |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 647 | static void free_ioctx_users(struct percpu_ref *ref) |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 648 | { |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 649 | struct kioctx *ctx = container_of(ref, struct kioctx, users); |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 650 | struct aio_kiocb *req; |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 651 | |
| 652 | spin_lock_irq(&ctx->ctx_lock); |
| 653 | |
| 654 | while (!list_empty(&ctx->active_reqs)) { |
| 655 | req = list_first_entry(&ctx->active_reqs, |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 656 | struct aio_kiocb, ki_list); |
Christoph Hellwig | 888933f | 2018-05-23 14:11:02 +0200 | [diff] [blame] | 657 | req->ki_cancel(&req->rw); |
Al Viro | 4faa9996 | 2018-05-23 22:53:22 -0400 | [diff] [blame] | 658 | list_del_init(&req->ki_list); |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 659 | } |
| 660 | |
| 661 | spin_unlock_irq(&ctx->ctx_lock); |
| 662 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 663 | percpu_ref_kill(&ctx->reqs); |
| 664 | percpu_ref_put(&ctx->reqs); |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 665 | } |
| 666 | |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 667 | static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) |
| 668 | { |
| 669 | unsigned i, new_nr; |
| 670 | struct kioctx_table *table, *old; |
| 671 | struct aio_ring *ring; |
| 672 | |
| 673 | spin_lock(&mm->ioctx_lock); |
Oleg Nesterov | 855ef0d | 2014-04-30 16:16:36 +0200 | [diff] [blame] | 674 | table = rcu_dereference_raw(mm->ioctx_table); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 675 | |
| 676 | while (1) { |
| 677 | if (table) |
| 678 | for (i = 0; i < table->nr; i++) |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 679 | if (!rcu_access_pointer(table->table[i])) { |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 680 | ctx->id = i; |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 681 | rcu_assign_pointer(table->table[i], ctx); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 682 | spin_unlock(&mm->ioctx_lock); |
| 683 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 684 | /* While kioctx setup is in progress, |
| 685 | * we are protected from page migration |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 686 | * changes ring_folios by ->ring_lock. |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 687 | */ |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 688 | ring = folio_address(ctx->ring_folios[0]); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 689 | ring->id = ctx->id; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 690 | return 0; |
| 691 | } |
| 692 | |
| 693 | new_nr = (table ? table->nr : 1) * 4; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 694 | spin_unlock(&mm->ioctx_lock); |
| 695 | |
Len Baker | 6446c4f | 2021-09-19 11:45:39 +0200 | [diff] [blame] | 696 | table = kzalloc(struct_size(table, table, new_nr), GFP_KERNEL); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 697 | if (!table) |
| 698 | return -ENOMEM; |
| 699 | |
| 700 | table->nr = new_nr; |
| 701 | |
| 702 | spin_lock(&mm->ioctx_lock); |
Oleg Nesterov | 855ef0d | 2014-04-30 16:16:36 +0200 | [diff] [blame] | 703 | old = rcu_dereference_raw(mm->ioctx_table); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 704 | |
| 705 | if (!old) { |
| 706 | rcu_assign_pointer(mm->ioctx_table, table); |
| 707 | } else if (table->nr > old->nr) { |
| 708 | memcpy(table->table, old->table, |
| 709 | old->nr * sizeof(struct kioctx *)); |
| 710 | |
| 711 | rcu_assign_pointer(mm->ioctx_table, table); |
| 712 | kfree_rcu(old, rcu); |
| 713 | } else { |
| 714 | kfree(table); |
| 715 | table = old; |
| 716 | } |
| 717 | } |
| 718 | } |
| 719 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 720 | static void aio_nr_sub(unsigned nr) |
| 721 | { |
| 722 | spin_lock(&aio_nr_lock); |
| 723 | if (WARN_ON(aio_nr - nr > aio_nr)) |
| 724 | aio_nr = 0; |
| 725 | else |
| 726 | aio_nr -= nr; |
| 727 | spin_unlock(&aio_nr_lock); |
| 728 | } |
| 729 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 730 | /* ioctx_alloc |
| 731 | * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed. |
| 732 | */ |
| 733 | static struct kioctx *ioctx_alloc(unsigned nr_events) |
| 734 | { |
Zach Brown | 41003a7 | 2013-05-07 16:18:25 -0700 | [diff] [blame] | 735 | struct mm_struct *mm = current->mm; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 736 | struct kioctx *ctx; |
Al Viro | e23754f | 2012-03-06 14:33:22 -0500 | [diff] [blame] | 737 | int err = -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 738 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 739 | /* |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 740 | * Store the original nr_events -- what userspace passed to io_setup(), |
| 741 | * for counting against the global limit -- before it changes. |
| 742 | */ |
| 743 | unsigned int max_reqs = nr_events; |
| 744 | |
| 745 | /* |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 746 | * We keep track of the number of available ringbuffer slots, to prevent |
| 747 | * overflow (reqs_available), and we also use percpu counters for this. |
| 748 | * |
| 749 | * So since up to half the slots might be on other cpu's percpu counters |
| 750 | * and unavailable, double nr_events so userspace sees what they |
| 751 | * expected: additionally, we move req_batch slots to/from percpu |
| 752 | * counters at a time, so make sure that isn't 0: |
| 753 | */ |
| 754 | nr_events = max(nr_events, num_possible_cpus() * 4); |
| 755 | nr_events *= 2; |
| 756 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 757 | /* Prevent overflows */ |
Al Viro | 08397ac | 2015-03-31 11:43:52 -0400 | [diff] [blame] | 758 | if (nr_events > (0x10000000U / sizeof(struct io_event))) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 759 | pr_debug("ENOMEM: nr_events too high\n"); |
| 760 | return ERR_PTR(-EINVAL); |
| 761 | } |
| 762 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 763 | if (!nr_events || (unsigned long)max_reqs > aio_max_nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 764 | return ERR_PTR(-EAGAIN); |
| 765 | |
Robert P. J. Day | c376222 | 2007-02-10 01:45:03 -0800 | [diff] [blame] | 766 | ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 767 | if (!ctx) |
| 768 | return ERR_PTR(-ENOMEM); |
| 769 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 770 | ctx->max_reqs = max_reqs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 771 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 772 | spin_lock_init(&ctx->ctx_lock); |
| 773 | spin_lock_init(&ctx->completion_lock); |
| 774 | mutex_init(&ctx->ring_lock); |
| 775 | /* Protect against page migration throughout kiotx setup by keeping |
| 776 | * the ring_lock mutex held until setup is complete. */ |
| 777 | mutex_lock(&ctx->ring_lock); |
| 778 | init_waitqueue_head(&ctx->wait); |
| 779 | |
| 780 | INIT_LIST_HEAD(&ctx->active_reqs); |
| 781 | |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 782 | if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL)) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 783 | goto err; |
| 784 | |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 785 | if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL)) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 786 | goto err; |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 787 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 788 | ctx->cpu = alloc_percpu(struct kioctx_cpu); |
| 789 | if (!ctx->cpu) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 790 | goto err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 791 | |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 792 | err = aio_setup_ring(ctx, nr_events); |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 793 | if (err < 0) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 794 | goto err; |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 795 | |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 796 | atomic_set(&ctx->reqs_available, ctx->nr_events - 1); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 797 | ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4); |
Benjamin LaHaise | 6878ea7 | 2013-07-31 10:34:18 -0400 | [diff] [blame] | 798 | if (ctx->req_batch < 1) |
| 799 | ctx->req_batch = 1; |
Kent Overstreet | 34e83fc | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 800 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 801 | /* limit the number of system wide aios */ |
Al Viro | 9fa1cb3 | 2012-03-10 23:14:05 -0500 | [diff] [blame] | 802 | spin_lock(&aio_nr_lock); |
Mauricio Faria de Oliveira | 2a8a986 | 2017-07-05 10:53:16 -0300 | [diff] [blame] | 803 | if (aio_nr + ctx->max_reqs > aio_max_nr || |
| 804 | aio_nr + ctx->max_reqs < aio_nr) { |
Al Viro | 9fa1cb3 | 2012-03-10 23:14:05 -0500 | [diff] [blame] | 805 | spin_unlock(&aio_nr_lock); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 806 | err = -EAGAIN; |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 807 | goto err_ctx; |
Al Viro | 2dd542b7 | 2012-03-10 23:10:35 -0500 | [diff] [blame] | 808 | } |
| 809 | aio_nr += ctx->max_reqs; |
Al Viro | 9fa1cb3 | 2012-03-10 23:14:05 -0500 | [diff] [blame] | 810 | spin_unlock(&aio_nr_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 811 | |
Benjamin LaHaise | 1881686 | 2013-12-21 15:49:28 -0500 | [diff] [blame] | 812 | percpu_ref_get(&ctx->users); /* io_setup() will drop this ref */ |
| 813 | percpu_ref_get(&ctx->reqs); /* free_ioctx_users() will drop this */ |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 814 | |
Benjamin LaHaise | da90382 | 2013-08-05 13:21:43 -0400 | [diff] [blame] | 815 | err = ioctx_add_table(ctx, mm); |
| 816 | if (err) |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 817 | goto err_cleanup; |
Benjamin LaHaise | da90382 | 2013-08-05 13:21:43 -0400 | [diff] [blame] | 818 | |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 819 | /* Release the ring_lock mutex now that all setup is complete. */ |
| 820 | mutex_unlock(&ctx->ring_lock); |
| 821 | |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 822 | pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n", |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 823 | ctx, ctx->user_id, mm, ctx->nr_events); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 824 | return ctx; |
| 825 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 826 | err_cleanup: |
| 827 | aio_nr_sub(ctx->max_reqs); |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 828 | err_ctx: |
Al Viro | deeb852 | 2015-04-06 17:57:44 -0400 | [diff] [blame] | 829 | atomic_set(&ctx->dead, 1); |
| 830 | if (ctx->mmap_size) |
| 831 | vm_munmap(ctx->mmap_base, ctx->mmap_size); |
Gu Zheng | d1b9432 | 2013-12-04 18:19:06 +0800 | [diff] [blame] | 832 | aio_free_ring(ctx); |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 833 | err: |
Benjamin LaHaise | fa8a53c | 2014-03-28 10:14:45 -0400 | [diff] [blame] | 834 | mutex_unlock(&ctx->ring_lock); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 835 | free_percpu(ctx->cpu); |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 836 | percpu_ref_exit(&ctx->reqs); |
| 837 | percpu_ref_exit(&ctx->users); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 838 | kmem_cache_free(kioctx_cachep, ctx); |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 839 | pr_debug("error allocating ioctx %d\n", err); |
Al Viro | e23754f | 2012-03-06 14:33:22 -0500 | [diff] [blame] | 840 | return ERR_PTR(err); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 841 | } |
| 842 | |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 843 | /* kill_ioctx |
| 844 | * Cancels all outstanding aio requests on an aio context. Used |
| 845 | * when the processes owning a context have all exited to encourage |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 846 | * the rapid destruction of the kioctx. |
| 847 | */ |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 848 | static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx, |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 849 | struct ctx_rq_wait *wait) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 850 | { |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 851 | struct kioctx_table *table; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 852 | |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 853 | spin_lock(&mm->ioctx_lock); |
Al Viro | b2edffd | 2015-04-06 17:48:54 -0400 | [diff] [blame] | 854 | if (atomic_xchg(&ctx->dead, 1)) { |
| 855 | spin_unlock(&mm->ioctx_lock); |
| 856 | return -EINVAL; |
| 857 | } |
| 858 | |
Oleg Nesterov | 855ef0d | 2014-04-30 16:16:36 +0200 | [diff] [blame] | 859 | table = rcu_dereference_raw(mm->ioctx_table); |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 860 | WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id])); |
| 861 | RCU_INIT_POINTER(table->table[ctx->id], NULL); |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 862 | spin_unlock(&mm->ioctx_lock); |
Kent Overstreet | 4fcc712 | 2013-06-12 14:04:59 -0700 | [diff] [blame] | 863 | |
Tejun Heo | a6d7cff | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 864 | /* free_ioctx_reqs() will do the necessary RCU synchronization */ |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 865 | wake_up_all(&ctx->wait); |
Kent Overstreet | 4fcc712 | 2013-06-12 14:04:59 -0700 | [diff] [blame] | 866 | |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 867 | /* |
| 868 | * It'd be more correct to do this in free_ioctx(), after all |
| 869 | * the outstanding kiocbs have finished - but by then io_destroy |
| 870 | * has already returned, so io_setup() could potentially return |
| 871 | * -EAGAIN with no ioctxs actually in use (as far as userspace |
| 872 | * could tell). |
| 873 | */ |
| 874 | aio_nr_sub(ctx->max_reqs); |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 875 | |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 876 | if (ctx->mmap_size) |
| 877 | vm_munmap(ctx->mmap_base, ctx->mmap_size); |
| 878 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 879 | ctx->rq_wait = wait; |
Benjamin LaHaise | fa88b6f | 2014-04-29 12:55:48 -0400 | [diff] [blame] | 880 | percpu_ref_kill(&ctx->users); |
| 881 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 882 | } |
| 883 | |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 884 | /* |
| 885 | * exit_aio: called when the last user of mm goes away. At this point, there is |
| 886 | * no way for any new requests to be submited or any of the io_* syscalls to be |
| 887 | * called on the context. |
| 888 | * |
| 889 | * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on |
| 890 | * them. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 891 | */ |
Harvey Harrison | fc9b52c | 2008-02-08 04:19:52 -0800 | [diff] [blame] | 892 | void exit_aio(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 893 | { |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 894 | struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table); |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 895 | struct ctx_rq_wait wait; |
| 896 | int i, skipped; |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 897 | |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 898 | if (!table) |
| 899 | return; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 900 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 901 | atomic_set(&wait.count, table->nr); |
| 902 | init_completion(&wait.comp); |
| 903 | |
| 904 | skipped = 0; |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 905 | for (i = 0; i < table->nr; ++i) { |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 906 | struct kioctx *ctx = |
| 907 | rcu_dereference_protected(table->table[i], true); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 908 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 909 | if (!ctx) { |
| 910 | skipped++; |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 911 | continue; |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 912 | } |
| 913 | |
Al Viro | 936af15 | 2012-04-20 21:49:41 -0400 | [diff] [blame] | 914 | /* |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 915 | * We don't need to bother with munmap() here - exit_mmap(mm) |
| 916 | * is coming and it'll unmap everything. And we simply can't, |
| 917 | * this is not necessarily our ->mm. |
| 918 | * Since kill_ioctx() uses non-zero ->mmap_size as indicator |
| 919 | * that it needs to unmap the area, just set it to 0. |
Al Viro | 936af15 | 2012-04-20 21:49:41 -0400 | [diff] [blame] | 920 | */ |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 921 | ctx->mmap_size = 0; |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 922 | kill_ioctx(mm, ctx, &wait); |
| 923 | } |
Kent Overstreet | 36f5588 | 2013-05-07 16:18:41 -0700 | [diff] [blame] | 924 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 925 | if (!atomic_sub_and_test(skipped, &wait.count)) { |
Gu Zheng | 6098b45 | 2014-09-03 17:45:44 +0800 | [diff] [blame] | 926 | /* Wait until all IO for the context are done. */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 927 | wait_for_completion(&wait.comp); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 928 | } |
Oleg Nesterov | 4b70ac5 | 2014-04-30 19:02:48 +0200 | [diff] [blame] | 929 | |
| 930 | RCU_INIT_POINTER(mm->ioctx_table, NULL); |
| 931 | kfree(table); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 932 | } |
| 933 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 934 | static void put_reqs_available(struct kioctx *ctx, unsigned nr) |
| 935 | { |
| 936 | struct kioctx_cpu *kcpu; |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 937 | unsigned long flags; |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 938 | |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 939 | local_irq_save(flags); |
Benjamin LaHaise | be6fb45 | 2014-07-22 09:56:56 -0400 | [diff] [blame] | 940 | kcpu = this_cpu_ptr(ctx->cpu); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 941 | kcpu->reqs_available += nr; |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 942 | |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 943 | while (kcpu->reqs_available >= ctx->req_batch * 2) { |
| 944 | kcpu->reqs_available -= ctx->req_batch; |
| 945 | atomic_add(ctx->req_batch, &ctx->reqs_available); |
| 946 | } |
| 947 | |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 948 | local_irq_restore(flags); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 949 | } |
| 950 | |
Christoph Hellwig | 432c799 | 2018-11-19 15:57:42 -0700 | [diff] [blame] | 951 | static bool __get_reqs_available(struct kioctx *ctx) |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 952 | { |
| 953 | struct kioctx_cpu *kcpu; |
| 954 | bool ret = false; |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 955 | unsigned long flags; |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 956 | |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 957 | local_irq_save(flags); |
Benjamin LaHaise | be6fb45 | 2014-07-22 09:56:56 -0400 | [diff] [blame] | 958 | kcpu = this_cpu_ptr(ctx->cpu); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 959 | if (!kcpu->reqs_available) { |
Uros Bizjak | 38ace0d | 2022-07-14 18:48:51 +0200 | [diff] [blame] | 960 | int avail = atomic_read(&ctx->reqs_available); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 961 | |
| 962 | do { |
| 963 | if (avail < ctx->req_batch) |
| 964 | goto out; |
Uros Bizjak | 38ace0d | 2022-07-14 18:48:51 +0200 | [diff] [blame] | 965 | } while (!atomic_try_cmpxchg(&ctx->reqs_available, |
| 966 | &avail, avail - ctx->req_batch)); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 967 | |
| 968 | kcpu->reqs_available += ctx->req_batch; |
| 969 | } |
| 970 | |
| 971 | ret = true; |
| 972 | kcpu->reqs_available--; |
| 973 | out: |
Benjamin LaHaise | 263782c | 2014-07-14 12:49:26 -0400 | [diff] [blame] | 974 | local_irq_restore(flags); |
Kent Overstreet | e1bdd5f | 2013-04-26 10:58:39 +1000 | [diff] [blame] | 975 | return ret; |
| 976 | } |
| 977 | |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 978 | /* refill_reqs_available |
| 979 | * Updates the reqs_available reference counts used for tracking the |
| 980 | * number of free slots in the completion ring. This can be called |
| 981 | * from aio_complete() (to optimistically update reqs_available) or |
| 982 | * from aio_get_req() (the we're out of events case). It must be |
| 983 | * called holding ctx->completion_lock. |
| 984 | */ |
| 985 | static void refill_reqs_available(struct kioctx *ctx, unsigned head, |
| 986 | unsigned tail) |
| 987 | { |
| 988 | unsigned events_in_ring, completed; |
| 989 | |
| 990 | /* Clamp head since userland can write to it. */ |
| 991 | head %= ctx->nr_events; |
| 992 | if (head <= tail) |
| 993 | events_in_ring = tail - head; |
| 994 | else |
| 995 | events_in_ring = ctx->nr_events - (head - tail); |
| 996 | |
| 997 | completed = ctx->completed_events; |
| 998 | if (events_in_ring < completed) |
| 999 | completed -= events_in_ring; |
| 1000 | else |
| 1001 | completed = 0; |
| 1002 | |
| 1003 | if (!completed) |
| 1004 | return; |
| 1005 | |
| 1006 | ctx->completed_events -= completed; |
| 1007 | put_reqs_available(ctx, completed); |
| 1008 | } |
| 1009 | |
| 1010 | /* user_refill_reqs_available |
| 1011 | * Called to refill reqs_available when aio_get_req() encounters an |
| 1012 | * out of space in the completion ring. |
| 1013 | */ |
| 1014 | static void user_refill_reqs_available(struct kioctx *ctx) |
| 1015 | { |
| 1016 | spin_lock_irq(&ctx->completion_lock); |
| 1017 | if (ctx->completed_events) { |
| 1018 | struct aio_ring *ring; |
| 1019 | unsigned head; |
| 1020 | |
| 1021 | /* Access of ring->head may race with aio_read_events_ring() |
| 1022 | * here, but that's okay since whether we read the old version |
| 1023 | * or the new version, and either will be valid. The important |
| 1024 | * part is that head cannot pass tail since we prevent |
| 1025 | * aio_complete() from updating tail by holding |
| 1026 | * ctx->completion_lock. Even if head is invalid, the check |
| 1027 | * against ctx->completed_events below will make sure we do the |
| 1028 | * safe/right thing. |
| 1029 | */ |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1030 | ring = folio_address(ctx->ring_folios[0]); |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 1031 | head = ring->head; |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 1032 | |
| 1033 | refill_reqs_available(ctx, head, ctx->tail); |
| 1034 | } |
| 1035 | |
| 1036 | spin_unlock_irq(&ctx->completion_lock); |
| 1037 | } |
| 1038 | |
Christoph Hellwig | 432c799 | 2018-11-19 15:57:42 -0700 | [diff] [blame] | 1039 | static bool get_reqs_available(struct kioctx *ctx) |
| 1040 | { |
| 1041 | if (__get_reqs_available(ctx)) |
| 1042 | return true; |
| 1043 | user_refill_reqs_available(ctx); |
| 1044 | return __get_reqs_available(ctx); |
| 1045 | } |
| 1046 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1047 | /* aio_get_req |
Kent Overstreet | 57282d8 | 2013-05-13 13:42:52 -0700 | [diff] [blame] | 1048 | * Allocate a slot for an aio request. |
| 1049 | * Returns NULL if no requests are free. |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1050 | * |
| 1051 | * The refcount is initialized to 2 - one for the async op completion, |
| 1052 | * one for the synchronous code that does this. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1053 | */ |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 1054 | static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1055 | { |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 1056 | struct aio_kiocb *req; |
Kent Overstreet | a1c8eae | 2013-05-07 16:18:53 -0700 | [diff] [blame] | 1057 | |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1058 | req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1059 | if (unlikely(!req)) |
Christoph Hellwig | 432c799 | 2018-11-19 15:57:42 -0700 | [diff] [blame] | 1060 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1061 | |
Al Viro | fa0ca2a | 2019-03-06 18:21:08 -0500 | [diff] [blame] | 1062 | if (unlikely(!get_reqs_available(ctx))) { |
Wei Yongjun | 6af1c84 | 2019-04-04 08:44:05 +0000 | [diff] [blame] | 1063 | kmem_cache_free(kiocb_cachep, req); |
Al Viro | fa0ca2a | 2019-03-06 18:21:08 -0500 | [diff] [blame] | 1064 | return NULL; |
| 1065 | } |
| 1066 | |
Kent Overstreet | e34ecee | 2013-10-10 19:31:47 -0700 | [diff] [blame] | 1067 | percpu_ref_get(&ctx->reqs); |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1068 | req->ki_ctx = ctx; |
Christoph Hellwig | 75321b5 | 2018-04-09 14:57:56 +0200 | [diff] [blame] | 1069 | INIT_LIST_HEAD(&req->ki_list); |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1070 | refcount_set(&req->ki_refcnt, 2); |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1071 | req->ki_eventfd = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1072 | return req; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1073 | } |
| 1074 | |
Adrian Bunk | d5470b5 | 2008-04-29 00:58:57 -0700 | [diff] [blame] | 1075 | static struct kioctx *lookup_ioctx(unsigned long ctx_id) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1076 | { |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1077 | struct aio_ring __user *ring = (void __user *)ctx_id; |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1078 | struct mm_struct *mm = current->mm; |
Jeff Moyer | 65c2449 | 2009-03-18 17:04:21 -0700 | [diff] [blame] | 1079 | struct kioctx *ctx, *ret = NULL; |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1080 | struct kioctx_table *table; |
| 1081 | unsigned id; |
| 1082 | |
| 1083 | if (get_user(id, &ring->id)) |
| 1084 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1085 | |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1086 | rcu_read_lock(); |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1087 | table = rcu_dereference(mm->ioctx_table); |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1088 | |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1089 | if (!table || id >= table->nr) |
| 1090 | goto out; |
| 1091 | |
Jeff Moyer | a538e3f | 2018-12-11 12:37:49 -0500 | [diff] [blame] | 1092 | id = array_index_nospec(id, table->nr); |
Tejun Heo | d0264c0 | 2018-03-14 12:10:17 -0700 | [diff] [blame] | 1093 | ctx = rcu_dereference(table->table[id]); |
Benjamin LaHaise | f30d704 | 2013-08-07 18:23:48 -0400 | [diff] [blame] | 1094 | if (ctx && ctx->user_id == ctx_id) { |
Al Viro | baf1056 | 2018-05-20 16:46:23 -0400 | [diff] [blame] | 1095 | if (percpu_ref_tryget_live(&ctx->users)) |
| 1096 | ret = ctx; |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1097 | } |
Benjamin LaHaise | db446a0 | 2013-07-30 12:54:40 -0400 | [diff] [blame] | 1098 | out: |
Jens Axboe | abf137d | 2008-12-09 08:11:22 +0100 | [diff] [blame] | 1099 | rcu_read_unlock(); |
Jeff Moyer | 65c2449 | 2009-03-18 17:04:21 -0700 | [diff] [blame] | 1100 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1101 | } |
| 1102 | |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1103 | static inline void iocb_destroy(struct aio_kiocb *iocb) |
| 1104 | { |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1105 | if (iocb->ki_eventfd) |
| 1106 | eventfd_ctx_put(iocb->ki_eventfd); |
Linus Torvalds | b53119f | 2019-03-06 20:22:54 -0500 | [diff] [blame] | 1107 | if (iocb->ki_filp) |
| 1108 | fput(iocb->ki_filp); |
| 1109 | percpu_ref_put(&iocb->ki_ctx->reqs); |
| 1110 | kmem_cache_free(kiocb_cachep, iocb); |
| 1111 | } |
| 1112 | |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1113 | struct aio_waiter { |
| 1114 | struct wait_queue_entry w; |
| 1115 | size_t min_nr; |
| 1116 | }; |
| 1117 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1118 | /* aio_complete |
| 1119 | * Called when the io request on the given iocb is complete. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1120 | */ |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1121 | static void aio_complete(struct aio_kiocb *iocb) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1122 | { |
| 1123 | struct kioctx *ctx = iocb->ki_ctx; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1124 | struct aio_ring *ring; |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1125 | struct io_event *ev_page, *event; |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1126 | unsigned tail, pos, head, avail; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1127 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1128 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1129 | /* |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 1130 | * Add a completion event to the ring buffer. Must be done holding |
Tang Chen | 4b30f07 | 2013-07-03 15:09:16 -0700 | [diff] [blame] | 1131 | * ctx->completion_lock to prevent other code from messing with the tail |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 1132 | * pointer since we might be called from irq context. |
| 1133 | */ |
| 1134 | spin_lock_irqsave(&ctx->completion_lock, flags); |
| 1135 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1136 | tail = ctx->tail; |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1137 | pos = tail + AIO_EVENTS_OFFSET; |
| 1138 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1139 | if (++tail >= ctx->nr_events) |
Ken Chen | 4bf69b2 | 2005-05-01 08:59:15 -0700 | [diff] [blame] | 1140 | tail = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1141 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1142 | ev_page = folio_address(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]); |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1143 | event = ev_page + pos % AIO_EVENTS_PER_PAGE; |
| 1144 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 1145 | *event = iocb->ki_res; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1146 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1147 | flush_dcache_folio(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]); |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1148 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 1149 | pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb, |
| 1150 | (void __user *)(unsigned long)iocb->ki_res.obj, |
| 1151 | iocb->ki_res.data, iocb->ki_res.res, iocb->ki_res.res2); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1152 | |
| 1153 | /* after flagging the request as done, we |
| 1154 | * must never even look at it again |
| 1155 | */ |
| 1156 | smp_wmb(); /* make event visible before updating tail */ |
| 1157 | |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1158 | ctx->tail = tail; |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1159 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1160 | ring = folio_address(ctx->ring_folios[0]); |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 1161 | head = ring->head; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1162 | ring->tail = tail; |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1163 | flush_dcache_folio(ctx->ring_folios[0]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1164 | |
Benjamin LaHaise | d856f32 | 2014-08-24 13:14:05 -0400 | [diff] [blame] | 1165 | ctx->completed_events++; |
| 1166 | if (ctx->completed_events > 1) |
| 1167 | refill_reqs_available(ctx, head, tail); |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1168 | |
| 1169 | avail = tail > head |
| 1170 | ? tail - head |
| 1171 | : tail + ctx->nr_events - head; |
Kent Overstreet | 0460fef | 2013-05-07 16:18:49 -0700 | [diff] [blame] | 1172 | spin_unlock_irqrestore(&ctx->completion_lock, flags); |
| 1173 | |
Kent Overstreet | 21b4020 | 2013-05-07 16:18:47 -0700 | [diff] [blame] | 1174 | pr_debug("added to ring %p at [%u]\n", iocb, tail); |
Davide Libenzi | 8d1c98b | 2008-04-10 21:29:19 -0700 | [diff] [blame] | 1175 | |
| 1176 | /* |
| 1177 | * Check if the user asked us to deliver the result through an |
| 1178 | * eventfd. The eventfd_signal() function is safe to be called |
| 1179 | * from IRQ context. |
| 1180 | */ |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1181 | if (iocb->ki_eventfd) |
Christian Brauner | 3652117 | 2023-11-22 13:48:23 +0100 | [diff] [blame] | 1182 | eventfd_signal(iocb->ki_eventfd); |
Davide Libenzi | 8d1c98b | 2008-04-10 21:29:19 -0700 | [diff] [blame] | 1183 | |
Quentin Barnes | 6cb2a21 | 2008-03-19 17:00:39 -0700 | [diff] [blame] | 1184 | /* |
| 1185 | * We have to order our ring_info tail store above and test |
| 1186 | * of the wait list below outside the wait lock. This is |
| 1187 | * like in wake_up_bit() where clearing a bit has to be |
| 1188 | * ordered with the unlocked test. |
| 1189 | */ |
| 1190 | smp_mb(); |
| 1191 | |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1192 | if (waitqueue_active(&ctx->wait)) { |
| 1193 | struct aio_waiter *curr, *next; |
| 1194 | unsigned long flags; |
| 1195 | |
| 1196 | spin_lock_irqsave(&ctx->wait.lock, flags); |
| 1197 | list_for_each_entry_safe(curr, next, &ctx->wait.head, w.entry) |
| 1198 | if (avail >= curr->min_nr) { |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1199 | wake_up_process(curr->w.private); |
Kent Overstreet | caeb4b0 | 2024-03-31 17:52:12 -0400 | [diff] [blame] | 1200 | list_del_init_careful(&curr->w.entry); |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1201 | } |
| 1202 | spin_unlock_irqrestore(&ctx->wait.lock, flags); |
| 1203 | } |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1204 | } |
| 1205 | |
| 1206 | static inline void iocb_put(struct aio_kiocb *iocb) |
| 1207 | { |
| 1208 | if (refcount_dec_and_test(&iocb->ki_refcnt)) { |
| 1209 | aio_complete(iocb); |
| 1210 | iocb_destroy(iocb); |
| 1211 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1212 | } |
| 1213 | |
Gu Zheng | 2be4e7d | 2014-07-23 18:03:53 +0800 | [diff] [blame] | 1214 | /* aio_read_events_ring |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1215 | * Pull an event off of the ioctx's event ring. Returns the number of |
| 1216 | * events fetched |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1217 | */ |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1218 | static long aio_read_events_ring(struct kioctx *ctx, |
| 1219 | struct io_event __user *event, long nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1220 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1221 | struct aio_ring *ring; |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1222 | unsigned head, tail, pos; |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1223 | long ret = 0; |
| 1224 | int copy_ret; |
| 1225 | |
Dave Chinner | 9c9ce76 | 2015-02-03 19:29:05 -0500 | [diff] [blame] | 1226 | /* |
| 1227 | * The mutex can block and wake us up and that will cause |
| 1228 | * wait_event_interruptible_hrtimeout() to schedule without sleeping |
| 1229 | * and repeat. This should be rare enough that it doesn't cause |
| 1230 | * peformance issues. See the comment in read_events() for more detail. |
| 1231 | */ |
| 1232 | sched_annotate_sleep(); |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1233 | mutex_lock(&ctx->ring_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1234 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1235 | /* Access to ->ring_folios here is protected by ctx->ring_lock. */ |
| 1236 | ring = folio_address(ctx->ring_folios[0]); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1237 | head = ring->head; |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1238 | tail = ring->tail; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1239 | |
Jeff Moyer | 2ff396be | 2014-09-02 13:17:00 -0400 | [diff] [blame] | 1240 | /* |
| 1241 | * Ensure that once we've read the current tail pointer, that |
| 1242 | * we also see the events that were stored up to the tail. |
| 1243 | */ |
| 1244 | smp_rmb(); |
| 1245 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1246 | pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1247 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1248 | if (head == tail) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1249 | goto out; |
| 1250 | |
Benjamin LaHaise | edfbbf3 | 2014-06-24 13:32:51 -0400 | [diff] [blame] | 1251 | head %= ctx->nr_events; |
| 1252 | tail %= ctx->nr_events; |
| 1253 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1254 | while (ret < nr) { |
| 1255 | long avail; |
| 1256 | struct io_event *ev; |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1257 | struct folio *folio; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1258 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1259 | avail = (head <= tail ? tail : ctx->nr_events) - head; |
| 1260 | if (head == tail) |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1261 | break; |
| 1262 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1263 | pos = head + AIO_EVENTS_OFFSET; |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1264 | folio = ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]; |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1265 | pos %= AIO_EVENTS_PER_PAGE; |
| 1266 | |
Al Viro | d2988bd4 | 2018-05-26 19:13:10 -0400 | [diff] [blame] | 1267 | avail = min(avail, nr - ret); |
| 1268 | avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); |
| 1269 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1270 | ev = folio_address(folio); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1271 | copy_ret = copy_to_user(event + ret, ev + pos, |
| 1272 | sizeof(*ev) * avail); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1273 | |
| 1274 | if (unlikely(copy_ret)) { |
| 1275 | ret = -EFAULT; |
| 1276 | goto out; |
| 1277 | } |
| 1278 | |
| 1279 | ret += avail; |
| 1280 | head += avail; |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1281 | head %= ctx->nr_events; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1282 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1283 | |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1284 | ring = folio_address(ctx->ring_folios[0]); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1285 | ring->head = head; |
Kefeng Wang | 16594e6 | 2024-03-21 21:16:40 +0800 | [diff] [blame] | 1286 | flush_dcache_folio(ctx->ring_folios[0]); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1287 | |
Kent Overstreet | 5ffac12 | 2013-05-09 15:36:07 -0700 | [diff] [blame] | 1288 | pr_debug("%li h%u t%u\n", ret, head, tail); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1289 | out: |
Kent Overstreet | 58c85dc | 2013-05-07 16:18:55 -0700 | [diff] [blame] | 1290 | mutex_unlock(&ctx->ring_lock); |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1291 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1292 | return ret; |
| 1293 | } |
| 1294 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1295 | static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr, |
| 1296 | struct io_event __user *event, long *i) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1297 | { |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1298 | long ret = aio_read_events_ring(ctx, event + *i, nr - *i); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1299 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1300 | if (ret > 0) |
| 1301 | *i += ret; |
| 1302 | |
| 1303 | if (unlikely(atomic_read(&ctx->dead))) |
| 1304 | ret = -EINVAL; |
| 1305 | |
| 1306 | if (!*i) |
| 1307 | *i = ret; |
| 1308 | |
| 1309 | return ret < 0 || *i >= min_nr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1310 | } |
| 1311 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1312 | static long read_events(struct kioctx *ctx, long min_nr, long nr, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1313 | struct io_event __user *event, |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 1314 | ktime_t until) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1315 | { |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1316 | struct hrtimer_sleeper t; |
| 1317 | struct aio_waiter w; |
| 1318 | long ret = 0, ret2 = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1319 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1320 | /* |
| 1321 | * Note that aio_read_events() is being called as the conditional - i.e. |
| 1322 | * we're calling it after prepare_to_wait() has set task state to |
| 1323 | * TASK_INTERRUPTIBLE. |
| 1324 | * |
| 1325 | * But aio_read_events() can block, and if it blocks it's going to flip |
| 1326 | * the task state back to TASK_RUNNING. |
| 1327 | * |
| 1328 | * This should be ok, provided it doesn't flip the state back to |
| 1329 | * TASK_RUNNING and return 0 too much - that causes us to spin. That |
| 1330 | * will only happen if the mutex_lock() call blocks, and we then find |
| 1331 | * the ringbuffer empty. So in practice we should be ok, but it's |
| 1332 | * something to be aware of when touching this code. |
| 1333 | */ |
Kent Overstreet | 71eb6b6 | 2023-11-22 18:42:53 -0500 | [diff] [blame] | 1334 | aio_read_events(ctx, min_nr, nr, event, &ret); |
| 1335 | if (until == 0 || ret < 0 || ret >= min_nr) |
| 1336 | return ret; |
| 1337 | |
| 1338 | hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 1339 | if (until != KTIME_MAX) { |
| 1340 | hrtimer_set_expires_range_ns(&t.timer, until, current->timer_slack_ns); |
| 1341 | hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL); |
| 1342 | } |
| 1343 | |
| 1344 | init_wait(&w.w); |
| 1345 | |
| 1346 | while (1) { |
| 1347 | unsigned long nr_got = ret; |
| 1348 | |
| 1349 | w.min_nr = min_nr - ret; |
| 1350 | |
| 1351 | ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE); |
| 1352 | if (!ret2 && !t.task) |
| 1353 | ret2 = -ETIME; |
| 1354 | |
| 1355 | if (aio_read_events(ctx, min_nr, nr, event, &ret) || ret2) |
| 1356 | break; |
| 1357 | |
| 1358 | if (nr_got == ret) |
| 1359 | schedule(); |
| 1360 | } |
| 1361 | |
| 1362 | finish_wait(&ctx->wait, &w.w); |
| 1363 | hrtimer_cancel(&t.timer); |
| 1364 | destroy_hrtimer_on_stack(&t.timer); |
| 1365 | |
Kent Overstreet | a31ad38 | 2013-05-07 16:18:45 -0700 | [diff] [blame] | 1366 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1367 | } |
| 1368 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1369 | /* sys_io_setup: |
| 1370 | * Create an aio_context capable of receiving at least nr_events. |
| 1371 | * ctxp must not point to an aio_context that already exists, and |
| 1372 | * must be initialized to 0 prior to the call. On successful |
| 1373 | * creation of the aio_context, *ctxp is filled in with the resulting |
| 1374 | * handle. May fail with -EINVAL if *ctxp is not initialized, |
| 1375 | * if the specified nr_events exceeds internal limits. May fail |
| 1376 | * with -EAGAIN if the specified nr_events exceeds the user's limit |
| 1377 | * of available events. May fail with -ENOMEM if insufficient kernel |
| 1378 | * resources are available. May fail with -EFAULT if an invalid |
| 1379 | * pointer is passed for ctxp. Will fail with -ENOSYS if not |
| 1380 | * implemented. |
| 1381 | */ |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 1382 | SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1383 | { |
| 1384 | struct kioctx *ioctx = NULL; |
| 1385 | unsigned long ctx; |
| 1386 | long ret; |
| 1387 | |
| 1388 | ret = get_user(ctx, ctxp); |
| 1389 | if (unlikely(ret)) |
| 1390 | goto out; |
| 1391 | |
| 1392 | ret = -EINVAL; |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 1393 | if (unlikely(ctx || nr_events == 0)) { |
Kinglong Mee | acd88d4 | 2015-02-04 21:15:59 +0800 | [diff] [blame] | 1394 | pr_debug("EINVAL: ctx %lu nr_events %u\n", |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 1395 | ctx, nr_events); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1396 | goto out; |
| 1397 | } |
| 1398 | |
| 1399 | ioctx = ioctx_alloc(nr_events); |
| 1400 | ret = PTR_ERR(ioctx); |
| 1401 | if (!IS_ERR(ioctx)) { |
| 1402 | ret = put_user(ioctx->user_id, ctxp); |
Al Viro | a2e1859 | 2012-03-20 16:27:57 -0400 | [diff] [blame] | 1403 | if (ret) |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1404 | kill_ioctx(current->mm, ioctx, NULL); |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 1405 | percpu_ref_put(&ioctx->users); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1406 | } |
| 1407 | |
| 1408 | out: |
| 1409 | return ret; |
| 1410 | } |
| 1411 | |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 1412 | #ifdef CONFIG_COMPAT |
| 1413 | COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p) |
| 1414 | { |
| 1415 | struct kioctx *ioctx = NULL; |
| 1416 | unsigned long ctx; |
| 1417 | long ret; |
| 1418 | |
| 1419 | ret = get_user(ctx, ctx32p); |
| 1420 | if (unlikely(ret)) |
| 1421 | goto out; |
| 1422 | |
| 1423 | ret = -EINVAL; |
| 1424 | if (unlikely(ctx || nr_events == 0)) { |
| 1425 | pr_debug("EINVAL: ctx %lu nr_events %u\n", |
| 1426 | ctx, nr_events); |
| 1427 | goto out; |
| 1428 | } |
| 1429 | |
| 1430 | ioctx = ioctx_alloc(nr_events); |
| 1431 | ret = PTR_ERR(ioctx); |
| 1432 | if (!IS_ERR(ioctx)) { |
| 1433 | /* truncating is ok because it's a user address */ |
| 1434 | ret = put_user((u32)ioctx->user_id, ctx32p); |
| 1435 | if (ret) |
| 1436 | kill_ioctx(current->mm, ioctx, NULL); |
| 1437 | percpu_ref_put(&ioctx->users); |
| 1438 | } |
| 1439 | |
| 1440 | out: |
| 1441 | return ret; |
| 1442 | } |
| 1443 | #endif |
| 1444 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1445 | /* sys_io_destroy: |
| 1446 | * Destroy the aio_context specified. May cancel any outstanding |
| 1447 | * AIOs and block on completion. Will fail with -ENOSYS if not |
Satoru Takeuchi | 642b512 | 2010-08-05 11:23:11 -0700 | [diff] [blame] | 1448 | * implemented. May fail with -EINVAL if the context pointed to |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1449 | * is invalid. |
| 1450 | */ |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 1451 | SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1452 | { |
| 1453 | struct kioctx *ioctx = lookup_ioctx(ctx); |
| 1454 | if (likely(NULL != ioctx)) { |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1455 | struct ctx_rq_wait wait; |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 1456 | int ret; |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1457 | |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1458 | init_completion(&wait.comp); |
| 1459 | atomic_set(&wait.count, 1); |
| 1460 | |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1461 | /* Pass requests_done to kill_ioctx() where it can be set |
| 1462 | * in a thread-safe way. If we try to set it here then we have |
| 1463 | * a race condition if two io_destroy() called simultaneously. |
| 1464 | */ |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1465 | ret = kill_ioctx(current->mm, ioctx, &wait); |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 1466 | percpu_ref_put(&ioctx->users); |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1467 | |
| 1468 | /* Wait until all IO for the context are done. Otherwise kernel |
| 1469 | * keep using user-space buffers even if user thinks the context |
| 1470 | * is destroyed. |
| 1471 | */ |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 1472 | if (!ret) |
Jens Axboe | dc48e56 | 2015-04-15 11:17:23 -0600 | [diff] [blame] | 1473 | wait_for_completion(&wait.comp); |
Anatol Pomozov | e02ba72 | 2014-04-15 11:31:33 -0700 | [diff] [blame] | 1474 | |
Benjamin LaHaise | fb2d448 | 2014-04-29 12:45:17 -0400 | [diff] [blame] | 1475 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1476 | } |
Kinglong Mee | acd88d4 | 2015-02-04 21:15:59 +0800 | [diff] [blame] | 1477 | pr_debug("EINVAL: invalid context id\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1478 | return -EINVAL; |
| 1479 | } |
| 1480 | |
Al Viro | 3c96c7f | 2018-05-28 13:37:43 -0400 | [diff] [blame] | 1481 | static void aio_remove_iocb(struct aio_kiocb *iocb) |
| 1482 | { |
| 1483 | struct kioctx *ctx = iocb->ki_ctx; |
| 1484 | unsigned long flags; |
| 1485 | |
| 1486 | spin_lock_irqsave(&ctx->ctx_lock, flags); |
| 1487 | list_del(&iocb->ki_list); |
| 1488 | spin_unlock_irqrestore(&ctx->ctx_lock, flags); |
| 1489 | } |
| 1490 | |
Jens Axboe | 6b19b76 | 2021-10-21 09:22:35 -0600 | [diff] [blame] | 1491 | static void aio_complete_rw(struct kiocb *kiocb, long res) |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1492 | { |
| 1493 | struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); |
| 1494 | |
Al Viro | 3c96c7f | 2018-05-28 13:37:43 -0400 | [diff] [blame] | 1495 | if (!list_empty_careful(&iocb->ki_list)) |
| 1496 | aio_remove_iocb(iocb); |
| 1497 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1498 | if (kiocb->ki_flags & IOCB_WRITE) { |
| 1499 | struct inode *inode = file_inode(kiocb->ki_filp); |
| 1500 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1501 | if (S_ISREG(inode->i_mode)) |
Amir Goldstein | 8c3cfa8 | 2023-08-17 17:13:35 +0300 | [diff] [blame] | 1502 | kiocb_end_write(kiocb); |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1503 | } |
| 1504 | |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1505 | iocb->ki_res.res = res; |
Jens Axboe | 6b19b76 | 2021-10-21 09:22:35 -0600 | [diff] [blame] | 1506 | iocb->ki_res.res2 = 0; |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1507 | iocb_put(iocb); |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1508 | } |
| 1509 | |
Prasad Singamsetty | c34fc6f | 2024-06-20 12:53:52 +0000 | [diff] [blame] | 1510 | static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb, int rw_type) |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1511 | { |
| 1512 | int ret; |
| 1513 | |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1514 | req->ki_complete = aio_complete_rw; |
Mike Marshall | ec51f8e | 2019-02-05 14:13:35 -0500 | [diff] [blame] | 1515 | req->private = NULL; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1516 | req->ki_pos = iocb->aio_offset; |
Bart Van Assche | b820de7 | 2024-02-15 12:47:38 -0800 | [diff] [blame] | 1517 | req->ki_flags = req->ki_filp->f_iocb_flags | IOCB_AIO_RW; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1518 | if (iocb->aio_flags & IOCB_FLAG_RESFD) |
| 1519 | req->ki_flags |= IOCB_EVENTFD; |
Adam Manzanares | d9a08a9 | 2018-05-22 10:52:19 -0700 | [diff] [blame] | 1520 | if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { |
| 1521 | /* |
| 1522 | * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then |
| 1523 | * aio_reqprio is interpreted as an I/O scheduling |
| 1524 | * class and priority. |
| 1525 | */ |
| 1526 | ret = ioprio_check_cap(iocb->aio_reqprio); |
| 1527 | if (ret) { |
Adam Manzanares | 9a6d9a6 | 2018-06-04 10:59:57 -0700 | [diff] [blame] | 1528 | pr_debug("aio ioprio check cap error: %d\n", ret); |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1529 | return ret; |
Adam Manzanares | d9a08a9 | 2018-05-22 10:52:19 -0700 | [diff] [blame] | 1530 | } |
| 1531 | |
| 1532 | req->ki_ioprio = iocb->aio_reqprio; |
| 1533 | } else |
Damien Le Moal | 76dc891 | 2018-11-20 10:52:36 +0900 | [diff] [blame] | 1534 | req->ki_ioprio = get_current_ioprio(); |
Adam Manzanares | d9a08a9 | 2018-05-22 10:52:19 -0700 | [diff] [blame] | 1535 | |
Prasad Singamsetty | c34fc6f | 2024-06-20 12:53:52 +0000 | [diff] [blame] | 1536 | ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags, rw_type); |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1537 | if (unlikely(ret)) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1538 | return ret; |
Christoph Hellwig | 154989e | 2018-11-22 16:44:07 +0100 | [diff] [blame] | 1539 | |
| 1540 | req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ |
| 1541 | return 0; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1542 | } |
| 1543 | |
Jens Axboe | 87e5e6d | 2019-05-14 16:02:22 -0600 | [diff] [blame] | 1544 | static ssize_t aio_setup_rw(int rw, const struct iocb *iocb, |
| 1545 | struct iovec **iovec, bool vectored, bool compat, |
| 1546 | struct iov_iter *iter) |
Badari Pulavarty | eed4e51 | 2006-09-30 23:28:49 -0700 | [diff] [blame] | 1547 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1548 | void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf; |
| 1549 | size_t len = iocb->aio_nbytes; |
| 1550 | |
| 1551 | if (!vectored) { |
Jens Axboe | 9fd7874 | 2023-12-04 10:47:50 -0700 | [diff] [blame] | 1552 | ssize_t ret = import_ubuf(rw, buf, len, iter); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1553 | *iovec = NULL; |
| 1554 | return ret; |
| 1555 | } |
Christoph Hellwig | 89cd35c | 2020-09-25 06:51:41 +0200 | [diff] [blame] | 1556 | |
| 1557 | return __import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter, compat); |
Badari Pulavarty | eed4e51 | 2006-09-30 23:28:49 -0700 | [diff] [blame] | 1558 | } |
| 1559 | |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1560 | static inline void aio_rw_done(struct kiocb *req, ssize_t ret) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1561 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1562 | switch (ret) { |
| 1563 | case -EIOCBQUEUED: |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1564 | break; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1565 | case -ERESTARTSYS: |
| 1566 | case -ERESTARTNOINTR: |
| 1567 | case -ERESTARTNOHAND: |
| 1568 | case -ERESTART_RESTARTBLOCK: |
Kent Overstreet | 41ef4eb | 2013-05-07 16:19:11 -0700 | [diff] [blame] | 1569 | /* |
| 1570 | * There's no easy way to restart the syscall since other AIO's |
| 1571 | * may be already running. Just fail this IO with EINTR. |
| 1572 | */ |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1573 | ret = -EINTR; |
Gustavo A. R. Silva | df561f66 | 2020-08-23 17:36:59 -0500 | [diff] [blame] | 1574 | fallthrough; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1575 | default: |
Jens Axboe | 6b19b76 | 2021-10-21 09:22:35 -0600 | [diff] [blame] | 1576 | req->ki_complete(req, ret); |
Kent Overstreet | 41ef4eb | 2013-05-07 16:19:11 -0700 | [diff] [blame] | 1577 | } |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1578 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1579 | |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1580 | static int aio_read(struct kiocb *req, const struct iocb *iocb, |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1581 | bool vectored, bool compat) |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1582 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1583 | struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; |
| 1584 | struct iov_iter iter; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1585 | struct file *file; |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1586 | int ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1587 | |
Prasad Singamsetty | c34fc6f | 2024-06-20 12:53:52 +0000 | [diff] [blame] | 1588 | ret = aio_prep_rw(req, iocb, READ); |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1589 | if (ret) |
| 1590 | return ret; |
| 1591 | file = req->ki_filp; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1592 | if (unlikely(!(file->f_mode & FMODE_READ))) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1593 | return -EBADF; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1594 | if (unlikely(!file->f_op->read_iter)) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1595 | return -EINVAL; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1596 | |
Al Viro | de4eda9 | 2022-09-15 20:25:47 -0400 | [diff] [blame] | 1597 | ret = aio_setup_rw(ITER_DEST, iocb, &iovec, vectored, compat, &iter); |
Jens Axboe | 87e5e6d | 2019-05-14 16:02:22 -0600 | [diff] [blame] | 1598 | if (ret < 0) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1599 | return ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1600 | ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); |
| 1601 | if (!ret) |
Miklos Szeredi | 7c98f7c | 2023-08-28 17:13:18 +0200 | [diff] [blame] | 1602 | aio_rw_done(req, file->f_op->read_iter(req, &iter)); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1603 | kfree(iovec); |
| 1604 | return ret; |
| 1605 | } |
| 1606 | |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1607 | static int aio_write(struct kiocb *req, const struct iocb *iocb, |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1608 | bool vectored, bool compat) |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1609 | { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1610 | struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; |
| 1611 | struct iov_iter iter; |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1612 | struct file *file; |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1613 | int ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1614 | |
Prasad Singamsetty | c34fc6f | 2024-06-20 12:53:52 +0000 | [diff] [blame] | 1615 | ret = aio_prep_rw(req, iocb, WRITE); |
Christoph Hellwig | 54843f8 | 2018-05-02 19:57:21 +0200 | [diff] [blame] | 1616 | if (ret) |
| 1617 | return ret; |
| 1618 | file = req->ki_filp; |
| 1619 | |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1620 | if (unlikely(!(file->f_mode & FMODE_WRITE))) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1621 | return -EBADF; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1622 | if (unlikely(!file->f_op->write_iter)) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1623 | return -EINVAL; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1624 | |
Al Viro | de4eda9 | 2022-09-15 20:25:47 -0400 | [diff] [blame] | 1625 | ret = aio_setup_rw(ITER_SOURCE, iocb, &iovec, vectored, compat, &iter); |
Jens Axboe | 87e5e6d | 2019-05-14 16:02:22 -0600 | [diff] [blame] | 1626 | if (ret < 0) |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1627 | return ret; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1628 | ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); |
| 1629 | if (!ret) { |
Amir Goldstein | 8c3cfa8 | 2023-08-17 17:13:35 +0300 | [diff] [blame] | 1630 | if (S_ISREG(file_inode(file)->i_mode)) |
| 1631 | kiocb_start_write(req); |
Christoph Hellwig | 92ce472 | 2018-04-06 09:28:17 +0200 | [diff] [blame] | 1632 | req->ki_flags |= IOCB_WRITE; |
Miklos Szeredi | 7c98f7c | 2023-08-28 17:13:18 +0200 | [diff] [blame] | 1633 | aio_rw_done(req, file->f_op->write_iter(req, &iter)); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 1634 | } |
| 1635 | kfree(iovec); |
| 1636 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1637 | } |
| 1638 | |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1639 | static void aio_fsync_work(struct work_struct *work) |
| 1640 | { |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1641 | struct aio_kiocb *iocb = container_of(work, struct aio_kiocb, fsync.work); |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 1642 | const struct cred *old_cred = override_creds(iocb->fsync.creds); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1643 | |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1644 | iocb->ki_res.res = vfs_fsync(iocb->fsync.file, iocb->fsync.datasync); |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 1645 | revert_creds(old_cred); |
| 1646 | put_cred(iocb->fsync.creds); |
Al Viro | 2bb874c | 2019-03-07 19:49:55 -0500 | [diff] [blame] | 1647 | iocb_put(iocb); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1648 | } |
| 1649 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1650 | static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, |
| 1651 | bool datasync) |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1652 | { |
| 1653 | if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || |
| 1654 | iocb->aio_rw_flags)) |
| 1655 | return -EINVAL; |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 1656 | |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1657 | if (unlikely(!req->file->f_op->fsync)) |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1658 | return -EINVAL; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1659 | |
Miklos Szeredi | 530f32f | 2020-05-14 16:44:24 +0200 | [diff] [blame] | 1660 | req->creds = prepare_creds(); |
| 1661 | if (!req->creds) |
| 1662 | return -ENOMEM; |
| 1663 | |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1664 | req->datasync = datasync; |
| 1665 | INIT_WORK(&req->work, aio_fsync_work); |
| 1666 | schedule_work(&req->work); |
Al Viro | 9061d14 | 2018-05-26 19:11:40 -0400 | [diff] [blame] | 1667 | return 0; |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 1668 | } |
| 1669 | |
Jens Axboe | 01d7a35 | 2020-02-03 10:33:42 -0700 | [diff] [blame] | 1670 | static void aio_poll_put_work(struct work_struct *work) |
| 1671 | { |
| 1672 | struct poll_iocb *req = container_of(work, struct poll_iocb, work); |
| 1673 | struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); |
| 1674 | |
| 1675 | iocb_put(iocb); |
| 1676 | } |
| 1677 | |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1678 | /* |
| 1679 | * Safely lock the waitqueue which the request is on, synchronizing with the |
| 1680 | * case where the ->poll() provider decides to free its waitqueue early. |
| 1681 | * |
| 1682 | * Returns true on success, meaning that req->head->lock was locked, req->wait |
| 1683 | * is on req->head, and an RCU read lock was taken. Returns false if the |
| 1684 | * request was already removed from its waitqueue (which might no longer exist). |
| 1685 | */ |
| 1686 | static bool poll_iocb_lock_wq(struct poll_iocb *req) |
| 1687 | { |
| 1688 | wait_queue_head_t *head; |
| 1689 | |
| 1690 | /* |
| 1691 | * While we hold the waitqueue lock and the waitqueue is nonempty, |
| 1692 | * wake_up_pollfree() will wait for us. However, taking the waitqueue |
| 1693 | * lock in the first place can race with the waitqueue being freed. |
| 1694 | * |
| 1695 | * We solve this as eventpoll does: by taking advantage of the fact that |
| 1696 | * all users of wake_up_pollfree() will RCU-delay the actual free. If |
| 1697 | * we enter rcu_read_lock() and see that the pointer to the queue is |
| 1698 | * non-NULL, we can then lock it without the memory being freed out from |
| 1699 | * under us, then check whether the request is still on the queue. |
| 1700 | * |
| 1701 | * Keep holding rcu_read_lock() as long as we hold the queue lock, in |
| 1702 | * case the caller deletes the entry from the queue, leaving it empty. |
| 1703 | * In that case, only RCU prevents the queue memory from being freed. |
| 1704 | */ |
| 1705 | rcu_read_lock(); |
| 1706 | head = smp_load_acquire(&req->head); |
| 1707 | if (head) { |
| 1708 | spin_lock(&head->lock); |
| 1709 | if (!list_empty(&req->wait.entry)) |
| 1710 | return true; |
| 1711 | spin_unlock(&head->lock); |
| 1712 | } |
| 1713 | rcu_read_unlock(); |
| 1714 | return false; |
| 1715 | } |
| 1716 | |
| 1717 | static void poll_iocb_unlock_wq(struct poll_iocb *req) |
| 1718 | { |
| 1719 | spin_unlock(&req->head->lock); |
| 1720 | rcu_read_unlock(); |
| 1721 | } |
| 1722 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1723 | static void aio_poll_complete_work(struct work_struct *work) |
| 1724 | { |
| 1725 | struct poll_iocb *req = container_of(work, struct poll_iocb, work); |
| 1726 | struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); |
| 1727 | struct poll_table_struct pt = { ._key = req->events }; |
| 1728 | struct kioctx *ctx = iocb->ki_ctx; |
| 1729 | __poll_t mask = 0; |
| 1730 | |
| 1731 | if (!READ_ONCE(req->cancelled)) |
| 1732 | mask = vfs_poll(req->file, &pt) & req->events; |
| 1733 | |
| 1734 | /* |
| 1735 | * Note that ->ki_cancel callers also delete iocb from active_reqs after |
| 1736 | * calling ->ki_cancel. We need the ctx_lock roundtrip here to |
| 1737 | * synchronize with them. In the cancellation case the list_del_init |
| 1738 | * itself is not actually needed, but harmless so we keep it in to |
| 1739 | * avoid further branches in the fast path. |
| 1740 | */ |
| 1741 | spin_lock_irq(&ctx->ctx_lock); |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1742 | if (poll_iocb_lock_wq(req)) { |
| 1743 | if (!mask && !READ_ONCE(req->cancelled)) { |
| 1744 | /* |
| 1745 | * The request isn't actually ready to be completed yet. |
| 1746 | * Reschedule completion if another wakeup came in. |
| 1747 | */ |
| 1748 | if (req->work_need_resched) { |
| 1749 | schedule_work(&req->work); |
| 1750 | req->work_need_resched = false; |
| 1751 | } else { |
| 1752 | req->work_scheduled = false; |
| 1753 | } |
| 1754 | poll_iocb_unlock_wq(req); |
| 1755 | spin_unlock_irq(&ctx->ctx_lock); |
| 1756 | return; |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1757 | } |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1758 | list_del_init(&req->wait.entry); |
| 1759 | poll_iocb_unlock_wq(req); |
| 1760 | } /* else, POLLFREE has freed the waitqueue, so we must complete */ |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1761 | list_del_init(&iocb->ki_list); |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1762 | iocb->ki_res.res = mangle_poll(mask); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1763 | spin_unlock_irq(&ctx->ctx_lock); |
| 1764 | |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1765 | iocb_put(iocb); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1766 | } |
| 1767 | |
| 1768 | /* assumes we are called with irqs disabled */ |
| 1769 | static int aio_poll_cancel(struct kiocb *iocb) |
| 1770 | { |
| 1771 | struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw); |
| 1772 | struct poll_iocb *req = &aiocb->poll; |
| 1773 | |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1774 | if (poll_iocb_lock_wq(req)) { |
| 1775 | WRITE_ONCE(req->cancelled, true); |
| 1776 | if (!req->work_scheduled) { |
| 1777 | schedule_work(&aiocb->poll.work); |
| 1778 | req->work_scheduled = true; |
| 1779 | } |
| 1780 | poll_iocb_unlock_wq(req); |
| 1781 | } /* else, the request was force-cancelled by POLLFREE already */ |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1782 | |
| 1783 | return 0; |
| 1784 | } |
| 1785 | |
| 1786 | static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, |
| 1787 | void *key) |
| 1788 | { |
| 1789 | struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); |
Christoph Hellwig | e8693bc | 2018-07-16 12:25:17 +0200 | [diff] [blame] | 1790 | struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1791 | __poll_t mask = key_to_poll(key); |
Bart Van Assche | d3d6a18 | 2019-02-08 16:59:49 -0800 | [diff] [blame] | 1792 | unsigned long flags; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1793 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1794 | /* for instances that support it check for an event match first: */ |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1795 | if (mask && !(mask & req->events)) |
| 1796 | return 0; |
Christoph Hellwig | e8693bc | 2018-07-16 12:25:17 +0200 | [diff] [blame] | 1797 | |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1798 | /* |
| 1799 | * Complete the request inline if possible. This requires that three |
| 1800 | * conditions be met: |
| 1801 | * 1. An event mask must have been passed. If a plain wakeup was done |
| 1802 | * instead, then mask == 0 and we have to call vfs_poll() to get |
| 1803 | * the events, so inline completion isn't possible. |
| 1804 | * 2. The completion work must not have already been scheduled. |
| 1805 | * 3. ctx_lock must not be busy. We have to use trylock because we |
| 1806 | * already hold the waitqueue lock, so this inverts the normal |
| 1807 | * locking order. Use irqsave/irqrestore because not all |
| 1808 | * filesystems (e.g. fuse) call this function with IRQs disabled, |
| 1809 | * yet IRQs have to be disabled before ctx_lock is obtained. |
| 1810 | */ |
| 1811 | if (mask && !req->work_scheduled && |
| 1812 | spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) { |
Jens Axboe | 01d7a35 | 2020-02-03 10:33:42 -0700 | [diff] [blame] | 1813 | struct kioctx *ctx = iocb->ki_ctx; |
| 1814 | |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1815 | list_del_init(&req->wait.entry); |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1816 | list_del(&iocb->ki_list); |
| 1817 | iocb->ki_res.res = mangle_poll(mask); |
Xie Yongji | 4b37498 | 2021-09-13 19:19:28 +0800 | [diff] [blame] | 1818 | if (iocb->ki_eventfd && !eventfd_signal_allowed()) { |
Jens Axboe | 01d7a35 | 2020-02-03 10:33:42 -0700 | [diff] [blame] | 1819 | iocb = NULL; |
| 1820 | INIT_WORK(&req->work, aio_poll_put_work); |
| 1821 | schedule_work(&req->work); |
| 1822 | } |
| 1823 | spin_unlock_irqrestore(&ctx->ctx_lock, flags); |
| 1824 | if (iocb) |
| 1825 | iocb_put(iocb); |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1826 | } else { |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1827 | /* |
| 1828 | * Schedule the completion work if needed. If it was already |
| 1829 | * scheduled, record that another wakeup came in. |
| 1830 | * |
| 1831 | * Don't remove the request from the waitqueue here, as it might |
| 1832 | * not actually be complete yet (we won't know until vfs_poll() |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1833 | * is called), and we must not miss any wakeups. POLLFREE is an |
| 1834 | * exception to this; see below. |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1835 | */ |
| 1836 | if (req->work_scheduled) { |
| 1837 | req->work_need_resched = true; |
| 1838 | } else { |
| 1839 | schedule_work(&req->work); |
| 1840 | req->work_scheduled = true; |
| 1841 | } |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1842 | |
| 1843 | /* |
| 1844 | * If the waitqueue is being freed early but we can't complete |
| 1845 | * the request inline, we have to tear down the request as best |
| 1846 | * we can. That means immediately removing the request from its |
| 1847 | * waitqueue and preventing all further accesses to the |
| 1848 | * waitqueue via the request. We also need to schedule the |
| 1849 | * completion work (done above). Also mark the request as |
| 1850 | * cancelled, to potentially skip an unneeded call to ->poll(). |
| 1851 | */ |
| 1852 | if (mask & POLLFREE) { |
| 1853 | WRITE_ONCE(req->cancelled, true); |
| 1854 | list_del_init(&req->wait.entry); |
| 1855 | |
| 1856 | /* |
| 1857 | * Careful: this *must* be the last step, since as soon |
| 1858 | * as req->head is NULL'ed out, the request can be |
| 1859 | * completed and freed, since aio_poll_complete_work() |
| 1860 | * will no longer need to take the waitqueue lock. |
| 1861 | */ |
| 1862 | smp_store_release(&req->head, NULL); |
| 1863 | } |
Christoph Hellwig | e8693bc | 2018-07-16 12:25:17 +0200 | [diff] [blame] | 1864 | } |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1865 | return 1; |
| 1866 | } |
| 1867 | |
| 1868 | struct aio_poll_table { |
| 1869 | struct poll_table_struct pt; |
| 1870 | struct aio_kiocb *iocb; |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1871 | bool queued; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1872 | int error; |
| 1873 | }; |
| 1874 | |
| 1875 | static void |
| 1876 | aio_poll_queue_proc(struct file *file, struct wait_queue_head *head, |
| 1877 | struct poll_table_struct *p) |
| 1878 | { |
| 1879 | struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt); |
| 1880 | |
| 1881 | /* multiple wait queues per file are not supported */ |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1882 | if (unlikely(pt->queued)) { |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1883 | pt->error = -EINVAL; |
| 1884 | return; |
| 1885 | } |
| 1886 | |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1887 | pt->queued = true; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1888 | pt->error = 0; |
| 1889 | pt->iocb->poll.head = head; |
| 1890 | add_wait_queue(head, &pt->iocb->poll.wait); |
| 1891 | } |
| 1892 | |
Al Viro | 958c13c | 2019-03-06 18:13:00 -0500 | [diff] [blame] | 1893 | static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1894 | { |
| 1895 | struct kioctx *ctx = aiocb->ki_ctx; |
| 1896 | struct poll_iocb *req = &aiocb->poll; |
| 1897 | struct aio_poll_table apt; |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1898 | bool cancel = false; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1899 | __poll_t mask; |
| 1900 | |
| 1901 | /* reject any unknown events outside the normal event mask. */ |
| 1902 | if ((u16)iocb->aio_buf != iocb->aio_buf) |
| 1903 | return -EINVAL; |
| 1904 | /* reject fields that are not defined for poll */ |
| 1905 | if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags) |
| 1906 | return -EINVAL; |
| 1907 | |
| 1908 | INIT_WORK(&req->work, aio_poll_complete_work); |
| 1909 | req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1910 | |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1911 | req->head = NULL; |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1912 | req->cancelled = false; |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1913 | req->work_scheduled = false; |
| 1914 | req->work_need_resched = false; |
Jens Axboe | 2bc4ca9 | 2018-12-04 09:44:49 -0700 | [diff] [blame] | 1915 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1916 | apt.pt._qproc = aio_poll_queue_proc; |
| 1917 | apt.pt._key = req->events; |
| 1918 | apt.iocb = aiocb; |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1919 | apt.queued = false; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1920 | apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */ |
| 1921 | |
| 1922 | /* initialized the list so that we can do list_empty checks */ |
| 1923 | INIT_LIST_HEAD(&req->wait.entry); |
| 1924 | init_waitqueue_func_entry(&req->wait, aio_poll_wake); |
| 1925 | |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1926 | mask = vfs_poll(req->file, &apt.pt) & req->events; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1927 | spin_lock_irq(&ctx->ctx_lock); |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1928 | if (likely(apt.queued)) { |
| 1929 | bool on_queue = poll_iocb_lock_wq(req); |
| 1930 | |
| 1931 | if (!on_queue || req->work_scheduled) { |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1932 | /* |
| 1933 | * aio_poll_wake() already either scheduled the async |
| 1934 | * completion work, or completed the request inline. |
| 1935 | */ |
| 1936 | if (apt.error) /* unsupported case: multiple queues */ |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1937 | cancel = true; |
| 1938 | apt.error = 0; |
| 1939 | mask = 0; |
| 1940 | } |
| 1941 | if (mask || apt.error) { |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1942 | /* Steal to complete synchronously. */ |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1943 | list_del_init(&req->wait.entry); |
| 1944 | } else if (cancel) { |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1945 | /* Cancel if possible (may be too late though). */ |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1946 | WRITE_ONCE(req->cancelled, true); |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1947 | } else if (on_queue) { |
Eric Biggers | 363bee2 | 2021-12-08 17:04:54 -0800 | [diff] [blame] | 1948 | /* |
| 1949 | * Actually waiting for an event, so add the request to |
| 1950 | * active_reqs so that it can be cancelled if needed. |
| 1951 | */ |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1952 | list_add_tail(&aiocb->ki_list, &ctx->active_reqs); |
| 1953 | aiocb->ki_cancel = aio_poll_cancel; |
| 1954 | } |
Eric Biggers | 50252e4 | 2021-12-08 17:04:55 -0800 | [diff] [blame] | 1955 | if (on_queue) |
| 1956 | poll_iocb_unlock_wq(req); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1957 | } |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1958 | if (mask) { /* no async, we'd stolen it */ |
| 1959 | aiocb->ki_res.res = mangle_poll(mask); |
| 1960 | apt.error = 0; |
| 1961 | } |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1962 | spin_unlock_irq(&ctx->ctx_lock); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1963 | if (mask) |
Al Viro | af5c72b | 2019-03-07 21:45:41 -0500 | [diff] [blame] | 1964 | iocb_put(aiocb); |
| 1965 | return apt.error; |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 1966 | } |
| 1967 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1968 | static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb, |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1969 | struct iocb __user *user_iocb, struct aio_kiocb *req, |
| 1970 | bool compat) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1971 | { |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1972 | req->ki_filp = fget(iocb->aio_fildes); |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1973 | if (unlikely(!req->ki_filp)) |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1974 | return -EBADF; |
Linus Torvalds | 84c4e1f8 | 2019-03-03 14:23:33 -0800 | [diff] [blame] | 1975 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 1976 | if (iocb->aio_flags & IOCB_FLAG_RESFD) { |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1977 | struct eventfd_ctx *eventfd; |
Davide Libenzi | 9c3060b | 2007-05-10 22:23:21 -0700 | [diff] [blame] | 1978 | /* |
| 1979 | * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an |
| 1980 | * instance of the file* now. The file descriptor must be |
| 1981 | * an eventfd() fd, and will be signaled for each completed |
| 1982 | * event using the eventfd_signal() function. |
| 1983 | */ |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1984 | eventfd = eventfd_ctx_fdget(iocb->aio_resfd); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1985 | if (IS_ERR(eventfd)) |
Dan Carpenter | 18bfb9c | 2019-04-03 09:22:35 +0300 | [diff] [blame] | 1986 | return PTR_ERR(eventfd); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1987 | |
Al Viro | 7425970 | 2019-03-06 18:18:31 -0500 | [diff] [blame] | 1988 | req->ki_eventfd = eventfd; |
Goldwyn Rodrigues | 9830f4b | 2017-06-20 07:05:42 -0500 | [diff] [blame] | 1989 | } |
| 1990 | |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1991 | if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) { |
Kent Overstreet | caf4167 | 2013-05-07 16:18:35 -0700 | [diff] [blame] | 1992 | pr_debug("EFAULT: aio_key\n"); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 1993 | return -EFAULT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1994 | } |
| 1995 | |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 1996 | req->ki_res.obj = (u64)(unsigned long)user_iocb; |
| 1997 | req->ki_res.data = iocb->aio_data; |
| 1998 | req->ki_res.res = 0; |
| 1999 | req->ki_res.res2 = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2000 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 2001 | switch (iocb->aio_lio_opcode) { |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 2002 | case IOCB_CMD_PREAD: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2003 | return aio_read(&req->rw, iocb, false, compat); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 2004 | case IOCB_CMD_PWRITE: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2005 | return aio_write(&req->rw, iocb, false, compat); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 2006 | case IOCB_CMD_PREADV: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2007 | return aio_read(&req->rw, iocb, true, compat); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 2008 | case IOCB_CMD_PWRITEV: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2009 | return aio_write(&req->rw, iocb, true, compat); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 2010 | case IOCB_CMD_FSYNC: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2011 | return aio_fsync(&req->fsync, iocb, false); |
Christoph Hellwig | a3c0d43 | 2018-03-27 19:18:57 +0200 | [diff] [blame] | 2012 | case IOCB_CMD_FDSYNC: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2013 | return aio_fsync(&req->fsync, iocb, true); |
Christoph Hellwig | bfe4037 | 2018-07-16 09:08:20 +0200 | [diff] [blame] | 2014 | case IOCB_CMD_POLL: |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2015 | return aio_poll(req, iocb); |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 2016 | default: |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 2017 | pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2018 | return -EINVAL; |
Christoph Hellwig | 89319d31 | 2016-10-30 11:42:03 -0500 | [diff] [blame] | 2019 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2020 | } |
| 2021 | |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 2022 | static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, |
| 2023 | bool compat) |
| 2024 | { |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2025 | struct aio_kiocb *req; |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 2026 | struct iocb iocb; |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2027 | int err; |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 2028 | |
| 2029 | if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb)))) |
| 2030 | return -EFAULT; |
| 2031 | |
Al Viro | 7316b49 | 2019-03-06 18:24:51 -0500 | [diff] [blame] | 2032 | /* enforce forwards compatibility on users */ |
| 2033 | if (unlikely(iocb.aio_reserved2)) { |
| 2034 | pr_debug("EINVAL: reserve field set\n"); |
| 2035 | return -EINVAL; |
| 2036 | } |
| 2037 | |
| 2038 | /* prevent overflows */ |
| 2039 | if (unlikely( |
| 2040 | (iocb.aio_buf != (unsigned long)iocb.aio_buf) || |
| 2041 | (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) || |
| 2042 | ((ssize_t)iocb.aio_nbytes < 0) |
| 2043 | )) { |
| 2044 | pr_debug("EINVAL: overflow check\n"); |
| 2045 | return -EINVAL; |
| 2046 | } |
| 2047 | |
| 2048 | req = aio_get_req(ctx); |
| 2049 | if (unlikely(!req)) |
| 2050 | return -EAGAIN; |
| 2051 | |
| 2052 | err = __io_submit_one(ctx, &iocb, user_iocb, req, compat); |
| 2053 | |
| 2054 | /* Done with the synchronous reference */ |
| 2055 | iocb_put(req); |
| 2056 | |
| 2057 | /* |
| 2058 | * If err is 0, we'd either done aio_complete() ourselves or have |
| 2059 | * arranged for that to be done asynchronously. Anything non-zero |
| 2060 | * means that we need to destroy req ourselves. |
| 2061 | */ |
| 2062 | if (unlikely(err)) { |
| 2063 | iocb_destroy(req); |
| 2064 | put_reqs_available(ctx, 1); |
| 2065 | } |
| 2066 | return err; |
Jens Axboe | 88a6f18 | 2018-11-24 14:46:14 -0700 | [diff] [blame] | 2067 | } |
| 2068 | |
Jeff Moyer | 9d85cba7 | 2010-05-26 14:44:26 -0700 | [diff] [blame] | 2069 | /* sys_io_submit: |
| 2070 | * Queue the nr iocbs pointed to by iocbpp for processing. Returns |
| 2071 | * the number of iocbs queued. May return -EINVAL if the aio_context |
| 2072 | * specified by ctx_id is invalid, if nr is < 0, if the iocb at |
| 2073 | * *iocbpp[0] is not properly initialized, if the operation specified |
| 2074 | * is invalid for the file descriptor in the iocb. May fail with |
| 2075 | * -EFAULT if any of the data structures point to invalid data. May |
| 2076 | * fail with -EBADF if the file descriptor specified in the first |
| 2077 | * iocb is invalid. May fail with -EAGAIN if insufficient resources |
| 2078 | * are available to queue any iocbs. Will return 0 if nr is 0. Will |
| 2079 | * fail with -ENOSYS if not implemented. |
| 2080 | */ |
| 2081 | SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, |
| 2082 | struct iocb __user * __user *, iocbpp) |
| 2083 | { |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2084 | struct kioctx *ctx; |
| 2085 | long ret = 0; |
| 2086 | int i = 0; |
| 2087 | struct blk_plug plug; |
| 2088 | |
| 2089 | if (unlikely(nr < 0)) |
| 2090 | return -EINVAL; |
| 2091 | |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2092 | ctx = lookup_ioctx(ctx_id); |
| 2093 | if (unlikely(!ctx)) { |
| 2094 | pr_debug("EINVAL: invalid context id\n"); |
| 2095 | return -EINVAL; |
| 2096 | } |
| 2097 | |
Al Viro | 1da9277 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2098 | if (nr > ctx->nr_events) |
| 2099 | nr = ctx->nr_events; |
| 2100 | |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 2101 | if (nr > AIO_PLUG_THRESHOLD) |
| 2102 | blk_start_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2103 | for (i = 0; i < nr; i++) { |
| 2104 | struct iocb __user *user_iocb; |
| 2105 | |
| 2106 | if (unlikely(get_user(user_iocb, iocbpp + i))) { |
| 2107 | ret = -EFAULT; |
| 2108 | break; |
| 2109 | } |
| 2110 | |
| 2111 | ret = io_submit_one(ctx, user_iocb, false); |
| 2112 | if (ret) |
| 2113 | break; |
| 2114 | } |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 2115 | if (nr > AIO_PLUG_THRESHOLD) |
| 2116 | blk_finish_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2117 | |
| 2118 | percpu_ref_put(&ctx->users); |
| 2119 | return i ? i : ret; |
Jeff Moyer | 9d85cba7 | 2010-05-26 14:44:26 -0700 | [diff] [blame] | 2120 | } |
| 2121 | |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2122 | #ifdef CONFIG_COMPAT |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2123 | COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id, |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2124 | int, nr, compat_uptr_t __user *, iocbpp) |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2125 | { |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2126 | struct kioctx *ctx; |
| 2127 | long ret = 0; |
| 2128 | int i = 0; |
| 2129 | struct blk_plug plug; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2130 | |
| 2131 | if (unlikely(nr < 0)) |
| 2132 | return -EINVAL; |
| 2133 | |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2134 | ctx = lookup_ioctx(ctx_id); |
| 2135 | if (unlikely(!ctx)) { |
| 2136 | pr_debug("EINVAL: invalid context id\n"); |
| 2137 | return -EINVAL; |
| 2138 | } |
| 2139 | |
Al Viro | 1da9277 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2140 | if (nr > ctx->nr_events) |
| 2141 | nr = ctx->nr_events; |
| 2142 | |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 2143 | if (nr > AIO_PLUG_THRESHOLD) |
| 2144 | blk_start_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2145 | for (i = 0; i < nr; i++) { |
| 2146 | compat_uptr_t user_iocb; |
| 2147 | |
| 2148 | if (unlikely(get_user(user_iocb, iocbpp + i))) { |
| 2149 | ret = -EFAULT; |
| 2150 | break; |
| 2151 | } |
| 2152 | |
| 2153 | ret = io_submit_one(ctx, compat_ptr(user_iocb), true); |
| 2154 | if (ret) |
| 2155 | break; |
| 2156 | } |
Jens Axboe | a79d40e | 2018-12-04 09:45:32 -0700 | [diff] [blame] | 2157 | if (nr > AIO_PLUG_THRESHOLD) |
| 2158 | blk_finish_plug(&plug); |
Al Viro | 67ba049 | 2018-05-26 20:10:07 -0400 | [diff] [blame] | 2159 | |
| 2160 | percpu_ref_put(&ctx->users); |
| 2161 | return i ? i : ret; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2162 | } |
| 2163 | #endif |
| 2164 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2165 | /* sys_io_cancel: |
Bart Van Assche | 28468cb | 2024-03-04 10:29:44 -0800 | [diff] [blame] | 2166 | * Attempts to cancel an iocb previously passed to io_submit. If |
| 2167 | * the operation is successfully cancelled, the resulting event is |
| 2168 | * copied into the memory pointed to by result without being placed |
| 2169 | * into the completion queue and 0 is returned. May fail with |
| 2170 | * -EFAULT if any of the data structures pointed to are invalid. |
| 2171 | * May fail with -EINVAL if aio_context specified by ctx_id is |
| 2172 | * invalid. May fail with -EAGAIN if the iocb specified was not |
| 2173 | * cancelled. Will fail with -ENOSYS if not implemented. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2174 | */ |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 2175 | SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, |
| 2176 | struct io_event __user *, result) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2177 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2178 | struct kioctx *ctx; |
Christoph Hellwig | 04b2fa9 | 2015-02-02 14:49:06 +0100 | [diff] [blame] | 2179 | struct aio_kiocb *kiocb; |
Christoph Hellwig | 888933f | 2018-05-23 14:11:02 +0200 | [diff] [blame] | 2180 | int ret = -EINVAL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2181 | u32 key; |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 2182 | u64 obj = (u64)(unsigned long)iocb; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2183 | |
Christoph Hellwig | f3a2752 | 2018-03-30 11:19:25 +0200 | [diff] [blame] | 2184 | if (unlikely(get_user(key, &iocb->aio_key))) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2185 | return -EFAULT; |
Christoph Hellwig | f3a2752 | 2018-03-30 11:19:25 +0200 | [diff] [blame] | 2186 | if (unlikely(key != KIOCB_KEY)) |
| 2187 | return -EINVAL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2188 | |
| 2189 | ctx = lookup_ioctx(ctx_id); |
| 2190 | if (unlikely(!ctx)) |
| 2191 | return -EINVAL; |
| 2192 | |
| 2193 | spin_lock_irq(&ctx->ctx_lock); |
Al Viro | 833f415 | 2019-03-11 19:00:36 -0400 | [diff] [blame] | 2194 | /* TODO: use a hash or array, this sucks. */ |
| 2195 | list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) { |
Al Viro | a9339b7 | 2019-03-07 19:43:45 -0500 | [diff] [blame] | 2196 | if (kiocb->ki_res.obj == obj) { |
Al Viro | 833f415 | 2019-03-11 19:00:36 -0400 | [diff] [blame] | 2197 | ret = kiocb->ki_cancel(&kiocb->rw); |
| 2198 | list_del_init(&kiocb->ki_list); |
| 2199 | break; |
| 2200 | } |
Christoph Hellwig | 888933f | 2018-05-23 14:11:02 +0200 | [diff] [blame] | 2201 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2202 | spin_unlock_irq(&ctx->ctx_lock); |
| 2203 | |
Bart Van Assche | 28468cb | 2024-03-04 10:29:44 -0800 | [diff] [blame] | 2204 | if (!ret) { |
| 2205 | /* |
| 2206 | * The result argument is no longer used - the io_event is |
| 2207 | * always delivered via the ring buffer. -EINPROGRESS indicates |
| 2208 | * cancellation is progress: |
| 2209 | */ |
| 2210 | ret = -EINPROGRESS; |
| 2211 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2212 | |
Kent Overstreet | 723be6e | 2013-05-28 15:14:48 -0700 | [diff] [blame] | 2213 | percpu_ref_put(&ctx->users); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2214 | |
| 2215 | return ret; |
| 2216 | } |
| 2217 | |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2218 | static long do_io_getevents(aio_context_t ctx_id, |
| 2219 | long min_nr, |
| 2220 | long nr, |
| 2221 | struct io_event __user *events, |
| 2222 | struct timespec64 *ts) |
| 2223 | { |
| 2224 | ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX; |
| 2225 | struct kioctx *ioctx = lookup_ioctx(ctx_id); |
| 2226 | long ret = -EINVAL; |
| 2227 | |
| 2228 | if (likely(ioctx)) { |
| 2229 | if (likely(min_nr <= nr && min_nr >= 0)) |
| 2230 | ret = read_events(ioctx, min_nr, nr, events, until); |
| 2231 | percpu_ref_put(&ioctx->users); |
| 2232 | } |
| 2233 | |
| 2234 | return ret; |
| 2235 | } |
| 2236 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2237 | /* io_getevents: |
| 2238 | * Attempts to read at least min_nr events and up to nr events from |
Satoru Takeuchi | 642b512 | 2010-08-05 11:23:11 -0700 | [diff] [blame] | 2239 | * the completion queue for the aio_context specified by ctx_id. If |
| 2240 | * it succeeds, the number of read events is returned. May fail with |
| 2241 | * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is |
| 2242 | * out of range, if timeout is out of range. May fail with -EFAULT |
| 2243 | * if any of the memory specified is invalid. May return 0 or |
| 2244 | * < min_nr if the timeout specified by timeout has elapsed |
| 2245 | * before sufficient events are available, where timeout == NULL |
| 2246 | * specifies an infinite timeout. Note that the timeout pointed to by |
Jeff Moyer | 6900807 | 2013-05-24 15:55:24 -0700 | [diff] [blame] | 2247 | * timeout is relative. Will fail with -ENOSYS if not implemented. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2248 | */ |
Arnd Bergmann | 3ca47e9 | 2019-04-23 17:43:50 +0200 | [diff] [blame] | 2249 | #ifdef CONFIG_64BIT |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2250 | |
Heiko Carstens | 002c897 | 2009-01-14 14:14:18 +0100 | [diff] [blame] | 2251 | SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, |
| 2252 | long, min_nr, |
| 2253 | long, nr, |
| 2254 | struct io_event __user *, events, |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2255 | struct __kernel_timespec __user *, timeout) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2256 | { |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2257 | struct timespec64 ts; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2258 | int ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2259 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2260 | if (timeout && unlikely(get_timespec64(&ts, timeout))) |
| 2261 | return -EFAULT; |
| 2262 | |
| 2263 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); |
| 2264 | if (!ret && signal_pending(current)) |
| 2265 | ret = -EINTR; |
| 2266 | return ret; |
| 2267 | } |
| 2268 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2269 | #endif |
| 2270 | |
Christoph Hellwig | 9ba546c | 2018-07-11 15:48:46 +0200 | [diff] [blame] | 2271 | struct __aio_sigset { |
| 2272 | const sigset_t __user *sigmask; |
| 2273 | size_t sigsetsize; |
| 2274 | }; |
| 2275 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2276 | SYSCALL_DEFINE6(io_pgetevents, |
| 2277 | aio_context_t, ctx_id, |
| 2278 | long, min_nr, |
| 2279 | long, nr, |
| 2280 | struct io_event __user *, events, |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2281 | struct __kernel_timespec __user *, timeout, |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2282 | const struct __aio_sigset __user *, usig) |
| 2283 | { |
| 2284 | struct __aio_sigset ksig = { NULL, }; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2285 | struct timespec64 ts; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2286 | bool interrupted; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2287 | int ret; |
| 2288 | |
| 2289 | if (timeout && unlikely(get_timespec64(&ts, timeout))) |
| 2290 | return -EFAULT; |
| 2291 | |
| 2292 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2293 | return -EFAULT; |
| 2294 | |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2295 | ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2296 | if (ret) |
| 2297 | return ret; |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2298 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2299 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2300 | |
| 2301 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2302 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2303 | if (interrupted && !ret) |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2304 | ret = -ERESTARTNOHAND; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2305 | |
| 2306 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2307 | } |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2308 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2309 | #if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT) |
| 2310 | |
| 2311 | SYSCALL_DEFINE6(io_pgetevents_time32, |
| 2312 | aio_context_t, ctx_id, |
| 2313 | long, min_nr, |
| 2314 | long, nr, |
| 2315 | struct io_event __user *, events, |
| 2316 | struct old_timespec32 __user *, timeout, |
| 2317 | const struct __aio_sigset __user *, usig) |
| 2318 | { |
| 2319 | struct __aio_sigset ksig = { NULL, }; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2320 | struct timespec64 ts; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2321 | bool interrupted; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2322 | int ret; |
| 2323 | |
| 2324 | if (timeout && unlikely(get_old_timespec32(&ts, timeout))) |
| 2325 | return -EFAULT; |
| 2326 | |
| 2327 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2328 | return -EFAULT; |
| 2329 | |
Deepa Dinamani | ded653c | 2018-09-19 21:41:04 -0700 | [diff] [blame] | 2330 | |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2331 | ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); |
Deepa Dinamani | ded653c | 2018-09-19 21:41:04 -0700 | [diff] [blame] | 2332 | if (ret) |
| 2333 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2334 | |
| 2335 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2336 | |
| 2337 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2338 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2339 | if (interrupted && !ret) |
Deepa Dinamani | 854a6ed | 2018-09-19 21:41:05 -0700 | [diff] [blame] | 2340 | ret = -ERESTARTNOHAND; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2341 | |
| 2342 | return ret; |
| 2343 | } |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2344 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2345 | #endif |
| 2346 | |
| 2347 | #if defined(CONFIG_COMPAT_32BIT_TIME) |
| 2348 | |
Arnd Bergmann | 8dabe72 | 2019-01-07 00:33:08 +0100 | [diff] [blame] | 2349 | SYSCALL_DEFINE5(io_getevents_time32, __u32, ctx_id, |
| 2350 | __s32, min_nr, |
| 2351 | __s32, nr, |
| 2352 | struct io_event __user *, events, |
| 2353 | struct old_timespec32 __user *, timeout) |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2354 | { |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2355 | struct timespec64 t; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2356 | int ret; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2357 | |
Arnd Bergmann | 9afc5ee | 2018-07-13 12:52:28 +0200 | [diff] [blame] | 2358 | if (timeout && get_old_timespec32(&t, timeout)) |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2359 | return -EFAULT; |
| 2360 | |
| 2361 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); |
| 2362 | if (!ret && signal_pending(current)) |
| 2363 | ret = -EINTR; |
| 2364 | return ret; |
| 2365 | } |
| 2366 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2367 | #endif |
| 2368 | |
| 2369 | #ifdef CONFIG_COMPAT |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2370 | |
| 2371 | struct __compat_aio_sigset { |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2372 | compat_uptr_t sigmask; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2373 | compat_size_t sigsetsize; |
| 2374 | }; |
| 2375 | |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2376 | #if defined(CONFIG_COMPAT_32BIT_TIME) |
| 2377 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2378 | COMPAT_SYSCALL_DEFINE6(io_pgetevents, |
| 2379 | compat_aio_context_t, ctx_id, |
| 2380 | compat_long_t, min_nr, |
| 2381 | compat_long_t, nr, |
| 2382 | struct io_event __user *, events, |
Arnd Bergmann | 9afc5ee | 2018-07-13 12:52:28 +0200 | [diff] [blame] | 2383 | struct old_timespec32 __user *, timeout, |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2384 | const struct __compat_aio_sigset __user *, usig) |
| 2385 | { |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2386 | struct __compat_aio_sigset ksig = { 0, }; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2387 | struct timespec64 t; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2388 | bool interrupted; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2389 | int ret; |
| 2390 | |
Arnd Bergmann | 9afc5ee | 2018-07-13 12:52:28 +0200 | [diff] [blame] | 2391 | if (timeout && get_old_timespec32(&t, timeout)) |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2392 | return -EFAULT; |
| 2393 | |
| 2394 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2395 | return -EFAULT; |
| 2396 | |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2397 | ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize); |
Deepa Dinamani | ded653c | 2018-09-19 21:41:04 -0700 | [diff] [blame] | 2398 | if (ret) |
| 2399 | return ret; |
Deepa Dinamani | fa2e62a | 2017-08-04 21:12:32 -0700 | [diff] [blame] | 2400 | |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2401 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2402 | |
| 2403 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2404 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2405 | if (interrupted && !ret) |
Deepa Dinamani | 854a6ed | 2018-09-19 21:41:05 -0700 | [diff] [blame] | 2406 | ret = -ERESTARTNOHAND; |
Christoph Hellwig | 7a074e9 | 2018-05-02 19:51:00 +0200 | [diff] [blame] | 2407 | |
| 2408 | return ret; |
Al Viro | c00d2c7 | 2016-12-20 07:04:57 -0500 | [diff] [blame] | 2409 | } |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2410 | |
| 2411 | #endif |
| 2412 | |
| 2413 | COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64, |
| 2414 | compat_aio_context_t, ctx_id, |
| 2415 | compat_long_t, min_nr, |
| 2416 | compat_long_t, nr, |
| 2417 | struct io_event __user *, events, |
| 2418 | struct __kernel_timespec __user *, timeout, |
| 2419 | const struct __compat_aio_sigset __user *, usig) |
| 2420 | { |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2421 | struct __compat_aio_sigset ksig = { 0, }; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2422 | struct timespec64 t; |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2423 | bool interrupted; |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2424 | int ret; |
| 2425 | |
| 2426 | if (timeout && get_timespec64(&t, timeout)) |
| 2427 | return -EFAULT; |
| 2428 | |
| 2429 | if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) |
| 2430 | return -EFAULT; |
| 2431 | |
Guillem Jover | 97eba80 | 2019-08-21 05:38:20 +0200 | [diff] [blame] | 2432 | ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize); |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2433 | if (ret) |
| 2434 | return ret; |
| 2435 | |
| 2436 | ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2437 | |
| 2438 | interrupted = signal_pending(current); |
Oleg Nesterov | b772434 | 2019-07-16 16:29:53 -0700 | [diff] [blame] | 2439 | restore_saved_sigmask_unless(interrupted); |
Oleg Nesterov | 97abc88 | 2019-06-28 12:06:50 -0700 | [diff] [blame] | 2440 | if (interrupted && !ret) |
Deepa Dinamani | 7a35397 | 2018-09-19 21:41:08 -0700 | [diff] [blame] | 2441 | ret = -ERESTARTNOHAND; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2442 | |
| 2443 | return ret; |
| 2444 | } |
| 2445 | #endif |