blob: 97f9835929256bfd6027d6958e14fb3ec64d34da [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * An async IO implementation for Linux
3 * Written by Benjamin LaHaise <bcrl@kvack.org>
4 *
5 * Implements an efficient asynchronous io interface.
6 *
7 * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved.
Christoph Hellwigbfe40372018-07-16 09:08:20 +02008 * Copyright 2018 Christoph Hellwig.
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 *
10 * See ../COPYING for licensing terms.
11 */
Kent Overstreetcaf41672013-05-07 16:18:35 -070012#define pr_fmt(fmt) "%s: " fmt, __func__
13
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/kernel.h>
15#include <linux/init.h>
16#include <linux/errno.h>
17#include <linux/time.h>
18#include <linux/aio_abi.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050019#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/syscalls.h>
Jens Axboeb9d128f2009-10-29 13:59:26 +010021#include <linux/backing-dev.h>
Christoph Hellwig9018ccc2018-07-24 11:36:37 +020022#include <linux/refcount.h>
Badari Pulavarty027445c2006-09-30 23:28:46 -070023#include <linux/uio.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Ingo Molnar174cd4b2017-02-02 19:15:33 +010025#include <linux/sched/signal.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/fs.h>
27#include <linux/file.h>
28#include <linux/mm.h>
29#include <linux/mman.h>
Michael S. Tsirkin3d2d8272009-09-21 17:03:51 -070030#include <linux/mmu_context.h>
Kent Overstreete1bdd5f2013-04-26 10:58:39 +100031#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032#include <linux/slab.h>
33#include <linux/timer.h>
34#include <linux/aio.h>
35#include <linux/highmem.h>
36#include <linux/workqueue.h>
37#include <linux/security.h>
Davide Libenzi9c3060b2007-05-10 22:23:21 -070038#include <linux/eventfd.h>
Jeff Moyercfb1e332009-10-02 18:57:36 -040039#include <linux/blkdev.h>
Jeff Moyer9d85cba72010-05-26 14:44:26 -070040#include <linux/compat.h>
Gu Zheng36bc08c2013-07-16 17:56:16 +080041#include <linux/migrate.h>
42#include <linux/ramfs.h>
Kent Overstreet723be6e2013-05-28 15:14:48 -070043#include <linux/percpu-refcount.h>
Benjamin LaHaise71ad7492013-09-17 10:18:25 -040044#include <linux/mount.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045
46#include <asm/kmap_types.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080047#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
Al Viro68d70d02013-06-19 15:26:04 +040049#include "internal.h"
50
Christoph Hellwigf3a27522018-03-30 11:19:25 +020051#define KIOCB_KEY 0
52
Kent Overstreet4e179bc2013-05-07 16:18:33 -070053#define AIO_RING_MAGIC 0xa10a10a1
54#define AIO_RING_COMPAT_FEATURES 1
55#define AIO_RING_INCOMPAT_FEATURES 0
56struct aio_ring {
57 unsigned id; /* kernel internal index number */
58 unsigned nr; /* number of io_events */
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -040059 unsigned head; /* Written to by userland or under ring_lock
60 * mutex by aio_read_events_ring(). */
Kent Overstreet4e179bc2013-05-07 16:18:33 -070061 unsigned tail;
62
63 unsigned magic;
64 unsigned compat_features;
65 unsigned incompat_features;
66 unsigned header_length; /* size of aio_ring */
67
68
69 struct io_event io_events[0];
70}; /* 128 bytes + ring size */
71
72#define AIO_RING_PAGES 8
Kent Overstreet4e179bc2013-05-07 16:18:33 -070073
Benjamin LaHaisedb446a02013-07-30 12:54:40 -040074struct kioctx_table {
Tejun Heod0264c02018-03-14 12:10:17 -070075 struct rcu_head rcu;
76 unsigned nr;
77 struct kioctx __rcu *table[];
Benjamin LaHaisedb446a02013-07-30 12:54:40 -040078};
79
Kent Overstreete1bdd5f2013-04-26 10:58:39 +100080struct kioctx_cpu {
81 unsigned reqs_available;
82};
83
Jens Axboedc48e562015-04-15 11:17:23 -060084struct ctx_rq_wait {
85 struct completion comp;
86 atomic_t count;
87};
88
Kent Overstreet4e179bc2013-05-07 16:18:33 -070089struct kioctx {
Kent Overstreet723be6e2013-05-28 15:14:48 -070090 struct percpu_ref users;
Kent Overstreet36f55882013-05-07 16:18:41 -070091 atomic_t dead;
Kent Overstreet4e179bc2013-05-07 16:18:33 -070092
Kent Overstreete34ecee2013-10-10 19:31:47 -070093 struct percpu_ref reqs;
94
Kent Overstreet4e179bc2013-05-07 16:18:33 -070095 unsigned long user_id;
Kent Overstreet4e179bc2013-05-07 16:18:33 -070096
Kent Overstreete1bdd5f2013-04-26 10:58:39 +100097 struct __percpu kioctx_cpu *cpu;
98
99 /*
100 * For percpu reqs_available, number of slots we move to/from global
101 * counter at a time:
102 */
103 unsigned req_batch;
Kent Overstreet3e845ce2013-05-07 16:18:51 -0700104 /*
105 * This is what userspace passed to io_setup(), it's not used for
106 * anything but counting against the global max_reqs quota.
107 *
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700108 * The real limit is nr_events - 1, which will be larger (see
Kent Overstreet3e845ce2013-05-07 16:18:51 -0700109 * aio_setup_ring())
110 */
Kent Overstreet4e179bc2013-05-07 16:18:33 -0700111 unsigned max_reqs;
112
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700113 /* Size of ringbuffer, in units of struct io_event */
114 unsigned nr_events;
Kent Overstreet4e179bc2013-05-07 16:18:33 -0700115
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700116 unsigned long mmap_base;
117 unsigned long mmap_size;
118
119 struct page **ring_pages;
120 long nr_pages;
121
Tejun Heof7298632018-03-14 12:45:15 -0700122 struct rcu_work free_rwork; /* see free_ioctx() */
Kent Overstreet4e23bca2013-05-07 16:18:56 -0700123
Anatol Pomozove02ba722014-04-15 11:31:33 -0700124 /*
125 * signals when all in-flight requests are done
126 */
Jens Axboedc48e562015-04-15 11:17:23 -0600127 struct ctx_rq_wait *rq_wait;
Anatol Pomozove02ba722014-04-15 11:31:33 -0700128
Kent Overstreet4e23bca2013-05-07 16:18:56 -0700129 struct {
Kent Overstreet34e83fc2013-04-26 10:58:39 +1000130 /*
131 * This counts the number of available slots in the ringbuffer,
132 * so we avoid overflowing it: it's decremented (if positive)
133 * when allocating a kiocb and incremented when the resulting
134 * io_event is pulled off the ringbuffer.
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000135 *
136 * We batch accesses to it with a percpu version.
Kent Overstreet34e83fc2013-04-26 10:58:39 +1000137 */
138 atomic_t reqs_available;
Kent Overstreet4e23bca2013-05-07 16:18:56 -0700139 } ____cacheline_aligned_in_smp;
140
141 struct {
142 spinlock_t ctx_lock;
143 struct list_head active_reqs; /* used for cancellation */
144 } ____cacheline_aligned_in_smp;
145
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700146 struct {
147 struct mutex ring_lock;
Kent Overstreet4e23bca2013-05-07 16:18:56 -0700148 wait_queue_head_t wait;
149 } ____cacheline_aligned_in_smp;
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700150
151 struct {
152 unsigned tail;
Benjamin LaHaised856f322014-08-24 13:14:05 -0400153 unsigned completed_events;
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700154 spinlock_t completion_lock;
Kent Overstreet4e23bca2013-05-07 16:18:56 -0700155 } ____cacheline_aligned_in_smp;
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700156
157 struct page *internal_pages[AIO_RING_PAGES];
Gu Zheng36bc08c2013-07-16 17:56:16 +0800158 struct file *aio_ring_file;
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400159
160 unsigned id;
Kent Overstreet4e179bc2013-05-07 16:18:33 -0700161};
162
Christoph Hellwiga3c0d432018-03-27 19:18:57 +0200163struct fsync_iocb {
164 struct work_struct work;
165 struct file *file;
166 bool datasync;
167};
168
Christoph Hellwigbfe40372018-07-16 09:08:20 +0200169struct poll_iocb {
170 struct file *file;
171 struct wait_queue_head *head;
172 __poll_t events;
173 bool woken;
174 bool cancelled;
175 struct wait_queue_entry wait;
176 struct work_struct work;
177};
178
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100179struct aio_kiocb {
Christoph Hellwig54843f82018-05-02 19:57:21 +0200180 union {
181 struct kiocb rw;
Christoph Hellwiga3c0d432018-03-27 19:18:57 +0200182 struct fsync_iocb fsync;
Christoph Hellwigbfe40372018-07-16 09:08:20 +0200183 struct poll_iocb poll;
Christoph Hellwig54843f82018-05-02 19:57:21 +0200184 };
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100185
186 struct kioctx *ki_ctx;
187 kiocb_cancel_fn *ki_cancel;
188
189 struct iocb __user *ki_user_iocb; /* user's aiocb */
190 __u64 ki_user_data; /* user's data for completion */
191
192 struct list_head ki_list; /* the aio core uses this
193 * for cancellation */
Christoph Hellwig9018ccc2018-07-24 11:36:37 +0200194 refcount_t ki_refcnt;
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100195
196 /*
197 * If the aio_resfd field of the userspace iocb is not zero,
198 * this is the underlying eventfd context to deliver events to.
199 */
200 struct eventfd_ctx *ki_eventfd;
201};
202
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203/*------ sysctl variables----*/
Zach Brownd55b5fd2005-11-07 00:59:31 -0800204static DEFINE_SPINLOCK(aio_nr_lock);
205unsigned long aio_nr; /* current system wide number of aio requests */
206unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207/*----end sysctl variables---*/
208
Christoph Lametere18b8902006-12-06 20:33:20 -0800209static struct kmem_cache *kiocb_cachep;
210static struct kmem_cache *kioctx_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400212static struct vfsmount *aio_mnt;
213
214static const struct file_operations aio_ring_fops;
215static const struct address_space_operations aio_ctx_aops;
216
217static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
218{
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400219 struct file *file;
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400220 struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);
Dan Carpenter7f626562013-11-13 10:49:40 +0300221 if (IS_ERR(inode))
222 return ERR_CAST(inode);
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400223
224 inode->i_mapping->a_ops = &aio_ctx_aops;
225 inode->i_mapping->private_data = ctx;
226 inode->i_size = PAGE_SIZE * nr_pages;
227
Al Virod93aa9d2018-06-09 09:40:05 -0400228 file = alloc_file_pseudo(inode, aio_mnt, "[aio]",
229 O_RDWR, &aio_ring_fops);
Al Viroc9c554f2018-07-11 14:19:04 -0400230 if (IS_ERR(file))
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400231 iput(inode);
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400232 return file;
233}
234
235static struct dentry *aio_mount(struct file_system_type *fs_type,
236 int flags, const char *dev_name, void *data)
237{
Al Virod93aa9d2018-06-09 09:40:05 -0400238 struct dentry *root = mount_pseudo(fs_type, "aio:", NULL, NULL,
Jann Horn22f6b4d2016-09-16 00:31:22 +0200239 AIO_RING_MAGIC);
240
241 if (!IS_ERR(root))
242 root->d_sb->s_iflags |= SB_I_NOEXEC;
243 return root;
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400244}
245
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246/* aio_setup
247 * Creates the slab caches used by the aio routines, panic on
248 * failure as this is done early during the boot sequence.
249 */
250static int __init aio_setup(void)
251{
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400252 static struct file_system_type aio_fs = {
253 .name = "aio",
254 .mount = aio_mount,
255 .kill_sb = kill_anon_super,
256 };
257 aio_mnt = kern_mount(&aio_fs);
258 if (IS_ERR(aio_mnt))
259 panic("Failed to create aio fs mount.");
260
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100261 kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
Christoph Lameter0a31bd52007-05-06 14:49:57 -0700262 kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 return 0;
264}
H Hartley Sweeten385773e2009-09-22 16:43:53 -0700265__initcall(aio_setup);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400267static void put_aio_ring_file(struct kioctx *ctx)
268{
269 struct file *aio_ring_file = ctx->aio_ring_file;
Rasmus Villemoesde04e762016-09-15 00:25:03 +0200270 struct address_space *i_mapping;
271
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400272 if (aio_ring_file) {
Al Viro45063092016-12-04 18:24:56 -0500273 truncate_setsize(file_inode(aio_ring_file), 0);
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400274
275 /* Prevent further access to the kioctx from migratepages */
Al Viro45063092016-12-04 18:24:56 -0500276 i_mapping = aio_ring_file->f_mapping;
Rasmus Villemoesde04e762016-09-15 00:25:03 +0200277 spin_lock(&i_mapping->private_lock);
278 i_mapping->private_data = NULL;
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400279 ctx->aio_ring_file = NULL;
Rasmus Villemoesde04e762016-09-15 00:25:03 +0200280 spin_unlock(&i_mapping->private_lock);
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400281
282 fput(aio_ring_file);
283 }
284}
285
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286static void aio_free_ring(struct kioctx *ctx)
287{
Gu Zheng36bc08c2013-07-16 17:56:16 +0800288 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400290 /* Disconnect the kiotx from the ring file. This prevents future
291 * accesses to the kioctx from page migration.
292 */
293 put_aio_ring_file(ctx);
294
Gu Zheng36bc08c2013-07-16 17:56:16 +0800295 for (i = 0; i < ctx->nr_pages; i++) {
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500296 struct page *page;
Gu Zheng36bc08c2013-07-16 17:56:16 +0800297 pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i,
298 page_count(ctx->ring_pages[i]));
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500299 page = ctx->ring_pages[i];
300 if (!page)
301 continue;
302 ctx->ring_pages[i] = NULL;
303 put_page(page);
Gu Zheng36bc08c2013-07-16 17:56:16 +0800304 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305
Sasha Levinddb8c452013-11-19 17:33:03 -0500306 if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700307 kfree(ctx->ring_pages);
Sasha Levinddb8c452013-11-19 17:33:03 -0500308 ctx->ring_pages = NULL;
309 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310}
311
Oleg Nesterov5477e702015-09-04 15:48:04 -0700312static int aio_ring_mremap(struct vm_area_struct *vma)
Gu Zheng36bc08c2013-07-16 17:56:16 +0800313{
Oleg Nesterov5477e702015-09-04 15:48:04 -0700314 struct file *file = vma->vm_file;
Pavel Emelyanove4a0d3e2014-09-18 19:56:17 +0400315 struct mm_struct *mm = vma->vm_mm;
316 struct kioctx_table *table;
Al Virob2edffd2015-04-06 17:48:54 -0400317 int i, res = -EINVAL;
Pavel Emelyanove4a0d3e2014-09-18 19:56:17 +0400318
319 spin_lock(&mm->ioctx_lock);
320 rcu_read_lock();
321 table = rcu_dereference(mm->ioctx_table);
322 for (i = 0; i < table->nr; i++) {
323 struct kioctx *ctx;
324
Tejun Heod0264c02018-03-14 12:10:17 -0700325 ctx = rcu_dereference(table->table[i]);
Pavel Emelyanove4a0d3e2014-09-18 19:56:17 +0400326 if (ctx && ctx->aio_ring_file == file) {
Al Virob2edffd2015-04-06 17:48:54 -0400327 if (!atomic_read(&ctx->dead)) {
328 ctx->user_id = ctx->mmap_base = vma->vm_start;
329 res = 0;
330 }
Pavel Emelyanove4a0d3e2014-09-18 19:56:17 +0400331 break;
332 }
333 }
334
335 rcu_read_unlock();
336 spin_unlock(&mm->ioctx_lock);
Al Virob2edffd2015-04-06 17:48:54 -0400337 return res;
Pavel Emelyanove4a0d3e2014-09-18 19:56:17 +0400338}
339
Oleg Nesterov5477e702015-09-04 15:48:04 -0700340static const struct vm_operations_struct aio_ring_vm_ops = {
341 .mremap = aio_ring_mremap,
342#if IS_ENABLED(CONFIG_MMU)
343 .fault = filemap_fault,
344 .map_pages = filemap_map_pages,
345 .page_mkwrite = filemap_page_mkwrite,
346#endif
347};
348
349static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
350{
351 vma->vm_flags |= VM_DONTEXPAND;
352 vma->vm_ops = &aio_ring_vm_ops;
353 return 0;
354}
355
Gu Zheng36bc08c2013-07-16 17:56:16 +0800356static const struct file_operations aio_ring_fops = {
357 .mmap = aio_ring_mmap,
358};
359
Benjamin LaHaise0c453552013-07-17 09:34:24 -0400360#if IS_ENABLED(CONFIG_MIGRATION)
Gu Zheng36bc08c2013-07-16 17:56:16 +0800361static int aio_migratepage(struct address_space *mapping, struct page *new,
362 struct page *old, enum migrate_mode mode)
363{
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400364 struct kioctx *ctx;
Gu Zheng36bc08c2013-07-16 17:56:16 +0800365 unsigned long flags;
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400366 pgoff_t idx;
Gu Zheng36bc08c2013-07-16 17:56:16 +0800367 int rc;
368
Jérôme Glisse2916ecc2017-09-08 16:12:06 -0700369 /*
370 * We cannot support the _NO_COPY case here, because copy needs to
371 * happen under the ctx->completion_lock. That does not work with the
372 * migration workflow of MIGRATE_SYNC_NO_COPY.
373 */
374 if (mode == MIGRATE_SYNC_NO_COPY)
375 return -EINVAL;
376
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500377 rc = 0;
378
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400379 /* mapping->private_lock here protects against the kioctx teardown. */
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500380 spin_lock(&mapping->private_lock);
381 ctx = mapping->private_data;
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400382 if (!ctx) {
383 rc = -EINVAL;
384 goto out;
385 }
386
387 /* The ring_lock mutex. The prevents aio_read_events() from writing
388 * to the ring's head, and prevents page migration from mucking in
389 * a partially initialized kiotx.
390 */
391 if (!mutex_trylock(&ctx->ring_lock)) {
392 rc = -EAGAIN;
393 goto out;
394 }
395
396 idx = old->index;
397 if (idx < (pgoff_t)ctx->nr_pages) {
398 /* Make sure the old page hasn't already been changed */
399 if (ctx->ring_pages[idx] != old)
400 rc = -EAGAIN;
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500401 } else
402 rc = -EINVAL;
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500403
404 if (rc != 0)
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400405 goto out_unlock;
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500406
Gu Zheng36bc08c2013-07-16 17:56:16 +0800407 /* Writeback must be complete */
408 BUG_ON(PageWriteback(old));
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500409 get_page(new);
Gu Zheng36bc08c2013-07-16 17:56:16 +0800410
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500411 rc = migrate_page_move_mapping(mapping, new, old, NULL, mode, 1);
Gu Zheng36bc08c2013-07-16 17:56:16 +0800412 if (rc != MIGRATEPAGE_SUCCESS) {
Benjamin LaHaise8e321fe2013-12-21 17:56:08 -0500413 put_page(new);
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400414 goto out_unlock;
Gu Zheng36bc08c2013-07-16 17:56:16 +0800415 }
416
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400417 /* Take completion_lock to prevent other writes to the ring buffer
418 * while the old page is copied to the new. This prevents new
419 * events from being lost.
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400420 */
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400421 spin_lock_irqsave(&ctx->completion_lock, flags);
422 migrate_page_copy(new, old);
423 BUG_ON(ctx->ring_pages[idx] != old);
424 ctx->ring_pages[idx] = new;
425 spin_unlock_irqrestore(&ctx->completion_lock, flags);
426
427 /* The old page is no longer accessible. */
428 put_page(old);
429
430out_unlock:
431 mutex_unlock(&ctx->ring_lock);
432out:
Benjamin LaHaise5e9ae2e2013-09-26 20:34:51 -0400433 spin_unlock(&mapping->private_lock);
Gu Zheng36bc08c2013-07-16 17:56:16 +0800434 return rc;
435}
Benjamin LaHaise0c453552013-07-17 09:34:24 -0400436#endif
Gu Zheng36bc08c2013-07-16 17:56:16 +0800437
438static const struct address_space_operations aio_ctx_aops = {
Gu Zheng835f2522014-11-06 17:46:21 +0800439 .set_page_dirty = __set_page_dirty_no_writeback,
Benjamin LaHaise0c453552013-07-17 09:34:24 -0400440#if IS_ENABLED(CONFIG_MIGRATION)
Gu Zheng36bc08c2013-07-16 17:56:16 +0800441 .migratepage = aio_migratepage,
Benjamin LaHaise0c453552013-07-17 09:34:24 -0400442#endif
Gu Zheng36bc08c2013-07-16 17:56:16 +0800443};
444
Mauricio Faria de Oliveira2a8a9862017-07-05 10:53:16 -0300445static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446{
447 struct aio_ring *ring;
Zach Brown41003a72013-05-07 16:18:25 -0700448 struct mm_struct *mm = current->mm;
Linus Torvalds3dc9acb2013-12-20 05:11:12 +0900449 unsigned long size, unused;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 int nr_pages;
Gu Zheng36bc08c2013-07-16 17:56:16 +0800451 int i;
452 struct file *file;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
454 /* Compensate for the ring buffer's head/tail overlap entry */
455 nr_events += 2; /* 1 is required, 2 for good luck */
456
457 size = sizeof(struct aio_ring);
458 size += sizeof(struct io_event) * nr_events;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459
Gu Zheng36bc08c2013-07-16 17:56:16 +0800460 nr_pages = PFN_UP(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 if (nr_pages < 0)
462 return -EINVAL;
463
Benjamin LaHaise71ad7492013-09-17 10:18:25 -0400464 file = aio_private_file(ctx, nr_pages);
Gu Zheng36bc08c2013-07-16 17:56:16 +0800465 if (IS_ERR(file)) {
466 ctx->aio_ring_file = NULL;
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400467 return -ENOMEM;
Gu Zheng36bc08c2013-07-16 17:56:16 +0800468 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
Gu Zheng36bc08c2013-07-16 17:56:16 +0800470 ctx->aio_ring_file = file;
471 nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))
472 / sizeof(struct io_event);
473
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700474 ctx->ring_pages = ctx->internal_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 if (nr_pages > AIO_RING_PAGES) {
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700476 ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),
477 GFP_KERNEL);
Gu Zhengd1b94322013-12-04 18:19:06 +0800478 if (!ctx->ring_pages) {
479 put_aio_ring_file(ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 return -ENOMEM;
Gu Zhengd1b94322013-12-04 18:19:06 +0800481 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482 }
483
Linus Torvalds3dc9acb2013-12-20 05:11:12 +0900484 for (i = 0; i < nr_pages; i++) {
485 struct page *page;
Al Viro45063092016-12-04 18:24:56 -0500486 page = find_or_create_page(file->f_mapping,
Linus Torvalds3dc9acb2013-12-20 05:11:12 +0900487 i, GFP_HIGHUSER | __GFP_ZERO);
488 if (!page)
489 break;
490 pr_debug("pid(%d) page[%d]->count=%d\n",
491 current->pid, i, page_count(page));
492 SetPageUptodate(page);
Linus Torvalds3dc9acb2013-12-20 05:11:12 +0900493 unlock_page(page);
494
495 ctx->ring_pages[i] = page;
496 }
497 ctx->nr_pages = i;
498
499 if (unlikely(i != nr_pages)) {
500 aio_free_ring(ctx);
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400501 return -ENOMEM;
Linus Torvalds3dc9acb2013-12-20 05:11:12 +0900502 }
503
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700504 ctx->mmap_size = nr_pages * PAGE_SIZE;
505 pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);
Gu Zheng36bc08c2013-07-16 17:56:16 +0800506
Michal Hocko013373e2016-05-23 16:25:59 -0700507 if (down_write_killable(&mm->mmap_sem)) {
508 ctx->mmap_size = 0;
509 aio_free_ring(ctx);
510 return -EINTR;
511 }
512
Gu Zheng36bc08c2013-07-16 17:56:16 +0800513 ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size,
514 PROT_READ | PROT_WRITE,
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800515 MAP_SHARED, 0, &unused, NULL);
Linus Torvalds3dc9acb2013-12-20 05:11:12 +0900516 up_write(&mm->mmap_sem);
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700517 if (IS_ERR((void *)ctx->mmap_base)) {
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700518 ctx->mmap_size = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 aio_free_ring(ctx);
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400520 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 }
522
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700523 pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);
Benjamin LaHaised6c355c2013-09-09 11:57:59 -0400524
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700525 ctx->user_id = ctx->mmap_base;
526 ctx->nr_events = nr_events; /* trusted copy */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700528 ring = kmap_atomic(ctx->ring_pages[0]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 ring->nr = nr_events; /* user copy */
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400530 ring->id = ~0U;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 ring->head = ring->tail = 0;
532 ring->magic = AIO_RING_MAGIC;
533 ring->compat_features = AIO_RING_COMPAT_FEATURES;
534 ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
535 ring->header_length = sizeof(struct aio_ring);
Cong Wange8e3c3d2011-11-25 23:14:27 +0800536 kunmap_atomic(ring);
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700537 flush_dcache_page(ctx->ring_pages[0]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538
539 return 0;
540}
541
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542#define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
543#define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
544#define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
545
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100546void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
Kent Overstreet0460fef2013-05-07 16:18:49 -0700547{
Christoph Hellwig54843f82018-05-02 19:57:21 +0200548 struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw);
Kent Overstreet0460fef2013-05-07 16:18:49 -0700549 struct kioctx *ctx = req->ki_ctx;
550 unsigned long flags;
551
Christoph Hellwig75321b52018-04-09 14:57:56 +0200552 if (WARN_ON_ONCE(!list_empty(&req->ki_list)))
553 return;
554
Kent Overstreet0460fef2013-05-07 16:18:49 -0700555 spin_lock_irqsave(&ctx->ctx_lock, flags);
Christoph Hellwig75321b52018-04-09 14:57:56 +0200556 list_add_tail(&req->ki_list, &ctx->active_reqs);
Kent Overstreet0460fef2013-05-07 16:18:49 -0700557 req->ki_cancel = cancel;
Kent Overstreet0460fef2013-05-07 16:18:49 -0700558 spin_unlock_irqrestore(&ctx->ctx_lock, flags);
559}
560EXPORT_SYMBOL(kiocb_set_cancel_fn);
561
Tejun Heoa6d7cff2018-03-14 12:10:17 -0700562/*
563 * free_ioctx() should be RCU delayed to synchronize against the RCU
564 * protected lookup_ioctx() and also needs process context to call
Tejun Heof7298632018-03-14 12:45:15 -0700565 * aio_free_ring(). Use rcu_work.
Tejun Heoa6d7cff2018-03-14 12:10:17 -0700566 */
Kent Overstreete34ecee2013-10-10 19:31:47 -0700567static void free_ioctx(struct work_struct *work)
Kent Overstreet36f55882013-05-07 16:18:41 -0700568{
Tejun Heof7298632018-03-14 12:45:15 -0700569 struct kioctx *ctx = container_of(to_rcu_work(work), struct kioctx,
570 free_rwork);
Kent Overstreete34ecee2013-10-10 19:31:47 -0700571 pr_debug("freeing %p\n", ctx);
572
573 aio_free_ring(ctx);
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000574 free_percpu(ctx->cpu);
Tejun Heo9a1049d2014-06-28 08:10:14 -0400575 percpu_ref_exit(&ctx->reqs);
576 percpu_ref_exit(&ctx->users);
Kent Overstreet36f55882013-05-07 16:18:41 -0700577 kmem_cache_free(kioctx_cachep, ctx);
578}
579
Kent Overstreete34ecee2013-10-10 19:31:47 -0700580static void free_ioctx_reqs(struct percpu_ref *ref)
581{
582 struct kioctx *ctx = container_of(ref, struct kioctx, reqs);
583
Anatol Pomozove02ba722014-04-15 11:31:33 -0700584 /* At this point we know that there are no any in-flight requests */
Jens Axboedc48e562015-04-15 11:17:23 -0600585 if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count))
586 complete(&ctx->rq_wait->comp);
Anatol Pomozove02ba722014-04-15 11:31:33 -0700587
Tejun Heoa6d7cff2018-03-14 12:10:17 -0700588 /* Synchronize against RCU protected table->table[] dereferences */
Tejun Heof7298632018-03-14 12:45:15 -0700589 INIT_RCU_WORK(&ctx->free_rwork, free_ioctx);
590 queue_rcu_work(system_wq, &ctx->free_rwork);
Kent Overstreete34ecee2013-10-10 19:31:47 -0700591}
592
Kent Overstreet36f55882013-05-07 16:18:41 -0700593/*
594 * When this function runs, the kioctx has been removed from the "hash table"
595 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
596 * now it's safe to cancel any that need to be.
597 */
Kent Overstreete34ecee2013-10-10 19:31:47 -0700598static void free_ioctx_users(struct percpu_ref *ref)
Kent Overstreet36f55882013-05-07 16:18:41 -0700599{
Kent Overstreete34ecee2013-10-10 19:31:47 -0700600 struct kioctx *ctx = container_of(ref, struct kioctx, users);
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100601 struct aio_kiocb *req;
Kent Overstreet36f55882013-05-07 16:18:41 -0700602
603 spin_lock_irq(&ctx->ctx_lock);
604
605 while (!list_empty(&ctx->active_reqs)) {
606 req = list_first_entry(&ctx->active_reqs,
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100607 struct aio_kiocb, ki_list);
Christoph Hellwig888933f2018-05-23 14:11:02 +0200608 req->ki_cancel(&req->rw);
Al Viro4faa99962018-05-23 22:53:22 -0400609 list_del_init(&req->ki_list);
Kent Overstreet36f55882013-05-07 16:18:41 -0700610 }
611
612 spin_unlock_irq(&ctx->ctx_lock);
613
Kent Overstreete34ecee2013-10-10 19:31:47 -0700614 percpu_ref_kill(&ctx->reqs);
615 percpu_ref_put(&ctx->reqs);
Kent Overstreet36f55882013-05-07 16:18:41 -0700616}
617
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400618static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
619{
620 unsigned i, new_nr;
621 struct kioctx_table *table, *old;
622 struct aio_ring *ring;
623
624 spin_lock(&mm->ioctx_lock);
Oleg Nesterov855ef0d2014-04-30 16:16:36 +0200625 table = rcu_dereference_raw(mm->ioctx_table);
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400626
627 while (1) {
628 if (table)
629 for (i = 0; i < table->nr; i++)
Tejun Heod0264c02018-03-14 12:10:17 -0700630 if (!rcu_access_pointer(table->table[i])) {
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400631 ctx->id = i;
Tejun Heod0264c02018-03-14 12:10:17 -0700632 rcu_assign_pointer(table->table[i], ctx);
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400633 spin_unlock(&mm->ioctx_lock);
634
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400635 /* While kioctx setup is in progress,
636 * we are protected from page migration
637 * changes ring_pages by ->ring_lock.
638 */
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400639 ring = kmap_atomic(ctx->ring_pages[0]);
640 ring->id = ctx->id;
641 kunmap_atomic(ring);
642 return 0;
643 }
644
645 new_nr = (table ? table->nr : 1) * 4;
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400646 spin_unlock(&mm->ioctx_lock);
647
648 table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) *
649 new_nr, GFP_KERNEL);
650 if (!table)
651 return -ENOMEM;
652
653 table->nr = new_nr;
654
655 spin_lock(&mm->ioctx_lock);
Oleg Nesterov855ef0d2014-04-30 16:16:36 +0200656 old = rcu_dereference_raw(mm->ioctx_table);
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400657
658 if (!old) {
659 rcu_assign_pointer(mm->ioctx_table, table);
660 } else if (table->nr > old->nr) {
661 memcpy(table->table, old->table,
662 old->nr * sizeof(struct kioctx *));
663
664 rcu_assign_pointer(mm->ioctx_table, table);
665 kfree_rcu(old, rcu);
666 } else {
667 kfree(table);
668 table = old;
669 }
670 }
671}
672
Kent Overstreete34ecee2013-10-10 19:31:47 -0700673static void aio_nr_sub(unsigned nr)
674{
675 spin_lock(&aio_nr_lock);
676 if (WARN_ON(aio_nr - nr > aio_nr))
677 aio_nr = 0;
678 else
679 aio_nr -= nr;
680 spin_unlock(&aio_nr_lock);
681}
682
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683/* ioctx_alloc
684 * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed.
685 */
686static struct kioctx *ioctx_alloc(unsigned nr_events)
687{
Zach Brown41003a72013-05-07 16:18:25 -0700688 struct mm_struct *mm = current->mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 struct kioctx *ctx;
Al Viroe23754f2012-03-06 14:33:22 -0500690 int err = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000692 /*
Mauricio Faria de Oliveira2a8a9862017-07-05 10:53:16 -0300693 * Store the original nr_events -- what userspace passed to io_setup(),
694 * for counting against the global limit -- before it changes.
695 */
696 unsigned int max_reqs = nr_events;
697
698 /*
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000699 * We keep track of the number of available ringbuffer slots, to prevent
700 * overflow (reqs_available), and we also use percpu counters for this.
701 *
702 * So since up to half the slots might be on other cpu's percpu counters
703 * and unavailable, double nr_events so userspace sees what they
704 * expected: additionally, we move req_batch slots to/from percpu
705 * counters at a time, so make sure that isn't 0:
706 */
707 nr_events = max(nr_events, num_possible_cpus() * 4);
708 nr_events *= 2;
709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 /* Prevent overflows */
Al Viro08397ac2015-03-31 11:43:52 -0400711 if (nr_events > (0x10000000U / sizeof(struct io_event))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 pr_debug("ENOMEM: nr_events too high\n");
713 return ERR_PTR(-EINVAL);
714 }
715
Mauricio Faria de Oliveira2a8a9862017-07-05 10:53:16 -0300716 if (!nr_events || (unsigned long)max_reqs > aio_max_nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 return ERR_PTR(-EAGAIN);
718
Robert P. J. Dayc3762222007-02-10 01:45:03 -0800719 ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 if (!ctx)
721 return ERR_PTR(-ENOMEM);
722
Mauricio Faria de Oliveira2a8a9862017-07-05 10:53:16 -0300723 ctx->max_reqs = max_reqs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400725 spin_lock_init(&ctx->ctx_lock);
726 spin_lock_init(&ctx->completion_lock);
727 mutex_init(&ctx->ring_lock);
728 /* Protect against page migration throughout kiotx setup by keeping
729 * the ring_lock mutex held until setup is complete. */
730 mutex_lock(&ctx->ring_lock);
731 init_waitqueue_head(&ctx->wait);
732
733 INIT_LIST_HEAD(&ctx->active_reqs);
734
Tejun Heo2aad2a82014-09-24 13:31:50 -0400735 if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL))
Kent Overstreete34ecee2013-10-10 19:31:47 -0700736 goto err;
737
Tejun Heo2aad2a82014-09-24 13:31:50 -0400738 if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL))
Kent Overstreete34ecee2013-10-10 19:31:47 -0700739 goto err;
Kent Overstreet723be6e2013-05-28 15:14:48 -0700740
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000741 ctx->cpu = alloc_percpu(struct kioctx_cpu);
742 if (!ctx->cpu)
Kent Overstreete34ecee2013-10-10 19:31:47 -0700743 goto err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744
Mauricio Faria de Oliveira2a8a9862017-07-05 10:53:16 -0300745 err = aio_setup_ring(ctx, nr_events);
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400746 if (err < 0)
Kent Overstreete34ecee2013-10-10 19:31:47 -0700747 goto err;
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000748
Kent Overstreet34e83fc2013-04-26 10:58:39 +1000749 atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000750 ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
Benjamin LaHaise6878ea72013-07-31 10:34:18 -0400751 if (ctx->req_batch < 1)
752 ctx->req_batch = 1;
Kent Overstreet34e83fc2013-04-26 10:58:39 +1000753
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 /* limit the number of system wide aios */
Al Viro9fa1cb32012-03-10 23:14:05 -0500755 spin_lock(&aio_nr_lock);
Mauricio Faria de Oliveira2a8a9862017-07-05 10:53:16 -0300756 if (aio_nr + ctx->max_reqs > aio_max_nr ||
757 aio_nr + ctx->max_reqs < aio_nr) {
Al Viro9fa1cb32012-03-10 23:14:05 -0500758 spin_unlock(&aio_nr_lock);
Kent Overstreete34ecee2013-10-10 19:31:47 -0700759 err = -EAGAIN;
Gu Zhengd1b94322013-12-04 18:19:06 +0800760 goto err_ctx;
Al Viro2dd542b72012-03-10 23:10:35 -0500761 }
762 aio_nr += ctx->max_reqs;
Al Viro9fa1cb32012-03-10 23:14:05 -0500763 spin_unlock(&aio_nr_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764
Benjamin LaHaise18816862013-12-21 15:49:28 -0500765 percpu_ref_get(&ctx->users); /* io_setup() will drop this ref */
766 percpu_ref_get(&ctx->reqs); /* free_ioctx_users() will drop this */
Kent Overstreet723be6e2013-05-28 15:14:48 -0700767
Benjamin LaHaiseda903822013-08-05 13:21:43 -0400768 err = ioctx_add_table(ctx, mm);
769 if (err)
Kent Overstreete34ecee2013-10-10 19:31:47 -0700770 goto err_cleanup;
Benjamin LaHaiseda903822013-08-05 13:21:43 -0400771
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400772 /* Release the ring_lock mutex now that all setup is complete. */
773 mutex_unlock(&ctx->ring_lock);
774
Kent Overstreetcaf41672013-05-07 16:18:35 -0700775 pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700776 ctx, ctx->user_id, mm, ctx->nr_events);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 return ctx;
778
Kent Overstreete34ecee2013-10-10 19:31:47 -0700779err_cleanup:
780 aio_nr_sub(ctx->max_reqs);
Gu Zhengd1b94322013-12-04 18:19:06 +0800781err_ctx:
Al Virodeeb8522015-04-06 17:57:44 -0400782 atomic_set(&ctx->dead, 1);
783 if (ctx->mmap_size)
784 vm_munmap(ctx->mmap_base, ctx->mmap_size);
Gu Zhengd1b94322013-12-04 18:19:06 +0800785 aio_free_ring(ctx);
Kent Overstreete34ecee2013-10-10 19:31:47 -0700786err:
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -0400787 mutex_unlock(&ctx->ring_lock);
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000788 free_percpu(ctx->cpu);
Tejun Heo9a1049d2014-06-28 08:10:14 -0400789 percpu_ref_exit(&ctx->reqs);
790 percpu_ref_exit(&ctx->users);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 kmem_cache_free(kioctx_cachep, ctx);
Kent Overstreetcaf41672013-05-07 16:18:35 -0700792 pr_debug("error allocating ioctx %d\n", err);
Al Viroe23754f2012-03-06 14:33:22 -0500793 return ERR_PTR(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794}
795
Kent Overstreet36f55882013-05-07 16:18:41 -0700796/* kill_ioctx
797 * Cancels all outstanding aio requests on an aio context. Used
798 * when the processes owning a context have all exited to encourage
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 * the rapid destruction of the kioctx.
800 */
Benjamin LaHaisefb2d4482014-04-29 12:45:17 -0400801static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
Jens Axboedc48e562015-04-15 11:17:23 -0600802 struct ctx_rq_wait *wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803{
Benjamin LaHaisefa88b6f2014-04-29 12:55:48 -0400804 struct kioctx_table *table;
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400805
Benjamin LaHaisefa88b6f2014-04-29 12:55:48 -0400806 spin_lock(&mm->ioctx_lock);
Al Virob2edffd2015-04-06 17:48:54 -0400807 if (atomic_xchg(&ctx->dead, 1)) {
808 spin_unlock(&mm->ioctx_lock);
809 return -EINVAL;
810 }
811
Oleg Nesterov855ef0d2014-04-30 16:16:36 +0200812 table = rcu_dereference_raw(mm->ioctx_table);
Tejun Heod0264c02018-03-14 12:10:17 -0700813 WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id]));
814 RCU_INIT_POINTER(table->table[ctx->id], NULL);
Benjamin LaHaisefa88b6f2014-04-29 12:55:48 -0400815 spin_unlock(&mm->ioctx_lock);
Kent Overstreet4fcc7122013-06-12 14:04:59 -0700816
Tejun Heoa6d7cff2018-03-14 12:10:17 -0700817 /* free_ioctx_reqs() will do the necessary RCU synchronization */
Benjamin LaHaisefa88b6f2014-04-29 12:55:48 -0400818 wake_up_all(&ctx->wait);
Kent Overstreet4fcc7122013-06-12 14:04:59 -0700819
Benjamin LaHaisefa88b6f2014-04-29 12:55:48 -0400820 /*
821 * It'd be more correct to do this in free_ioctx(), after all
822 * the outstanding kiocbs have finished - but by then io_destroy
823 * has already returned, so io_setup() could potentially return
824 * -EAGAIN with no ioctxs actually in use (as far as userspace
825 * could tell).
826 */
827 aio_nr_sub(ctx->max_reqs);
Benjamin LaHaisefb2d4482014-04-29 12:45:17 -0400828
Benjamin LaHaisefa88b6f2014-04-29 12:55:48 -0400829 if (ctx->mmap_size)
830 vm_munmap(ctx->mmap_base, ctx->mmap_size);
831
Jens Axboedc48e562015-04-15 11:17:23 -0600832 ctx->rq_wait = wait;
Benjamin LaHaisefa88b6f2014-04-29 12:55:48 -0400833 percpu_ref_kill(&ctx->users);
834 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835}
836
Kent Overstreet36f55882013-05-07 16:18:41 -0700837/*
838 * exit_aio: called when the last user of mm goes away. At this point, there is
839 * no way for any new requests to be submited or any of the io_* syscalls to be
840 * called on the context.
841 *
842 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
843 * them.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844 */
Harvey Harrisonfc9b52c2008-02-08 04:19:52 -0800845void exit_aio(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846{
Oleg Nesterov4b70ac52014-04-30 19:02:48 +0200847 struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table);
Jens Axboedc48e562015-04-15 11:17:23 -0600848 struct ctx_rq_wait wait;
849 int i, skipped;
Jens Axboeabf137d2008-12-09 08:11:22 +0100850
Oleg Nesterov4b70ac52014-04-30 19:02:48 +0200851 if (!table)
852 return;
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400853
Jens Axboedc48e562015-04-15 11:17:23 -0600854 atomic_set(&wait.count, table->nr);
855 init_completion(&wait.comp);
856
857 skipped = 0;
Oleg Nesterov4b70ac52014-04-30 19:02:48 +0200858 for (i = 0; i < table->nr; ++i) {
Tejun Heod0264c02018-03-14 12:10:17 -0700859 struct kioctx *ctx =
860 rcu_dereference_protected(table->table[i], true);
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400861
Jens Axboedc48e562015-04-15 11:17:23 -0600862 if (!ctx) {
863 skipped++;
Oleg Nesterov4b70ac52014-04-30 19:02:48 +0200864 continue;
Jens Axboedc48e562015-04-15 11:17:23 -0600865 }
866
Al Viro936af152012-04-20 21:49:41 -0400867 /*
Oleg Nesterov4b70ac52014-04-30 19:02:48 +0200868 * We don't need to bother with munmap() here - exit_mmap(mm)
869 * is coming and it'll unmap everything. And we simply can't,
870 * this is not necessarily our ->mm.
871 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
872 * that it needs to unmap the area, just set it to 0.
Al Viro936af152012-04-20 21:49:41 -0400873 */
Kent Overstreet58c85dc2013-05-07 16:18:55 -0700874 ctx->mmap_size = 0;
Jens Axboedc48e562015-04-15 11:17:23 -0600875 kill_ioctx(mm, ctx, &wait);
876 }
Kent Overstreet36f55882013-05-07 16:18:41 -0700877
Jens Axboedc48e562015-04-15 11:17:23 -0600878 if (!atomic_sub_and_test(skipped, &wait.count)) {
Gu Zheng6098b452014-09-03 17:45:44 +0800879 /* Wait until all IO for the context are done. */
Jens Axboedc48e562015-04-15 11:17:23 -0600880 wait_for_completion(&wait.comp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881 }
Oleg Nesterov4b70ac52014-04-30 19:02:48 +0200882
883 RCU_INIT_POINTER(mm->ioctx_table, NULL);
884 kfree(table);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885}
886
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000887static void put_reqs_available(struct kioctx *ctx, unsigned nr)
888{
889 struct kioctx_cpu *kcpu;
Benjamin LaHaise263782c2014-07-14 12:49:26 -0400890 unsigned long flags;
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000891
Benjamin LaHaise263782c2014-07-14 12:49:26 -0400892 local_irq_save(flags);
Benjamin LaHaisebe6fb452014-07-22 09:56:56 -0400893 kcpu = this_cpu_ptr(ctx->cpu);
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000894 kcpu->reqs_available += nr;
Benjamin LaHaise263782c2014-07-14 12:49:26 -0400895
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000896 while (kcpu->reqs_available >= ctx->req_batch * 2) {
897 kcpu->reqs_available -= ctx->req_batch;
898 atomic_add(ctx->req_batch, &ctx->reqs_available);
899 }
900
Benjamin LaHaise263782c2014-07-14 12:49:26 -0400901 local_irq_restore(flags);
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000902}
903
904static bool get_reqs_available(struct kioctx *ctx)
905{
906 struct kioctx_cpu *kcpu;
907 bool ret = false;
Benjamin LaHaise263782c2014-07-14 12:49:26 -0400908 unsigned long flags;
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000909
Benjamin LaHaise263782c2014-07-14 12:49:26 -0400910 local_irq_save(flags);
Benjamin LaHaisebe6fb452014-07-22 09:56:56 -0400911 kcpu = this_cpu_ptr(ctx->cpu);
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000912 if (!kcpu->reqs_available) {
913 int old, avail = atomic_read(&ctx->reqs_available);
914
915 do {
916 if (avail < ctx->req_batch)
917 goto out;
918
919 old = avail;
920 avail = atomic_cmpxchg(&ctx->reqs_available,
921 avail, avail - ctx->req_batch);
922 } while (avail != old);
923
924 kcpu->reqs_available += ctx->req_batch;
925 }
926
927 ret = true;
928 kcpu->reqs_available--;
929out:
Benjamin LaHaise263782c2014-07-14 12:49:26 -0400930 local_irq_restore(flags);
Kent Overstreete1bdd5f2013-04-26 10:58:39 +1000931 return ret;
932}
933
Benjamin LaHaised856f322014-08-24 13:14:05 -0400934/* refill_reqs_available
935 * Updates the reqs_available reference counts used for tracking the
936 * number of free slots in the completion ring. This can be called
937 * from aio_complete() (to optimistically update reqs_available) or
938 * from aio_get_req() (the we're out of events case). It must be
939 * called holding ctx->completion_lock.
940 */
941static void refill_reqs_available(struct kioctx *ctx, unsigned head,
942 unsigned tail)
943{
944 unsigned events_in_ring, completed;
945
946 /* Clamp head since userland can write to it. */
947 head %= ctx->nr_events;
948 if (head <= tail)
949 events_in_ring = tail - head;
950 else
951 events_in_ring = ctx->nr_events - (head - tail);
952
953 completed = ctx->completed_events;
954 if (events_in_ring < completed)
955 completed -= events_in_ring;
956 else
957 completed = 0;
958
959 if (!completed)
960 return;
961
962 ctx->completed_events -= completed;
963 put_reqs_available(ctx, completed);
964}
965
966/* user_refill_reqs_available
967 * Called to refill reqs_available when aio_get_req() encounters an
968 * out of space in the completion ring.
969 */
970static void user_refill_reqs_available(struct kioctx *ctx)
971{
972 spin_lock_irq(&ctx->completion_lock);
973 if (ctx->completed_events) {
974 struct aio_ring *ring;
975 unsigned head;
976
977 /* Access of ring->head may race with aio_read_events_ring()
978 * here, but that's okay since whether we read the old version
979 * or the new version, and either will be valid. The important
980 * part is that head cannot pass tail since we prevent
981 * aio_complete() from updating tail by holding
982 * ctx->completion_lock. Even if head is invalid, the check
983 * against ctx->completed_events below will make sure we do the
984 * safe/right thing.
985 */
986 ring = kmap_atomic(ctx->ring_pages[0]);
987 head = ring->head;
988 kunmap_atomic(ring);
989
990 refill_reqs_available(ctx, head, ctx->tail);
991 }
992
993 spin_unlock_irq(&ctx->completion_lock);
994}
995
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996/* aio_get_req
Kent Overstreet57282d82013-05-13 13:42:52 -0700997 * Allocate a slot for an aio request.
998 * Returns NULL if no requests are free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 */
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001000static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001{
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001002 struct aio_kiocb *req;
Kent Overstreeta1c8eae2013-05-07 16:18:53 -07001003
Benjamin LaHaised856f322014-08-24 13:14:05 -04001004 if (!get_reqs_available(ctx)) {
1005 user_refill_reqs_available(ctx);
1006 if (!get_reqs_available(ctx))
1007 return NULL;
1008 }
Kent Overstreeta1c8eae2013-05-07 16:18:53 -07001009
Kent Overstreet0460fef2013-05-07 16:18:49 -07001010 req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 if (unlikely(!req))
Kent Overstreeta1c8eae2013-05-07 16:18:53 -07001012 goto out_put;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013
Kent Overstreete34ecee2013-10-10 19:31:47 -07001014 percpu_ref_get(&ctx->reqs);
Christoph Hellwig75321b52018-04-09 14:57:56 +02001015 INIT_LIST_HEAD(&req->ki_list);
Christoph Hellwig9018ccc2018-07-24 11:36:37 +02001016 refcount_set(&req->ki_refcnt, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 req->ki_ctx = ctx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 return req;
Kent Overstreeta1c8eae2013-05-07 16:18:53 -07001019out_put:
Kent Overstreete1bdd5f2013-04-26 10:58:39 +10001020 put_reqs_available(ctx, 1);
Kent Overstreeta1c8eae2013-05-07 16:18:53 -07001021 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022}
1023
Adrian Bunkd5470b52008-04-29 00:58:57 -07001024static struct kioctx *lookup_ioctx(unsigned long ctx_id)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025{
Benjamin LaHaisedb446a02013-07-30 12:54:40 -04001026 struct aio_ring __user *ring = (void __user *)ctx_id;
Jens Axboeabf137d2008-12-09 08:11:22 +01001027 struct mm_struct *mm = current->mm;
Jeff Moyer65c24492009-03-18 17:04:21 -07001028 struct kioctx *ctx, *ret = NULL;
Benjamin LaHaisedb446a02013-07-30 12:54:40 -04001029 struct kioctx_table *table;
1030 unsigned id;
1031
1032 if (get_user(id, &ring->id))
1033 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
Jens Axboeabf137d2008-12-09 08:11:22 +01001035 rcu_read_lock();
Benjamin LaHaisedb446a02013-07-30 12:54:40 -04001036 table = rcu_dereference(mm->ioctx_table);
Jens Axboeabf137d2008-12-09 08:11:22 +01001037
Benjamin LaHaisedb446a02013-07-30 12:54:40 -04001038 if (!table || id >= table->nr)
1039 goto out;
1040
Tejun Heod0264c02018-03-14 12:10:17 -07001041 ctx = rcu_dereference(table->table[id]);
Benjamin LaHaisef30d7042013-08-07 18:23:48 -04001042 if (ctx && ctx->user_id == ctx_id) {
Al Virobaf10562018-05-20 16:46:23 -04001043 if (percpu_ref_tryget_live(&ctx->users))
1044 ret = ctx;
Jens Axboeabf137d2008-12-09 08:11:22 +01001045 }
Benjamin LaHaisedb446a02013-07-30 12:54:40 -04001046out:
Jens Axboeabf137d2008-12-09 08:11:22 +01001047 rcu_read_unlock();
Jeff Moyer65c24492009-03-18 17:04:21 -07001048 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049}
1050
Christoph Hellwig9018ccc2018-07-24 11:36:37 +02001051static inline void iocb_put(struct aio_kiocb *iocb)
1052{
1053 if (refcount_read(&iocb->ki_refcnt) == 0 ||
1054 refcount_dec_and_test(&iocb->ki_refcnt)) {
1055 percpu_ref_put(&iocb->ki_ctx->reqs);
1056 kmem_cache_free(kiocb_cachep, iocb);
1057 }
1058}
1059
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060/* aio_complete
1061 * Called when the io request on the given iocb is complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 */
Christoph Hellwig54843f82018-05-02 19:57:21 +02001063static void aio_complete(struct aio_kiocb *iocb, long res, long res2)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064{
1065 struct kioctx *ctx = iocb->ki_ctx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 struct aio_ring *ring;
Kent Overstreet21b40202013-05-07 16:18:47 -07001067 struct io_event *ev_page, *event;
Benjamin LaHaised856f322014-08-24 13:14:05 -04001068 unsigned tail, pos, head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071 /*
Kent Overstreet0460fef2013-05-07 16:18:49 -07001072 * Add a completion event to the ring buffer. Must be done holding
Tang Chen4b30f072013-07-03 15:09:16 -07001073 * ctx->completion_lock to prevent other code from messing with the tail
Kent Overstreet0460fef2013-05-07 16:18:49 -07001074 * pointer since we might be called from irq context.
1075 */
1076 spin_lock_irqsave(&ctx->completion_lock, flags);
1077
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001078 tail = ctx->tail;
Kent Overstreet21b40202013-05-07 16:18:47 -07001079 pos = tail + AIO_EVENTS_OFFSET;
1080
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001081 if (++tail >= ctx->nr_events)
Ken Chen4bf69b22005-05-01 08:59:15 -07001082 tail = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001084 ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
Kent Overstreet21b40202013-05-07 16:18:47 -07001085 event = ev_page + pos % AIO_EVENTS_PER_PAGE;
1086
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001087 event->obj = (u64)(unsigned long)iocb->ki_user_iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088 event->data = iocb->ki_user_data;
1089 event->res = res;
1090 event->res2 = res2;
1091
Kent Overstreet21b40202013-05-07 16:18:47 -07001092 kunmap_atomic(ev_page);
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001093 flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
Kent Overstreet21b40202013-05-07 16:18:47 -07001094
1095 pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001096 ctx, tail, iocb, iocb->ki_user_iocb, iocb->ki_user_data,
Kent Overstreetcaf41672013-05-07 16:18:35 -07001097 res, res2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098
1099 /* after flagging the request as done, we
1100 * must never even look at it again
1101 */
1102 smp_wmb(); /* make event visible before updating tail */
1103
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001104 ctx->tail = tail;
Kent Overstreet21b40202013-05-07 16:18:47 -07001105
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001106 ring = kmap_atomic(ctx->ring_pages[0]);
Benjamin LaHaised856f322014-08-24 13:14:05 -04001107 head = ring->head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 ring->tail = tail;
Cong Wange8e3c3d2011-11-25 23:14:27 +08001109 kunmap_atomic(ring);
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001110 flush_dcache_page(ctx->ring_pages[0]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111
Benjamin LaHaised856f322014-08-24 13:14:05 -04001112 ctx->completed_events++;
1113 if (ctx->completed_events > 1)
1114 refill_reqs_available(ctx, head, tail);
Kent Overstreet0460fef2013-05-07 16:18:49 -07001115 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1116
Kent Overstreet21b40202013-05-07 16:18:47 -07001117 pr_debug("added to ring %p at [%u]\n", iocb, tail);
Davide Libenzi8d1c98b2008-04-10 21:29:19 -07001118
1119 /*
1120 * Check if the user asked us to deliver the result through an
1121 * eventfd. The eventfd_signal() function is safe to be called
1122 * from IRQ context.
1123 */
Christoph Hellwig54843f82018-05-02 19:57:21 +02001124 if (iocb->ki_eventfd) {
Davide Libenzi8d1c98b2008-04-10 21:29:19 -07001125 eventfd_signal(iocb->ki_eventfd, 1);
Christoph Hellwig54843f82018-05-02 19:57:21 +02001126 eventfd_ctx_put(iocb->ki_eventfd);
1127 }
Davide Libenzi8d1c98b2008-04-10 21:29:19 -07001128
Quentin Barnes6cb2a212008-03-19 17:00:39 -07001129 /*
1130 * We have to order our ring_info tail store above and test
1131 * of the wait list below outside the wait lock. This is
1132 * like in wake_up_bit() where clearing a bit has to be
1133 * ordered with the unlocked test.
1134 */
1135 smp_mb();
1136
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 if (waitqueue_active(&ctx->wait))
1138 wake_up(&ctx->wait);
Christoph Hellwig9018ccc2018-07-24 11:36:37 +02001139 iocb_put(iocb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140}
1141
Gu Zheng2be4e7d2014-07-23 18:03:53 +08001142/* aio_read_events_ring
Kent Overstreeta31ad382013-05-07 16:18:45 -07001143 * Pull an event off of the ioctx's event ring. Returns the number of
1144 * events fetched
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 */
Kent Overstreeta31ad382013-05-07 16:18:45 -07001146static long aio_read_events_ring(struct kioctx *ctx,
1147 struct io_event __user *event, long nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149 struct aio_ring *ring;
Kent Overstreet5ffac122013-05-09 15:36:07 -07001150 unsigned head, tail, pos;
Kent Overstreeta31ad382013-05-07 16:18:45 -07001151 long ret = 0;
1152 int copy_ret;
1153
Dave Chinner9c9ce762015-02-03 19:29:05 -05001154 /*
1155 * The mutex can block and wake us up and that will cause
1156 * wait_event_interruptible_hrtimeout() to schedule without sleeping
1157 * and repeat. This should be rare enough that it doesn't cause
1158 * peformance issues. See the comment in read_events() for more detail.
1159 */
1160 sched_annotate_sleep();
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001161 mutex_lock(&ctx->ring_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162
Benjamin LaHaisefa8a53c2014-03-28 10:14:45 -04001163 /* Access to ->ring_pages here is protected by ctx->ring_lock. */
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001164 ring = kmap_atomic(ctx->ring_pages[0]);
Kent Overstreeta31ad382013-05-07 16:18:45 -07001165 head = ring->head;
Kent Overstreet5ffac122013-05-09 15:36:07 -07001166 tail = ring->tail;
Kent Overstreeta31ad382013-05-07 16:18:45 -07001167 kunmap_atomic(ring);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168
Jeff Moyer2ff396be2014-09-02 13:17:00 -04001169 /*
1170 * Ensure that once we've read the current tail pointer, that
1171 * we also see the events that were stored up to the tail.
1172 */
1173 smp_rmb();
1174
Kent Overstreet5ffac122013-05-09 15:36:07 -07001175 pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);
Kent Overstreeta31ad382013-05-07 16:18:45 -07001176
Kent Overstreet5ffac122013-05-09 15:36:07 -07001177 if (head == tail)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 goto out;
1179
Benjamin LaHaiseedfbbf32014-06-24 13:32:51 -04001180 head %= ctx->nr_events;
1181 tail %= ctx->nr_events;
1182
Kent Overstreeta31ad382013-05-07 16:18:45 -07001183 while (ret < nr) {
1184 long avail;
1185 struct io_event *ev;
1186 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187
Kent Overstreet5ffac122013-05-09 15:36:07 -07001188 avail = (head <= tail ? tail : ctx->nr_events) - head;
1189 if (head == tail)
Kent Overstreeta31ad382013-05-07 16:18:45 -07001190 break;
1191
Kent Overstreeta31ad382013-05-07 16:18:45 -07001192 pos = head + AIO_EVENTS_OFFSET;
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001193 page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE];
Kent Overstreeta31ad382013-05-07 16:18:45 -07001194 pos %= AIO_EVENTS_PER_PAGE;
1195
Al Virod2988bd42018-05-26 19:13:10 -04001196 avail = min(avail, nr - ret);
1197 avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos);
1198
Kent Overstreeta31ad382013-05-07 16:18:45 -07001199 ev = kmap(page);
1200 copy_ret = copy_to_user(event + ret, ev + pos,
1201 sizeof(*ev) * avail);
1202 kunmap(page);
1203
1204 if (unlikely(copy_ret)) {
1205 ret = -EFAULT;
1206 goto out;
1207 }
1208
1209 ret += avail;
1210 head += avail;
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001211 head %= ctx->nr_events;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001214 ring = kmap_atomic(ctx->ring_pages[0]);
Kent Overstreeta31ad382013-05-07 16:18:45 -07001215 ring->head = head;
Zhao Hongjiang91d80a82013-04-26 11:03:53 +08001216 kunmap_atomic(ring);
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001217 flush_dcache_page(ctx->ring_pages[0]);
Kent Overstreeta31ad382013-05-07 16:18:45 -07001218
Kent Overstreet5ffac122013-05-09 15:36:07 -07001219 pr_debug("%li h%u t%u\n", ret, head, tail);
Kent Overstreeta31ad382013-05-07 16:18:45 -07001220out:
Kent Overstreet58c85dc2013-05-07 16:18:55 -07001221 mutex_unlock(&ctx->ring_lock);
Kent Overstreeta31ad382013-05-07 16:18:45 -07001222
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223 return ret;
1224}
1225
Kent Overstreeta31ad382013-05-07 16:18:45 -07001226static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr,
1227 struct io_event __user *event, long *i)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228{
Kent Overstreeta31ad382013-05-07 16:18:45 -07001229 long ret = aio_read_events_ring(ctx, event + *i, nr - *i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
Kent Overstreeta31ad382013-05-07 16:18:45 -07001231 if (ret > 0)
1232 *i += ret;
1233
1234 if (unlikely(atomic_read(&ctx->dead)))
1235 ret = -EINVAL;
1236
1237 if (!*i)
1238 *i = ret;
1239
1240 return ret < 0 || *i >= min_nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241}
1242
Kent Overstreeta31ad382013-05-07 16:18:45 -07001243static long read_events(struct kioctx *ctx, long min_nr, long nr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244 struct io_event __user *event,
Deepa Dinamanifa2e62a2017-08-04 21:12:32 -07001245 ktime_t until)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246{
Kent Overstreeta31ad382013-05-07 16:18:45 -07001247 long ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248
Kent Overstreeta31ad382013-05-07 16:18:45 -07001249 /*
1250 * Note that aio_read_events() is being called as the conditional - i.e.
1251 * we're calling it after prepare_to_wait() has set task state to
1252 * TASK_INTERRUPTIBLE.
1253 *
1254 * But aio_read_events() can block, and if it blocks it's going to flip
1255 * the task state back to TASK_RUNNING.
1256 *
1257 * This should be ok, provided it doesn't flip the state back to
1258 * TASK_RUNNING and return 0 too much - that causes us to spin. That
1259 * will only happen if the mutex_lock() call blocks, and we then find
1260 * the ringbuffer empty. So in practice we should be ok, but it's
1261 * something to be aware of when touching this code.
1262 */
Thomas Gleixner2456e852016-12-25 11:38:40 +01001263 if (until == 0)
Fam Zheng5f785de2014-11-06 20:44:36 +08001264 aio_read_events(ctx, min_nr, nr, event, &ret);
1265 else
1266 wait_event_interruptible_hrtimeout(ctx->wait,
1267 aio_read_events(ctx, min_nr, nr, event, &ret),
1268 until);
Kent Overstreeta31ad382013-05-07 16:18:45 -07001269 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270}
1271
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272/* sys_io_setup:
1273 * Create an aio_context capable of receiving at least nr_events.
1274 * ctxp must not point to an aio_context that already exists, and
1275 * must be initialized to 0 prior to the call. On successful
1276 * creation of the aio_context, *ctxp is filled in with the resulting
1277 * handle. May fail with -EINVAL if *ctxp is not initialized,
1278 * if the specified nr_events exceeds internal limits. May fail
1279 * with -EAGAIN if the specified nr_events exceeds the user's limit
1280 * of available events. May fail with -ENOMEM if insufficient kernel
1281 * resources are available. May fail with -EFAULT if an invalid
1282 * pointer is passed for ctxp. Will fail with -ENOSYS if not
1283 * implemented.
1284 */
Heiko Carstens002c8972009-01-14 14:14:18 +01001285SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286{
1287 struct kioctx *ioctx = NULL;
1288 unsigned long ctx;
1289 long ret;
1290
1291 ret = get_user(ctx, ctxp);
1292 if (unlikely(ret))
1293 goto out;
1294
1295 ret = -EINVAL;
Zach Brownd55b5fd2005-11-07 00:59:31 -08001296 if (unlikely(ctx || nr_events == 0)) {
Kinglong Meeacd88d42015-02-04 21:15:59 +08001297 pr_debug("EINVAL: ctx %lu nr_events %u\n",
Zach Brownd55b5fd2005-11-07 00:59:31 -08001298 ctx, nr_events);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 goto out;
1300 }
1301
1302 ioctx = ioctx_alloc(nr_events);
1303 ret = PTR_ERR(ioctx);
1304 if (!IS_ERR(ioctx)) {
1305 ret = put_user(ioctx->user_id, ctxp);
Al Viroa2e18592012-03-20 16:27:57 -04001306 if (ret)
Anatol Pomozove02ba722014-04-15 11:31:33 -07001307 kill_ioctx(current->mm, ioctx, NULL);
Kent Overstreet723be6e2013-05-28 15:14:48 -07001308 percpu_ref_put(&ioctx->users);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309 }
1310
1311out:
1312 return ret;
1313}
1314
Al Viroc00d2c72016-12-20 07:04:57 -05001315#ifdef CONFIG_COMPAT
1316COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p)
1317{
1318 struct kioctx *ioctx = NULL;
1319 unsigned long ctx;
1320 long ret;
1321
1322 ret = get_user(ctx, ctx32p);
1323 if (unlikely(ret))
1324 goto out;
1325
1326 ret = -EINVAL;
1327 if (unlikely(ctx || nr_events == 0)) {
1328 pr_debug("EINVAL: ctx %lu nr_events %u\n",
1329 ctx, nr_events);
1330 goto out;
1331 }
1332
1333 ioctx = ioctx_alloc(nr_events);
1334 ret = PTR_ERR(ioctx);
1335 if (!IS_ERR(ioctx)) {
1336 /* truncating is ok because it's a user address */
1337 ret = put_user((u32)ioctx->user_id, ctx32p);
1338 if (ret)
1339 kill_ioctx(current->mm, ioctx, NULL);
1340 percpu_ref_put(&ioctx->users);
1341 }
1342
1343out:
1344 return ret;
1345}
1346#endif
1347
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348/* sys_io_destroy:
1349 * Destroy the aio_context specified. May cancel any outstanding
1350 * AIOs and block on completion. Will fail with -ENOSYS if not
Satoru Takeuchi642b5122010-08-05 11:23:11 -07001351 * implemented. May fail with -EINVAL if the context pointed to
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 * is invalid.
1353 */
Heiko Carstens002c8972009-01-14 14:14:18 +01001354SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355{
1356 struct kioctx *ioctx = lookup_ioctx(ctx);
1357 if (likely(NULL != ioctx)) {
Jens Axboedc48e562015-04-15 11:17:23 -06001358 struct ctx_rq_wait wait;
Benjamin LaHaisefb2d4482014-04-29 12:45:17 -04001359 int ret;
Anatol Pomozove02ba722014-04-15 11:31:33 -07001360
Jens Axboedc48e562015-04-15 11:17:23 -06001361 init_completion(&wait.comp);
1362 atomic_set(&wait.count, 1);
1363
Anatol Pomozove02ba722014-04-15 11:31:33 -07001364 /* Pass requests_done to kill_ioctx() where it can be set
1365 * in a thread-safe way. If we try to set it here then we have
1366 * a race condition if two io_destroy() called simultaneously.
1367 */
Jens Axboedc48e562015-04-15 11:17:23 -06001368 ret = kill_ioctx(current->mm, ioctx, &wait);
Kent Overstreet723be6e2013-05-28 15:14:48 -07001369 percpu_ref_put(&ioctx->users);
Anatol Pomozove02ba722014-04-15 11:31:33 -07001370
1371 /* Wait until all IO for the context are done. Otherwise kernel
1372 * keep using user-space buffers even if user thinks the context
1373 * is destroyed.
1374 */
Benjamin LaHaisefb2d4482014-04-29 12:45:17 -04001375 if (!ret)
Jens Axboedc48e562015-04-15 11:17:23 -06001376 wait_for_completion(&wait.comp);
Anatol Pomozove02ba722014-04-15 11:31:33 -07001377
Benjamin LaHaisefb2d4482014-04-29 12:45:17 -04001378 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 }
Kinglong Meeacd88d42015-02-04 21:15:59 +08001380 pr_debug("EINVAL: invalid context id\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381 return -EINVAL;
1382}
1383
Al Viro3c96c7f2018-05-28 13:37:43 -04001384static void aio_remove_iocb(struct aio_kiocb *iocb)
1385{
1386 struct kioctx *ctx = iocb->ki_ctx;
1387 unsigned long flags;
1388
1389 spin_lock_irqsave(&ctx->ctx_lock, flags);
1390 list_del(&iocb->ki_list);
1391 spin_unlock_irqrestore(&ctx->ctx_lock, flags);
1392}
1393
Christoph Hellwig54843f82018-05-02 19:57:21 +02001394static void aio_complete_rw(struct kiocb *kiocb, long res, long res2)
1395{
1396 struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw);
1397
Al Viro3c96c7f2018-05-28 13:37:43 -04001398 if (!list_empty_careful(&iocb->ki_list))
1399 aio_remove_iocb(iocb);
1400
Christoph Hellwig54843f82018-05-02 19:57:21 +02001401 if (kiocb->ki_flags & IOCB_WRITE) {
1402 struct inode *inode = file_inode(kiocb->ki_filp);
1403
1404 /*
1405 * Tell lockdep we inherited freeze protection from submission
1406 * thread.
1407 */
1408 if (S_ISREG(inode->i_mode))
1409 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
1410 file_end_write(kiocb->ki_filp);
1411 }
1412
1413 fput(kiocb->ki_filp);
1414 aio_complete(iocb, res, res2);
1415}
1416
1417static int aio_prep_rw(struct kiocb *req, struct iocb *iocb)
1418{
1419 int ret;
1420
1421 req->ki_filp = fget(iocb->aio_fildes);
1422 if (unlikely(!req->ki_filp))
1423 return -EBADF;
1424 req->ki_complete = aio_complete_rw;
1425 req->ki_pos = iocb->aio_offset;
1426 req->ki_flags = iocb_flags(req->ki_filp);
1427 if (iocb->aio_flags & IOCB_FLAG_RESFD)
1428 req->ki_flags |= IOCB_EVENTFD;
Adam Manzanaresfc287242018-05-22 10:52:18 -07001429 req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp));
Adam Manzanaresd9a08a92018-05-22 10:52:19 -07001430 if (iocb->aio_flags & IOCB_FLAG_IOPRIO) {
1431 /*
1432 * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then
1433 * aio_reqprio is interpreted as an I/O scheduling
1434 * class and priority.
1435 */
1436 ret = ioprio_check_cap(iocb->aio_reqprio);
1437 if (ret) {
Adam Manzanares9a6d9a62018-06-04 10:59:57 -07001438 pr_debug("aio ioprio check cap error: %d\n", ret);
Jens Axboe53fffe22018-11-17 07:43:42 -07001439 fput(req->ki_filp);
Adam Manzanares9a6d9a62018-06-04 10:59:57 -07001440 return ret;
Adam Manzanaresd9a08a92018-05-22 10:52:19 -07001441 }
1442
1443 req->ki_ioprio = iocb->aio_reqprio;
1444 } else
1445 req->ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
1446
Christoph Hellwig54843f82018-05-02 19:57:21 +02001447 ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags);
1448 if (unlikely(ret))
1449 fput(req->ki_filp);
1450 return ret;
1451}
1452
Christoph Hellwig89319d312016-10-30 11:42:03 -05001453static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec,
1454 bool vectored, bool compat, struct iov_iter *iter)
Badari Pulavartyeed4e512006-09-30 23:28:49 -07001455{
Christoph Hellwig89319d312016-10-30 11:42:03 -05001456 void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf;
1457 size_t len = iocb->aio_nbytes;
1458
1459 if (!vectored) {
1460 ssize_t ret = import_single_range(rw, buf, len, *iovec, iter);
1461 *iovec = NULL;
1462 return ret;
1463 }
Jeff Moyer9d85cba72010-05-26 14:44:26 -07001464#ifdef CONFIG_COMPAT
1465 if (compat)
Christoph Hellwig89319d312016-10-30 11:42:03 -05001466 return compat_import_iovec(rw, buf, len, UIO_FASTIOV, iovec,
1467 iter);
Jeff Moyer9d85cba72010-05-26 14:44:26 -07001468#endif
Christoph Hellwig89319d312016-10-30 11:42:03 -05001469 return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter);
Badari Pulavartyeed4e512006-09-30 23:28:49 -07001470}
1471
Al Viro9061d142018-05-26 19:11:40 -04001472static inline void aio_rw_done(struct kiocb *req, ssize_t ret)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473{
Christoph Hellwig89319d312016-10-30 11:42:03 -05001474 switch (ret) {
1475 case -EIOCBQUEUED:
Al Viro9061d142018-05-26 19:11:40 -04001476 break;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001477 case -ERESTARTSYS:
1478 case -ERESTARTNOINTR:
1479 case -ERESTARTNOHAND:
1480 case -ERESTART_RESTARTBLOCK:
Kent Overstreet41ef4eb2013-05-07 16:19:11 -07001481 /*
1482 * There's no easy way to restart the syscall since other AIO's
1483 * may be already running. Just fail this IO with EINTR.
1484 */
Christoph Hellwig89319d312016-10-30 11:42:03 -05001485 ret = -EINTR;
1486 /*FALLTHRU*/
1487 default:
Christoph Hellwig54843f82018-05-02 19:57:21 +02001488 aio_complete_rw(req, ret, 0);
Kent Overstreet41ef4eb2013-05-07 16:19:11 -07001489 }
Christoph Hellwig89319d312016-10-30 11:42:03 -05001490}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491
Christoph Hellwig89319d312016-10-30 11:42:03 -05001492static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored,
1493 bool compat)
1494{
Christoph Hellwig89319d312016-10-30 11:42:03 -05001495 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1496 struct iov_iter iter;
Christoph Hellwig54843f82018-05-02 19:57:21 +02001497 struct file *file;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001498 ssize_t ret;
1499
Christoph Hellwig54843f82018-05-02 19:57:21 +02001500 ret = aio_prep_rw(req, iocb);
1501 if (ret)
1502 return ret;
1503 file = req->ki_filp;
1504
1505 ret = -EBADF;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001506 if (unlikely(!(file->f_mode & FMODE_READ)))
Christoph Hellwig54843f82018-05-02 19:57:21 +02001507 goto out_fput;
1508 ret = -EINVAL;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001509 if (unlikely(!file->f_op->read_iter))
Christoph Hellwig54843f82018-05-02 19:57:21 +02001510 goto out_fput;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001511
1512 ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);
1513 if (ret)
Christoph Hellwig54843f82018-05-02 19:57:21 +02001514 goto out_fput;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001515 ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
1516 if (!ret)
Al Viro9061d142018-05-26 19:11:40 -04001517 aio_rw_done(req, call_read_iter(file, req, &iter));
Christoph Hellwig89319d312016-10-30 11:42:03 -05001518 kfree(iovec);
Christoph Hellwig54843f82018-05-02 19:57:21 +02001519out_fput:
Al Viro9061d142018-05-26 19:11:40 -04001520 if (unlikely(ret))
Christoph Hellwig54843f82018-05-02 19:57:21 +02001521 fput(file);
Christoph Hellwig89319d312016-10-30 11:42:03 -05001522 return ret;
1523}
1524
1525static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored,
1526 bool compat)
1527{
Christoph Hellwig89319d312016-10-30 11:42:03 -05001528 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1529 struct iov_iter iter;
Christoph Hellwig54843f82018-05-02 19:57:21 +02001530 struct file *file;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001531 ssize_t ret;
1532
Christoph Hellwig54843f82018-05-02 19:57:21 +02001533 ret = aio_prep_rw(req, iocb);
1534 if (ret)
1535 return ret;
1536 file = req->ki_filp;
1537
1538 ret = -EBADF;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001539 if (unlikely(!(file->f_mode & FMODE_WRITE)))
Christoph Hellwig54843f82018-05-02 19:57:21 +02001540 goto out_fput;
1541 ret = -EINVAL;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001542 if (unlikely(!file->f_op->write_iter))
Christoph Hellwig54843f82018-05-02 19:57:21 +02001543 goto out_fput;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001544
1545 ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);
1546 if (ret)
Christoph Hellwig54843f82018-05-02 19:57:21 +02001547 goto out_fput;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001548 ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));
1549 if (!ret) {
Jan Kara70fe2f42016-10-30 11:42:04 -05001550 /*
Christoph Hellwig92ce4722018-04-06 09:28:17 +02001551 * Open-code file_start_write here to grab freeze protection,
Christoph Hellwig54843f82018-05-02 19:57:21 +02001552 * which will be released by another thread in
1553 * aio_complete_rw(). Fool lockdep by telling it the lock got
1554 * released so that it doesn't complain about the held lock when
1555 * we return to userspace.
Jan Kara70fe2f42016-10-30 11:42:04 -05001556 */
Christoph Hellwig92ce4722018-04-06 09:28:17 +02001557 if (S_ISREG(file_inode(file)->i_mode)) {
1558 __sb_start_write(file_inode(file)->i_sb, SB_FREEZE_WRITE, true);
Shaohua Lia12f1ae2016-12-13 12:09:56 -08001559 __sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE);
Christoph Hellwig92ce4722018-04-06 09:28:17 +02001560 }
1561 req->ki_flags |= IOCB_WRITE;
Al Viro9061d142018-05-26 19:11:40 -04001562 aio_rw_done(req, call_write_iter(file, req, &iter));
Christoph Hellwig89319d312016-10-30 11:42:03 -05001563 }
1564 kfree(iovec);
Christoph Hellwig54843f82018-05-02 19:57:21 +02001565out_fput:
Al Viro9061d142018-05-26 19:11:40 -04001566 if (unlikely(ret))
Christoph Hellwig54843f82018-05-02 19:57:21 +02001567 fput(file);
Christoph Hellwig89319d312016-10-30 11:42:03 -05001568 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569}
1570
Christoph Hellwiga3c0d432018-03-27 19:18:57 +02001571static void aio_fsync_work(struct work_struct *work)
1572{
1573 struct fsync_iocb *req = container_of(work, struct fsync_iocb, work);
1574 int ret;
1575
1576 ret = vfs_fsync(req->file, req->datasync);
1577 fput(req->file);
1578 aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0);
1579}
1580
1581static int aio_fsync(struct fsync_iocb *req, struct iocb *iocb, bool datasync)
1582{
1583 if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes ||
1584 iocb->aio_rw_flags))
1585 return -EINVAL;
Linus Torvaldsa11e1d42018-06-28 09:43:44 -07001586
Christoph Hellwiga3c0d432018-03-27 19:18:57 +02001587 req->file = fget(iocb->aio_fildes);
1588 if (unlikely(!req->file))
1589 return -EBADF;
1590 if (unlikely(!req->file->f_op->fsync)) {
1591 fput(req->file);
1592 return -EINVAL;
1593 }
1594
1595 req->datasync = datasync;
1596 INIT_WORK(&req->work, aio_fsync_work);
1597 schedule_work(&req->work);
Al Viro9061d142018-05-26 19:11:40 -04001598 return 0;
Christoph Hellwiga3c0d432018-03-27 19:18:57 +02001599}
1600
Christoph Hellwigbfe40372018-07-16 09:08:20 +02001601static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask)
1602{
1603 struct file *file = iocb->poll.file;
1604
1605 aio_complete(iocb, mangle_poll(mask), 0);
1606 fput(file);
1607}
1608
1609static void aio_poll_complete_work(struct work_struct *work)
1610{
1611 struct poll_iocb *req = container_of(work, struct poll_iocb, work);
1612 struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
1613 struct poll_table_struct pt = { ._key = req->events };
1614 struct kioctx *ctx = iocb->ki_ctx;
1615 __poll_t mask = 0;
1616
1617 if (!READ_ONCE(req->cancelled))
1618 mask = vfs_poll(req->file, &pt) & req->events;
1619
1620 /*
1621 * Note that ->ki_cancel callers also delete iocb from active_reqs after
1622 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
1623 * synchronize with them. In the cancellation case the list_del_init
1624 * itself is not actually needed, but harmless so we keep it in to
1625 * avoid further branches in the fast path.
1626 */
1627 spin_lock_irq(&ctx->ctx_lock);
1628 if (!mask && !READ_ONCE(req->cancelled)) {
1629 add_wait_queue(req->head, &req->wait);
1630 spin_unlock_irq(&ctx->ctx_lock);
1631 return;
1632 }
1633 list_del_init(&iocb->ki_list);
1634 spin_unlock_irq(&ctx->ctx_lock);
1635
1636 aio_poll_complete(iocb, mask);
1637}
1638
1639/* assumes we are called with irqs disabled */
1640static int aio_poll_cancel(struct kiocb *iocb)
1641{
1642 struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw);
1643 struct poll_iocb *req = &aiocb->poll;
1644
1645 spin_lock(&req->head->lock);
1646 WRITE_ONCE(req->cancelled, true);
1647 if (!list_empty(&req->wait.entry)) {
1648 list_del_init(&req->wait.entry);
1649 schedule_work(&aiocb->poll.work);
1650 }
1651 spin_unlock(&req->head->lock);
1652
1653 return 0;
1654}
1655
1656static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
1657 void *key)
1658{
1659 struct poll_iocb *req = container_of(wait, struct poll_iocb, wait);
Christoph Hellwige8693bc2018-07-16 12:25:17 +02001660 struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
Christoph Hellwigbfe40372018-07-16 09:08:20 +02001661 __poll_t mask = key_to_poll(key);
1662
1663 req->woken = true;
1664
1665 /* for instances that support it check for an event match first: */
Christoph Hellwige8693bc2018-07-16 12:25:17 +02001666 if (mask) {
1667 if (!(mask & req->events))
1668 return 0;
1669
1670 /* try to complete the iocb inline if we can: */
1671 if (spin_trylock(&iocb->ki_ctx->ctx_lock)) {
1672 list_del(&iocb->ki_list);
1673 spin_unlock(&iocb->ki_ctx->ctx_lock);
1674
1675 list_del_init(&req->wait.entry);
1676 aio_poll_complete(iocb, mask);
1677 return 1;
1678 }
1679 }
Christoph Hellwigbfe40372018-07-16 09:08:20 +02001680
1681 list_del_init(&req->wait.entry);
1682 schedule_work(&req->work);
1683 return 1;
1684}
1685
1686struct aio_poll_table {
1687 struct poll_table_struct pt;
1688 struct aio_kiocb *iocb;
1689 int error;
1690};
1691
1692static void
1693aio_poll_queue_proc(struct file *file, struct wait_queue_head *head,
1694 struct poll_table_struct *p)
1695{
1696 struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt);
1697
1698 /* multiple wait queues per file are not supported */
1699 if (unlikely(pt->iocb->poll.head)) {
1700 pt->error = -EINVAL;
1701 return;
1702 }
1703
1704 pt->error = 0;
1705 pt->iocb->poll.head = head;
1706 add_wait_queue(head, &pt->iocb->poll.wait);
1707}
1708
1709static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb)
1710{
1711 struct kioctx *ctx = aiocb->ki_ctx;
1712 struct poll_iocb *req = &aiocb->poll;
1713 struct aio_poll_table apt;
1714 __poll_t mask;
1715
1716 /* reject any unknown events outside the normal event mask. */
1717 if ((u16)iocb->aio_buf != iocb->aio_buf)
1718 return -EINVAL;
1719 /* reject fields that are not defined for poll */
1720 if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags)
1721 return -EINVAL;
1722
1723 INIT_WORK(&req->work, aio_poll_complete_work);
1724 req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;
1725 req->file = fget(iocb->aio_fildes);
1726 if (unlikely(!req->file))
1727 return -EBADF;
1728
1729 apt.pt._qproc = aio_poll_queue_proc;
1730 apt.pt._key = req->events;
1731 apt.iocb = aiocb;
1732 apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
1733
1734 /* initialized the list so that we can do list_empty checks */
1735 INIT_LIST_HEAD(&req->wait.entry);
1736 init_waitqueue_func_entry(&req->wait, aio_poll_wake);
1737
1738 /* one for removal from waitqueue, one for this function */
1739 refcount_set(&aiocb->ki_refcnt, 2);
1740
1741 mask = vfs_poll(req->file, &apt.pt) & req->events;
1742 if (unlikely(!req->head)) {
1743 /* we did not manage to set up a waitqueue, done */
1744 goto out;
1745 }
1746
1747 spin_lock_irq(&ctx->ctx_lock);
1748 spin_lock(&req->head->lock);
1749 if (req->woken) {
1750 /* wake_up context handles the rest */
1751 mask = 0;
1752 apt.error = 0;
1753 } else if (mask || apt.error) {
1754 /* if we get an error or a mask we are done */
1755 WARN_ON_ONCE(list_empty(&req->wait.entry));
1756 list_del_init(&req->wait.entry);
1757 } else {
1758 /* actually waiting for an event */
1759 list_add_tail(&aiocb->ki_list, &ctx->active_reqs);
1760 aiocb->ki_cancel = aio_poll_cancel;
1761 }
1762 spin_unlock(&req->head->lock);
1763 spin_unlock_irq(&ctx->ctx_lock);
1764
1765out:
1766 if (unlikely(apt.error)) {
1767 fput(req->file);
1768 return apt.error;
1769 }
1770
1771 if (mask)
1772 aio_poll_complete(aiocb, mask);
1773 iocb_put(aiocb);
1774 return 0;
1775}
1776
Adrian Bunkd5470b52008-04-29 00:58:57 -07001777static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
Al Viro95af8492018-05-26 19:43:16 -04001778 bool compat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779{
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001780 struct aio_kiocb *req;
Al Viro95af8492018-05-26 19:43:16 -04001781 struct iocb iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782 ssize_t ret;
1783
Al Viro95af8492018-05-26 19:43:16 -04001784 if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb))))
1785 return -EFAULT;
1786
Linus Torvalds1da177e2005-04-16 15:20:36 -07001787 /* enforce forwards compatibility on users */
Al Viro95af8492018-05-26 19:43:16 -04001788 if (unlikely(iocb.aio_reserved2)) {
Kent Overstreetcaf41672013-05-07 16:18:35 -07001789 pr_debug("EINVAL: reserve field set\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790 return -EINVAL;
1791 }
1792
1793 /* prevent overflows */
1794 if (unlikely(
Al Viro95af8492018-05-26 19:43:16 -04001795 (iocb.aio_buf != (unsigned long)iocb.aio_buf) ||
1796 (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) ||
1797 ((ssize_t)iocb.aio_nbytes < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 )) {
Kinglong Meeacd88d42015-02-04 21:15:59 +08001799 pr_debug("EINVAL: overflow check\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800 return -EINVAL;
1801 }
1802
Kent Overstreet41ef4eb2013-05-07 16:19:11 -07001803 req = aio_get_req(ctx);
Kent Overstreet1d98ebf2013-05-07 16:18:37 -07001804 if (unlikely(!req))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805 return -EAGAIN;
Kent Overstreet1d98ebf2013-05-07 16:18:37 -07001806
Al Viro95af8492018-05-26 19:43:16 -04001807 if (iocb.aio_flags & IOCB_FLAG_RESFD) {
Davide Libenzi9c3060b2007-05-10 22:23:21 -07001808 /*
1809 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
1810 * instance of the file* now. The file descriptor must be
1811 * an eventfd() fd, and will be signaled for each completed
1812 * event using the eventfd_signal() function.
1813 */
Al Viro95af8492018-05-26 19:43:16 -04001814 req->ki_eventfd = eventfd_ctx_fdget((int) iocb.aio_resfd);
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07001815 if (IS_ERR(req->ki_eventfd)) {
Davide Libenzi9c3060b2007-05-10 22:23:21 -07001816 ret = PTR_ERR(req->ki_eventfd);
Davide Libenzi87c3a862009-03-18 17:04:19 -07001817 req->ki_eventfd = NULL;
Davide Libenzi9c3060b2007-05-10 22:23:21 -07001818 goto out_put_req;
1819 }
Goldwyn Rodrigues9830f4b2017-06-20 07:05:42 -05001820 }
1821
Kent Overstreet8a660892013-05-07 16:19:10 -07001822 ret = put_user(KIOCB_KEY, &user_iocb->aio_key);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001823 if (unlikely(ret)) {
Kent Overstreetcaf41672013-05-07 16:18:35 -07001824 pr_debug("EFAULT: aio_key\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825 goto out_put_req;
1826 }
1827
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001828 req->ki_user_iocb = user_iocb;
Al Viro95af8492018-05-26 19:43:16 -04001829 req->ki_user_data = iocb.aio_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830
Al Viro95af8492018-05-26 19:43:16 -04001831 switch (iocb.aio_lio_opcode) {
Christoph Hellwig89319d312016-10-30 11:42:03 -05001832 case IOCB_CMD_PREAD:
Al Viro95af8492018-05-26 19:43:16 -04001833 ret = aio_read(&req->rw, &iocb, false, compat);
Christoph Hellwig89319d312016-10-30 11:42:03 -05001834 break;
1835 case IOCB_CMD_PWRITE:
Al Viro95af8492018-05-26 19:43:16 -04001836 ret = aio_write(&req->rw, &iocb, false, compat);
Christoph Hellwig89319d312016-10-30 11:42:03 -05001837 break;
1838 case IOCB_CMD_PREADV:
Al Viro95af8492018-05-26 19:43:16 -04001839 ret = aio_read(&req->rw, &iocb, true, compat);
Christoph Hellwig89319d312016-10-30 11:42:03 -05001840 break;
1841 case IOCB_CMD_PWRITEV:
Al Viro95af8492018-05-26 19:43:16 -04001842 ret = aio_write(&req->rw, &iocb, true, compat);
Christoph Hellwig89319d312016-10-30 11:42:03 -05001843 break;
Christoph Hellwiga3c0d432018-03-27 19:18:57 +02001844 case IOCB_CMD_FSYNC:
Al Viro95af8492018-05-26 19:43:16 -04001845 ret = aio_fsync(&req->fsync, &iocb, false);
Christoph Hellwiga3c0d432018-03-27 19:18:57 +02001846 break;
1847 case IOCB_CMD_FDSYNC:
Al Viro95af8492018-05-26 19:43:16 -04001848 ret = aio_fsync(&req->fsync, &iocb, true);
Christoph Hellwigac060cb2018-05-28 07:19:49 +02001849 break;
Christoph Hellwigbfe40372018-07-16 09:08:20 +02001850 case IOCB_CMD_POLL:
1851 ret = aio_poll(req, &iocb);
1852 break;
Christoph Hellwig89319d312016-10-30 11:42:03 -05001853 default:
Al Viro95af8492018-05-26 19:43:16 -04001854 pr_debug("invalid aio operation %d\n", iocb.aio_lio_opcode);
Christoph Hellwig89319d312016-10-30 11:42:03 -05001855 ret = -EINVAL;
1856 break;
1857 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001858
Christoph Hellwig92ce4722018-04-06 09:28:17 +02001859 /*
Al Viro9061d142018-05-26 19:11:40 -04001860 * If ret is 0, we'd either done aio_complete() ourselves or have
1861 * arranged for that to be done asynchronously. Anything non-zero
1862 * means that we need to destroy req ourselves.
Christoph Hellwig92ce4722018-04-06 09:28:17 +02001863 */
Al Viro9061d142018-05-26 19:11:40 -04001864 if (ret)
Christoph Hellwig89319d312016-10-30 11:42:03 -05001865 goto out_put_req;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001867out_put_req:
Kent Overstreete1bdd5f2013-04-26 10:58:39 +10001868 put_reqs_available(ctx, 1);
Kent Overstreete34ecee2013-10-10 19:31:47 -07001869 percpu_ref_put(&ctx->reqs);
Christoph Hellwig54843f82018-05-02 19:57:21 +02001870 if (req->ki_eventfd)
1871 eventfd_ctx_put(req->ki_eventfd);
1872 kmem_cache_free(kiocb_cachep, req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873 return ret;
1874}
1875
Jeff Moyer9d85cba72010-05-26 14:44:26 -07001876/* sys_io_submit:
1877 * Queue the nr iocbs pointed to by iocbpp for processing. Returns
1878 * the number of iocbs queued. May return -EINVAL if the aio_context
1879 * specified by ctx_id is invalid, if nr is < 0, if the iocb at
1880 * *iocbpp[0] is not properly initialized, if the operation specified
1881 * is invalid for the file descriptor in the iocb. May fail with
1882 * -EFAULT if any of the data structures point to invalid data. May
1883 * fail with -EBADF if the file descriptor specified in the first
1884 * iocb is invalid. May fail with -EAGAIN if insufficient resources
1885 * are available to queue any iocbs. Will return 0 if nr is 0. Will
1886 * fail with -ENOSYS if not implemented.
1887 */
1888SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
1889 struct iocb __user * __user *, iocbpp)
1890{
Al Viro67ba0492018-05-26 20:10:07 -04001891 struct kioctx *ctx;
1892 long ret = 0;
1893 int i = 0;
1894 struct blk_plug plug;
1895
1896 if (unlikely(nr < 0))
1897 return -EINVAL;
1898
Al Viro67ba0492018-05-26 20:10:07 -04001899 ctx = lookup_ioctx(ctx_id);
1900 if (unlikely(!ctx)) {
1901 pr_debug("EINVAL: invalid context id\n");
1902 return -EINVAL;
1903 }
1904
Al Viro1da92772018-05-26 20:10:07 -04001905 if (nr > ctx->nr_events)
1906 nr = ctx->nr_events;
1907
Al Viro67ba0492018-05-26 20:10:07 -04001908 blk_start_plug(&plug);
1909 for (i = 0; i < nr; i++) {
1910 struct iocb __user *user_iocb;
1911
1912 if (unlikely(get_user(user_iocb, iocbpp + i))) {
1913 ret = -EFAULT;
1914 break;
1915 }
1916
1917 ret = io_submit_one(ctx, user_iocb, false);
1918 if (ret)
1919 break;
1920 }
1921 blk_finish_plug(&plug);
1922
1923 percpu_ref_put(&ctx->users);
1924 return i ? i : ret;
Jeff Moyer9d85cba72010-05-26 14:44:26 -07001925}
1926
Al Viroc00d2c72016-12-20 07:04:57 -05001927#ifdef CONFIG_COMPAT
Al Viroc00d2c72016-12-20 07:04:57 -05001928COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id,
Al Viro67ba0492018-05-26 20:10:07 -04001929 int, nr, compat_uptr_t __user *, iocbpp)
Al Viroc00d2c72016-12-20 07:04:57 -05001930{
Al Viro67ba0492018-05-26 20:10:07 -04001931 struct kioctx *ctx;
1932 long ret = 0;
1933 int i = 0;
1934 struct blk_plug plug;
Al Viroc00d2c72016-12-20 07:04:57 -05001935
1936 if (unlikely(nr < 0))
1937 return -EINVAL;
1938
Al Viro67ba0492018-05-26 20:10:07 -04001939 ctx = lookup_ioctx(ctx_id);
1940 if (unlikely(!ctx)) {
1941 pr_debug("EINVAL: invalid context id\n");
1942 return -EINVAL;
1943 }
1944
Al Viro1da92772018-05-26 20:10:07 -04001945 if (nr > ctx->nr_events)
1946 nr = ctx->nr_events;
1947
Al Viro67ba0492018-05-26 20:10:07 -04001948 blk_start_plug(&plug);
1949 for (i = 0; i < nr; i++) {
1950 compat_uptr_t user_iocb;
1951
1952 if (unlikely(get_user(user_iocb, iocbpp + i))) {
1953 ret = -EFAULT;
1954 break;
1955 }
1956
1957 ret = io_submit_one(ctx, compat_ptr(user_iocb), true);
1958 if (ret)
1959 break;
1960 }
1961 blk_finish_plug(&plug);
1962
1963 percpu_ref_put(&ctx->users);
1964 return i ? i : ret;
Al Viroc00d2c72016-12-20 07:04:57 -05001965}
1966#endif
1967
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968/* lookup_kiocb
1969 * Finds a given iocb for cancellation.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001970 */
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001971static struct aio_kiocb *
Christoph Hellwigf3a27522018-03-30 11:19:25 +02001972lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001973{
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001974 struct aio_kiocb *kiocb;
Zach Brownd00689a2005-11-13 16:07:34 -08001975
1976 assert_spin_locked(&ctx->ctx_lock);
1977
Linus Torvalds1da177e2005-04-16 15:20:36 -07001978 /* TODO: use a hash or array, this sucks. */
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01001979 list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) {
1980 if (kiocb->ki_user_iocb == iocb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981 return kiocb;
1982 }
1983 return NULL;
1984}
1985
1986/* sys_io_cancel:
1987 * Attempts to cancel an iocb previously passed to io_submit. If
1988 * the operation is successfully cancelled, the resulting event is
1989 * copied into the memory pointed to by result without being placed
1990 * into the completion queue and 0 is returned. May fail with
1991 * -EFAULT if any of the data structures pointed to are invalid.
1992 * May fail with -EINVAL if aio_context specified by ctx_id is
1993 * invalid. May fail with -EAGAIN if the iocb specified was not
1994 * cancelled. Will fail with -ENOSYS if not implemented.
1995 */
Heiko Carstens002c8972009-01-14 14:14:18 +01001996SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
1997 struct io_event __user *, result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001999 struct kioctx *ctx;
Christoph Hellwig04b2fa92015-02-02 14:49:06 +01002000 struct aio_kiocb *kiocb;
Christoph Hellwig888933f2018-05-23 14:11:02 +02002001 int ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002002 u32 key;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002003
Christoph Hellwigf3a27522018-03-30 11:19:25 +02002004 if (unlikely(get_user(key, &iocb->aio_key)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005 return -EFAULT;
Christoph Hellwigf3a27522018-03-30 11:19:25 +02002006 if (unlikely(key != KIOCB_KEY))
2007 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002008
2009 ctx = lookup_ioctx(ctx_id);
2010 if (unlikely(!ctx))
2011 return -EINVAL;
2012
2013 spin_lock_irq(&ctx->ctx_lock);
Christoph Hellwigf3a27522018-03-30 11:19:25 +02002014 kiocb = lookup_kiocb(ctx, iocb);
Christoph Hellwig888933f2018-05-23 14:11:02 +02002015 if (kiocb) {
2016 ret = kiocb->ki_cancel(&kiocb->rw);
2017 list_del_init(&kiocb->ki_list);
2018 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 spin_unlock_irq(&ctx->ctx_lock);
2020
Kent Overstreet906b9732013-05-07 16:18:31 -07002021 if (!ret) {
Kent Overstreetbec68faa2013-05-13 14:45:08 -07002022 /*
2023 * The result argument is no longer used - the io_event is
2024 * always delivered via the ring buffer. -EINPROGRESS indicates
2025 * cancellation is progress:
Kent Overstreet906b9732013-05-07 16:18:31 -07002026 */
Kent Overstreetbec68faa2013-05-13 14:45:08 -07002027 ret = -EINPROGRESS;
Kent Overstreet906b9732013-05-07 16:18:31 -07002028 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029
Kent Overstreet723be6e2013-05-28 15:14:48 -07002030 percpu_ref_put(&ctx->users);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031
2032 return ret;
2033}
2034
Deepa Dinamanifa2e62a2017-08-04 21:12:32 -07002035static long do_io_getevents(aio_context_t ctx_id,
2036 long min_nr,
2037 long nr,
2038 struct io_event __user *events,
2039 struct timespec64 *ts)
2040{
2041 ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX;
2042 struct kioctx *ioctx = lookup_ioctx(ctx_id);
2043 long ret = -EINVAL;
2044
2045 if (likely(ioctx)) {
2046 if (likely(min_nr <= nr && min_nr >= 0))
2047 ret = read_events(ioctx, min_nr, nr, events, until);
2048 percpu_ref_put(&ioctx->users);
2049 }
2050
2051 return ret;
2052}
2053
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054/* io_getevents:
2055 * Attempts to read at least min_nr events and up to nr events from
Satoru Takeuchi642b5122010-08-05 11:23:11 -07002056 * the completion queue for the aio_context specified by ctx_id. If
2057 * it succeeds, the number of read events is returned. May fail with
2058 * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
2059 * out of range, if timeout is out of range. May fail with -EFAULT
2060 * if any of the memory specified is invalid. May return 0 or
2061 * < min_nr if the timeout specified by timeout has elapsed
2062 * before sufficient events are available, where timeout == NULL
2063 * specifies an infinite timeout. Note that the timeout pointed to by
Jeff Moyer69008072013-05-24 15:55:24 -07002064 * timeout is relative. Will fail with -ENOSYS if not implemented.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065 */
Heiko Carstens002c8972009-01-14 14:14:18 +01002066SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
2067 long, min_nr,
2068 long, nr,
2069 struct io_event __user *, events,
2070 struct timespec __user *, timeout)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002071{
Deepa Dinamanifa2e62a2017-08-04 21:12:32 -07002072 struct timespec64 ts;
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002073 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002075 if (timeout && unlikely(get_timespec64(&ts, timeout)))
2076 return -EFAULT;
2077
2078 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);
2079 if (!ret && signal_pending(current))
2080 ret = -EINTR;
2081 return ret;
2082}
2083
Christoph Hellwig9ba546c2018-07-11 15:48:46 +02002084struct __aio_sigset {
2085 const sigset_t __user *sigmask;
2086 size_t sigsetsize;
2087};
2088
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002089SYSCALL_DEFINE6(io_pgetevents,
2090 aio_context_t, ctx_id,
2091 long, min_nr,
2092 long, nr,
2093 struct io_event __user *, events,
2094 struct timespec __user *, timeout,
2095 const struct __aio_sigset __user *, usig)
2096{
2097 struct __aio_sigset ksig = { NULL, };
2098 sigset_t ksigmask, sigsaved;
2099 struct timespec64 ts;
2100 int ret;
2101
2102 if (timeout && unlikely(get_timespec64(&ts, timeout)))
2103 return -EFAULT;
2104
2105 if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
2106 return -EFAULT;
2107
2108 if (ksig.sigmask) {
2109 if (ksig.sigsetsize != sizeof(sigset_t))
2110 return -EINVAL;
2111 if (copy_from_user(&ksigmask, ksig.sigmask, sizeof(ksigmask)))
Deepa Dinamanifa2e62a2017-08-04 21:12:32 -07002112 return -EFAULT;
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002113 sigdelsetmask(&ksigmask, sigmask(SIGKILL) | sigmask(SIGSTOP));
2114 sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002115 }
Deepa Dinamanifa2e62a2017-08-04 21:12:32 -07002116
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002117 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);
2118 if (signal_pending(current)) {
2119 if (ksig.sigmask) {
2120 current->saved_sigmask = sigsaved;
2121 set_restore_sigmask();
2122 }
2123
2124 if (!ret)
2125 ret = -ERESTARTNOHAND;
2126 } else {
2127 if (ksig.sigmask)
2128 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2129 }
2130
2131 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002132}
Al Viroc00d2c72016-12-20 07:04:57 -05002133
2134#ifdef CONFIG_COMPAT
2135COMPAT_SYSCALL_DEFINE5(io_getevents, compat_aio_context_t, ctx_id,
2136 compat_long_t, min_nr,
2137 compat_long_t, nr,
2138 struct io_event __user *, events,
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02002139 struct old_timespec32 __user *, timeout)
Al Viroc00d2c72016-12-20 07:04:57 -05002140{
Deepa Dinamanifa2e62a2017-08-04 21:12:32 -07002141 struct timespec64 t;
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002142 int ret;
Al Viroc00d2c72016-12-20 07:04:57 -05002143
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02002144 if (timeout && get_old_timespec32(&t, timeout))
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002145 return -EFAULT;
2146
2147 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);
2148 if (!ret && signal_pending(current))
2149 ret = -EINTR;
2150 return ret;
2151}
2152
2153
2154struct __compat_aio_sigset {
2155 compat_sigset_t __user *sigmask;
2156 compat_size_t sigsetsize;
2157};
2158
2159COMPAT_SYSCALL_DEFINE6(io_pgetevents,
2160 compat_aio_context_t, ctx_id,
2161 compat_long_t, min_nr,
2162 compat_long_t, nr,
2163 struct io_event __user *, events,
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02002164 struct old_timespec32 __user *, timeout,
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002165 const struct __compat_aio_sigset __user *, usig)
2166{
2167 struct __compat_aio_sigset ksig = { NULL, };
2168 sigset_t ksigmask, sigsaved;
2169 struct timespec64 t;
2170 int ret;
2171
Arnd Bergmann9afc5ee2018-07-13 12:52:28 +02002172 if (timeout && get_old_timespec32(&t, timeout))
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002173 return -EFAULT;
2174
2175 if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
2176 return -EFAULT;
2177
2178 if (ksig.sigmask) {
2179 if (ksig.sigsetsize != sizeof(compat_sigset_t))
2180 return -EINVAL;
2181 if (get_compat_sigset(&ksigmask, ksig.sigmask))
Al Viroc00d2c72016-12-20 07:04:57 -05002182 return -EFAULT;
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002183 sigdelsetmask(&ksigmask, sigmask(SIGKILL) | sigmask(SIGSTOP));
2184 sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved);
Al Viroc00d2c72016-12-20 07:04:57 -05002185 }
Deepa Dinamanifa2e62a2017-08-04 21:12:32 -07002186
Christoph Hellwig7a074e92018-05-02 19:51:00 +02002187 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);
2188 if (signal_pending(current)) {
2189 if (ksig.sigmask) {
2190 current->saved_sigmask = sigsaved;
2191 set_restore_sigmask();
2192 }
2193 if (!ret)
2194 ret = -ERESTARTNOHAND;
2195 } else {
2196 if (ksig.sigmask)
2197 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2198 }
2199
2200 return ret;
Al Viroc00d2c72016-12-20 07:04:57 -05002201}
2202#endif