blob: 046fc4e1e1552ee3387f65fa006e798935610dd1 [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
47#include <linux/refcount.h>
48#include <linux/uio.h>
49
50#include <linux/sched/signal.h>
51#include <linux/fs.h>
52#include <linux/file.h>
53#include <linux/fdtable.h>
54#include <linux/mm.h>
55#include <linux/mman.h>
56#include <linux/mmu_context.h>
57#include <linux/percpu.h>
58#include <linux/slab.h>
59#include <linux/workqueue.h>
Jens Axboe6c271ce2019-01-10 11:22:30 -070060#include <linux/kthread.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070061#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070062#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070063#include <linux/net.h>
64#include <net/sock.h>
65#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070066#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070067#include <linux/anon_inodes.h>
68#include <linux/sched/mm.h>
69#include <linux/uaccess.h>
70#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070071#include <linux/sizes.h>
72#include <linux/hugetlb.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070073
74#include <uapi/linux/io_uring.h>
75
76#include "internal.h"
77
78#define IORING_MAX_ENTRIES 4096
Jens Axboe6b063142019-01-10 22:13:58 -070079#define IORING_MAX_FIXED_FILES 1024
Jens Axboe2b188cc2019-01-07 10:46:33 -070080
81struct io_uring {
82 u32 head ____cacheline_aligned_in_smp;
83 u32 tail ____cacheline_aligned_in_smp;
84};
85
Stefan Bühler1e84b972019-04-24 23:54:16 +020086/*
87 * This data is shared with the application through the mmap at offset
88 * IORING_OFF_SQ_RING.
89 *
90 * The offsets to the member fields are published through struct
91 * io_sqring_offsets when calling io_uring_setup.
92 */
Jens Axboe2b188cc2019-01-07 10:46:33 -070093struct io_sq_ring {
Stefan Bühler1e84b972019-04-24 23:54:16 +020094 /*
95 * Head and tail offsets into the ring; the offsets need to be
96 * masked to get valid indices.
97 *
98 * The kernel controls head and the application controls tail.
99 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700100 struct io_uring r;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200101 /*
102 * Bitmask to apply to head and tail offsets (constant, equals
103 * ring_entries - 1)
104 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700105 u32 ring_mask;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200106 /* Ring size (constant, power of 2) */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700107 u32 ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200108 /*
109 * Number of invalid entries dropped by the kernel due to
110 * invalid index stored in array
111 *
112 * Written by the kernel, shouldn't be modified by the
113 * application (i.e. get number of "new events" by comparing to
114 * cached value).
115 *
116 * After a new SQ head value was read by the application this
117 * counter includes all submissions that were dropped reaching
118 * the new SQ head (and possibly more).
119 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700120 u32 dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200121 /*
122 * Runtime flags
123 *
124 * Written by the kernel, shouldn't be modified by the
125 * application.
126 *
127 * The application needs a full memory barrier before checking
128 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
129 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700130 u32 flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200131 /*
132 * Ring buffer of indices into array of io_uring_sqe, which is
133 * mmapped by the application using the IORING_OFF_SQES offset.
134 *
135 * This indirection could e.g. be used to assign fixed
136 * io_uring_sqe entries to operations and only submit them to
137 * the queue when needed.
138 *
139 * The kernel modifies neither the indices array nor the entries
140 * array.
141 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700142 u32 array[];
143};
144
Stefan Bühler1e84b972019-04-24 23:54:16 +0200145/*
146 * This data is shared with the application through the mmap at offset
147 * IORING_OFF_CQ_RING.
148 *
149 * The offsets to the member fields are published through struct
150 * io_cqring_offsets when calling io_uring_setup.
151 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700152struct io_cq_ring {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200153 /*
154 * Head and tail offsets into the ring; the offsets need to be
155 * masked to get valid indices.
156 *
157 * The application controls head and the kernel tail.
158 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700159 struct io_uring r;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200160 /*
161 * Bitmask to apply to head and tail offsets (constant, equals
162 * ring_entries - 1)
163 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700164 u32 ring_mask;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200165 /* Ring size (constant, power of 2) */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700166 u32 ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200167 /*
168 * Number of completion events lost because the queue was full;
169 * this should be avoided by the application by making sure
170 * there are not more requests pending thatn there is space in
171 * the completion queue.
172 *
173 * Written by the kernel, shouldn't be modified by the
174 * application (i.e. get number of "new events" by comparing to
175 * cached value).
176 *
177 * As completion events come in out of order this counter is not
178 * ordered with any other data.
179 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700180 u32 overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200181 /*
182 * Ring buffer of completion events.
183 *
184 * The kernel writes completion events fresh every time they are
185 * produced, so the application is allowed to modify pending
186 * entries.
187 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700188 struct io_uring_cqe cqes[];
189};
190
Jens Axboeedafcce2019-01-09 09:16:05 -0700191struct io_mapped_ubuf {
192 u64 ubuf;
193 size_t len;
194 struct bio_vec *bvec;
195 unsigned int nr_bvecs;
196};
197
Jens Axboe31b51512019-01-18 22:56:34 -0700198struct async_list {
199 spinlock_t lock;
200 atomic_t cnt;
201 struct list_head list;
202
203 struct file *file;
204 off_t io_end;
205 size_t io_pages;
206};
207
Jens Axboe2b188cc2019-01-07 10:46:33 -0700208struct io_ring_ctx {
209 struct {
210 struct percpu_ref refs;
211 } ____cacheline_aligned_in_smp;
212
213 struct {
214 unsigned int flags;
215 bool compat;
216 bool account_mem;
217
218 /* SQ ring */
219 struct io_sq_ring *sq_ring;
220 unsigned cached_sq_head;
221 unsigned sq_entries;
222 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700223 unsigned sq_thread_idle;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700224 struct io_uring_sqe *sq_sqes;
225 } ____cacheline_aligned_in_smp;
226
227 /* IO offload */
228 struct workqueue_struct *sqo_wq;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700229 struct task_struct *sqo_thread; /* if using sq thread polling */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700230 struct mm_struct *sqo_mm;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700231 wait_queue_head_t sqo_wait;
232 unsigned sqo_stop;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700233
234 struct {
235 /* CQ ring */
236 struct io_cq_ring *cq_ring;
237 unsigned cached_cq_tail;
238 unsigned cq_entries;
239 unsigned cq_mask;
240 struct wait_queue_head cq_wait;
241 struct fasync_struct *cq_fasync;
242 } ____cacheline_aligned_in_smp;
243
Jens Axboe6b063142019-01-10 22:13:58 -0700244 /*
245 * If used, fixed file set. Writers must ensure that ->refs is dead,
246 * readers must ensure that ->refs is alive as long as the file* is
247 * used. Only updated through io_uring_register(2).
248 */
249 struct file **user_files;
250 unsigned nr_user_files;
251
Jens Axboeedafcce2019-01-09 09:16:05 -0700252 /* if used, fixed mapped user buffers */
253 unsigned nr_user_bufs;
254 struct io_mapped_ubuf *user_bufs;
255
Jens Axboe2b188cc2019-01-07 10:46:33 -0700256 struct user_struct *user;
257
258 struct completion ctx_done;
259
260 struct {
261 struct mutex uring_lock;
262 wait_queue_head_t wait;
263 } ____cacheline_aligned_in_smp;
264
265 struct {
266 spinlock_t completion_lock;
Jens Axboedef596e2019-01-09 08:59:42 -0700267 bool poll_multi_file;
268 /*
269 * ->poll_list is protected by the ctx->uring_lock for
270 * io_uring instances that don't use IORING_SETUP_SQPOLL.
271 * For SQPOLL, only the single threaded io_sq_thread() will
272 * manipulate the list, hence no extra locking is needed there.
273 */
274 struct list_head poll_list;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700275 struct list_head cancel_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700276 } ____cacheline_aligned_in_smp;
277
Jens Axboe31b51512019-01-18 22:56:34 -0700278 struct async_list pending_async[2];
279
Jens Axboe2b188cc2019-01-07 10:46:33 -0700280#if defined(CONFIG_UNIX)
281 struct socket *ring_sock;
282#endif
283};
284
285struct sqe_submit {
286 const struct io_uring_sqe *sqe;
287 unsigned short index;
288 bool has_user;
Jens Axboedef596e2019-01-09 08:59:42 -0700289 bool needs_lock;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700290 bool needs_fixed_file;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700291};
292
Jens Axboe09bb8392019-03-13 12:39:28 -0600293/*
294 * First field must be the file pointer in all the
295 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
296 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700297struct io_poll_iocb {
298 struct file *file;
299 struct wait_queue_head *head;
300 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600301 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700302 bool canceled;
303 struct wait_queue_entry wait;
304};
305
Jens Axboe09bb8392019-03-13 12:39:28 -0600306/*
307 * NOTE! Each of the iocb union members has the file pointer
308 * as the first entry in their struct definition. So you can
309 * access the file pointer through any of the sub-structs,
310 * or directly as just 'ki_filp' in this struct.
311 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700312struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700313 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600314 struct file *file;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700315 struct kiocb rw;
316 struct io_poll_iocb poll;
317 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700318
319 struct sqe_submit submit;
320
321 struct io_ring_ctx *ctx;
322 struct list_head list;
323 unsigned int flags;
Jens Axboec16361c2019-01-17 08:39:48 -0700324 refcount_t refs;
Stefan Bühler8449eed2019-04-27 20:34:19 +0200325#define REQ_F_NOWAIT 1 /* must not punt to workers */
Jens Axboedef596e2019-01-09 08:59:42 -0700326#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
Jens Axboe6b063142019-01-10 22:13:58 -0700327#define REQ_F_FIXED_FILE 4 /* ctx owns file */
Jens Axboe31b51512019-01-18 22:56:34 -0700328#define REQ_F_SEQ_PREV 8 /* sequential with previous */
Jens Axboed530a402019-03-13 12:15:01 -0600329#define REQ_F_PREPPED 16 /* prep already done */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700330 u64 user_data;
Jens Axboedef596e2019-01-09 08:59:42 -0700331 u64 error;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700332
333 struct work_struct work;
334};
335
336#define IO_PLUG_THRESHOLD 2
Jens Axboedef596e2019-01-09 08:59:42 -0700337#define IO_IOPOLL_BATCH 8
Jens Axboe2b188cc2019-01-07 10:46:33 -0700338
Jens Axboe9a56a232019-01-09 09:06:50 -0700339struct io_submit_state {
340 struct blk_plug plug;
341
342 /*
Jens Axboe2579f912019-01-09 09:10:43 -0700343 * io_kiocb alloc cache
344 */
345 void *reqs[IO_IOPOLL_BATCH];
346 unsigned int free_reqs;
347 unsigned int cur_req;
348
349 /*
Jens Axboe9a56a232019-01-09 09:06:50 -0700350 * File reference cache
351 */
352 struct file *file;
353 unsigned int fd;
354 unsigned int has_refs;
355 unsigned int used_refs;
356 unsigned int ios_left;
357};
358
Jens Axboe2b188cc2019-01-07 10:46:33 -0700359static struct kmem_cache *req_cachep;
360
361static const struct file_operations io_uring_fops;
362
363struct sock *io_uring_get_socket(struct file *file)
364{
365#if defined(CONFIG_UNIX)
366 if (file->f_op == &io_uring_fops) {
367 struct io_ring_ctx *ctx = file->private_data;
368
369 return ctx->ring_sock->sk;
370 }
371#endif
372 return NULL;
373}
374EXPORT_SYMBOL(io_uring_get_socket);
375
376static void io_ring_ctx_ref_free(struct percpu_ref *ref)
377{
378 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
379
380 complete(&ctx->ctx_done);
381}
382
383static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
384{
385 struct io_ring_ctx *ctx;
Jens Axboe31b51512019-01-18 22:56:34 -0700386 int i;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700387
388 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
389 if (!ctx)
390 return NULL;
391
392 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, 0, GFP_KERNEL)) {
393 kfree(ctx);
394 return NULL;
395 }
396
397 ctx->flags = p->flags;
398 init_waitqueue_head(&ctx->cq_wait);
399 init_completion(&ctx->ctx_done);
400 mutex_init(&ctx->uring_lock);
401 init_waitqueue_head(&ctx->wait);
Jens Axboe31b51512019-01-18 22:56:34 -0700402 for (i = 0; i < ARRAY_SIZE(ctx->pending_async); i++) {
403 spin_lock_init(&ctx->pending_async[i].lock);
404 INIT_LIST_HEAD(&ctx->pending_async[i].list);
405 atomic_set(&ctx->pending_async[i].cnt, 0);
406 }
Jens Axboe2b188cc2019-01-07 10:46:33 -0700407 spin_lock_init(&ctx->completion_lock);
Jens Axboedef596e2019-01-09 08:59:42 -0700408 INIT_LIST_HEAD(&ctx->poll_list);
Jens Axboe221c5eb2019-01-17 09:41:58 -0700409 INIT_LIST_HEAD(&ctx->cancel_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700410 return ctx;
411}
412
413static void io_commit_cqring(struct io_ring_ctx *ctx)
414{
415 struct io_cq_ring *ring = ctx->cq_ring;
416
417 if (ctx->cached_cq_tail != READ_ONCE(ring->r.tail)) {
418 /* order cqe stores with ring update */
419 smp_store_release(&ring->r.tail, ctx->cached_cq_tail);
420
Jens Axboe2b188cc2019-01-07 10:46:33 -0700421 if (wq_has_sleeper(&ctx->cq_wait)) {
422 wake_up_interruptible(&ctx->cq_wait);
423 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
424 }
425 }
426}
427
428static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
429{
430 struct io_cq_ring *ring = ctx->cq_ring;
431 unsigned tail;
432
433 tail = ctx->cached_cq_tail;
Stefan Bühler115e12e2019-04-24 23:54:18 +0200434 /*
435 * writes to the cq entry need to come after reading head; the
436 * control dependency is enough as we're using WRITE_ONCE to
437 * fill the cq entry
438 */
Jens Axboe74f464e2019-04-17 08:57:48 -0600439 if (tail - READ_ONCE(ring->r.head) == ring->ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700440 return NULL;
441
442 ctx->cached_cq_tail++;
443 return &ring->cqes[tail & ctx->cq_mask];
444}
445
446static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
447 long res, unsigned ev_flags)
448{
449 struct io_uring_cqe *cqe;
450
451 /*
452 * If we can't get a cq entry, userspace overflowed the
453 * submission (by quite a lot). Increment the overflow count in
454 * the ring.
455 */
456 cqe = io_get_cqring(ctx);
457 if (cqe) {
458 WRITE_ONCE(cqe->user_data, ki_user_data);
459 WRITE_ONCE(cqe->res, res);
460 WRITE_ONCE(cqe->flags, ev_flags);
461 } else {
462 unsigned overflow = READ_ONCE(ctx->cq_ring->overflow);
463
464 WRITE_ONCE(ctx->cq_ring->overflow, overflow + 1);
465 }
466}
467
Jens Axboe8c838782019-03-12 15:48:16 -0600468static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
469{
470 if (waitqueue_active(&ctx->wait))
471 wake_up(&ctx->wait);
472 if (waitqueue_active(&ctx->sqo_wait))
473 wake_up(&ctx->sqo_wait);
474}
475
476static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data,
Jens Axboe2b188cc2019-01-07 10:46:33 -0700477 long res, unsigned ev_flags)
478{
479 unsigned long flags;
480
481 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe8c838782019-03-12 15:48:16 -0600482 io_cqring_fill_event(ctx, user_data, res, ev_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700483 io_commit_cqring(ctx);
484 spin_unlock_irqrestore(&ctx->completion_lock, flags);
485
Jens Axboe8c838782019-03-12 15:48:16 -0600486 io_cqring_ev_posted(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700487}
488
489static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
490{
491 percpu_ref_put_many(&ctx->refs, refs);
492
493 if (waitqueue_active(&ctx->wait))
494 wake_up(&ctx->wait);
495}
496
Jens Axboe2579f912019-01-09 09:10:43 -0700497static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
498 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700499{
Jens Axboefd6fab22019-03-14 16:30:06 -0600500 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700501 struct io_kiocb *req;
502
503 if (!percpu_ref_tryget(&ctx->refs))
504 return NULL;
505
Jens Axboe2579f912019-01-09 09:10:43 -0700506 if (!state) {
Jens Axboefd6fab22019-03-14 16:30:06 -0600507 req = kmem_cache_alloc(req_cachep, gfp);
Jens Axboe2579f912019-01-09 09:10:43 -0700508 if (unlikely(!req))
509 goto out;
510 } else if (!state->free_reqs) {
511 size_t sz;
512 int ret;
513
514 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
Jens Axboefd6fab22019-03-14 16:30:06 -0600515 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
516
517 /*
518 * Bulk alloc is all-or-nothing. If we fail to get a batch,
519 * retry single alloc to be on the safe side.
520 */
521 if (unlikely(ret <= 0)) {
522 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
523 if (!state->reqs[0])
524 goto out;
525 ret = 1;
526 }
Jens Axboe2579f912019-01-09 09:10:43 -0700527 state->free_reqs = ret - 1;
528 state->cur_req = 1;
529 req = state->reqs[0];
530 } else {
531 req = state->reqs[state->cur_req];
532 state->free_reqs--;
533 state->cur_req++;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700534 }
535
Jens Axboe2579f912019-01-09 09:10:43 -0700536 req->ctx = ctx;
537 req->flags = 0;
Jens Axboee65ef562019-03-12 10:16:44 -0600538 /* one is dropped after submission, the other at completion */
539 refcount_set(&req->refs, 2);
Jens Axboe2579f912019-01-09 09:10:43 -0700540 return req;
541out:
Jens Axboe2b188cc2019-01-07 10:46:33 -0700542 io_ring_drop_ctx_refs(ctx, 1);
543 return NULL;
544}
545
Jens Axboedef596e2019-01-09 08:59:42 -0700546static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
547{
548 if (*nr) {
549 kmem_cache_free_bulk(req_cachep, *nr, reqs);
550 io_ring_drop_ctx_refs(ctx, *nr);
551 *nr = 0;
552 }
553}
554
Jens Axboe2b188cc2019-01-07 10:46:33 -0700555static void io_free_req(struct io_kiocb *req)
556{
Jens Axboe09bb8392019-03-13 12:39:28 -0600557 if (req->file && !(req->flags & REQ_F_FIXED_FILE))
558 fput(req->file);
Jens Axboee65ef562019-03-12 10:16:44 -0600559 io_ring_drop_ctx_refs(req->ctx, 1);
560 kmem_cache_free(req_cachep, req);
561}
562
563static void io_put_req(struct io_kiocb *req)
564{
565 if (refcount_dec_and_test(&req->refs))
566 io_free_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700567}
568
Jens Axboedef596e2019-01-09 08:59:42 -0700569/*
570 * Find and free completed poll iocbs
571 */
572static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
573 struct list_head *done)
574{
575 void *reqs[IO_IOPOLL_BATCH];
576 struct io_kiocb *req;
Jens Axboe09bb8392019-03-13 12:39:28 -0600577 int to_free;
Jens Axboedef596e2019-01-09 08:59:42 -0700578
Jens Axboe09bb8392019-03-13 12:39:28 -0600579 to_free = 0;
Jens Axboedef596e2019-01-09 08:59:42 -0700580 while (!list_empty(done)) {
581 req = list_first_entry(done, struct io_kiocb, list);
582 list_del(&req->list);
583
584 io_cqring_fill_event(ctx, req->user_data, req->error, 0);
Jens Axboedef596e2019-01-09 08:59:42 -0700585 (*nr_events)++;
586
Jens Axboe09bb8392019-03-13 12:39:28 -0600587 if (refcount_dec_and_test(&req->refs)) {
588 /* If we're not using fixed files, we have to pair the
589 * completion part with the file put. Use regular
590 * completions for those, only batch free for fixed
591 * file.
592 */
593 if (req->flags & REQ_F_FIXED_FILE) {
594 reqs[to_free++] = req;
595 if (to_free == ARRAY_SIZE(reqs))
596 io_free_req_many(ctx, reqs, &to_free);
Jens Axboe6b063142019-01-10 22:13:58 -0700597 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -0600598 io_free_req(req);
Jens Axboe6b063142019-01-10 22:13:58 -0700599 }
Jens Axboe9a56a232019-01-09 09:06:50 -0700600 }
Jens Axboedef596e2019-01-09 08:59:42 -0700601 }
Jens Axboedef596e2019-01-09 08:59:42 -0700602
Jens Axboe09bb8392019-03-13 12:39:28 -0600603 io_commit_cqring(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -0700604 io_free_req_many(ctx, reqs, &to_free);
605}
606
607static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
608 long min)
609{
610 struct io_kiocb *req, *tmp;
611 LIST_HEAD(done);
612 bool spin;
613 int ret;
614
615 /*
616 * Only spin for completions if we don't have multiple devices hanging
617 * off our complete list, and we're under the requested amount.
618 */
619 spin = !ctx->poll_multi_file && *nr_events < min;
620
621 ret = 0;
622 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
623 struct kiocb *kiocb = &req->rw;
624
625 /*
626 * Move completed entries to our local list. If we find a
627 * request that requires polling, break out and complete
628 * the done list first, if we have entries there.
629 */
630 if (req->flags & REQ_F_IOPOLL_COMPLETED) {
631 list_move_tail(&req->list, &done);
632 continue;
633 }
634 if (!list_empty(&done))
635 break;
636
637 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
638 if (ret < 0)
639 break;
640
641 if (ret && spin)
642 spin = false;
643 ret = 0;
644 }
645
646 if (!list_empty(&done))
647 io_iopoll_complete(ctx, nr_events, &done);
648
649 return ret;
650}
651
652/*
653 * Poll for a mininum of 'min' events. Note that if min == 0 we consider that a
654 * non-spinning poll check - we'll still enter the driver poll loop, but only
655 * as a non-spinning completion check.
656 */
657static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
658 long min)
659{
660 while (!list_empty(&ctx->poll_list)) {
661 int ret;
662
663 ret = io_do_iopoll(ctx, nr_events, min);
664 if (ret < 0)
665 return ret;
666 if (!min || *nr_events >= min)
667 return 0;
668 }
669
670 return 1;
671}
672
673/*
674 * We can't just wait for polled events to come to us, we have to actively
675 * find and complete them.
676 */
677static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
678{
679 if (!(ctx->flags & IORING_SETUP_IOPOLL))
680 return;
681
682 mutex_lock(&ctx->uring_lock);
683 while (!list_empty(&ctx->poll_list)) {
684 unsigned int nr_events = 0;
685
686 io_iopoll_getevents(ctx, &nr_events, 1);
687 }
688 mutex_unlock(&ctx->uring_lock);
689}
690
691static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
692 long min)
693{
694 int ret = 0;
695
696 do {
697 int tmin = 0;
698
699 if (*nr_events < min)
700 tmin = min - *nr_events;
701
702 ret = io_iopoll_getevents(ctx, nr_events, tmin);
703 if (ret <= 0)
704 break;
705 ret = 0;
706 } while (min && !*nr_events && !need_resched());
707
708 return ret;
709}
710
Jens Axboe2b188cc2019-01-07 10:46:33 -0700711static void kiocb_end_write(struct kiocb *kiocb)
712{
713 if (kiocb->ki_flags & IOCB_WRITE) {
714 struct inode *inode = file_inode(kiocb->ki_filp);
715
716 /*
717 * Tell lockdep we inherited freeze protection from submission
718 * thread.
719 */
720 if (S_ISREG(inode->i_mode))
721 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
722 file_end_write(kiocb->ki_filp);
723 }
724}
725
726static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
727{
728 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
729
730 kiocb_end_write(kiocb);
731
Jens Axboe2b188cc2019-01-07 10:46:33 -0700732 io_cqring_add_event(req->ctx, req->user_data, res, 0);
Jens Axboee65ef562019-03-12 10:16:44 -0600733 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700734}
735
Jens Axboedef596e2019-01-09 08:59:42 -0700736static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
737{
738 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
739
740 kiocb_end_write(kiocb);
741
742 req->error = res;
743 if (res != -EAGAIN)
744 req->flags |= REQ_F_IOPOLL_COMPLETED;
745}
746
747/*
748 * After the iocb has been issued, it's safe to be found on the poll list.
749 * Adding the kiocb to the list AFTER submission ensures that we don't
750 * find it from a io_iopoll_getevents() thread before the issuer is done
751 * accessing the kiocb cookie.
752 */
753static void io_iopoll_req_issued(struct io_kiocb *req)
754{
755 struct io_ring_ctx *ctx = req->ctx;
756
757 /*
758 * Track whether we have multiple files in our lists. This will impact
759 * how we do polling eventually, not spinning if we're on potentially
760 * different devices.
761 */
762 if (list_empty(&ctx->poll_list)) {
763 ctx->poll_multi_file = false;
764 } else if (!ctx->poll_multi_file) {
765 struct io_kiocb *list_req;
766
767 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
768 list);
769 if (list_req->rw.ki_filp != req->rw.ki_filp)
770 ctx->poll_multi_file = true;
771 }
772
773 /*
774 * For fast devices, IO may have already completed. If it has, add
775 * it to the front so we find it first.
776 */
777 if (req->flags & REQ_F_IOPOLL_COMPLETED)
778 list_add(&req->list, &ctx->poll_list);
779 else
780 list_add_tail(&req->list, &ctx->poll_list);
781}
782
Jens Axboe3d6770f2019-04-13 11:50:54 -0600783static void io_file_put(struct io_submit_state *state)
Jens Axboe9a56a232019-01-09 09:06:50 -0700784{
Jens Axboe3d6770f2019-04-13 11:50:54 -0600785 if (state->file) {
Jens Axboe9a56a232019-01-09 09:06:50 -0700786 int diff = state->has_refs - state->used_refs;
787
788 if (diff)
789 fput_many(state->file, diff);
790 state->file = NULL;
791 }
792}
793
794/*
795 * Get as many references to a file as we have IOs left in this submission,
796 * assuming most submissions are for one file, or at least that each file
797 * has more than one submission.
798 */
799static struct file *io_file_get(struct io_submit_state *state, int fd)
800{
801 if (!state)
802 return fget(fd);
803
804 if (state->file) {
805 if (state->fd == fd) {
806 state->used_refs++;
807 state->ios_left--;
808 return state->file;
809 }
Jens Axboe3d6770f2019-04-13 11:50:54 -0600810 io_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -0700811 }
812 state->file = fget_many(fd, state->ios_left);
813 if (!state->file)
814 return NULL;
815
816 state->fd = fd;
817 state->has_refs = state->ios_left;
818 state->used_refs = 1;
819 state->ios_left--;
820 return state->file;
821}
822
Jens Axboe2b188cc2019-01-07 10:46:33 -0700823/*
824 * If we tracked the file through the SCM inflight mechanism, we could support
825 * any file. For now, just ensure that anything potentially problematic is done
826 * inline.
827 */
828static bool io_file_supports_async(struct file *file)
829{
830 umode_t mode = file_inode(file)->i_mode;
831
832 if (S_ISBLK(mode) || S_ISCHR(mode))
833 return true;
834 if (S_ISREG(mode) && file->f_op != &io_uring_fops)
835 return true;
836
837 return false;
838}
839
Jens Axboe6c271ce2019-01-10 11:22:30 -0700840static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s,
Jens Axboe8358e3a2019-04-23 08:17:58 -0600841 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700842{
Jens Axboe6c271ce2019-01-10 11:22:30 -0700843 const struct io_uring_sqe *sqe = s->sqe;
Jens Axboedef596e2019-01-09 08:59:42 -0700844 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700845 struct kiocb *kiocb = &req->rw;
Jens Axboe09bb8392019-03-13 12:39:28 -0600846 unsigned ioprio;
847 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700848
Jens Axboe09bb8392019-03-13 12:39:28 -0600849 if (!req->file)
850 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700851 /* For -EAGAIN retry, everything is already prepped */
Jens Axboed530a402019-03-13 12:15:01 -0600852 if (req->flags & REQ_F_PREPPED)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700853 return 0;
854
Jens Axboe09bb8392019-03-13 12:39:28 -0600855 if (force_nonblock && !io_file_supports_async(req->file))
856 force_nonblock = false;
Jens Axboe6b063142019-01-10 22:13:58 -0700857
Jens Axboe2b188cc2019-01-07 10:46:33 -0700858 kiocb->ki_pos = READ_ONCE(sqe->off);
859 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
860 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
861
862 ioprio = READ_ONCE(sqe->ioprio);
863 if (ioprio) {
864 ret = ioprio_check_cap(ioprio);
865 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -0600866 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700867
868 kiocb->ki_ioprio = ioprio;
869 } else
870 kiocb->ki_ioprio = get_current_ioprio();
871
872 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
873 if (unlikely(ret))
Jens Axboe09bb8392019-03-13 12:39:28 -0600874 return ret;
Stefan Bühler8449eed2019-04-27 20:34:19 +0200875
876 /* don't allow async punt if RWF_NOWAIT was requested */
877 if (kiocb->ki_flags & IOCB_NOWAIT)
878 req->flags |= REQ_F_NOWAIT;
879
880 if (force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700881 kiocb->ki_flags |= IOCB_NOWAIT;
Stefan Bühler8449eed2019-04-27 20:34:19 +0200882
Jens Axboedef596e2019-01-09 08:59:42 -0700883 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -0700884 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
885 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -0600886 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700887
Jens Axboedef596e2019-01-09 08:59:42 -0700888 req->error = 0;
889 kiocb->ki_flags |= IOCB_HIPRI;
890 kiocb->ki_complete = io_complete_rw_iopoll;
891 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -0600892 if (kiocb->ki_flags & IOCB_HIPRI)
893 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -0700894 kiocb->ki_complete = io_complete_rw;
895 }
Jens Axboed530a402019-03-13 12:15:01 -0600896 req->flags |= REQ_F_PREPPED;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700897 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700898}
899
900static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
901{
902 switch (ret) {
903 case -EIOCBQUEUED:
904 break;
905 case -ERESTARTSYS:
906 case -ERESTARTNOINTR:
907 case -ERESTARTNOHAND:
908 case -ERESTART_RESTARTBLOCK:
909 /*
910 * We can't just restart the syscall, since previously
911 * submitted sqes may already be in progress. Just fail this
912 * IO with EINTR.
913 */
914 ret = -EINTR;
915 /* fall through */
916 default:
917 kiocb->ki_complete(kiocb, ret, 0);
918 }
919}
920
Jens Axboeedafcce2019-01-09 09:16:05 -0700921static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
922 const struct io_uring_sqe *sqe,
923 struct iov_iter *iter)
924{
925 size_t len = READ_ONCE(sqe->len);
926 struct io_mapped_ubuf *imu;
927 unsigned index, buf_index;
928 size_t offset;
929 u64 buf_addr;
930
931 /* attempt to use fixed buffers without having provided iovecs */
932 if (unlikely(!ctx->user_bufs))
933 return -EFAULT;
934
935 buf_index = READ_ONCE(sqe->buf_index);
936 if (unlikely(buf_index >= ctx->nr_user_bufs))
937 return -EFAULT;
938
939 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
940 imu = &ctx->user_bufs[index];
941 buf_addr = READ_ONCE(sqe->addr);
942
943 /* overflow */
944 if (buf_addr + len < buf_addr)
945 return -EFAULT;
946 /* not inside the mapped region */
947 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
948 return -EFAULT;
949
950 /*
951 * May not be a start of buffer, set size appropriately
952 * and advance us to the beginning.
953 */
954 offset = buf_addr - imu->ubuf;
955 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
956 if (offset)
957 iov_iter_advance(iter, offset);
Jens Axboe875f1d02019-02-27 13:05:25 -0700958
959 /* don't drop a reference to these pages */
960 iter->type |= ITER_BVEC_FLAG_NO_REF;
Jens Axboeedafcce2019-01-09 09:16:05 -0700961 return 0;
962}
963
Jens Axboe2b188cc2019-01-07 10:46:33 -0700964static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
965 const struct sqe_submit *s, struct iovec **iovec,
966 struct iov_iter *iter)
967{
968 const struct io_uring_sqe *sqe = s->sqe;
969 void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
970 size_t sqe_len = READ_ONCE(sqe->len);
Jens Axboeedafcce2019-01-09 09:16:05 -0700971 u8 opcode;
972
973 /*
974 * We're reading ->opcode for the second time, but the first read
975 * doesn't care whether it's _FIXED or not, so it doesn't matter
976 * whether ->opcode changes concurrently. The first read does care
977 * about whether it is a READ or a WRITE, so we don't trust this read
978 * for that purpose and instead let the caller pass in the read/write
979 * flag.
980 */
981 opcode = READ_ONCE(sqe->opcode);
982 if (opcode == IORING_OP_READ_FIXED ||
983 opcode == IORING_OP_WRITE_FIXED) {
Jens Axboee0c5c572019-03-12 10:18:47 -0600984 int ret = io_import_fixed(ctx, rw, sqe, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -0700985 *iovec = NULL;
986 return ret;
987 }
Jens Axboe2b188cc2019-01-07 10:46:33 -0700988
989 if (!s->has_user)
990 return -EFAULT;
991
992#ifdef CONFIG_COMPAT
993 if (ctx->compat)
994 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
995 iovec, iter);
996#endif
997
998 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
999}
1000
Jens Axboe31b51512019-01-18 22:56:34 -07001001/*
1002 * Make a note of the last file/offset/direction we punted to async
1003 * context. We'll use this information to see if we can piggy back a
1004 * sequential request onto the previous one, if it's still hasn't been
1005 * completed by the async worker.
1006 */
1007static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
1008{
1009 struct async_list *async_list = &req->ctx->pending_async[rw];
1010 struct kiocb *kiocb = &req->rw;
1011 struct file *filp = kiocb->ki_filp;
1012 off_t io_end = kiocb->ki_pos + len;
1013
1014 if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) {
1015 unsigned long max_pages;
1016
1017 /* Use 8x RA size as a decent limiter for both reads/writes */
1018 max_pages = filp->f_ra.ra_pages;
1019 if (!max_pages)
Nikolay Borisovb5420232019-03-11 23:28:13 -07001020 max_pages = VM_READAHEAD_PAGES;
Jens Axboe31b51512019-01-18 22:56:34 -07001021 max_pages *= 8;
1022
1023 /* If max pages are exceeded, reset the state */
1024 len >>= PAGE_SHIFT;
1025 if (async_list->io_pages + len <= max_pages) {
1026 req->flags |= REQ_F_SEQ_PREV;
1027 async_list->io_pages += len;
1028 } else {
1029 io_end = 0;
1030 async_list->io_pages = 0;
1031 }
1032 }
1033
1034 /* New file? Reset state. */
1035 if (async_list->file != filp) {
1036 async_list->io_pages = 0;
1037 async_list->file = filp;
1038 }
1039 async_list->io_end = io_end;
1040}
1041
Jens Axboee0c5c572019-03-12 10:18:47 -06001042static int io_read(struct io_kiocb *req, const struct sqe_submit *s,
Jens Axboe8358e3a2019-04-23 08:17:58 -06001043 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001044{
1045 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1046 struct kiocb *kiocb = &req->rw;
1047 struct iov_iter iter;
1048 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001049 size_t iov_count;
Jens Axboee0c5c572019-03-12 10:18:47 -06001050 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001051
Jens Axboe8358e3a2019-04-23 08:17:58 -06001052 ret = io_prep_rw(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001053 if (ret)
1054 return ret;
1055 file = kiocb->ki_filp;
1056
Jens Axboe2b188cc2019-01-07 10:46:33 -07001057 if (unlikely(!(file->f_mode & FMODE_READ)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001058 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001059 if (unlikely(!file->f_op->read_iter))
Jens Axboe09bb8392019-03-13 12:39:28 -06001060 return -EINVAL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001061
1062 ret = io_import_iovec(req->ctx, READ, s, &iovec, &iter);
1063 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06001064 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001065
Jens Axboe31b51512019-01-18 22:56:34 -07001066 iov_count = iov_iter_count(&iter);
1067 ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001068 if (!ret) {
1069 ssize_t ret2;
1070
1071 /* Catch -EAGAIN return for forced non-blocking submission */
1072 ret2 = call_read_iter(file, kiocb, &iter);
Jens Axboe31b51512019-01-18 22:56:34 -07001073 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001074 io_rw_done(kiocb, ret2);
Jens Axboe31b51512019-01-18 22:56:34 -07001075 } else {
1076 /*
1077 * If ->needs_lock is true, we're already in async
1078 * context.
1079 */
1080 if (!s->needs_lock)
1081 io_async_list_note(READ, req, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001082 ret = -EAGAIN;
Jens Axboe31b51512019-01-18 22:56:34 -07001083 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001084 }
1085 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001086 return ret;
1087}
1088
Jens Axboee0c5c572019-03-12 10:18:47 -06001089static int io_write(struct io_kiocb *req, const struct sqe_submit *s,
Jens Axboe8358e3a2019-04-23 08:17:58 -06001090 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001091{
1092 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1093 struct kiocb *kiocb = &req->rw;
1094 struct iov_iter iter;
1095 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001096 size_t iov_count;
Jens Axboee0c5c572019-03-12 10:18:47 -06001097 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001098
Jens Axboe8358e3a2019-04-23 08:17:58 -06001099 ret = io_prep_rw(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001100 if (ret)
1101 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001102
Jens Axboe2b188cc2019-01-07 10:46:33 -07001103 file = kiocb->ki_filp;
1104 if (unlikely(!(file->f_mode & FMODE_WRITE)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001105 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001106 if (unlikely(!file->f_op->write_iter))
Jens Axboe09bb8392019-03-13 12:39:28 -06001107 return -EINVAL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001108
1109 ret = io_import_iovec(req->ctx, WRITE, s, &iovec, &iter);
1110 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06001111 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001112
Jens Axboe31b51512019-01-18 22:56:34 -07001113 iov_count = iov_iter_count(&iter);
1114
1115 ret = -EAGAIN;
1116 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT)) {
1117 /* If ->needs_lock is true, we're already in async context. */
1118 if (!s->needs_lock)
1119 io_async_list_note(WRITE, req, iov_count);
1120 goto out_free;
1121 }
1122
1123 ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001124 if (!ret) {
Roman Penyaev9bf79332019-03-25 20:09:24 +01001125 ssize_t ret2;
1126
Jens Axboe2b188cc2019-01-07 10:46:33 -07001127 /*
1128 * Open-code file_start_write here to grab freeze protection,
1129 * which will be released by another thread in
1130 * io_complete_rw(). Fool lockdep by telling it the lock got
1131 * released so that it doesn't complain about the held lock when
1132 * we return to userspace.
1133 */
1134 if (S_ISREG(file_inode(file)->i_mode)) {
1135 __sb_start_write(file_inode(file)->i_sb,
1136 SB_FREEZE_WRITE, true);
1137 __sb_writers_release(file_inode(file)->i_sb,
1138 SB_FREEZE_WRITE);
1139 }
1140 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01001141
1142 ret2 = call_write_iter(file, kiocb, &iter);
1143 if (!force_nonblock || ret2 != -EAGAIN) {
1144 io_rw_done(kiocb, ret2);
1145 } else {
1146 /*
1147 * If ->needs_lock is true, we're already in async
1148 * context.
1149 */
1150 if (!s->needs_lock)
1151 io_async_list_note(WRITE, req, iov_count);
1152 ret = -EAGAIN;
1153 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001154 }
Jens Axboe31b51512019-01-18 22:56:34 -07001155out_free:
Jens Axboe2b188cc2019-01-07 10:46:33 -07001156 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001157 return ret;
1158}
1159
1160/*
1161 * IORING_OP_NOP just posts a completion event, nothing else.
1162 */
1163static int io_nop(struct io_kiocb *req, u64 user_data)
1164{
1165 struct io_ring_ctx *ctx = req->ctx;
1166 long err = 0;
1167
Jens Axboedef596e2019-01-09 08:59:42 -07001168 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1169 return -EINVAL;
1170
Jens Axboe2b188cc2019-01-07 10:46:33 -07001171 io_cqring_add_event(ctx, user_data, err, 0);
Jens Axboee65ef562019-03-12 10:16:44 -06001172 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001173 return 0;
1174}
1175
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001176static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1177{
Jens Axboe6b063142019-01-10 22:13:58 -07001178 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001179
Jens Axboe09bb8392019-03-13 12:39:28 -06001180 if (!req->file)
1181 return -EBADF;
Jens Axboed530a402019-03-13 12:15:01 -06001182 /* Prep already done (EAGAIN retry) */
1183 if (req->flags & REQ_F_PREPPED)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001184 return 0;
1185
Jens Axboe6b063142019-01-10 22:13:58 -07001186 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07001187 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07001188 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001189 return -EINVAL;
1190
Jens Axboed530a402019-03-13 12:15:01 -06001191 req->flags |= REQ_F_PREPPED;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001192 return 0;
1193}
1194
1195static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1196 bool force_nonblock)
1197{
1198 loff_t sqe_off = READ_ONCE(sqe->off);
1199 loff_t sqe_len = READ_ONCE(sqe->len);
1200 loff_t end = sqe_off + sqe_len;
1201 unsigned fsync_flags;
1202 int ret;
1203
1204 fsync_flags = READ_ONCE(sqe->fsync_flags);
1205 if (unlikely(fsync_flags & ~IORING_FSYNC_DATASYNC))
1206 return -EINVAL;
1207
1208 ret = io_prep_fsync(req, sqe);
1209 if (ret)
1210 return ret;
1211
1212 /* fsync always requires a blocking context */
1213 if (force_nonblock)
1214 return -EAGAIN;
1215
1216 ret = vfs_fsync_range(req->rw.ki_filp, sqe_off,
1217 end > 0 ? end : LLONG_MAX,
1218 fsync_flags & IORING_FSYNC_DATASYNC);
1219
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001220 io_cqring_add_event(req->ctx, sqe->user_data, ret, 0);
Jens Axboee65ef562019-03-12 10:16:44 -06001221 io_put_req(req);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001222 return 0;
1223}
1224
Jens Axboe221c5eb2019-01-17 09:41:58 -07001225static void io_poll_remove_one(struct io_kiocb *req)
1226{
1227 struct io_poll_iocb *poll = &req->poll;
1228
1229 spin_lock(&poll->head->lock);
1230 WRITE_ONCE(poll->canceled, true);
1231 if (!list_empty(&poll->wait.entry)) {
1232 list_del_init(&poll->wait.entry);
1233 queue_work(req->ctx->sqo_wq, &req->work);
1234 }
1235 spin_unlock(&poll->head->lock);
1236
1237 list_del_init(&req->list);
1238}
1239
1240static void io_poll_remove_all(struct io_ring_ctx *ctx)
1241{
1242 struct io_kiocb *req;
1243
1244 spin_lock_irq(&ctx->completion_lock);
1245 while (!list_empty(&ctx->cancel_list)) {
1246 req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list);
1247 io_poll_remove_one(req);
1248 }
1249 spin_unlock_irq(&ctx->completion_lock);
1250}
1251
1252/*
1253 * Find a running poll command that matches one specified in sqe->addr,
1254 * and remove it if found.
1255 */
1256static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1257{
1258 struct io_ring_ctx *ctx = req->ctx;
1259 struct io_kiocb *poll_req, *next;
1260 int ret = -ENOENT;
1261
1262 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1263 return -EINVAL;
1264 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
1265 sqe->poll_events)
1266 return -EINVAL;
1267
1268 spin_lock_irq(&ctx->completion_lock);
1269 list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
1270 if (READ_ONCE(sqe->addr) == poll_req->user_data) {
1271 io_poll_remove_one(poll_req);
1272 ret = 0;
1273 break;
1274 }
1275 }
1276 spin_unlock_irq(&ctx->completion_lock);
1277
1278 io_cqring_add_event(req->ctx, sqe->user_data, ret, 0);
Jens Axboee65ef562019-03-12 10:16:44 -06001279 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001280 return 0;
1281}
1282
Jens Axboe8c838782019-03-12 15:48:16 -06001283static void io_poll_complete(struct io_ring_ctx *ctx, struct io_kiocb *req,
1284 __poll_t mask)
Jens Axboe221c5eb2019-01-17 09:41:58 -07001285{
Jens Axboe8c838782019-03-12 15:48:16 -06001286 req->poll.done = true;
1287 io_cqring_fill_event(ctx, req->user_data, mangle_poll(mask), 0);
1288 io_commit_cqring(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001289}
1290
1291static void io_poll_complete_work(struct work_struct *work)
1292{
1293 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
1294 struct io_poll_iocb *poll = &req->poll;
1295 struct poll_table_struct pt = { ._key = poll->events };
1296 struct io_ring_ctx *ctx = req->ctx;
1297 __poll_t mask = 0;
1298
1299 if (!READ_ONCE(poll->canceled))
1300 mask = vfs_poll(poll->file, &pt) & poll->events;
1301
1302 /*
1303 * Note that ->ki_cancel callers also delete iocb from active_reqs after
1304 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
1305 * synchronize with them. In the cancellation case the list_del_init
1306 * itself is not actually needed, but harmless so we keep it in to
1307 * avoid further branches in the fast path.
1308 */
1309 spin_lock_irq(&ctx->completion_lock);
1310 if (!mask && !READ_ONCE(poll->canceled)) {
1311 add_wait_queue(poll->head, &poll->wait);
1312 spin_unlock_irq(&ctx->completion_lock);
1313 return;
1314 }
1315 list_del_init(&req->list);
Jens Axboe8c838782019-03-12 15:48:16 -06001316 io_poll_complete(ctx, req, mask);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001317 spin_unlock_irq(&ctx->completion_lock);
1318
Jens Axboe8c838782019-03-12 15:48:16 -06001319 io_cqring_ev_posted(ctx);
1320 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001321}
1322
1323static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
1324 void *key)
1325{
1326 struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
1327 wait);
1328 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
1329 struct io_ring_ctx *ctx = req->ctx;
1330 __poll_t mask = key_to_poll(key);
Jens Axboe8c838782019-03-12 15:48:16 -06001331 unsigned long flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001332
1333 /* for instances that support it check for an event match first: */
Jens Axboe8c838782019-03-12 15:48:16 -06001334 if (mask && !(mask & poll->events))
1335 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001336
1337 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06001338
1339 if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) {
1340 list_del(&req->list);
1341 io_poll_complete(ctx, req, mask);
1342 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1343
1344 io_cqring_ev_posted(ctx);
1345 io_put_req(req);
1346 } else {
1347 queue_work(ctx->sqo_wq, &req->work);
1348 }
1349
Jens Axboe221c5eb2019-01-17 09:41:58 -07001350 return 1;
1351}
1352
1353struct io_poll_table {
1354 struct poll_table_struct pt;
1355 struct io_kiocb *req;
1356 int error;
1357};
1358
1359static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
1360 struct poll_table_struct *p)
1361{
1362 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
1363
1364 if (unlikely(pt->req->poll.head)) {
1365 pt->error = -EINVAL;
1366 return;
1367 }
1368
1369 pt->error = 0;
1370 pt->req->poll.head = head;
1371 add_wait_queue(head, &pt->req->poll.wait);
1372}
1373
1374static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1375{
1376 struct io_poll_iocb *poll = &req->poll;
1377 struct io_ring_ctx *ctx = req->ctx;
1378 struct io_poll_table ipt;
Jens Axboe8c838782019-03-12 15:48:16 -06001379 bool cancel = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001380 __poll_t mask;
1381 u16 events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001382
1383 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1384 return -EINVAL;
1385 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
1386 return -EINVAL;
Jens Axboe09bb8392019-03-13 12:39:28 -06001387 if (!poll->file)
1388 return -EBADF;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001389
1390 INIT_WORK(&req->work, io_poll_complete_work);
1391 events = READ_ONCE(sqe->poll_events);
1392 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
1393
Jens Axboe221c5eb2019-01-17 09:41:58 -07001394 poll->head = NULL;
Jens Axboe8c838782019-03-12 15:48:16 -06001395 poll->done = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001396 poll->canceled = false;
1397
1398 ipt.pt._qproc = io_poll_queue_proc;
1399 ipt.pt._key = poll->events;
1400 ipt.req = req;
1401 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
1402
1403 /* initialized the list so that we can do list_empty checks */
1404 INIT_LIST_HEAD(&poll->wait.entry);
1405 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
1406
Jens Axboe221c5eb2019-01-17 09:41:58 -07001407 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001408
1409 spin_lock_irq(&ctx->completion_lock);
Jens Axboe8c838782019-03-12 15:48:16 -06001410 if (likely(poll->head)) {
1411 spin_lock(&poll->head->lock);
1412 if (unlikely(list_empty(&poll->wait.entry))) {
1413 if (ipt.error)
1414 cancel = true;
1415 ipt.error = 0;
1416 mask = 0;
1417 }
1418 if (mask || ipt.error)
1419 list_del_init(&poll->wait.entry);
1420 else if (cancel)
1421 WRITE_ONCE(poll->canceled, true);
1422 else if (!poll->done) /* actually waiting for an event */
1423 list_add_tail(&req->list, &ctx->cancel_list);
1424 spin_unlock(&poll->head->lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001425 }
Jens Axboe8c838782019-03-12 15:48:16 -06001426 if (mask) { /* no async, we'd stolen it */
1427 req->error = mangle_poll(mask);
1428 ipt.error = 0;
1429 io_poll_complete(ctx, req, mask);
1430 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07001431 spin_unlock_irq(&ctx->completion_lock);
1432
Jens Axboe8c838782019-03-12 15:48:16 -06001433 if (mask) {
1434 io_cqring_ev_posted(ctx);
Jens Axboee65ef562019-03-12 10:16:44 -06001435 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001436 }
Jens Axboe8c838782019-03-12 15:48:16 -06001437 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001438}
1439
Jens Axboe2b188cc2019-01-07 10:46:33 -07001440static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Jens Axboe8358e3a2019-04-23 08:17:58 -06001441 const struct sqe_submit *s, bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001442{
Jens Axboee0c5c572019-03-12 10:18:47 -06001443 int ret, opcode;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001444
1445 if (unlikely(s->index >= ctx->sq_entries))
1446 return -EINVAL;
1447 req->user_data = READ_ONCE(s->sqe->user_data);
1448
1449 opcode = READ_ONCE(s->sqe->opcode);
1450 switch (opcode) {
1451 case IORING_OP_NOP:
1452 ret = io_nop(req, req->user_data);
1453 break;
1454 case IORING_OP_READV:
Jens Axboeedafcce2019-01-09 09:16:05 -07001455 if (unlikely(s->sqe->buf_index))
1456 return -EINVAL;
Jens Axboe8358e3a2019-04-23 08:17:58 -06001457 ret = io_read(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001458 break;
1459 case IORING_OP_WRITEV:
Jens Axboeedafcce2019-01-09 09:16:05 -07001460 if (unlikely(s->sqe->buf_index))
1461 return -EINVAL;
Jens Axboe8358e3a2019-04-23 08:17:58 -06001462 ret = io_write(req, s, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07001463 break;
1464 case IORING_OP_READ_FIXED:
Jens Axboe8358e3a2019-04-23 08:17:58 -06001465 ret = io_read(req, s, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07001466 break;
1467 case IORING_OP_WRITE_FIXED:
Jens Axboe8358e3a2019-04-23 08:17:58 -06001468 ret = io_write(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001469 break;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001470 case IORING_OP_FSYNC:
1471 ret = io_fsync(req, s->sqe, force_nonblock);
1472 break;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001473 case IORING_OP_POLL_ADD:
1474 ret = io_poll_add(req, s->sqe);
1475 break;
1476 case IORING_OP_POLL_REMOVE:
1477 ret = io_poll_remove(req, s->sqe);
1478 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001479 default:
1480 ret = -EINVAL;
1481 break;
1482 }
1483
Jens Axboedef596e2019-01-09 08:59:42 -07001484 if (ret)
1485 return ret;
1486
1487 if (ctx->flags & IORING_SETUP_IOPOLL) {
1488 if (req->error == -EAGAIN)
1489 return -EAGAIN;
1490
1491 /* workqueue context doesn't hold uring_lock, grab it now */
1492 if (s->needs_lock)
1493 mutex_lock(&ctx->uring_lock);
1494 io_iopoll_req_issued(req);
1495 if (s->needs_lock)
1496 mutex_unlock(&ctx->uring_lock);
1497 }
1498
1499 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001500}
1501
Jens Axboe31b51512019-01-18 22:56:34 -07001502static struct async_list *io_async_list_from_sqe(struct io_ring_ctx *ctx,
1503 const struct io_uring_sqe *sqe)
1504{
1505 switch (sqe->opcode) {
1506 case IORING_OP_READV:
1507 case IORING_OP_READ_FIXED:
1508 return &ctx->pending_async[READ];
1509 case IORING_OP_WRITEV:
1510 case IORING_OP_WRITE_FIXED:
1511 return &ctx->pending_async[WRITE];
1512 default:
1513 return NULL;
1514 }
1515}
1516
Jens Axboeedafcce2019-01-09 09:16:05 -07001517static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
1518{
1519 u8 opcode = READ_ONCE(sqe->opcode);
1520
1521 return !(opcode == IORING_OP_READ_FIXED ||
1522 opcode == IORING_OP_WRITE_FIXED);
1523}
1524
Jens Axboe2b188cc2019-01-07 10:46:33 -07001525static void io_sq_wq_submit_work(struct work_struct *work)
1526{
1527 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001528 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe31b51512019-01-18 22:56:34 -07001529 struct mm_struct *cur_mm = NULL;
1530 struct async_list *async_list;
1531 LIST_HEAD(req_list);
Jens Axboeedafcce2019-01-09 09:16:05 -07001532 mm_segment_t old_fs;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001533 int ret;
1534
Jens Axboe31b51512019-01-18 22:56:34 -07001535 async_list = io_async_list_from_sqe(ctx, req->submit.sqe);
1536restart:
1537 do {
1538 struct sqe_submit *s = &req->submit;
1539 const struct io_uring_sqe *sqe = s->sqe;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001540
Stefan Bühler8449eed2019-04-27 20:34:19 +02001541 /* Ensure we clear previously set non-block flag */
Jens Axboe31b51512019-01-18 22:56:34 -07001542 req->rw.ki_flags &= ~IOCB_NOWAIT;
1543
1544 ret = 0;
1545 if (io_sqe_needs_user(sqe) && !cur_mm) {
1546 if (!mmget_not_zero(ctx->sqo_mm)) {
1547 ret = -EFAULT;
1548 } else {
1549 cur_mm = ctx->sqo_mm;
1550 use_mm(cur_mm);
1551 old_fs = get_fs();
1552 set_fs(USER_DS);
1553 }
1554 }
1555
1556 if (!ret) {
1557 s->has_user = cur_mm != NULL;
1558 s->needs_lock = true;
1559 do {
Jens Axboe8358e3a2019-04-23 08:17:58 -06001560 ret = __io_submit_sqe(ctx, req, s, false);
Jens Axboe31b51512019-01-18 22:56:34 -07001561 /*
1562 * We can get EAGAIN for polled IO even though
1563 * we're forcing a sync submission from here,
1564 * since we can't wait for request slots on the
1565 * block side.
1566 */
1567 if (ret != -EAGAIN)
1568 break;
1569 cond_resched();
1570 } while (1);
Jens Axboee65ef562019-03-12 10:16:44 -06001571
1572 /* drop submission reference */
1573 io_put_req(req);
Jens Axboe31b51512019-01-18 22:56:34 -07001574 }
1575 if (ret) {
1576 io_cqring_add_event(ctx, sqe->user_data, ret, 0);
Jens Axboee65ef562019-03-12 10:16:44 -06001577 io_put_req(req);
Jens Axboe31b51512019-01-18 22:56:34 -07001578 }
1579
1580 /* async context always use a copy of the sqe */
1581 kfree(sqe);
1582
1583 if (!async_list)
1584 break;
1585 if (!list_empty(&req_list)) {
1586 req = list_first_entry(&req_list, struct io_kiocb,
1587 list);
1588 list_del(&req->list);
1589 continue;
1590 }
1591 if (list_empty(&async_list->list))
1592 break;
1593
1594 req = NULL;
1595 spin_lock(&async_list->lock);
1596 if (list_empty(&async_list->list)) {
1597 spin_unlock(&async_list->lock);
1598 break;
1599 }
1600 list_splice_init(&async_list->list, &req_list);
1601 spin_unlock(&async_list->lock);
1602
1603 req = list_first_entry(&req_list, struct io_kiocb, list);
1604 list_del(&req->list);
1605 } while (req);
Jens Axboeedafcce2019-01-09 09:16:05 -07001606
1607 /*
Jens Axboe31b51512019-01-18 22:56:34 -07001608 * Rare case of racing with a submitter. If we find the count has
1609 * dropped to zero AND we have pending work items, then restart
1610 * the processing. This is a tiny race window.
Jens Axboeedafcce2019-01-09 09:16:05 -07001611 */
Jens Axboe31b51512019-01-18 22:56:34 -07001612 if (async_list) {
1613 ret = atomic_dec_return(&async_list->cnt);
1614 while (!ret && !list_empty(&async_list->list)) {
1615 spin_lock(&async_list->lock);
1616 atomic_inc(&async_list->cnt);
1617 list_splice_init(&async_list->list, &req_list);
1618 spin_unlock(&async_list->lock);
1619
1620 if (!list_empty(&req_list)) {
1621 req = list_first_entry(&req_list,
1622 struct io_kiocb, list);
1623 list_del(&req->list);
1624 goto restart;
1625 }
1626 ret = atomic_dec_return(&async_list->cnt);
Jens Axboeedafcce2019-01-09 09:16:05 -07001627 }
Jens Axboeedafcce2019-01-09 09:16:05 -07001628 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001629
Jens Axboe31b51512019-01-18 22:56:34 -07001630 if (cur_mm) {
Jens Axboeedafcce2019-01-09 09:16:05 -07001631 set_fs(old_fs);
Jens Axboe31b51512019-01-18 22:56:34 -07001632 unuse_mm(cur_mm);
1633 mmput(cur_mm);
Jens Axboeedafcce2019-01-09 09:16:05 -07001634 }
Jens Axboe31b51512019-01-18 22:56:34 -07001635}
Jens Axboe2b188cc2019-01-07 10:46:33 -07001636
Jens Axboe31b51512019-01-18 22:56:34 -07001637/*
1638 * See if we can piggy back onto previously submitted work, that is still
1639 * running. We currently only allow this if the new request is sequential
1640 * to the previous one we punted.
1641 */
1642static bool io_add_to_prev_work(struct async_list *list, struct io_kiocb *req)
1643{
1644 bool ret = false;
1645
1646 if (!list)
1647 return false;
1648 if (!(req->flags & REQ_F_SEQ_PREV))
1649 return false;
1650 if (!atomic_read(&list->cnt))
1651 return false;
1652
1653 ret = true;
1654 spin_lock(&list->lock);
1655 list_add_tail(&req->list, &list->list);
1656 if (!atomic_read(&list->cnt)) {
1657 list_del_init(&req->list);
1658 ret = false;
1659 }
1660 spin_unlock(&list->lock);
1661 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001662}
1663
Jens Axboe09bb8392019-03-13 12:39:28 -06001664static bool io_op_needs_file(const struct io_uring_sqe *sqe)
1665{
1666 int op = READ_ONCE(sqe->opcode);
1667
1668 switch (op) {
1669 case IORING_OP_NOP:
1670 case IORING_OP_POLL_REMOVE:
1671 return false;
1672 default:
1673 return true;
1674 }
1675}
1676
1677static int io_req_set_file(struct io_ring_ctx *ctx, const struct sqe_submit *s,
1678 struct io_submit_state *state, struct io_kiocb *req)
1679{
1680 unsigned flags;
1681 int fd;
1682
1683 flags = READ_ONCE(s->sqe->flags);
1684 fd = READ_ONCE(s->sqe->fd);
1685
1686 if (!io_op_needs_file(s->sqe)) {
1687 req->file = NULL;
1688 return 0;
1689 }
1690
1691 if (flags & IOSQE_FIXED_FILE) {
1692 if (unlikely(!ctx->user_files ||
1693 (unsigned) fd >= ctx->nr_user_files))
1694 return -EBADF;
1695 req->file = ctx->user_files[fd];
1696 req->flags |= REQ_F_FIXED_FILE;
1697 } else {
1698 if (s->needs_fixed_file)
1699 return -EBADF;
1700 req->file = io_file_get(state, fd);
1701 if (unlikely(!req->file))
1702 return -EBADF;
1703 }
1704
1705 return 0;
1706}
1707
Jens Axboe9a56a232019-01-09 09:06:50 -07001708static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
1709 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001710{
1711 struct io_kiocb *req;
Jens Axboee0c5c572019-03-12 10:18:47 -06001712 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001713
1714 /* enforce forwards compatibility on users */
Jens Axboe6b063142019-01-10 22:13:58 -07001715 if (unlikely(s->sqe->flags & ~IOSQE_FIXED_FILE))
Jens Axboe2b188cc2019-01-07 10:46:33 -07001716 return -EINVAL;
1717
Jens Axboe2579f912019-01-09 09:10:43 -07001718 req = io_get_req(ctx, state);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001719 if (unlikely(!req))
1720 return -EAGAIN;
1721
Jens Axboe09bb8392019-03-13 12:39:28 -06001722 ret = io_req_set_file(ctx, s, state, req);
1723 if (unlikely(ret))
1724 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001725
Jens Axboe8358e3a2019-04-23 08:17:58 -06001726 ret = __io_submit_sqe(ctx, req, s, true);
Stefan Bühler8449eed2019-04-27 20:34:19 +02001727 if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001728 struct io_uring_sqe *sqe_copy;
1729
1730 sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL);
1731 if (sqe_copy) {
Jens Axboe31b51512019-01-18 22:56:34 -07001732 struct async_list *list;
1733
Jens Axboe2b188cc2019-01-07 10:46:33 -07001734 memcpy(sqe_copy, s->sqe, sizeof(*sqe_copy));
1735 s->sqe = sqe_copy;
1736
1737 memcpy(&req->submit, s, sizeof(*s));
Jens Axboe31b51512019-01-18 22:56:34 -07001738 list = io_async_list_from_sqe(ctx, s->sqe);
1739 if (!io_add_to_prev_work(list, req)) {
1740 if (list)
1741 atomic_inc(&list->cnt);
1742 INIT_WORK(&req->work, io_sq_wq_submit_work);
1743 queue_work(ctx->sqo_wq, &req->work);
1744 }
Jens Axboee65ef562019-03-12 10:16:44 -06001745
1746 /*
1747 * Queued up for async execution, worker will release
1748 * submit reference when the iocb is actually
1749 * submitted.
1750 */
1751 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001752 }
1753 }
Jens Axboee65ef562019-03-12 10:16:44 -06001754
Jens Axboe09bb8392019-03-13 12:39:28 -06001755out:
Jens Axboee65ef562019-03-12 10:16:44 -06001756 /* drop submission reference */
1757 io_put_req(req);
1758
1759 /* and drop final reference, if we failed */
Jens Axboe2b188cc2019-01-07 10:46:33 -07001760 if (ret)
Jens Axboee65ef562019-03-12 10:16:44 -06001761 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001762
1763 return ret;
1764}
1765
Jens Axboe9a56a232019-01-09 09:06:50 -07001766/*
1767 * Batched submission is done, ensure local IO is flushed out.
1768 */
1769static void io_submit_state_end(struct io_submit_state *state)
1770{
1771 blk_finish_plug(&state->plug);
Jens Axboe3d6770f2019-04-13 11:50:54 -06001772 io_file_put(state);
Jens Axboe2579f912019-01-09 09:10:43 -07001773 if (state->free_reqs)
1774 kmem_cache_free_bulk(req_cachep, state->free_reqs,
1775 &state->reqs[state->cur_req]);
Jens Axboe9a56a232019-01-09 09:06:50 -07001776}
1777
1778/*
1779 * Start submission side cache.
1780 */
1781static void io_submit_state_start(struct io_submit_state *state,
1782 struct io_ring_ctx *ctx, unsigned max_ios)
1783{
1784 blk_start_plug(&state->plug);
Jens Axboe2579f912019-01-09 09:10:43 -07001785 state->free_reqs = 0;
Jens Axboe9a56a232019-01-09 09:06:50 -07001786 state->file = NULL;
1787 state->ios_left = max_ios;
1788}
1789
Jens Axboe2b188cc2019-01-07 10:46:33 -07001790static void io_commit_sqring(struct io_ring_ctx *ctx)
1791{
1792 struct io_sq_ring *ring = ctx->sq_ring;
1793
1794 if (ctx->cached_sq_head != READ_ONCE(ring->r.head)) {
1795 /*
1796 * Ensure any loads from the SQEs are done at this point,
1797 * since once we write the new head, the application could
1798 * write new data to them.
1799 */
1800 smp_store_release(&ring->r.head, ctx->cached_sq_head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001801 }
1802}
1803
1804/*
Jens Axboe2b188cc2019-01-07 10:46:33 -07001805 * Fetch an sqe, if one is available. Note that s->sqe will point to memory
1806 * that is mapped by userspace. This means that care needs to be taken to
1807 * ensure that reads are stable, as we cannot rely on userspace always
1808 * being a good citizen. If members of the sqe are validated and then later
1809 * used, it's important that those reads are done through READ_ONCE() to
1810 * prevent a re-load down the line.
1811 */
1812static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
1813{
1814 struct io_sq_ring *ring = ctx->sq_ring;
1815 unsigned head;
1816
1817 /*
1818 * The cached sq head (or cq tail) serves two purposes:
1819 *
1820 * 1) allows us to batch the cost of updating the user visible
1821 * head updates.
1822 * 2) allows the kernel side to track the head on its own, even
1823 * though the application is the one updating it.
1824 */
1825 head = ctx->cached_sq_head;
Stefan Bühlere523a292019-04-19 11:57:44 +02001826 /* make sure SQ entry isn't read before tail */
1827 if (head == smp_load_acquire(&ring->r.tail))
Jens Axboe2b188cc2019-01-07 10:46:33 -07001828 return false;
1829
1830 head = READ_ONCE(ring->array[head & ctx->sq_mask]);
1831 if (head < ctx->sq_entries) {
1832 s->index = head;
1833 s->sqe = &ctx->sq_sqes[head];
1834 ctx->cached_sq_head++;
1835 return true;
1836 }
1837
1838 /* drop invalid entries */
1839 ctx->cached_sq_head++;
1840 ring->dropped++;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001841 return false;
1842}
1843
Jens Axboe6c271ce2019-01-10 11:22:30 -07001844static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes,
1845 unsigned int nr, bool has_user, bool mm_fault)
1846{
1847 struct io_submit_state state, *statep = NULL;
1848 int ret, i, submitted = 0;
1849
1850 if (nr > IO_PLUG_THRESHOLD) {
1851 io_submit_state_start(&state, ctx, nr);
1852 statep = &state;
1853 }
1854
1855 for (i = 0; i < nr; i++) {
1856 if (unlikely(mm_fault)) {
1857 ret = -EFAULT;
1858 } else {
1859 sqes[i].has_user = has_user;
1860 sqes[i].needs_lock = true;
1861 sqes[i].needs_fixed_file = true;
1862 ret = io_submit_sqe(ctx, &sqes[i], statep);
1863 }
1864 if (!ret) {
1865 submitted++;
1866 continue;
1867 }
1868
1869 io_cqring_add_event(ctx, sqes[i].sqe->user_data, ret, 0);
1870 }
1871
1872 if (statep)
1873 io_submit_state_end(&state);
1874
1875 return submitted;
1876}
1877
1878static int io_sq_thread(void *data)
1879{
1880 struct sqe_submit sqes[IO_IOPOLL_BATCH];
1881 struct io_ring_ctx *ctx = data;
1882 struct mm_struct *cur_mm = NULL;
1883 mm_segment_t old_fs;
1884 DEFINE_WAIT(wait);
1885 unsigned inflight;
1886 unsigned long timeout;
1887
1888 old_fs = get_fs();
1889 set_fs(USER_DS);
1890
1891 timeout = inflight = 0;
1892 while (!kthread_should_stop() && !ctx->sqo_stop) {
1893 bool all_fixed, mm_fault = false;
1894 int i;
1895
1896 if (inflight) {
1897 unsigned nr_events = 0;
1898
1899 if (ctx->flags & IORING_SETUP_IOPOLL) {
1900 /*
1901 * We disallow the app entering submit/complete
1902 * with polling, but we still need to lock the
1903 * ring to prevent racing with polled issue
1904 * that got punted to a workqueue.
1905 */
1906 mutex_lock(&ctx->uring_lock);
1907 io_iopoll_check(ctx, &nr_events, 0);
1908 mutex_unlock(&ctx->uring_lock);
1909 } else {
1910 /*
1911 * Normal IO, just pretend everything completed.
1912 * We don't have to poll completions for that.
1913 */
1914 nr_events = inflight;
1915 }
1916
1917 inflight -= nr_events;
1918 if (!inflight)
1919 timeout = jiffies + ctx->sq_thread_idle;
1920 }
1921
1922 if (!io_get_sqring(ctx, &sqes[0])) {
1923 /*
1924 * We're polling. If we're within the defined idle
1925 * period, then let us spin without work before going
1926 * to sleep.
1927 */
1928 if (inflight || !time_after(jiffies, timeout)) {
1929 cpu_relax();
1930 continue;
1931 }
1932
1933 /*
1934 * Drop cur_mm before scheduling, we can't hold it for
1935 * long periods (or over schedule()). Do this before
1936 * adding ourselves to the waitqueue, as the unuse/drop
1937 * may sleep.
1938 */
1939 if (cur_mm) {
1940 unuse_mm(cur_mm);
1941 mmput(cur_mm);
1942 cur_mm = NULL;
1943 }
1944
1945 prepare_to_wait(&ctx->sqo_wait, &wait,
1946 TASK_INTERRUPTIBLE);
1947
1948 /* Tell userspace we may need a wakeup call */
1949 ctx->sq_ring->flags |= IORING_SQ_NEED_WAKEUP;
Stefan Bühler0d7bae62019-04-19 11:57:45 +02001950 /* make sure to read SQ tail after writing flags */
1951 smp_mb();
Jens Axboe6c271ce2019-01-10 11:22:30 -07001952
1953 if (!io_get_sqring(ctx, &sqes[0])) {
1954 if (kthread_should_stop()) {
1955 finish_wait(&ctx->sqo_wait, &wait);
1956 break;
1957 }
1958 if (signal_pending(current))
1959 flush_signals(current);
1960 schedule();
1961 finish_wait(&ctx->sqo_wait, &wait);
1962
1963 ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07001964 continue;
1965 }
1966 finish_wait(&ctx->sqo_wait, &wait);
1967
1968 ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07001969 }
1970
1971 i = 0;
1972 all_fixed = true;
1973 do {
1974 if (all_fixed && io_sqe_needs_user(sqes[i].sqe))
1975 all_fixed = false;
1976
1977 i++;
1978 if (i == ARRAY_SIZE(sqes))
1979 break;
1980 } while (io_get_sqring(ctx, &sqes[i]));
1981
1982 /* Unless all new commands are FIXED regions, grab mm */
1983 if (!all_fixed && !cur_mm) {
1984 mm_fault = !mmget_not_zero(ctx->sqo_mm);
1985 if (!mm_fault) {
1986 use_mm(ctx->sqo_mm);
1987 cur_mm = ctx->sqo_mm;
1988 }
1989 }
1990
1991 inflight += io_submit_sqes(ctx, sqes, i, cur_mm != NULL,
1992 mm_fault);
1993
1994 /* Commit SQ ring head once we've consumed all SQEs */
1995 io_commit_sqring(ctx);
1996 }
1997
1998 set_fs(old_fs);
1999 if (cur_mm) {
2000 unuse_mm(cur_mm);
2001 mmput(cur_mm);
2002 }
Jens Axboe06058632019-04-13 09:26:03 -06002003
2004 if (kthread_should_park())
2005 kthread_parkme();
2006
Jens Axboe6c271ce2019-01-10 11:22:30 -07002007 return 0;
2008}
2009
Jens Axboe2b188cc2019-01-07 10:46:33 -07002010static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
2011{
Jens Axboe9a56a232019-01-09 09:06:50 -07002012 struct io_submit_state state, *statep = NULL;
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002013 int i, submit = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002014
Jens Axboe9a56a232019-01-09 09:06:50 -07002015 if (to_submit > IO_PLUG_THRESHOLD) {
2016 io_submit_state_start(&state, ctx, to_submit);
2017 statep = &state;
2018 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002019
2020 for (i = 0; i < to_submit; i++) {
2021 struct sqe_submit s;
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002022 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002023
2024 if (!io_get_sqring(ctx, &s))
2025 break;
2026
2027 s.has_user = true;
Jens Axboedef596e2019-01-09 08:59:42 -07002028 s.needs_lock = false;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002029 s.needs_fixed_file = false;
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002030 submit++;
Jens Axboedef596e2019-01-09 08:59:42 -07002031
Jens Axboe9a56a232019-01-09 09:06:50 -07002032 ret = io_submit_sqe(ctx, &s, statep);
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002033 if (ret)
2034 io_cqring_add_event(ctx, s.sqe->user_data, ret, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002035 }
2036 io_commit_sqring(ctx);
2037
Jens Axboe9a56a232019-01-09 09:06:50 -07002038 if (statep)
2039 io_submit_state_end(statep);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002040
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002041 return submit;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002042}
2043
2044static unsigned io_cqring_events(struct io_cq_ring *ring)
2045{
2046 return READ_ONCE(ring->r.tail) - READ_ONCE(ring->r.head);
2047}
2048
2049/*
2050 * Wait until events become available, if we don't already have some. The
2051 * application must reap them itself, as they reside on the shared cq ring.
2052 */
2053static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
2054 const sigset_t __user *sig, size_t sigsz)
2055{
2056 struct io_cq_ring *ring = ctx->cq_ring;
2057 sigset_t ksigmask, sigsaved;
2058 DEFINE_WAIT(wait);
2059 int ret;
2060
2061 /* See comment at the top of this file */
2062 smp_rmb();
2063 if (io_cqring_events(ring) >= min_events)
2064 return 0;
2065
2066 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01002067#ifdef CONFIG_COMPAT
2068 if (in_compat_syscall())
2069 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
2070 &ksigmask, &sigsaved, sigsz);
2071 else
2072#endif
2073 ret = set_user_sigmask(sig, &ksigmask,
2074 &sigsaved, sigsz);
2075
Jens Axboe2b188cc2019-01-07 10:46:33 -07002076 if (ret)
2077 return ret;
2078 }
2079
2080 do {
2081 prepare_to_wait(&ctx->wait, &wait, TASK_INTERRUPTIBLE);
2082
2083 ret = 0;
2084 /* See comment at the top of this file */
2085 smp_rmb();
2086 if (io_cqring_events(ring) >= min_events)
2087 break;
2088
2089 schedule();
2090
2091 ret = -EINTR;
2092 if (signal_pending(current))
2093 break;
2094 } while (1);
2095
2096 finish_wait(&ctx->wait, &wait);
2097
2098 if (sig)
2099 restore_user_sigmask(sig, &sigsaved);
2100
2101 return READ_ONCE(ring->r.head) == READ_ONCE(ring->r.tail) ? ret : 0;
2102}
2103
Jens Axboe6b063142019-01-10 22:13:58 -07002104static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
2105{
2106#if defined(CONFIG_UNIX)
2107 if (ctx->ring_sock) {
2108 struct sock *sock = ctx->ring_sock->sk;
2109 struct sk_buff *skb;
2110
2111 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
2112 kfree_skb(skb);
2113 }
2114#else
2115 int i;
2116
2117 for (i = 0; i < ctx->nr_user_files; i++)
2118 fput(ctx->user_files[i]);
2119#endif
2120}
2121
2122static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
2123{
2124 if (!ctx->user_files)
2125 return -ENXIO;
2126
2127 __io_sqe_files_unregister(ctx);
2128 kfree(ctx->user_files);
2129 ctx->user_files = NULL;
2130 ctx->nr_user_files = 0;
2131 return 0;
2132}
2133
Jens Axboe6c271ce2019-01-10 11:22:30 -07002134static void io_sq_thread_stop(struct io_ring_ctx *ctx)
2135{
2136 if (ctx->sqo_thread) {
2137 ctx->sqo_stop = 1;
2138 mb();
Jens Axboe06058632019-04-13 09:26:03 -06002139 kthread_park(ctx->sqo_thread);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002140 kthread_stop(ctx->sqo_thread);
2141 ctx->sqo_thread = NULL;
2142 }
2143}
2144
Jens Axboe6b063142019-01-10 22:13:58 -07002145static void io_finish_async(struct io_ring_ctx *ctx)
2146{
Jens Axboe6c271ce2019-01-10 11:22:30 -07002147 io_sq_thread_stop(ctx);
2148
Jens Axboe6b063142019-01-10 22:13:58 -07002149 if (ctx->sqo_wq) {
2150 destroy_workqueue(ctx->sqo_wq);
2151 ctx->sqo_wq = NULL;
2152 }
2153}
2154
2155#if defined(CONFIG_UNIX)
2156static void io_destruct_skb(struct sk_buff *skb)
2157{
2158 struct io_ring_ctx *ctx = skb->sk->sk_user_data;
2159
2160 io_finish_async(ctx);
2161 unix_destruct_scm(skb);
2162}
2163
2164/*
2165 * Ensure the UNIX gc is aware of our file set, so we are certain that
2166 * the io_uring can be safely unregistered on process exit, even if we have
2167 * loops in the file referencing.
2168 */
2169static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
2170{
2171 struct sock *sk = ctx->ring_sock->sk;
2172 struct scm_fp_list *fpl;
2173 struct sk_buff *skb;
2174 int i;
2175
2176 if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
2177 unsigned long inflight = ctx->user->unix_inflight + nr;
2178
2179 if (inflight > task_rlimit(current, RLIMIT_NOFILE))
2180 return -EMFILE;
2181 }
2182
2183 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
2184 if (!fpl)
2185 return -ENOMEM;
2186
2187 skb = alloc_skb(0, GFP_KERNEL);
2188 if (!skb) {
2189 kfree(fpl);
2190 return -ENOMEM;
2191 }
2192
2193 skb->sk = sk;
2194 skb->destructor = io_destruct_skb;
2195
2196 fpl->user = get_uid(ctx->user);
2197 for (i = 0; i < nr; i++) {
2198 fpl->fp[i] = get_file(ctx->user_files[i + offset]);
2199 unix_inflight(fpl->user, fpl->fp[i]);
2200 }
2201
2202 fpl->max = fpl->count = nr;
2203 UNIXCB(skb).fp = fpl;
2204 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
2205 skb_queue_head(&sk->sk_receive_queue, skb);
2206
2207 for (i = 0; i < nr; i++)
2208 fput(fpl->fp[i]);
2209
2210 return 0;
2211}
2212
2213/*
2214 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
2215 * causes regular reference counting to break down. We rely on the UNIX
2216 * garbage collection to take care of this problem for us.
2217 */
2218static int io_sqe_files_scm(struct io_ring_ctx *ctx)
2219{
2220 unsigned left, total;
2221 int ret = 0;
2222
2223 total = 0;
2224 left = ctx->nr_user_files;
2225 while (left) {
2226 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
2227 int ret;
2228
2229 ret = __io_sqe_files_scm(ctx, this_files, total);
2230 if (ret)
2231 break;
2232 left -= this_files;
2233 total += this_files;
2234 }
2235
2236 if (!ret)
2237 return 0;
2238
2239 while (total < ctx->nr_user_files) {
2240 fput(ctx->user_files[total]);
2241 total++;
2242 }
2243
2244 return ret;
2245}
2246#else
2247static int io_sqe_files_scm(struct io_ring_ctx *ctx)
2248{
2249 return 0;
2250}
2251#endif
2252
2253static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
2254 unsigned nr_args)
2255{
2256 __s32 __user *fds = (__s32 __user *) arg;
2257 int fd, ret = 0;
2258 unsigned i;
2259
2260 if (ctx->user_files)
2261 return -EBUSY;
2262 if (!nr_args)
2263 return -EINVAL;
2264 if (nr_args > IORING_MAX_FIXED_FILES)
2265 return -EMFILE;
2266
2267 ctx->user_files = kcalloc(nr_args, sizeof(struct file *), GFP_KERNEL);
2268 if (!ctx->user_files)
2269 return -ENOMEM;
2270
2271 for (i = 0; i < nr_args; i++) {
2272 ret = -EFAULT;
2273 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
2274 break;
2275
2276 ctx->user_files[i] = fget(fd);
2277
2278 ret = -EBADF;
2279 if (!ctx->user_files[i])
2280 break;
2281 /*
2282 * Don't allow io_uring instances to be registered. If UNIX
2283 * isn't enabled, then this causes a reference cycle and this
2284 * instance can never get freed. If UNIX is enabled we'll
2285 * handle it just fine, but there's still no point in allowing
2286 * a ring fd as it doesn't support regular read/write anyway.
2287 */
2288 if (ctx->user_files[i]->f_op == &io_uring_fops) {
2289 fput(ctx->user_files[i]);
2290 break;
2291 }
2292 ctx->nr_user_files++;
2293 ret = 0;
2294 }
2295
2296 if (ret) {
2297 for (i = 0; i < ctx->nr_user_files; i++)
2298 fput(ctx->user_files[i]);
2299
2300 kfree(ctx->user_files);
Jens Axboe25adf502019-04-03 09:52:40 -06002301 ctx->user_files = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07002302 ctx->nr_user_files = 0;
2303 return ret;
2304 }
2305
2306 ret = io_sqe_files_scm(ctx);
2307 if (ret)
2308 io_sqe_files_unregister(ctx);
2309
2310 return ret;
2311}
2312
Jens Axboe6c271ce2019-01-10 11:22:30 -07002313static int io_sq_offload_start(struct io_ring_ctx *ctx,
2314 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002315{
2316 int ret;
2317
Jens Axboe6c271ce2019-01-10 11:22:30 -07002318 init_waitqueue_head(&ctx->sqo_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002319 mmgrab(current->mm);
2320 ctx->sqo_mm = current->mm;
2321
Jens Axboe6c271ce2019-01-10 11:22:30 -07002322 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe3ec482d2019-04-08 10:51:01 -06002323 ret = -EPERM;
2324 if (!capable(CAP_SYS_ADMIN))
2325 goto err;
2326
Jens Axboe917257d2019-04-13 09:28:55 -06002327 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
2328 if (!ctx->sq_thread_idle)
2329 ctx->sq_thread_idle = HZ;
2330
Jens Axboe6c271ce2019-01-10 11:22:30 -07002331 if (p->flags & IORING_SETUP_SQ_AFF) {
Mark Rutland975554b2019-04-30 13:34:51 +01002332 int cpu = array_index_nospec(p->sq_thread_cpu,
2333 nr_cpu_ids);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002334
Jens Axboe917257d2019-04-13 09:28:55 -06002335 ret = -EINVAL;
Mark Rutland975554b2019-04-30 13:34:51 +01002336 if (!cpu_possible(cpu))
Jens Axboe917257d2019-04-13 09:28:55 -06002337 goto err;
2338
Jens Axboe6c271ce2019-01-10 11:22:30 -07002339 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
2340 ctx, cpu,
2341 "io_uring-sq");
2342 } else {
2343 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
2344 "io_uring-sq");
2345 }
2346 if (IS_ERR(ctx->sqo_thread)) {
2347 ret = PTR_ERR(ctx->sqo_thread);
2348 ctx->sqo_thread = NULL;
2349 goto err;
2350 }
2351 wake_up_process(ctx->sqo_thread);
2352 } else if (p->flags & IORING_SETUP_SQ_AFF) {
2353 /* Can't have SQ_AFF without SQPOLL */
2354 ret = -EINVAL;
2355 goto err;
2356 }
2357
Jens Axboe2b188cc2019-01-07 10:46:33 -07002358 /* Do QD, or 2 * CPUS, whatever is smallest */
2359 ctx->sqo_wq = alloc_workqueue("io_ring-wq", WQ_UNBOUND | WQ_FREEZABLE,
2360 min(ctx->sq_entries - 1, 2 * num_online_cpus()));
2361 if (!ctx->sqo_wq) {
2362 ret = -ENOMEM;
2363 goto err;
2364 }
2365
2366 return 0;
2367err:
Jens Axboe6c271ce2019-01-10 11:22:30 -07002368 io_sq_thread_stop(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002369 mmdrop(ctx->sqo_mm);
2370 ctx->sqo_mm = NULL;
2371 return ret;
2372}
2373
2374static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
2375{
2376 atomic_long_sub(nr_pages, &user->locked_vm);
2377}
2378
2379static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
2380{
2381 unsigned long page_limit, cur_pages, new_pages;
2382
2383 /* Don't allow more pages than we can safely lock */
2384 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
2385
2386 do {
2387 cur_pages = atomic_long_read(&user->locked_vm);
2388 new_pages = cur_pages + nr_pages;
2389 if (new_pages > page_limit)
2390 return -ENOMEM;
2391 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
2392 new_pages) != cur_pages);
2393
2394 return 0;
2395}
2396
2397static void io_mem_free(void *ptr)
2398{
Mark Rutland52e04ef2019-04-30 17:30:21 +01002399 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002400
Mark Rutland52e04ef2019-04-30 17:30:21 +01002401 if (!ptr)
2402 return;
2403
2404 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002405 if (put_page_testzero(page))
2406 free_compound_page(page);
2407}
2408
2409static void *io_mem_alloc(size_t size)
2410{
2411 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
2412 __GFP_NORETRY;
2413
2414 return (void *) __get_free_pages(gfp_flags, get_order(size));
2415}
2416
2417static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
2418{
2419 struct io_sq_ring *sq_ring;
2420 struct io_cq_ring *cq_ring;
2421 size_t bytes;
2422
2423 bytes = struct_size(sq_ring, array, sq_entries);
2424 bytes += array_size(sizeof(struct io_uring_sqe), sq_entries);
2425 bytes += struct_size(cq_ring, cqes, cq_entries);
2426
2427 return (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
2428}
2429
Jens Axboeedafcce2019-01-09 09:16:05 -07002430static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
2431{
2432 int i, j;
2433
2434 if (!ctx->user_bufs)
2435 return -ENXIO;
2436
2437 for (i = 0; i < ctx->nr_user_bufs; i++) {
2438 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
2439
2440 for (j = 0; j < imu->nr_bvecs; j++)
2441 put_page(imu->bvec[j].bv_page);
2442
2443 if (ctx->account_mem)
2444 io_unaccount_mem(ctx->user, imu->nr_bvecs);
2445 kfree(imu->bvec);
2446 imu->nr_bvecs = 0;
2447 }
2448
2449 kfree(ctx->user_bufs);
2450 ctx->user_bufs = NULL;
2451 ctx->nr_user_bufs = 0;
2452 return 0;
2453}
2454
2455static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
2456 void __user *arg, unsigned index)
2457{
2458 struct iovec __user *src;
2459
2460#ifdef CONFIG_COMPAT
2461 if (ctx->compat) {
2462 struct compat_iovec __user *ciovs;
2463 struct compat_iovec ciov;
2464
2465 ciovs = (struct compat_iovec __user *) arg;
2466 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
2467 return -EFAULT;
2468
2469 dst->iov_base = (void __user *) (unsigned long) ciov.iov_base;
2470 dst->iov_len = ciov.iov_len;
2471 return 0;
2472 }
2473#endif
2474 src = (struct iovec __user *) arg;
2475 if (copy_from_user(dst, &src[index], sizeof(*dst)))
2476 return -EFAULT;
2477 return 0;
2478}
2479
2480static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
2481 unsigned nr_args)
2482{
2483 struct vm_area_struct **vmas = NULL;
2484 struct page **pages = NULL;
2485 int i, j, got_pages = 0;
2486 int ret = -EINVAL;
2487
2488 if (ctx->user_bufs)
2489 return -EBUSY;
2490 if (!nr_args || nr_args > UIO_MAXIOV)
2491 return -EINVAL;
2492
2493 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
2494 GFP_KERNEL);
2495 if (!ctx->user_bufs)
2496 return -ENOMEM;
2497
2498 for (i = 0; i < nr_args; i++) {
2499 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
2500 unsigned long off, start, end, ubuf;
2501 int pret, nr_pages;
2502 struct iovec iov;
2503 size_t size;
2504
2505 ret = io_copy_iov(ctx, &iov, arg, i);
2506 if (ret)
2507 break;
2508
2509 /*
2510 * Don't impose further limits on the size and buffer
2511 * constraints here, we'll -EINVAL later when IO is
2512 * submitted if they are wrong.
2513 */
2514 ret = -EFAULT;
2515 if (!iov.iov_base || !iov.iov_len)
2516 goto err;
2517
2518 /* arbitrary limit, but we need something */
2519 if (iov.iov_len > SZ_1G)
2520 goto err;
2521
2522 ubuf = (unsigned long) iov.iov_base;
2523 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2524 start = ubuf >> PAGE_SHIFT;
2525 nr_pages = end - start;
2526
2527 if (ctx->account_mem) {
2528 ret = io_account_mem(ctx->user, nr_pages);
2529 if (ret)
2530 goto err;
2531 }
2532
2533 ret = 0;
2534 if (!pages || nr_pages > got_pages) {
2535 kfree(vmas);
2536 kfree(pages);
2537 pages = kmalloc_array(nr_pages, sizeof(struct page *),
2538 GFP_KERNEL);
2539 vmas = kmalloc_array(nr_pages,
2540 sizeof(struct vm_area_struct *),
2541 GFP_KERNEL);
2542 if (!pages || !vmas) {
2543 ret = -ENOMEM;
2544 if (ctx->account_mem)
2545 io_unaccount_mem(ctx->user, nr_pages);
2546 goto err;
2547 }
2548 got_pages = nr_pages;
2549 }
2550
2551 imu->bvec = kmalloc_array(nr_pages, sizeof(struct bio_vec),
2552 GFP_KERNEL);
2553 ret = -ENOMEM;
2554 if (!imu->bvec) {
2555 if (ctx->account_mem)
2556 io_unaccount_mem(ctx->user, nr_pages);
2557 goto err;
2558 }
2559
2560 ret = 0;
2561 down_read(&current->mm->mmap_sem);
2562 pret = get_user_pages_longterm(ubuf, nr_pages, FOLL_WRITE,
2563 pages, vmas);
2564 if (pret == nr_pages) {
2565 /* don't support file backed memory */
2566 for (j = 0; j < nr_pages; j++) {
2567 struct vm_area_struct *vma = vmas[j];
2568
2569 if (vma->vm_file &&
2570 !is_file_hugepages(vma->vm_file)) {
2571 ret = -EOPNOTSUPP;
2572 break;
2573 }
2574 }
2575 } else {
2576 ret = pret < 0 ? pret : -EFAULT;
2577 }
2578 up_read(&current->mm->mmap_sem);
2579 if (ret) {
2580 /*
2581 * if we did partial map, or found file backed vmas,
2582 * release any pages we did get
2583 */
2584 if (pret > 0) {
2585 for (j = 0; j < pret; j++)
2586 put_page(pages[j]);
2587 }
2588 if (ctx->account_mem)
2589 io_unaccount_mem(ctx->user, nr_pages);
2590 goto err;
2591 }
2592
2593 off = ubuf & ~PAGE_MASK;
2594 size = iov.iov_len;
2595 for (j = 0; j < nr_pages; j++) {
2596 size_t vec_len;
2597
2598 vec_len = min_t(size_t, size, PAGE_SIZE - off);
2599 imu->bvec[j].bv_page = pages[j];
2600 imu->bvec[j].bv_len = vec_len;
2601 imu->bvec[j].bv_offset = off;
2602 off = 0;
2603 size -= vec_len;
2604 }
2605 /* store original address for later verification */
2606 imu->ubuf = ubuf;
2607 imu->len = iov.iov_len;
2608 imu->nr_bvecs = nr_pages;
2609
2610 ctx->nr_user_bufs++;
2611 }
2612 kfree(pages);
2613 kfree(vmas);
2614 return 0;
2615err:
2616 kfree(pages);
2617 kfree(vmas);
2618 io_sqe_buffer_unregister(ctx);
2619 return ret;
2620}
2621
Jens Axboe2b188cc2019-01-07 10:46:33 -07002622static void io_ring_ctx_free(struct io_ring_ctx *ctx)
2623{
Jens Axboe6b063142019-01-10 22:13:58 -07002624 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002625 if (ctx->sqo_mm)
2626 mmdrop(ctx->sqo_mm);
Jens Axboedef596e2019-01-09 08:59:42 -07002627
2628 io_iopoll_reap_events(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -07002629 io_sqe_buffer_unregister(ctx);
Jens Axboe6b063142019-01-10 22:13:58 -07002630 io_sqe_files_unregister(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07002631
Jens Axboe2b188cc2019-01-07 10:46:33 -07002632#if defined(CONFIG_UNIX)
2633 if (ctx->ring_sock)
2634 sock_release(ctx->ring_sock);
2635#endif
2636
2637 io_mem_free(ctx->sq_ring);
2638 io_mem_free(ctx->sq_sqes);
2639 io_mem_free(ctx->cq_ring);
2640
2641 percpu_ref_exit(&ctx->refs);
2642 if (ctx->account_mem)
2643 io_unaccount_mem(ctx->user,
2644 ring_pages(ctx->sq_entries, ctx->cq_entries));
2645 free_uid(ctx->user);
2646 kfree(ctx);
2647}
2648
2649static __poll_t io_uring_poll(struct file *file, poll_table *wait)
2650{
2651 struct io_ring_ctx *ctx = file->private_data;
2652 __poll_t mask = 0;
2653
2654 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02002655 /*
2656 * synchronizes with barrier from wq_has_sleeper call in
2657 * io_commit_cqring
2658 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07002659 smp_rmb();
Stefan Bühlerfb775fa2019-04-19 11:57:46 +02002660 if (READ_ONCE(ctx->sq_ring->r.tail) - ctx->cached_sq_head !=
2661 ctx->sq_ring->ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002662 mask |= EPOLLOUT | EPOLLWRNORM;
2663 if (READ_ONCE(ctx->cq_ring->r.head) != ctx->cached_cq_tail)
2664 mask |= EPOLLIN | EPOLLRDNORM;
2665
2666 return mask;
2667}
2668
2669static int io_uring_fasync(int fd, struct file *file, int on)
2670{
2671 struct io_ring_ctx *ctx = file->private_data;
2672
2673 return fasync_helper(fd, file, on, &ctx->cq_fasync);
2674}
2675
2676static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
2677{
2678 mutex_lock(&ctx->uring_lock);
2679 percpu_ref_kill(&ctx->refs);
2680 mutex_unlock(&ctx->uring_lock);
2681
Jens Axboe221c5eb2019-01-17 09:41:58 -07002682 io_poll_remove_all(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07002683 io_iopoll_reap_events(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002684 wait_for_completion(&ctx->ctx_done);
2685 io_ring_ctx_free(ctx);
2686}
2687
2688static int io_uring_release(struct inode *inode, struct file *file)
2689{
2690 struct io_ring_ctx *ctx = file->private_data;
2691
2692 file->private_data = NULL;
2693 io_ring_ctx_wait_and_kill(ctx);
2694 return 0;
2695}
2696
2697static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
2698{
2699 loff_t offset = (loff_t) vma->vm_pgoff << PAGE_SHIFT;
2700 unsigned long sz = vma->vm_end - vma->vm_start;
2701 struct io_ring_ctx *ctx = file->private_data;
2702 unsigned long pfn;
2703 struct page *page;
2704 void *ptr;
2705
2706 switch (offset) {
2707 case IORING_OFF_SQ_RING:
2708 ptr = ctx->sq_ring;
2709 break;
2710 case IORING_OFF_SQES:
2711 ptr = ctx->sq_sqes;
2712 break;
2713 case IORING_OFF_CQ_RING:
2714 ptr = ctx->cq_ring;
2715 break;
2716 default:
2717 return -EINVAL;
2718 }
2719
2720 page = virt_to_head_page(ptr);
2721 if (sz > (PAGE_SIZE << compound_order(page)))
2722 return -EINVAL;
2723
2724 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
2725 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
2726}
2727
2728SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
2729 u32, min_complete, u32, flags, const sigset_t __user *, sig,
2730 size_t, sigsz)
2731{
2732 struct io_ring_ctx *ctx;
2733 long ret = -EBADF;
2734 int submitted = 0;
2735 struct fd f;
2736
Jens Axboe6c271ce2019-01-10 11:22:30 -07002737 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002738 return -EINVAL;
2739
2740 f = fdget(fd);
2741 if (!f.file)
2742 return -EBADF;
2743
2744 ret = -EOPNOTSUPP;
2745 if (f.file->f_op != &io_uring_fops)
2746 goto out_fput;
2747
2748 ret = -ENXIO;
2749 ctx = f.file->private_data;
2750 if (!percpu_ref_tryget(&ctx->refs))
2751 goto out_fput;
2752
Jens Axboe6c271ce2019-01-10 11:22:30 -07002753 /*
2754 * For SQ polling, the thread will do all submissions and completions.
2755 * Just return the requested submit count, and wake the thread if
2756 * we were asked to.
2757 */
2758 if (ctx->flags & IORING_SETUP_SQPOLL) {
2759 if (flags & IORING_ENTER_SQ_WAKEUP)
2760 wake_up(&ctx->sqo_wait);
2761 submitted = to_submit;
2762 goto out_ctx;
2763 }
2764
Jens Axboe2b188cc2019-01-07 10:46:33 -07002765 ret = 0;
2766 if (to_submit) {
2767 to_submit = min(to_submit, ctx->sq_entries);
2768
2769 mutex_lock(&ctx->uring_lock);
2770 submitted = io_ring_submit(ctx, to_submit);
2771 mutex_unlock(&ctx->uring_lock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002772 }
2773 if (flags & IORING_ENTER_GETEVENTS) {
Jens Axboedef596e2019-01-09 08:59:42 -07002774 unsigned nr_events = 0;
2775
Jens Axboe2b188cc2019-01-07 10:46:33 -07002776 min_complete = min(min_complete, ctx->cq_entries);
2777
Jens Axboedef596e2019-01-09 08:59:42 -07002778 if (ctx->flags & IORING_SETUP_IOPOLL) {
2779 mutex_lock(&ctx->uring_lock);
2780 ret = io_iopoll_check(ctx, &nr_events, min_complete);
2781 mutex_unlock(&ctx->uring_lock);
2782 } else {
2783 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
2784 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002785 }
2786
2787out_ctx:
2788 io_ring_drop_ctx_refs(ctx, 1);
2789out_fput:
2790 fdput(f);
2791 return submitted ? submitted : ret;
2792}
2793
2794static const struct file_operations io_uring_fops = {
2795 .release = io_uring_release,
2796 .mmap = io_uring_mmap,
2797 .poll = io_uring_poll,
2798 .fasync = io_uring_fasync,
2799};
2800
2801static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
2802 struct io_uring_params *p)
2803{
2804 struct io_sq_ring *sq_ring;
2805 struct io_cq_ring *cq_ring;
2806 size_t size;
2807
2808 sq_ring = io_mem_alloc(struct_size(sq_ring, array, p->sq_entries));
2809 if (!sq_ring)
2810 return -ENOMEM;
2811
2812 ctx->sq_ring = sq_ring;
2813 sq_ring->ring_mask = p->sq_entries - 1;
2814 sq_ring->ring_entries = p->sq_entries;
2815 ctx->sq_mask = sq_ring->ring_mask;
2816 ctx->sq_entries = sq_ring->ring_entries;
2817
2818 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
2819 if (size == SIZE_MAX)
2820 return -EOVERFLOW;
2821
2822 ctx->sq_sqes = io_mem_alloc(size);
Mark Rutland52e04ef2019-04-30 17:30:21 +01002823 if (!ctx->sq_sqes)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002824 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002825
2826 cq_ring = io_mem_alloc(struct_size(cq_ring, cqes, p->cq_entries));
Mark Rutland52e04ef2019-04-30 17:30:21 +01002827 if (!cq_ring)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002828 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002829
2830 ctx->cq_ring = cq_ring;
2831 cq_ring->ring_mask = p->cq_entries - 1;
2832 cq_ring->ring_entries = p->cq_entries;
2833 ctx->cq_mask = cq_ring->ring_mask;
2834 ctx->cq_entries = cq_ring->ring_entries;
2835 return 0;
2836}
2837
2838/*
2839 * Allocate an anonymous fd, this is what constitutes the application
2840 * visible backing of an io_uring instance. The application mmaps this
2841 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
2842 * we have to tie this fd to a socket for file garbage collection purposes.
2843 */
2844static int io_uring_get_fd(struct io_ring_ctx *ctx)
2845{
2846 struct file *file;
2847 int ret;
2848
2849#if defined(CONFIG_UNIX)
2850 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
2851 &ctx->ring_sock);
2852 if (ret)
2853 return ret;
2854#endif
2855
2856 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
2857 if (ret < 0)
2858 goto err;
2859
2860 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
2861 O_RDWR | O_CLOEXEC);
2862 if (IS_ERR(file)) {
2863 put_unused_fd(ret);
2864 ret = PTR_ERR(file);
2865 goto err;
2866 }
2867
2868#if defined(CONFIG_UNIX)
2869 ctx->ring_sock->file = file;
Jens Axboe6b063142019-01-10 22:13:58 -07002870 ctx->ring_sock->sk->sk_user_data = ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002871#endif
2872 fd_install(ret, file);
2873 return ret;
2874err:
2875#if defined(CONFIG_UNIX)
2876 sock_release(ctx->ring_sock);
2877 ctx->ring_sock = NULL;
2878#endif
2879 return ret;
2880}
2881
2882static int io_uring_create(unsigned entries, struct io_uring_params *p)
2883{
2884 struct user_struct *user = NULL;
2885 struct io_ring_ctx *ctx;
2886 bool account_mem;
2887 int ret;
2888
2889 if (!entries || entries > IORING_MAX_ENTRIES)
2890 return -EINVAL;
2891
2892 /*
2893 * Use twice as many entries for the CQ ring. It's possible for the
2894 * application to drive a higher depth than the size of the SQ ring,
2895 * since the sqes are only used at submission time. This allows for
2896 * some flexibility in overcommitting a bit.
2897 */
2898 p->sq_entries = roundup_pow_of_two(entries);
2899 p->cq_entries = 2 * p->sq_entries;
2900
2901 user = get_uid(current_user());
2902 account_mem = !capable(CAP_IPC_LOCK);
2903
2904 if (account_mem) {
2905 ret = io_account_mem(user,
2906 ring_pages(p->sq_entries, p->cq_entries));
2907 if (ret) {
2908 free_uid(user);
2909 return ret;
2910 }
2911 }
2912
2913 ctx = io_ring_ctx_alloc(p);
2914 if (!ctx) {
2915 if (account_mem)
2916 io_unaccount_mem(user, ring_pages(p->sq_entries,
2917 p->cq_entries));
2918 free_uid(user);
2919 return -ENOMEM;
2920 }
2921 ctx->compat = in_compat_syscall();
2922 ctx->account_mem = account_mem;
2923 ctx->user = user;
2924
2925 ret = io_allocate_scq_urings(ctx, p);
2926 if (ret)
2927 goto err;
2928
Jens Axboe6c271ce2019-01-10 11:22:30 -07002929 ret = io_sq_offload_start(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002930 if (ret)
2931 goto err;
2932
2933 ret = io_uring_get_fd(ctx);
2934 if (ret < 0)
2935 goto err;
2936
2937 memset(&p->sq_off, 0, sizeof(p->sq_off));
2938 p->sq_off.head = offsetof(struct io_sq_ring, r.head);
2939 p->sq_off.tail = offsetof(struct io_sq_ring, r.tail);
2940 p->sq_off.ring_mask = offsetof(struct io_sq_ring, ring_mask);
2941 p->sq_off.ring_entries = offsetof(struct io_sq_ring, ring_entries);
2942 p->sq_off.flags = offsetof(struct io_sq_ring, flags);
2943 p->sq_off.dropped = offsetof(struct io_sq_ring, dropped);
2944 p->sq_off.array = offsetof(struct io_sq_ring, array);
2945
2946 memset(&p->cq_off, 0, sizeof(p->cq_off));
2947 p->cq_off.head = offsetof(struct io_cq_ring, r.head);
2948 p->cq_off.tail = offsetof(struct io_cq_ring, r.tail);
2949 p->cq_off.ring_mask = offsetof(struct io_cq_ring, ring_mask);
2950 p->cq_off.ring_entries = offsetof(struct io_cq_ring, ring_entries);
2951 p->cq_off.overflow = offsetof(struct io_cq_ring, overflow);
2952 p->cq_off.cqes = offsetof(struct io_cq_ring, cqes);
2953 return ret;
2954err:
2955 io_ring_ctx_wait_and_kill(ctx);
2956 return ret;
2957}
2958
2959/*
2960 * Sets up an aio uring context, and returns the fd. Applications asks for a
2961 * ring size, we return the actual sq/cq ring sizes (among other things) in the
2962 * params structure passed in.
2963 */
2964static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
2965{
2966 struct io_uring_params p;
2967 long ret;
2968 int i;
2969
2970 if (copy_from_user(&p, params, sizeof(p)))
2971 return -EFAULT;
2972 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
2973 if (p.resv[i])
2974 return -EINVAL;
2975 }
2976
Jens Axboe6c271ce2019-01-10 11:22:30 -07002977 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
2978 IORING_SETUP_SQ_AFF))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002979 return -EINVAL;
2980
2981 ret = io_uring_create(entries, &p);
2982 if (ret < 0)
2983 return ret;
2984
2985 if (copy_to_user(params, &p, sizeof(p)))
2986 return -EFAULT;
2987
2988 return ret;
2989}
2990
2991SYSCALL_DEFINE2(io_uring_setup, u32, entries,
2992 struct io_uring_params __user *, params)
2993{
2994 return io_uring_setup(entries, params);
2995}
2996
Jens Axboeedafcce2019-01-09 09:16:05 -07002997static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
2998 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06002999 __releases(ctx->uring_lock)
3000 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07003001{
3002 int ret;
3003
Jens Axboe35fa71a2019-04-22 10:23:23 -06003004 /*
3005 * We're inside the ring mutex, if the ref is already dying, then
3006 * someone else killed the ctx or is already going through
3007 * io_uring_register().
3008 */
3009 if (percpu_ref_is_dying(&ctx->refs))
3010 return -ENXIO;
3011
Jens Axboeedafcce2019-01-09 09:16:05 -07003012 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -06003013
3014 /*
3015 * Drop uring mutex before waiting for references to exit. If another
3016 * thread is currently inside io_uring_enter() it might need to grab
3017 * the uring_lock to make progress. If we hold it here across the drain
3018 * wait, then we can deadlock. It's safe to drop the mutex here, since
3019 * no new references will come in after we've killed the percpu ref.
3020 */
3021 mutex_unlock(&ctx->uring_lock);
Jens Axboeedafcce2019-01-09 09:16:05 -07003022 wait_for_completion(&ctx->ctx_done);
Jens Axboeb19062a2019-04-15 10:49:38 -06003023 mutex_lock(&ctx->uring_lock);
Jens Axboeedafcce2019-01-09 09:16:05 -07003024
3025 switch (opcode) {
3026 case IORING_REGISTER_BUFFERS:
3027 ret = io_sqe_buffer_register(ctx, arg, nr_args);
3028 break;
3029 case IORING_UNREGISTER_BUFFERS:
3030 ret = -EINVAL;
3031 if (arg || nr_args)
3032 break;
3033 ret = io_sqe_buffer_unregister(ctx);
3034 break;
Jens Axboe6b063142019-01-10 22:13:58 -07003035 case IORING_REGISTER_FILES:
3036 ret = io_sqe_files_register(ctx, arg, nr_args);
3037 break;
3038 case IORING_UNREGISTER_FILES:
3039 ret = -EINVAL;
3040 if (arg || nr_args)
3041 break;
3042 ret = io_sqe_files_unregister(ctx);
3043 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07003044 default:
3045 ret = -EINVAL;
3046 break;
3047 }
3048
3049 /* bring the ctx back to life */
3050 reinit_completion(&ctx->ctx_done);
3051 percpu_ref_reinit(&ctx->refs);
3052 return ret;
3053}
3054
3055SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
3056 void __user *, arg, unsigned int, nr_args)
3057{
3058 struct io_ring_ctx *ctx;
3059 long ret = -EBADF;
3060 struct fd f;
3061
3062 f = fdget(fd);
3063 if (!f.file)
3064 return -EBADF;
3065
3066 ret = -EOPNOTSUPP;
3067 if (f.file->f_op != &io_uring_fops)
3068 goto out_fput;
3069
3070 ctx = f.file->private_data;
3071
3072 mutex_lock(&ctx->uring_lock);
3073 ret = __io_uring_register(ctx, opcode, arg, nr_args);
3074 mutex_unlock(&ctx->uring_lock);
3075out_fput:
3076 fdput(f);
3077 return ret;
3078}
3079
Jens Axboe2b188cc2019-01-07 10:46:33 -07003080static int __init io_uring_init(void)
3081{
3082 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
3083 return 0;
3084};
3085__initcall(io_uring_init);