blob: f9a38998f2fc515c94c4cc292e00450a55aaa912 [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
47#include <linux/refcount.h>
48#include <linux/uio.h>
49
50#include <linux/sched/signal.h>
51#include <linux/fs.h>
52#include <linux/file.h>
53#include <linux/fdtable.h>
54#include <linux/mm.h>
55#include <linux/mman.h>
56#include <linux/mmu_context.h>
57#include <linux/percpu.h>
58#include <linux/slab.h>
59#include <linux/workqueue.h>
Jens Axboe6c271ce2019-01-10 11:22:30 -070060#include <linux/kthread.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070061#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070062#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070063#include <linux/net.h>
64#include <net/sock.h>
65#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070066#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070067#include <linux/anon_inodes.h>
68#include <linux/sched/mm.h>
69#include <linux/uaccess.h>
70#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070071#include <linux/sizes.h>
72#include <linux/hugetlb.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070073
74#include <uapi/linux/io_uring.h>
75
76#include "internal.h"
77
Daniel Xu5277dea2019-09-14 14:23:45 -070078#define IORING_MAX_ENTRIES 32768
Jens Axboe6b063142019-01-10 22:13:58 -070079#define IORING_MAX_FIXED_FILES 1024
Jens Axboe2b188cc2019-01-07 10:46:33 -070080
81struct io_uring {
82 u32 head ____cacheline_aligned_in_smp;
83 u32 tail ____cacheline_aligned_in_smp;
84};
85
Stefan Bühler1e84b972019-04-24 23:54:16 +020086/*
Hristo Venev75b28af2019-08-26 17:23:46 +000087 * This data is shared with the application through the mmap at offsets
88 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +020089 *
90 * The offsets to the member fields are published through struct
91 * io_sqring_offsets when calling io_uring_setup.
92 */
Hristo Venev75b28af2019-08-26 17:23:46 +000093struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +020094 /*
95 * Head and tail offsets into the ring; the offsets need to be
96 * masked to get valid indices.
97 *
Hristo Venev75b28af2019-08-26 17:23:46 +000098 * The kernel controls head of the sq ring and the tail of the cq ring,
99 * and the application controls tail of the sq ring and the head of the
100 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200101 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000102 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200103 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000104 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200105 * ring_entries - 1)
106 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000107 u32 sq_ring_mask, cq_ring_mask;
108 /* Ring sizes (constant, power of 2) */
109 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200110 /*
111 * Number of invalid entries dropped by the kernel due to
112 * invalid index stored in array
113 *
114 * Written by the kernel, shouldn't be modified by the
115 * application (i.e. get number of "new events" by comparing to
116 * cached value).
117 *
118 * After a new SQ head value was read by the application this
119 * counter includes all submissions that were dropped reaching
120 * the new SQ head (and possibly more).
121 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000122 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200123 /*
124 * Runtime flags
125 *
126 * Written by the kernel, shouldn't be modified by the
127 * application.
128 *
129 * The application needs a full memory barrier before checking
130 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
131 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000132 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200133 /*
134 * Number of completion events lost because the queue was full;
135 * this should be avoided by the application by making sure
136 * there are not more requests pending thatn there is space in
137 * the completion queue.
138 *
139 * Written by the kernel, shouldn't be modified by the
140 * application (i.e. get number of "new events" by comparing to
141 * cached value).
142 *
143 * As completion events come in out of order this counter is not
144 * ordered with any other data.
145 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000146 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200147 /*
148 * Ring buffer of completion events.
149 *
150 * The kernel writes completion events fresh every time they are
151 * produced, so the application is allowed to modify pending
152 * entries.
153 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000154 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700155};
156
Jens Axboeedafcce2019-01-09 09:16:05 -0700157struct io_mapped_ubuf {
158 u64 ubuf;
159 size_t len;
160 struct bio_vec *bvec;
161 unsigned int nr_bvecs;
162};
163
Jens Axboe31b51512019-01-18 22:56:34 -0700164struct async_list {
165 spinlock_t lock;
166 atomic_t cnt;
167 struct list_head list;
168
169 struct file *file;
Jens Axboe6d5d5ac2019-09-11 10:16:13 -0600170 off_t io_start;
Zhengyuan Liu9310a7ba2019-07-22 10:23:27 +0800171 size_t io_len;
Jens Axboe31b51512019-01-18 22:56:34 -0700172};
173
Jens Axboe2b188cc2019-01-07 10:46:33 -0700174struct io_ring_ctx {
175 struct {
176 struct percpu_ref refs;
177 } ____cacheline_aligned_in_smp;
178
179 struct {
180 unsigned int flags;
181 bool compat;
182 bool account_mem;
183
Hristo Venev75b28af2019-08-26 17:23:46 +0000184 /*
185 * Ring buffer of indices into array of io_uring_sqe, which is
186 * mmapped by the application using the IORING_OFF_SQES offset.
187 *
188 * This indirection could e.g. be used to assign fixed
189 * io_uring_sqe entries to operations and only submit them to
190 * the queue when needed.
191 *
192 * The kernel modifies neither the indices array nor the entries
193 * array.
194 */
195 u32 *sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700196 unsigned cached_sq_head;
197 unsigned sq_entries;
198 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700199 unsigned sq_thread_idle;
Jens Axboe498ccd92019-10-25 10:04:25 -0600200 unsigned cached_sq_dropped;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700201 struct io_uring_sqe *sq_sqes;
Jens Axboede0617e2019-04-06 21:51:27 -0600202
203 struct list_head defer_list;
Jens Axboe5262f562019-09-17 12:26:57 -0600204 struct list_head timeout_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700205 } ____cacheline_aligned_in_smp;
206
207 /* IO offload */
Jens Axboe54a91f32019-09-10 09:15:04 -0600208 struct workqueue_struct *sqo_wq[2];
Jens Axboe6c271ce2019-01-10 11:22:30 -0700209 struct task_struct *sqo_thread; /* if using sq thread polling */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700210 struct mm_struct *sqo_mm;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700211 wait_queue_head_t sqo_wait;
Jackie Liua4c0b3d2019-07-08 13:41:12 +0800212 struct completion sqo_thread_started;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700213
214 struct {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700215 unsigned cached_cq_tail;
Jens Axboe498ccd92019-10-25 10:04:25 -0600216 atomic_t cached_cq_overflow;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700217 unsigned cq_entries;
218 unsigned cq_mask;
219 struct wait_queue_head cq_wait;
220 struct fasync_struct *cq_fasync;
Jens Axboe9b402842019-04-11 11:45:41 -0600221 struct eventfd_ctx *cq_ev_fd;
Jens Axboe5262f562019-09-17 12:26:57 -0600222 atomic_t cq_timeouts;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700223 } ____cacheline_aligned_in_smp;
224
Hristo Venev75b28af2019-08-26 17:23:46 +0000225 struct io_rings *rings;
226
Jens Axboe6b063142019-01-10 22:13:58 -0700227 /*
228 * If used, fixed file set. Writers must ensure that ->refs is dead,
229 * readers must ensure that ->refs is alive as long as the file* is
230 * used. Only updated through io_uring_register(2).
231 */
232 struct file **user_files;
233 unsigned nr_user_files;
234
Jens Axboeedafcce2019-01-09 09:16:05 -0700235 /* if used, fixed mapped user buffers */
236 unsigned nr_user_bufs;
237 struct io_mapped_ubuf *user_bufs;
238
Jens Axboe2b188cc2019-01-07 10:46:33 -0700239 struct user_struct *user;
240
241 struct completion ctx_done;
242
243 struct {
244 struct mutex uring_lock;
245 wait_queue_head_t wait;
246 } ____cacheline_aligned_in_smp;
247
248 struct {
249 spinlock_t completion_lock;
Jens Axboedef596e2019-01-09 08:59:42 -0700250 bool poll_multi_file;
251 /*
252 * ->poll_list is protected by the ctx->uring_lock for
253 * io_uring instances that don't use IORING_SETUP_SQPOLL.
254 * For SQPOLL, only the single threaded io_sq_thread() will
255 * manipulate the list, hence no extra locking is needed there.
256 */
257 struct list_head poll_list;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700258 struct list_head cancel_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700259 } ____cacheline_aligned_in_smp;
260
Jens Axboe31b51512019-01-18 22:56:34 -0700261 struct async_list pending_async[2];
262
Jens Axboe2b188cc2019-01-07 10:46:33 -0700263#if defined(CONFIG_UNIX)
264 struct socket *ring_sock;
265#endif
266};
267
268struct sqe_submit {
269 const struct io_uring_sqe *sqe;
270 unsigned short index;
Jackie Liu8776f3f2019-09-09 20:50:39 +0800271 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700272 bool has_user;
Jens Axboedef596e2019-01-09 08:59:42 -0700273 bool needs_lock;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700274 bool needs_fixed_file;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700275};
276
Jens Axboe09bb8392019-03-13 12:39:28 -0600277/*
278 * First field must be the file pointer in all the
279 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
280 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700281struct io_poll_iocb {
282 struct file *file;
283 struct wait_queue_head *head;
284 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600285 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700286 bool canceled;
287 struct wait_queue_entry wait;
288};
289
Jens Axboe5262f562019-09-17 12:26:57 -0600290struct io_timeout {
291 struct file *file;
292 struct hrtimer timer;
293};
294
Jens Axboe09bb8392019-03-13 12:39:28 -0600295/*
296 * NOTE! Each of the iocb union members has the file pointer
297 * as the first entry in their struct definition. So you can
298 * access the file pointer through any of the sub-structs,
299 * or directly as just 'ki_filp' in this struct.
300 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700301struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700302 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600303 struct file *file;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700304 struct kiocb rw;
305 struct io_poll_iocb poll;
Jens Axboe5262f562019-09-17 12:26:57 -0600306 struct io_timeout timeout;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700307 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700308
309 struct sqe_submit submit;
310
311 struct io_ring_ctx *ctx;
312 struct list_head list;
Jens Axboe9e645e112019-05-10 16:07:28 -0600313 struct list_head link_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700314 unsigned int flags;
Jens Axboec16361c2019-01-17 08:39:48 -0700315 refcount_t refs;
Stefan Bühler8449eed2019-04-27 20:34:19 +0200316#define REQ_F_NOWAIT 1 /* must not punt to workers */
Jens Axboedef596e2019-01-09 08:59:42 -0700317#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
Jens Axboe6b063142019-01-10 22:13:58 -0700318#define REQ_F_FIXED_FILE 4 /* ctx owns file */
Jens Axboe31b51512019-01-18 22:56:34 -0700319#define REQ_F_SEQ_PREV 8 /* sequential with previous */
Stefan Bühlere2033e32019-05-11 19:08:01 +0200320#define REQ_F_IO_DRAIN 16 /* drain existing IO first */
321#define REQ_F_IO_DRAINED 32 /* drain done */
Jens Axboe9e645e112019-05-10 16:07:28 -0600322#define REQ_F_LINK 64 /* linked sqes */
Zhengyuan Liuf7b76ac2019-07-16 23:26:14 +0800323#define REQ_F_LINK_DONE 128 /* linked sqes done */
324#define REQ_F_FAIL_LINK 256 /* fail rest of links */
Jackie Liu4fe2c962019-09-09 20:50:40 +0800325#define REQ_F_SHADOW_DRAIN 512 /* link-drain shadow req */
Jens Axboe5262f562019-09-17 12:26:57 -0600326#define REQ_F_TIMEOUT 1024 /* timeout request */
Jens Axboe491381ce2019-10-17 09:20:46 -0600327#define REQ_F_ISREG 2048 /* regular file */
328#define REQ_F_MUST_PUNT 4096 /* must be punted even for NONBLOCK */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700329 u64 user_data;
Jens Axboe9e645e112019-05-10 16:07:28 -0600330 u32 result;
Jens Axboede0617e2019-04-06 21:51:27 -0600331 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700332
333 struct work_struct work;
334};
335
336#define IO_PLUG_THRESHOLD 2
Jens Axboedef596e2019-01-09 08:59:42 -0700337#define IO_IOPOLL_BATCH 8
Jens Axboe2b188cc2019-01-07 10:46:33 -0700338
Jens Axboe9a56a232019-01-09 09:06:50 -0700339struct io_submit_state {
340 struct blk_plug plug;
341
342 /*
Jens Axboe2579f912019-01-09 09:10:43 -0700343 * io_kiocb alloc cache
344 */
345 void *reqs[IO_IOPOLL_BATCH];
346 unsigned int free_reqs;
347 unsigned int cur_req;
348
349 /*
Jens Axboe9a56a232019-01-09 09:06:50 -0700350 * File reference cache
351 */
352 struct file *file;
353 unsigned int fd;
354 unsigned int has_refs;
355 unsigned int used_refs;
356 unsigned int ios_left;
357};
358
Jens Axboede0617e2019-04-06 21:51:27 -0600359static void io_sq_wq_submit_work(struct work_struct *work);
Jens Axboe5262f562019-09-17 12:26:57 -0600360static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
361 long res);
Jackie Liu4fe2c962019-09-09 20:50:40 +0800362static void __io_free_req(struct io_kiocb *req);
Jens Axboede0617e2019-04-06 21:51:27 -0600363
Jens Axboe2b188cc2019-01-07 10:46:33 -0700364static struct kmem_cache *req_cachep;
365
366static const struct file_operations io_uring_fops;
367
368struct sock *io_uring_get_socket(struct file *file)
369{
370#if defined(CONFIG_UNIX)
371 if (file->f_op == &io_uring_fops) {
372 struct io_ring_ctx *ctx = file->private_data;
373
374 return ctx->ring_sock->sk;
375 }
376#endif
377 return NULL;
378}
379EXPORT_SYMBOL(io_uring_get_socket);
380
381static void io_ring_ctx_ref_free(struct percpu_ref *ref)
382{
383 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
384
385 complete(&ctx->ctx_done);
386}
387
388static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
389{
390 struct io_ring_ctx *ctx;
Jens Axboe31b51512019-01-18 22:56:34 -0700391 int i;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700392
393 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
394 if (!ctx)
395 return NULL;
396
Roman Gushchin21482892019-05-07 10:01:48 -0700397 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
398 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700399 kfree(ctx);
400 return NULL;
401 }
402
403 ctx->flags = p->flags;
404 init_waitqueue_head(&ctx->cq_wait);
405 init_completion(&ctx->ctx_done);
Jackie Liua4c0b3d2019-07-08 13:41:12 +0800406 init_completion(&ctx->sqo_thread_started);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700407 mutex_init(&ctx->uring_lock);
408 init_waitqueue_head(&ctx->wait);
Jens Axboe31b51512019-01-18 22:56:34 -0700409 for (i = 0; i < ARRAY_SIZE(ctx->pending_async); i++) {
410 spin_lock_init(&ctx->pending_async[i].lock);
411 INIT_LIST_HEAD(&ctx->pending_async[i].list);
412 atomic_set(&ctx->pending_async[i].cnt, 0);
413 }
Jens Axboe2b188cc2019-01-07 10:46:33 -0700414 spin_lock_init(&ctx->completion_lock);
Jens Axboedef596e2019-01-09 08:59:42 -0700415 INIT_LIST_HEAD(&ctx->poll_list);
Jens Axboe221c5eb2019-01-17 09:41:58 -0700416 INIT_LIST_HEAD(&ctx->cancel_list);
Jens Axboede0617e2019-04-06 21:51:27 -0600417 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -0600418 INIT_LIST_HEAD(&ctx->timeout_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700419 return ctx;
420}
421
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600422static inline bool __io_sequence_defer(struct io_ring_ctx *ctx,
423 struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -0600424{
Jens Axboe498ccd92019-10-25 10:04:25 -0600425 return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
426 + atomic_read(&ctx->cached_cq_overflow);
Jens Axboede0617e2019-04-06 21:51:27 -0600427}
428
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600429static inline bool io_sequence_defer(struct io_ring_ctx *ctx,
430 struct io_kiocb *req)
431{
432 if ((req->flags & (REQ_F_IO_DRAIN|REQ_F_IO_DRAINED)) != REQ_F_IO_DRAIN)
433 return false;
434
435 return __io_sequence_defer(ctx, req);
436}
437
438static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -0600439{
440 struct io_kiocb *req;
441
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600442 req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
443 if (req && !io_sequence_defer(ctx, req)) {
Jens Axboede0617e2019-04-06 21:51:27 -0600444 list_del_init(&req->list);
445 return req;
446 }
447
448 return NULL;
449}
450
Jens Axboe5262f562019-09-17 12:26:57 -0600451static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
452{
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600453 struct io_kiocb *req;
454
455 req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
456 if (req && !__io_sequence_defer(ctx, req)) {
457 list_del_init(&req->list);
458 return req;
459 }
460
461 return NULL;
Jens Axboe5262f562019-09-17 12:26:57 -0600462}
463
Jens Axboede0617e2019-04-06 21:51:27 -0600464static void __io_commit_cqring(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700465{
Hristo Venev75b28af2019-08-26 17:23:46 +0000466 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700467
Hristo Venev75b28af2019-08-26 17:23:46 +0000468 if (ctx->cached_cq_tail != READ_ONCE(rings->cq.tail)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700469 /* order cqe stores with ring update */
Hristo Venev75b28af2019-08-26 17:23:46 +0000470 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700471
Jens Axboe2b188cc2019-01-07 10:46:33 -0700472 if (wq_has_sleeper(&ctx->cq_wait)) {
473 wake_up_interruptible(&ctx->cq_wait);
474 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
475 }
476 }
477}
478
Jens Axboe18d9be12019-09-10 09:13:05 -0600479static inline void io_queue_async_work(struct io_ring_ctx *ctx,
480 struct io_kiocb *req)
481{
Jens Axboe6cc47d12019-09-18 11:18:23 -0600482 int rw = 0;
Jens Axboe54a91f32019-09-10 09:15:04 -0600483
Jens Axboe6cc47d12019-09-18 11:18:23 -0600484 if (req->submit.sqe) {
485 switch (req->submit.sqe->opcode) {
486 case IORING_OP_WRITEV:
487 case IORING_OP_WRITE_FIXED:
488 rw = !(req->rw.ki_flags & IOCB_DIRECT);
489 break;
490 }
Jens Axboe54a91f32019-09-10 09:15:04 -0600491 }
492
493 queue_work(ctx->sqo_wq[rw], &req->work);
Jens Axboe18d9be12019-09-10 09:13:05 -0600494}
495
Jens Axboe5262f562019-09-17 12:26:57 -0600496static void io_kill_timeout(struct io_kiocb *req)
497{
498 int ret;
499
500 ret = hrtimer_try_to_cancel(&req->timeout.timer);
501 if (ret != -1) {
502 atomic_inc(&req->ctx->cq_timeouts);
503 list_del(&req->list);
504 io_cqring_fill_event(req->ctx, req->user_data, 0);
505 __io_free_req(req);
506 }
507}
508
509static void io_kill_timeouts(struct io_ring_ctx *ctx)
510{
511 struct io_kiocb *req, *tmp;
512
513 spin_lock_irq(&ctx->completion_lock);
514 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
515 io_kill_timeout(req);
516 spin_unlock_irq(&ctx->completion_lock);
517}
518
Jens Axboede0617e2019-04-06 21:51:27 -0600519static void io_commit_cqring(struct io_ring_ctx *ctx)
520{
521 struct io_kiocb *req;
522
Jens Axboe5262f562019-09-17 12:26:57 -0600523 while ((req = io_get_timeout_req(ctx)) != NULL)
524 io_kill_timeout(req);
525
Jens Axboede0617e2019-04-06 21:51:27 -0600526 __io_commit_cqring(ctx);
527
528 while ((req = io_get_deferred_req(ctx)) != NULL) {
Jackie Liu4fe2c962019-09-09 20:50:40 +0800529 if (req->flags & REQ_F_SHADOW_DRAIN) {
530 /* Just for drain, free it. */
531 __io_free_req(req);
532 continue;
533 }
Jens Axboede0617e2019-04-06 21:51:27 -0600534 req->flags |= REQ_F_IO_DRAINED;
Jens Axboe18d9be12019-09-10 09:13:05 -0600535 io_queue_async_work(ctx, req);
Jens Axboede0617e2019-04-06 21:51:27 -0600536 }
537}
538
Jens Axboe2b188cc2019-01-07 10:46:33 -0700539static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
540{
Hristo Venev75b28af2019-08-26 17:23:46 +0000541 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700542 unsigned tail;
543
544 tail = ctx->cached_cq_tail;
Stefan Bühler115e12e2019-04-24 23:54:18 +0200545 /*
546 * writes to the cq entry need to come after reading head; the
547 * control dependency is enough as we're using WRITE_ONCE to
548 * fill the cq entry
549 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000550 if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700551 return NULL;
552
553 ctx->cached_cq_tail++;
Hristo Venev75b28af2019-08-26 17:23:46 +0000554 return &rings->cqes[tail & ctx->cq_mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -0700555}
556
557static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
Jens Axboec71ffb62019-05-13 20:58:29 -0600558 long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700559{
560 struct io_uring_cqe *cqe;
561
562 /*
563 * If we can't get a cq entry, userspace overflowed the
564 * submission (by quite a lot). Increment the overflow count in
565 * the ring.
566 */
567 cqe = io_get_cqring(ctx);
568 if (cqe) {
569 WRITE_ONCE(cqe->user_data, ki_user_data);
570 WRITE_ONCE(cqe->res, res);
Jens Axboec71ffb62019-05-13 20:58:29 -0600571 WRITE_ONCE(cqe->flags, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700572 } else {
Jens Axboe498ccd92019-10-25 10:04:25 -0600573 WRITE_ONCE(ctx->rings->cq_overflow,
574 atomic_inc_return(&ctx->cached_cq_overflow));
Jens Axboe2b188cc2019-01-07 10:46:33 -0700575 }
576}
577
Jens Axboe8c838782019-03-12 15:48:16 -0600578static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
579{
580 if (waitqueue_active(&ctx->wait))
581 wake_up(&ctx->wait);
582 if (waitqueue_active(&ctx->sqo_wait))
583 wake_up(&ctx->sqo_wait);
Jens Axboe9b402842019-04-11 11:45:41 -0600584 if (ctx->cq_ev_fd)
585 eventfd_signal(ctx->cq_ev_fd, 1);
Jens Axboe8c838782019-03-12 15:48:16 -0600586}
587
588static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data,
Jens Axboec71ffb62019-05-13 20:58:29 -0600589 long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700590{
591 unsigned long flags;
592
593 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboec71ffb62019-05-13 20:58:29 -0600594 io_cqring_fill_event(ctx, user_data, res);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700595 io_commit_cqring(ctx);
596 spin_unlock_irqrestore(&ctx->completion_lock, flags);
597
Jens Axboe8c838782019-03-12 15:48:16 -0600598 io_cqring_ev_posted(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700599}
600
Jens Axboe2579f912019-01-09 09:10:43 -0700601static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
602 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700603{
Jens Axboefd6fab22019-03-14 16:30:06 -0600604 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700605 struct io_kiocb *req;
606
607 if (!percpu_ref_tryget(&ctx->refs))
608 return NULL;
609
Jens Axboe2579f912019-01-09 09:10:43 -0700610 if (!state) {
Jens Axboefd6fab22019-03-14 16:30:06 -0600611 req = kmem_cache_alloc(req_cachep, gfp);
Jens Axboe2579f912019-01-09 09:10:43 -0700612 if (unlikely(!req))
613 goto out;
614 } else if (!state->free_reqs) {
615 size_t sz;
616 int ret;
617
618 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
Jens Axboefd6fab22019-03-14 16:30:06 -0600619 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
620
621 /*
622 * Bulk alloc is all-or-nothing. If we fail to get a batch,
623 * retry single alloc to be on the safe side.
624 */
625 if (unlikely(ret <= 0)) {
626 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
627 if (!state->reqs[0])
628 goto out;
629 ret = 1;
630 }
Jens Axboe2579f912019-01-09 09:10:43 -0700631 state->free_reqs = ret - 1;
632 state->cur_req = 1;
633 req = state->reqs[0];
634 } else {
635 req = state->reqs[state->cur_req];
636 state->free_reqs--;
637 state->cur_req++;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700638 }
639
Jens Axboe60c112b2019-06-21 10:20:18 -0600640 req->file = NULL;
Jens Axboe2579f912019-01-09 09:10:43 -0700641 req->ctx = ctx;
642 req->flags = 0;
Jens Axboee65ef562019-03-12 10:16:44 -0600643 /* one is dropped after submission, the other at completion */
644 refcount_set(&req->refs, 2);
Jens Axboe9e645e112019-05-10 16:07:28 -0600645 req->result = 0;
Jens Axboe2579f912019-01-09 09:10:43 -0700646 return req;
647out:
Pavel Begunkov6805b32e2019-10-08 02:18:42 +0300648 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700649 return NULL;
650}
651
Jens Axboedef596e2019-01-09 08:59:42 -0700652static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
653{
654 if (*nr) {
655 kmem_cache_free_bulk(req_cachep, *nr, reqs);
Pavel Begunkov6805b32e2019-10-08 02:18:42 +0300656 percpu_ref_put_many(&ctx->refs, *nr);
Jens Axboedef596e2019-01-09 08:59:42 -0700657 *nr = 0;
658 }
659}
660
Jens Axboe9e645e112019-05-10 16:07:28 -0600661static void __io_free_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700662{
Jens Axboe09bb8392019-03-13 12:39:28 -0600663 if (req->file && !(req->flags & REQ_F_FIXED_FILE))
664 fput(req->file);
Pavel Begunkov6805b32e2019-10-08 02:18:42 +0300665 percpu_ref_put(&req->ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -0600666 kmem_cache_free(req_cachep, req);
667}
668
Jens Axboe9e645e112019-05-10 16:07:28 -0600669static void io_req_link_next(struct io_kiocb *req)
670{
671 struct io_kiocb *nxt;
672
673 /*
674 * The list should never be empty when we are called here. But could
675 * potentially happen if the chain is messed up, check to be on the
676 * safe side.
677 */
678 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
679 if (nxt) {
680 list_del(&nxt->list);
681 if (!list_empty(&req->link_list)) {
682 INIT_LIST_HEAD(&nxt->link_list);
683 list_splice(&req->link_list, &nxt->link_list);
684 nxt->flags |= REQ_F_LINK;
685 }
686
Zhengyuan Liuf7b76ac2019-07-16 23:26:14 +0800687 nxt->flags |= REQ_F_LINK_DONE;
Jens Axboe9e645e112019-05-10 16:07:28 -0600688 INIT_WORK(&nxt->work, io_sq_wq_submit_work);
Jens Axboe18d9be12019-09-10 09:13:05 -0600689 io_queue_async_work(req->ctx, nxt);
Jens Axboe9e645e112019-05-10 16:07:28 -0600690 }
691}
692
693/*
694 * Called if REQ_F_LINK is set, and we fail the head request
695 */
696static void io_fail_links(struct io_kiocb *req)
697{
698 struct io_kiocb *link;
699
700 while (!list_empty(&req->link_list)) {
701 link = list_first_entry(&req->link_list, struct io_kiocb, list);
702 list_del(&link->list);
703
704 io_cqring_add_event(req->ctx, link->user_data, -ECANCELED);
705 __io_free_req(link);
706 }
707}
708
709static void io_free_req(struct io_kiocb *req)
710{
711 /*
712 * If LINK is set, we have dependent requests in this chain. If we
713 * didn't fail this request, queue the first one up, moving any other
714 * dependencies to the next request. In case of failure, fail the rest
715 * of the chain.
716 */
717 if (req->flags & REQ_F_LINK) {
718 if (req->flags & REQ_F_FAIL_LINK)
719 io_fail_links(req);
720 else
721 io_req_link_next(req);
722 }
723
724 __io_free_req(req);
725}
726
Jens Axboee65ef562019-03-12 10:16:44 -0600727static void io_put_req(struct io_kiocb *req)
728{
729 if (refcount_dec_and_test(&req->refs))
730 io_free_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700731}
732
Hristo Venev75b28af2019-08-26 17:23:46 +0000733static unsigned io_cqring_events(struct io_rings *rings)
Jens Axboea3a0e432019-08-20 11:03:11 -0600734{
735 /* See comment at the top of this file */
736 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +0000737 return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
Jens Axboea3a0e432019-08-20 11:03:11 -0600738}
739
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +0300740static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
741{
742 struct io_rings *rings = ctx->rings;
743
744 /* make sure SQ entry isn't read before tail */
745 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
746}
747
Jens Axboedef596e2019-01-09 08:59:42 -0700748/*
749 * Find and free completed poll iocbs
750 */
751static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
752 struct list_head *done)
753{
754 void *reqs[IO_IOPOLL_BATCH];
755 struct io_kiocb *req;
Jens Axboe09bb8392019-03-13 12:39:28 -0600756 int to_free;
Jens Axboedef596e2019-01-09 08:59:42 -0700757
Jens Axboe09bb8392019-03-13 12:39:28 -0600758 to_free = 0;
Jens Axboedef596e2019-01-09 08:59:42 -0700759 while (!list_empty(done)) {
760 req = list_first_entry(done, struct io_kiocb, list);
761 list_del(&req->list);
762
Jens Axboe9e645e112019-05-10 16:07:28 -0600763 io_cqring_fill_event(ctx, req->user_data, req->result);
Jens Axboedef596e2019-01-09 08:59:42 -0700764 (*nr_events)++;
765
Jens Axboe09bb8392019-03-13 12:39:28 -0600766 if (refcount_dec_and_test(&req->refs)) {
767 /* If we're not using fixed files, we have to pair the
768 * completion part with the file put. Use regular
769 * completions for those, only batch free for fixed
Jens Axboe9e645e112019-05-10 16:07:28 -0600770 * file and non-linked commands.
Jens Axboe09bb8392019-03-13 12:39:28 -0600771 */
Jens Axboe9e645e112019-05-10 16:07:28 -0600772 if ((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
773 REQ_F_FIXED_FILE) {
Jens Axboe09bb8392019-03-13 12:39:28 -0600774 reqs[to_free++] = req;
775 if (to_free == ARRAY_SIZE(reqs))
776 io_free_req_many(ctx, reqs, &to_free);
Jens Axboe6b063142019-01-10 22:13:58 -0700777 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -0600778 io_free_req(req);
Jens Axboe6b063142019-01-10 22:13:58 -0700779 }
Jens Axboe9a56a232019-01-09 09:06:50 -0700780 }
Jens Axboedef596e2019-01-09 08:59:42 -0700781 }
Jens Axboedef596e2019-01-09 08:59:42 -0700782
Jens Axboe09bb8392019-03-13 12:39:28 -0600783 io_commit_cqring(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -0700784 io_free_req_many(ctx, reqs, &to_free);
785}
786
787static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
788 long min)
789{
790 struct io_kiocb *req, *tmp;
791 LIST_HEAD(done);
792 bool spin;
793 int ret;
794
795 /*
796 * Only spin for completions if we don't have multiple devices hanging
797 * off our complete list, and we're under the requested amount.
798 */
799 spin = !ctx->poll_multi_file && *nr_events < min;
800
801 ret = 0;
802 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
803 struct kiocb *kiocb = &req->rw;
804
805 /*
806 * Move completed entries to our local list. If we find a
807 * request that requires polling, break out and complete
808 * the done list first, if we have entries there.
809 */
810 if (req->flags & REQ_F_IOPOLL_COMPLETED) {
811 list_move_tail(&req->list, &done);
812 continue;
813 }
814 if (!list_empty(&done))
815 break;
816
817 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
818 if (ret < 0)
819 break;
820
821 if (ret && spin)
822 spin = false;
823 ret = 0;
824 }
825
826 if (!list_empty(&done))
827 io_iopoll_complete(ctx, nr_events, &done);
828
829 return ret;
830}
831
832/*
833 * Poll for a mininum of 'min' events. Note that if min == 0 we consider that a
834 * non-spinning poll check - we'll still enter the driver poll loop, but only
835 * as a non-spinning completion check.
836 */
837static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
838 long min)
839{
Jens Axboe08f54392019-08-21 22:19:11 -0600840 while (!list_empty(&ctx->poll_list) && !need_resched()) {
Jens Axboedef596e2019-01-09 08:59:42 -0700841 int ret;
842
843 ret = io_do_iopoll(ctx, nr_events, min);
844 if (ret < 0)
845 return ret;
846 if (!min || *nr_events >= min)
847 return 0;
848 }
849
850 return 1;
851}
852
853/*
854 * We can't just wait for polled events to come to us, we have to actively
855 * find and complete them.
856 */
857static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
858{
859 if (!(ctx->flags & IORING_SETUP_IOPOLL))
860 return;
861
862 mutex_lock(&ctx->uring_lock);
863 while (!list_empty(&ctx->poll_list)) {
864 unsigned int nr_events = 0;
865
866 io_iopoll_getevents(ctx, &nr_events, 1);
Jens Axboe08f54392019-08-21 22:19:11 -0600867
868 /*
869 * Ensure we allow local-to-the-cpu processing to take place,
870 * in this case we need to ensure that we reap all events.
871 */
872 cond_resched();
Jens Axboedef596e2019-01-09 08:59:42 -0700873 }
874 mutex_unlock(&ctx->uring_lock);
875}
876
Jens Axboe2b2ed972019-10-25 10:06:15 -0600877static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
878 long min)
Jens Axboedef596e2019-01-09 08:59:42 -0700879{
Jens Axboe2b2ed972019-10-25 10:06:15 -0600880 int iters = 0, ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -0700881
882 do {
883 int tmin = 0;
884
Jens Axboe500f9fb2019-08-19 12:15:59 -0600885 /*
Jens Axboea3a0e432019-08-20 11:03:11 -0600886 * Don't enter poll loop if we already have events pending.
887 * If we do, we can potentially be spinning for commands that
888 * already triggered a CQE (eg in error).
889 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000890 if (io_cqring_events(ctx->rings))
Jens Axboea3a0e432019-08-20 11:03:11 -0600891 break;
892
893 /*
Jens Axboe500f9fb2019-08-19 12:15:59 -0600894 * If a submit got punted to a workqueue, we can have the
895 * application entering polling for a command before it gets
896 * issued. That app will hold the uring_lock for the duration
897 * of the poll right here, so we need to take a breather every
898 * now and then to ensure that the issue has a chance to add
899 * the poll to the issued list. Otherwise we can spin here
900 * forever, while the workqueue is stuck trying to acquire the
901 * very same mutex.
902 */
903 if (!(++iters & 7)) {
904 mutex_unlock(&ctx->uring_lock);
905 mutex_lock(&ctx->uring_lock);
906 }
907
Jens Axboedef596e2019-01-09 08:59:42 -0700908 if (*nr_events < min)
909 tmin = min - *nr_events;
910
911 ret = io_iopoll_getevents(ctx, nr_events, tmin);
912 if (ret <= 0)
913 break;
914 ret = 0;
915 } while (min && !*nr_events && !need_resched());
916
Jens Axboe2b2ed972019-10-25 10:06:15 -0600917 return ret;
918}
919
920static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
921 long min)
922{
923 int ret;
924
925 /*
926 * We disallow the app entering submit/complete with polling, but we
927 * still need to lock the ring to prevent racing with polled issue
928 * that got punted to a workqueue.
929 */
930 mutex_lock(&ctx->uring_lock);
931 ret = __io_iopoll_check(ctx, nr_events, min);
Jens Axboe500f9fb2019-08-19 12:15:59 -0600932 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -0700933 return ret;
934}
935
Jens Axboe491381ce2019-10-17 09:20:46 -0600936static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700937{
Jens Axboe491381ce2019-10-17 09:20:46 -0600938 /*
939 * Tell lockdep we inherited freeze protection from submission
940 * thread.
941 */
942 if (req->flags & REQ_F_ISREG) {
943 struct inode *inode = file_inode(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700944
Jens Axboe491381ce2019-10-17 09:20:46 -0600945 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700946 }
Jens Axboe491381ce2019-10-17 09:20:46 -0600947 file_end_write(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700948}
949
950static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
951{
952 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
953
Jens Axboe491381ce2019-10-17 09:20:46 -0600954 if (kiocb->ki_flags & IOCB_WRITE)
955 kiocb_end_write(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700956
Jens Axboe9e645e112019-05-10 16:07:28 -0600957 if ((req->flags & REQ_F_LINK) && res != req->result)
958 req->flags |= REQ_F_FAIL_LINK;
Jens Axboec71ffb62019-05-13 20:58:29 -0600959 io_cqring_add_event(req->ctx, req->user_data, res);
Jens Axboee65ef562019-03-12 10:16:44 -0600960 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700961}
962
Jens Axboedef596e2019-01-09 08:59:42 -0700963static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
964{
965 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
966
Jens Axboe491381ce2019-10-17 09:20:46 -0600967 if (kiocb->ki_flags & IOCB_WRITE)
968 kiocb_end_write(req);
Jens Axboedef596e2019-01-09 08:59:42 -0700969
Jens Axboe9e645e112019-05-10 16:07:28 -0600970 if ((req->flags & REQ_F_LINK) && res != req->result)
971 req->flags |= REQ_F_FAIL_LINK;
972 req->result = res;
Jens Axboedef596e2019-01-09 08:59:42 -0700973 if (res != -EAGAIN)
974 req->flags |= REQ_F_IOPOLL_COMPLETED;
975}
976
977/*
978 * After the iocb has been issued, it's safe to be found on the poll list.
979 * Adding the kiocb to the list AFTER submission ensures that we don't
980 * find it from a io_iopoll_getevents() thread before the issuer is done
981 * accessing the kiocb cookie.
982 */
983static void io_iopoll_req_issued(struct io_kiocb *req)
984{
985 struct io_ring_ctx *ctx = req->ctx;
986
987 /*
988 * Track whether we have multiple files in our lists. This will impact
989 * how we do polling eventually, not spinning if we're on potentially
990 * different devices.
991 */
992 if (list_empty(&ctx->poll_list)) {
993 ctx->poll_multi_file = false;
994 } else if (!ctx->poll_multi_file) {
995 struct io_kiocb *list_req;
996
997 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
998 list);
999 if (list_req->rw.ki_filp != req->rw.ki_filp)
1000 ctx->poll_multi_file = true;
1001 }
1002
1003 /*
1004 * For fast devices, IO may have already completed. If it has, add
1005 * it to the front so we find it first.
1006 */
1007 if (req->flags & REQ_F_IOPOLL_COMPLETED)
1008 list_add(&req->list, &ctx->poll_list);
1009 else
1010 list_add_tail(&req->list, &ctx->poll_list);
1011}
1012
Jens Axboe3d6770f2019-04-13 11:50:54 -06001013static void io_file_put(struct io_submit_state *state)
Jens Axboe9a56a232019-01-09 09:06:50 -07001014{
Jens Axboe3d6770f2019-04-13 11:50:54 -06001015 if (state->file) {
Jens Axboe9a56a232019-01-09 09:06:50 -07001016 int diff = state->has_refs - state->used_refs;
1017
1018 if (diff)
1019 fput_many(state->file, diff);
1020 state->file = NULL;
1021 }
1022}
1023
1024/*
1025 * Get as many references to a file as we have IOs left in this submission,
1026 * assuming most submissions are for one file, or at least that each file
1027 * has more than one submission.
1028 */
1029static struct file *io_file_get(struct io_submit_state *state, int fd)
1030{
1031 if (!state)
1032 return fget(fd);
1033
1034 if (state->file) {
1035 if (state->fd == fd) {
1036 state->used_refs++;
1037 state->ios_left--;
1038 return state->file;
1039 }
Jens Axboe3d6770f2019-04-13 11:50:54 -06001040 io_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07001041 }
1042 state->file = fget_many(fd, state->ios_left);
1043 if (!state->file)
1044 return NULL;
1045
1046 state->fd = fd;
1047 state->has_refs = state->ios_left;
1048 state->used_refs = 1;
1049 state->ios_left--;
1050 return state->file;
1051}
1052
Jens Axboe2b188cc2019-01-07 10:46:33 -07001053/*
1054 * If we tracked the file through the SCM inflight mechanism, we could support
1055 * any file. For now, just ensure that anything potentially problematic is done
1056 * inline.
1057 */
1058static bool io_file_supports_async(struct file *file)
1059{
1060 umode_t mode = file_inode(file)->i_mode;
1061
1062 if (S_ISBLK(mode) || S_ISCHR(mode))
1063 return true;
1064 if (S_ISREG(mode) && file->f_op != &io_uring_fops)
1065 return true;
1066
1067 return false;
1068}
1069
Jens Axboe6c271ce2019-01-10 11:22:30 -07001070static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s,
Jens Axboe8358e3a2019-04-23 08:17:58 -06001071 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001072{
Jens Axboe6c271ce2019-01-10 11:22:30 -07001073 const struct io_uring_sqe *sqe = s->sqe;
Jens Axboedef596e2019-01-09 08:59:42 -07001074 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001075 struct kiocb *kiocb = &req->rw;
Jens Axboe09bb8392019-03-13 12:39:28 -06001076 unsigned ioprio;
1077 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001078
Jens Axboe09bb8392019-03-13 12:39:28 -06001079 if (!req->file)
1080 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001081
Jens Axboe491381ce2019-10-17 09:20:46 -06001082 if (S_ISREG(file_inode(req->file)->i_mode))
1083 req->flags |= REQ_F_ISREG;
1084
1085 /*
1086 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
1087 * we know to async punt it even if it was opened O_NONBLOCK
1088 */
1089 if (force_nonblock && !io_file_supports_async(req->file)) {
1090 req->flags |= REQ_F_MUST_PUNT;
1091 return -EAGAIN;
1092 }
Jens Axboe6b063142019-01-10 22:13:58 -07001093
Jens Axboe2b188cc2019-01-07 10:46:33 -07001094 kiocb->ki_pos = READ_ONCE(sqe->off);
1095 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
1096 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
1097
1098 ioprio = READ_ONCE(sqe->ioprio);
1099 if (ioprio) {
1100 ret = ioprio_check_cap(ioprio);
1101 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06001102 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001103
1104 kiocb->ki_ioprio = ioprio;
1105 } else
1106 kiocb->ki_ioprio = get_current_ioprio();
1107
1108 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
1109 if (unlikely(ret))
Jens Axboe09bb8392019-03-13 12:39:28 -06001110 return ret;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001111
1112 /* don't allow async punt if RWF_NOWAIT was requested */
Jens Axboe491381ce2019-10-17 09:20:46 -06001113 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
1114 (req->file->f_flags & O_NONBLOCK))
Stefan Bühler8449eed2019-04-27 20:34:19 +02001115 req->flags |= REQ_F_NOWAIT;
1116
1117 if (force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001118 kiocb->ki_flags |= IOCB_NOWAIT;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001119
Jens Axboedef596e2019-01-09 08:59:42 -07001120 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07001121 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
1122 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06001123 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001124
Jens Axboedef596e2019-01-09 08:59:42 -07001125 kiocb->ki_flags |= IOCB_HIPRI;
1126 kiocb->ki_complete = io_complete_rw_iopoll;
Jens Axboe6873e0b2019-10-30 13:53:09 -06001127 req->result = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001128 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06001129 if (kiocb->ki_flags & IOCB_HIPRI)
1130 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07001131 kiocb->ki_complete = io_complete_rw;
1132 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001133 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001134}
1135
1136static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
1137{
1138 switch (ret) {
1139 case -EIOCBQUEUED:
1140 break;
1141 case -ERESTARTSYS:
1142 case -ERESTARTNOINTR:
1143 case -ERESTARTNOHAND:
1144 case -ERESTART_RESTARTBLOCK:
1145 /*
1146 * We can't just restart the syscall, since previously
1147 * submitted sqes may already be in progress. Just fail this
1148 * IO with EINTR.
1149 */
1150 ret = -EINTR;
1151 /* fall through */
1152 default:
1153 kiocb->ki_complete(kiocb, ret, 0);
1154 }
1155}
1156
Jens Axboeedafcce2019-01-09 09:16:05 -07001157static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
1158 const struct io_uring_sqe *sqe,
1159 struct iov_iter *iter)
1160{
1161 size_t len = READ_ONCE(sqe->len);
1162 struct io_mapped_ubuf *imu;
1163 unsigned index, buf_index;
1164 size_t offset;
1165 u64 buf_addr;
1166
1167 /* attempt to use fixed buffers without having provided iovecs */
1168 if (unlikely(!ctx->user_bufs))
1169 return -EFAULT;
1170
1171 buf_index = READ_ONCE(sqe->buf_index);
1172 if (unlikely(buf_index >= ctx->nr_user_bufs))
1173 return -EFAULT;
1174
1175 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
1176 imu = &ctx->user_bufs[index];
1177 buf_addr = READ_ONCE(sqe->addr);
1178
1179 /* overflow */
1180 if (buf_addr + len < buf_addr)
1181 return -EFAULT;
1182 /* not inside the mapped region */
1183 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
1184 return -EFAULT;
1185
1186 /*
1187 * May not be a start of buffer, set size appropriately
1188 * and advance us to the beginning.
1189 */
1190 offset = buf_addr - imu->ubuf;
1191 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06001192
1193 if (offset) {
1194 /*
1195 * Don't use iov_iter_advance() here, as it's really slow for
1196 * using the latter parts of a big fixed buffer - it iterates
1197 * over each segment manually. We can cheat a bit here, because
1198 * we know that:
1199 *
1200 * 1) it's a BVEC iter, we set it up
1201 * 2) all bvecs are PAGE_SIZE in size, except potentially the
1202 * first and last bvec
1203 *
1204 * So just find our index, and adjust the iterator afterwards.
1205 * If the offset is within the first bvec (or the whole first
1206 * bvec, just use iov_iter_advance(). This makes it easier
1207 * since we can just skip the first segment, which may not
1208 * be PAGE_SIZE aligned.
1209 */
1210 const struct bio_vec *bvec = imu->bvec;
1211
1212 if (offset <= bvec->bv_len) {
1213 iov_iter_advance(iter, offset);
1214 } else {
1215 unsigned long seg_skip;
1216
1217 /* skip first vec */
1218 offset -= bvec->bv_len;
1219 seg_skip = 1 + (offset >> PAGE_SHIFT);
1220
1221 iter->bvec = bvec + seg_skip;
1222 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02001223 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06001224 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06001225 }
1226 }
1227
Jens Axboeedafcce2019-01-09 09:16:05 -07001228 return 0;
1229}
1230
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001231static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw,
1232 const struct sqe_submit *s, struct iovec **iovec,
1233 struct iov_iter *iter)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001234{
1235 const struct io_uring_sqe *sqe = s->sqe;
1236 void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
1237 size_t sqe_len = READ_ONCE(sqe->len);
Jens Axboeedafcce2019-01-09 09:16:05 -07001238 u8 opcode;
1239
1240 /*
1241 * We're reading ->opcode for the second time, but the first read
1242 * doesn't care whether it's _FIXED or not, so it doesn't matter
1243 * whether ->opcode changes concurrently. The first read does care
1244 * about whether it is a READ or a WRITE, so we don't trust this read
1245 * for that purpose and instead let the caller pass in the read/write
1246 * flag.
1247 */
1248 opcode = READ_ONCE(sqe->opcode);
1249 if (opcode == IORING_OP_READ_FIXED ||
1250 opcode == IORING_OP_WRITE_FIXED) {
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001251 ssize_t ret = io_import_fixed(ctx, rw, sqe, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07001252 *iovec = NULL;
1253 return ret;
1254 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001255
1256 if (!s->has_user)
1257 return -EFAULT;
1258
1259#ifdef CONFIG_COMPAT
1260 if (ctx->compat)
1261 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
1262 iovec, iter);
1263#endif
1264
1265 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
1266}
1267
Jens Axboe6d5d5ac2019-09-11 10:16:13 -06001268static inline bool io_should_merge(struct async_list *al, struct kiocb *kiocb)
1269{
1270 if (al->file == kiocb->ki_filp) {
1271 off_t start, end;
1272
1273 /*
1274 * Allow merging if we're anywhere in the range of the same
1275 * page. Generally this happens for sub-page reads or writes,
1276 * and it's beneficial to allow the first worker to bring the
1277 * page in and the piggy backed work can then work on the
1278 * cached page.
1279 */
1280 start = al->io_start & PAGE_MASK;
1281 end = (al->io_start + al->io_len + PAGE_SIZE - 1) & PAGE_MASK;
1282 if (kiocb->ki_pos >= start && kiocb->ki_pos <= end)
1283 return true;
1284 }
1285
1286 al->file = NULL;
1287 return false;
1288}
1289
Jens Axboe31b51512019-01-18 22:56:34 -07001290/*
1291 * Make a note of the last file/offset/direction we punted to async
1292 * context. We'll use this information to see if we can piggy back a
1293 * sequential request onto the previous one, if it's still hasn't been
1294 * completed by the async worker.
1295 */
1296static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
1297{
1298 struct async_list *async_list = &req->ctx->pending_async[rw];
1299 struct kiocb *kiocb = &req->rw;
1300 struct file *filp = kiocb->ki_filp;
Jens Axboe31b51512019-01-18 22:56:34 -07001301
Jens Axboe6d5d5ac2019-09-11 10:16:13 -06001302 if (io_should_merge(async_list, kiocb)) {
Zhengyuan Liu9310a7ba2019-07-22 10:23:27 +08001303 unsigned long max_bytes;
Jens Axboe31b51512019-01-18 22:56:34 -07001304
1305 /* Use 8x RA size as a decent limiter for both reads/writes */
Zhengyuan Liu9310a7ba2019-07-22 10:23:27 +08001306 max_bytes = filp->f_ra.ra_pages << (PAGE_SHIFT + 3);
1307 if (!max_bytes)
1308 max_bytes = VM_READAHEAD_PAGES << (PAGE_SHIFT + 3);
Jens Axboe31b51512019-01-18 22:56:34 -07001309
Zhengyuan Liu9310a7ba2019-07-22 10:23:27 +08001310 /* If max len are exceeded, reset the state */
1311 if (async_list->io_len + len <= max_bytes) {
Jens Axboe31b51512019-01-18 22:56:34 -07001312 req->flags |= REQ_F_SEQ_PREV;
Zhengyuan Liu9310a7ba2019-07-22 10:23:27 +08001313 async_list->io_len += len;
Jens Axboe31b51512019-01-18 22:56:34 -07001314 } else {
Jens Axboe6d5d5ac2019-09-11 10:16:13 -06001315 async_list->file = NULL;
Jens Axboe31b51512019-01-18 22:56:34 -07001316 }
1317 }
1318
1319 /* New file? Reset state. */
1320 if (async_list->file != filp) {
Jens Axboe6d5d5ac2019-09-11 10:16:13 -06001321 async_list->io_start = kiocb->ki_pos;
1322 async_list->io_len = len;
Jens Axboe31b51512019-01-18 22:56:34 -07001323 async_list->file = filp;
1324 }
Jens Axboe31b51512019-01-18 22:56:34 -07001325}
1326
Jens Axboe32960612019-09-23 11:05:34 -06001327/*
1328 * For files that don't have ->read_iter() and ->write_iter(), handle them
1329 * by looping over ->read() or ->write() manually.
1330 */
1331static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
1332 struct iov_iter *iter)
1333{
1334 ssize_t ret = 0;
1335
1336 /*
1337 * Don't support polled IO through this interface, and we can't
1338 * support non-blocking either. For the latter, this just causes
1339 * the kiocb to be handled from an async context.
1340 */
1341 if (kiocb->ki_flags & IOCB_HIPRI)
1342 return -EOPNOTSUPP;
1343 if (kiocb->ki_flags & IOCB_NOWAIT)
1344 return -EAGAIN;
1345
1346 while (iov_iter_count(iter)) {
1347 struct iovec iovec = iov_iter_iovec(iter);
1348 ssize_t nr;
1349
1350 if (rw == READ) {
1351 nr = file->f_op->read(file, iovec.iov_base,
1352 iovec.iov_len, &kiocb->ki_pos);
1353 } else {
1354 nr = file->f_op->write(file, iovec.iov_base,
1355 iovec.iov_len, &kiocb->ki_pos);
1356 }
1357
1358 if (nr < 0) {
1359 if (!ret)
1360 ret = nr;
1361 break;
1362 }
1363 ret += nr;
1364 if (nr != iovec.iov_len)
1365 break;
1366 iov_iter_advance(iter, nr);
1367 }
1368
1369 return ret;
1370}
1371
Jens Axboee0c5c572019-03-12 10:18:47 -06001372static int io_read(struct io_kiocb *req, const struct sqe_submit *s,
Jens Axboe8358e3a2019-04-23 08:17:58 -06001373 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001374{
1375 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1376 struct kiocb *kiocb = &req->rw;
1377 struct iov_iter iter;
1378 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001379 size_t iov_count;
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001380 ssize_t read_size, ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001381
Jens Axboe8358e3a2019-04-23 08:17:58 -06001382 ret = io_prep_rw(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001383 if (ret)
1384 return ret;
1385 file = kiocb->ki_filp;
1386
Jens Axboe2b188cc2019-01-07 10:46:33 -07001387 if (unlikely(!(file->f_mode & FMODE_READ)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001388 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001389
1390 ret = io_import_iovec(req->ctx, READ, s, &iovec, &iter);
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001391 if (ret < 0)
Jens Axboe09bb8392019-03-13 12:39:28 -06001392 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001393
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001394 read_size = ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06001395 if (req->flags & REQ_F_LINK)
1396 req->result = read_size;
1397
Jens Axboe31b51512019-01-18 22:56:34 -07001398 iov_count = iov_iter_count(&iter);
1399 ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001400 if (!ret) {
1401 ssize_t ret2;
1402
Jens Axboe32960612019-09-23 11:05:34 -06001403 if (file->f_op->read_iter)
1404 ret2 = call_read_iter(file, kiocb, &iter);
1405 else
1406 ret2 = loop_rw_iter(READ, file, kiocb, &iter);
1407
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001408 /*
1409 * In case of a short read, punt to async. This can happen
1410 * if we have data partially cached. Alternatively we can
1411 * return the short read, in which case the application will
1412 * need to issue another SQE and wait for it. That SQE will
1413 * need async punt anyway, so it's more efficient to do it
1414 * here.
1415 */
Jens Axboe491381ce2019-10-17 09:20:46 -06001416 if (force_nonblock && !(req->flags & REQ_F_NOWAIT) &&
1417 (req->flags & REQ_F_ISREG) &&
1418 ret2 > 0 && ret2 < read_size)
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001419 ret2 = -EAGAIN;
1420 /* Catch -EAGAIN return for forced non-blocking submission */
Jens Axboe31b51512019-01-18 22:56:34 -07001421 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001422 io_rw_done(kiocb, ret2);
Jens Axboe31b51512019-01-18 22:56:34 -07001423 } else {
1424 /*
1425 * If ->needs_lock is true, we're already in async
1426 * context.
1427 */
1428 if (!s->needs_lock)
1429 io_async_list_note(READ, req, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001430 ret = -EAGAIN;
Jens Axboe31b51512019-01-18 22:56:34 -07001431 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001432 }
1433 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001434 return ret;
1435}
1436
Jens Axboee0c5c572019-03-12 10:18:47 -06001437static int io_write(struct io_kiocb *req, const struct sqe_submit *s,
Jens Axboe8358e3a2019-04-23 08:17:58 -06001438 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001439{
1440 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1441 struct kiocb *kiocb = &req->rw;
1442 struct iov_iter iter;
1443 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001444 size_t iov_count;
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001445 ssize_t ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001446
Jens Axboe8358e3a2019-04-23 08:17:58 -06001447 ret = io_prep_rw(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001448 if (ret)
1449 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001450
Jens Axboe2b188cc2019-01-07 10:46:33 -07001451 file = kiocb->ki_filp;
1452 if (unlikely(!(file->f_mode & FMODE_WRITE)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001453 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001454
1455 ret = io_import_iovec(req->ctx, WRITE, s, &iovec, &iter);
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001456 if (ret < 0)
Jens Axboe09bb8392019-03-13 12:39:28 -06001457 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001458
Jens Axboe9e645e112019-05-10 16:07:28 -06001459 if (req->flags & REQ_F_LINK)
1460 req->result = ret;
1461
Jens Axboe31b51512019-01-18 22:56:34 -07001462 iov_count = iov_iter_count(&iter);
1463
1464 ret = -EAGAIN;
1465 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT)) {
1466 /* If ->needs_lock is true, we're already in async context. */
1467 if (!s->needs_lock)
1468 io_async_list_note(WRITE, req, iov_count);
1469 goto out_free;
1470 }
1471
1472 ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001473 if (!ret) {
Roman Penyaev9bf79332019-03-25 20:09:24 +01001474 ssize_t ret2;
1475
Jens Axboe2b188cc2019-01-07 10:46:33 -07001476 /*
1477 * Open-code file_start_write here to grab freeze protection,
1478 * which will be released by another thread in
1479 * io_complete_rw(). Fool lockdep by telling it the lock got
1480 * released so that it doesn't complain about the held lock when
1481 * we return to userspace.
1482 */
Jens Axboe491381ce2019-10-17 09:20:46 -06001483 if (req->flags & REQ_F_ISREG) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001484 __sb_start_write(file_inode(file)->i_sb,
1485 SB_FREEZE_WRITE, true);
1486 __sb_writers_release(file_inode(file)->i_sb,
1487 SB_FREEZE_WRITE);
1488 }
1489 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01001490
Jens Axboe32960612019-09-23 11:05:34 -06001491 if (file->f_op->write_iter)
1492 ret2 = call_write_iter(file, kiocb, &iter);
1493 else
1494 ret2 = loop_rw_iter(WRITE, file, kiocb, &iter);
Roman Penyaev9bf79332019-03-25 20:09:24 +01001495 if (!force_nonblock || ret2 != -EAGAIN) {
1496 io_rw_done(kiocb, ret2);
1497 } else {
1498 /*
1499 * If ->needs_lock is true, we're already in async
1500 * context.
1501 */
1502 if (!s->needs_lock)
1503 io_async_list_note(WRITE, req, iov_count);
1504 ret = -EAGAIN;
1505 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001506 }
Jens Axboe31b51512019-01-18 22:56:34 -07001507out_free:
Jens Axboe2b188cc2019-01-07 10:46:33 -07001508 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001509 return ret;
1510}
1511
1512/*
1513 * IORING_OP_NOP just posts a completion event, nothing else.
1514 */
1515static int io_nop(struct io_kiocb *req, u64 user_data)
1516{
1517 struct io_ring_ctx *ctx = req->ctx;
1518 long err = 0;
1519
Jens Axboedef596e2019-01-09 08:59:42 -07001520 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1521 return -EINVAL;
1522
Jens Axboec71ffb62019-05-13 20:58:29 -06001523 io_cqring_add_event(ctx, user_data, err);
Jens Axboee65ef562019-03-12 10:16:44 -06001524 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001525 return 0;
1526}
1527
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001528static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1529{
Jens Axboe6b063142019-01-10 22:13:58 -07001530 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001531
Jens Axboe09bb8392019-03-13 12:39:28 -06001532 if (!req->file)
1533 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001534
Jens Axboe6b063142019-01-10 22:13:58 -07001535 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07001536 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07001537 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001538 return -EINVAL;
1539
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001540 return 0;
1541}
1542
1543static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1544 bool force_nonblock)
1545{
1546 loff_t sqe_off = READ_ONCE(sqe->off);
1547 loff_t sqe_len = READ_ONCE(sqe->len);
1548 loff_t end = sqe_off + sqe_len;
1549 unsigned fsync_flags;
1550 int ret;
1551
1552 fsync_flags = READ_ONCE(sqe->fsync_flags);
1553 if (unlikely(fsync_flags & ~IORING_FSYNC_DATASYNC))
1554 return -EINVAL;
1555
1556 ret = io_prep_fsync(req, sqe);
1557 if (ret)
1558 return ret;
1559
1560 /* fsync always requires a blocking context */
1561 if (force_nonblock)
1562 return -EAGAIN;
1563
1564 ret = vfs_fsync_range(req->rw.ki_filp, sqe_off,
1565 end > 0 ? end : LLONG_MAX,
1566 fsync_flags & IORING_FSYNC_DATASYNC);
1567
Jens Axboe9e645e112019-05-10 16:07:28 -06001568 if (ret < 0 && (req->flags & REQ_F_LINK))
1569 req->flags |= REQ_F_FAIL_LINK;
Jens Axboec71ffb62019-05-13 20:58:29 -06001570 io_cqring_add_event(req->ctx, sqe->user_data, ret);
Jens Axboee65ef562019-03-12 10:16:44 -06001571 io_put_req(req);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001572 return 0;
1573}
1574
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001575static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1576{
1577 struct io_ring_ctx *ctx = req->ctx;
1578 int ret = 0;
1579
1580 if (!req->file)
1581 return -EBADF;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001582
1583 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1584 return -EINVAL;
1585 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
1586 return -EINVAL;
1587
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001588 return ret;
1589}
1590
1591static int io_sync_file_range(struct io_kiocb *req,
1592 const struct io_uring_sqe *sqe,
1593 bool force_nonblock)
1594{
1595 loff_t sqe_off;
1596 loff_t sqe_len;
1597 unsigned flags;
1598 int ret;
1599
1600 ret = io_prep_sfr(req, sqe);
1601 if (ret)
1602 return ret;
1603
1604 /* sync_file_range always requires a blocking context */
1605 if (force_nonblock)
1606 return -EAGAIN;
1607
1608 sqe_off = READ_ONCE(sqe->off);
1609 sqe_len = READ_ONCE(sqe->len);
1610 flags = READ_ONCE(sqe->sync_range_flags);
1611
1612 ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags);
1613
Jens Axboe9e645e112019-05-10 16:07:28 -06001614 if (ret < 0 && (req->flags & REQ_F_LINK))
1615 req->flags |= REQ_F_FAIL_LINK;
Jens Axboec71ffb62019-05-13 20:58:29 -06001616 io_cqring_add_event(req->ctx, sqe->user_data, ret);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001617 io_put_req(req);
1618 return 0;
1619}
1620
Jens Axboe0fa03c62019-04-19 13:34:07 -06001621#if defined(CONFIG_NET)
Jens Axboeaa1fa282019-04-19 13:38:09 -06001622static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1623 bool force_nonblock,
1624 long (*fn)(struct socket *, struct user_msghdr __user *,
1625 unsigned int))
1626{
Jens Axboe0fa03c62019-04-19 13:34:07 -06001627 struct socket *sock;
1628 int ret;
1629
1630 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1631 return -EINVAL;
1632
1633 sock = sock_from_file(req->file, &ret);
1634 if (sock) {
1635 struct user_msghdr __user *msg;
1636 unsigned flags;
1637
1638 flags = READ_ONCE(sqe->msg_flags);
1639 if (flags & MSG_DONTWAIT)
1640 req->flags |= REQ_F_NOWAIT;
1641 else if (force_nonblock)
1642 flags |= MSG_DONTWAIT;
1643
1644 msg = (struct user_msghdr __user *) (unsigned long)
1645 READ_ONCE(sqe->addr);
1646
Jens Axboeaa1fa282019-04-19 13:38:09 -06001647 ret = fn(sock, msg, flags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001648 if (force_nonblock && ret == -EAGAIN)
1649 return ret;
1650 }
1651
1652 io_cqring_add_event(req->ctx, sqe->user_data, ret);
1653 io_put_req(req);
1654 return 0;
Jens Axboeaa1fa282019-04-19 13:38:09 -06001655}
1656#endif
1657
1658static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1659 bool force_nonblock)
1660{
1661#if defined(CONFIG_NET)
1662 return io_send_recvmsg(req, sqe, force_nonblock, __sys_sendmsg_sock);
1663#else
1664 return -EOPNOTSUPP;
1665#endif
1666}
1667
1668static int io_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1669 bool force_nonblock)
1670{
1671#if defined(CONFIG_NET)
1672 return io_send_recvmsg(req, sqe, force_nonblock, __sys_recvmsg_sock);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001673#else
1674 return -EOPNOTSUPP;
1675#endif
1676}
1677
Jens Axboe221c5eb2019-01-17 09:41:58 -07001678static void io_poll_remove_one(struct io_kiocb *req)
1679{
1680 struct io_poll_iocb *poll = &req->poll;
1681
1682 spin_lock(&poll->head->lock);
1683 WRITE_ONCE(poll->canceled, true);
1684 if (!list_empty(&poll->wait.entry)) {
1685 list_del_init(&poll->wait.entry);
Jens Axboe18d9be12019-09-10 09:13:05 -06001686 io_queue_async_work(req->ctx, req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001687 }
1688 spin_unlock(&poll->head->lock);
1689
1690 list_del_init(&req->list);
1691}
1692
1693static void io_poll_remove_all(struct io_ring_ctx *ctx)
1694{
1695 struct io_kiocb *req;
1696
1697 spin_lock_irq(&ctx->completion_lock);
1698 while (!list_empty(&ctx->cancel_list)) {
1699 req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list);
1700 io_poll_remove_one(req);
1701 }
1702 spin_unlock_irq(&ctx->completion_lock);
1703}
1704
1705/*
1706 * Find a running poll command that matches one specified in sqe->addr,
1707 * and remove it if found.
1708 */
1709static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1710{
1711 struct io_ring_ctx *ctx = req->ctx;
1712 struct io_kiocb *poll_req, *next;
1713 int ret = -ENOENT;
1714
1715 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1716 return -EINVAL;
1717 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
1718 sqe->poll_events)
1719 return -EINVAL;
1720
1721 spin_lock_irq(&ctx->completion_lock);
1722 list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
1723 if (READ_ONCE(sqe->addr) == poll_req->user_data) {
1724 io_poll_remove_one(poll_req);
1725 ret = 0;
1726 break;
1727 }
1728 }
1729 spin_unlock_irq(&ctx->completion_lock);
1730
Jens Axboec71ffb62019-05-13 20:58:29 -06001731 io_cqring_add_event(req->ctx, sqe->user_data, ret);
Jens Axboee65ef562019-03-12 10:16:44 -06001732 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001733 return 0;
1734}
1735
Jens Axboe8c838782019-03-12 15:48:16 -06001736static void io_poll_complete(struct io_ring_ctx *ctx, struct io_kiocb *req,
1737 __poll_t mask)
Jens Axboe221c5eb2019-01-17 09:41:58 -07001738{
Jens Axboe8c838782019-03-12 15:48:16 -06001739 req->poll.done = true;
Jens Axboec71ffb62019-05-13 20:58:29 -06001740 io_cqring_fill_event(ctx, req->user_data, mangle_poll(mask));
Jens Axboe8c838782019-03-12 15:48:16 -06001741 io_commit_cqring(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001742}
1743
1744static void io_poll_complete_work(struct work_struct *work)
1745{
1746 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
1747 struct io_poll_iocb *poll = &req->poll;
1748 struct poll_table_struct pt = { ._key = poll->events };
1749 struct io_ring_ctx *ctx = req->ctx;
1750 __poll_t mask = 0;
1751
1752 if (!READ_ONCE(poll->canceled))
1753 mask = vfs_poll(poll->file, &pt) & poll->events;
1754
1755 /*
1756 * Note that ->ki_cancel callers also delete iocb from active_reqs after
1757 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
1758 * synchronize with them. In the cancellation case the list_del_init
1759 * itself is not actually needed, but harmless so we keep it in to
1760 * avoid further branches in the fast path.
1761 */
1762 spin_lock_irq(&ctx->completion_lock);
1763 if (!mask && !READ_ONCE(poll->canceled)) {
1764 add_wait_queue(poll->head, &poll->wait);
1765 spin_unlock_irq(&ctx->completion_lock);
1766 return;
1767 }
1768 list_del_init(&req->list);
Jens Axboe8c838782019-03-12 15:48:16 -06001769 io_poll_complete(ctx, req, mask);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001770 spin_unlock_irq(&ctx->completion_lock);
1771
Jens Axboe8c838782019-03-12 15:48:16 -06001772 io_cqring_ev_posted(ctx);
1773 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001774}
1775
1776static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
1777 void *key)
1778{
1779 struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
1780 wait);
1781 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
1782 struct io_ring_ctx *ctx = req->ctx;
1783 __poll_t mask = key_to_poll(key);
Jens Axboe8c838782019-03-12 15:48:16 -06001784 unsigned long flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001785
1786 /* for instances that support it check for an event match first: */
Jens Axboe8c838782019-03-12 15:48:16 -06001787 if (mask && !(mask & poll->events))
1788 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001789
1790 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06001791
1792 if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) {
1793 list_del(&req->list);
1794 io_poll_complete(ctx, req, mask);
1795 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1796
1797 io_cqring_ev_posted(ctx);
1798 io_put_req(req);
1799 } else {
Jens Axboe18d9be12019-09-10 09:13:05 -06001800 io_queue_async_work(ctx, req);
Jens Axboe8c838782019-03-12 15:48:16 -06001801 }
1802
Jens Axboe221c5eb2019-01-17 09:41:58 -07001803 return 1;
1804}
1805
1806struct io_poll_table {
1807 struct poll_table_struct pt;
1808 struct io_kiocb *req;
1809 int error;
1810};
1811
1812static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
1813 struct poll_table_struct *p)
1814{
1815 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
1816
1817 if (unlikely(pt->req->poll.head)) {
1818 pt->error = -EINVAL;
1819 return;
1820 }
1821
1822 pt->error = 0;
1823 pt->req->poll.head = head;
1824 add_wait_queue(head, &pt->req->poll.wait);
1825}
1826
1827static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1828{
1829 struct io_poll_iocb *poll = &req->poll;
1830 struct io_ring_ctx *ctx = req->ctx;
1831 struct io_poll_table ipt;
Jens Axboe8c838782019-03-12 15:48:16 -06001832 bool cancel = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001833 __poll_t mask;
1834 u16 events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001835
1836 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1837 return -EINVAL;
1838 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
1839 return -EINVAL;
Jens Axboe09bb8392019-03-13 12:39:28 -06001840 if (!poll->file)
1841 return -EBADF;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001842
Jens Axboe6cc47d12019-09-18 11:18:23 -06001843 req->submit.sqe = NULL;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001844 INIT_WORK(&req->work, io_poll_complete_work);
1845 events = READ_ONCE(sqe->poll_events);
1846 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
1847
Jens Axboe221c5eb2019-01-17 09:41:58 -07001848 poll->head = NULL;
Jens Axboe8c838782019-03-12 15:48:16 -06001849 poll->done = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001850 poll->canceled = false;
1851
1852 ipt.pt._qproc = io_poll_queue_proc;
1853 ipt.pt._key = poll->events;
1854 ipt.req = req;
1855 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
1856
1857 /* initialized the list so that we can do list_empty checks */
1858 INIT_LIST_HEAD(&poll->wait.entry);
1859 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
1860
Jens Axboe36703242019-07-25 10:20:18 -06001861 INIT_LIST_HEAD(&req->list);
1862
Jens Axboe221c5eb2019-01-17 09:41:58 -07001863 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001864
1865 spin_lock_irq(&ctx->completion_lock);
Jens Axboe8c838782019-03-12 15:48:16 -06001866 if (likely(poll->head)) {
1867 spin_lock(&poll->head->lock);
1868 if (unlikely(list_empty(&poll->wait.entry))) {
1869 if (ipt.error)
1870 cancel = true;
1871 ipt.error = 0;
1872 mask = 0;
1873 }
1874 if (mask || ipt.error)
1875 list_del_init(&poll->wait.entry);
1876 else if (cancel)
1877 WRITE_ONCE(poll->canceled, true);
1878 else if (!poll->done) /* actually waiting for an event */
1879 list_add_tail(&req->list, &ctx->cancel_list);
1880 spin_unlock(&poll->head->lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001881 }
Jens Axboe8c838782019-03-12 15:48:16 -06001882 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06001883 ipt.error = 0;
1884 io_poll_complete(ctx, req, mask);
1885 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07001886 spin_unlock_irq(&ctx->completion_lock);
1887
Jens Axboe8c838782019-03-12 15:48:16 -06001888 if (mask) {
1889 io_cqring_ev_posted(ctx);
Jens Axboee65ef562019-03-12 10:16:44 -06001890 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001891 }
Jens Axboe8c838782019-03-12 15:48:16 -06001892 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001893}
1894
Jens Axboe5262f562019-09-17 12:26:57 -06001895static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
1896{
1897 struct io_ring_ctx *ctx;
zhangyi (F)ef036812019-10-23 15:10:08 +08001898 struct io_kiocb *req, *prev;
Jens Axboe5262f562019-09-17 12:26:57 -06001899 unsigned long flags;
1900
1901 req = container_of(timer, struct io_kiocb, timeout.timer);
1902 ctx = req->ctx;
1903 atomic_inc(&ctx->cq_timeouts);
1904
1905 spin_lock_irqsave(&ctx->completion_lock, flags);
zhangyi (F)ef036812019-10-23 15:10:08 +08001906 /*
1907 * Adjust the reqs sequence before the current one because it
1908 * will consume a slot in the cq_ring and the the cq_tail pointer
1909 * will be increased, otherwise other timeout reqs may return in
1910 * advance without waiting for enough wait_nr.
1911 */
1912 prev = req;
1913 list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
1914 prev->sequence++;
Jens Axboe5262f562019-09-17 12:26:57 -06001915 list_del(&req->list);
1916
1917 io_cqring_fill_event(ctx, req->user_data, -ETIME);
1918 io_commit_cqring(ctx);
1919 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1920
1921 io_cqring_ev_posted(ctx);
1922
1923 io_put_req(req);
1924 return HRTIMER_NORESTART;
1925}
1926
1927static int io_timeout(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1928{
yangerkun5da0fb12019-10-15 21:59:29 +08001929 unsigned count;
Jens Axboe5262f562019-09-17 12:26:57 -06001930 struct io_ring_ctx *ctx = req->ctx;
1931 struct list_head *entry;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06001932 struct timespec64 ts;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08001933 unsigned span = 0;
Jens Axboe5262f562019-09-17 12:26:57 -06001934
1935 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1936 return -EINVAL;
1937 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->timeout_flags ||
1938 sqe->len != 1)
1939 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06001940
1941 if (get_timespec64(&ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06001942 return -EFAULT;
1943
1944 /*
1945 * sqe->off holds how many events that need to occur for this
1946 * timeout event to be satisfied.
1947 */
1948 count = READ_ONCE(sqe->off);
1949 if (!count)
1950 count = 1;
1951
1952 req->sequence = ctx->cached_sq_head + count - 1;
yangerkun5da0fb12019-10-15 21:59:29 +08001953 /* reuse it to store the count */
1954 req->submit.sequence = count;
Jens Axboe5262f562019-09-17 12:26:57 -06001955 req->flags |= REQ_F_TIMEOUT;
1956
1957 /*
1958 * Insertion sort, ensuring the first entry in the list is always
1959 * the one we need first.
1960 */
Jens Axboe5262f562019-09-17 12:26:57 -06001961 spin_lock_irq(&ctx->completion_lock);
1962 list_for_each_prev(entry, &ctx->timeout_list) {
1963 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
yangerkun5da0fb12019-10-15 21:59:29 +08001964 unsigned nxt_sq_head;
1965 long long tmp, tmp_nxt;
Jens Axboe5262f562019-09-17 12:26:57 -06001966
yangerkun5da0fb12019-10-15 21:59:29 +08001967 /*
1968 * Since cached_sq_head + count - 1 can overflow, use type long
1969 * long to store it.
1970 */
1971 tmp = (long long)ctx->cached_sq_head + count - 1;
1972 nxt_sq_head = nxt->sequence - nxt->submit.sequence + 1;
1973 tmp_nxt = (long long)nxt_sq_head + nxt->submit.sequence - 1;
1974
1975 /*
1976 * cached_sq_head may overflow, and it will never overflow twice
1977 * once there is some timeout req still be valid.
1978 */
1979 if (ctx->cached_sq_head < nxt_sq_head)
yangerkun8b07a652019-10-17 12:12:35 +08001980 tmp += UINT_MAX;
yangerkun5da0fb12019-10-15 21:59:29 +08001981
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08001982 if (tmp > tmp_nxt)
Jens Axboe5262f562019-09-17 12:26:57 -06001983 break;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08001984
1985 /*
1986 * Sequence of reqs after the insert one and itself should
1987 * be adjusted because each timeout req consumes a slot.
1988 */
1989 span++;
1990 nxt->sequence++;
Jens Axboe5262f562019-09-17 12:26:57 -06001991 }
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08001992 req->sequence -= span;
Jens Axboe5262f562019-09-17 12:26:57 -06001993 list_add(&req->list, entry);
1994 spin_unlock_irq(&ctx->completion_lock);
1995
1996 hrtimer_init(&req->timeout.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1997 req->timeout.timer.function = io_timeout_fn;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06001998 hrtimer_start(&req->timeout.timer, timespec64_to_ktime(ts),
Jens Axboe5262f562019-09-17 12:26:57 -06001999 HRTIMER_MODE_REL);
2000 return 0;
2001}
2002
Jens Axboede0617e2019-04-06 21:51:27 -06002003static int io_req_defer(struct io_ring_ctx *ctx, struct io_kiocb *req,
2004 const struct io_uring_sqe *sqe)
2005{
2006 struct io_uring_sqe *sqe_copy;
2007
2008 if (!io_sequence_defer(ctx, req) && list_empty(&ctx->defer_list))
2009 return 0;
2010
2011 sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL);
2012 if (!sqe_copy)
2013 return -EAGAIN;
2014
2015 spin_lock_irq(&ctx->completion_lock);
2016 if (!io_sequence_defer(ctx, req) && list_empty(&ctx->defer_list)) {
2017 spin_unlock_irq(&ctx->completion_lock);
2018 kfree(sqe_copy);
2019 return 0;
2020 }
2021
2022 memcpy(sqe_copy, sqe, sizeof(*sqe_copy));
2023 req->submit.sqe = sqe_copy;
2024
2025 INIT_WORK(&req->work, io_sq_wq_submit_work);
2026 list_add_tail(&req->list, &ctx->defer_list);
2027 spin_unlock_irq(&ctx->completion_lock);
2028 return -EIOCBQUEUED;
2029}
2030
Jens Axboe2b188cc2019-01-07 10:46:33 -07002031static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Jens Axboe8358e3a2019-04-23 08:17:58 -06002032 const struct sqe_submit *s, bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002033{
Jens Axboee0c5c572019-03-12 10:18:47 -06002034 int ret, opcode;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002035
Jens Axboe9e645e112019-05-10 16:07:28 -06002036 req->user_data = READ_ONCE(s->sqe->user_data);
2037
Jens Axboe2b188cc2019-01-07 10:46:33 -07002038 if (unlikely(s->index >= ctx->sq_entries))
2039 return -EINVAL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002040
2041 opcode = READ_ONCE(s->sqe->opcode);
2042 switch (opcode) {
2043 case IORING_OP_NOP:
2044 ret = io_nop(req, req->user_data);
2045 break;
2046 case IORING_OP_READV:
Jens Axboeedafcce2019-01-09 09:16:05 -07002047 if (unlikely(s->sqe->buf_index))
2048 return -EINVAL;
Jens Axboe8358e3a2019-04-23 08:17:58 -06002049 ret = io_read(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002050 break;
2051 case IORING_OP_WRITEV:
Jens Axboeedafcce2019-01-09 09:16:05 -07002052 if (unlikely(s->sqe->buf_index))
2053 return -EINVAL;
Jens Axboe8358e3a2019-04-23 08:17:58 -06002054 ret = io_write(req, s, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07002055 break;
2056 case IORING_OP_READ_FIXED:
Jens Axboe8358e3a2019-04-23 08:17:58 -06002057 ret = io_read(req, s, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07002058 break;
2059 case IORING_OP_WRITE_FIXED:
Jens Axboe8358e3a2019-04-23 08:17:58 -06002060 ret = io_write(req, s, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002061 break;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002062 case IORING_OP_FSYNC:
2063 ret = io_fsync(req, s->sqe, force_nonblock);
2064 break;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002065 case IORING_OP_POLL_ADD:
2066 ret = io_poll_add(req, s->sqe);
2067 break;
2068 case IORING_OP_POLL_REMOVE:
2069 ret = io_poll_remove(req, s->sqe);
2070 break;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002071 case IORING_OP_SYNC_FILE_RANGE:
2072 ret = io_sync_file_range(req, s->sqe, force_nonblock);
2073 break;
Jens Axboe0fa03c62019-04-19 13:34:07 -06002074 case IORING_OP_SENDMSG:
2075 ret = io_sendmsg(req, s->sqe, force_nonblock);
2076 break;
Jens Axboeaa1fa282019-04-19 13:38:09 -06002077 case IORING_OP_RECVMSG:
2078 ret = io_recvmsg(req, s->sqe, force_nonblock);
2079 break;
Jens Axboe5262f562019-09-17 12:26:57 -06002080 case IORING_OP_TIMEOUT:
2081 ret = io_timeout(req, s->sqe);
2082 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002083 default:
2084 ret = -EINVAL;
2085 break;
2086 }
2087
Jens Axboedef596e2019-01-09 08:59:42 -07002088 if (ret)
2089 return ret;
2090
2091 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe9e645e112019-05-10 16:07:28 -06002092 if (req->result == -EAGAIN)
Jens Axboedef596e2019-01-09 08:59:42 -07002093 return -EAGAIN;
2094
2095 /* workqueue context doesn't hold uring_lock, grab it now */
2096 if (s->needs_lock)
2097 mutex_lock(&ctx->uring_lock);
2098 io_iopoll_req_issued(req);
2099 if (s->needs_lock)
2100 mutex_unlock(&ctx->uring_lock);
2101 }
2102
2103 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002104}
2105
Jens Axboe31b51512019-01-18 22:56:34 -07002106static struct async_list *io_async_list_from_sqe(struct io_ring_ctx *ctx,
2107 const struct io_uring_sqe *sqe)
2108{
2109 switch (sqe->opcode) {
2110 case IORING_OP_READV:
2111 case IORING_OP_READ_FIXED:
2112 return &ctx->pending_async[READ];
2113 case IORING_OP_WRITEV:
2114 case IORING_OP_WRITE_FIXED:
2115 return &ctx->pending_async[WRITE];
2116 default:
2117 return NULL;
2118 }
2119}
2120
Jens Axboeedafcce2019-01-09 09:16:05 -07002121static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
2122{
2123 u8 opcode = READ_ONCE(sqe->opcode);
2124
2125 return !(opcode == IORING_OP_READ_FIXED ||
2126 opcode == IORING_OP_WRITE_FIXED);
2127}
2128
Jens Axboe2b188cc2019-01-07 10:46:33 -07002129static void io_sq_wq_submit_work(struct work_struct *work)
2130{
2131 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002132 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe31b51512019-01-18 22:56:34 -07002133 struct mm_struct *cur_mm = NULL;
2134 struct async_list *async_list;
2135 LIST_HEAD(req_list);
Jens Axboeedafcce2019-01-09 09:16:05 -07002136 mm_segment_t old_fs;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002137 int ret;
2138
Jens Axboe31b51512019-01-18 22:56:34 -07002139 async_list = io_async_list_from_sqe(ctx, req->submit.sqe);
2140restart:
2141 do {
2142 struct sqe_submit *s = &req->submit;
2143 const struct io_uring_sqe *sqe = s->sqe;
Jackie Liud0ee8792019-07-31 14:39:33 +08002144 unsigned int flags = req->flags;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002145
Stefan Bühler8449eed2019-04-27 20:34:19 +02002146 /* Ensure we clear previously set non-block flag */
Jens Axboe31b51512019-01-18 22:56:34 -07002147 req->rw.ki_flags &= ~IOCB_NOWAIT;
2148
2149 ret = 0;
2150 if (io_sqe_needs_user(sqe) && !cur_mm) {
2151 if (!mmget_not_zero(ctx->sqo_mm)) {
2152 ret = -EFAULT;
2153 } else {
2154 cur_mm = ctx->sqo_mm;
2155 use_mm(cur_mm);
2156 old_fs = get_fs();
2157 set_fs(USER_DS);
2158 }
2159 }
2160
2161 if (!ret) {
2162 s->has_user = cur_mm != NULL;
2163 s->needs_lock = true;
2164 do {
Jens Axboe8358e3a2019-04-23 08:17:58 -06002165 ret = __io_submit_sqe(ctx, req, s, false);
Jens Axboe31b51512019-01-18 22:56:34 -07002166 /*
2167 * We can get EAGAIN for polled IO even though
2168 * we're forcing a sync submission from here,
2169 * since we can't wait for request slots on the
2170 * block side.
2171 */
2172 if (ret != -EAGAIN)
2173 break;
2174 cond_resched();
2175 } while (1);
2176 }
Jens Axboe817869d2019-04-30 14:44:05 -06002177
2178 /* drop submission reference */
2179 io_put_req(req);
2180
Jens Axboe31b51512019-01-18 22:56:34 -07002181 if (ret) {
Jens Axboec71ffb62019-05-13 20:58:29 -06002182 io_cqring_add_event(ctx, sqe->user_data, ret);
Jens Axboee65ef562019-03-12 10:16:44 -06002183 io_put_req(req);
Jens Axboe31b51512019-01-18 22:56:34 -07002184 }
2185
2186 /* async context always use a copy of the sqe */
2187 kfree(sqe);
2188
Zhengyuan Liuf7b76ac2019-07-16 23:26:14 +08002189 /* req from defer and link list needn't decrease async cnt */
Jackie Liud0ee8792019-07-31 14:39:33 +08002190 if (flags & (REQ_F_IO_DRAINED | REQ_F_LINK_DONE))
Zhengyuan Liuf7b76ac2019-07-16 23:26:14 +08002191 goto out;
2192
Jens Axboe31b51512019-01-18 22:56:34 -07002193 if (!async_list)
2194 break;
2195 if (!list_empty(&req_list)) {
2196 req = list_first_entry(&req_list, struct io_kiocb,
2197 list);
2198 list_del(&req->list);
2199 continue;
2200 }
2201 if (list_empty(&async_list->list))
2202 break;
2203
2204 req = NULL;
2205 spin_lock(&async_list->lock);
2206 if (list_empty(&async_list->list)) {
2207 spin_unlock(&async_list->lock);
2208 break;
2209 }
2210 list_splice_init(&async_list->list, &req_list);
2211 spin_unlock(&async_list->lock);
2212
2213 req = list_first_entry(&req_list, struct io_kiocb, list);
2214 list_del(&req->list);
2215 } while (req);
Jens Axboeedafcce2019-01-09 09:16:05 -07002216
2217 /*
Jens Axboe31b51512019-01-18 22:56:34 -07002218 * Rare case of racing with a submitter. If we find the count has
2219 * dropped to zero AND we have pending work items, then restart
2220 * the processing. This is a tiny race window.
Jens Axboeedafcce2019-01-09 09:16:05 -07002221 */
Jens Axboe31b51512019-01-18 22:56:34 -07002222 if (async_list) {
2223 ret = atomic_dec_return(&async_list->cnt);
2224 while (!ret && !list_empty(&async_list->list)) {
2225 spin_lock(&async_list->lock);
2226 atomic_inc(&async_list->cnt);
2227 list_splice_init(&async_list->list, &req_list);
2228 spin_unlock(&async_list->lock);
2229
2230 if (!list_empty(&req_list)) {
2231 req = list_first_entry(&req_list,
2232 struct io_kiocb, list);
2233 list_del(&req->list);
2234 goto restart;
2235 }
2236 ret = atomic_dec_return(&async_list->cnt);
Jens Axboeedafcce2019-01-09 09:16:05 -07002237 }
Jens Axboeedafcce2019-01-09 09:16:05 -07002238 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002239
Zhengyuan Liuf7b76ac2019-07-16 23:26:14 +08002240out:
Jens Axboe31b51512019-01-18 22:56:34 -07002241 if (cur_mm) {
Jens Axboeedafcce2019-01-09 09:16:05 -07002242 set_fs(old_fs);
Jens Axboe31b51512019-01-18 22:56:34 -07002243 unuse_mm(cur_mm);
2244 mmput(cur_mm);
Jens Axboeedafcce2019-01-09 09:16:05 -07002245 }
Jens Axboe31b51512019-01-18 22:56:34 -07002246}
Jens Axboe2b188cc2019-01-07 10:46:33 -07002247
Jens Axboe31b51512019-01-18 22:56:34 -07002248/*
2249 * See if we can piggy back onto previously submitted work, that is still
2250 * running. We currently only allow this if the new request is sequential
2251 * to the previous one we punted.
2252 */
2253static bool io_add_to_prev_work(struct async_list *list, struct io_kiocb *req)
2254{
Jens Axboe6d5d5ac2019-09-11 10:16:13 -06002255 bool ret;
Jens Axboe31b51512019-01-18 22:56:34 -07002256
2257 if (!list)
2258 return false;
2259 if (!(req->flags & REQ_F_SEQ_PREV))
2260 return false;
2261 if (!atomic_read(&list->cnt))
2262 return false;
2263
2264 ret = true;
2265 spin_lock(&list->lock);
2266 list_add_tail(&req->list, &list->list);
Zhengyuan Liuc0e48f92019-07-18 20:44:00 +08002267 /*
2268 * Ensure we see a simultaneous modification from io_sq_wq_submit_work()
2269 */
2270 smp_mb();
Jens Axboe31b51512019-01-18 22:56:34 -07002271 if (!atomic_read(&list->cnt)) {
2272 list_del_init(&req->list);
2273 ret = false;
2274 }
2275 spin_unlock(&list->lock);
2276 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002277}
2278
Jens Axboe09bb8392019-03-13 12:39:28 -06002279static bool io_op_needs_file(const struct io_uring_sqe *sqe)
2280{
2281 int op = READ_ONCE(sqe->opcode);
2282
2283 switch (op) {
2284 case IORING_OP_NOP:
2285 case IORING_OP_POLL_REMOVE:
2286 return false;
2287 default:
2288 return true;
2289 }
2290}
2291
2292static int io_req_set_file(struct io_ring_ctx *ctx, const struct sqe_submit *s,
2293 struct io_submit_state *state, struct io_kiocb *req)
2294{
2295 unsigned flags;
2296 int fd;
2297
2298 flags = READ_ONCE(s->sqe->flags);
2299 fd = READ_ONCE(s->sqe->fd);
2300
Jackie Liu4fe2c962019-09-09 20:50:40 +08002301 if (flags & IOSQE_IO_DRAIN)
Jens Axboede0617e2019-04-06 21:51:27 -06002302 req->flags |= REQ_F_IO_DRAIN;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002303 /*
2304 * All io need record the previous position, if LINK vs DARIN,
2305 * it can be used to mark the position of the first IO in the
2306 * link list.
2307 */
2308 req->sequence = s->sequence;
Jens Axboede0617e2019-04-06 21:51:27 -06002309
Jens Axboe60c112b2019-06-21 10:20:18 -06002310 if (!io_op_needs_file(s->sqe))
Jens Axboe09bb8392019-03-13 12:39:28 -06002311 return 0;
Jens Axboe09bb8392019-03-13 12:39:28 -06002312
2313 if (flags & IOSQE_FIXED_FILE) {
2314 if (unlikely(!ctx->user_files ||
2315 (unsigned) fd >= ctx->nr_user_files))
2316 return -EBADF;
2317 req->file = ctx->user_files[fd];
2318 req->flags |= REQ_F_FIXED_FILE;
2319 } else {
2320 if (s->needs_fixed_file)
2321 return -EBADF;
2322 req->file = io_file_get(state, fd);
2323 if (unlikely(!req->file))
2324 return -EBADF;
2325 }
2326
2327 return 0;
2328}
2329
Jackie Liu4fe2c962019-09-09 20:50:40 +08002330static int __io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Jens Axboebc808bc2019-10-22 13:14:37 -06002331 struct sqe_submit *s)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002332{
Jens Axboee0c5c572019-03-12 10:18:47 -06002333 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002334
Jens Axboebc808bc2019-10-22 13:14:37 -06002335 ret = __io_submit_sqe(ctx, req, s, true);
Jens Axboe491381ce2019-10-17 09:20:46 -06002336
2337 /*
2338 * We async punt it if the file wasn't marked NOWAIT, or if the file
2339 * doesn't support non-blocking read/write attempts
2340 */
2341 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
2342 (req->flags & REQ_F_MUST_PUNT))) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07002343 struct io_uring_sqe *sqe_copy;
2344
Jackie Liu954dab12019-09-18 10:37:52 +08002345 sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002346 if (sqe_copy) {
Jens Axboe31b51512019-01-18 22:56:34 -07002347 struct async_list *list;
2348
Jens Axboe2b188cc2019-01-07 10:46:33 -07002349 s->sqe = sqe_copy;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002350 memcpy(&req->submit, s, sizeof(*s));
Jens Axboe31b51512019-01-18 22:56:34 -07002351 list = io_async_list_from_sqe(ctx, s->sqe);
2352 if (!io_add_to_prev_work(list, req)) {
2353 if (list)
2354 atomic_inc(&list->cnt);
2355 INIT_WORK(&req->work, io_sq_wq_submit_work);
Jens Axboe18d9be12019-09-10 09:13:05 -06002356 io_queue_async_work(ctx, req);
Jens Axboe31b51512019-01-18 22:56:34 -07002357 }
Jens Axboee65ef562019-03-12 10:16:44 -06002358
2359 /*
2360 * Queued up for async execution, worker will release
Jens Axboe9e645e112019-05-10 16:07:28 -06002361 * submit reference when the iocb is actually submitted.
Jens Axboee65ef562019-03-12 10:16:44 -06002362 */
2363 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002364 }
2365 }
Jens Axboee65ef562019-03-12 10:16:44 -06002366
2367 /* drop submission reference */
2368 io_put_req(req);
2369
2370 /* and drop final reference, if we failed */
Jens Axboe9e645e112019-05-10 16:07:28 -06002371 if (ret) {
2372 io_cqring_add_event(ctx, req->user_data, ret);
2373 if (req->flags & REQ_F_LINK)
2374 req->flags |= REQ_F_FAIL_LINK;
Jens Axboee65ef562019-03-12 10:16:44 -06002375 io_put_req(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002376 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002377
2378 return ret;
2379}
2380
Jackie Liu4fe2c962019-09-09 20:50:40 +08002381static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Jens Axboebc808bc2019-10-22 13:14:37 -06002382 struct sqe_submit *s)
Jackie Liu4fe2c962019-09-09 20:50:40 +08002383{
2384 int ret;
2385
2386 ret = io_req_defer(ctx, req, s->sqe);
2387 if (ret) {
2388 if (ret != -EIOCBQUEUED) {
2389 io_free_req(req);
2390 io_cqring_add_event(ctx, s->sqe->user_data, ret);
2391 }
2392 return 0;
2393 }
2394
Jens Axboebc808bc2019-10-22 13:14:37 -06002395 return __io_queue_sqe(ctx, req, s);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002396}
2397
2398static int io_queue_link_head(struct io_ring_ctx *ctx, struct io_kiocb *req,
Jens Axboebc808bc2019-10-22 13:14:37 -06002399 struct sqe_submit *s, struct io_kiocb *shadow)
Jackie Liu4fe2c962019-09-09 20:50:40 +08002400{
2401 int ret;
2402 int need_submit = false;
2403
2404 if (!shadow)
Jens Axboebc808bc2019-10-22 13:14:37 -06002405 return io_queue_sqe(ctx, req, s);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002406
2407 /*
2408 * Mark the first IO in link list as DRAIN, let all the following
2409 * IOs enter the defer list. all IO needs to be completed before link
2410 * list.
2411 */
2412 req->flags |= REQ_F_IO_DRAIN;
2413 ret = io_req_defer(ctx, req, s->sqe);
2414 if (ret) {
2415 if (ret != -EIOCBQUEUED) {
2416 io_free_req(req);
Pavel Begunkov7b202382019-10-27 22:10:36 +03002417 __io_free_req(shadow);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002418 io_cqring_add_event(ctx, s->sqe->user_data, ret);
2419 return 0;
2420 }
2421 } else {
2422 /*
2423 * If ret == 0 means that all IOs in front of link io are
2424 * running done. let's queue link head.
2425 */
2426 need_submit = true;
2427 }
2428
2429 /* Insert shadow req to defer_list, blocking next IOs */
2430 spin_lock_irq(&ctx->completion_lock);
2431 list_add_tail(&shadow->list, &ctx->defer_list);
2432 spin_unlock_irq(&ctx->completion_lock);
2433
2434 if (need_submit)
Jens Axboebc808bc2019-10-22 13:14:37 -06002435 return __io_queue_sqe(ctx, req, s);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002436
2437 return 0;
2438}
2439
Jens Axboe9e645e112019-05-10 16:07:28 -06002440#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)
2441
2442static void io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
Jens Axboebc808bc2019-10-22 13:14:37 -06002443 struct io_submit_state *state, struct io_kiocb **link)
Jens Axboe9e645e112019-05-10 16:07:28 -06002444{
2445 struct io_uring_sqe *sqe_copy;
2446 struct io_kiocb *req;
2447 int ret;
2448
2449 /* enforce forwards compatibility on users */
2450 if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
2451 ret = -EINVAL;
2452 goto err;
2453 }
2454
2455 req = io_get_req(ctx, state);
2456 if (unlikely(!req)) {
2457 ret = -EAGAIN;
2458 goto err;
2459 }
2460
2461 ret = io_req_set_file(ctx, s, state, req);
2462 if (unlikely(ret)) {
2463err_req:
2464 io_free_req(req);
2465err:
2466 io_cqring_add_event(ctx, s->sqe->user_data, ret);
2467 return;
2468 }
2469
Pavel Begunkov84d55dc2019-10-25 12:31:29 +03002470 req->user_data = s->sqe->user_data;
2471
Jens Axboe9e645e112019-05-10 16:07:28 -06002472 /*
2473 * If we already have a head request, queue this one for async
2474 * submittal once the head completes. If we don't have a head but
2475 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
2476 * submitted sync once the chain is complete. If none of those
2477 * conditions are true (normal request), then just queue it.
2478 */
2479 if (*link) {
2480 struct io_kiocb *prev = *link;
2481
2482 sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
2483 if (!sqe_copy) {
2484 ret = -EAGAIN;
2485 goto err_req;
2486 }
2487
2488 s->sqe = sqe_copy;
2489 memcpy(&req->submit, s, sizeof(*s));
2490 list_add_tail(&req->list, &prev->link_list);
2491 } else if (s->sqe->flags & IOSQE_IO_LINK) {
2492 req->flags |= REQ_F_LINK;
2493
2494 memcpy(&req->submit, s, sizeof(*s));
2495 INIT_LIST_HEAD(&req->link_list);
2496 *link = req;
2497 } else {
Jens Axboebc808bc2019-10-22 13:14:37 -06002498 io_queue_sqe(ctx, req, s);
Jens Axboe9e645e112019-05-10 16:07:28 -06002499 }
2500}
2501
Jens Axboe9a56a232019-01-09 09:06:50 -07002502/*
2503 * Batched submission is done, ensure local IO is flushed out.
2504 */
2505static void io_submit_state_end(struct io_submit_state *state)
2506{
2507 blk_finish_plug(&state->plug);
Jens Axboe3d6770f2019-04-13 11:50:54 -06002508 io_file_put(state);
Jens Axboe2579f912019-01-09 09:10:43 -07002509 if (state->free_reqs)
2510 kmem_cache_free_bulk(req_cachep, state->free_reqs,
2511 &state->reqs[state->cur_req]);
Jens Axboe9a56a232019-01-09 09:06:50 -07002512}
2513
2514/*
2515 * Start submission side cache.
2516 */
2517static void io_submit_state_start(struct io_submit_state *state,
2518 struct io_ring_ctx *ctx, unsigned max_ios)
2519{
2520 blk_start_plug(&state->plug);
Jens Axboe2579f912019-01-09 09:10:43 -07002521 state->free_reqs = 0;
Jens Axboe9a56a232019-01-09 09:06:50 -07002522 state->file = NULL;
2523 state->ios_left = max_ios;
2524}
2525
Jens Axboe2b188cc2019-01-07 10:46:33 -07002526static void io_commit_sqring(struct io_ring_ctx *ctx)
2527{
Hristo Venev75b28af2019-08-26 17:23:46 +00002528 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002529
Hristo Venev75b28af2019-08-26 17:23:46 +00002530 if (ctx->cached_sq_head != READ_ONCE(rings->sq.head)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07002531 /*
2532 * Ensure any loads from the SQEs are done at this point,
2533 * since once we write the new head, the application could
2534 * write new data to them.
2535 */
Hristo Venev75b28af2019-08-26 17:23:46 +00002536 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002537 }
2538}
2539
2540/*
Jens Axboe2b188cc2019-01-07 10:46:33 -07002541 * Fetch an sqe, if one is available. Note that s->sqe will point to memory
2542 * that is mapped by userspace. This means that care needs to be taken to
2543 * ensure that reads are stable, as we cannot rely on userspace always
2544 * being a good citizen. If members of the sqe are validated and then later
2545 * used, it's important that those reads are done through READ_ONCE() to
2546 * prevent a re-load down the line.
2547 */
2548static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
2549{
Hristo Venev75b28af2019-08-26 17:23:46 +00002550 struct io_rings *rings = ctx->rings;
2551 u32 *sq_array = ctx->sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002552 unsigned head;
2553
2554 /*
2555 * The cached sq head (or cq tail) serves two purposes:
2556 *
2557 * 1) allows us to batch the cost of updating the user visible
2558 * head updates.
2559 * 2) allows the kernel side to track the head on its own, even
2560 * though the application is the one updating it.
2561 */
2562 head = ctx->cached_sq_head;
Stefan Bühlere523a292019-04-19 11:57:44 +02002563 /* make sure SQ entry isn't read before tail */
Hristo Venev75b28af2019-08-26 17:23:46 +00002564 if (head == smp_load_acquire(&rings->sq.tail))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002565 return false;
2566
Hristo Venev75b28af2019-08-26 17:23:46 +00002567 head = READ_ONCE(sq_array[head & ctx->sq_mask]);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002568 if (head < ctx->sq_entries) {
2569 s->index = head;
2570 s->sqe = &ctx->sq_sqes[head];
Jackie Liu8776f3f2019-09-09 20:50:39 +08002571 s->sequence = ctx->cached_sq_head;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002572 ctx->cached_sq_head++;
2573 return true;
2574 }
2575
2576 /* drop invalid entries */
2577 ctx->cached_sq_head++;
Jens Axboe498ccd92019-10-25 10:04:25 -06002578 ctx->cached_sq_dropped++;
2579 WRITE_ONCE(rings->sq_dropped, ctx->cached_sq_dropped);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002580 return false;
2581}
2582
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002583static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
2584 bool has_user, bool mm_fault)
Jens Axboe6c271ce2019-01-10 11:22:30 -07002585{
2586 struct io_submit_state state, *statep = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002587 struct io_kiocb *link = NULL;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002588 struct io_kiocb *shadow_req = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002589 bool prev_was_link = false;
2590 int i, submitted = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002591
2592 if (nr > IO_PLUG_THRESHOLD) {
2593 io_submit_state_start(&state, ctx, nr);
2594 statep = &state;
2595 }
2596
2597 for (i = 0; i < nr; i++) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002598 struct sqe_submit s;
2599
2600 if (!io_get_sqring(ctx, &s))
2601 break;
2602
Jens Axboe9e645e112019-05-10 16:07:28 -06002603 /*
2604 * If previous wasn't linked and we have a linked command,
2605 * that's the end of the chain. Submit the previous link.
2606 */
2607 if (!prev_was_link && link) {
Jens Axboebc808bc2019-10-22 13:14:37 -06002608 io_queue_link_head(ctx, link, &link->submit, shadow_req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002609 link = NULL;
Jackie Liu5f5ad9c2019-09-18 10:37:53 +08002610 shadow_req = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002611 }
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002612 prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
Jens Axboe9e645e112019-05-10 16:07:28 -06002613
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002614 if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08002615 if (!shadow_req) {
2616 shadow_req = io_get_req(ctx, NULL);
Jackie Liua1041c22019-09-18 17:25:52 +08002617 if (unlikely(!shadow_req))
2618 goto out;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002619 shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
2620 refcount_dec(&shadow_req->refs);
2621 }
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002622 shadow_req->sequence = s.sequence;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002623 }
2624
Jackie Liua1041c22019-09-18 17:25:52 +08002625out:
Jens Axboe6c271ce2019-01-10 11:22:30 -07002626 if (unlikely(mm_fault)) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002627 io_cqring_add_event(ctx, s.sqe->user_data,
Jens Axboe9e645e112019-05-10 16:07:28 -06002628 -EFAULT);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002629 } else {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002630 s.has_user = has_user;
2631 s.needs_lock = true;
2632 s.needs_fixed_file = true;
2633 io_submit_sqe(ctx, &s, statep, &link);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002634 submitted++;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002635 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07002636 }
2637
Jens Axboe9e645e112019-05-10 16:07:28 -06002638 if (link)
Jens Axboebc808bc2019-10-22 13:14:37 -06002639 io_queue_link_head(ctx, link, &link->submit, shadow_req);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002640 if (statep)
2641 io_submit_state_end(&state);
2642
2643 return submitted;
2644}
2645
2646static int io_sq_thread(void *data)
2647{
Jens Axboe6c271ce2019-01-10 11:22:30 -07002648 struct io_ring_ctx *ctx = data;
2649 struct mm_struct *cur_mm = NULL;
2650 mm_segment_t old_fs;
2651 DEFINE_WAIT(wait);
2652 unsigned inflight;
2653 unsigned long timeout;
2654
Jackie Liua4c0b3d2019-07-08 13:41:12 +08002655 complete(&ctx->sqo_thread_started);
2656
Jens Axboe6c271ce2019-01-10 11:22:30 -07002657 old_fs = get_fs();
2658 set_fs(USER_DS);
2659
2660 timeout = inflight = 0;
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02002661 while (!kthread_should_park()) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002662 bool mm_fault = false;
2663 unsigned int to_submit;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002664
2665 if (inflight) {
2666 unsigned nr_events = 0;
2667
2668 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe2b2ed972019-10-25 10:06:15 -06002669 /*
2670 * inflight is the count of the maximum possible
2671 * entries we submitted, but it can be smaller
2672 * if we dropped some of them. If we don't have
2673 * poll entries available, then we know that we
2674 * have nothing left to poll for. Reset the
2675 * inflight count to zero in that case.
2676 */
2677 mutex_lock(&ctx->uring_lock);
2678 if (!list_empty(&ctx->poll_list))
2679 __io_iopoll_check(ctx, &nr_events, 0);
2680 else
2681 inflight = 0;
2682 mutex_unlock(&ctx->uring_lock);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002683 } else {
2684 /*
2685 * Normal IO, just pretend everything completed.
2686 * We don't have to poll completions for that.
2687 */
2688 nr_events = inflight;
2689 }
2690
2691 inflight -= nr_events;
2692 if (!inflight)
2693 timeout = jiffies + ctx->sq_thread_idle;
2694 }
2695
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002696 to_submit = io_sqring_entries(ctx);
2697 if (!to_submit) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07002698 /*
2699 * We're polling. If we're within the defined idle
2700 * period, then let us spin without work before going
2701 * to sleep.
2702 */
2703 if (inflight || !time_after(jiffies, timeout)) {
Jens Axboe9831a902019-09-19 09:48:55 -06002704 cond_resched();
Jens Axboe6c271ce2019-01-10 11:22:30 -07002705 continue;
2706 }
2707
2708 /*
2709 * Drop cur_mm before scheduling, we can't hold it for
2710 * long periods (or over schedule()). Do this before
2711 * adding ourselves to the waitqueue, as the unuse/drop
2712 * may sleep.
2713 */
2714 if (cur_mm) {
2715 unuse_mm(cur_mm);
2716 mmput(cur_mm);
2717 cur_mm = NULL;
2718 }
2719
2720 prepare_to_wait(&ctx->sqo_wait, &wait,
2721 TASK_INTERRUPTIBLE);
2722
2723 /* Tell userspace we may need a wakeup call */
Hristo Venev75b28af2019-08-26 17:23:46 +00002724 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
Stefan Bühler0d7bae62019-04-19 11:57:45 +02002725 /* make sure to read SQ tail after writing flags */
2726 smp_mb();
Jens Axboe6c271ce2019-01-10 11:22:30 -07002727
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002728 to_submit = io_sqring_entries(ctx);
2729 if (!to_submit) {
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02002730 if (kthread_should_park()) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07002731 finish_wait(&ctx->sqo_wait, &wait);
2732 break;
2733 }
2734 if (signal_pending(current))
2735 flush_signals(current);
2736 schedule();
2737 finish_wait(&ctx->sqo_wait, &wait);
2738
Hristo Venev75b28af2019-08-26 17:23:46 +00002739 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002740 continue;
2741 }
2742 finish_wait(&ctx->sqo_wait, &wait);
2743
Hristo Venev75b28af2019-08-26 17:23:46 +00002744 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002745 }
2746
Jens Axboe6c271ce2019-01-10 11:22:30 -07002747 /* Unless all new commands are FIXED regions, grab mm */
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002748 if (!cur_mm) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07002749 mm_fault = !mmget_not_zero(ctx->sqo_mm);
2750 if (!mm_fault) {
2751 use_mm(ctx->sqo_mm);
2752 cur_mm = ctx->sqo_mm;
2753 }
2754 }
2755
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002756 to_submit = min(to_submit, ctx->sq_entries);
2757 inflight += io_submit_sqes(ctx, to_submit, cur_mm != NULL,
2758 mm_fault);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002759
2760 /* Commit SQ ring head once we've consumed all SQEs */
2761 io_commit_sqring(ctx);
2762 }
2763
2764 set_fs(old_fs);
2765 if (cur_mm) {
2766 unuse_mm(cur_mm);
2767 mmput(cur_mm);
2768 }
Jens Axboe06058632019-04-13 09:26:03 -06002769
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02002770 kthread_parkme();
Jens Axboe06058632019-04-13 09:26:03 -06002771
Jens Axboe6c271ce2019-01-10 11:22:30 -07002772 return 0;
2773}
2774
Jens Axboebc808bc2019-10-22 13:14:37 -06002775static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002776{
Jens Axboe9a56a232019-01-09 09:06:50 -07002777 struct io_submit_state state, *statep = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002778 struct io_kiocb *link = NULL;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002779 struct io_kiocb *shadow_req = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002780 bool prev_was_link = false;
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002781 int i, submit = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002782
Jens Axboe9a56a232019-01-09 09:06:50 -07002783 if (to_submit > IO_PLUG_THRESHOLD) {
2784 io_submit_state_start(&state, ctx, to_submit);
2785 statep = &state;
2786 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002787
2788 for (i = 0; i < to_submit; i++) {
2789 struct sqe_submit s;
2790
2791 if (!io_get_sqring(ctx, &s))
2792 break;
2793
Jens Axboe9e645e112019-05-10 16:07:28 -06002794 /*
2795 * If previous wasn't linked and we have a linked command,
2796 * that's the end of the chain. Submit the previous link.
2797 */
2798 if (!prev_was_link && link) {
Jens Axboebc808bc2019-10-22 13:14:37 -06002799 io_queue_link_head(ctx, link, &link->submit, shadow_req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002800 link = NULL;
Jackie Liu5f5ad9c2019-09-18 10:37:53 +08002801 shadow_req = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002802 }
2803 prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
2804
Jackie Liu4fe2c962019-09-09 20:50:40 +08002805 if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
2806 if (!shadow_req) {
2807 shadow_req = io_get_req(ctx, NULL);
Jackie Liua1041c22019-09-18 17:25:52 +08002808 if (unlikely(!shadow_req))
2809 goto out;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002810 shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
2811 refcount_dec(&shadow_req->refs);
2812 }
2813 shadow_req->sequence = s.sequence;
2814 }
2815
Jackie Liua1041c22019-09-18 17:25:52 +08002816out:
Jens Axboe2b188cc2019-01-07 10:46:33 -07002817 s.has_user = true;
Jens Axboedef596e2019-01-09 08:59:42 -07002818 s.needs_lock = false;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002819 s.needs_fixed_file = false;
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002820 submit++;
Jens Axboebc808bc2019-10-22 13:14:37 -06002821 io_submit_sqe(ctx, &s, statep, &link);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002822 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002823
Jens Axboe9e645e112019-05-10 16:07:28 -06002824 if (link)
Jens Axboebc808bc2019-10-22 13:14:37 -06002825 io_queue_link_head(ctx, link, &link->submit, shadow_req);
Jens Axboe9a56a232019-01-09 09:06:50 -07002826 if (statep)
2827 io_submit_state_end(statep);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002828
Pavel Begunkov935d1e42019-10-25 12:31:31 +03002829 io_commit_sqring(ctx);
2830
Jens Axboe5c8b0b52019-04-30 10:16:07 -06002831 return submit;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002832}
2833
Jens Axboebda52162019-09-24 13:47:15 -06002834struct io_wait_queue {
2835 struct wait_queue_entry wq;
2836 struct io_ring_ctx *ctx;
2837 unsigned to_wait;
2838 unsigned nr_timeouts;
2839};
2840
2841static inline bool io_should_wake(struct io_wait_queue *iowq)
2842{
2843 struct io_ring_ctx *ctx = iowq->ctx;
2844
2845 /*
2846 * Wake up if we have enough events, or if a timeout occured since we
2847 * started waiting. For timeouts, we always want to return to userspace,
2848 * regardless of event count.
2849 */
2850 return io_cqring_events(ctx->rings) >= iowq->to_wait ||
2851 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
2852}
2853
2854static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
2855 int wake_flags, void *key)
2856{
2857 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
2858 wq);
2859
2860 if (!io_should_wake(iowq))
2861 return -1;
2862
2863 return autoremove_wake_function(curr, mode, wake_flags, key);
2864}
2865
Jens Axboe2b188cc2019-01-07 10:46:33 -07002866/*
2867 * Wait until events become available, if we don't already have some. The
2868 * application must reap them itself, as they reside on the shared cq ring.
2869 */
2870static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
2871 const sigset_t __user *sig, size_t sigsz)
2872{
Jens Axboebda52162019-09-24 13:47:15 -06002873 struct io_wait_queue iowq = {
2874 .wq = {
2875 .private = current,
2876 .func = io_wake_function,
2877 .entry = LIST_HEAD_INIT(iowq.wq.entry),
2878 },
2879 .ctx = ctx,
2880 .to_wait = min_events,
2881 };
Hristo Venev75b28af2019-08-26 17:23:46 +00002882 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002883 int ret;
2884
Hristo Venev75b28af2019-08-26 17:23:46 +00002885 if (io_cqring_events(rings) >= min_events)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002886 return 0;
2887
2888 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01002889#ifdef CONFIG_COMPAT
2890 if (in_compat_syscall())
2891 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07002892 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01002893 else
2894#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07002895 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01002896
Jens Axboe2b188cc2019-01-07 10:46:33 -07002897 if (ret)
2898 return ret;
2899 }
2900
Jens Axboebda52162019-09-24 13:47:15 -06002901 ret = 0;
2902 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
2903 do {
2904 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
2905 TASK_INTERRUPTIBLE);
2906 if (io_should_wake(&iowq))
2907 break;
2908 schedule();
2909 if (signal_pending(current)) {
2910 ret = -ERESTARTSYS;
2911 break;
2912 }
2913 } while (1);
2914 finish_wait(&ctx->wait, &iowq.wq);
2915
Oleg Nesterovb7724342019-07-16 16:29:53 -07002916 restore_saved_sigmask_unless(ret == -ERESTARTSYS);
Oleg Nesterov97abc882019-06-28 12:06:50 -07002917 if (ret == -ERESTARTSYS)
2918 ret = -EINTR;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002919
Hristo Venev75b28af2019-08-26 17:23:46 +00002920 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002921}
2922
Jens Axboe6b063142019-01-10 22:13:58 -07002923static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
2924{
2925#if defined(CONFIG_UNIX)
2926 if (ctx->ring_sock) {
2927 struct sock *sock = ctx->ring_sock->sk;
2928 struct sk_buff *skb;
2929
2930 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
2931 kfree_skb(skb);
2932 }
2933#else
2934 int i;
2935
2936 for (i = 0; i < ctx->nr_user_files; i++)
2937 fput(ctx->user_files[i]);
2938#endif
2939}
2940
2941static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
2942{
2943 if (!ctx->user_files)
2944 return -ENXIO;
2945
2946 __io_sqe_files_unregister(ctx);
2947 kfree(ctx->user_files);
2948 ctx->user_files = NULL;
2949 ctx->nr_user_files = 0;
2950 return 0;
2951}
2952
Jens Axboe6c271ce2019-01-10 11:22:30 -07002953static void io_sq_thread_stop(struct io_ring_ctx *ctx)
2954{
2955 if (ctx->sqo_thread) {
Jackie Liua4c0b3d2019-07-08 13:41:12 +08002956 wait_for_completion(&ctx->sqo_thread_started);
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02002957 /*
2958 * The park is a bit of a work-around, without it we get
2959 * warning spews on shutdown with SQPOLL set and affinity
2960 * set to a single CPU.
2961 */
Jens Axboe06058632019-04-13 09:26:03 -06002962 kthread_park(ctx->sqo_thread);
Jens Axboe6c271ce2019-01-10 11:22:30 -07002963 kthread_stop(ctx->sqo_thread);
2964 ctx->sqo_thread = NULL;
2965 }
2966}
2967
Jens Axboe6b063142019-01-10 22:13:58 -07002968static void io_finish_async(struct io_ring_ctx *ctx)
2969{
Jens Axboe54a91f32019-09-10 09:15:04 -06002970 int i;
2971
Jens Axboe6c271ce2019-01-10 11:22:30 -07002972 io_sq_thread_stop(ctx);
2973
Jens Axboe54a91f32019-09-10 09:15:04 -06002974 for (i = 0; i < ARRAY_SIZE(ctx->sqo_wq); i++) {
2975 if (ctx->sqo_wq[i]) {
2976 destroy_workqueue(ctx->sqo_wq[i]);
2977 ctx->sqo_wq[i] = NULL;
2978 }
Jens Axboe6b063142019-01-10 22:13:58 -07002979 }
2980}
2981
2982#if defined(CONFIG_UNIX)
2983static void io_destruct_skb(struct sk_buff *skb)
2984{
2985 struct io_ring_ctx *ctx = skb->sk->sk_user_data;
Jens Axboe8a997342019-10-09 14:40:13 -06002986 int i;
Jens Axboe6b063142019-01-10 22:13:58 -07002987
Jens Axboe8a997342019-10-09 14:40:13 -06002988 for (i = 0; i < ARRAY_SIZE(ctx->sqo_wq); i++)
2989 if (ctx->sqo_wq[i])
2990 flush_workqueue(ctx->sqo_wq[i]);
2991
Jens Axboe6b063142019-01-10 22:13:58 -07002992 unix_destruct_scm(skb);
2993}
2994
2995/*
2996 * Ensure the UNIX gc is aware of our file set, so we are certain that
2997 * the io_uring can be safely unregistered on process exit, even if we have
2998 * loops in the file referencing.
2999 */
3000static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
3001{
3002 struct sock *sk = ctx->ring_sock->sk;
3003 struct scm_fp_list *fpl;
3004 struct sk_buff *skb;
3005 int i;
3006
3007 if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
3008 unsigned long inflight = ctx->user->unix_inflight + nr;
3009
3010 if (inflight > task_rlimit(current, RLIMIT_NOFILE))
3011 return -EMFILE;
3012 }
3013
3014 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
3015 if (!fpl)
3016 return -ENOMEM;
3017
3018 skb = alloc_skb(0, GFP_KERNEL);
3019 if (!skb) {
3020 kfree(fpl);
3021 return -ENOMEM;
3022 }
3023
3024 skb->sk = sk;
3025 skb->destructor = io_destruct_skb;
3026
3027 fpl->user = get_uid(ctx->user);
3028 for (i = 0; i < nr; i++) {
3029 fpl->fp[i] = get_file(ctx->user_files[i + offset]);
3030 unix_inflight(fpl->user, fpl->fp[i]);
3031 }
3032
3033 fpl->max = fpl->count = nr;
3034 UNIXCB(skb).fp = fpl;
3035 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
3036 skb_queue_head(&sk->sk_receive_queue, skb);
3037
3038 for (i = 0; i < nr; i++)
3039 fput(fpl->fp[i]);
3040
3041 return 0;
3042}
3043
3044/*
3045 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
3046 * causes regular reference counting to break down. We rely on the UNIX
3047 * garbage collection to take care of this problem for us.
3048 */
3049static int io_sqe_files_scm(struct io_ring_ctx *ctx)
3050{
3051 unsigned left, total;
3052 int ret = 0;
3053
3054 total = 0;
3055 left = ctx->nr_user_files;
3056 while (left) {
3057 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07003058
3059 ret = __io_sqe_files_scm(ctx, this_files, total);
3060 if (ret)
3061 break;
3062 left -= this_files;
3063 total += this_files;
3064 }
3065
3066 if (!ret)
3067 return 0;
3068
3069 while (total < ctx->nr_user_files) {
3070 fput(ctx->user_files[total]);
3071 total++;
3072 }
3073
3074 return ret;
3075}
3076#else
3077static int io_sqe_files_scm(struct io_ring_ctx *ctx)
3078{
3079 return 0;
3080}
3081#endif
3082
3083static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
3084 unsigned nr_args)
3085{
3086 __s32 __user *fds = (__s32 __user *) arg;
3087 int fd, ret = 0;
3088 unsigned i;
3089
3090 if (ctx->user_files)
3091 return -EBUSY;
3092 if (!nr_args)
3093 return -EINVAL;
3094 if (nr_args > IORING_MAX_FIXED_FILES)
3095 return -EMFILE;
3096
3097 ctx->user_files = kcalloc(nr_args, sizeof(struct file *), GFP_KERNEL);
3098 if (!ctx->user_files)
3099 return -ENOMEM;
3100
3101 for (i = 0; i < nr_args; i++) {
3102 ret = -EFAULT;
3103 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
3104 break;
3105
3106 ctx->user_files[i] = fget(fd);
3107
3108 ret = -EBADF;
3109 if (!ctx->user_files[i])
3110 break;
3111 /*
3112 * Don't allow io_uring instances to be registered. If UNIX
3113 * isn't enabled, then this causes a reference cycle and this
3114 * instance can never get freed. If UNIX is enabled we'll
3115 * handle it just fine, but there's still no point in allowing
3116 * a ring fd as it doesn't support regular read/write anyway.
3117 */
3118 if (ctx->user_files[i]->f_op == &io_uring_fops) {
3119 fput(ctx->user_files[i]);
3120 break;
3121 }
3122 ctx->nr_user_files++;
3123 ret = 0;
3124 }
3125
3126 if (ret) {
3127 for (i = 0; i < ctx->nr_user_files; i++)
3128 fput(ctx->user_files[i]);
3129
3130 kfree(ctx->user_files);
Jens Axboe25adf502019-04-03 09:52:40 -06003131 ctx->user_files = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07003132 ctx->nr_user_files = 0;
3133 return ret;
3134 }
3135
3136 ret = io_sqe_files_scm(ctx);
3137 if (ret)
3138 io_sqe_files_unregister(ctx);
3139
3140 return ret;
3141}
3142
Jens Axboe6c271ce2019-01-10 11:22:30 -07003143static int io_sq_offload_start(struct io_ring_ctx *ctx,
3144 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003145{
3146 int ret;
3147
Jens Axboe6c271ce2019-01-10 11:22:30 -07003148 init_waitqueue_head(&ctx->sqo_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003149 mmgrab(current->mm);
3150 ctx->sqo_mm = current->mm;
3151
Jens Axboe6c271ce2019-01-10 11:22:30 -07003152 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe3ec482d2019-04-08 10:51:01 -06003153 ret = -EPERM;
3154 if (!capable(CAP_SYS_ADMIN))
3155 goto err;
3156
Jens Axboe917257d2019-04-13 09:28:55 -06003157 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
3158 if (!ctx->sq_thread_idle)
3159 ctx->sq_thread_idle = HZ;
3160
Jens Axboe6c271ce2019-01-10 11:22:30 -07003161 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06003162 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003163
Jens Axboe917257d2019-04-13 09:28:55 -06003164 ret = -EINVAL;
Jens Axboe44a9bd12019-05-14 20:00:30 -06003165 if (cpu >= nr_cpu_ids)
3166 goto err;
Shenghui Wang7889f442019-05-07 16:03:19 +08003167 if (!cpu_online(cpu))
Jens Axboe917257d2019-04-13 09:28:55 -06003168 goto err;
3169
Jens Axboe6c271ce2019-01-10 11:22:30 -07003170 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
3171 ctx, cpu,
3172 "io_uring-sq");
3173 } else {
3174 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
3175 "io_uring-sq");
3176 }
3177 if (IS_ERR(ctx->sqo_thread)) {
3178 ret = PTR_ERR(ctx->sqo_thread);
3179 ctx->sqo_thread = NULL;
3180 goto err;
3181 }
3182 wake_up_process(ctx->sqo_thread);
3183 } else if (p->flags & IORING_SETUP_SQ_AFF) {
3184 /* Can't have SQ_AFF without SQPOLL */
3185 ret = -EINVAL;
3186 goto err;
3187 }
3188
Jens Axboe2b188cc2019-01-07 10:46:33 -07003189 /* Do QD, or 2 * CPUS, whatever is smallest */
Jens Axboe54a91f32019-09-10 09:15:04 -06003190 ctx->sqo_wq[0] = alloc_workqueue("io_ring-wq",
3191 WQ_UNBOUND | WQ_FREEZABLE,
Jens Axboe2b188cc2019-01-07 10:46:33 -07003192 min(ctx->sq_entries - 1, 2 * num_online_cpus()));
Jens Axboe54a91f32019-09-10 09:15:04 -06003193 if (!ctx->sqo_wq[0]) {
3194 ret = -ENOMEM;
3195 goto err;
3196 }
3197
3198 /*
3199 * This is for buffered writes, where we want to limit the parallelism
3200 * due to file locking in file systems. As "normal" buffered writes
3201 * should parellelize on writeout quite nicely, limit us to having 2
3202 * pending. This avoids massive contention on the inode when doing
3203 * buffered async writes.
3204 */
3205 ctx->sqo_wq[1] = alloc_workqueue("io_ring-write-wq",
3206 WQ_UNBOUND | WQ_FREEZABLE, 2);
3207 if (!ctx->sqo_wq[1]) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07003208 ret = -ENOMEM;
3209 goto err;
3210 }
3211
3212 return 0;
3213err:
Jens Axboe54a91f32019-09-10 09:15:04 -06003214 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003215 mmdrop(ctx->sqo_mm);
3216 ctx->sqo_mm = NULL;
3217 return ret;
3218}
3219
3220static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
3221{
3222 atomic_long_sub(nr_pages, &user->locked_vm);
3223}
3224
3225static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
3226{
3227 unsigned long page_limit, cur_pages, new_pages;
3228
3229 /* Don't allow more pages than we can safely lock */
3230 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
3231
3232 do {
3233 cur_pages = atomic_long_read(&user->locked_vm);
3234 new_pages = cur_pages + nr_pages;
3235 if (new_pages > page_limit)
3236 return -ENOMEM;
3237 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
3238 new_pages) != cur_pages);
3239
3240 return 0;
3241}
3242
3243static void io_mem_free(void *ptr)
3244{
Mark Rutland52e04ef2019-04-30 17:30:21 +01003245 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003246
Mark Rutland52e04ef2019-04-30 17:30:21 +01003247 if (!ptr)
3248 return;
3249
3250 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003251 if (put_page_testzero(page))
3252 free_compound_page(page);
3253}
3254
3255static void *io_mem_alloc(size_t size)
3256{
3257 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
3258 __GFP_NORETRY;
3259
3260 return (void *) __get_free_pages(gfp_flags, get_order(size));
3261}
3262
Hristo Venev75b28af2019-08-26 17:23:46 +00003263static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
3264 size_t *sq_offset)
3265{
3266 struct io_rings *rings;
3267 size_t off, sq_array_size;
3268
3269 off = struct_size(rings, cqes, cq_entries);
3270 if (off == SIZE_MAX)
3271 return SIZE_MAX;
3272
3273#ifdef CONFIG_SMP
3274 off = ALIGN(off, SMP_CACHE_BYTES);
3275 if (off == 0)
3276 return SIZE_MAX;
3277#endif
3278
3279 sq_array_size = array_size(sizeof(u32), sq_entries);
3280 if (sq_array_size == SIZE_MAX)
3281 return SIZE_MAX;
3282
3283 if (check_add_overflow(off, sq_array_size, &off))
3284 return SIZE_MAX;
3285
3286 if (sq_offset)
3287 *sq_offset = off;
3288
3289 return off;
3290}
3291
Jens Axboe2b188cc2019-01-07 10:46:33 -07003292static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
3293{
Hristo Venev75b28af2019-08-26 17:23:46 +00003294 size_t pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003295
Hristo Venev75b28af2019-08-26 17:23:46 +00003296 pages = (size_t)1 << get_order(
3297 rings_size(sq_entries, cq_entries, NULL));
3298 pages += (size_t)1 << get_order(
3299 array_size(sizeof(struct io_uring_sqe), sq_entries));
Jens Axboe2b188cc2019-01-07 10:46:33 -07003300
Hristo Venev75b28af2019-08-26 17:23:46 +00003301 return pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003302}
3303
Jens Axboeedafcce2019-01-09 09:16:05 -07003304static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
3305{
3306 int i, j;
3307
3308 if (!ctx->user_bufs)
3309 return -ENXIO;
3310
3311 for (i = 0; i < ctx->nr_user_bufs; i++) {
3312 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
3313
3314 for (j = 0; j < imu->nr_bvecs; j++)
John Hubbard27c4d3a2019-08-04 19:32:06 -07003315 put_user_page(imu->bvec[j].bv_page);
Jens Axboeedafcce2019-01-09 09:16:05 -07003316
3317 if (ctx->account_mem)
3318 io_unaccount_mem(ctx->user, imu->nr_bvecs);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003319 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07003320 imu->nr_bvecs = 0;
3321 }
3322
3323 kfree(ctx->user_bufs);
3324 ctx->user_bufs = NULL;
3325 ctx->nr_user_bufs = 0;
3326 return 0;
3327}
3328
3329static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
3330 void __user *arg, unsigned index)
3331{
3332 struct iovec __user *src;
3333
3334#ifdef CONFIG_COMPAT
3335 if (ctx->compat) {
3336 struct compat_iovec __user *ciovs;
3337 struct compat_iovec ciov;
3338
3339 ciovs = (struct compat_iovec __user *) arg;
3340 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
3341 return -EFAULT;
3342
3343 dst->iov_base = (void __user *) (unsigned long) ciov.iov_base;
3344 dst->iov_len = ciov.iov_len;
3345 return 0;
3346 }
3347#endif
3348 src = (struct iovec __user *) arg;
3349 if (copy_from_user(dst, &src[index], sizeof(*dst)))
3350 return -EFAULT;
3351 return 0;
3352}
3353
3354static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
3355 unsigned nr_args)
3356{
3357 struct vm_area_struct **vmas = NULL;
3358 struct page **pages = NULL;
3359 int i, j, got_pages = 0;
3360 int ret = -EINVAL;
3361
3362 if (ctx->user_bufs)
3363 return -EBUSY;
3364 if (!nr_args || nr_args > UIO_MAXIOV)
3365 return -EINVAL;
3366
3367 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
3368 GFP_KERNEL);
3369 if (!ctx->user_bufs)
3370 return -ENOMEM;
3371
3372 for (i = 0; i < nr_args; i++) {
3373 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
3374 unsigned long off, start, end, ubuf;
3375 int pret, nr_pages;
3376 struct iovec iov;
3377 size_t size;
3378
3379 ret = io_copy_iov(ctx, &iov, arg, i);
3380 if (ret)
Pavel Begunkova2786822019-05-26 12:35:47 +03003381 goto err;
Jens Axboeedafcce2019-01-09 09:16:05 -07003382
3383 /*
3384 * Don't impose further limits on the size and buffer
3385 * constraints here, we'll -EINVAL later when IO is
3386 * submitted if they are wrong.
3387 */
3388 ret = -EFAULT;
3389 if (!iov.iov_base || !iov.iov_len)
3390 goto err;
3391
3392 /* arbitrary limit, but we need something */
3393 if (iov.iov_len > SZ_1G)
3394 goto err;
3395
3396 ubuf = (unsigned long) iov.iov_base;
3397 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
3398 start = ubuf >> PAGE_SHIFT;
3399 nr_pages = end - start;
3400
3401 if (ctx->account_mem) {
3402 ret = io_account_mem(ctx->user, nr_pages);
3403 if (ret)
3404 goto err;
3405 }
3406
3407 ret = 0;
3408 if (!pages || nr_pages > got_pages) {
3409 kfree(vmas);
3410 kfree(pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003411 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
Jens Axboeedafcce2019-01-09 09:16:05 -07003412 GFP_KERNEL);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003413 vmas = kvmalloc_array(nr_pages,
Jens Axboeedafcce2019-01-09 09:16:05 -07003414 sizeof(struct vm_area_struct *),
3415 GFP_KERNEL);
3416 if (!pages || !vmas) {
3417 ret = -ENOMEM;
3418 if (ctx->account_mem)
3419 io_unaccount_mem(ctx->user, nr_pages);
3420 goto err;
3421 }
3422 got_pages = nr_pages;
3423 }
3424
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003425 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
Jens Axboeedafcce2019-01-09 09:16:05 -07003426 GFP_KERNEL);
3427 ret = -ENOMEM;
3428 if (!imu->bvec) {
3429 if (ctx->account_mem)
3430 io_unaccount_mem(ctx->user, nr_pages);
3431 goto err;
3432 }
3433
3434 ret = 0;
3435 down_read(&current->mm->mmap_sem);
Ira Weiny932f4a62019-05-13 17:17:03 -07003436 pret = get_user_pages(ubuf, nr_pages,
3437 FOLL_WRITE | FOLL_LONGTERM,
3438 pages, vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07003439 if (pret == nr_pages) {
3440 /* don't support file backed memory */
3441 for (j = 0; j < nr_pages; j++) {
3442 struct vm_area_struct *vma = vmas[j];
3443
3444 if (vma->vm_file &&
3445 !is_file_hugepages(vma->vm_file)) {
3446 ret = -EOPNOTSUPP;
3447 break;
3448 }
3449 }
3450 } else {
3451 ret = pret < 0 ? pret : -EFAULT;
3452 }
3453 up_read(&current->mm->mmap_sem);
3454 if (ret) {
3455 /*
3456 * if we did partial map, or found file backed vmas,
3457 * release any pages we did get
3458 */
John Hubbard27c4d3a2019-08-04 19:32:06 -07003459 if (pret > 0)
3460 put_user_pages(pages, pret);
Jens Axboeedafcce2019-01-09 09:16:05 -07003461 if (ctx->account_mem)
3462 io_unaccount_mem(ctx->user, nr_pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003463 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07003464 goto err;
3465 }
3466
3467 off = ubuf & ~PAGE_MASK;
3468 size = iov.iov_len;
3469 for (j = 0; j < nr_pages; j++) {
3470 size_t vec_len;
3471
3472 vec_len = min_t(size_t, size, PAGE_SIZE - off);
3473 imu->bvec[j].bv_page = pages[j];
3474 imu->bvec[j].bv_len = vec_len;
3475 imu->bvec[j].bv_offset = off;
3476 off = 0;
3477 size -= vec_len;
3478 }
3479 /* store original address for later verification */
3480 imu->ubuf = ubuf;
3481 imu->len = iov.iov_len;
3482 imu->nr_bvecs = nr_pages;
3483
3484 ctx->nr_user_bufs++;
3485 }
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003486 kvfree(pages);
3487 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07003488 return 0;
3489err:
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003490 kvfree(pages);
3491 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07003492 io_sqe_buffer_unregister(ctx);
3493 return ret;
3494}
3495
Jens Axboe9b402842019-04-11 11:45:41 -06003496static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
3497{
3498 __s32 __user *fds = arg;
3499 int fd;
3500
3501 if (ctx->cq_ev_fd)
3502 return -EBUSY;
3503
3504 if (copy_from_user(&fd, fds, sizeof(*fds)))
3505 return -EFAULT;
3506
3507 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
3508 if (IS_ERR(ctx->cq_ev_fd)) {
3509 int ret = PTR_ERR(ctx->cq_ev_fd);
3510 ctx->cq_ev_fd = NULL;
3511 return ret;
3512 }
3513
3514 return 0;
3515}
3516
3517static int io_eventfd_unregister(struct io_ring_ctx *ctx)
3518{
3519 if (ctx->cq_ev_fd) {
3520 eventfd_ctx_put(ctx->cq_ev_fd);
3521 ctx->cq_ev_fd = NULL;
3522 return 0;
3523 }
3524
3525 return -ENXIO;
3526}
3527
Jens Axboe2b188cc2019-01-07 10:46:33 -07003528static void io_ring_ctx_free(struct io_ring_ctx *ctx)
3529{
Jens Axboe6b063142019-01-10 22:13:58 -07003530 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003531 if (ctx->sqo_mm)
3532 mmdrop(ctx->sqo_mm);
Jens Axboedef596e2019-01-09 08:59:42 -07003533
3534 io_iopoll_reap_events(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -07003535 io_sqe_buffer_unregister(ctx);
Jens Axboe6b063142019-01-10 22:13:58 -07003536 io_sqe_files_unregister(ctx);
Jens Axboe9b402842019-04-11 11:45:41 -06003537 io_eventfd_unregister(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07003538
Jens Axboe2b188cc2019-01-07 10:46:33 -07003539#if defined(CONFIG_UNIX)
Eric Biggers355e8d262019-06-12 14:58:43 -07003540 if (ctx->ring_sock) {
3541 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07003542 sock_release(ctx->ring_sock);
Eric Biggers355e8d262019-06-12 14:58:43 -07003543 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003544#endif
3545
Hristo Venev75b28af2019-08-26 17:23:46 +00003546 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003547 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003548
3549 percpu_ref_exit(&ctx->refs);
3550 if (ctx->account_mem)
3551 io_unaccount_mem(ctx->user,
3552 ring_pages(ctx->sq_entries, ctx->cq_entries));
3553 free_uid(ctx->user);
3554 kfree(ctx);
3555}
3556
3557static __poll_t io_uring_poll(struct file *file, poll_table *wait)
3558{
3559 struct io_ring_ctx *ctx = file->private_data;
3560 __poll_t mask = 0;
3561
3562 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02003563 /*
3564 * synchronizes with barrier from wq_has_sleeper call in
3565 * io_commit_cqring
3566 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07003567 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +00003568 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
3569 ctx->rings->sq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003570 mask |= EPOLLOUT | EPOLLWRNORM;
yangerkundaa5de52019-09-24 20:53:34 +08003571 if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003572 mask |= EPOLLIN | EPOLLRDNORM;
3573
3574 return mask;
3575}
3576
3577static int io_uring_fasync(int fd, struct file *file, int on)
3578{
3579 struct io_ring_ctx *ctx = file->private_data;
3580
3581 return fasync_helper(fd, file, on, &ctx->cq_fasync);
3582}
3583
3584static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
3585{
3586 mutex_lock(&ctx->uring_lock);
3587 percpu_ref_kill(&ctx->refs);
3588 mutex_unlock(&ctx->uring_lock);
3589
Jens Axboe5262f562019-09-17 12:26:57 -06003590 io_kill_timeouts(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003591 io_poll_remove_all(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07003592 io_iopoll_reap_events(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003593 wait_for_completion(&ctx->ctx_done);
3594 io_ring_ctx_free(ctx);
3595}
3596
3597static int io_uring_release(struct inode *inode, struct file *file)
3598{
3599 struct io_ring_ctx *ctx = file->private_data;
3600
3601 file->private_data = NULL;
3602 io_ring_ctx_wait_and_kill(ctx);
3603 return 0;
3604}
3605
3606static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
3607{
3608 loff_t offset = (loff_t) vma->vm_pgoff << PAGE_SHIFT;
3609 unsigned long sz = vma->vm_end - vma->vm_start;
3610 struct io_ring_ctx *ctx = file->private_data;
3611 unsigned long pfn;
3612 struct page *page;
3613 void *ptr;
3614
3615 switch (offset) {
3616 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00003617 case IORING_OFF_CQ_RING:
3618 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003619 break;
3620 case IORING_OFF_SQES:
3621 ptr = ctx->sq_sqes;
3622 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003623 default:
3624 return -EINVAL;
3625 }
3626
3627 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07003628 if (sz > page_size(page))
Jens Axboe2b188cc2019-01-07 10:46:33 -07003629 return -EINVAL;
3630
3631 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
3632 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
3633}
3634
3635SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
3636 u32, min_complete, u32, flags, const sigset_t __user *, sig,
3637 size_t, sigsz)
3638{
3639 struct io_ring_ctx *ctx;
3640 long ret = -EBADF;
3641 int submitted = 0;
3642 struct fd f;
3643
Jens Axboe6c271ce2019-01-10 11:22:30 -07003644 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
Jens Axboe2b188cc2019-01-07 10:46:33 -07003645 return -EINVAL;
3646
3647 f = fdget(fd);
3648 if (!f.file)
3649 return -EBADF;
3650
3651 ret = -EOPNOTSUPP;
3652 if (f.file->f_op != &io_uring_fops)
3653 goto out_fput;
3654
3655 ret = -ENXIO;
3656 ctx = f.file->private_data;
3657 if (!percpu_ref_tryget(&ctx->refs))
3658 goto out_fput;
3659
Jens Axboe6c271ce2019-01-10 11:22:30 -07003660 /*
3661 * For SQ polling, the thread will do all submissions and completions.
3662 * Just return the requested submit count, and wake the thread if
3663 * we were asked to.
3664 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06003665 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003666 if (ctx->flags & IORING_SETUP_SQPOLL) {
3667 if (flags & IORING_ENTER_SQ_WAKEUP)
3668 wake_up(&ctx->sqo_wait);
3669 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06003670 } else if (to_submit) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07003671 to_submit = min(to_submit, ctx->sq_entries);
3672
3673 mutex_lock(&ctx->uring_lock);
Jens Axboebc808bc2019-10-22 13:14:37 -06003674 submitted = io_ring_submit(ctx, to_submit);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003675 mutex_unlock(&ctx->uring_lock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003676 }
3677 if (flags & IORING_ENTER_GETEVENTS) {
Jens Axboedef596e2019-01-09 08:59:42 -07003678 unsigned nr_events = 0;
3679
Jens Axboe2b188cc2019-01-07 10:46:33 -07003680 min_complete = min(min_complete, ctx->cq_entries);
3681
Jens Axboedef596e2019-01-09 08:59:42 -07003682 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07003683 ret = io_iopoll_check(ctx, &nr_events, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07003684 } else {
3685 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
3686 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003687 }
3688
Pavel Begunkov6805b32e2019-10-08 02:18:42 +03003689 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003690out_fput:
3691 fdput(f);
3692 return submitted ? submitted : ret;
3693}
3694
3695static const struct file_operations io_uring_fops = {
3696 .release = io_uring_release,
3697 .mmap = io_uring_mmap,
3698 .poll = io_uring_poll,
3699 .fasync = io_uring_fasync,
3700};
3701
3702static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
3703 struct io_uring_params *p)
3704{
Hristo Venev75b28af2019-08-26 17:23:46 +00003705 struct io_rings *rings;
3706 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003707
Hristo Venev75b28af2019-08-26 17:23:46 +00003708 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
3709 if (size == SIZE_MAX)
3710 return -EOVERFLOW;
3711
3712 rings = io_mem_alloc(size);
3713 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003714 return -ENOMEM;
3715
Hristo Venev75b28af2019-08-26 17:23:46 +00003716 ctx->rings = rings;
3717 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
3718 rings->sq_ring_mask = p->sq_entries - 1;
3719 rings->cq_ring_mask = p->cq_entries - 1;
3720 rings->sq_ring_entries = p->sq_entries;
3721 rings->cq_ring_entries = p->cq_entries;
3722 ctx->sq_mask = rings->sq_ring_mask;
3723 ctx->cq_mask = rings->cq_ring_mask;
3724 ctx->sq_entries = rings->sq_ring_entries;
3725 ctx->cq_entries = rings->cq_ring_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003726
3727 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
3728 if (size == SIZE_MAX)
3729 return -EOVERFLOW;
3730
3731 ctx->sq_sqes = io_mem_alloc(size);
Mark Rutland52e04ef2019-04-30 17:30:21 +01003732 if (!ctx->sq_sqes)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003733 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003734
Jens Axboe2b188cc2019-01-07 10:46:33 -07003735 return 0;
3736}
3737
3738/*
3739 * Allocate an anonymous fd, this is what constitutes the application
3740 * visible backing of an io_uring instance. The application mmaps this
3741 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
3742 * we have to tie this fd to a socket for file garbage collection purposes.
3743 */
3744static int io_uring_get_fd(struct io_ring_ctx *ctx)
3745{
3746 struct file *file;
3747 int ret;
3748
3749#if defined(CONFIG_UNIX)
3750 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
3751 &ctx->ring_sock);
3752 if (ret)
3753 return ret;
3754#endif
3755
3756 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
3757 if (ret < 0)
3758 goto err;
3759
3760 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
3761 O_RDWR | O_CLOEXEC);
3762 if (IS_ERR(file)) {
3763 put_unused_fd(ret);
3764 ret = PTR_ERR(file);
3765 goto err;
3766 }
3767
3768#if defined(CONFIG_UNIX)
3769 ctx->ring_sock->file = file;
Jens Axboe6b063142019-01-10 22:13:58 -07003770 ctx->ring_sock->sk->sk_user_data = ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003771#endif
3772 fd_install(ret, file);
3773 return ret;
3774err:
3775#if defined(CONFIG_UNIX)
3776 sock_release(ctx->ring_sock);
3777 ctx->ring_sock = NULL;
3778#endif
3779 return ret;
3780}
3781
3782static int io_uring_create(unsigned entries, struct io_uring_params *p)
3783{
3784 struct user_struct *user = NULL;
3785 struct io_ring_ctx *ctx;
3786 bool account_mem;
3787 int ret;
3788
3789 if (!entries || entries > IORING_MAX_ENTRIES)
3790 return -EINVAL;
3791
3792 /*
3793 * Use twice as many entries for the CQ ring. It's possible for the
3794 * application to drive a higher depth than the size of the SQ ring,
3795 * since the sqes are only used at submission time. This allows for
3796 * some flexibility in overcommitting a bit.
3797 */
3798 p->sq_entries = roundup_pow_of_two(entries);
3799 p->cq_entries = 2 * p->sq_entries;
3800
3801 user = get_uid(current_user());
3802 account_mem = !capable(CAP_IPC_LOCK);
3803
3804 if (account_mem) {
3805 ret = io_account_mem(user,
3806 ring_pages(p->sq_entries, p->cq_entries));
3807 if (ret) {
3808 free_uid(user);
3809 return ret;
3810 }
3811 }
3812
3813 ctx = io_ring_ctx_alloc(p);
3814 if (!ctx) {
3815 if (account_mem)
3816 io_unaccount_mem(user, ring_pages(p->sq_entries,
3817 p->cq_entries));
3818 free_uid(user);
3819 return -ENOMEM;
3820 }
3821 ctx->compat = in_compat_syscall();
3822 ctx->account_mem = account_mem;
3823 ctx->user = user;
3824
3825 ret = io_allocate_scq_urings(ctx, p);
3826 if (ret)
3827 goto err;
3828
Jens Axboe6c271ce2019-01-10 11:22:30 -07003829 ret = io_sq_offload_start(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003830 if (ret)
3831 goto err;
3832
Jens Axboe2b188cc2019-01-07 10:46:33 -07003833 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00003834 p->sq_off.head = offsetof(struct io_rings, sq.head);
3835 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
3836 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
3837 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
3838 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
3839 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
3840 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003841
3842 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00003843 p->cq_off.head = offsetof(struct io_rings, cq.head);
3844 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
3845 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
3846 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
3847 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
3848 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Jens Axboeac90f242019-09-06 10:26:21 -06003849
Jens Axboe044c1ab2019-10-28 09:15:33 -06003850 /*
3851 * Install ring fd as the very last thing, so we don't risk someone
3852 * having closed it before we finish setup
3853 */
3854 ret = io_uring_get_fd(ctx);
3855 if (ret < 0)
3856 goto err;
3857
Jens Axboeac90f242019-09-06 10:26:21 -06003858 p->features = IORING_FEAT_SINGLE_MMAP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003859 return ret;
3860err:
3861 io_ring_ctx_wait_and_kill(ctx);
3862 return ret;
3863}
3864
3865/*
3866 * Sets up an aio uring context, and returns the fd. Applications asks for a
3867 * ring size, we return the actual sq/cq ring sizes (among other things) in the
3868 * params structure passed in.
3869 */
3870static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
3871{
3872 struct io_uring_params p;
3873 long ret;
3874 int i;
3875
3876 if (copy_from_user(&p, params, sizeof(p)))
3877 return -EFAULT;
3878 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
3879 if (p.resv[i])
3880 return -EINVAL;
3881 }
3882
Jens Axboe6c271ce2019-01-10 11:22:30 -07003883 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
3884 IORING_SETUP_SQ_AFF))
Jens Axboe2b188cc2019-01-07 10:46:33 -07003885 return -EINVAL;
3886
3887 ret = io_uring_create(entries, &p);
3888 if (ret < 0)
3889 return ret;
3890
3891 if (copy_to_user(params, &p, sizeof(p)))
3892 return -EFAULT;
3893
3894 return ret;
3895}
3896
3897SYSCALL_DEFINE2(io_uring_setup, u32, entries,
3898 struct io_uring_params __user *, params)
3899{
3900 return io_uring_setup(entries, params);
3901}
3902
Jens Axboeedafcce2019-01-09 09:16:05 -07003903static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
3904 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06003905 __releases(ctx->uring_lock)
3906 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07003907{
3908 int ret;
3909
Jens Axboe35fa71a2019-04-22 10:23:23 -06003910 /*
3911 * We're inside the ring mutex, if the ref is already dying, then
3912 * someone else killed the ctx or is already going through
3913 * io_uring_register().
3914 */
3915 if (percpu_ref_is_dying(&ctx->refs))
3916 return -ENXIO;
3917
Jens Axboeedafcce2019-01-09 09:16:05 -07003918 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -06003919
3920 /*
3921 * Drop uring mutex before waiting for references to exit. If another
3922 * thread is currently inside io_uring_enter() it might need to grab
3923 * the uring_lock to make progress. If we hold it here across the drain
3924 * wait, then we can deadlock. It's safe to drop the mutex here, since
3925 * no new references will come in after we've killed the percpu ref.
3926 */
3927 mutex_unlock(&ctx->uring_lock);
Jens Axboeedafcce2019-01-09 09:16:05 -07003928 wait_for_completion(&ctx->ctx_done);
Jens Axboeb19062a2019-04-15 10:49:38 -06003929 mutex_lock(&ctx->uring_lock);
Jens Axboeedafcce2019-01-09 09:16:05 -07003930
3931 switch (opcode) {
3932 case IORING_REGISTER_BUFFERS:
3933 ret = io_sqe_buffer_register(ctx, arg, nr_args);
3934 break;
3935 case IORING_UNREGISTER_BUFFERS:
3936 ret = -EINVAL;
3937 if (arg || nr_args)
3938 break;
3939 ret = io_sqe_buffer_unregister(ctx);
3940 break;
Jens Axboe6b063142019-01-10 22:13:58 -07003941 case IORING_REGISTER_FILES:
3942 ret = io_sqe_files_register(ctx, arg, nr_args);
3943 break;
3944 case IORING_UNREGISTER_FILES:
3945 ret = -EINVAL;
3946 if (arg || nr_args)
3947 break;
3948 ret = io_sqe_files_unregister(ctx);
3949 break;
Jens Axboe9b402842019-04-11 11:45:41 -06003950 case IORING_REGISTER_EVENTFD:
3951 ret = -EINVAL;
3952 if (nr_args != 1)
3953 break;
3954 ret = io_eventfd_register(ctx, arg);
3955 break;
3956 case IORING_UNREGISTER_EVENTFD:
3957 ret = -EINVAL;
3958 if (arg || nr_args)
3959 break;
3960 ret = io_eventfd_unregister(ctx);
3961 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07003962 default:
3963 ret = -EINVAL;
3964 break;
3965 }
3966
3967 /* bring the ctx back to life */
3968 reinit_completion(&ctx->ctx_done);
3969 percpu_ref_reinit(&ctx->refs);
3970 return ret;
3971}
3972
3973SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
3974 void __user *, arg, unsigned int, nr_args)
3975{
3976 struct io_ring_ctx *ctx;
3977 long ret = -EBADF;
3978 struct fd f;
3979
3980 f = fdget(fd);
3981 if (!f.file)
3982 return -EBADF;
3983
3984 ret = -EOPNOTSUPP;
3985 if (f.file->f_op != &io_uring_fops)
3986 goto out_fput;
3987
3988 ctx = f.file->private_data;
3989
3990 mutex_lock(&ctx->uring_lock);
3991 ret = __io_uring_register(ctx, opcode, arg, nr_args);
3992 mutex_unlock(&ctx->uring_lock);
3993out_fput:
3994 fdput(f);
3995 return ret;
3996}
3997
Jens Axboe2b188cc2019-01-07 10:46:33 -07003998static int __init io_uring_init(void)
3999{
4000 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
4001 return 0;
4002};
4003__initcall(io_uring_init);