Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | |
| 3 | #include <linux/net.h> |
| 4 | #include <linux/uio.h> |
| 5 | #include <net/sock.h> |
| 6 | #include <linux/nospec.h> |
| 7 | |
Pavel Begunkov | 6a9ce66 | 2022-07-25 10:52:05 +0100 | [diff] [blame] | 8 | #include "rsrc.h" |
| 9 | |
Pavel Begunkov | eb4a299 | 2022-07-12 21:52:39 +0100 | [diff] [blame] | 10 | #define IO_NOTIF_SPLICE_BATCH 32 |
| 11 | |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 12 | struct io_notif_data { |
| 13 | struct file *file; |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 14 | struct ubuf_info uarg; |
Pavel Begunkov | 6a9ce66 | 2022-07-25 10:52:05 +0100 | [diff] [blame] | 15 | unsigned long account_pages; |
Stefan Metzmacher | e307e66 | 2022-10-27 20:34:45 +0200 | [diff] [blame] | 16 | bool zc_report; |
| 17 | bool zc_used; |
| 18 | bool zc_copied; |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 19 | }; |
| 20 | |
Pavel Begunkov | b48c312 | 2022-09-01 11:54:04 +0100 | [diff] [blame] | 21 | struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx); |
Pavel Begunkov | 40725d1 | 2022-11-04 10:59:45 +0000 | [diff] [blame] | 22 | void io_notif_set_extended(struct io_kiocb *notif); |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 23 | |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 24 | static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) |
| 25 | { |
Stefan Metzmacher | f2ccb5a | 2022-08-11 09:11:15 +0200 | [diff] [blame] | 26 | return io_kiocb_to_cmd(notif, struct io_notif_data); |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 27 | } |
| 28 | |
Pavel Begunkov | bedd20b | 2022-11-04 10:59:44 +0000 | [diff] [blame] | 29 | static inline void io_notif_flush(struct io_kiocb *notif) |
| 30 | __must_hold(¬if->ctx->uring_lock) |
| 31 | { |
| 32 | struct io_notif_data *nd = io_notif_to_data(notif); |
| 33 | |
| 34 | /* drop slot's master ref */ |
| 35 | if (refcount_dec_and_test(&nd->uarg.refcnt)) |
| 36 | io_req_task_work_add(notif); |
| 37 | } |
| 38 | |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 39 | static inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len) |
Pavel Begunkov | 6a9ce66 | 2022-07-25 10:52:05 +0100 | [diff] [blame] | 40 | { |
| 41 | struct io_ring_ctx *ctx = notif->ctx; |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 42 | struct io_notif_data *nd = io_notif_to_data(notif); |
Pavel Begunkov | 6a9ce66 | 2022-07-25 10:52:05 +0100 | [diff] [blame] | 43 | unsigned nr_pages = (len >> PAGE_SHIFT) + 2; |
| 44 | int ret; |
| 45 | |
| 46 | if (ctx->user) { |
| 47 | ret = __io_account_mem(ctx->user, nr_pages); |
| 48 | if (ret) |
| 49 | return ret; |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 50 | nd->account_pages += nr_pages; |
Pavel Begunkov | 6a9ce66 | 2022-07-25 10:52:05 +0100 | [diff] [blame] | 51 | } |
| 52 | return 0; |
| 53 | } |