blob: c88c800cd89d22918b4ee61fbef1dd188af4f8bf [file] [log] [blame]
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +01001// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/net.h>
4#include <linux/uio.h>
5#include <net/sock.h>
6#include <linux/nospec.h>
7
Pavel Begunkov6a9ce662022-07-25 10:52:05 +01008#include "rsrc.h"
9
Pavel Begunkoveb4a2992022-07-12 21:52:39 +010010#define IO_NOTIF_SPLICE_BATCH 32
11
Pavel Begunkov14b146b2022-07-27 10:30:41 +010012struct io_notif_data {
13 struct file *file;
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010014 struct ubuf_info uarg;
Pavel Begunkov6a9ce662022-07-25 10:52:05 +010015 unsigned long account_pages;
Stefan Metzmachere307e662022-10-27 20:34:45 +020016 bool zc_report;
17 bool zc_used;
18 bool zc_copied;
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010019};
20
Pavel Begunkovb48c3122022-09-01 11:54:04 +010021struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx);
Pavel Begunkov40725d12022-11-04 10:59:45 +000022void io_notif_set_extended(struct io_kiocb *notif);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010023
Pavel Begunkov14b146b2022-07-27 10:30:41 +010024static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif)
25{
Stefan Metzmacherf2ccb5a2022-08-11 09:11:15 +020026 return io_kiocb_to_cmd(notif, struct io_notif_data);
Pavel Begunkov14b146b2022-07-27 10:30:41 +010027}
28
Pavel Begunkovbedd20b2022-11-04 10:59:44 +000029static inline void io_notif_flush(struct io_kiocb *notif)
30 __must_hold(&notif->ctx->uring_lock)
31{
32 struct io_notif_data *nd = io_notif_to_data(notif);
33
34 /* drop slot's master ref */
35 if (refcount_dec_and_test(&nd->uarg.refcnt))
36 io_req_task_work_add(notif);
37}
38
Pavel Begunkov14b146b2022-07-27 10:30:41 +010039static inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len)
Pavel Begunkov6a9ce662022-07-25 10:52:05 +010040{
41 struct io_ring_ctx *ctx = notif->ctx;
Pavel Begunkov14b146b2022-07-27 10:30:41 +010042 struct io_notif_data *nd = io_notif_to_data(notif);
Pavel Begunkov6a9ce662022-07-25 10:52:05 +010043 unsigned nr_pages = (len >> PAGE_SHIFT) + 2;
44 int ret;
45
46 if (ctx->user) {
47 ret = __io_account_mem(ctx->user, nr_pages);
48 if (ret)
49 return ret;
Pavel Begunkov14b146b2022-07-27 10:30:41 +010050 nd->account_pages += nr_pages;
Pavel Begunkov6a9ce662022-07-25 10:52:05 +010051 }
52 return 0;
53}