blob: 977736e82c1aab5c5870619b7bc18c70bfa94448 [file] [log] [blame]
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +01001#include <linux/kernel.h>
2#include <linux/errno.h>
3#include <linux/file.h>
4#include <linux/slab.h>
5#include <linux/net.h>
6#include <linux/io_uring.h>
7
8#include "io_uring.h"
9#include "notif.h"
Pavel Begunkov68ef5572022-07-12 21:52:41 +010010#include "rsrc.h"
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010011
Pavel Begunkov14b146b2022-07-27 10:30:41 +010012static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010013{
Pavel Begunkov14b146b2022-07-27 10:30:41 +010014 struct io_notif_data *nd = io_notif_to_data(notif);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010015 struct io_ring_ctx *ctx = notif->ctx;
16
Pavel Begunkov14b146b2022-07-27 10:30:41 +010017 if (nd->account_pages && ctx->user) {
18 __io_unaccount_mem(ctx->user, nd->account_pages);
19 nd->account_pages = 0;
Pavel Begunkove29e3bd2022-07-12 21:52:44 +010020 }
Pavel Begunkov14b146b2022-07-27 10:30:41 +010021 io_req_task_complete(notif, locked);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010022}
23
Pavel Begunkov14b146b2022-07-27 10:30:41 +010024static inline void io_notif_complete(struct io_kiocb *notif)
25 __must_hold(&notif->ctx->uring_lock)
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010026{
Pavel Begunkov14b146b2022-07-27 10:30:41 +010027 bool locked = true;
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010028
Pavel Begunkov14b146b2022-07-27 10:30:41 +010029 __io_notif_complete_tw(notif, &locked);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010030}
31
32static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,
33 struct ubuf_info *uarg,
34 bool success)
35{
Pavel Begunkov14b146b2022-07-27 10:30:41 +010036 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
37 struct io_kiocb *notif = cmd_to_io_kiocb(nd);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010038
Pavel Begunkov14b146b2022-07-27 10:30:41 +010039 if (refcount_dec_and_test(&uarg->refcnt)) {
40 notif->io_task_work.func = __io_notif_complete_tw;
41 io_req_task_work_add(notif);
Pavel Begunkoveb4a2992022-07-12 21:52:39 +010042 }
43}
44
Pavel Begunkov14b146b2022-07-27 10:30:41 +010045struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx,
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010046 struct io_notif_slot *slot)
47 __must_hold(&ctx->uring_lock)
48{
Pavel Begunkov14b146b2022-07-27 10:30:41 +010049 struct io_kiocb *notif;
50 struct io_notif_data *nd;
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010051
Pavel Begunkov14b146b2022-07-27 10:30:41 +010052 if (unlikely(!io_alloc_req_refill(ctx)))
53 return NULL;
54 notif = io_alloc_req(ctx);
55 notif->opcode = IORING_OP_NOP;
56 notif->flags = 0;
57 notif->file = NULL;
58 notif->task = current;
59 io_get_task_refs(1);
60 notif->rsrc_node = NULL;
61 io_req_set_rsrc_node(notif, ctx, 0);
62 notif->cqe.user_data = slot->tag;
63 notif->cqe.flags = slot->seq++;
64 notif->cqe.res = 0;
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010065
Pavel Begunkov14b146b2022-07-27 10:30:41 +010066 nd = io_notif_to_data(notif);
67 nd->account_pages = 0;
68 nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
69 nd->uarg.callback = io_uring_tx_zerocopy_callback;
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010070 /* master ref owned by io_notif_slot, will be dropped on flush */
Pavel Begunkov14b146b2022-07-27 10:30:41 +010071 refcount_set(&nd->uarg.refcnt, 1);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010072 return notif;
73}
74
Pavel Begunkov63809132022-07-12 21:52:47 +010075void io_notif_slot_flush(struct io_notif_slot *slot)
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010076 __must_hold(&ctx->uring_lock)
77{
Pavel Begunkov14b146b2022-07-27 10:30:41 +010078 struct io_kiocb *notif = slot->notif;
79 struct io_notif_data *nd = io_notif_to_data(notif);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010080
81 slot->notif = NULL;
82
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010083 /* drop slot's master ref */
Pavel Begunkov14b146b2022-07-27 10:30:41 +010084 if (refcount_dec_and_test(&nd->uarg.refcnt))
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +010085 io_notif_complete(notif);
86}
87
88__cold int io_notif_unregister(struct io_ring_ctx *ctx)
89 __must_hold(&ctx->uring_lock)
90{
91 int i;
92
93 if (!ctx->notif_slots)
94 return -ENXIO;
95
96 for (i = 0; i < ctx->nr_notif_slots; i++) {
97 struct io_notif_slot *slot = &ctx->notif_slots[i];
Pavel Begunkov14b146b2022-07-27 10:30:41 +010098 struct io_kiocb *notif = slot->notif;
99 struct io_notif_data *nd;
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +0100100
Pavel Begunkov14b146b2022-07-27 10:30:41 +0100101 if (!notif)
Pavel Begunkove58d4982022-07-12 21:52:40 +0100102 continue;
Stefan Metzmacherda2634e2022-08-11 09:11:14 +0200103 nd = io_notif_to_data(notif);
Pavel Begunkov14b146b2022-07-27 10:30:41 +0100104 slot->notif = NULL;
105 if (!refcount_dec_and_test(&nd->uarg.refcnt))
106 continue;
107 notif->io_task_work.func = __io_notif_complete_tw;
108 io_req_task_work_add(notif);
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +0100109 }
110
111 kvfree(ctx->notif_slots);
112 ctx->notif_slots = NULL;
113 ctx->nr_notif_slots = 0;
Pavel Begunkovbc24d6b2022-07-12 21:52:42 +0100114 return 0;
115}
116
117__cold int io_notif_register(struct io_ring_ctx *ctx,
118 void __user *arg, unsigned int size)
119 __must_hold(&ctx->uring_lock)
120{
121 struct io_uring_notification_slot __user *slots;
122 struct io_uring_notification_slot slot;
123 struct io_uring_notification_register reg;
124 unsigned i;
125
126 if (ctx->nr_notif_slots)
127 return -EBUSY;
128 if (size != sizeof(reg))
129 return -EINVAL;
130 if (copy_from_user(&reg, arg, sizeof(reg)))
131 return -EFAULT;
132 if (!reg.nr_slots || reg.nr_slots > IORING_MAX_NOTIF_SLOTS)
133 return -EINVAL;
134 if (reg.resv || reg.resv2 || reg.resv3)
135 return -EINVAL;
136
137 slots = u64_to_user_ptr(reg.data);
138 ctx->notif_slots = kvcalloc(reg.nr_slots, sizeof(ctx->notif_slots[0]),
139 GFP_KERNEL_ACCOUNT);
140 if (!ctx->notif_slots)
141 return -ENOMEM;
142
143 for (i = 0; i < reg.nr_slots; i++, ctx->nr_notif_slots++) {
144 struct io_notif_slot *notif_slot = &ctx->notif_slots[i];
145
146 if (copy_from_user(&slot, &slots[i], sizeof(slot))) {
147 io_notif_unregister(ctx);
148 return -EFAULT;
149 }
150 if (slot.resv[0] | slot.resv[1] | slot.resv[2]) {
151 io_notif_unregister(ctx);
152 return -EINVAL;
153 }
154 notif_slot->tag = slot.tag;
155 }
Pavel Begunkoveb42ceb2022-07-12 21:52:38 +0100156 return 0;
Pavel Begunkove58d4982022-07-12 21:52:40 +0100157}