Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 1 | #include <linux/kernel.h> |
| 2 | #include <linux/errno.h> |
| 3 | #include <linux/file.h> |
| 4 | #include <linux/slab.h> |
| 5 | #include <linux/net.h> |
| 6 | #include <linux/io_uring.h> |
| 7 | |
| 8 | #include "io_uring.h" |
| 9 | #include "notif.h" |
Pavel Begunkov | 68ef557 | 2022-07-12 21:52:41 +0100 | [diff] [blame] | 10 | #include "rsrc.h" |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 11 | |
Pavel Begunkov | 6fe4220 | 2024-04-19 12:08:42 +0100 | [diff] [blame] | 12 | static const struct ubuf_info_ops io_ubuf_ops; |
| 13 | |
Pavel Begunkov | 5a56946 | 2024-04-19 12:08:41 +0100 | [diff] [blame] | 14 | static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts) |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 15 | { |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 16 | struct io_notif_data *nd = io_notif_to_data(notif); |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 17 | |
Pavel Begunkov | 6fe4220 | 2024-04-19 12:08:42 +0100 | [diff] [blame] | 18 | do { |
| 19 | notif = cmd_to_io_kiocb(nd); |
Pavel Begunkov | 42385b0 | 2022-11-04 10:59:46 +0000 | [diff] [blame] | 20 | |
Pavel Begunkov | 6fe4220 | 2024-04-19 12:08:42 +0100 | [diff] [blame] | 21 | lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0); |
| 22 | |
| 23 | if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used)) |
| 24 | notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED; |
| 25 | |
| 26 | if (nd->account_pages && notif->ctx->user) { |
| 27 | __io_unaccount_mem(notif->ctx->user, nd->account_pages); |
| 28 | nd->account_pages = 0; |
| 29 | } |
| 30 | |
| 31 | nd = nd->next; |
| 32 | io_req_task_complete(notif, ts); |
| 33 | } while (nd); |
Pavel Begunkov | 40725d1 | 2022-11-04 10:59:45 +0000 | [diff] [blame] | 34 | } |
| 35 | |
Pavel Begunkov | 5a56946 | 2024-04-19 12:08:41 +0100 | [diff] [blame] | 36 | void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg, |
| 37 | bool success) |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 38 | { |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 39 | struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); |
| 40 | struct io_kiocb *notif = cmd_to_io_kiocb(nd); |
Pavel Begunkov | 19352a1 | 2024-04-30 16:42:31 +0100 | [diff] [blame] | 41 | unsigned tw_flags; |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 42 | |
Stefan Metzmacher | e307e66 | 2022-10-27 20:34:45 +0200 | [diff] [blame] | 43 | if (nd->zc_report) { |
| 44 | if (success && !nd->zc_used && skb) |
| 45 | WRITE_ONCE(nd->zc_used, true); |
| 46 | else if (!success && !nd->zc_copied) |
| 47 | WRITE_ONCE(nd->zc_copied, true); |
| 48 | } |
Pavel Begunkov | 9986329 | 2024-04-08 00:54:55 +0100 | [diff] [blame] | 49 | |
Pavel Begunkov | 7e58d0a | 2024-04-15 13:50:11 +0100 | [diff] [blame] | 50 | if (!refcount_dec_and_test(&uarg->refcnt)) |
| 51 | return; |
| 52 | |
Pavel Begunkov | 6fe4220 | 2024-04-19 12:08:42 +0100 | [diff] [blame] | 53 | if (nd->head != nd) { |
| 54 | io_tx_ubuf_complete(skb, &nd->head->uarg, success); |
| 55 | return; |
| 56 | } |
Pavel Begunkov | 19352a1 | 2024-04-30 16:42:31 +0100 | [diff] [blame] | 57 | |
| 58 | tw_flags = nd->next ? 0 : IOU_F_TWQ_LAZY_WAKE; |
Pavel Begunkov | 7e58d0a | 2024-04-15 13:50:11 +0100 | [diff] [blame] | 59 | notif->io_task_work.func = io_notif_tw_complete; |
Pavel Begunkov | 19352a1 | 2024-04-30 16:42:31 +0100 | [diff] [blame] | 60 | __io_req_task_work_add(notif, tw_flags); |
Pavel Begunkov | eb4a299 | 2022-07-12 21:52:39 +0100 | [diff] [blame] | 61 | } |
| 62 | |
Pavel Begunkov | 6fe4220 | 2024-04-19 12:08:42 +0100 | [diff] [blame] | 63 | static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg) |
| 64 | { |
| 65 | struct io_notif_data *nd, *prev_nd; |
| 66 | struct io_kiocb *prev_notif, *notif; |
| 67 | struct ubuf_info *prev_uarg = skb_zcopy(skb); |
| 68 | |
| 69 | nd = container_of(uarg, struct io_notif_data, uarg); |
| 70 | notif = cmd_to_io_kiocb(nd); |
| 71 | |
| 72 | if (!prev_uarg) { |
| 73 | net_zcopy_get(&nd->uarg); |
| 74 | skb_zcopy_init(skb, &nd->uarg); |
| 75 | return 0; |
| 76 | } |
| 77 | /* handle it separately as we can't link a notif to itself */ |
| 78 | if (unlikely(prev_uarg == &nd->uarg)) |
| 79 | return 0; |
| 80 | /* we can't join two links together, just request a fresh skb */ |
| 81 | if (unlikely(nd->head != nd || nd->next)) |
| 82 | return -EEXIST; |
| 83 | /* don't mix zc providers */ |
| 84 | if (unlikely(prev_uarg->ops != &io_ubuf_ops)) |
| 85 | return -EEXIST; |
| 86 | |
| 87 | prev_nd = container_of(prev_uarg, struct io_notif_data, uarg); |
| 88 | prev_notif = cmd_to_io_kiocb(nd); |
| 89 | |
| 90 | /* make sure all noifications can be finished in the same task_work */ |
| 91 | if (unlikely(notif->ctx != prev_notif->ctx || |
| 92 | notif->task != prev_notif->task)) |
| 93 | return -EEXIST; |
| 94 | |
| 95 | nd->head = prev_nd->head; |
| 96 | nd->next = prev_nd->next; |
| 97 | prev_nd->next = nd; |
| 98 | net_zcopy_get(&nd->head->uarg); |
| 99 | return 0; |
| 100 | } |
| 101 | |
Pavel Begunkov | 7ab4f16 | 2024-04-19 12:08:39 +0100 | [diff] [blame] | 102 | static const struct ubuf_info_ops io_ubuf_ops = { |
| 103 | .complete = io_tx_ubuf_complete, |
Pavel Begunkov | 6fe4220 | 2024-04-19 12:08:42 +0100 | [diff] [blame] | 104 | .link_skb = io_link_skb, |
Pavel Begunkov | 7ab4f16 | 2024-04-19 12:08:39 +0100 | [diff] [blame] | 105 | }; |
| 106 | |
Pavel Begunkov | b48c312 | 2022-09-01 11:54:04 +0100 | [diff] [blame] | 107 | struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 108 | __must_hold(&ctx->uring_lock) |
| 109 | { |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 110 | struct io_kiocb *notif; |
| 111 | struct io_notif_data *nd; |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 112 | |
Pavel Begunkov | c8576f3 | 2023-01-23 14:37:16 +0000 | [diff] [blame] | 113 | if (unlikely(!io_alloc_req(ctx, ¬if))) |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 114 | return NULL; |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 115 | notif->opcode = IORING_OP_NOP; |
| 116 | notif->flags = 0; |
| 117 | notif->file = NULL; |
| 118 | notif->task = current; |
| 119 | io_get_task_refs(1); |
| 120 | notif->rsrc_node = NULL; |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 121 | |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 122 | nd = io_notif_to_data(notif); |
Pavel Begunkov | 9986329 | 2024-04-08 00:54:55 +0100 | [diff] [blame] | 123 | nd->zc_report = false; |
| 124 | nd->account_pages = 0; |
Pavel Begunkov | 6fe4220 | 2024-04-19 12:08:42 +0100 | [diff] [blame] | 125 | nd->next = NULL; |
| 126 | nd->head = nd; |
| 127 | |
Pavel Begunkov | 519760d | 2023-04-15 14:20:08 +0100 | [diff] [blame] | 128 | nd->uarg.flags = IO_NOTIF_UBUF_FLAGS; |
Pavel Begunkov | 7ab4f16 | 2024-04-19 12:08:39 +0100 | [diff] [blame] | 129 | nd->uarg.ops = &io_ubuf_ops; |
Pavel Begunkov | 14b146b | 2022-07-27 10:30:41 +0100 | [diff] [blame] | 130 | refcount_set(&nd->uarg.refcnt, 1); |
Pavel Begunkov | eb42ceb | 2022-07-12 21:52:38 +0100 | [diff] [blame] | 131 | return notif; |
| 132 | } |