| #include <sys/types.h> |
| #include <sys/stat.h> |
| #include <sys/mman.h> |
| #include <unistd.h> |
| #include <errno.h> |
| #include <string.h> |
| |
| #include "liburing.h" |
| #include "barrier.h" |
| |
| static int __io_uring_get_cqe(struct io_uring *ring, |
| struct io_uring_cqe **cqe_ptr, int wait) |
| { |
| struct io_uring_cq *cq = &ring->cq; |
| const unsigned mask = *cq->kring_mask; |
| unsigned head; |
| int ret; |
| |
| *cqe_ptr = NULL; |
| head = *cq->khead; |
| do { |
| /* |
| * It's necessary to use a read_barrier() before reading |
| * the CQ tail, since the kernel updates it locklessly. The |
| * kernel has the matching store barrier for the update. The |
| * kernel also ensures that previous stores to CQEs are ordered |
| * with the tail update. |
| */ |
| read_barrier(); |
| if (head != *cq->ktail) { |
| *cqe_ptr = &cq->cqes[head & mask]; |
| break; |
| } |
| if (!wait) |
| break; |
| ret = io_uring_enter(ring->ring_fd, 0, 1, |
| IORING_ENTER_GETEVENTS, NULL); |
| if (ret < 0) |
| return -errno; |
| } while (1); |
| |
| return 0; |
| } |
| |
| /* |
| * Return an IO completion, if one is readily available. Returns 0 with |
| * cqe_ptr filled in on success, -errno on failure. |
| */ |
| int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) |
| { |
| return __io_uring_get_cqe(ring, cqe_ptr, 0); |
| } |
| |
| /* |
| * Return an IO completion, waiting for it if necessary. Returns 0 with |
| * cqe_ptr filled in on success, -errno on failure. |
| */ |
| int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) |
| { |
| return __io_uring_get_cqe(ring, cqe_ptr, 1); |
| } |
| |
| /* |
| * Submit sqes acquired from io_uring_get_sqe() to the kernel. |
| * |
| * Returns number of sqes submitted |
| */ |
| int io_uring_submit(struct io_uring *ring) |
| { |
| struct io_uring_sq *sq = &ring->sq; |
| const unsigned mask = *sq->kring_mask; |
| unsigned ktail, ktail_next, submitted, to_submit; |
| int ret; |
| |
| /* |
| * If we have pending IO in the kring, submit it first. We need a |
| * read barrier here to match the kernels store barrier when updating |
| * the SQ head. |
| */ |
| read_barrier(); |
| if (*sq->khead != *sq->ktail) { |
| submitted = *sq->kring_entries; |
| goto submit; |
| } |
| |
| if (sq->sqe_head == sq->sqe_tail) |
| return 0; |
| |
| /* |
| * Fill in sqes that we have queued up, adding them to the kernel ring |
| */ |
| submitted = 0; |
| ktail = ktail_next = *sq->ktail; |
| to_submit = sq->sqe_tail - sq->sqe_head; |
| while (to_submit--) { |
| ktail_next++; |
| read_barrier(); |
| |
| sq->array[ktail & mask] = sq->sqe_head & mask; |
| ktail = ktail_next; |
| |
| sq->sqe_head++; |
| submitted++; |
| } |
| |
| if (!submitted) |
| return 0; |
| |
| if (*sq->ktail != ktail) { |
| /* |
| * First write barrier ensures that the SQE stores are updated |
| * with the tail update. This is needed so that the kernel |
| * will never see a tail update without the preceeding sQE |
| * stores being done. |
| */ |
| write_barrier(); |
| *sq->ktail = ktail; |
| /* |
| * The kernel has the matching read barrier for reading the |
| * SQ tail. |
| */ |
| write_barrier(); |
| } |
| |
| submit: |
| ret = io_uring_enter(ring->ring_fd, submitted, 0, |
| IORING_ENTER_GETEVENTS, NULL); |
| if (ret < 0) |
| return -errno; |
| |
| return ret; |
| } |
| |
| /* |
| * Return an sqe to fill. Application must later call io_uring_submit() |
| * when it's ready to tell the kernel about it. The caller may call this |
| * function multiple times before calling io_uring_submit(). |
| * |
| * Returns a vacant sqe, or NULL if we're full. |
| */ |
| struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) |
| { |
| struct io_uring_sq *sq = &ring->sq; |
| unsigned next = sq->sqe_tail + 1; |
| struct io_uring_sqe *sqe; |
| |
| /* |
| * All sqes are used |
| */ |
| if (next - sq->sqe_head > *sq->kring_entries) |
| return NULL; |
| |
| sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask]; |
| sq->sqe_tail = next; |
| return sqe; |
| } |