commit 394918ebb889f99d89db6843bcc93279b2b745f9
author Jens Axboe <axboe@kernel.dk> Mon Mar 08 11:40:23 2021 -0700
committer Jens Axboe <axboe@kernel.dk> Mon Aug 23 13:44:55 2021 -0600
tree fdceeeefee346bd281fd73a52d8cb615fa0034c4
parent be863b9e4348a791e360d25611a1bdde2c9595ed

io_uring: enable use of bio alloc cache

Mark polled IO as being safe for dipping into the bio allocation
cache, in case the targeted bio_set has it enabled.

This brings an IOPOLL gen2 Optane QD=128 workload from ~3.2M IOPS to
~3.5M IOPS.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

fs/io_uring.c

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 4f5a007..504aede 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2737,7 +2737,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 		    !kiocb->ki_filp->f_op->iopoll)
 			return -EOPNOTSUPP;
 
-		kiocb->ki_flags |= IOCB_HIPRI;
+		kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;
 		kiocb->ki_complete = io_complete_rw_iopoll;
 		req->iopoll_completed = 0;
 	} else {