blob: 4e3483a16b7575ec1d6f1cf9b92ebc7dfca1e9e5 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Jens Axboed6d48192008-01-29 14:04:06 +01002/*
3 * Functions related to segment and merge handling
4 */
5#include <linux/kernel.h>
6#include <linux/module.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>
Christoph Hellwigfe45e632021-09-20 14:33:27 +02009#include <linux/blk-integrity.h>
Jens Axboed6d48192008-01-29 14:04:06 +010010#include <linux/scatterlist.h>
Christoph Hellwig82d981d2021-11-23 19:53:12 +010011#include <linux/part_stat.h>
Tejun Heo6b2b0452022-03-14 14:30:11 -100012#include <linux/blk-cgroup.h>
Jens Axboed6d48192008-01-29 14:04:06 +010013
Mike Krinkincda22642015-12-03 17:32:30 +030014#include <trace/events/block.h>
15
Jens Axboed6d48192008-01-29 14:04:06 +010016#include "blk.h"
Christoph Hellwig2aa77452021-11-23 19:53:08 +010017#include "blk-mq-sched.h"
Baolin Wang8e756372020-08-28 10:52:54 +080018#include "blk-rq-qos.h"
Jens Axboea7b36ee2021-10-05 09:11:56 -060019#include "blk-throttle.h"
Jens Axboed6d48192008-01-29 14:04:06 +010020
Christoph Hellwigff18d772021-10-12 18:18:03 +020021static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
22{
23 *bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
24}
25
26static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
27{
28 struct bvec_iter iter = bio->bi_iter;
29 int idx;
30
31 bio_get_first_bvec(bio, bv);
32 if (bv->bv_len == bio->bi_iter.bi_size)
33 return; /* this bio only has a single bvec */
34
35 bio_advance_iter(bio, &iter, iter.bi_size);
36
37 if (!iter.bi_bvec_done)
38 idx = iter.bi_idx - 1;
39 else /* in the middle of bvec */
40 idx = iter.bi_idx;
41
42 *bv = bio->bi_io_vec[idx];
43
44 /*
45 * iter.bi_bvec_done records actual length of the last bvec
46 * if this bio ends in the middle of one io vector
47 */
48 if (iter.bi_bvec_done)
49 bv->bv_len = iter.bi_bvec_done;
50}
51
Christoph Hellwige9907002018-09-24 09:43:48 +020052static inline bool bio_will_gap(struct request_queue *q,
53 struct request *prev_rq, struct bio *prev, struct bio *next)
54{
55 struct bio_vec pb, nb;
56
57 if (!bio_has_data(prev) || !queue_virt_boundary(q))
58 return false;
59
60 /*
61 * Don't merge if the 1st bio starts with non-zero offset, otherwise it
62 * is quite difficult to respect the sg gap limit. We work hard to
63 * merge a huge number of small single bios in case of mkfs.
64 */
65 if (prev_rq)
66 bio_get_first_bvec(prev_rq->bio, &pb);
67 else
68 bio_get_first_bvec(prev, &pb);
Johannes Thumshirndf376b22018-11-07 14:58:14 +010069 if (pb.bv_offset & queue_virt_boundary(q))
Christoph Hellwige9907002018-09-24 09:43:48 +020070 return true;
71
72 /*
73 * We don't need to worry about the situation that the merged segment
74 * ends in unaligned virt boundary:
75 *
76 * - if 'pb' ends aligned, the merged segment ends aligned
77 * - if 'pb' ends unaligned, the next bio must include
78 * one single bvec of 'nb', otherwise the 'nb' can't
79 * merge with 'pb'
80 */
81 bio_get_last_bvec(prev, &pb);
82 bio_get_first_bvec(next, &nb);
Christoph Hellwig200a9af2019-05-21 09:01:42 +020083 if (biovec_phys_mergeable(q, &pb, &nb))
Christoph Hellwige9907002018-09-24 09:43:48 +020084 return false;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -040085 return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset);
Christoph Hellwige9907002018-09-24 09:43:48 +020086}
87
88static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
89{
90 return bio_will_gap(req->q, req, req->biotail, bio);
91}
92
93static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
94{
95 return bio_will_gap(req->q, NULL, bio, req->bio);
96}
97
Christoph Hellwigb6dc6192022-07-27 12:22:59 -040098/*
99 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
100 * is defined as 'unsigned int', meantime it has to be aligned to with the
101 * logical block size, which is the minimum accepted unit by hardware.
102 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700103static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim)
Christoph Hellwigb6dc6192022-07-27 12:22:59 -0400104{
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400105 return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT;
Christoph Hellwigb6dc6192022-07-27 12:22:59 -0400106}
107
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700108static struct bio *bio_split_discard(struct bio *bio,
109 const struct queue_limits *lim,
110 unsigned *nsegs, struct bio_set *bs)
Kent Overstreet54efd502015-04-23 22:37:18 -0700111{
112 unsigned int max_discard_sectors, granularity;
Kent Overstreet54efd502015-04-23 22:37:18 -0700113 sector_t tmp;
114 unsigned split_sectors;
115
Ming Leibdced432015-10-20 23:13:52 +0800116 *nsegs = 1;
117
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400118 granularity = max(lim->discard_granularity >> 9, 1U);
Kent Overstreet54efd502015-04-23 22:37:18 -0700119
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400120 max_discard_sectors =
121 min(lim->max_discard_sectors, bio_allowed_max_sectors(lim));
Kent Overstreet54efd502015-04-23 22:37:18 -0700122 max_discard_sectors -= max_discard_sectors % granularity;
Christoph Hellwig928a5dd2023-12-28 07:55:37 +0000123 if (unlikely(!max_discard_sectors))
Kent Overstreet54efd502015-04-23 22:37:18 -0700124 return NULL;
Kent Overstreet54efd502015-04-23 22:37:18 -0700125
126 if (bio_sectors(bio) <= max_discard_sectors)
127 return NULL;
128
129 split_sectors = max_discard_sectors;
130
131 /*
132 * If the next starting sector would be misaligned, stop the discard at
133 * the previous aligned sector.
134 */
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400135 tmp = bio->bi_iter.bi_sector + split_sectors -
136 ((lim->discard_alignment >> 9) % granularity);
Kent Overstreet54efd502015-04-23 22:37:18 -0700137 tmp = sector_div(tmp, granularity);
138
139 if (split_sectors > tmp)
140 split_sectors -= tmp;
141
142 return bio_split(bio, split_sectors, GFP_NOIO, bs);
143}
144
Christoph Hellwig5a978062022-07-27 12:22:55 -0400145static struct bio *bio_split_write_zeroes(struct bio *bio,
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700146 const struct queue_limits *lim,
147 unsigned *nsegs, struct bio_set *bs)
Christoph Hellwig885fa132017-04-05 19:21:01 +0200148{
Christoph Hellwigd665e122019-07-03 05:24:35 -0700149 *nsegs = 0;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400150 if (!lim->max_write_zeroes_sectors)
Christoph Hellwig885fa132017-04-05 19:21:01 +0200151 return NULL;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400152 if (bio_sectors(bio) <= lim->max_write_zeroes_sectors)
Christoph Hellwig885fa132017-04-05 19:21:01 +0200153 return NULL;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400154 return bio_split(bio, lim->max_write_zeroes_sectors, GFP_NOIO, bs);
Christoph Hellwig885fa132017-04-05 19:21:01 +0200155}
156
Bart Van Assche9cc51692019-08-01 15:50:44 -0700157/*
158 * Return the maximum number of sectors from the start of a bio that may be
159 * submitted as a single request to a block device. If enough sectors remain,
160 * align the end to the physical block size. Otherwise align the end to the
161 * logical block size. This approach minimizes the number of non-aligned
162 * requests that are submitted to a block device if the start of a bio is not
163 * aligned to a physical block boundary.
164 */
Christoph Hellwig5a978062022-07-27 12:22:55 -0400165static inline unsigned get_max_io_size(struct bio *bio,
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700166 const struct queue_limits *lim)
Ming Leid0e5fbb2016-01-23 08:05:33 +0800167{
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400168 unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
169 unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
170 unsigned max_sectors = lim->max_sectors, start, end;
Ming Leid0e5fbb2016-01-23 08:05:33 +0800171
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400172 if (lim->chunk_sectors) {
Christoph Hellwigefef7392022-06-14 11:09:33 +0200173 max_sectors = min(max_sectors,
174 blk_chunk_sectors_left(bio->bi_iter.bi_sector,
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400175 lim->chunk_sectors));
Christoph Hellwigefef7392022-06-14 11:09:33 +0200176 }
Ming Leid0e5fbb2016-01-23 08:05:33 +0800177
Christoph Hellwig84613be2022-06-14 11:09:32 +0200178 start = bio->bi_iter.bi_sector & (pbs - 1);
179 end = (start + max_sectors) & ~(pbs - 1);
180 if (end > start)
181 return end - start;
182 return max_sectors & ~(lbs - 1);
Ming Leid0e5fbb2016-01-23 08:05:33 +0800183}
184
Bart Van Assche95465312022-10-25 12:17:55 -0700185/**
186 * get_max_segment_size() - maximum number of bytes to add as a single segment
187 * @lim: Request queue limits.
188 * @start_page: See below.
189 * @offset: Offset from @start_page where to add a segment.
190 *
191 * Returns the maximum number of bytes that can be added as a single segment.
192 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700193static inline unsigned get_max_segment_size(const struct queue_limits *lim,
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400194 struct page *start_page, unsigned long offset)
Ming Leidcebd752019-02-15 19:13:12 +0800195{
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400196 unsigned long mask = lim->seg_boundary_mask;
Ming Leidcebd752019-02-15 19:13:12 +0800197
Ming Lei429120f2019-12-29 10:32:30 +0800198 offset = mask & (page_to_phys(start_page) + offset);
Ming Lei4a2f704e2020-01-11 20:57:43 +0800199
200 /*
Bart Van Assche95465312022-10-25 12:17:55 -0700201 * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1
202 * after having calculated the minimum.
Ming Lei4a2f704e2020-01-11 20:57:43 +0800203 */
Bart Van Assche95465312022-10-25 12:17:55 -0700204 return min(mask - offset, (unsigned long)lim->max_segment_size - 1) + 1;
Ming Leidcebd752019-02-15 19:13:12 +0800205}
206
Bart Van Assche708b25b2019-08-01 15:50:43 -0700207/**
208 * bvec_split_segs - verify whether or not a bvec should be split in the middle
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400209 * @lim: [in] queue limits to split based on
Bart Van Assche708b25b2019-08-01 15:50:43 -0700210 * @bv: [in] bvec to examine
211 * @nsegs: [in,out] Number of segments in the bio being built. Incremented
212 * by the number of segments from @bv that may be appended to that
213 * bio without exceeding @max_segs
Keith Busch67927d22022-06-10 12:58:25 -0700214 * @bytes: [in,out] Number of bytes in the bio being built. Incremented
215 * by the number of bytes from @bv that may be appended to that
216 * bio without exceeding @max_bytes
Bart Van Assche708b25b2019-08-01 15:50:43 -0700217 * @max_segs: [in] upper bound for *@nsegs
Keith Busch67927d22022-06-10 12:58:25 -0700218 * @max_bytes: [in] upper bound for *@bytes
Bart Van Assche708b25b2019-08-01 15:50:43 -0700219 *
220 * When splitting a bio, it can happen that a bvec is encountered that is too
221 * big to fit in a single segment and hence that it has to be split in the
222 * middle. This function verifies whether or not that should happen. The value
223 * %true is returned if and only if appending the entire @bv to a bio with
224 * *@nsegs segments and *@sectors sectors would make that bio unacceptable for
225 * the block driver.
Ming Leidcebd752019-02-15 19:13:12 +0800226 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700227static bool bvec_split_segs(const struct queue_limits *lim,
228 const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes,
229 unsigned max_segs, unsigned max_bytes)
Ming Leidcebd752019-02-15 19:13:12 +0800230{
Keith Busch67927d22022-06-10 12:58:25 -0700231 unsigned max_len = min(max_bytes, UINT_MAX) - *bytes;
Bart Van Assche708b25b2019-08-01 15:50:43 -0700232 unsigned len = min(bv->bv_len, max_len);
Ming Leidcebd752019-02-15 19:13:12 +0800233 unsigned total_len = 0;
Bart Van Asscheff9811b2019-08-01 15:50:42 -0700234 unsigned seg_size = 0;
Ming Leidcebd752019-02-15 19:13:12 +0800235
Bart Van Asscheff9811b2019-08-01 15:50:42 -0700236 while (len && *nsegs < max_segs) {
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400237 seg_size = get_max_segment_size(lim, bv->bv_page,
Ming Lei429120f2019-12-29 10:32:30 +0800238 bv->bv_offset + total_len);
Ming Leidcebd752019-02-15 19:13:12 +0800239 seg_size = min(seg_size, len);
240
Bart Van Asscheff9811b2019-08-01 15:50:42 -0700241 (*nsegs)++;
Ming Leidcebd752019-02-15 19:13:12 +0800242 total_len += seg_size;
243 len -= seg_size;
244
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400245 if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
Ming Leidcebd752019-02-15 19:13:12 +0800246 break;
247 }
248
Keith Busch67927d22022-06-10 12:58:25 -0700249 *bytes += total_len;
Ming Leidcebd752019-02-15 19:13:12 +0800250
Bart Van Assche708b25b2019-08-01 15:50:43 -0700251 /* tell the caller to split the bvec if it is too big to fit */
252 return len > 0 || bv->bv_len > max_len;
Ming Leidcebd752019-02-15 19:13:12 +0800253}
254
Bart Van Asschedad77582019-08-01 15:50:41 -0700255/**
Christoph Hellwig5a978062022-07-27 12:22:55 -0400256 * bio_split_rw - split a bio in two bios
Bart Van Asschedad77582019-08-01 15:50:41 -0700257 * @bio: [in] bio to be split
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400258 * @lim: [in] queue limits to split based on
Bart Van Asschedad77582019-08-01 15:50:41 -0700259 * @segs: [out] number of segments in the bio with the first half of the sectors
Christoph Hellwig5a978062022-07-27 12:22:55 -0400260 * @bs: [in] bio set to allocate the clone from
Christoph Hellwiga85b3632022-07-27 12:22:58 -0400261 * @max_bytes: [in] maximum number of bytes per bio
Bart Van Asschedad77582019-08-01 15:50:41 -0700262 *
263 * Clone @bio, update the bi_iter of the clone to represent the first sectors
264 * of @bio and update @bio->bi_iter to represent the remaining sectors. The
265 * following is guaranteed for the cloned bio:
Christoph Hellwiga85b3632022-07-27 12:22:58 -0400266 * - That it has at most @max_bytes worth of data
Bart Van Asschedad77582019-08-01 15:50:41 -0700267 * - That it has at most queue_max_segments(@q) segments.
268 *
269 * Except for discard requests the cloned bio will point at the bi_io_vec of
270 * the original bio. It is the responsibility of the caller to ensure that the
271 * original bio is not freed before the cloned bio. The caller is also
272 * responsible for ensuring that @bs is only destroyed after processing of the
273 * split bio has finished.
274 */
Christoph Hellwigfd8f8ed2023-01-21 07:49:58 +0100275struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
Christoph Hellwiga85b3632022-07-27 12:22:58 -0400276 unsigned *segs, struct bio_set *bs, unsigned max_bytes)
Kent Overstreet54efd502015-04-23 22:37:18 -0700277{
Jens Axboe5014c312015-09-02 16:46:02 -0600278 struct bio_vec bv, bvprv, *bvprvp = NULL;
Kent Overstreet54efd502015-04-23 22:37:18 -0700279 struct bvec_iter iter;
Keith Busch67927d22022-06-10 12:58:25 -0700280 unsigned nsegs = 0, bytes = 0;
Kent Overstreet54efd502015-04-23 22:37:18 -0700281
Ming Leidcebd752019-02-15 19:13:12 +0800282 bio_for_each_bvec(bv, bio, iter) {
Kent Overstreet54efd502015-04-23 22:37:18 -0700283 /*
284 * If the queue doesn't support SG gaps and adding this
285 * offset would create a gap, disallow it.
286 */
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400287 if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
Kent Overstreet54efd502015-04-23 22:37:18 -0700288 goto split;
289
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400290 if (nsegs < lim->max_segments &&
Keith Busch67927d22022-06-10 12:58:25 -0700291 bytes + bv.bv_len <= max_bytes &&
Bart Van Assche708b25b2019-08-01 15:50:43 -0700292 bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
293 nsegs++;
Keith Busch67927d22022-06-10 12:58:25 -0700294 bytes += bv.bv_len;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400295 } else {
296 if (bvec_split_segs(lim, &bv, &nsegs, &bytes,
297 lim->max_segments, max_bytes))
298 goto split;
Keith Busche36f6202016-01-12 15:08:39 -0700299 }
300
Kent Overstreet54efd502015-04-23 22:37:18 -0700301 bvprv = bv;
Ming Lei578270b2015-11-24 10:35:29 +0800302 bvprvp = &bvprv;
Kent Overstreet54efd502015-04-23 22:37:18 -0700303 }
304
Christoph Hellwigd6270652019-06-06 12:29:03 +0200305 *segs = nsegs;
306 return NULL;
Kent Overstreet54efd502015-04-23 22:37:18 -0700307split:
Jens Axboe9cea62b2023-01-04 08:52:06 -0700308 /*
309 * We can't sanely support splitting for a REQ_NOWAIT bio. End it
310 * with EAGAIN if splitting is required and return an error pointer.
311 */
312 if (bio->bi_opf & REQ_NOWAIT) {
313 bio->bi_status = BLK_STS_AGAIN;
314 bio_endio(bio);
315 return ERR_PTR(-EAGAIN);
316 }
317
Ming Leibdced432015-10-20 23:13:52 +0800318 *segs = nsegs;
Jeffle Xucc29e1b2020-11-26 17:18:52 +0800319
320 /*
Keith Busch67927d22022-06-10 12:58:25 -0700321 * Individual bvecs might not be logical block aligned. Round down the
322 * split size so that each bio is properly block size aligned, even if
323 * we do not use the full hardware limits.
324 */
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400325 bytes = ALIGN_DOWN(bytes, lim->logical_block_size);
Keith Busch67927d22022-06-10 12:58:25 -0700326
327 /*
Jeffle Xucc29e1b2020-11-26 17:18:52 +0800328 * Bio splitting may cause subtle trouble such as hang when doing sync
329 * iopoll in direct IO routine. Given performance gain of iopoll for
330 * big IO can be trival, disable iopoll when split needed.
331 */
Christoph Hellwig6ce913f2021-10-12 13:12:21 +0200332 bio_clear_polled(bio);
Keith Busch67927d22022-06-10 12:58:25 -0700333 return bio_split(bio, bytes >> SECTOR_SHIFT, GFP_NOIO, bs);
Kent Overstreet54efd502015-04-23 22:37:18 -0700334}
Christoph Hellwigfd8f8ed2023-01-21 07:49:58 +0100335EXPORT_SYMBOL_GPL(bio_split_rw);
Kent Overstreet54efd502015-04-23 22:37:18 -0700336
Bart Van Asschedad77582019-08-01 15:50:41 -0700337/**
Christoph Hellwig5a978062022-07-27 12:22:55 -0400338 * __bio_split_to_limits - split a bio to fit the queue limits
339 * @bio: bio to be split
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400340 * @lim: queue limits to split based on
Christoph Hellwig5a978062022-07-27 12:22:55 -0400341 * @nr_segs: returns the number of segments in the returned bio
Bart Van Asschedad77582019-08-01 15:50:41 -0700342 *
Christoph Hellwig5a978062022-07-27 12:22:55 -0400343 * Check if @bio needs splitting based on the queue limits, and if so split off
344 * a bio fitting the limits from the beginning of @bio and return it. @bio is
345 * shortened to the remainder and re-submitted.
346 *
347 * The split bio is allocated from @q->bio_split, which is provided by the
348 * block layer.
Bart Van Asschedad77582019-08-01 15:50:41 -0700349 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700350struct bio *__bio_split_to_limits(struct bio *bio,
351 const struct queue_limits *lim,
352 unsigned int *nr_segs)
Kent Overstreet54efd502015-04-23 22:37:18 -0700353{
Christoph Hellwig46754bd2022-07-27 12:22:57 -0400354 struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
Christoph Hellwig5a978062022-07-27 12:22:55 -0400355 struct bio *split;
Kent Overstreet54efd502015-04-23 22:37:18 -0700356
Christoph Hellwig5a978062022-07-27 12:22:55 -0400357 switch (bio_op(bio)) {
Adrian Hunter7afafc82016-08-16 10:59:35 +0300358 case REQ_OP_DISCARD:
359 case REQ_OP_SECURE_ERASE:
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400360 split = bio_split_discard(bio, lim, nr_segs, bs);
Adrian Hunter7afafc82016-08-16 10:59:35 +0300361 break;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800362 case REQ_OP_WRITE_ZEROES:
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400363 split = bio_split_write_zeroes(bio, lim, nr_segs, bs);
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800364 break;
Adrian Hunter7afafc82016-08-16 10:59:35 +0300365 default:
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400366 split = bio_split_rw(bio, lim, nr_segs, bs,
367 get_max_io_size(bio, lim) << SECTOR_SHIFT);
Jens Axboe613b1482023-01-04 08:51:19 -0700368 if (IS_ERR(split))
369 return NULL;
Adrian Hunter7afafc82016-08-16 10:59:35 +0300370 break;
371 }
Ming Leibdced432015-10-20 23:13:52 +0800372
Kent Overstreet54efd502015-04-23 22:37:18 -0700373 if (split) {
Jens Axboe613b1482023-01-04 08:51:19 -0700374 /* there isn't chance to merge the split bio */
Jens Axboe1eff9d32016-08-05 15:35:16 -0600375 split->bi_opf |= REQ_NOMERGE;
Ming Lei6ac45ae2015-10-20 23:13:53 +0800376
Muchun Song957a2b32022-07-13 22:02:26 +0800377 blkcg_bio_issue_init(split);
Christoph Hellwig5a978062022-07-27 12:22:55 -0400378 bio_chain(split, bio);
379 trace_block_split(split, bio->bi_iter.bi_sector);
380 submit_bio_noacct(bio);
381 return split;
Kent Overstreet54efd502015-04-23 22:37:18 -0700382 }
Christoph Hellwig5a978062022-07-27 12:22:55 -0400383 return bio;
Kent Overstreet54efd502015-04-23 22:37:18 -0700384}
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200385
Bart Van Asschedad77582019-08-01 15:50:41 -0700386/**
Christoph Hellwig5a978062022-07-27 12:22:55 -0400387 * bio_split_to_limits - split a bio to fit the queue limits
388 * @bio: bio to be split
Bart Van Asschedad77582019-08-01 15:50:41 -0700389 *
Christoph Hellwig5a978062022-07-27 12:22:55 -0400390 * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and
391 * if so split off a bio fitting the limits from the beginning of @bio and
392 * return it. @bio is shortened to the remainder and re-submitted.
393 *
394 * The split bio is allocated from @q->bio_split, which is provided by the
395 * block layer.
Bart Van Asschedad77582019-08-01 15:50:41 -0700396 */
Christoph Hellwig5a978062022-07-27 12:22:55 -0400397struct bio *bio_split_to_limits(struct bio *bio)
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200398{
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700399 const struct queue_limits *lim = &bdev_get_queue(bio->bi_bdev)->limits;
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200400 unsigned int nr_segs;
401
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400402 if (bio_may_exceed_limits(bio, lim))
403 return __bio_split_to_limits(bio, lim, &nr_segs);
Christoph Hellwig5a978062022-07-27 12:22:55 -0400404 return bio;
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200405}
Christoph Hellwig5a978062022-07-27 12:22:55 -0400406EXPORT_SYMBOL(bio_split_to_limits);
Kent Overstreet54efd502015-04-23 22:37:18 -0700407
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200408unsigned int blk_recalc_rq_segments(struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100409{
Christoph Hellwig68698752019-05-21 09:01:43 +0200410 unsigned int nr_phys_segs = 0;
Keith Busch67927d22022-06-10 12:58:25 -0700411 unsigned int bytes = 0;
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200412 struct req_iterator iter;
Christoph Hellwig68698752019-05-21 09:01:43 +0200413 struct bio_vec bv;
Jens Axboed6d48192008-01-29 14:04:06 +0100414
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200415 if (!rq->bio)
Jens Axboe1e428072009-02-23 09:03:10 +0100416 return 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100417
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200418 switch (bio_op(rq->bio)) {
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800419 case REQ_OP_DISCARD:
420 case REQ_OP_SECURE_ERASE:
David Jefferya958937f2021-02-11 09:38:07 -0500421 if (queue_max_discard_segments(rq->q) > 1) {
422 struct bio *bio = rq->bio;
423
424 for_each_bio(bio)
425 nr_phys_segs++;
426 return nr_phys_segs;
427 }
428 return 1;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800429 case REQ_OP_WRITE_ZEROES:
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700430 return 0;
Bart Van Assche2d9b02b2022-07-14 11:06:30 -0700431 default:
432 break;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800433 }
Kent Overstreet5cb88502014-02-07 13:53:46 -0700434
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200435 rq_for_each_bvec(bv, rq, iter)
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400436 bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes,
Bart Van Assche708b25b2019-08-01 15:50:43 -0700437 UINT_MAX, UINT_MAX);
Jens Axboe1e428072009-02-23 09:03:10 +0100438 return nr_phys_segs;
439}
440
Ming Lei48d77272019-02-27 20:40:11 +0800441static inline struct scatterlist *blk_next_sg(struct scatterlist **sg,
Ming Lei862e5a52019-02-15 19:13:13 +0800442 struct scatterlist *sglist)
443{
444 if (!*sg)
445 return sglist;
446
447 /*
448 * If the driver previously mapped a shorter list, we could see a
449 * termination bit prematurely unless it fully inits the sg table
450 * on each mapping. We KNOW that there must be more entries here
451 * or the driver would be buggy, so force clear the termination bit
452 * to avoid doing a full sg_init_table() in drivers for each command.
453 */
454 sg_unmark_end(*sg);
455 return sg_next(*sg);
456}
457
458static unsigned blk_bvec_map_sg(struct request_queue *q,
459 struct bio_vec *bvec, struct scatterlist *sglist,
460 struct scatterlist **sg)
461{
462 unsigned nbytes = bvec->bv_len;
Christoph Hellwig8a96a0e2019-04-11 08:23:27 +0200463 unsigned nsegs = 0, total = 0;
Ming Lei862e5a52019-02-15 19:13:13 +0800464
465 while (nbytes > 0) {
Christoph Hellwig8a96a0e2019-04-11 08:23:27 +0200466 unsigned offset = bvec->bv_offset + total;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400467 unsigned len = min(get_max_segment_size(&q->limits,
468 bvec->bv_page, offset), nbytes);
Christoph Hellwigf9f76872019-04-19 08:56:24 +0200469 struct page *page = bvec->bv_page;
470
471 /*
472 * Unfortunately a fair number of drivers barf on scatterlists
473 * that have an offset larger than PAGE_SIZE, despite other
474 * subsystems dealing with that invariant just fine. For now
475 * stick to the legacy format where we never present those from
476 * the block layer, but the code below should be removed once
477 * these offenders (mostly MMC/SD drivers) are fixed.
478 */
479 page += (offset >> PAGE_SHIFT);
480 offset &= ~PAGE_MASK;
Ming Lei862e5a52019-02-15 19:13:13 +0800481
482 *sg = blk_next_sg(sg, sglist);
Christoph Hellwigf9f76872019-04-19 08:56:24 +0200483 sg_set_page(*sg, page, len, offset);
Ming Lei862e5a52019-02-15 19:13:13 +0800484
Christoph Hellwig8a96a0e2019-04-11 08:23:27 +0200485 total += len;
486 nbytes -= len;
Ming Lei862e5a52019-02-15 19:13:13 +0800487 nsegs++;
488 }
489
490 return nsegs;
491}
492
Ming Lei16e3e412019-03-17 18:01:11 +0800493static inline int __blk_bvec_map_sg(struct bio_vec bv,
494 struct scatterlist *sglist, struct scatterlist **sg)
495{
496 *sg = blk_next_sg(sg, sglist);
497 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
498 return 1;
499}
500
Ming Leif6970f82019-03-17 18:01:12 +0800501/* only try to merge bvecs into one sg if they are from two bios */
502static inline bool
503__blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec,
504 struct bio_vec *bvprv, struct scatterlist **sg)
Asias He963ab9e52012-08-02 23:42:03 +0200505{
506
507 int nbytes = bvec->bv_len;
508
Ming Leif6970f82019-03-17 18:01:12 +0800509 if (!*sg)
510 return false;
Asias He963ab9e52012-08-02 23:42:03 +0200511
Ming Leif6970f82019-03-17 18:01:12 +0800512 if ((*sg)->length + nbytes > queue_max_segment_size(q))
513 return false;
514
515 if (!biovec_phys_mergeable(q, bvprv, bvec))
516 return false;
517
518 (*sg)->length += nbytes;
519
520 return true;
Asias He963ab9e52012-08-02 23:42:03 +0200521}
522
Kent Overstreet5cb88502014-02-07 13:53:46 -0700523static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio,
524 struct scatterlist *sglist,
525 struct scatterlist **sg)
526{
Kees Cook3f649ab2020-06-03 13:09:38 -0700527 struct bio_vec bvec, bvprv = { NULL };
Kent Overstreet5cb88502014-02-07 13:53:46 -0700528 struct bvec_iter iter;
Christoph Hellwig38417462018-12-13 16:17:10 +0100529 int nsegs = 0;
Ming Leif6970f82019-03-17 18:01:12 +0800530 bool new_bio = false;
Kent Overstreet5cb88502014-02-07 13:53:46 -0700531
Ming Leif6970f82019-03-17 18:01:12 +0800532 for_each_bio(bio) {
533 bio_for_each_bvec(bvec, bio, iter) {
534 /*
535 * Only try to merge bvecs from two bios given we
536 * have done bio internal merge when adding pages
537 * to bio
538 */
539 if (new_bio &&
540 __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg))
541 goto next_bvec;
542
543 if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE)
544 nsegs += __blk_bvec_map_sg(bvec, sglist, sg);
545 else
546 nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg);
547 next_bvec:
548 new_bio = false;
549 }
Ming Leib21e11c2019-04-02 10:26:44 +0800550 if (likely(bio->bi_iter.bi_size)) {
551 bvprv = bvec;
552 new_bio = true;
553 }
Ming Leif6970f82019-03-17 18:01:12 +0800554 }
Kent Overstreet5cb88502014-02-07 13:53:46 -0700555
556 return nsegs;
557}
558
Jens Axboed6d48192008-01-29 14:04:06 +0100559/*
560 * map a request to scatterlist, return number of sg entries setup. Caller
561 * must make sure sg can hold rq->nr_phys_segments entries
562 */
Christoph Hellwig89de1502020-04-14 09:42:22 +0200563int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
564 struct scatterlist *sglist, struct scatterlist **last_sg)
Jens Axboed6d48192008-01-29 14:04:06 +0100565{
Kent Overstreet5cb88502014-02-07 13:53:46 -0700566 int nsegs = 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100567
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700568 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
Christoph Hellwig89de1502020-04-14 09:42:22 +0200569 nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, last_sg);
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700570 else if (rq->bio)
Christoph Hellwig89de1502020-04-14 09:42:22 +0200571 nsegs = __blk_bios_map_sg(q, rq->bio, sglist, last_sg);
FUJITA Tomonorif18573a2008-04-11 12:56:52 +0200572
Christoph Hellwig89de1502020-04-14 09:42:22 +0200573 if (*last_sg)
574 sg_mark_end(*last_sg);
Jens Axboed6d48192008-01-29 14:04:06 +0100575
Ming Lei12e57f52015-11-24 10:35:31 +0800576 /*
577 * Something must have been wrong if the figured number of
578 * segment is bigger than number of req's physical segments
579 */
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700580 WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
Ming Lei12e57f52015-11-24 10:35:31 +0800581
Jens Axboed6d48192008-01-29 14:04:06 +0100582 return nsegs;
583}
Christoph Hellwig89de1502020-04-14 09:42:22 +0200584EXPORT_SYMBOL(__blk_rq_map_sg);
Jens Axboed6d48192008-01-29 14:04:06 +0100585
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200586static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
587 sector_t offset)
588{
589 struct request_queue *q = rq->q;
Christoph Hellwigc8875192022-06-14 11:09:31 +0200590 unsigned int max_sectors;
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200591
592 if (blk_rq_is_passthrough(rq))
593 return q->limits.max_hw_sectors;
594
Christoph Hellwigc8875192022-06-14 11:09:31 +0200595 max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200596 if (!q->limits.chunk_sectors ||
597 req_op(rq) == REQ_OP_DISCARD ||
598 req_op(rq) == REQ_OP_SECURE_ERASE)
Christoph Hellwigc8875192022-06-14 11:09:31 +0200599 return max_sectors;
600 return min(max_sectors,
601 blk_chunk_sectors_left(offset, q->limits.chunk_sectors));
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200602}
603
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200604static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
605 unsigned int nr_phys_segs)
Jens Axboed6d48192008-01-29 14:04:06 +0100606{
Tejun Heo6b2b0452022-03-14 14:30:11 -1000607 if (!blk_cgroup_mergeable(req, bio))
608 goto no_merge;
609
Ming Lei2705dfb2021-06-28 10:33:12 +0800610 if (blk_integrity_merge_bio(req->q, req, bio) == false)
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200611 goto no_merge;
612
Ming Lei2705dfb2021-06-28 10:33:12 +0800613 /* discard request merge won't add new segment */
614 if (req_op(req) == REQ_OP_DISCARD)
615 return 1;
616
617 if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req))
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200618 goto no_merge;
Jens Axboed6d48192008-01-29 14:04:06 +0100619
620 /*
621 * This will form the start of a new hw segment. Bump both
622 * counters.
623 */
Jens Axboed6d48192008-01-29 14:04:06 +0100624 req->nr_phys_segments += nr_phys_segs;
625 return 1;
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200626
627no_merge:
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200628 req_set_nomerge(req->q, req);
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200629 return 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100630}
631
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200632int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
Jens Axboed6d48192008-01-29 14:04:06 +0100633{
Jens Axboe5e7c4272015-09-03 19:28:20 +0300634 if (req_gap_back_merge(req, bio))
635 return 0;
Sagi Grimberg7f39add2015-09-11 09:03:04 -0600636 if (blk_integrity_rq(req) &&
637 integrity_req_gap_back_merge(req, bio))
638 return 0;
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000639 if (!bio_crypt_ctx_back_mergeable(req, bio))
640 return 0;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400641 if (blk_rq_sectors(req) + bio_sectors(bio) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600642 blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200643 req_set_nomerge(req->q, req);
Jens Axboed6d48192008-01-29 14:04:06 +0100644 return 0;
645 }
Jens Axboed6d48192008-01-29 14:04:06 +0100646
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200647 return ll_new_hw_segment(req, bio, nr_segs);
Jens Axboed6d48192008-01-29 14:04:06 +0100648}
649
Christoph Hellwigeda5cc92020-10-06 09:07:19 +0200650static int ll_front_merge_fn(struct request *req, struct bio *bio,
651 unsigned int nr_segs)
Jens Axboed6d48192008-01-29 14:04:06 +0100652{
Jens Axboe5e7c4272015-09-03 19:28:20 +0300653 if (req_gap_front_merge(req, bio))
654 return 0;
Sagi Grimberg7f39add2015-09-11 09:03:04 -0600655 if (blk_integrity_rq(req) &&
656 integrity_req_gap_front_merge(req, bio))
657 return 0;
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000658 if (!bio_crypt_ctx_front_mergeable(req, bio))
659 return 0;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400660 if (blk_rq_sectors(req) + bio_sectors(bio) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600661 blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200662 req_set_nomerge(req->q, req);
Jens Axboed6d48192008-01-29 14:04:06 +0100663 return 0;
664 }
Jens Axboed6d48192008-01-29 14:04:06 +0100665
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200666 return ll_new_hw_segment(req, bio, nr_segs);
Jens Axboed6d48192008-01-29 14:04:06 +0100667}
668
Jens Axboe445251d2018-02-01 14:01:02 -0700669static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
670 struct request *next)
671{
672 unsigned short segments = blk_rq_nr_discard_segments(req);
673
674 if (segments >= queue_max_discard_segments(q))
675 goto no_merge;
676 if (blk_rq_sectors(req) + bio_sectors(next->bio) >
677 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
678 goto no_merge;
679
680 req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
681 return true;
682no_merge:
683 req_set_nomerge(q, req);
684 return false;
685}
686
Jens Axboed6d48192008-01-29 14:04:06 +0100687static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
688 struct request *next)
689{
690 int total_phys_segments;
Jens Axboed6d48192008-01-29 14:04:06 +0100691
Jens Axboe5e7c4272015-09-03 19:28:20 +0300692 if (req_gap_back_merge(req, next->bio))
Keith Busch854fbb92015-02-11 08:20:13 -0700693 return 0;
694
Jens Axboed6d48192008-01-29 14:04:06 +0100695 /*
696 * Will it become too large?
697 */
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400698 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600699 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
Jens Axboed6d48192008-01-29 14:04:06 +0100700 return 0;
701
702 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
Ming Lei943b40c2020-08-17 17:52:39 +0800703 if (total_phys_segments > blk_rq_get_max_segments(req))
Jens Axboed6d48192008-01-29 14:04:06 +0100704 return 0;
705
Tejun Heo6b2b0452022-03-14 14:30:11 -1000706 if (!blk_cgroup_mergeable(req, next->bio))
707 return 0;
708
Martin K. Petersen4eaf99b2014-09-26 19:20:06 -0400709 if (blk_integrity_merge_rq(q, req, next) == false)
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200710 return 0;
711
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000712 if (!bio_crypt_ctx_merge_rq(req, next))
713 return 0;
714
Jens Axboed6d48192008-01-29 14:04:06 +0100715 /* Merge is OK... */
716 req->nr_phys_segments = total_phys_segments;
Jens Axboed6d48192008-01-29 14:04:06 +0100717 return 1;
718}
719
Tejun Heo80a761f2009-07-03 17:48:17 +0900720/**
721 * blk_rq_set_mixed_merge - mark a request as mixed merge
722 * @rq: request to mark as mixed merge
723 *
724 * Description:
725 * @rq is about to be mixed merged. Make sure the attributes
726 * which can be mixed are set in each bio and mark @rq as mixed
727 * merged.
728 */
John Garrydc53d9e2024-03-25 08:35:01 +0000729static void blk_rq_set_mixed_merge(struct request *rq)
Tejun Heo80a761f2009-07-03 17:48:17 +0900730{
Bart Van Assche16458cf2022-07-14 11:06:32 -0700731 blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
Tejun Heo80a761f2009-07-03 17:48:17 +0900732 struct bio *bio;
733
Christoph Hellwige8064022016-10-20 15:12:13 +0200734 if (rq->rq_flags & RQF_MIXED_MERGE)
Tejun Heo80a761f2009-07-03 17:48:17 +0900735 return;
736
737 /*
738 * @rq will no longer represent mixable attributes for all the
739 * contained bios. It will just track those of the first one.
740 * Distributes the attributs to each bio.
741 */
742 for (bio = rq->bio; bio; bio = bio->bi_next) {
Jens Axboe1eff9d32016-08-05 15:35:16 -0600743 WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
744 (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
745 bio->bi_opf |= ff;
Tejun Heo80a761f2009-07-03 17:48:17 +0900746 }
Christoph Hellwige8064022016-10-20 15:12:13 +0200747 rq->rq_flags |= RQF_MIXED_MERGE;
Tejun Heo80a761f2009-07-03 17:48:17 +0900748}
749
Jens Axboef3ca7382023-02-16 19:39:15 -0700750static inline blk_opf_t bio_failfast(const struct bio *bio)
Ming Lei3ce6a112023-02-09 20:55:27 +0800751{
752 if (bio->bi_opf & REQ_RAHEAD)
753 return REQ_FAILFAST_MASK;
754
755 return bio->bi_opf & REQ_FAILFAST_MASK;
756}
757
758/*
759 * After we are marked as MIXED_MERGE, any new RA bio has to be updated
760 * as failfast, and request's failfast has to be updated in case of
761 * front merge.
762 */
763static inline void blk_update_mixed_merge(struct request *req,
764 struct bio *bio, bool front_merge)
765{
766 if (req->rq_flags & RQF_MIXED_MERGE) {
767 if (bio->bi_opf & REQ_RAHEAD)
768 bio->bi_opf |= REQ_FAILFAST_MASK;
769
770 if (front_merge) {
771 req->cmd_flags &= ~REQ_FAILFAST_MASK;
772 req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK;
773 }
774 }
775}
776
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200777static void blk_account_io_merge_request(struct request *req)
Jerome Marchand26308ea2009-03-27 10:31:51 +0100778{
779 if (blk_do_io_stat(req)) {
Mike Snitzer112f1582018-12-06 11:41:18 -0500780 part_stat_lock();
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200781 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
Jerome Marchand26308ea2009-03-27 10:31:51 +0100782 part_stat_unlock();
783 }
784}
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200785
Eric Biggerse96c0d82018-11-14 17:19:46 -0800786static enum elv_merge blk_try_req_merge(struct request *req,
787 struct request *next)
Jianchao Wang698404662018-10-27 19:52:14 +0800788{
789 if (blk_discard_mergable(req))
790 return ELEVATOR_DISCARD_MERGE;
791 else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
792 return ELEVATOR_BACK_MERGE;
793
794 return ELEVATOR_NO_MERGE;
795}
Jerome Marchand26308ea2009-03-27 10:31:51 +0100796
Jens Axboed6d48192008-01-29 14:04:06 +0100797/*
Jens Axboeb973cb72017-02-02 08:54:40 -0700798 * For non-mq, this has to be called with the request spinlock acquired.
799 * For mq with scheduling, the appropriate queue wide lock should be held.
Jens Axboed6d48192008-01-29 14:04:06 +0100800 */
Jens Axboeb973cb72017-02-02 08:54:40 -0700801static struct request *attempt_merge(struct request_queue *q,
802 struct request *req, struct request *next)
Jens Axboed6d48192008-01-29 14:04:06 +0100803{
804 if (!rq_mergeable(req) || !rq_mergeable(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700805 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100806
Christoph Hellwig288dab82016-06-09 16:00:36 +0200807 if (req_op(req) != req_op(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700808 return NULL;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400809
Christoph Hellwig79bb1db2021-11-26 13:17:59 +0100810 if (rq_data_dir(req) != rq_data_dir(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700811 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100812
Bart Van Assche44981352024-02-02 12:39:25 -0800813 /* Don't merge requests with different write hints. */
814 if (req->write_hint != next->write_hint)
815 return NULL;
816
Damien Le Moal668ffc02018-11-20 10:52:37 +0900817 if (req->ioprio != next->ioprio)
818 return NULL;
819
Jens Axboecb6934f2017-06-27 09:22:02 -0600820 /*
Jens Axboed6d48192008-01-29 14:04:06 +0100821 * If we are allowed to merge, then append bio list
822 * from next to rq and release next. merge_requests_fn
823 * will have updated segment counts, update sector
Jens Axboe445251d2018-02-01 14:01:02 -0700824 * counts here. Handle DISCARDs separately, as they
825 * have separate settings.
Jens Axboed6d48192008-01-29 14:04:06 +0100826 */
Jianchao Wang698404662018-10-27 19:52:14 +0800827
828 switch (blk_try_req_merge(req, next)) {
829 case ELEVATOR_DISCARD_MERGE:
Jens Axboe445251d2018-02-01 14:01:02 -0700830 if (!req_attempt_discard_merge(q, req, next))
831 return NULL;
Jianchao Wang698404662018-10-27 19:52:14 +0800832 break;
833 case ELEVATOR_BACK_MERGE:
834 if (!ll_merge_requests_fn(q, req, next))
835 return NULL;
836 break;
837 default:
Jens Axboeb973cb72017-02-02 08:54:40 -0700838 return NULL;
Jianchao Wang698404662018-10-27 19:52:14 +0800839 }
Jens Axboed6d48192008-01-29 14:04:06 +0100840
841 /*
Tejun Heo80a761f2009-07-03 17:48:17 +0900842 * If failfast settings disagree or any of the two is already
843 * a mixed merge, mark both as mixed before proceeding. This
844 * makes sure that all involved bios have mixable attributes
845 * set properly.
846 */
Christoph Hellwige8064022016-10-20 15:12:13 +0200847 if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
Tejun Heo80a761f2009-07-03 17:48:17 +0900848 (req->cmd_flags & REQ_FAILFAST_MASK) !=
849 (next->cmd_flags & REQ_FAILFAST_MASK)) {
850 blk_rq_set_mixed_merge(req);
851 blk_rq_set_mixed_merge(next);
852 }
853
854 /*
Omar Sandoval522a7772018-05-09 02:08:53 -0700855 * At this point we have either done a back merge or front merge. We
856 * need the smaller start_time_ns of the merged requests to be the
857 * current request for accounting purposes.
Jens Axboed6d48192008-01-29 14:04:06 +0100858 */
Omar Sandoval522a7772018-05-09 02:08:53 -0700859 if (next->start_time_ns < req->start_time_ns)
860 req->start_time_ns = next->start_time_ns;
Jens Axboed6d48192008-01-29 14:04:06 +0100861
862 req->biotail->bi_next = next->bio;
863 req->biotail = next->biotail;
864
Tejun Heoa2dec7b2009-05-07 22:24:44 +0900865 req->__data_len += blk_rq_bytes(next);
Jens Axboed6d48192008-01-29 14:04:06 +0100866
Ming Lei2a5cf352018-12-01 00:38:18 +0800867 if (!blk_discard_mergable(req))
Jens Axboe445251d2018-02-01 14:01:02 -0700868 elv_merge_requests(q, req, next);
Jens Axboed6d48192008-01-29 14:04:06 +0100869
Eric Biggers9cd1e562023-03-15 11:39:02 -0700870 blk_crypto_rq_put_keyslot(next);
871
Jerome Marchand42dad762009-04-22 14:01:49 +0200872 /*
873 * 'next' is going away, so update stats accordingly
874 */
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200875 blk_account_io_merge_request(next);
Jens Axboed6d48192008-01-29 14:04:06 +0100876
Christoph Hellwiga54895f2020-12-03 17:21:39 +0100877 trace_block_rq_merge(next);
Jan Karaf3bdc622020-06-17 15:58:23 +0200878
Jens Axboee4d750c2017-02-03 09:48:28 -0700879 /*
880 * ownership of bio passed from next to req, return 'next' for
881 * the caller to free
882 */
Boaz Harrosh1cd96c22009-03-24 12:35:07 +0100883 next->bio = NULL;
Jens Axboeb973cb72017-02-02 08:54:40 -0700884 return next;
Jens Axboed6d48192008-01-29 14:04:06 +0100885}
886
Christoph Hellwigeda5cc92020-10-06 09:07:19 +0200887static struct request *attempt_back_merge(struct request_queue *q,
888 struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100889{
890 struct request *next = elv_latter_request(q, rq);
891
892 if (next)
893 return attempt_merge(q, rq, next);
894
Jens Axboeb973cb72017-02-02 08:54:40 -0700895 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100896}
897
Christoph Hellwigeda5cc92020-10-06 09:07:19 +0200898static struct request *attempt_front_merge(struct request_queue *q,
899 struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100900{
901 struct request *prev = elv_former_request(q, rq);
902
903 if (prev)
904 return attempt_merge(q, prev, rq);
905
Jens Axboeb973cb72017-02-02 08:54:40 -0700906 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100907}
Jens Axboe5e84ea32011-03-21 10:14:27 +0100908
Jan Karafd2ef392021-06-23 11:36:34 +0200909/*
910 * Try to merge 'next' into 'rq'. Return true if the merge happened, false
911 * otherwise. The caller is responsible for freeing 'next' if the merge
912 * happened.
913 */
914bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
915 struct request *next)
Jens Axboe5e84ea32011-03-21 10:14:27 +0100916{
Jan Karafd2ef392021-06-23 11:36:34 +0200917 return attempt_merge(q, rq, next);
Jens Axboe5e84ea32011-03-21 10:14:27 +0100918}
Tejun Heo050c8ea2012-02-08 09:19:38 +0100919
920bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
921{
Martin K. Petersene2a60da2012-09-18 12:19:25 -0400922 if (!rq_mergeable(rq) || !bio_mergeable(bio))
Tejun Heo050c8ea2012-02-08 09:19:38 +0100923 return false;
924
Christoph Hellwig288dab82016-06-09 16:00:36 +0200925 if (req_op(rq) != bio_op(bio))
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400926 return false;
927
Tejun Heo050c8ea2012-02-08 09:19:38 +0100928 /* different data direction or already started, don't merge */
929 if (bio_data_dir(bio) != rq_data_dir(rq))
930 return false;
931
Tejun Heo6b2b0452022-03-14 14:30:11 -1000932 /* don't merge across cgroup boundaries */
933 if (!blk_cgroup_mergeable(rq, bio))
934 return false;
935
Tejun Heo050c8ea2012-02-08 09:19:38 +0100936 /* only merge integrity protected bio into ditto rq */
Martin K. Petersen4eaf99b2014-09-26 19:20:06 -0400937 if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100938 return false;
939
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000940 /* Only merge if the crypt contexts are compatible */
941 if (!bio_crypt_rq_ctx_compatible(rq, bio))
942 return false;
943
Bart Van Assche44981352024-02-02 12:39:25 -0800944 /* Don't merge requests with different write hints. */
945 if (rq->write_hint != bio->bi_write_hint)
946 return false;
947
Damien Le Moal668ffc02018-11-20 10:52:37 +0900948 if (rq->ioprio != bio_prio(bio))
949 return false;
950
Tejun Heo050c8ea2012-02-08 09:19:38 +0100951 return true;
952}
953
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100954enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100955{
Jianchao Wang698404662018-10-27 19:52:14 +0800956 if (blk_discard_mergable(rq))
Christoph Hellwig1e739732017-02-08 14:46:49 +0100957 return ELEVATOR_DISCARD_MERGE;
958 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100959 return ELEVATOR_BACK_MERGE;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700960 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100961 return ELEVATOR_FRONT_MERGE;
962 return ELEVATOR_NO_MERGE;
963}
Baolin Wang8e756372020-08-28 10:52:54 +0800964
965static void blk_account_io_merge_bio(struct request *req)
966{
967 if (!blk_do_io_stat(req))
968 return;
969
970 part_stat_lock();
971 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
972 part_stat_unlock();
973}
974
Christoph Hellwigeda5cc92020-10-06 09:07:19 +0200975enum bio_merge_status {
976 BIO_MERGE_OK,
977 BIO_MERGE_NONE,
978 BIO_MERGE_FAILED,
979};
980
981static enum bio_merge_status bio_attempt_back_merge(struct request *req,
982 struct bio *bio, unsigned int nr_segs)
Baolin Wang8e756372020-08-28 10:52:54 +0800983{
Ming Lei3ce6a112023-02-09 20:55:27 +0800984 const blk_opf_t ff = bio_failfast(bio);
Baolin Wang8e756372020-08-28 10:52:54 +0800985
986 if (!ll_back_merge_fn(req, bio, nr_segs))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +0800987 return BIO_MERGE_FAILED;
Baolin Wang8e756372020-08-28 10:52:54 +0800988
Christoph Hellwige8a676d2020-12-03 17:21:36 +0100989 trace_block_bio_backmerge(bio);
Baolin Wang8e756372020-08-28 10:52:54 +0800990 rq_qos_merge(req->q, req, bio);
991
992 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
993 blk_rq_set_mixed_merge(req);
994
Ming Lei3ce6a112023-02-09 20:55:27 +0800995 blk_update_mixed_merge(req, bio, false);
996
Baolin Wang8e756372020-08-28 10:52:54 +0800997 req->biotail->bi_next = bio;
998 req->biotail = bio;
999 req->__data_len += bio->bi_iter.bi_size;
1000
1001 bio_crypt_free_ctx(bio);
1002
1003 blk_account_io_merge_bio(req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001004 return BIO_MERGE_OK;
Baolin Wang8e756372020-08-28 10:52:54 +08001005}
1006
Christoph Hellwigeda5cc92020-10-06 09:07:19 +02001007static enum bio_merge_status bio_attempt_front_merge(struct request *req,
1008 struct bio *bio, unsigned int nr_segs)
Baolin Wang8e756372020-08-28 10:52:54 +08001009{
Ming Lei3ce6a112023-02-09 20:55:27 +08001010 const blk_opf_t ff = bio_failfast(bio);
Baolin Wang8e756372020-08-28 10:52:54 +08001011
1012 if (!ll_front_merge_fn(req, bio, nr_segs))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001013 return BIO_MERGE_FAILED;
Baolin Wang8e756372020-08-28 10:52:54 +08001014
Christoph Hellwige8a676d2020-12-03 17:21:36 +01001015 trace_block_bio_frontmerge(bio);
Baolin Wang8e756372020-08-28 10:52:54 +08001016 rq_qos_merge(req->q, req, bio);
1017
1018 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1019 blk_rq_set_mixed_merge(req);
1020
Ming Lei3ce6a112023-02-09 20:55:27 +08001021 blk_update_mixed_merge(req, bio, true);
1022
Baolin Wang8e756372020-08-28 10:52:54 +08001023 bio->bi_next = req->bio;
1024 req->bio = bio;
1025
1026 req->__sector = bio->bi_iter.bi_sector;
1027 req->__data_len += bio->bi_iter.bi_size;
1028
1029 bio_crypt_do_front_merge(req, bio);
1030
1031 blk_account_io_merge_bio(req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001032 return BIO_MERGE_OK;
Baolin Wang8e756372020-08-28 10:52:54 +08001033}
1034
Christoph Hellwigeda5cc92020-10-06 09:07:19 +02001035static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
1036 struct request *req, struct bio *bio)
Baolin Wang8e756372020-08-28 10:52:54 +08001037{
1038 unsigned short segments = blk_rq_nr_discard_segments(req);
1039
1040 if (segments >= queue_max_discard_segments(q))
1041 goto no_merge;
1042 if (blk_rq_sectors(req) + bio_sectors(bio) >
1043 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1044 goto no_merge;
1045
1046 rq_qos_merge(q, req, bio);
1047
1048 req->biotail->bi_next = bio;
1049 req->biotail = bio;
1050 req->__data_len += bio->bi_iter.bi_size;
1051 req->nr_phys_segments = segments + 1;
1052
1053 blk_account_io_merge_bio(req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001054 return BIO_MERGE_OK;
Baolin Wang8e756372020-08-28 10:52:54 +08001055no_merge:
1056 req_set_nomerge(q, req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001057 return BIO_MERGE_FAILED;
1058}
1059
1060static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
1061 struct request *rq,
1062 struct bio *bio,
1063 unsigned int nr_segs,
1064 bool sched_allow_merge)
1065{
1066 if (!blk_rq_merge_ok(rq, bio))
1067 return BIO_MERGE_NONE;
1068
1069 switch (blk_try_merge(rq, bio)) {
1070 case ELEVATOR_BACK_MERGE:
Baolin Wang265600b2020-09-02 09:45:25 +08001071 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001072 return bio_attempt_back_merge(rq, bio, nr_segs);
1073 break;
1074 case ELEVATOR_FRONT_MERGE:
Baolin Wang265600b2020-09-02 09:45:25 +08001075 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001076 return bio_attempt_front_merge(rq, bio, nr_segs);
1077 break;
1078 case ELEVATOR_DISCARD_MERGE:
1079 return bio_attempt_discard_merge(q, rq, bio);
1080 default:
1081 return BIO_MERGE_NONE;
1082 }
1083
1084 return BIO_MERGE_FAILED;
Baolin Wang8e756372020-08-28 10:52:54 +08001085}
1086
1087/**
1088 * blk_attempt_plug_merge - try to merge with %current's plugged list
1089 * @q: request_queue new bio is being queued at
1090 * @bio: new bio being queued
1091 * @nr_segs: number of segments in @bio
Jens Axboe87c037d2021-10-18 10:07:09 -06001092 * from the passed in @q already in the plug list
Baolin Wang8e756372020-08-28 10:52:54 +08001093 *
Jens Axboed38a9c02021-10-14 07:24:07 -06001094 * Determine whether @bio being queued on @q can be merged with the previous
1095 * request on %current's plugged list. Returns %true if merge was successful,
Baolin Wang8e756372020-08-28 10:52:54 +08001096 * otherwise %false.
1097 *
1098 * Plugging coalesces IOs from the same issuer for the same purpose without
1099 * going through @q->queue_lock. As such it's more of an issuing mechanism
1100 * than scheduling, and the request, while may have elvpriv data, is not
1101 * added on the elevator at this point. In addition, we don't have
1102 * reliable access to the elevator outside queue lock. Only check basic
1103 * merging parameters without querying the elevator.
1104 *
1105 * Caller must ensure !blk_queue_nomerges(q) beforehand.
1106 */
1107bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
Christoph Hellwig0c5bcc92021-11-23 17:04:41 +01001108 unsigned int nr_segs)
Baolin Wang8e756372020-08-28 10:52:54 +08001109{
1110 struct blk_plug *plug;
1111 struct request *rq;
Baolin Wang8e756372020-08-28 10:52:54 +08001112
Christoph Hellwig6deacb32022-07-06 09:03:38 +02001113 plug = blk_mq_plug(bio);
Jens Axboebc490f82021-10-18 10:12:12 -06001114 if (!plug || rq_list_empty(plug->mq_list))
Baolin Wang8e756372020-08-28 10:52:54 +08001115 return false;
1116
Jens Axboe5b205072022-03-11 10:21:43 -07001117 rq_list_for_each(&plug->mq_list, rq) {
1118 if (rq->q == q) {
1119 if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
1120 BIO_MERGE_OK)
1121 return true;
1122 break;
1123 }
1124
1125 /*
1126 * Only keep iterating plug list for merges if we have multiple
1127 * queues
1128 */
1129 if (!plug->multiple_queues)
1130 break;
Baolin Wang8e756372020-08-28 10:52:54 +08001131 }
Baolin Wang8e756372020-08-28 10:52:54 +08001132 return false;
1133}
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001134
1135/*
1136 * Iterate list of requests and see if we can merge this bio with any
1137 * of them.
1138 */
1139bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
1140 struct bio *bio, unsigned int nr_segs)
1141{
1142 struct request *rq;
1143 int checked = 8;
1144
1145 list_for_each_entry_reverse(rq, list, queuelist) {
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001146 if (!checked--)
1147 break;
1148
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001149 switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {
1150 case BIO_MERGE_NONE:
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001151 continue;
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001152 case BIO_MERGE_OK:
1153 return true;
1154 case BIO_MERGE_FAILED:
1155 return false;
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001156 }
1157
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001158 }
1159
1160 return false;
1161}
1162EXPORT_SYMBOL_GPL(blk_bio_list_merge);
Christoph Hellwigeda5cc92020-10-06 09:07:19 +02001163
1164bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
1165 unsigned int nr_segs, struct request **merged_request)
1166{
1167 struct request *rq;
1168
1169 switch (elv_merge(q, &rq, bio)) {
1170 case ELEVATOR_BACK_MERGE:
1171 if (!blk_mq_sched_allow_merge(q, rq, bio))
1172 return false;
1173 if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1174 return false;
1175 *merged_request = attempt_back_merge(q, rq);
1176 if (!*merged_request)
1177 elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
1178 return true;
1179 case ELEVATOR_FRONT_MERGE:
1180 if (!blk_mq_sched_allow_merge(q, rq, bio))
1181 return false;
1182 if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1183 return false;
1184 *merged_request = attempt_front_merge(q, rq);
1185 if (!*merged_request)
1186 elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
1187 return true;
1188 case ELEVATOR_DISCARD_MERGE:
1189 return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
1190 default:
1191 return false;
1192 }
1193}
1194EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);