blob: e0b28e9298c9fb5315225e6a1daacc50ba4786fa [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Jens Axboed6d48192008-01-29 14:04:06 +01002/*
3 * Functions related to segment and merge handling
4 */
5#include <linux/kernel.h>
6#include <linux/module.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>
Christoph Hellwigfe45e632021-09-20 14:33:27 +02009#include <linux/blk-integrity.h>
Jens Axboed6d48192008-01-29 14:04:06 +010010#include <linux/scatterlist.h>
Christoph Hellwig82d981d2021-11-23 19:53:12 +010011#include <linux/part_stat.h>
Tejun Heo6b2b0452022-03-14 14:30:11 -100012#include <linux/blk-cgroup.h>
Jens Axboed6d48192008-01-29 14:04:06 +010013
Mike Krinkincda22642015-12-03 17:32:30 +030014#include <trace/events/block.h>
15
Jens Axboed6d48192008-01-29 14:04:06 +010016#include "blk.h"
Christoph Hellwig2aa77452021-11-23 19:53:08 +010017#include "blk-mq-sched.h"
Baolin Wang8e756372020-08-28 10:52:54 +080018#include "blk-rq-qos.h"
Jens Axboea7b36ee2021-10-05 09:11:56 -060019#include "blk-throttle.h"
Jens Axboed6d48192008-01-29 14:04:06 +010020
Christoph Hellwigff18d772021-10-12 18:18:03 +020021static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
22{
23 *bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
24}
25
26static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
27{
28 struct bvec_iter iter = bio->bi_iter;
29 int idx;
30
31 bio_get_first_bvec(bio, bv);
32 if (bv->bv_len == bio->bi_iter.bi_size)
33 return; /* this bio only has a single bvec */
34
35 bio_advance_iter(bio, &iter, iter.bi_size);
36
37 if (!iter.bi_bvec_done)
38 idx = iter.bi_idx - 1;
39 else /* in the middle of bvec */
40 idx = iter.bi_idx;
41
42 *bv = bio->bi_io_vec[idx];
43
44 /*
45 * iter.bi_bvec_done records actual length of the last bvec
46 * if this bio ends in the middle of one io vector
47 */
48 if (iter.bi_bvec_done)
49 bv->bv_len = iter.bi_bvec_done;
50}
51
Christoph Hellwige9907002018-09-24 09:43:48 +020052static inline bool bio_will_gap(struct request_queue *q,
53 struct request *prev_rq, struct bio *prev, struct bio *next)
54{
55 struct bio_vec pb, nb;
56
57 if (!bio_has_data(prev) || !queue_virt_boundary(q))
58 return false;
59
60 /*
61 * Don't merge if the 1st bio starts with non-zero offset, otherwise it
62 * is quite difficult to respect the sg gap limit. We work hard to
63 * merge a huge number of small single bios in case of mkfs.
64 */
65 if (prev_rq)
66 bio_get_first_bvec(prev_rq->bio, &pb);
67 else
68 bio_get_first_bvec(prev, &pb);
Johannes Thumshirndf376b22018-11-07 14:58:14 +010069 if (pb.bv_offset & queue_virt_boundary(q))
Christoph Hellwige9907002018-09-24 09:43:48 +020070 return true;
71
72 /*
73 * We don't need to worry about the situation that the merged segment
74 * ends in unaligned virt boundary:
75 *
76 * - if 'pb' ends aligned, the merged segment ends aligned
77 * - if 'pb' ends unaligned, the next bio must include
78 * one single bvec of 'nb', otherwise the 'nb' can't
79 * merge with 'pb'
80 */
81 bio_get_last_bvec(prev, &pb);
82 bio_get_first_bvec(next, &nb);
Christoph Hellwig200a9af2019-05-21 09:01:42 +020083 if (biovec_phys_mergeable(q, &pb, &nb))
Christoph Hellwige9907002018-09-24 09:43:48 +020084 return false;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -040085 return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset);
Christoph Hellwige9907002018-09-24 09:43:48 +020086}
87
88static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
89{
90 return bio_will_gap(req->q, req, req->biotail, bio);
91}
92
93static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
94{
95 return bio_will_gap(req->q, NULL, bio, req->bio);
96}
97
Christoph Hellwigb6dc6192022-07-27 12:22:59 -040098/*
99 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
100 * is defined as 'unsigned int', meantime it has to be aligned to with the
101 * logical block size, which is the minimum accepted unit by hardware.
102 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700103static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim)
Christoph Hellwigb6dc6192022-07-27 12:22:59 -0400104{
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400105 return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT;
Christoph Hellwigb6dc6192022-07-27 12:22:59 -0400106}
107
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200108static struct bio *bio_submit_split(struct bio *bio, int split_sectors)
109{
John Garry6eb09682024-11-11 11:21:47 +0000110 if (unlikely(split_sectors < 0))
111 goto error;
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200112
113 if (split_sectors) {
114 struct bio *split;
115
116 split = bio_split(bio, split_sectors, GFP_NOIO,
117 &bio->bi_bdev->bd_disk->bio_split);
John Garry6eb09682024-11-11 11:21:47 +0000118 if (IS_ERR(split)) {
119 split_sectors = PTR_ERR(split);
120 goto error;
121 }
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200122 split->bi_opf |= REQ_NOMERGE;
123 blkcg_bio_issue_init(split);
124 bio_chain(split, bio);
125 trace_block_split(split, bio->bi_iter.bi_sector);
126 WARN_ON_ONCE(bio_zone_write_plugging(bio));
127 submit_bio_noacct(bio);
128 return split;
129 }
130
131 return bio;
John Garry6eb09682024-11-11 11:21:47 +0000132error:
133 bio->bi_status = errno_to_blk_status(split_sectors);
134 bio_endio(bio);
135 return NULL;
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200136}
137
138struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
139 unsigned *nsegs)
Kent Overstreet54efd502015-04-23 22:37:18 -0700140{
141 unsigned int max_discard_sectors, granularity;
Kent Overstreet54efd502015-04-23 22:37:18 -0700142 sector_t tmp;
143 unsigned split_sectors;
144
Ming Leibdced432015-10-20 23:13:52 +0800145 *nsegs = 1;
146
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400147 granularity = max(lim->discard_granularity >> 9, 1U);
Kent Overstreet54efd502015-04-23 22:37:18 -0700148
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400149 max_discard_sectors =
150 min(lim->max_discard_sectors, bio_allowed_max_sectors(lim));
Kent Overstreet54efd502015-04-23 22:37:18 -0700151 max_discard_sectors -= max_discard_sectors % granularity;
Christoph Hellwig928a5dd2023-12-28 07:55:37 +0000152 if (unlikely(!max_discard_sectors))
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200153 return bio;
Kent Overstreet54efd502015-04-23 22:37:18 -0700154
155 if (bio_sectors(bio) <= max_discard_sectors)
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200156 return bio;
Kent Overstreet54efd502015-04-23 22:37:18 -0700157
158 split_sectors = max_discard_sectors;
159
160 /*
161 * If the next starting sector would be misaligned, stop the discard at
162 * the previous aligned sector.
163 */
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400164 tmp = bio->bi_iter.bi_sector + split_sectors -
165 ((lim->discard_alignment >> 9) % granularity);
Kent Overstreet54efd502015-04-23 22:37:18 -0700166 tmp = sector_div(tmp, granularity);
167
168 if (split_sectors > tmp)
169 split_sectors -= tmp;
170
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200171 return bio_submit_split(bio, split_sectors);
Kent Overstreet54efd502015-04-23 22:37:18 -0700172}
173
John Garry9da3d1e2024-06-20 12:53:54 +0000174static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim,
175 bool is_atomic)
John Garryf70167a2024-06-20 12:53:51 +0000176{
John Garry9da3d1e2024-06-20 12:53:54 +0000177 /*
178 * chunk_sectors must be a multiple of atomic_write_boundary_sectors if
179 * both non-zero.
180 */
181 if (is_atomic && lim->atomic_write_boundary_sectors)
182 return lim->atomic_write_boundary_sectors;
183
John Garryf70167a2024-06-20 12:53:51 +0000184 return lim->chunk_sectors;
185}
186
Bart Van Assche9cc51692019-08-01 15:50:44 -0700187/*
188 * Return the maximum number of sectors from the start of a bio that may be
189 * submitted as a single request to a block device. If enough sectors remain,
190 * align the end to the physical block size. Otherwise align the end to the
191 * logical block size. This approach minimizes the number of non-aligned
192 * requests that are submitted to a block device if the start of a bio is not
193 * aligned to a physical block boundary.
194 */
Christoph Hellwig5a978062022-07-27 12:22:55 -0400195static inline unsigned get_max_io_size(struct bio *bio,
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700196 const struct queue_limits *lim)
Ming Leid0e5fbb2016-01-23 08:05:33 +0800197{
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400198 unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
199 unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
John Garry9da3d1e2024-06-20 12:53:54 +0000200 bool is_atomic = bio->bi_opf & REQ_ATOMIC;
201 unsigned boundary_sectors = blk_boundary_sectors(lim, is_atomic);
202 unsigned max_sectors, start, end;
203
204 /*
205 * We ignore lim->max_sectors for atomic writes because it may less
206 * than the actual bio size, which we cannot tolerate.
207 */
Christoph Hellwig60dc5ea2024-11-04 07:26:29 +0100208 if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
209 max_sectors = lim->max_write_zeroes_sectors;
210 else if (is_atomic)
John Garry9da3d1e2024-06-20 12:53:54 +0000211 max_sectors = lim->atomic_write_max_sectors;
212 else
213 max_sectors = lim->max_sectors;
Ming Leid0e5fbb2016-01-23 08:05:33 +0800214
John Garryf70167a2024-06-20 12:53:51 +0000215 if (boundary_sectors) {
Christoph Hellwigefef7392022-06-14 11:09:33 +0200216 max_sectors = min(max_sectors,
John Garryf70167a2024-06-20 12:53:51 +0000217 blk_boundary_sectors_left(bio->bi_iter.bi_sector,
218 boundary_sectors));
Christoph Hellwigefef7392022-06-14 11:09:33 +0200219 }
Ming Leid0e5fbb2016-01-23 08:05:33 +0800220
Christoph Hellwig84613be2022-06-14 11:09:32 +0200221 start = bio->bi_iter.bi_sector & (pbs - 1);
222 end = (start + max_sectors) & ~(pbs - 1);
223 if (end > start)
224 return end - start;
225 return max_sectors & ~(lbs - 1);
Ming Leid0e5fbb2016-01-23 08:05:33 +0800226}
227
Bart Van Assche95465312022-10-25 12:17:55 -0700228/**
229 * get_max_segment_size() - maximum number of bytes to add as a single segment
230 * @lim: Request queue limits.
Christoph Hellwig09595e02024-07-06 09:52:18 +0200231 * @paddr: address of the range to add
Chaitanya Kulkarni0ffc46e2024-07-08 21:54:32 -0700232 * @len: maximum length available to add at @paddr
Bart Van Assche95465312022-10-25 12:17:55 -0700233 *
Christoph Hellwig09595e02024-07-06 09:52:18 +0200234 * Returns the maximum number of bytes of the range starting at @paddr that can
235 * be added to a single segment.
Bart Van Assche95465312022-10-25 12:17:55 -0700236 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700237static inline unsigned get_max_segment_size(const struct queue_limits *lim,
Christoph Hellwig09595e02024-07-06 09:52:18 +0200238 phys_addr_t paddr, unsigned int len)
Ming Leidcebd752019-02-15 19:13:12 +0800239{
Ming Lei4a2f704e2020-01-11 20:57:43 +0800240 /*
Bart Van Assche95465312022-10-25 12:17:55 -0700241 * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1
242 * after having calculated the minimum.
Ming Lei4a2f704e2020-01-11 20:57:43 +0800243 */
Christoph Hellwig09595e02024-07-06 09:52:18 +0200244 return min_t(unsigned long, len,
245 min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr),
246 (unsigned long)lim->max_segment_size - 1) + 1);
Ming Leidcebd752019-02-15 19:13:12 +0800247}
248
Bart Van Assche708b25b2019-08-01 15:50:43 -0700249/**
250 * bvec_split_segs - verify whether or not a bvec should be split in the middle
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400251 * @lim: [in] queue limits to split based on
Bart Van Assche708b25b2019-08-01 15:50:43 -0700252 * @bv: [in] bvec to examine
253 * @nsegs: [in,out] Number of segments in the bio being built. Incremented
254 * by the number of segments from @bv that may be appended to that
255 * bio without exceeding @max_segs
Keith Busch67927d22022-06-10 12:58:25 -0700256 * @bytes: [in,out] Number of bytes in the bio being built. Incremented
257 * by the number of bytes from @bv that may be appended to that
258 * bio without exceeding @max_bytes
Bart Van Assche708b25b2019-08-01 15:50:43 -0700259 * @max_segs: [in] upper bound for *@nsegs
Keith Busch67927d22022-06-10 12:58:25 -0700260 * @max_bytes: [in] upper bound for *@bytes
Bart Van Assche708b25b2019-08-01 15:50:43 -0700261 *
262 * When splitting a bio, it can happen that a bvec is encountered that is too
263 * big to fit in a single segment and hence that it has to be split in the
264 * middle. This function verifies whether or not that should happen. The value
265 * %true is returned if and only if appending the entire @bv to a bio with
266 * *@nsegs segments and *@sectors sectors would make that bio unacceptable for
267 * the block driver.
Ming Leidcebd752019-02-15 19:13:12 +0800268 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700269static bool bvec_split_segs(const struct queue_limits *lim,
270 const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes,
271 unsigned max_segs, unsigned max_bytes)
Ming Leidcebd752019-02-15 19:13:12 +0800272{
Keith Busch67927d22022-06-10 12:58:25 -0700273 unsigned max_len = min(max_bytes, UINT_MAX) - *bytes;
Bart Van Assche708b25b2019-08-01 15:50:43 -0700274 unsigned len = min(bv->bv_len, max_len);
Ming Leidcebd752019-02-15 19:13:12 +0800275 unsigned total_len = 0;
Bart Van Asscheff9811b2019-08-01 15:50:42 -0700276 unsigned seg_size = 0;
Ming Leidcebd752019-02-15 19:13:12 +0800277
Bart Van Asscheff9811b2019-08-01 15:50:42 -0700278 while (len && *nsegs < max_segs) {
Christoph Hellwig09595e02024-07-06 09:52:18 +0200279 seg_size = get_max_segment_size(lim, bvec_phys(bv) + total_len, len);
Ming Leidcebd752019-02-15 19:13:12 +0800280
Bart Van Asscheff9811b2019-08-01 15:50:42 -0700281 (*nsegs)++;
Ming Leidcebd752019-02-15 19:13:12 +0800282 total_len += seg_size;
283 len -= seg_size;
284
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400285 if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
Ming Leidcebd752019-02-15 19:13:12 +0800286 break;
287 }
288
Keith Busch67927d22022-06-10 12:58:25 -0700289 *bytes += total_len;
Ming Leidcebd752019-02-15 19:13:12 +0800290
Bart Van Assche708b25b2019-08-01 15:50:43 -0700291 /* tell the caller to split the bvec if it is too big to fit */
292 return len > 0 || bv->bv_len > max_len;
Ming Leidcebd752019-02-15 19:13:12 +0800293}
294
Christoph Hellwig7ecd2cd2024-11-04 07:26:30 +0100295static unsigned int bio_split_alignment(struct bio *bio,
296 const struct queue_limits *lim)
297{
298 if (op_is_write(bio_op(bio)) && lim->zone_write_granularity)
299 return lim->zone_write_granularity;
300 return lim->logical_block_size;
301}
302
Bart Van Asschedad77582019-08-01 15:50:41 -0700303/**
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200304 * bio_split_rw_at - check if and where to split a read/write bio
Bart Van Asschedad77582019-08-01 15:50:41 -0700305 * @bio: [in] bio to be split
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400306 * @lim: [in] queue limits to split based on
Bart Van Asschedad77582019-08-01 15:50:41 -0700307 * @segs: [out] number of segments in the bio with the first half of the sectors
Christoph Hellwiga85b3632022-07-27 12:22:58 -0400308 * @max_bytes: [in] maximum number of bytes per bio
Bart Van Asschedad77582019-08-01 15:50:41 -0700309 *
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200310 * Find out if @bio needs to be split to fit the queue limits in @lim and a
311 * maximum size of @max_bytes. Returns a negative error number if @bio can't be
312 * split, 0 if the bio doesn't have to be split, or a positive sector offset if
313 * @bio needs to be split.
Bart Van Asschedad77582019-08-01 15:50:41 -0700314 */
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200315int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim,
316 unsigned *segs, unsigned max_bytes)
Kent Overstreet54efd502015-04-23 22:37:18 -0700317{
Jens Axboe5014c312015-09-02 16:46:02 -0600318 struct bio_vec bv, bvprv, *bvprvp = NULL;
Kent Overstreet54efd502015-04-23 22:37:18 -0700319 struct bvec_iter iter;
Keith Busch67927d22022-06-10 12:58:25 -0700320 unsigned nsegs = 0, bytes = 0;
Kent Overstreet54efd502015-04-23 22:37:18 -0700321
Ming Leidcebd752019-02-15 19:13:12 +0800322 bio_for_each_bvec(bv, bio, iter) {
Kent Overstreet54efd502015-04-23 22:37:18 -0700323 /*
324 * If the queue doesn't support SG gaps and adding this
325 * offset would create a gap, disallow it.
326 */
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400327 if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
Kent Overstreet54efd502015-04-23 22:37:18 -0700328 goto split;
329
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400330 if (nsegs < lim->max_segments &&
Keith Busch67927d22022-06-10 12:58:25 -0700331 bytes + bv.bv_len <= max_bytes &&
Bart Van Assche708b25b2019-08-01 15:50:43 -0700332 bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
333 nsegs++;
Keith Busch67927d22022-06-10 12:58:25 -0700334 bytes += bv.bv_len;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400335 } else {
336 if (bvec_split_segs(lim, &bv, &nsegs, &bytes,
337 lim->max_segments, max_bytes))
338 goto split;
Keith Busche36f6202016-01-12 15:08:39 -0700339 }
340
Kent Overstreet54efd502015-04-23 22:37:18 -0700341 bvprv = bv;
Ming Lei578270b2015-11-24 10:35:29 +0800342 bvprvp = &bvprv;
Kent Overstreet54efd502015-04-23 22:37:18 -0700343 }
344
Christoph Hellwigd6270652019-06-06 12:29:03 +0200345 *segs = nsegs;
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200346 return 0;
Kent Overstreet54efd502015-04-23 22:37:18 -0700347split:
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200348 if (bio->bi_opf & REQ_ATOMIC)
349 return -EINVAL;
350
Jens Axboe9cea62b2023-01-04 08:52:06 -0700351 /*
352 * We can't sanely support splitting for a REQ_NOWAIT bio. End it
353 * with EAGAIN if splitting is required and return an error pointer.
354 */
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200355 if (bio->bi_opf & REQ_NOWAIT)
356 return -EAGAIN;
Jens Axboe9cea62b2023-01-04 08:52:06 -0700357
Ming Leibdced432015-10-20 23:13:52 +0800358 *segs = nsegs;
Jeffle Xucc29e1b2020-11-26 17:18:52 +0800359
360 /*
Keith Busch67927d22022-06-10 12:58:25 -0700361 * Individual bvecs might not be logical block aligned. Round down the
362 * split size so that each bio is properly block size aligned, even if
363 * we do not use the full hardware limits.
364 */
Christoph Hellwig7ecd2cd2024-11-04 07:26:30 +0100365 bytes = ALIGN_DOWN(bytes, bio_split_alignment(bio, lim));
Keith Busch67927d22022-06-10 12:58:25 -0700366
367 /*
Jeffle Xucc29e1b2020-11-26 17:18:52 +0800368 * Bio splitting may cause subtle trouble such as hang when doing sync
369 * iopoll in direct IO routine. Given performance gain of iopoll for
370 * big IO can be trival, disable iopoll when split needed.
371 */
Christoph Hellwig6ce913f2021-10-12 13:12:21 +0200372 bio_clear_polled(bio);
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200373 return bytes >> SECTOR_SHIFT;
Kent Overstreet54efd502015-04-23 22:37:18 -0700374}
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200375EXPORT_SYMBOL_GPL(bio_split_rw_at);
Kent Overstreet54efd502015-04-23 22:37:18 -0700376
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200377struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
378 unsigned *nr_segs)
Kent Overstreet54efd502015-04-23 22:37:18 -0700379{
Christoph Hellwigb35243a2024-08-26 19:37:54 +0200380 return bio_submit_split(bio,
381 bio_split_rw_at(bio, lim, nr_segs,
382 get_max_io_size(bio, lim) << SECTOR_SHIFT));
Kent Overstreet54efd502015-04-23 22:37:18 -0700383}
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200384
Christoph Hellwig1e8a7f62024-08-26 19:37:56 +0200385/*
386 * REQ_OP_ZONE_APPEND bios must never be split by the block layer.
387 *
388 * But we want the nr_segs calculation provided by bio_split_rw_at, and having
389 * a good sanity check that the submitter built the bio correctly is nice to
390 * have as well.
391 */
392struct bio *bio_split_zone_append(struct bio *bio,
393 const struct queue_limits *lim, unsigned *nr_segs)
394{
Christoph Hellwig1e8a7f62024-08-26 19:37:56 +0200395 int split_sectors;
396
397 split_sectors = bio_split_rw_at(bio, lim, nr_segs,
Christoph Hellwig559218d2024-11-08 16:46:51 +0100398 lim->max_zone_append_sectors << SECTOR_SHIFT);
Christoph Hellwig1e8a7f62024-08-26 19:37:56 +0200399 if (WARN_ON_ONCE(split_sectors > 0))
400 split_sectors = -EINVAL;
401 return bio_submit_split(bio, split_sectors);
402}
403
Christoph Hellwig60dc5ea2024-11-04 07:26:29 +0100404struct bio *bio_split_write_zeroes(struct bio *bio,
405 const struct queue_limits *lim, unsigned *nsegs)
406{
407 unsigned int max_sectors = get_max_io_size(bio, lim);
408
409 *nsegs = 0;
410
411 /*
412 * An unset limit should normally not happen, as bio submission is keyed
413 * off having a non-zero limit. But SCSI can clear the limit in the
414 * I/O completion handler, and we can race and see this. Splitting to a
415 * zero limit obviously doesn't make sense, so band-aid it here.
416 */
417 if (!max_sectors)
418 return bio;
419 if (bio_sectors(bio) <= max_sectors)
420 return bio;
421 return bio_submit_split(bio, max_sectors);
422}
423
Bart Van Asschedad77582019-08-01 15:50:41 -0700424/**
Christoph Hellwig5a978062022-07-27 12:22:55 -0400425 * bio_split_to_limits - split a bio to fit the queue limits
426 * @bio: bio to be split
Bart Van Asschedad77582019-08-01 15:50:41 -0700427 *
Christoph Hellwig5a978062022-07-27 12:22:55 -0400428 * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and
429 * if so split off a bio fitting the limits from the beginning of @bio and
430 * return it. @bio is shortened to the remainder and re-submitted.
431 *
432 * The split bio is allocated from @q->bio_split, which is provided by the
433 * block layer.
Bart Van Asschedad77582019-08-01 15:50:41 -0700434 */
Christoph Hellwig5a978062022-07-27 12:22:55 -0400435struct bio *bio_split_to_limits(struct bio *bio)
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200436{
437 unsigned int nr_segs;
438
Christoph Hellwig2f5a65e2024-10-29 15:19:37 +0100439 return __bio_split_to_limits(bio, bdev_limits(bio->bi_bdev), &nr_segs);
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200440}
Christoph Hellwig5a978062022-07-27 12:22:55 -0400441EXPORT_SYMBOL(bio_split_to_limits);
Kent Overstreet54efd502015-04-23 22:37:18 -0700442
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200443unsigned int blk_recalc_rq_segments(struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100444{
Christoph Hellwig68698752019-05-21 09:01:43 +0200445 unsigned int nr_phys_segs = 0;
Keith Busch67927d22022-06-10 12:58:25 -0700446 unsigned int bytes = 0;
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200447 struct req_iterator iter;
Christoph Hellwig68698752019-05-21 09:01:43 +0200448 struct bio_vec bv;
Jens Axboed6d48192008-01-29 14:04:06 +0100449
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200450 if (!rq->bio)
Jens Axboe1e428072009-02-23 09:03:10 +0100451 return 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100452
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200453 switch (bio_op(rq->bio)) {
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800454 case REQ_OP_DISCARD:
455 case REQ_OP_SECURE_ERASE:
David Jefferya958937f2021-02-11 09:38:07 -0500456 if (queue_max_discard_segments(rq->q) > 1) {
457 struct bio *bio = rq->bio;
458
459 for_each_bio(bio)
460 nr_phys_segs++;
461 return nr_phys_segs;
462 }
463 return 1;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800464 case REQ_OP_WRITE_ZEROES:
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700465 return 0;
Bart Van Assche2d9b02b2022-07-14 11:06:30 -0700466 default:
467 break;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800468 }
Kent Overstreet5cb88502014-02-07 13:53:46 -0700469
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200470 rq_for_each_bvec(bv, rq, iter)
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400471 bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes,
Bart Van Assche708b25b2019-08-01 15:50:43 -0700472 UINT_MAX, UINT_MAX);
Jens Axboe1e428072009-02-23 09:03:10 +0100473 return nr_phys_segs;
474}
475
Ming Lei48d77272019-02-27 20:40:11 +0800476static inline struct scatterlist *blk_next_sg(struct scatterlist **sg,
Ming Lei862e5a52019-02-15 19:13:13 +0800477 struct scatterlist *sglist)
478{
479 if (!*sg)
480 return sglist;
481
482 /*
483 * If the driver previously mapped a shorter list, we could see a
484 * termination bit prematurely unless it fully inits the sg table
485 * on each mapping. We KNOW that there must be more entries here
486 * or the driver would be buggy, so force clear the termination bit
487 * to avoid doing a full sg_init_table() in drivers for each command.
488 */
489 sg_unmark_end(*sg);
490 return sg_next(*sg);
491}
492
493static unsigned blk_bvec_map_sg(struct request_queue *q,
494 struct bio_vec *bvec, struct scatterlist *sglist,
495 struct scatterlist **sg)
496{
497 unsigned nbytes = bvec->bv_len;
Christoph Hellwig8a96a0e2019-04-11 08:23:27 +0200498 unsigned nsegs = 0, total = 0;
Ming Lei862e5a52019-02-15 19:13:13 +0800499
500 while (nbytes > 0) {
Christoph Hellwig8a96a0e2019-04-11 08:23:27 +0200501 unsigned offset = bvec->bv_offset + total;
Christoph Hellwig61353a62024-07-09 09:01:25 +0200502 unsigned len = get_max_segment_size(&q->limits,
503 bvec_phys(bvec) + total, nbytes);
Christoph Hellwigf9f76872019-04-19 08:56:24 +0200504 struct page *page = bvec->bv_page;
505
506 /*
507 * Unfortunately a fair number of drivers barf on scatterlists
508 * that have an offset larger than PAGE_SIZE, despite other
509 * subsystems dealing with that invariant just fine. For now
510 * stick to the legacy format where we never present those from
511 * the block layer, but the code below should be removed once
512 * these offenders (mostly MMC/SD drivers) are fixed.
513 */
514 page += (offset >> PAGE_SHIFT);
515 offset &= ~PAGE_MASK;
Ming Lei862e5a52019-02-15 19:13:13 +0800516
517 *sg = blk_next_sg(sg, sglist);
Christoph Hellwigf9f76872019-04-19 08:56:24 +0200518 sg_set_page(*sg, page, len, offset);
Ming Lei862e5a52019-02-15 19:13:13 +0800519
Christoph Hellwig8a96a0e2019-04-11 08:23:27 +0200520 total += len;
521 nbytes -= len;
Ming Lei862e5a52019-02-15 19:13:13 +0800522 nsegs++;
523 }
524
525 return nsegs;
526}
527
Ming Lei16e3e412019-03-17 18:01:11 +0800528static inline int __blk_bvec_map_sg(struct bio_vec bv,
529 struct scatterlist *sglist, struct scatterlist **sg)
530{
531 *sg = blk_next_sg(sg, sglist);
532 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
533 return 1;
534}
535
Ming Leif6970f82019-03-17 18:01:12 +0800536/* only try to merge bvecs into one sg if they are from two bios */
537static inline bool
538__blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec,
539 struct bio_vec *bvprv, struct scatterlist **sg)
Asias He963ab9e52012-08-02 23:42:03 +0200540{
541
542 int nbytes = bvec->bv_len;
543
Ming Leif6970f82019-03-17 18:01:12 +0800544 if (!*sg)
545 return false;
Asias He963ab9e52012-08-02 23:42:03 +0200546
Ming Leif6970f82019-03-17 18:01:12 +0800547 if ((*sg)->length + nbytes > queue_max_segment_size(q))
548 return false;
549
550 if (!biovec_phys_mergeable(q, bvprv, bvec))
551 return false;
552
553 (*sg)->length += nbytes;
554
555 return true;
Asias He963ab9e52012-08-02 23:42:03 +0200556}
557
Kent Overstreet5cb88502014-02-07 13:53:46 -0700558static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio,
559 struct scatterlist *sglist,
560 struct scatterlist **sg)
561{
Kees Cook3f649ab2020-06-03 13:09:38 -0700562 struct bio_vec bvec, bvprv = { NULL };
Kent Overstreet5cb88502014-02-07 13:53:46 -0700563 struct bvec_iter iter;
Christoph Hellwig38417462018-12-13 16:17:10 +0100564 int nsegs = 0;
Ming Leif6970f82019-03-17 18:01:12 +0800565 bool new_bio = false;
Kent Overstreet5cb88502014-02-07 13:53:46 -0700566
Ming Leif6970f82019-03-17 18:01:12 +0800567 for_each_bio(bio) {
568 bio_for_each_bvec(bvec, bio, iter) {
569 /*
570 * Only try to merge bvecs from two bios given we
571 * have done bio internal merge when adding pages
572 * to bio
573 */
574 if (new_bio &&
575 __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg))
576 goto next_bvec;
577
578 if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE)
579 nsegs += __blk_bvec_map_sg(bvec, sglist, sg);
580 else
581 nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg);
582 next_bvec:
583 new_bio = false;
584 }
Ming Leib21e11c2019-04-02 10:26:44 +0800585 if (likely(bio->bi_iter.bi_size)) {
586 bvprv = bvec;
587 new_bio = true;
588 }
Ming Leif6970f82019-03-17 18:01:12 +0800589 }
Kent Overstreet5cb88502014-02-07 13:53:46 -0700590
591 return nsegs;
592}
593
Jens Axboed6d48192008-01-29 14:04:06 +0100594/*
595 * map a request to scatterlist, return number of sg entries setup. Caller
596 * must make sure sg can hold rq->nr_phys_segments entries
597 */
Christoph Hellwig89de1502020-04-14 09:42:22 +0200598int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
599 struct scatterlist *sglist, struct scatterlist **last_sg)
Jens Axboed6d48192008-01-29 14:04:06 +0100600{
Kent Overstreet5cb88502014-02-07 13:53:46 -0700601 int nsegs = 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100602
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700603 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
Christoph Hellwig89de1502020-04-14 09:42:22 +0200604 nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, last_sg);
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700605 else if (rq->bio)
Christoph Hellwig89de1502020-04-14 09:42:22 +0200606 nsegs = __blk_bios_map_sg(q, rq->bio, sglist, last_sg);
FUJITA Tomonorif18573a2008-04-11 12:56:52 +0200607
Christoph Hellwig89de1502020-04-14 09:42:22 +0200608 if (*last_sg)
609 sg_mark_end(*last_sg);
Jens Axboed6d48192008-01-29 14:04:06 +0100610
Ming Lei12e57f52015-11-24 10:35:31 +0800611 /*
612 * Something must have been wrong if the figured number of
613 * segment is bigger than number of req's physical segments
614 */
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700615 WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
Ming Lei12e57f52015-11-24 10:35:31 +0800616
Jens Axboed6d48192008-01-29 14:04:06 +0100617 return nsegs;
618}
Christoph Hellwig89de1502020-04-14 09:42:22 +0200619EXPORT_SYMBOL(__blk_rq_map_sg);
Jens Axboed6d48192008-01-29 14:04:06 +0100620
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200621static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
622 sector_t offset)
623{
624 struct request_queue *q = rq->q;
John Garryf70167a2024-06-20 12:53:51 +0000625 struct queue_limits *lim = &q->limits;
626 unsigned int max_sectors, boundary_sectors;
John Garry9da3d1e2024-06-20 12:53:54 +0000627 bool is_atomic = rq->cmd_flags & REQ_ATOMIC;
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200628
629 if (blk_rq_is_passthrough(rq))
630 return q->limits.max_hw_sectors;
631
John Garry9da3d1e2024-06-20 12:53:54 +0000632 boundary_sectors = blk_boundary_sectors(lim, is_atomic);
John Garry8d1dfd52024-06-20 12:53:50 +0000633 max_sectors = blk_queue_get_max_sectors(rq);
634
John Garryf70167a2024-06-20 12:53:51 +0000635 if (!boundary_sectors ||
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200636 req_op(rq) == REQ_OP_DISCARD ||
637 req_op(rq) == REQ_OP_SECURE_ERASE)
Christoph Hellwigc8875192022-06-14 11:09:31 +0200638 return max_sectors;
639 return min(max_sectors,
John Garryf70167a2024-06-20 12:53:51 +0000640 blk_boundary_sectors_left(offset, boundary_sectors));
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200641}
642
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200643static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
644 unsigned int nr_phys_segs)
Jens Axboed6d48192008-01-29 14:04:06 +0100645{
Tejun Heo6b2b0452022-03-14 14:30:11 -1000646 if (!blk_cgroup_mergeable(req, bio))
647 goto no_merge;
648
Ming Lei2705dfb2021-06-28 10:33:12 +0800649 if (blk_integrity_merge_bio(req->q, req, bio) == false)
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200650 goto no_merge;
651
Ming Lei2705dfb2021-06-28 10:33:12 +0800652 /* discard request merge won't add new segment */
653 if (req_op(req) == REQ_OP_DISCARD)
654 return 1;
655
656 if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req))
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200657 goto no_merge;
Jens Axboed6d48192008-01-29 14:04:06 +0100658
659 /*
660 * This will form the start of a new hw segment. Bump both
661 * counters.
662 */
Jens Axboed6d48192008-01-29 14:04:06 +0100663 req->nr_phys_segments += nr_phys_segs;
Keith Buschd148d7502024-09-13 11:28:48 -0700664 if (bio_integrity(bio))
665 req->nr_integrity_segments += blk_rq_count_integrity_sg(req->q,
666 bio);
Jens Axboed6d48192008-01-29 14:04:06 +0100667 return 1;
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200668
669no_merge:
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200670 req_set_nomerge(req->q, req);
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200671 return 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100672}
673
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200674int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
Jens Axboed6d48192008-01-29 14:04:06 +0100675{
Jens Axboe5e7c4272015-09-03 19:28:20 +0300676 if (req_gap_back_merge(req, bio))
677 return 0;
Sagi Grimberg7f39add2015-09-11 09:03:04 -0600678 if (blk_integrity_rq(req) &&
679 integrity_req_gap_back_merge(req, bio))
680 return 0;
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000681 if (!bio_crypt_ctx_back_mergeable(req, bio))
682 return 0;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400683 if (blk_rq_sectors(req) + bio_sectors(bio) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600684 blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200685 req_set_nomerge(req->q, req);
Jens Axboed6d48192008-01-29 14:04:06 +0100686 return 0;
687 }
Jens Axboed6d48192008-01-29 14:04:06 +0100688
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200689 return ll_new_hw_segment(req, bio, nr_segs);
Jens Axboed6d48192008-01-29 14:04:06 +0100690}
691
Christoph Hellwigeda5cc92020-10-06 09:07:19 +0200692static int ll_front_merge_fn(struct request *req, struct bio *bio,
693 unsigned int nr_segs)
Jens Axboed6d48192008-01-29 14:04:06 +0100694{
Jens Axboe5e7c4272015-09-03 19:28:20 +0300695 if (req_gap_front_merge(req, bio))
696 return 0;
Sagi Grimberg7f39add2015-09-11 09:03:04 -0600697 if (blk_integrity_rq(req) &&
698 integrity_req_gap_front_merge(req, bio))
699 return 0;
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000700 if (!bio_crypt_ctx_front_mergeable(req, bio))
701 return 0;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400702 if (blk_rq_sectors(req) + bio_sectors(bio) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600703 blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200704 req_set_nomerge(req->q, req);
Jens Axboed6d48192008-01-29 14:04:06 +0100705 return 0;
706 }
Jens Axboed6d48192008-01-29 14:04:06 +0100707
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200708 return ll_new_hw_segment(req, bio, nr_segs);
Jens Axboed6d48192008-01-29 14:04:06 +0100709}
710
Jens Axboe445251d2018-02-01 14:01:02 -0700711static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
712 struct request *next)
713{
714 unsigned short segments = blk_rq_nr_discard_segments(req);
715
716 if (segments >= queue_max_discard_segments(q))
717 goto no_merge;
718 if (blk_rq_sectors(req) + bio_sectors(next->bio) >
719 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
720 goto no_merge;
721
722 req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
723 return true;
724no_merge:
725 req_set_nomerge(q, req);
726 return false;
727}
728
Jens Axboed6d48192008-01-29 14:04:06 +0100729static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
730 struct request *next)
731{
732 int total_phys_segments;
Jens Axboed6d48192008-01-29 14:04:06 +0100733
Jens Axboe5e7c4272015-09-03 19:28:20 +0300734 if (req_gap_back_merge(req, next->bio))
Keith Busch854fbb92015-02-11 08:20:13 -0700735 return 0;
736
Jens Axboed6d48192008-01-29 14:04:06 +0100737 /*
738 * Will it become too large?
739 */
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400740 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600741 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
Jens Axboed6d48192008-01-29 14:04:06 +0100742 return 0;
743
744 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
Ming Lei943b40c2020-08-17 17:52:39 +0800745 if (total_phys_segments > blk_rq_get_max_segments(req))
Jens Axboed6d48192008-01-29 14:04:06 +0100746 return 0;
747
Tejun Heo6b2b0452022-03-14 14:30:11 -1000748 if (!blk_cgroup_mergeable(req, next->bio))
749 return 0;
750
Martin K. Petersen4eaf99b2014-09-26 19:20:06 -0400751 if (blk_integrity_merge_rq(q, req, next) == false)
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200752 return 0;
753
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000754 if (!bio_crypt_ctx_merge_rq(req, next))
755 return 0;
756
Jens Axboed6d48192008-01-29 14:04:06 +0100757 /* Merge is OK... */
758 req->nr_phys_segments = total_phys_segments;
Keith Buschd148d7502024-09-13 11:28:48 -0700759 req->nr_integrity_segments += next->nr_integrity_segments;
Jens Axboed6d48192008-01-29 14:04:06 +0100760 return 1;
761}
762
Tejun Heo80a761f2009-07-03 17:48:17 +0900763/**
764 * blk_rq_set_mixed_merge - mark a request as mixed merge
765 * @rq: request to mark as mixed merge
766 *
767 * Description:
768 * @rq is about to be mixed merged. Make sure the attributes
769 * which can be mixed are set in each bio and mark @rq as mixed
770 * merged.
771 */
John Garrydc53d9e2024-03-25 08:35:01 +0000772static void blk_rq_set_mixed_merge(struct request *rq)
Tejun Heo80a761f2009-07-03 17:48:17 +0900773{
Bart Van Assche16458cf2022-07-14 11:06:32 -0700774 blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
Tejun Heo80a761f2009-07-03 17:48:17 +0900775 struct bio *bio;
776
Christoph Hellwige8064022016-10-20 15:12:13 +0200777 if (rq->rq_flags & RQF_MIXED_MERGE)
Tejun Heo80a761f2009-07-03 17:48:17 +0900778 return;
779
780 /*
781 * @rq will no longer represent mixable attributes for all the
782 * contained bios. It will just track those of the first one.
783 * Distributes the attributs to each bio.
784 */
785 for (bio = rq->bio; bio; bio = bio->bi_next) {
Jens Axboe1eff9d32016-08-05 15:35:16 -0600786 WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
787 (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
788 bio->bi_opf |= ff;
Tejun Heo80a761f2009-07-03 17:48:17 +0900789 }
Christoph Hellwige8064022016-10-20 15:12:13 +0200790 rq->rq_flags |= RQF_MIXED_MERGE;
Tejun Heo80a761f2009-07-03 17:48:17 +0900791}
792
Jens Axboef3ca7382023-02-16 19:39:15 -0700793static inline blk_opf_t bio_failfast(const struct bio *bio)
Ming Lei3ce6a112023-02-09 20:55:27 +0800794{
795 if (bio->bi_opf & REQ_RAHEAD)
796 return REQ_FAILFAST_MASK;
797
798 return bio->bi_opf & REQ_FAILFAST_MASK;
799}
800
801/*
802 * After we are marked as MIXED_MERGE, any new RA bio has to be updated
803 * as failfast, and request's failfast has to be updated in case of
804 * front merge.
805 */
806static inline void blk_update_mixed_merge(struct request *req,
807 struct bio *bio, bool front_merge)
808{
809 if (req->rq_flags & RQF_MIXED_MERGE) {
810 if (bio->bi_opf & REQ_RAHEAD)
811 bio->bi_opf |= REQ_FAILFAST_MASK;
812
813 if (front_merge) {
814 req->cmd_flags &= ~REQ_FAILFAST_MASK;
815 req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK;
816 }
817 }
818}
819
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200820static void blk_account_io_merge_request(struct request *req)
Jerome Marchand26308ea2009-03-27 10:31:51 +0100821{
Jens Axboee3569ec2024-10-03 07:29:49 -0600822 if (req->rq_flags & RQF_IO_STAT) {
Mike Snitzer112f1582018-12-06 11:41:18 -0500823 part_stat_lock();
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200824 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
Yu Kuai99dc4222024-05-09 20:37:16 +0800825 part_stat_local_dec(req->part,
826 in_flight[op_is_write(req_op(req))]);
Jerome Marchand26308ea2009-03-27 10:31:51 +0100827 part_stat_unlock();
828 }
829}
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200830
Eric Biggerse96c0d82018-11-14 17:19:46 -0800831static enum elv_merge blk_try_req_merge(struct request *req,
832 struct request *next)
Jianchao Wang698404662018-10-27 19:52:14 +0800833{
834 if (blk_discard_mergable(req))
835 return ELEVATOR_DISCARD_MERGE;
836 else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
837 return ELEVATOR_BACK_MERGE;
838
839 return ELEVATOR_NO_MERGE;
840}
Jerome Marchand26308ea2009-03-27 10:31:51 +0100841
John Garry9da3d1e2024-06-20 12:53:54 +0000842static bool blk_atomic_write_mergeable_rq_bio(struct request *rq,
843 struct bio *bio)
844{
845 return (rq->cmd_flags & REQ_ATOMIC) == (bio->bi_opf & REQ_ATOMIC);
846}
847
848static bool blk_atomic_write_mergeable_rqs(struct request *rq,
849 struct request *next)
850{
851 return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC);
852}
853
Jens Axboed6d48192008-01-29 14:04:06 +0100854/*
Jens Axboeb973cb72017-02-02 08:54:40 -0700855 * For non-mq, this has to be called with the request spinlock acquired.
856 * For mq with scheduling, the appropriate queue wide lock should be held.
Jens Axboed6d48192008-01-29 14:04:06 +0100857 */
Jens Axboeb973cb72017-02-02 08:54:40 -0700858static struct request *attempt_merge(struct request_queue *q,
859 struct request *req, struct request *next)
Jens Axboed6d48192008-01-29 14:04:06 +0100860{
861 if (!rq_mergeable(req) || !rq_mergeable(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700862 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100863
Christoph Hellwig288dab82016-06-09 16:00:36 +0200864 if (req_op(req) != req_op(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700865 return NULL;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400866
Christoph Hellwig79bb1db2021-11-26 13:17:59 +0100867 if (rq_data_dir(req) != rq_data_dir(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700868 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100869
Christoph Hellwig61952bb2024-11-12 18:00:38 +0100870 if (req->bio && next->bio) {
871 /* Don't merge requests with different write hints. */
872 if (req->bio->bi_write_hint != next->bio->bi_write_hint)
873 return NULL;
Christoph Hellwig6975c1a2024-11-12 18:00:39 +0100874 if (req->bio->bi_ioprio != next->bio->bi_ioprio)
875 return NULL;
Christoph Hellwig61952bb2024-11-12 18:00:38 +0100876 }
Bart Van Assche44981352024-02-02 12:39:25 -0800877
John Garry9da3d1e2024-06-20 12:53:54 +0000878 if (!blk_atomic_write_mergeable_rqs(req, next))
879 return NULL;
880
Jens Axboecb6934f2017-06-27 09:22:02 -0600881 /*
Jens Axboed6d48192008-01-29 14:04:06 +0100882 * If we are allowed to merge, then append bio list
883 * from next to rq and release next. merge_requests_fn
884 * will have updated segment counts, update sector
Jens Axboe445251d2018-02-01 14:01:02 -0700885 * counts here. Handle DISCARDs separately, as they
886 * have separate settings.
Jens Axboed6d48192008-01-29 14:04:06 +0100887 */
Jianchao Wang698404662018-10-27 19:52:14 +0800888
889 switch (blk_try_req_merge(req, next)) {
890 case ELEVATOR_DISCARD_MERGE:
Jens Axboe445251d2018-02-01 14:01:02 -0700891 if (!req_attempt_discard_merge(q, req, next))
892 return NULL;
Jianchao Wang698404662018-10-27 19:52:14 +0800893 break;
894 case ELEVATOR_BACK_MERGE:
895 if (!ll_merge_requests_fn(q, req, next))
896 return NULL;
897 break;
898 default:
Jens Axboeb973cb72017-02-02 08:54:40 -0700899 return NULL;
Jianchao Wang698404662018-10-27 19:52:14 +0800900 }
Jens Axboed6d48192008-01-29 14:04:06 +0100901
902 /*
Tejun Heo80a761f2009-07-03 17:48:17 +0900903 * If failfast settings disagree or any of the two is already
904 * a mixed merge, mark both as mixed before proceeding. This
905 * makes sure that all involved bios have mixable attributes
906 * set properly.
907 */
Christoph Hellwige8064022016-10-20 15:12:13 +0200908 if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
Tejun Heo80a761f2009-07-03 17:48:17 +0900909 (req->cmd_flags & REQ_FAILFAST_MASK) !=
910 (next->cmd_flags & REQ_FAILFAST_MASK)) {
911 blk_rq_set_mixed_merge(req);
912 blk_rq_set_mixed_merge(next);
913 }
914
915 /*
Omar Sandoval522a7772018-05-09 02:08:53 -0700916 * At this point we have either done a back merge or front merge. We
917 * need the smaller start_time_ns of the merged requests to be the
918 * current request for accounting purposes.
Jens Axboed6d48192008-01-29 14:04:06 +0100919 */
Omar Sandoval522a7772018-05-09 02:08:53 -0700920 if (next->start_time_ns < req->start_time_ns)
921 req->start_time_ns = next->start_time_ns;
Jens Axboed6d48192008-01-29 14:04:06 +0100922
923 req->biotail->bi_next = next->bio;
924 req->biotail = next->biotail;
925
Tejun Heoa2dec7b2009-05-07 22:24:44 +0900926 req->__data_len += blk_rq_bytes(next);
Jens Axboed6d48192008-01-29 14:04:06 +0100927
Ming Lei2a5cf352018-12-01 00:38:18 +0800928 if (!blk_discard_mergable(req))
Jens Axboe445251d2018-02-01 14:01:02 -0700929 elv_merge_requests(q, req, next);
Jens Axboed6d48192008-01-29 14:04:06 +0100930
Eric Biggers9cd1e562023-03-15 11:39:02 -0700931 blk_crypto_rq_put_keyslot(next);
932
Jerome Marchand42dad762009-04-22 14:01:49 +0200933 /*
934 * 'next' is going away, so update stats accordingly
935 */
Konstantin Khlebnikovb9c54f52020-05-27 07:24:15 +0200936 blk_account_io_merge_request(next);
Jens Axboed6d48192008-01-29 14:04:06 +0100937
Christoph Hellwiga54895f2020-12-03 17:21:39 +0100938 trace_block_rq_merge(next);
Jan Karaf3bdc622020-06-17 15:58:23 +0200939
Jens Axboee4d750c2017-02-03 09:48:28 -0700940 /*
941 * ownership of bio passed from next to req, return 'next' for
942 * the caller to free
943 */
Boaz Harrosh1cd96c22009-03-24 12:35:07 +0100944 next->bio = NULL;
Jens Axboeb973cb72017-02-02 08:54:40 -0700945 return next;
Jens Axboed6d48192008-01-29 14:04:06 +0100946}
947
Christoph Hellwigeda5cc92020-10-06 09:07:19 +0200948static struct request *attempt_back_merge(struct request_queue *q,
949 struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100950{
951 struct request *next = elv_latter_request(q, rq);
952
953 if (next)
954 return attempt_merge(q, rq, next);
955
Jens Axboeb973cb72017-02-02 08:54:40 -0700956 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100957}
958
Christoph Hellwigeda5cc92020-10-06 09:07:19 +0200959static struct request *attempt_front_merge(struct request_queue *q,
960 struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100961{
962 struct request *prev = elv_former_request(q, rq);
963
964 if (prev)
965 return attempt_merge(q, prev, rq);
966
Jens Axboeb973cb72017-02-02 08:54:40 -0700967 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100968}
Jens Axboe5e84ea32011-03-21 10:14:27 +0100969
Jan Karafd2ef392021-06-23 11:36:34 +0200970/*
971 * Try to merge 'next' into 'rq'. Return true if the merge happened, false
972 * otherwise. The caller is responsible for freeing 'next' if the merge
973 * happened.
974 */
975bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
976 struct request *next)
Jens Axboe5e84ea32011-03-21 10:14:27 +0100977{
Jan Karafd2ef392021-06-23 11:36:34 +0200978 return attempt_merge(q, rq, next);
Jens Axboe5e84ea32011-03-21 10:14:27 +0100979}
Tejun Heo050c8ea2012-02-08 09:19:38 +0100980
981bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
982{
Martin K. Petersene2a60da2012-09-18 12:19:25 -0400983 if (!rq_mergeable(rq) || !bio_mergeable(bio))
Tejun Heo050c8ea2012-02-08 09:19:38 +0100984 return false;
985
Christoph Hellwig288dab82016-06-09 16:00:36 +0200986 if (req_op(rq) != bio_op(bio))
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400987 return false;
988
Tejun Heo050c8ea2012-02-08 09:19:38 +0100989 /* different data direction or already started, don't merge */
990 if (bio_data_dir(bio) != rq_data_dir(rq))
991 return false;
992
Tejun Heo6b2b0452022-03-14 14:30:11 -1000993 /* don't merge across cgroup boundaries */
994 if (!blk_cgroup_mergeable(rq, bio))
995 return false;
996
Tejun Heo050c8ea2012-02-08 09:19:38 +0100997 /* only merge integrity protected bio into ditto rq */
Martin K. Petersen4eaf99b2014-09-26 19:20:06 -0400998 if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100999 return false;
1000
Satya Tangiralaa892c8d2020-05-14 00:37:18 +00001001 /* Only merge if the crypt contexts are compatible */
1002 if (!bio_crypt_rq_ctx_compatible(rq, bio))
1003 return false;
1004
Christoph Hellwig61952bb2024-11-12 18:00:38 +01001005 if (rq->bio) {
1006 /* Don't merge requests with different write hints. */
1007 if (rq->bio->bi_write_hint != bio->bi_write_hint)
1008 return false;
Christoph Hellwig6975c1a2024-11-12 18:00:39 +01001009 if (rq->bio->bi_ioprio != bio->bi_ioprio)
1010 return false;
Christoph Hellwig61952bb2024-11-12 18:00:38 +01001011 }
Bart Van Assche44981352024-02-02 12:39:25 -08001012
John Garry9da3d1e2024-06-20 12:53:54 +00001013 if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false)
1014 return false;
1015
Tejun Heo050c8ea2012-02-08 09:19:38 +01001016 return true;
1017}
1018
Christoph Hellwig34fe7c02017-02-08 14:46:48 +01001019enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
Tejun Heo050c8ea2012-02-08 09:19:38 +01001020{
Jianchao Wang698404662018-10-27 19:52:14 +08001021 if (blk_discard_mergable(rq))
Christoph Hellwig1e739732017-02-08 14:46:49 +01001022 return ELEVATOR_DISCARD_MERGE;
1023 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
Tejun Heo050c8ea2012-02-08 09:19:38 +01001024 return ELEVATOR_BACK_MERGE;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001025 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
Tejun Heo050c8ea2012-02-08 09:19:38 +01001026 return ELEVATOR_FRONT_MERGE;
1027 return ELEVATOR_NO_MERGE;
1028}
Baolin Wang8e756372020-08-28 10:52:54 +08001029
1030static void blk_account_io_merge_bio(struct request *req)
1031{
Jens Axboee3569ec2024-10-03 07:29:49 -06001032 if (req->rq_flags & RQF_IO_STAT) {
1033 part_stat_lock();
1034 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
1035 part_stat_unlock();
1036 }
Baolin Wang8e756372020-08-28 10:52:54 +08001037}
1038
Damien Le Moaldd850ff2024-04-08 10:41:05 +09001039enum bio_merge_status bio_attempt_back_merge(struct request *req,
Christoph Hellwigeda5cc92020-10-06 09:07:19 +02001040 struct bio *bio, unsigned int nr_segs)
Baolin Wang8e756372020-08-28 10:52:54 +08001041{
Ming Lei3ce6a112023-02-09 20:55:27 +08001042 const blk_opf_t ff = bio_failfast(bio);
Baolin Wang8e756372020-08-28 10:52:54 +08001043
1044 if (!ll_back_merge_fn(req, bio, nr_segs))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001045 return BIO_MERGE_FAILED;
Baolin Wang8e756372020-08-28 10:52:54 +08001046
Christoph Hellwige8a676d2020-12-03 17:21:36 +01001047 trace_block_bio_backmerge(bio);
Baolin Wang8e756372020-08-28 10:52:54 +08001048 rq_qos_merge(req->q, req, bio);
1049
1050 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1051 blk_rq_set_mixed_merge(req);
1052
Ming Lei3ce6a112023-02-09 20:55:27 +08001053 blk_update_mixed_merge(req, bio, false);
1054
Damien Le Moaldd291d72024-04-08 10:41:07 +09001055 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
1056 blk_zone_write_plug_bio_merged(bio);
1057
Baolin Wang8e756372020-08-28 10:52:54 +08001058 req->biotail->bi_next = bio;
1059 req->biotail = bio;
1060 req->__data_len += bio->bi_iter.bi_size;
1061
1062 bio_crypt_free_ctx(bio);
1063
1064 blk_account_io_merge_bio(req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001065 return BIO_MERGE_OK;
Baolin Wang8e756372020-08-28 10:52:54 +08001066}
1067
Christoph Hellwigeda5cc92020-10-06 09:07:19 +02001068static enum bio_merge_status bio_attempt_front_merge(struct request *req,
1069 struct bio *bio, unsigned int nr_segs)
Baolin Wang8e756372020-08-28 10:52:54 +08001070{
Ming Lei3ce6a112023-02-09 20:55:27 +08001071 const blk_opf_t ff = bio_failfast(bio);
Baolin Wang8e756372020-08-28 10:52:54 +08001072
Damien Le Moaldd291d72024-04-08 10:41:07 +09001073 /*
1074 * A front merge for writes to sequential zones of a zoned block device
1075 * can happen only if the user submitted writes out of order. Do not
1076 * merge such write to let it fail.
1077 */
1078 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
1079 return BIO_MERGE_FAILED;
1080
Baolin Wang8e756372020-08-28 10:52:54 +08001081 if (!ll_front_merge_fn(req, bio, nr_segs))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001082 return BIO_MERGE_FAILED;
Baolin Wang8e756372020-08-28 10:52:54 +08001083
Christoph Hellwige8a676d2020-12-03 17:21:36 +01001084 trace_block_bio_frontmerge(bio);
Baolin Wang8e756372020-08-28 10:52:54 +08001085 rq_qos_merge(req->q, req, bio);
1086
1087 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1088 blk_rq_set_mixed_merge(req);
1089
Ming Lei3ce6a112023-02-09 20:55:27 +08001090 blk_update_mixed_merge(req, bio, true);
1091
Baolin Wang8e756372020-08-28 10:52:54 +08001092 bio->bi_next = req->bio;
1093 req->bio = bio;
1094
1095 req->__sector = bio->bi_iter.bi_sector;
1096 req->__data_len += bio->bi_iter.bi_size;
1097
1098 bio_crypt_do_front_merge(req, bio);
1099
1100 blk_account_io_merge_bio(req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001101 return BIO_MERGE_OK;
Baolin Wang8e756372020-08-28 10:52:54 +08001102}
1103
Christoph Hellwigeda5cc92020-10-06 09:07:19 +02001104static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
1105 struct request *req, struct bio *bio)
Baolin Wang8e756372020-08-28 10:52:54 +08001106{
1107 unsigned short segments = blk_rq_nr_discard_segments(req);
1108
1109 if (segments >= queue_max_discard_segments(q))
1110 goto no_merge;
1111 if (blk_rq_sectors(req) + bio_sectors(bio) >
1112 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1113 goto no_merge;
1114
1115 rq_qos_merge(q, req, bio);
1116
1117 req->biotail->bi_next = bio;
1118 req->biotail = bio;
1119 req->__data_len += bio->bi_iter.bi_size;
1120 req->nr_phys_segments = segments + 1;
1121
1122 blk_account_io_merge_bio(req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001123 return BIO_MERGE_OK;
Baolin Wang8e756372020-08-28 10:52:54 +08001124no_merge:
1125 req_set_nomerge(q, req);
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001126 return BIO_MERGE_FAILED;
1127}
1128
1129static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
1130 struct request *rq,
1131 struct bio *bio,
1132 unsigned int nr_segs,
1133 bool sched_allow_merge)
1134{
1135 if (!blk_rq_merge_ok(rq, bio))
1136 return BIO_MERGE_NONE;
1137
1138 switch (blk_try_merge(rq, bio)) {
1139 case ELEVATOR_BACK_MERGE:
Baolin Wang265600b2020-09-02 09:45:25 +08001140 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001141 return bio_attempt_back_merge(rq, bio, nr_segs);
1142 break;
1143 case ELEVATOR_FRONT_MERGE:
Baolin Wang265600b2020-09-02 09:45:25 +08001144 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001145 return bio_attempt_front_merge(rq, bio, nr_segs);
1146 break;
1147 case ELEVATOR_DISCARD_MERGE:
1148 return bio_attempt_discard_merge(q, rq, bio);
1149 default:
1150 return BIO_MERGE_NONE;
1151 }
1152
1153 return BIO_MERGE_FAILED;
Baolin Wang8e756372020-08-28 10:52:54 +08001154}
1155
1156/**
1157 * blk_attempt_plug_merge - try to merge with %current's plugged list
1158 * @q: request_queue new bio is being queued at
1159 * @bio: new bio being queued
1160 * @nr_segs: number of segments in @bio
Jens Axboe87c037d2021-10-18 10:07:09 -06001161 * from the passed in @q already in the plug list
Baolin Wang8e756372020-08-28 10:52:54 +08001162 *
Jens Axboed38a9c02021-10-14 07:24:07 -06001163 * Determine whether @bio being queued on @q can be merged with the previous
1164 * request on %current's plugged list. Returns %true if merge was successful,
Baolin Wang8e756372020-08-28 10:52:54 +08001165 * otherwise %false.
1166 *
1167 * Plugging coalesces IOs from the same issuer for the same purpose without
1168 * going through @q->queue_lock. As such it's more of an issuing mechanism
1169 * than scheduling, and the request, while may have elvpriv data, is not
1170 * added on the elevator at this point. In addition, we don't have
1171 * reliable access to the elevator outside queue lock. Only check basic
1172 * merging parameters without querying the elevator.
1173 *
1174 * Caller must ensure !blk_queue_nomerges(q) beforehand.
1175 */
1176bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
Christoph Hellwig0c5bcc92021-11-23 17:04:41 +01001177 unsigned int nr_segs)
Baolin Wang8e756372020-08-28 10:52:54 +08001178{
Damien Le Moal99a94762024-04-08 10:41:28 +09001179 struct blk_plug *plug = current->plug;
Baolin Wang8e756372020-08-28 10:52:54 +08001180 struct request *rq;
Baolin Wang8e756372020-08-28 10:52:54 +08001181
Christoph Hellwiga3396b92024-11-13 16:20:44 +01001182 if (!plug || rq_list_empty(&plug->mq_list))
Baolin Wang8e756372020-08-28 10:52:54 +08001183 return false;
1184
Jens Axboe5b205072022-03-11 10:21:43 -07001185 rq_list_for_each(&plug->mq_list, rq) {
1186 if (rq->q == q) {
1187 if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
1188 BIO_MERGE_OK)
1189 return true;
1190 break;
1191 }
1192
1193 /*
1194 * Only keep iterating plug list for merges if we have multiple
1195 * queues
1196 */
1197 if (!plug->multiple_queues)
1198 break;
Baolin Wang8e756372020-08-28 10:52:54 +08001199 }
Baolin Wang8e756372020-08-28 10:52:54 +08001200 return false;
1201}
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001202
1203/*
1204 * Iterate list of requests and see if we can merge this bio with any
1205 * of them.
1206 */
1207bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
1208 struct bio *bio, unsigned int nr_segs)
1209{
1210 struct request *rq;
1211 int checked = 8;
1212
1213 list_for_each_entry_reverse(rq, list, queuelist) {
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001214 if (!checked--)
1215 break;
1216
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001217 switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {
1218 case BIO_MERGE_NONE:
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001219 continue;
Baolin Wang7d7ca7c2020-08-28 10:52:56 +08001220 case BIO_MERGE_OK:
1221 return true;
1222 case BIO_MERGE_FAILED:
1223 return false;
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001224 }
1225
Baolin Wangbdc6a2872020-08-28 10:52:55 +08001226 }
1227
1228 return false;
1229}
1230EXPORT_SYMBOL_GPL(blk_bio_list_merge);
Christoph Hellwigeda5cc92020-10-06 09:07:19 +02001231
1232bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
1233 unsigned int nr_segs, struct request **merged_request)
1234{
1235 struct request *rq;
1236
1237 switch (elv_merge(q, &rq, bio)) {
1238 case ELEVATOR_BACK_MERGE:
1239 if (!blk_mq_sched_allow_merge(q, rq, bio))
1240 return false;
1241 if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1242 return false;
1243 *merged_request = attempt_back_merge(q, rq);
1244 if (!*merged_request)
1245 elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
1246 return true;
1247 case ELEVATOR_FRONT_MERGE:
1248 if (!blk_mq_sched_allow_merge(q, rq, bio))
1249 return false;
1250 if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1251 return false;
1252 *merged_request = attempt_front_merge(q, rq);
1253 if (!*merged_request)
1254 elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
1255 return true;
1256 case ELEVATOR_DISCARD_MERGE:
1257 return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
1258 default:
1259 return false;
1260 }
1261}
1262EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);