Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 2 | /* |
| 3 | * Functions related to segment and merge handling |
| 4 | */ |
| 5 | #include <linux/kernel.h> |
| 6 | #include <linux/module.h> |
| 7 | #include <linux/bio.h> |
| 8 | #include <linux/blkdev.h> |
Christoph Hellwig | fe45e63 | 2021-09-20 14:33:27 +0200 | [diff] [blame] | 9 | #include <linux/blk-integrity.h> |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 10 | #include <linux/scatterlist.h> |
Christoph Hellwig | 82d981d | 2021-11-23 19:53:12 +0100 | [diff] [blame] | 11 | #include <linux/part_stat.h> |
Tejun Heo | 6b2b045 | 2022-03-14 14:30:11 -1000 | [diff] [blame] | 12 | #include <linux/blk-cgroup.h> |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 13 | |
Mike Krinkin | cda2264 | 2015-12-03 17:32:30 +0300 | [diff] [blame] | 14 | #include <trace/events/block.h> |
| 15 | |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 16 | #include "blk.h" |
Christoph Hellwig | 2aa7745 | 2021-11-23 19:53:08 +0100 | [diff] [blame] | 17 | #include "blk-mq-sched.h" |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 18 | #include "blk-rq-qos.h" |
Jens Axboe | a7b36ee | 2021-10-05 09:11:56 -0600 | [diff] [blame] | 19 | #include "blk-throttle.h" |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 20 | |
Christoph Hellwig | ff18d77 | 2021-10-12 18:18:03 +0200 | [diff] [blame] | 21 | static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv) |
| 22 | { |
| 23 | *bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); |
| 24 | } |
| 25 | |
| 26 | static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv) |
| 27 | { |
| 28 | struct bvec_iter iter = bio->bi_iter; |
| 29 | int idx; |
| 30 | |
| 31 | bio_get_first_bvec(bio, bv); |
| 32 | if (bv->bv_len == bio->bi_iter.bi_size) |
| 33 | return; /* this bio only has a single bvec */ |
| 34 | |
| 35 | bio_advance_iter(bio, &iter, iter.bi_size); |
| 36 | |
| 37 | if (!iter.bi_bvec_done) |
| 38 | idx = iter.bi_idx - 1; |
| 39 | else /* in the middle of bvec */ |
| 40 | idx = iter.bi_idx; |
| 41 | |
| 42 | *bv = bio->bi_io_vec[idx]; |
| 43 | |
| 44 | /* |
| 45 | * iter.bi_bvec_done records actual length of the last bvec |
| 46 | * if this bio ends in the middle of one io vector |
| 47 | */ |
| 48 | if (iter.bi_bvec_done) |
| 49 | bv->bv_len = iter.bi_bvec_done; |
| 50 | } |
| 51 | |
Christoph Hellwig | e990700 | 2018-09-24 09:43:48 +0200 | [diff] [blame] | 52 | static inline bool bio_will_gap(struct request_queue *q, |
| 53 | struct request *prev_rq, struct bio *prev, struct bio *next) |
| 54 | { |
| 55 | struct bio_vec pb, nb; |
| 56 | |
| 57 | if (!bio_has_data(prev) || !queue_virt_boundary(q)) |
| 58 | return false; |
| 59 | |
| 60 | /* |
| 61 | * Don't merge if the 1st bio starts with non-zero offset, otherwise it |
| 62 | * is quite difficult to respect the sg gap limit. We work hard to |
| 63 | * merge a huge number of small single bios in case of mkfs. |
| 64 | */ |
| 65 | if (prev_rq) |
| 66 | bio_get_first_bvec(prev_rq->bio, &pb); |
| 67 | else |
| 68 | bio_get_first_bvec(prev, &pb); |
Johannes Thumshirn | df376b2 | 2018-11-07 14:58:14 +0100 | [diff] [blame] | 69 | if (pb.bv_offset & queue_virt_boundary(q)) |
Christoph Hellwig | e990700 | 2018-09-24 09:43:48 +0200 | [diff] [blame] | 70 | return true; |
| 71 | |
| 72 | /* |
| 73 | * We don't need to worry about the situation that the merged segment |
| 74 | * ends in unaligned virt boundary: |
| 75 | * |
| 76 | * - if 'pb' ends aligned, the merged segment ends aligned |
| 77 | * - if 'pb' ends unaligned, the next bio must include |
| 78 | * one single bvec of 'nb', otherwise the 'nb' can't |
| 79 | * merge with 'pb' |
| 80 | */ |
| 81 | bio_get_last_bvec(prev, &pb); |
| 82 | bio_get_first_bvec(next, &nb); |
Christoph Hellwig | 200a9af | 2019-05-21 09:01:42 +0200 | [diff] [blame] | 83 | if (biovec_phys_mergeable(q, &pb, &nb)) |
Christoph Hellwig | e990700 | 2018-09-24 09:43:48 +0200 | [diff] [blame] | 84 | return false; |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 85 | return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset); |
Christoph Hellwig | e990700 | 2018-09-24 09:43:48 +0200 | [diff] [blame] | 86 | } |
| 87 | |
| 88 | static inline bool req_gap_back_merge(struct request *req, struct bio *bio) |
| 89 | { |
| 90 | return bio_will_gap(req->q, req, req->biotail, bio); |
| 91 | } |
| 92 | |
| 93 | static inline bool req_gap_front_merge(struct request *req, struct bio *bio) |
| 94 | { |
| 95 | return bio_will_gap(req->q, NULL, bio, req->bio); |
| 96 | } |
| 97 | |
Christoph Hellwig | b6dc619 | 2022-07-27 12:22:59 -0400 | [diff] [blame] | 98 | /* |
| 99 | * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size |
| 100 | * is defined as 'unsigned int', meantime it has to be aligned to with the |
| 101 | * logical block size, which is the minimum accepted unit by hardware. |
| 102 | */ |
Bart Van Assche | aa261f2 | 2022-10-25 12:17:54 -0700 | [diff] [blame] | 103 | static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim) |
Christoph Hellwig | b6dc619 | 2022-07-27 12:22:59 -0400 | [diff] [blame] | 104 | { |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 105 | return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT; |
Christoph Hellwig | b6dc619 | 2022-07-27 12:22:59 -0400 | [diff] [blame] | 106 | } |
| 107 | |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 108 | static struct bio *bio_submit_split(struct bio *bio, int split_sectors) |
| 109 | { |
John Garry | 6eb0968 | 2024-11-11 11:21:47 +0000 | [diff] [blame] | 110 | if (unlikely(split_sectors < 0)) |
| 111 | goto error; |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 112 | |
| 113 | if (split_sectors) { |
| 114 | struct bio *split; |
| 115 | |
| 116 | split = bio_split(bio, split_sectors, GFP_NOIO, |
| 117 | &bio->bi_bdev->bd_disk->bio_split); |
John Garry | 6eb0968 | 2024-11-11 11:21:47 +0000 | [diff] [blame] | 118 | if (IS_ERR(split)) { |
| 119 | split_sectors = PTR_ERR(split); |
| 120 | goto error; |
| 121 | } |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 122 | split->bi_opf |= REQ_NOMERGE; |
| 123 | blkcg_bio_issue_init(split); |
| 124 | bio_chain(split, bio); |
| 125 | trace_block_split(split, bio->bi_iter.bi_sector); |
| 126 | WARN_ON_ONCE(bio_zone_write_plugging(bio)); |
| 127 | submit_bio_noacct(bio); |
| 128 | return split; |
| 129 | } |
| 130 | |
| 131 | return bio; |
John Garry | 6eb0968 | 2024-11-11 11:21:47 +0000 | [diff] [blame] | 132 | error: |
| 133 | bio->bi_status = errno_to_blk_status(split_sectors); |
| 134 | bio_endio(bio); |
| 135 | return NULL; |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 136 | } |
| 137 | |
| 138 | struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, |
| 139 | unsigned *nsegs) |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 140 | { |
| 141 | unsigned int max_discard_sectors, granularity; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 142 | sector_t tmp; |
| 143 | unsigned split_sectors; |
| 144 | |
Ming Lei | bdced43 | 2015-10-20 23:13:52 +0800 | [diff] [blame] | 145 | *nsegs = 1; |
| 146 | |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 147 | granularity = max(lim->discard_granularity >> 9, 1U); |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 148 | |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 149 | max_discard_sectors = |
| 150 | min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 151 | max_discard_sectors -= max_discard_sectors % granularity; |
Christoph Hellwig | 928a5dd | 2023-12-28 07:55:37 +0000 | [diff] [blame] | 152 | if (unlikely(!max_discard_sectors)) |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 153 | return bio; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 154 | |
| 155 | if (bio_sectors(bio) <= max_discard_sectors) |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 156 | return bio; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 157 | |
| 158 | split_sectors = max_discard_sectors; |
| 159 | |
| 160 | /* |
| 161 | * If the next starting sector would be misaligned, stop the discard at |
| 162 | * the previous aligned sector. |
| 163 | */ |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 164 | tmp = bio->bi_iter.bi_sector + split_sectors - |
| 165 | ((lim->discard_alignment >> 9) % granularity); |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 166 | tmp = sector_div(tmp, granularity); |
| 167 | |
| 168 | if (split_sectors > tmp) |
| 169 | split_sectors -= tmp; |
| 170 | |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 171 | return bio_submit_split(bio, split_sectors); |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 172 | } |
| 173 | |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 174 | static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, |
| 175 | bool is_atomic) |
John Garry | f70167a | 2024-06-20 12:53:51 +0000 | [diff] [blame] | 176 | { |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 177 | /* |
| 178 | * chunk_sectors must be a multiple of atomic_write_boundary_sectors if |
| 179 | * both non-zero. |
| 180 | */ |
| 181 | if (is_atomic && lim->atomic_write_boundary_sectors) |
| 182 | return lim->atomic_write_boundary_sectors; |
| 183 | |
John Garry | f70167a | 2024-06-20 12:53:51 +0000 | [diff] [blame] | 184 | return lim->chunk_sectors; |
| 185 | } |
| 186 | |
Bart Van Assche | 9cc5169 | 2019-08-01 15:50:44 -0700 | [diff] [blame] | 187 | /* |
| 188 | * Return the maximum number of sectors from the start of a bio that may be |
| 189 | * submitted as a single request to a block device. If enough sectors remain, |
| 190 | * align the end to the physical block size. Otherwise align the end to the |
| 191 | * logical block size. This approach minimizes the number of non-aligned |
| 192 | * requests that are submitted to a block device if the start of a bio is not |
| 193 | * aligned to a physical block boundary. |
| 194 | */ |
Christoph Hellwig | 5a97806 | 2022-07-27 12:22:55 -0400 | [diff] [blame] | 195 | static inline unsigned get_max_io_size(struct bio *bio, |
Bart Van Assche | aa261f2 | 2022-10-25 12:17:54 -0700 | [diff] [blame] | 196 | const struct queue_limits *lim) |
Ming Lei | d0e5fbb | 2016-01-23 08:05:33 +0800 | [diff] [blame] | 197 | { |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 198 | unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT; |
| 199 | unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT; |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 200 | bool is_atomic = bio->bi_opf & REQ_ATOMIC; |
| 201 | unsigned boundary_sectors = blk_boundary_sectors(lim, is_atomic); |
| 202 | unsigned max_sectors, start, end; |
| 203 | |
| 204 | /* |
| 205 | * We ignore lim->max_sectors for atomic writes because it may less |
| 206 | * than the actual bio size, which we cannot tolerate. |
| 207 | */ |
Christoph Hellwig | 60dc5ea | 2024-11-04 07:26:29 +0100 | [diff] [blame] | 208 | if (bio_op(bio) == REQ_OP_WRITE_ZEROES) |
| 209 | max_sectors = lim->max_write_zeroes_sectors; |
| 210 | else if (is_atomic) |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 211 | max_sectors = lim->atomic_write_max_sectors; |
| 212 | else |
| 213 | max_sectors = lim->max_sectors; |
Ming Lei | d0e5fbb | 2016-01-23 08:05:33 +0800 | [diff] [blame] | 214 | |
John Garry | f70167a | 2024-06-20 12:53:51 +0000 | [diff] [blame] | 215 | if (boundary_sectors) { |
Christoph Hellwig | efef739 | 2022-06-14 11:09:33 +0200 | [diff] [blame] | 216 | max_sectors = min(max_sectors, |
John Garry | f70167a | 2024-06-20 12:53:51 +0000 | [diff] [blame] | 217 | blk_boundary_sectors_left(bio->bi_iter.bi_sector, |
| 218 | boundary_sectors)); |
Christoph Hellwig | efef739 | 2022-06-14 11:09:33 +0200 | [diff] [blame] | 219 | } |
Ming Lei | d0e5fbb | 2016-01-23 08:05:33 +0800 | [diff] [blame] | 220 | |
Christoph Hellwig | 84613be | 2022-06-14 11:09:32 +0200 | [diff] [blame] | 221 | start = bio->bi_iter.bi_sector & (pbs - 1); |
| 222 | end = (start + max_sectors) & ~(pbs - 1); |
| 223 | if (end > start) |
| 224 | return end - start; |
| 225 | return max_sectors & ~(lbs - 1); |
Ming Lei | d0e5fbb | 2016-01-23 08:05:33 +0800 | [diff] [blame] | 226 | } |
| 227 | |
Bart Van Assche | 9546531 | 2022-10-25 12:17:55 -0700 | [diff] [blame] | 228 | /** |
| 229 | * get_max_segment_size() - maximum number of bytes to add as a single segment |
| 230 | * @lim: Request queue limits. |
Christoph Hellwig | 09595e0 | 2024-07-06 09:52:18 +0200 | [diff] [blame] | 231 | * @paddr: address of the range to add |
Chaitanya Kulkarni | 0ffc46e | 2024-07-08 21:54:32 -0700 | [diff] [blame] | 232 | * @len: maximum length available to add at @paddr |
Bart Van Assche | 9546531 | 2022-10-25 12:17:55 -0700 | [diff] [blame] | 233 | * |
Christoph Hellwig | 09595e0 | 2024-07-06 09:52:18 +0200 | [diff] [blame] | 234 | * Returns the maximum number of bytes of the range starting at @paddr that can |
| 235 | * be added to a single segment. |
Bart Van Assche | 9546531 | 2022-10-25 12:17:55 -0700 | [diff] [blame] | 236 | */ |
Bart Van Assche | aa261f2 | 2022-10-25 12:17:54 -0700 | [diff] [blame] | 237 | static inline unsigned get_max_segment_size(const struct queue_limits *lim, |
Christoph Hellwig | 09595e0 | 2024-07-06 09:52:18 +0200 | [diff] [blame] | 238 | phys_addr_t paddr, unsigned int len) |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 239 | { |
Ming Lei | 4a2f704e | 2020-01-11 20:57:43 +0800 | [diff] [blame] | 240 | /* |
Bart Van Assche | 9546531 | 2022-10-25 12:17:55 -0700 | [diff] [blame] | 241 | * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1 |
| 242 | * after having calculated the minimum. |
Ming Lei | 4a2f704e | 2020-01-11 20:57:43 +0800 | [diff] [blame] | 243 | */ |
Christoph Hellwig | 09595e0 | 2024-07-06 09:52:18 +0200 | [diff] [blame] | 244 | return min_t(unsigned long, len, |
| 245 | min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr), |
| 246 | (unsigned long)lim->max_segment_size - 1) + 1); |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 247 | } |
| 248 | |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 249 | /** |
| 250 | * bvec_split_segs - verify whether or not a bvec should be split in the middle |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 251 | * @lim: [in] queue limits to split based on |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 252 | * @bv: [in] bvec to examine |
| 253 | * @nsegs: [in,out] Number of segments in the bio being built. Incremented |
| 254 | * by the number of segments from @bv that may be appended to that |
| 255 | * bio without exceeding @max_segs |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 256 | * @bytes: [in,out] Number of bytes in the bio being built. Incremented |
| 257 | * by the number of bytes from @bv that may be appended to that |
| 258 | * bio without exceeding @max_bytes |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 259 | * @max_segs: [in] upper bound for *@nsegs |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 260 | * @max_bytes: [in] upper bound for *@bytes |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 261 | * |
| 262 | * When splitting a bio, it can happen that a bvec is encountered that is too |
| 263 | * big to fit in a single segment and hence that it has to be split in the |
| 264 | * middle. This function verifies whether or not that should happen. The value |
| 265 | * %true is returned if and only if appending the entire @bv to a bio with |
| 266 | * *@nsegs segments and *@sectors sectors would make that bio unacceptable for |
| 267 | * the block driver. |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 268 | */ |
Bart Van Assche | aa261f2 | 2022-10-25 12:17:54 -0700 | [diff] [blame] | 269 | static bool bvec_split_segs(const struct queue_limits *lim, |
| 270 | const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes, |
| 271 | unsigned max_segs, unsigned max_bytes) |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 272 | { |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 273 | unsigned max_len = min(max_bytes, UINT_MAX) - *bytes; |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 274 | unsigned len = min(bv->bv_len, max_len); |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 275 | unsigned total_len = 0; |
Bart Van Assche | ff9811b | 2019-08-01 15:50:42 -0700 | [diff] [blame] | 276 | unsigned seg_size = 0; |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 277 | |
Bart Van Assche | ff9811b | 2019-08-01 15:50:42 -0700 | [diff] [blame] | 278 | while (len && *nsegs < max_segs) { |
Christoph Hellwig | 09595e0 | 2024-07-06 09:52:18 +0200 | [diff] [blame] | 279 | seg_size = get_max_segment_size(lim, bvec_phys(bv) + total_len, len); |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 280 | |
Bart Van Assche | ff9811b | 2019-08-01 15:50:42 -0700 | [diff] [blame] | 281 | (*nsegs)++; |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 282 | total_len += seg_size; |
| 283 | len -= seg_size; |
| 284 | |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 285 | if ((bv->bv_offset + total_len) & lim->virt_boundary_mask) |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 286 | break; |
| 287 | } |
| 288 | |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 289 | *bytes += total_len; |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 290 | |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 291 | /* tell the caller to split the bvec if it is too big to fit */ |
| 292 | return len > 0 || bv->bv_len > max_len; |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 293 | } |
| 294 | |
Christoph Hellwig | 7ecd2cd | 2024-11-04 07:26:30 +0100 | [diff] [blame] | 295 | static unsigned int bio_split_alignment(struct bio *bio, |
| 296 | const struct queue_limits *lim) |
| 297 | { |
| 298 | if (op_is_write(bio_op(bio)) && lim->zone_write_granularity) |
| 299 | return lim->zone_write_granularity; |
| 300 | return lim->logical_block_size; |
| 301 | } |
| 302 | |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 303 | /** |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 304 | * bio_split_rw_at - check if and where to split a read/write bio |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 305 | * @bio: [in] bio to be split |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 306 | * @lim: [in] queue limits to split based on |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 307 | * @segs: [out] number of segments in the bio with the first half of the sectors |
Christoph Hellwig | a85b363 | 2022-07-27 12:22:58 -0400 | [diff] [blame] | 308 | * @max_bytes: [in] maximum number of bytes per bio |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 309 | * |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 310 | * Find out if @bio needs to be split to fit the queue limits in @lim and a |
| 311 | * maximum size of @max_bytes. Returns a negative error number if @bio can't be |
| 312 | * split, 0 if the bio doesn't have to be split, or a positive sector offset if |
| 313 | * @bio needs to be split. |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 314 | */ |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 315 | int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim, |
| 316 | unsigned *segs, unsigned max_bytes) |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 317 | { |
Jens Axboe | 5014c31 | 2015-09-02 16:46:02 -0600 | [diff] [blame] | 318 | struct bio_vec bv, bvprv, *bvprvp = NULL; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 319 | struct bvec_iter iter; |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 320 | unsigned nsegs = 0, bytes = 0; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 321 | |
Ming Lei | dcebd75 | 2019-02-15 19:13:12 +0800 | [diff] [blame] | 322 | bio_for_each_bvec(bv, bio, iter) { |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 323 | /* |
| 324 | * If the queue doesn't support SG gaps and adding this |
| 325 | * offset would create a gap, disallow it. |
| 326 | */ |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 327 | if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset)) |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 328 | goto split; |
| 329 | |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 330 | if (nsegs < lim->max_segments && |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 331 | bytes + bv.bv_len <= max_bytes && |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 332 | bv.bv_offset + bv.bv_len <= PAGE_SIZE) { |
| 333 | nsegs++; |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 334 | bytes += bv.bv_len; |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 335 | } else { |
| 336 | if (bvec_split_segs(lim, &bv, &nsegs, &bytes, |
| 337 | lim->max_segments, max_bytes)) |
| 338 | goto split; |
Keith Busch | e36f620 | 2016-01-12 15:08:39 -0700 | [diff] [blame] | 339 | } |
| 340 | |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 341 | bvprv = bv; |
Ming Lei | 578270b | 2015-11-24 10:35:29 +0800 | [diff] [blame] | 342 | bvprvp = &bvprv; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 343 | } |
| 344 | |
Christoph Hellwig | d627065 | 2019-06-06 12:29:03 +0200 | [diff] [blame] | 345 | *segs = nsegs; |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 346 | return 0; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 347 | split: |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 348 | if (bio->bi_opf & REQ_ATOMIC) |
| 349 | return -EINVAL; |
| 350 | |
Jens Axboe | 9cea62b | 2023-01-04 08:52:06 -0700 | [diff] [blame] | 351 | /* |
| 352 | * We can't sanely support splitting for a REQ_NOWAIT bio. End it |
| 353 | * with EAGAIN if splitting is required and return an error pointer. |
| 354 | */ |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 355 | if (bio->bi_opf & REQ_NOWAIT) |
| 356 | return -EAGAIN; |
Jens Axboe | 9cea62b | 2023-01-04 08:52:06 -0700 | [diff] [blame] | 357 | |
Ming Lei | bdced43 | 2015-10-20 23:13:52 +0800 | [diff] [blame] | 358 | *segs = nsegs; |
Jeffle Xu | cc29e1b | 2020-11-26 17:18:52 +0800 | [diff] [blame] | 359 | |
| 360 | /* |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 361 | * Individual bvecs might not be logical block aligned. Round down the |
| 362 | * split size so that each bio is properly block size aligned, even if |
| 363 | * we do not use the full hardware limits. |
| 364 | */ |
Christoph Hellwig | 7ecd2cd | 2024-11-04 07:26:30 +0100 | [diff] [blame] | 365 | bytes = ALIGN_DOWN(bytes, bio_split_alignment(bio, lim)); |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 366 | |
| 367 | /* |
Jeffle Xu | cc29e1b | 2020-11-26 17:18:52 +0800 | [diff] [blame] | 368 | * Bio splitting may cause subtle trouble such as hang when doing sync |
| 369 | * iopoll in direct IO routine. Given performance gain of iopoll for |
| 370 | * big IO can be trival, disable iopoll when split needed. |
| 371 | */ |
Christoph Hellwig | 6ce913f | 2021-10-12 13:12:21 +0200 | [diff] [blame] | 372 | bio_clear_polled(bio); |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 373 | return bytes >> SECTOR_SHIFT; |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 374 | } |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 375 | EXPORT_SYMBOL_GPL(bio_split_rw_at); |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 376 | |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 377 | struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, |
| 378 | unsigned *nr_segs) |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 379 | { |
Christoph Hellwig | b35243a | 2024-08-26 19:37:54 +0200 | [diff] [blame] | 380 | return bio_submit_split(bio, |
| 381 | bio_split_rw_at(bio, lim, nr_segs, |
| 382 | get_max_io_size(bio, lim) << SECTOR_SHIFT)); |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 383 | } |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 384 | |
Christoph Hellwig | 1e8a7f6 | 2024-08-26 19:37:56 +0200 | [diff] [blame] | 385 | /* |
| 386 | * REQ_OP_ZONE_APPEND bios must never be split by the block layer. |
| 387 | * |
| 388 | * But we want the nr_segs calculation provided by bio_split_rw_at, and having |
| 389 | * a good sanity check that the submitter built the bio correctly is nice to |
| 390 | * have as well. |
| 391 | */ |
| 392 | struct bio *bio_split_zone_append(struct bio *bio, |
| 393 | const struct queue_limits *lim, unsigned *nr_segs) |
| 394 | { |
Christoph Hellwig | 1e8a7f6 | 2024-08-26 19:37:56 +0200 | [diff] [blame] | 395 | int split_sectors; |
| 396 | |
| 397 | split_sectors = bio_split_rw_at(bio, lim, nr_segs, |
Christoph Hellwig | 559218d | 2024-11-08 16:46:51 +0100 | [diff] [blame] | 398 | lim->max_zone_append_sectors << SECTOR_SHIFT); |
Christoph Hellwig | 1e8a7f6 | 2024-08-26 19:37:56 +0200 | [diff] [blame] | 399 | if (WARN_ON_ONCE(split_sectors > 0)) |
| 400 | split_sectors = -EINVAL; |
| 401 | return bio_submit_split(bio, split_sectors); |
| 402 | } |
| 403 | |
Christoph Hellwig | 60dc5ea | 2024-11-04 07:26:29 +0100 | [diff] [blame] | 404 | struct bio *bio_split_write_zeroes(struct bio *bio, |
| 405 | const struct queue_limits *lim, unsigned *nsegs) |
| 406 | { |
| 407 | unsigned int max_sectors = get_max_io_size(bio, lim); |
| 408 | |
| 409 | *nsegs = 0; |
| 410 | |
| 411 | /* |
| 412 | * An unset limit should normally not happen, as bio submission is keyed |
| 413 | * off having a non-zero limit. But SCSI can clear the limit in the |
| 414 | * I/O completion handler, and we can race and see this. Splitting to a |
| 415 | * zero limit obviously doesn't make sense, so band-aid it here. |
| 416 | */ |
| 417 | if (!max_sectors) |
| 418 | return bio; |
| 419 | if (bio_sectors(bio) <= max_sectors) |
| 420 | return bio; |
| 421 | return bio_submit_split(bio, max_sectors); |
| 422 | } |
| 423 | |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 424 | /** |
Christoph Hellwig | 5a97806 | 2022-07-27 12:22:55 -0400 | [diff] [blame] | 425 | * bio_split_to_limits - split a bio to fit the queue limits |
| 426 | * @bio: bio to be split |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 427 | * |
Christoph Hellwig | 5a97806 | 2022-07-27 12:22:55 -0400 | [diff] [blame] | 428 | * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and |
| 429 | * if so split off a bio fitting the limits from the beginning of @bio and |
| 430 | * return it. @bio is shortened to the remainder and re-submitted. |
| 431 | * |
| 432 | * The split bio is allocated from @q->bio_split, which is provided by the |
| 433 | * block layer. |
Bart Van Assche | dad7758 | 2019-08-01 15:50:41 -0700 | [diff] [blame] | 434 | */ |
Christoph Hellwig | 5a97806 | 2022-07-27 12:22:55 -0400 | [diff] [blame] | 435 | struct bio *bio_split_to_limits(struct bio *bio) |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 436 | { |
| 437 | unsigned int nr_segs; |
| 438 | |
Christoph Hellwig | 2f5a65e | 2024-10-29 15:19:37 +0100 | [diff] [blame] | 439 | return __bio_split_to_limits(bio, bdev_limits(bio->bi_bdev), &nr_segs); |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 440 | } |
Christoph Hellwig | 5a97806 | 2022-07-27 12:22:55 -0400 | [diff] [blame] | 441 | EXPORT_SYMBOL(bio_split_to_limits); |
Kent Overstreet | 54efd50 | 2015-04-23 22:37:18 -0700 | [diff] [blame] | 442 | |
Christoph Hellwig | e9cd19c | 2019-06-06 12:29:02 +0200 | [diff] [blame] | 443 | unsigned int blk_recalc_rq_segments(struct request *rq) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 444 | { |
Christoph Hellwig | 6869875 | 2019-05-21 09:01:43 +0200 | [diff] [blame] | 445 | unsigned int nr_phys_segs = 0; |
Keith Busch | 67927d2 | 2022-06-10 12:58:25 -0700 | [diff] [blame] | 446 | unsigned int bytes = 0; |
Christoph Hellwig | e9cd19c | 2019-06-06 12:29:02 +0200 | [diff] [blame] | 447 | struct req_iterator iter; |
Christoph Hellwig | 6869875 | 2019-05-21 09:01:43 +0200 | [diff] [blame] | 448 | struct bio_vec bv; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 449 | |
Christoph Hellwig | e9cd19c | 2019-06-06 12:29:02 +0200 | [diff] [blame] | 450 | if (!rq->bio) |
Jens Axboe | 1e42807 | 2009-02-23 09:03:10 +0100 | [diff] [blame] | 451 | return 0; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 452 | |
Christoph Hellwig | e9cd19c | 2019-06-06 12:29:02 +0200 | [diff] [blame] | 453 | switch (bio_op(rq->bio)) { |
Chaitanya Kulkarni | a6f0788 | 2016-11-30 12:28:59 -0800 | [diff] [blame] | 454 | case REQ_OP_DISCARD: |
| 455 | case REQ_OP_SECURE_ERASE: |
David Jeffery | a958937f | 2021-02-11 09:38:07 -0500 | [diff] [blame] | 456 | if (queue_max_discard_segments(rq->q) > 1) { |
| 457 | struct bio *bio = rq->bio; |
| 458 | |
| 459 | for_each_bio(bio) |
| 460 | nr_phys_segs++; |
| 461 | return nr_phys_segs; |
| 462 | } |
| 463 | return 1; |
Chaitanya Kulkarni | a6f0788 | 2016-11-30 12:28:59 -0800 | [diff] [blame] | 464 | case REQ_OP_WRITE_ZEROES: |
Christoph Hellwig | f9d03f9 | 2016-12-08 15:20:32 -0700 | [diff] [blame] | 465 | return 0; |
Bart Van Assche | 2d9b02b | 2022-07-14 11:06:30 -0700 | [diff] [blame] | 466 | default: |
| 467 | break; |
Chaitanya Kulkarni | a6f0788 | 2016-11-30 12:28:59 -0800 | [diff] [blame] | 468 | } |
Kent Overstreet | 5cb8850 | 2014-02-07 13:53:46 -0700 | [diff] [blame] | 469 | |
Christoph Hellwig | e9cd19c | 2019-06-06 12:29:02 +0200 | [diff] [blame] | 470 | rq_for_each_bvec(bv, rq, iter) |
Christoph Hellwig | c55ddd9 | 2022-07-27 12:23:00 -0400 | [diff] [blame] | 471 | bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes, |
Bart Van Assche | 708b25b | 2019-08-01 15:50:43 -0700 | [diff] [blame] | 472 | UINT_MAX, UINT_MAX); |
Jens Axboe | 1e42807 | 2009-02-23 09:03:10 +0100 | [diff] [blame] | 473 | return nr_phys_segs; |
| 474 | } |
| 475 | |
Ming Lei | 48d7727 | 2019-02-27 20:40:11 +0800 | [diff] [blame] | 476 | static inline struct scatterlist *blk_next_sg(struct scatterlist **sg, |
Ming Lei | 862e5a5 | 2019-02-15 19:13:13 +0800 | [diff] [blame] | 477 | struct scatterlist *sglist) |
| 478 | { |
| 479 | if (!*sg) |
| 480 | return sglist; |
| 481 | |
| 482 | /* |
| 483 | * If the driver previously mapped a shorter list, we could see a |
| 484 | * termination bit prematurely unless it fully inits the sg table |
| 485 | * on each mapping. We KNOW that there must be more entries here |
| 486 | * or the driver would be buggy, so force clear the termination bit |
| 487 | * to avoid doing a full sg_init_table() in drivers for each command. |
| 488 | */ |
| 489 | sg_unmark_end(*sg); |
| 490 | return sg_next(*sg); |
| 491 | } |
| 492 | |
| 493 | static unsigned blk_bvec_map_sg(struct request_queue *q, |
| 494 | struct bio_vec *bvec, struct scatterlist *sglist, |
| 495 | struct scatterlist **sg) |
| 496 | { |
| 497 | unsigned nbytes = bvec->bv_len; |
Christoph Hellwig | 8a96a0e | 2019-04-11 08:23:27 +0200 | [diff] [blame] | 498 | unsigned nsegs = 0, total = 0; |
Ming Lei | 862e5a5 | 2019-02-15 19:13:13 +0800 | [diff] [blame] | 499 | |
| 500 | while (nbytes > 0) { |
Christoph Hellwig | 8a96a0e | 2019-04-11 08:23:27 +0200 | [diff] [blame] | 501 | unsigned offset = bvec->bv_offset + total; |
Christoph Hellwig | 61353a6 | 2024-07-09 09:01:25 +0200 | [diff] [blame] | 502 | unsigned len = get_max_segment_size(&q->limits, |
| 503 | bvec_phys(bvec) + total, nbytes); |
Christoph Hellwig | f9f7687 | 2019-04-19 08:56:24 +0200 | [diff] [blame] | 504 | struct page *page = bvec->bv_page; |
| 505 | |
| 506 | /* |
| 507 | * Unfortunately a fair number of drivers barf on scatterlists |
| 508 | * that have an offset larger than PAGE_SIZE, despite other |
| 509 | * subsystems dealing with that invariant just fine. For now |
| 510 | * stick to the legacy format where we never present those from |
| 511 | * the block layer, but the code below should be removed once |
| 512 | * these offenders (mostly MMC/SD drivers) are fixed. |
| 513 | */ |
| 514 | page += (offset >> PAGE_SHIFT); |
| 515 | offset &= ~PAGE_MASK; |
Ming Lei | 862e5a5 | 2019-02-15 19:13:13 +0800 | [diff] [blame] | 516 | |
| 517 | *sg = blk_next_sg(sg, sglist); |
Christoph Hellwig | f9f7687 | 2019-04-19 08:56:24 +0200 | [diff] [blame] | 518 | sg_set_page(*sg, page, len, offset); |
Ming Lei | 862e5a5 | 2019-02-15 19:13:13 +0800 | [diff] [blame] | 519 | |
Christoph Hellwig | 8a96a0e | 2019-04-11 08:23:27 +0200 | [diff] [blame] | 520 | total += len; |
| 521 | nbytes -= len; |
Ming Lei | 862e5a5 | 2019-02-15 19:13:13 +0800 | [diff] [blame] | 522 | nsegs++; |
| 523 | } |
| 524 | |
| 525 | return nsegs; |
| 526 | } |
| 527 | |
Ming Lei | 16e3e41 | 2019-03-17 18:01:11 +0800 | [diff] [blame] | 528 | static inline int __blk_bvec_map_sg(struct bio_vec bv, |
| 529 | struct scatterlist *sglist, struct scatterlist **sg) |
| 530 | { |
| 531 | *sg = blk_next_sg(sg, sglist); |
| 532 | sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); |
| 533 | return 1; |
| 534 | } |
| 535 | |
Ming Lei | f6970f8 | 2019-03-17 18:01:12 +0800 | [diff] [blame] | 536 | /* only try to merge bvecs into one sg if they are from two bios */ |
| 537 | static inline bool |
| 538 | __blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec, |
| 539 | struct bio_vec *bvprv, struct scatterlist **sg) |
Asias He | 963ab9e5 | 2012-08-02 23:42:03 +0200 | [diff] [blame] | 540 | { |
| 541 | |
| 542 | int nbytes = bvec->bv_len; |
| 543 | |
Ming Lei | f6970f8 | 2019-03-17 18:01:12 +0800 | [diff] [blame] | 544 | if (!*sg) |
| 545 | return false; |
Asias He | 963ab9e5 | 2012-08-02 23:42:03 +0200 | [diff] [blame] | 546 | |
Ming Lei | f6970f8 | 2019-03-17 18:01:12 +0800 | [diff] [blame] | 547 | if ((*sg)->length + nbytes > queue_max_segment_size(q)) |
| 548 | return false; |
| 549 | |
| 550 | if (!biovec_phys_mergeable(q, bvprv, bvec)) |
| 551 | return false; |
| 552 | |
| 553 | (*sg)->length += nbytes; |
| 554 | |
| 555 | return true; |
Asias He | 963ab9e5 | 2012-08-02 23:42:03 +0200 | [diff] [blame] | 556 | } |
| 557 | |
Kent Overstreet | 5cb8850 | 2014-02-07 13:53:46 -0700 | [diff] [blame] | 558 | static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, |
| 559 | struct scatterlist *sglist, |
| 560 | struct scatterlist **sg) |
| 561 | { |
Kees Cook | 3f649ab | 2020-06-03 13:09:38 -0700 | [diff] [blame] | 562 | struct bio_vec bvec, bvprv = { NULL }; |
Kent Overstreet | 5cb8850 | 2014-02-07 13:53:46 -0700 | [diff] [blame] | 563 | struct bvec_iter iter; |
Christoph Hellwig | 3841746 | 2018-12-13 16:17:10 +0100 | [diff] [blame] | 564 | int nsegs = 0; |
Ming Lei | f6970f8 | 2019-03-17 18:01:12 +0800 | [diff] [blame] | 565 | bool new_bio = false; |
Kent Overstreet | 5cb8850 | 2014-02-07 13:53:46 -0700 | [diff] [blame] | 566 | |
Ming Lei | f6970f8 | 2019-03-17 18:01:12 +0800 | [diff] [blame] | 567 | for_each_bio(bio) { |
| 568 | bio_for_each_bvec(bvec, bio, iter) { |
| 569 | /* |
| 570 | * Only try to merge bvecs from two bios given we |
| 571 | * have done bio internal merge when adding pages |
| 572 | * to bio |
| 573 | */ |
| 574 | if (new_bio && |
| 575 | __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg)) |
| 576 | goto next_bvec; |
| 577 | |
| 578 | if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE) |
| 579 | nsegs += __blk_bvec_map_sg(bvec, sglist, sg); |
| 580 | else |
| 581 | nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg); |
| 582 | next_bvec: |
| 583 | new_bio = false; |
| 584 | } |
Ming Lei | b21e11c | 2019-04-02 10:26:44 +0800 | [diff] [blame] | 585 | if (likely(bio->bi_iter.bi_size)) { |
| 586 | bvprv = bvec; |
| 587 | new_bio = true; |
| 588 | } |
Ming Lei | f6970f8 | 2019-03-17 18:01:12 +0800 | [diff] [blame] | 589 | } |
Kent Overstreet | 5cb8850 | 2014-02-07 13:53:46 -0700 | [diff] [blame] | 590 | |
| 591 | return nsegs; |
| 592 | } |
| 593 | |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 594 | /* |
| 595 | * map a request to scatterlist, return number of sg entries setup. Caller |
| 596 | * must make sure sg can hold rq->nr_phys_segments entries |
| 597 | */ |
Christoph Hellwig | 89de150 | 2020-04-14 09:42:22 +0200 | [diff] [blame] | 598 | int __blk_rq_map_sg(struct request_queue *q, struct request *rq, |
| 599 | struct scatterlist *sglist, struct scatterlist **last_sg) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 600 | { |
Kent Overstreet | 5cb8850 | 2014-02-07 13:53:46 -0700 | [diff] [blame] | 601 | int nsegs = 0; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 602 | |
Christoph Hellwig | f9d03f9 | 2016-12-08 15:20:32 -0700 | [diff] [blame] | 603 | if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) |
Christoph Hellwig | 89de150 | 2020-04-14 09:42:22 +0200 | [diff] [blame] | 604 | nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, last_sg); |
Christoph Hellwig | f9d03f9 | 2016-12-08 15:20:32 -0700 | [diff] [blame] | 605 | else if (rq->bio) |
Christoph Hellwig | 89de150 | 2020-04-14 09:42:22 +0200 | [diff] [blame] | 606 | nsegs = __blk_bios_map_sg(q, rq->bio, sglist, last_sg); |
FUJITA Tomonori | f18573a | 2008-04-11 12:56:52 +0200 | [diff] [blame] | 607 | |
Christoph Hellwig | 89de150 | 2020-04-14 09:42:22 +0200 | [diff] [blame] | 608 | if (*last_sg) |
| 609 | sg_mark_end(*last_sg); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 610 | |
Ming Lei | 12e57f5 | 2015-11-24 10:35:31 +0800 | [diff] [blame] | 611 | /* |
| 612 | * Something must have been wrong if the figured number of |
| 613 | * segment is bigger than number of req's physical segments |
| 614 | */ |
Christoph Hellwig | f9d03f9 | 2016-12-08 15:20:32 -0700 | [diff] [blame] | 615 | WARN_ON(nsegs > blk_rq_nr_phys_segments(rq)); |
Ming Lei | 12e57f5 | 2015-11-24 10:35:31 +0800 | [diff] [blame] | 616 | |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 617 | return nsegs; |
| 618 | } |
Christoph Hellwig | 89de150 | 2020-04-14 09:42:22 +0200 | [diff] [blame] | 619 | EXPORT_SYMBOL(__blk_rq_map_sg); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 620 | |
Christoph Hellwig | badf7f6 | 2021-09-20 14:33:26 +0200 | [diff] [blame] | 621 | static inline unsigned int blk_rq_get_max_sectors(struct request *rq, |
| 622 | sector_t offset) |
| 623 | { |
| 624 | struct request_queue *q = rq->q; |
John Garry | f70167a | 2024-06-20 12:53:51 +0000 | [diff] [blame] | 625 | struct queue_limits *lim = &q->limits; |
| 626 | unsigned int max_sectors, boundary_sectors; |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 627 | bool is_atomic = rq->cmd_flags & REQ_ATOMIC; |
Christoph Hellwig | badf7f6 | 2021-09-20 14:33:26 +0200 | [diff] [blame] | 628 | |
| 629 | if (blk_rq_is_passthrough(rq)) |
| 630 | return q->limits.max_hw_sectors; |
| 631 | |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 632 | boundary_sectors = blk_boundary_sectors(lim, is_atomic); |
John Garry | 8d1dfd5 | 2024-06-20 12:53:50 +0000 | [diff] [blame] | 633 | max_sectors = blk_queue_get_max_sectors(rq); |
| 634 | |
John Garry | f70167a | 2024-06-20 12:53:51 +0000 | [diff] [blame] | 635 | if (!boundary_sectors || |
Christoph Hellwig | badf7f6 | 2021-09-20 14:33:26 +0200 | [diff] [blame] | 636 | req_op(rq) == REQ_OP_DISCARD || |
| 637 | req_op(rq) == REQ_OP_SECURE_ERASE) |
Christoph Hellwig | c887519 | 2022-06-14 11:09:31 +0200 | [diff] [blame] | 638 | return max_sectors; |
| 639 | return min(max_sectors, |
John Garry | f70167a | 2024-06-20 12:53:51 +0000 | [diff] [blame] | 640 | blk_boundary_sectors_left(offset, boundary_sectors)); |
Christoph Hellwig | badf7f6 | 2021-09-20 14:33:26 +0200 | [diff] [blame] | 641 | } |
| 642 | |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 643 | static inline int ll_new_hw_segment(struct request *req, struct bio *bio, |
| 644 | unsigned int nr_phys_segs) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 645 | { |
Tejun Heo | 6b2b045 | 2022-03-14 14:30:11 -1000 | [diff] [blame] | 646 | if (!blk_cgroup_mergeable(req, bio)) |
| 647 | goto no_merge; |
| 648 | |
Ming Lei | 2705dfb | 2021-06-28 10:33:12 +0800 | [diff] [blame] | 649 | if (blk_integrity_merge_bio(req->q, req, bio) == false) |
Martin K. Petersen | 13f05c8 | 2010-09-10 20:50:10 +0200 | [diff] [blame] | 650 | goto no_merge; |
| 651 | |
Ming Lei | 2705dfb | 2021-06-28 10:33:12 +0800 | [diff] [blame] | 652 | /* discard request merge won't add new segment */ |
| 653 | if (req_op(req) == REQ_OP_DISCARD) |
| 654 | return 1; |
| 655 | |
| 656 | if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req)) |
Martin K. Petersen | 13f05c8 | 2010-09-10 20:50:10 +0200 | [diff] [blame] | 657 | goto no_merge; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 658 | |
| 659 | /* |
| 660 | * This will form the start of a new hw segment. Bump both |
| 661 | * counters. |
| 662 | */ |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 663 | req->nr_phys_segments += nr_phys_segs; |
Keith Busch | d148d750 | 2024-09-13 11:28:48 -0700 | [diff] [blame] | 664 | if (bio_integrity(bio)) |
| 665 | req->nr_integrity_segments += blk_rq_count_integrity_sg(req->q, |
| 666 | bio); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 667 | return 1; |
Martin K. Petersen | 13f05c8 | 2010-09-10 20:50:10 +0200 | [diff] [blame] | 668 | |
| 669 | no_merge: |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 670 | req_set_nomerge(req->q, req); |
Martin K. Petersen | 13f05c8 | 2010-09-10 20:50:10 +0200 | [diff] [blame] | 671 | return 0; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 672 | } |
| 673 | |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 674 | int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 675 | { |
Jens Axboe | 5e7c427 | 2015-09-03 19:28:20 +0300 | [diff] [blame] | 676 | if (req_gap_back_merge(req, bio)) |
| 677 | return 0; |
Sagi Grimberg | 7f39add | 2015-09-11 09:03:04 -0600 | [diff] [blame] | 678 | if (blk_integrity_rq(req) && |
| 679 | integrity_req_gap_back_merge(req, bio)) |
| 680 | return 0; |
Satya Tangirala | a892c8d | 2020-05-14 00:37:18 +0000 | [diff] [blame] | 681 | if (!bio_crypt_ctx_back_mergeable(req, bio)) |
| 682 | return 0; |
Martin K. Petersen | f31dc1c | 2012-09-18 12:19:26 -0400 | [diff] [blame] | 683 | if (blk_rq_sectors(req) + bio_sectors(bio) > |
Damien Le Moal | 17007f3 | 2016-07-20 21:40:47 -0600 | [diff] [blame] | 684 | blk_rq_get_max_sectors(req, blk_rq_pos(req))) { |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 685 | req_set_nomerge(req->q, req); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 686 | return 0; |
| 687 | } |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 688 | |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 689 | return ll_new_hw_segment(req, bio, nr_segs); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 690 | } |
| 691 | |
Christoph Hellwig | eda5cc9 | 2020-10-06 09:07:19 +0200 | [diff] [blame] | 692 | static int ll_front_merge_fn(struct request *req, struct bio *bio, |
| 693 | unsigned int nr_segs) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 694 | { |
Jens Axboe | 5e7c427 | 2015-09-03 19:28:20 +0300 | [diff] [blame] | 695 | if (req_gap_front_merge(req, bio)) |
| 696 | return 0; |
Sagi Grimberg | 7f39add | 2015-09-11 09:03:04 -0600 | [diff] [blame] | 697 | if (blk_integrity_rq(req) && |
| 698 | integrity_req_gap_front_merge(req, bio)) |
| 699 | return 0; |
Satya Tangirala | a892c8d | 2020-05-14 00:37:18 +0000 | [diff] [blame] | 700 | if (!bio_crypt_ctx_front_mergeable(req, bio)) |
| 701 | return 0; |
Martin K. Petersen | f31dc1c | 2012-09-18 12:19:26 -0400 | [diff] [blame] | 702 | if (blk_rq_sectors(req) + bio_sectors(bio) > |
Damien Le Moal | 17007f3 | 2016-07-20 21:40:47 -0600 | [diff] [blame] | 703 | blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) { |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 704 | req_set_nomerge(req->q, req); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 705 | return 0; |
| 706 | } |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 707 | |
Christoph Hellwig | 14ccb66 | 2019-06-06 12:29:01 +0200 | [diff] [blame] | 708 | return ll_new_hw_segment(req, bio, nr_segs); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 709 | } |
| 710 | |
Jens Axboe | 445251d | 2018-02-01 14:01:02 -0700 | [diff] [blame] | 711 | static bool req_attempt_discard_merge(struct request_queue *q, struct request *req, |
| 712 | struct request *next) |
| 713 | { |
| 714 | unsigned short segments = blk_rq_nr_discard_segments(req); |
| 715 | |
| 716 | if (segments >= queue_max_discard_segments(q)) |
| 717 | goto no_merge; |
| 718 | if (blk_rq_sectors(req) + bio_sectors(next->bio) > |
| 719 | blk_rq_get_max_sectors(req, blk_rq_pos(req))) |
| 720 | goto no_merge; |
| 721 | |
| 722 | req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next); |
| 723 | return true; |
| 724 | no_merge: |
| 725 | req_set_nomerge(q, req); |
| 726 | return false; |
| 727 | } |
| 728 | |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 729 | static int ll_merge_requests_fn(struct request_queue *q, struct request *req, |
| 730 | struct request *next) |
| 731 | { |
| 732 | int total_phys_segments; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 733 | |
Jens Axboe | 5e7c427 | 2015-09-03 19:28:20 +0300 | [diff] [blame] | 734 | if (req_gap_back_merge(req, next->bio)) |
Keith Busch | 854fbb9 | 2015-02-11 08:20:13 -0700 | [diff] [blame] | 735 | return 0; |
| 736 | |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 737 | /* |
| 738 | * Will it become too large? |
| 739 | */ |
Martin K. Petersen | f31dc1c | 2012-09-18 12:19:26 -0400 | [diff] [blame] | 740 | if ((blk_rq_sectors(req) + blk_rq_sectors(next)) > |
Damien Le Moal | 17007f3 | 2016-07-20 21:40:47 -0600 | [diff] [blame] | 741 | blk_rq_get_max_sectors(req, blk_rq_pos(req))) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 742 | return 0; |
| 743 | |
| 744 | total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; |
Ming Lei | 943b40c | 2020-08-17 17:52:39 +0800 | [diff] [blame] | 745 | if (total_phys_segments > blk_rq_get_max_segments(req)) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 746 | return 0; |
| 747 | |
Tejun Heo | 6b2b045 | 2022-03-14 14:30:11 -1000 | [diff] [blame] | 748 | if (!blk_cgroup_mergeable(req, next->bio)) |
| 749 | return 0; |
| 750 | |
Martin K. Petersen | 4eaf99b | 2014-09-26 19:20:06 -0400 | [diff] [blame] | 751 | if (blk_integrity_merge_rq(q, req, next) == false) |
Martin K. Petersen | 13f05c8 | 2010-09-10 20:50:10 +0200 | [diff] [blame] | 752 | return 0; |
| 753 | |
Satya Tangirala | a892c8d | 2020-05-14 00:37:18 +0000 | [diff] [blame] | 754 | if (!bio_crypt_ctx_merge_rq(req, next)) |
| 755 | return 0; |
| 756 | |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 757 | /* Merge is OK... */ |
| 758 | req->nr_phys_segments = total_phys_segments; |
Keith Busch | d148d750 | 2024-09-13 11:28:48 -0700 | [diff] [blame] | 759 | req->nr_integrity_segments += next->nr_integrity_segments; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 760 | return 1; |
| 761 | } |
| 762 | |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 763 | /** |
| 764 | * blk_rq_set_mixed_merge - mark a request as mixed merge |
| 765 | * @rq: request to mark as mixed merge |
| 766 | * |
| 767 | * Description: |
| 768 | * @rq is about to be mixed merged. Make sure the attributes |
| 769 | * which can be mixed are set in each bio and mark @rq as mixed |
| 770 | * merged. |
| 771 | */ |
John Garry | dc53d9e | 2024-03-25 08:35:01 +0000 | [diff] [blame] | 772 | static void blk_rq_set_mixed_merge(struct request *rq) |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 773 | { |
Bart Van Assche | 16458cf | 2022-07-14 11:06:32 -0700 | [diff] [blame] | 774 | blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK; |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 775 | struct bio *bio; |
| 776 | |
Christoph Hellwig | e806402 | 2016-10-20 15:12:13 +0200 | [diff] [blame] | 777 | if (rq->rq_flags & RQF_MIXED_MERGE) |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 778 | return; |
| 779 | |
| 780 | /* |
| 781 | * @rq will no longer represent mixable attributes for all the |
| 782 | * contained bios. It will just track those of the first one. |
| 783 | * Distributes the attributs to each bio. |
| 784 | */ |
| 785 | for (bio = rq->bio; bio; bio = bio->bi_next) { |
Jens Axboe | 1eff9d3 | 2016-08-05 15:35:16 -0600 | [diff] [blame] | 786 | WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) && |
| 787 | (bio->bi_opf & REQ_FAILFAST_MASK) != ff); |
| 788 | bio->bi_opf |= ff; |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 789 | } |
Christoph Hellwig | e806402 | 2016-10-20 15:12:13 +0200 | [diff] [blame] | 790 | rq->rq_flags |= RQF_MIXED_MERGE; |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 791 | } |
| 792 | |
Jens Axboe | f3ca738 | 2023-02-16 19:39:15 -0700 | [diff] [blame] | 793 | static inline blk_opf_t bio_failfast(const struct bio *bio) |
Ming Lei | 3ce6a11 | 2023-02-09 20:55:27 +0800 | [diff] [blame] | 794 | { |
| 795 | if (bio->bi_opf & REQ_RAHEAD) |
| 796 | return REQ_FAILFAST_MASK; |
| 797 | |
| 798 | return bio->bi_opf & REQ_FAILFAST_MASK; |
| 799 | } |
| 800 | |
| 801 | /* |
| 802 | * After we are marked as MIXED_MERGE, any new RA bio has to be updated |
| 803 | * as failfast, and request's failfast has to be updated in case of |
| 804 | * front merge. |
| 805 | */ |
| 806 | static inline void blk_update_mixed_merge(struct request *req, |
| 807 | struct bio *bio, bool front_merge) |
| 808 | { |
| 809 | if (req->rq_flags & RQF_MIXED_MERGE) { |
| 810 | if (bio->bi_opf & REQ_RAHEAD) |
| 811 | bio->bi_opf |= REQ_FAILFAST_MASK; |
| 812 | |
| 813 | if (front_merge) { |
| 814 | req->cmd_flags &= ~REQ_FAILFAST_MASK; |
| 815 | req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK; |
| 816 | } |
| 817 | } |
| 818 | } |
| 819 | |
Konstantin Khlebnikov | b9c54f5 | 2020-05-27 07:24:15 +0200 | [diff] [blame] | 820 | static void blk_account_io_merge_request(struct request *req) |
Jerome Marchand | 26308ea | 2009-03-27 10:31:51 +0100 | [diff] [blame] | 821 | { |
Jens Axboe | e3569ec | 2024-10-03 07:29:49 -0600 | [diff] [blame] | 822 | if (req->rq_flags & RQF_IO_STAT) { |
Mike Snitzer | 112f158 | 2018-12-06 11:41:18 -0500 | [diff] [blame] | 823 | part_stat_lock(); |
Konstantin Khlebnikov | b9c54f5 | 2020-05-27 07:24:15 +0200 | [diff] [blame] | 824 | part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); |
Yu Kuai | 99dc422 | 2024-05-09 20:37:16 +0800 | [diff] [blame] | 825 | part_stat_local_dec(req->part, |
| 826 | in_flight[op_is_write(req_op(req))]); |
Jerome Marchand | 26308ea | 2009-03-27 10:31:51 +0100 | [diff] [blame] | 827 | part_stat_unlock(); |
| 828 | } |
| 829 | } |
Konstantin Khlebnikov | b9c54f5 | 2020-05-27 07:24:15 +0200 | [diff] [blame] | 830 | |
Eric Biggers | e96c0d8 | 2018-11-14 17:19:46 -0800 | [diff] [blame] | 831 | static enum elv_merge blk_try_req_merge(struct request *req, |
| 832 | struct request *next) |
Jianchao Wang | 69840466 | 2018-10-27 19:52:14 +0800 | [diff] [blame] | 833 | { |
| 834 | if (blk_discard_mergable(req)) |
| 835 | return ELEVATOR_DISCARD_MERGE; |
| 836 | else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) |
| 837 | return ELEVATOR_BACK_MERGE; |
| 838 | |
| 839 | return ELEVATOR_NO_MERGE; |
| 840 | } |
Jerome Marchand | 26308ea | 2009-03-27 10:31:51 +0100 | [diff] [blame] | 841 | |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 842 | static bool blk_atomic_write_mergeable_rq_bio(struct request *rq, |
| 843 | struct bio *bio) |
| 844 | { |
| 845 | return (rq->cmd_flags & REQ_ATOMIC) == (bio->bi_opf & REQ_ATOMIC); |
| 846 | } |
| 847 | |
| 848 | static bool blk_atomic_write_mergeable_rqs(struct request *rq, |
| 849 | struct request *next) |
| 850 | { |
| 851 | return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC); |
| 852 | } |
| 853 | |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 854 | /* |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 855 | * For non-mq, this has to be called with the request spinlock acquired. |
| 856 | * For mq with scheduling, the appropriate queue wide lock should be held. |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 857 | */ |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 858 | static struct request *attempt_merge(struct request_queue *q, |
| 859 | struct request *req, struct request *next) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 860 | { |
| 861 | if (!rq_mergeable(req) || !rq_mergeable(next)) |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 862 | return NULL; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 863 | |
Christoph Hellwig | 288dab8 | 2016-06-09 16:00:36 +0200 | [diff] [blame] | 864 | if (req_op(req) != req_op(next)) |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 865 | return NULL; |
Martin K. Petersen | f31dc1c | 2012-09-18 12:19:26 -0400 | [diff] [blame] | 866 | |
Christoph Hellwig | 79bb1db | 2021-11-26 13:17:59 +0100 | [diff] [blame] | 867 | if (rq_data_dir(req) != rq_data_dir(next)) |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 868 | return NULL; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 869 | |
Christoph Hellwig | 61952bb | 2024-11-12 18:00:38 +0100 | [diff] [blame] | 870 | if (req->bio && next->bio) { |
| 871 | /* Don't merge requests with different write hints. */ |
| 872 | if (req->bio->bi_write_hint != next->bio->bi_write_hint) |
| 873 | return NULL; |
Christoph Hellwig | 6975c1a | 2024-11-12 18:00:39 +0100 | [diff] [blame] | 874 | if (req->bio->bi_ioprio != next->bio->bi_ioprio) |
| 875 | return NULL; |
Christoph Hellwig | 61952bb | 2024-11-12 18:00:38 +0100 | [diff] [blame] | 876 | } |
Bart Van Assche | 4498135 | 2024-02-02 12:39:25 -0800 | [diff] [blame] | 877 | |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 878 | if (!blk_atomic_write_mergeable_rqs(req, next)) |
| 879 | return NULL; |
| 880 | |
Jens Axboe | cb6934f | 2017-06-27 09:22:02 -0600 | [diff] [blame] | 881 | /* |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 882 | * If we are allowed to merge, then append bio list |
| 883 | * from next to rq and release next. merge_requests_fn |
| 884 | * will have updated segment counts, update sector |
Jens Axboe | 445251d | 2018-02-01 14:01:02 -0700 | [diff] [blame] | 885 | * counts here. Handle DISCARDs separately, as they |
| 886 | * have separate settings. |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 887 | */ |
Jianchao Wang | 69840466 | 2018-10-27 19:52:14 +0800 | [diff] [blame] | 888 | |
| 889 | switch (blk_try_req_merge(req, next)) { |
| 890 | case ELEVATOR_DISCARD_MERGE: |
Jens Axboe | 445251d | 2018-02-01 14:01:02 -0700 | [diff] [blame] | 891 | if (!req_attempt_discard_merge(q, req, next)) |
| 892 | return NULL; |
Jianchao Wang | 69840466 | 2018-10-27 19:52:14 +0800 | [diff] [blame] | 893 | break; |
| 894 | case ELEVATOR_BACK_MERGE: |
| 895 | if (!ll_merge_requests_fn(q, req, next)) |
| 896 | return NULL; |
| 897 | break; |
| 898 | default: |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 899 | return NULL; |
Jianchao Wang | 69840466 | 2018-10-27 19:52:14 +0800 | [diff] [blame] | 900 | } |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 901 | |
| 902 | /* |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 903 | * If failfast settings disagree or any of the two is already |
| 904 | * a mixed merge, mark both as mixed before proceeding. This |
| 905 | * makes sure that all involved bios have mixable attributes |
| 906 | * set properly. |
| 907 | */ |
Christoph Hellwig | e806402 | 2016-10-20 15:12:13 +0200 | [diff] [blame] | 908 | if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) || |
Tejun Heo | 80a761f | 2009-07-03 17:48:17 +0900 | [diff] [blame] | 909 | (req->cmd_flags & REQ_FAILFAST_MASK) != |
| 910 | (next->cmd_flags & REQ_FAILFAST_MASK)) { |
| 911 | blk_rq_set_mixed_merge(req); |
| 912 | blk_rq_set_mixed_merge(next); |
| 913 | } |
| 914 | |
| 915 | /* |
Omar Sandoval | 522a777 | 2018-05-09 02:08:53 -0700 | [diff] [blame] | 916 | * At this point we have either done a back merge or front merge. We |
| 917 | * need the smaller start_time_ns of the merged requests to be the |
| 918 | * current request for accounting purposes. |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 919 | */ |
Omar Sandoval | 522a777 | 2018-05-09 02:08:53 -0700 | [diff] [blame] | 920 | if (next->start_time_ns < req->start_time_ns) |
| 921 | req->start_time_ns = next->start_time_ns; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 922 | |
| 923 | req->biotail->bi_next = next->bio; |
| 924 | req->biotail = next->biotail; |
| 925 | |
Tejun Heo | a2dec7b | 2009-05-07 22:24:44 +0900 | [diff] [blame] | 926 | req->__data_len += blk_rq_bytes(next); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 927 | |
Ming Lei | 2a5cf35 | 2018-12-01 00:38:18 +0800 | [diff] [blame] | 928 | if (!blk_discard_mergable(req)) |
Jens Axboe | 445251d | 2018-02-01 14:01:02 -0700 | [diff] [blame] | 929 | elv_merge_requests(q, req, next); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 930 | |
Eric Biggers | 9cd1e56 | 2023-03-15 11:39:02 -0700 | [diff] [blame] | 931 | blk_crypto_rq_put_keyslot(next); |
| 932 | |
Jerome Marchand | 42dad76 | 2009-04-22 14:01:49 +0200 | [diff] [blame] | 933 | /* |
| 934 | * 'next' is going away, so update stats accordingly |
| 935 | */ |
Konstantin Khlebnikov | b9c54f5 | 2020-05-27 07:24:15 +0200 | [diff] [blame] | 936 | blk_account_io_merge_request(next); |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 937 | |
Christoph Hellwig | a54895f | 2020-12-03 17:21:39 +0100 | [diff] [blame] | 938 | trace_block_rq_merge(next); |
Jan Kara | f3bdc62 | 2020-06-17 15:58:23 +0200 | [diff] [blame] | 939 | |
Jens Axboe | e4d750c | 2017-02-03 09:48:28 -0700 | [diff] [blame] | 940 | /* |
| 941 | * ownership of bio passed from next to req, return 'next' for |
| 942 | * the caller to free |
| 943 | */ |
Boaz Harrosh | 1cd96c2 | 2009-03-24 12:35:07 +0100 | [diff] [blame] | 944 | next->bio = NULL; |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 945 | return next; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 946 | } |
| 947 | |
Christoph Hellwig | eda5cc9 | 2020-10-06 09:07:19 +0200 | [diff] [blame] | 948 | static struct request *attempt_back_merge(struct request_queue *q, |
| 949 | struct request *rq) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 950 | { |
| 951 | struct request *next = elv_latter_request(q, rq); |
| 952 | |
| 953 | if (next) |
| 954 | return attempt_merge(q, rq, next); |
| 955 | |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 956 | return NULL; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 957 | } |
| 958 | |
Christoph Hellwig | eda5cc9 | 2020-10-06 09:07:19 +0200 | [diff] [blame] | 959 | static struct request *attempt_front_merge(struct request_queue *q, |
| 960 | struct request *rq) |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 961 | { |
| 962 | struct request *prev = elv_former_request(q, rq); |
| 963 | |
| 964 | if (prev) |
| 965 | return attempt_merge(q, prev, rq); |
| 966 | |
Jens Axboe | b973cb7 | 2017-02-02 08:54:40 -0700 | [diff] [blame] | 967 | return NULL; |
Jens Axboe | d6d4819 | 2008-01-29 14:04:06 +0100 | [diff] [blame] | 968 | } |
Jens Axboe | 5e84ea3 | 2011-03-21 10:14:27 +0100 | [diff] [blame] | 969 | |
Jan Kara | fd2ef39 | 2021-06-23 11:36:34 +0200 | [diff] [blame] | 970 | /* |
| 971 | * Try to merge 'next' into 'rq'. Return true if the merge happened, false |
| 972 | * otherwise. The caller is responsible for freeing 'next' if the merge |
| 973 | * happened. |
| 974 | */ |
| 975 | bool blk_attempt_req_merge(struct request_queue *q, struct request *rq, |
| 976 | struct request *next) |
Jens Axboe | 5e84ea3 | 2011-03-21 10:14:27 +0100 | [diff] [blame] | 977 | { |
Jan Kara | fd2ef39 | 2021-06-23 11:36:34 +0200 | [diff] [blame] | 978 | return attempt_merge(q, rq, next); |
Jens Axboe | 5e84ea3 | 2011-03-21 10:14:27 +0100 | [diff] [blame] | 979 | } |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 980 | |
| 981 | bool blk_rq_merge_ok(struct request *rq, struct bio *bio) |
| 982 | { |
Martin K. Petersen | e2a60da | 2012-09-18 12:19:25 -0400 | [diff] [blame] | 983 | if (!rq_mergeable(rq) || !bio_mergeable(bio)) |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 984 | return false; |
| 985 | |
Christoph Hellwig | 288dab8 | 2016-06-09 16:00:36 +0200 | [diff] [blame] | 986 | if (req_op(rq) != bio_op(bio)) |
Martin K. Petersen | f31dc1c | 2012-09-18 12:19:26 -0400 | [diff] [blame] | 987 | return false; |
| 988 | |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 989 | /* different data direction or already started, don't merge */ |
| 990 | if (bio_data_dir(bio) != rq_data_dir(rq)) |
| 991 | return false; |
| 992 | |
Tejun Heo | 6b2b045 | 2022-03-14 14:30:11 -1000 | [diff] [blame] | 993 | /* don't merge across cgroup boundaries */ |
| 994 | if (!blk_cgroup_mergeable(rq, bio)) |
| 995 | return false; |
| 996 | |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 997 | /* only merge integrity protected bio into ditto rq */ |
Martin K. Petersen | 4eaf99b | 2014-09-26 19:20:06 -0400 | [diff] [blame] | 998 | if (blk_integrity_merge_bio(rq->q, rq, bio) == false) |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 999 | return false; |
| 1000 | |
Satya Tangirala | a892c8d | 2020-05-14 00:37:18 +0000 | [diff] [blame] | 1001 | /* Only merge if the crypt contexts are compatible */ |
| 1002 | if (!bio_crypt_rq_ctx_compatible(rq, bio)) |
| 1003 | return false; |
| 1004 | |
Christoph Hellwig | 61952bb | 2024-11-12 18:00:38 +0100 | [diff] [blame] | 1005 | if (rq->bio) { |
| 1006 | /* Don't merge requests with different write hints. */ |
| 1007 | if (rq->bio->bi_write_hint != bio->bi_write_hint) |
| 1008 | return false; |
Christoph Hellwig | 6975c1a | 2024-11-12 18:00:39 +0100 | [diff] [blame] | 1009 | if (rq->bio->bi_ioprio != bio->bi_ioprio) |
| 1010 | return false; |
Christoph Hellwig | 61952bb | 2024-11-12 18:00:38 +0100 | [diff] [blame] | 1011 | } |
Bart Van Assche | 4498135 | 2024-02-02 12:39:25 -0800 | [diff] [blame] | 1012 | |
John Garry | 9da3d1e | 2024-06-20 12:53:54 +0000 | [diff] [blame] | 1013 | if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false) |
| 1014 | return false; |
| 1015 | |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 1016 | return true; |
| 1017 | } |
| 1018 | |
Christoph Hellwig | 34fe7c0 | 2017-02-08 14:46:48 +0100 | [diff] [blame] | 1019 | enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 1020 | { |
Jianchao Wang | 69840466 | 2018-10-27 19:52:14 +0800 | [diff] [blame] | 1021 | if (blk_discard_mergable(rq)) |
Christoph Hellwig | 1e73973 | 2017-02-08 14:46:49 +0100 | [diff] [blame] | 1022 | return ELEVATOR_DISCARD_MERGE; |
| 1023 | else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 1024 | return ELEVATOR_BACK_MERGE; |
Kent Overstreet | 4f024f3 | 2013-10-11 15:44:27 -0700 | [diff] [blame] | 1025 | else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) |
Tejun Heo | 050c8ea | 2012-02-08 09:19:38 +0100 | [diff] [blame] | 1026 | return ELEVATOR_FRONT_MERGE; |
| 1027 | return ELEVATOR_NO_MERGE; |
| 1028 | } |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1029 | |
| 1030 | static void blk_account_io_merge_bio(struct request *req) |
| 1031 | { |
Jens Axboe | e3569ec | 2024-10-03 07:29:49 -0600 | [diff] [blame] | 1032 | if (req->rq_flags & RQF_IO_STAT) { |
| 1033 | part_stat_lock(); |
| 1034 | part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); |
| 1035 | part_stat_unlock(); |
| 1036 | } |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1037 | } |
| 1038 | |
Damien Le Moal | dd850ff | 2024-04-08 10:41:05 +0900 | [diff] [blame] | 1039 | enum bio_merge_status bio_attempt_back_merge(struct request *req, |
Christoph Hellwig | eda5cc9 | 2020-10-06 09:07:19 +0200 | [diff] [blame] | 1040 | struct bio *bio, unsigned int nr_segs) |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1041 | { |
Ming Lei | 3ce6a11 | 2023-02-09 20:55:27 +0800 | [diff] [blame] | 1042 | const blk_opf_t ff = bio_failfast(bio); |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1043 | |
| 1044 | if (!ll_back_merge_fn(req, bio, nr_segs)) |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1045 | return BIO_MERGE_FAILED; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1046 | |
Christoph Hellwig | e8a676d | 2020-12-03 17:21:36 +0100 | [diff] [blame] | 1047 | trace_block_bio_backmerge(bio); |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1048 | rq_qos_merge(req->q, req, bio); |
| 1049 | |
| 1050 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) |
| 1051 | blk_rq_set_mixed_merge(req); |
| 1052 | |
Ming Lei | 3ce6a11 | 2023-02-09 20:55:27 +0800 | [diff] [blame] | 1053 | blk_update_mixed_merge(req, bio, false); |
| 1054 | |
Damien Le Moal | dd291d7 | 2024-04-08 10:41:07 +0900 | [diff] [blame] | 1055 | if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING) |
| 1056 | blk_zone_write_plug_bio_merged(bio); |
| 1057 | |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1058 | req->biotail->bi_next = bio; |
| 1059 | req->biotail = bio; |
| 1060 | req->__data_len += bio->bi_iter.bi_size; |
| 1061 | |
| 1062 | bio_crypt_free_ctx(bio); |
| 1063 | |
| 1064 | blk_account_io_merge_bio(req); |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1065 | return BIO_MERGE_OK; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1066 | } |
| 1067 | |
Christoph Hellwig | eda5cc9 | 2020-10-06 09:07:19 +0200 | [diff] [blame] | 1068 | static enum bio_merge_status bio_attempt_front_merge(struct request *req, |
| 1069 | struct bio *bio, unsigned int nr_segs) |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1070 | { |
Ming Lei | 3ce6a11 | 2023-02-09 20:55:27 +0800 | [diff] [blame] | 1071 | const blk_opf_t ff = bio_failfast(bio); |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1072 | |
Damien Le Moal | dd291d7 | 2024-04-08 10:41:07 +0900 | [diff] [blame] | 1073 | /* |
| 1074 | * A front merge for writes to sequential zones of a zoned block device |
| 1075 | * can happen only if the user submitted writes out of order. Do not |
| 1076 | * merge such write to let it fail. |
| 1077 | */ |
| 1078 | if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING) |
| 1079 | return BIO_MERGE_FAILED; |
| 1080 | |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1081 | if (!ll_front_merge_fn(req, bio, nr_segs)) |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1082 | return BIO_MERGE_FAILED; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1083 | |
Christoph Hellwig | e8a676d | 2020-12-03 17:21:36 +0100 | [diff] [blame] | 1084 | trace_block_bio_frontmerge(bio); |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1085 | rq_qos_merge(req->q, req, bio); |
| 1086 | |
| 1087 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) |
| 1088 | blk_rq_set_mixed_merge(req); |
| 1089 | |
Ming Lei | 3ce6a11 | 2023-02-09 20:55:27 +0800 | [diff] [blame] | 1090 | blk_update_mixed_merge(req, bio, true); |
| 1091 | |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1092 | bio->bi_next = req->bio; |
| 1093 | req->bio = bio; |
| 1094 | |
| 1095 | req->__sector = bio->bi_iter.bi_sector; |
| 1096 | req->__data_len += bio->bi_iter.bi_size; |
| 1097 | |
| 1098 | bio_crypt_do_front_merge(req, bio); |
| 1099 | |
| 1100 | blk_account_io_merge_bio(req); |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1101 | return BIO_MERGE_OK; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1102 | } |
| 1103 | |
Christoph Hellwig | eda5cc9 | 2020-10-06 09:07:19 +0200 | [diff] [blame] | 1104 | static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q, |
| 1105 | struct request *req, struct bio *bio) |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1106 | { |
| 1107 | unsigned short segments = blk_rq_nr_discard_segments(req); |
| 1108 | |
| 1109 | if (segments >= queue_max_discard_segments(q)) |
| 1110 | goto no_merge; |
| 1111 | if (blk_rq_sectors(req) + bio_sectors(bio) > |
| 1112 | blk_rq_get_max_sectors(req, blk_rq_pos(req))) |
| 1113 | goto no_merge; |
| 1114 | |
| 1115 | rq_qos_merge(q, req, bio); |
| 1116 | |
| 1117 | req->biotail->bi_next = bio; |
| 1118 | req->biotail = bio; |
| 1119 | req->__data_len += bio->bi_iter.bi_size; |
| 1120 | req->nr_phys_segments = segments + 1; |
| 1121 | |
| 1122 | blk_account_io_merge_bio(req); |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1123 | return BIO_MERGE_OK; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1124 | no_merge: |
| 1125 | req_set_nomerge(q, req); |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1126 | return BIO_MERGE_FAILED; |
| 1127 | } |
| 1128 | |
| 1129 | static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q, |
| 1130 | struct request *rq, |
| 1131 | struct bio *bio, |
| 1132 | unsigned int nr_segs, |
| 1133 | bool sched_allow_merge) |
| 1134 | { |
| 1135 | if (!blk_rq_merge_ok(rq, bio)) |
| 1136 | return BIO_MERGE_NONE; |
| 1137 | |
| 1138 | switch (blk_try_merge(rq, bio)) { |
| 1139 | case ELEVATOR_BACK_MERGE: |
Baolin Wang | 265600b | 2020-09-02 09:45:25 +0800 | [diff] [blame] | 1140 | if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio)) |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1141 | return bio_attempt_back_merge(rq, bio, nr_segs); |
| 1142 | break; |
| 1143 | case ELEVATOR_FRONT_MERGE: |
Baolin Wang | 265600b | 2020-09-02 09:45:25 +0800 | [diff] [blame] | 1144 | if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio)) |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1145 | return bio_attempt_front_merge(rq, bio, nr_segs); |
| 1146 | break; |
| 1147 | case ELEVATOR_DISCARD_MERGE: |
| 1148 | return bio_attempt_discard_merge(q, rq, bio); |
| 1149 | default: |
| 1150 | return BIO_MERGE_NONE; |
| 1151 | } |
| 1152 | |
| 1153 | return BIO_MERGE_FAILED; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1154 | } |
| 1155 | |
| 1156 | /** |
| 1157 | * blk_attempt_plug_merge - try to merge with %current's plugged list |
| 1158 | * @q: request_queue new bio is being queued at |
| 1159 | * @bio: new bio being queued |
| 1160 | * @nr_segs: number of segments in @bio |
Jens Axboe | 87c037d | 2021-10-18 10:07:09 -0600 | [diff] [blame] | 1161 | * from the passed in @q already in the plug list |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1162 | * |
Jens Axboe | d38a9c0 | 2021-10-14 07:24:07 -0600 | [diff] [blame] | 1163 | * Determine whether @bio being queued on @q can be merged with the previous |
| 1164 | * request on %current's plugged list. Returns %true if merge was successful, |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1165 | * otherwise %false. |
| 1166 | * |
| 1167 | * Plugging coalesces IOs from the same issuer for the same purpose without |
| 1168 | * going through @q->queue_lock. As such it's more of an issuing mechanism |
| 1169 | * than scheduling, and the request, while may have elvpriv data, is not |
| 1170 | * added on the elevator at this point. In addition, we don't have |
| 1171 | * reliable access to the elevator outside queue lock. Only check basic |
| 1172 | * merging parameters without querying the elevator. |
| 1173 | * |
| 1174 | * Caller must ensure !blk_queue_nomerges(q) beforehand. |
| 1175 | */ |
| 1176 | bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, |
Christoph Hellwig | 0c5bcc9 | 2021-11-23 17:04:41 +0100 | [diff] [blame] | 1177 | unsigned int nr_segs) |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1178 | { |
Damien Le Moal | 99a9476 | 2024-04-08 10:41:28 +0900 | [diff] [blame] | 1179 | struct blk_plug *plug = current->plug; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1180 | struct request *rq; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1181 | |
Christoph Hellwig | a3396b9 | 2024-11-13 16:20:44 +0100 | [diff] [blame] | 1182 | if (!plug || rq_list_empty(&plug->mq_list)) |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1183 | return false; |
| 1184 | |
Jens Axboe | 5b20507 | 2022-03-11 10:21:43 -0700 | [diff] [blame] | 1185 | rq_list_for_each(&plug->mq_list, rq) { |
| 1186 | if (rq->q == q) { |
| 1187 | if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) == |
| 1188 | BIO_MERGE_OK) |
| 1189 | return true; |
| 1190 | break; |
| 1191 | } |
| 1192 | |
| 1193 | /* |
| 1194 | * Only keep iterating plug list for merges if we have multiple |
| 1195 | * queues |
| 1196 | */ |
| 1197 | if (!plug->multiple_queues) |
| 1198 | break; |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1199 | } |
Baolin Wang | 8e75637 | 2020-08-28 10:52:54 +0800 | [diff] [blame] | 1200 | return false; |
| 1201 | } |
Baolin Wang | bdc6a287 | 2020-08-28 10:52:55 +0800 | [diff] [blame] | 1202 | |
| 1203 | /* |
| 1204 | * Iterate list of requests and see if we can merge this bio with any |
| 1205 | * of them. |
| 1206 | */ |
| 1207 | bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, |
| 1208 | struct bio *bio, unsigned int nr_segs) |
| 1209 | { |
| 1210 | struct request *rq; |
| 1211 | int checked = 8; |
| 1212 | |
| 1213 | list_for_each_entry_reverse(rq, list, queuelist) { |
Baolin Wang | bdc6a287 | 2020-08-28 10:52:55 +0800 | [diff] [blame] | 1214 | if (!checked--) |
| 1215 | break; |
| 1216 | |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1217 | switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) { |
| 1218 | case BIO_MERGE_NONE: |
Baolin Wang | bdc6a287 | 2020-08-28 10:52:55 +0800 | [diff] [blame] | 1219 | continue; |
Baolin Wang | 7d7ca7c | 2020-08-28 10:52:56 +0800 | [diff] [blame] | 1220 | case BIO_MERGE_OK: |
| 1221 | return true; |
| 1222 | case BIO_MERGE_FAILED: |
| 1223 | return false; |
Baolin Wang | bdc6a287 | 2020-08-28 10:52:55 +0800 | [diff] [blame] | 1224 | } |
| 1225 | |
Baolin Wang | bdc6a287 | 2020-08-28 10:52:55 +0800 | [diff] [blame] | 1226 | } |
| 1227 | |
| 1228 | return false; |
| 1229 | } |
| 1230 | EXPORT_SYMBOL_GPL(blk_bio_list_merge); |
Christoph Hellwig | eda5cc9 | 2020-10-06 09:07:19 +0200 | [diff] [blame] | 1231 | |
| 1232 | bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, |
| 1233 | unsigned int nr_segs, struct request **merged_request) |
| 1234 | { |
| 1235 | struct request *rq; |
| 1236 | |
| 1237 | switch (elv_merge(q, &rq, bio)) { |
| 1238 | case ELEVATOR_BACK_MERGE: |
| 1239 | if (!blk_mq_sched_allow_merge(q, rq, bio)) |
| 1240 | return false; |
| 1241 | if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK) |
| 1242 | return false; |
| 1243 | *merged_request = attempt_back_merge(q, rq); |
| 1244 | if (!*merged_request) |
| 1245 | elv_merged_request(q, rq, ELEVATOR_BACK_MERGE); |
| 1246 | return true; |
| 1247 | case ELEVATOR_FRONT_MERGE: |
| 1248 | if (!blk_mq_sched_allow_merge(q, rq, bio)) |
| 1249 | return false; |
| 1250 | if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK) |
| 1251 | return false; |
| 1252 | *merged_request = attempt_front_merge(q, rq); |
| 1253 | if (!*merged_request) |
| 1254 | elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE); |
| 1255 | return true; |
| 1256 | case ELEVATOR_DISCARD_MERGE: |
| 1257 | return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK; |
| 1258 | default: |
| 1259 | return false; |
| 1260 | } |
| 1261 | } |
| 1262 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge); |