blob: 6be04ef8da5b758d051c74f3698ab6f7e9ff6d14 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Jens Axboed6d48192008-01-29 14:04:06 +01002/*
3 * Functions related to segment and merge handling
4 */
5#include <linux/kernel.h>
6#include <linux/module.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>
9#include <linux/scatterlist.h>
10
Mike Krinkincda22642015-12-03 17:32:30 +030011#include <trace/events/block.h>
12
Jens Axboed6d48192008-01-29 14:04:06 +010013#include "blk.h"
14
Christoph Hellwige9907002018-09-24 09:43:48 +020015/*
16 * Check if the two bvecs from two bios can be merged to one segment. If yes,
17 * no need to check gap between the two bios since the 1st bio and the 1st bvec
18 * in the 2nd bio can be handled in one segment.
19 */
20static inline bool bios_segs_mergeable(struct request_queue *q,
21 struct bio *prev, struct bio_vec *prev_last_bv,
22 struct bio_vec *next_first_bv)
23{
Christoph Hellwig3dccdae2018-09-24 09:43:52 +020024 if (!biovec_phys_mergeable(q, prev_last_bv, next_first_bv))
Christoph Hellwige9907002018-09-24 09:43:48 +020025 return false;
26 if (prev->bi_seg_back_size + next_first_bv->bv_len >
27 queue_max_segment_size(q))
28 return false;
29 return true;
30}
31
32static inline bool bio_will_gap(struct request_queue *q,
33 struct request *prev_rq, struct bio *prev, struct bio *next)
34{
35 struct bio_vec pb, nb;
36
37 if (!bio_has_data(prev) || !queue_virt_boundary(q))
38 return false;
39
40 /*
41 * Don't merge if the 1st bio starts with non-zero offset, otherwise it
42 * is quite difficult to respect the sg gap limit. We work hard to
43 * merge a huge number of small single bios in case of mkfs.
44 */
45 if (prev_rq)
46 bio_get_first_bvec(prev_rq->bio, &pb);
47 else
48 bio_get_first_bvec(prev, &pb);
Johannes Thumshirndf376b22018-11-07 14:58:14 +010049 if (pb.bv_offset & queue_virt_boundary(q))
Christoph Hellwige9907002018-09-24 09:43:48 +020050 return true;
51
52 /*
53 * We don't need to worry about the situation that the merged segment
54 * ends in unaligned virt boundary:
55 *
56 * - if 'pb' ends aligned, the merged segment ends aligned
57 * - if 'pb' ends unaligned, the next bio must include
58 * one single bvec of 'nb', otherwise the 'nb' can't
59 * merge with 'pb'
60 */
61 bio_get_last_bvec(prev, &pb);
62 bio_get_first_bvec(next, &nb);
63 if (bios_segs_mergeable(q, prev, &pb, &nb))
64 return false;
65 return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
66}
67
68static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
69{
70 return bio_will_gap(req->q, req, req->biotail, bio);
71}
72
73static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
74{
75 return bio_will_gap(req->q, NULL, bio, req->bio);
76}
77
Kent Overstreet54efd502015-04-23 22:37:18 -070078static struct bio *blk_bio_discard_split(struct request_queue *q,
79 struct bio *bio,
Ming Leibdced432015-10-20 23:13:52 +080080 struct bio_set *bs,
81 unsigned *nsegs)
Kent Overstreet54efd502015-04-23 22:37:18 -070082{
83 unsigned int max_discard_sectors, granularity;
84 int alignment;
85 sector_t tmp;
86 unsigned split_sectors;
87
Ming Leibdced432015-10-20 23:13:52 +080088 *nsegs = 1;
89
Kent Overstreet54efd502015-04-23 22:37:18 -070090 /* Zero-sector (unknown) and one-sector granularities are the same. */
91 granularity = max(q->limits.discard_granularity >> 9, 1U);
92
Ming Lei1adfc5e2018-10-29 20:57:17 +080093 max_discard_sectors = min(q->limits.max_discard_sectors,
94 bio_allowed_max_sectors(q));
Kent Overstreet54efd502015-04-23 22:37:18 -070095 max_discard_sectors -= max_discard_sectors % granularity;
96
97 if (unlikely(!max_discard_sectors)) {
98 /* XXX: warn */
99 return NULL;
100 }
101
102 if (bio_sectors(bio) <= max_discard_sectors)
103 return NULL;
104
105 split_sectors = max_discard_sectors;
106
107 /*
108 * If the next starting sector would be misaligned, stop the discard at
109 * the previous aligned sector.
110 */
111 alignment = (q->limits.discard_alignment >> 9) % granularity;
112
113 tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
114 tmp = sector_div(tmp, granularity);
115
116 if (split_sectors > tmp)
117 split_sectors -= tmp;
118
119 return bio_split(bio, split_sectors, GFP_NOIO, bs);
120}
121
Christoph Hellwig885fa132017-04-05 19:21:01 +0200122static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
123 struct bio *bio, struct bio_set *bs, unsigned *nsegs)
124{
125 *nsegs = 1;
126
127 if (!q->limits.max_write_zeroes_sectors)
128 return NULL;
129
130 if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors)
131 return NULL;
132
133 return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs);
134}
135
Kent Overstreet54efd502015-04-23 22:37:18 -0700136static struct bio *blk_bio_write_same_split(struct request_queue *q,
137 struct bio *bio,
Ming Leibdced432015-10-20 23:13:52 +0800138 struct bio_set *bs,
139 unsigned *nsegs)
Kent Overstreet54efd502015-04-23 22:37:18 -0700140{
Ming Leibdced432015-10-20 23:13:52 +0800141 *nsegs = 1;
142
Kent Overstreet54efd502015-04-23 22:37:18 -0700143 if (!q->limits.max_write_same_sectors)
144 return NULL;
145
146 if (bio_sectors(bio) <= q->limits.max_write_same_sectors)
147 return NULL;
148
149 return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);
150}
151
Ming Leid0e5fbb2016-01-23 08:05:33 +0800152static inline unsigned get_max_io_size(struct request_queue *q,
153 struct bio *bio)
154{
155 unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector);
156 unsigned mask = queue_logical_block_size(q) - 1;
157
158 /* aligned to logical block size */
159 sectors &= ~(mask >> 9);
160
161 return sectors;
162}
163
Kent Overstreet54efd502015-04-23 22:37:18 -0700164static struct bio *blk_bio_segment_split(struct request_queue *q,
165 struct bio *bio,
Ming Leibdced432015-10-20 23:13:52 +0800166 struct bio_set *bs,
167 unsigned *segs)
Kent Overstreet54efd502015-04-23 22:37:18 -0700168{
Jens Axboe5014c312015-09-02 16:46:02 -0600169 struct bio_vec bv, bvprv, *bvprvp = NULL;
Kent Overstreet54efd502015-04-23 22:37:18 -0700170 struct bvec_iter iter;
Kent Overstreet8ae12662015-04-27 23:48:34 -0700171 unsigned seg_size = 0, nsegs = 0, sectors = 0;
Ming Lei02e70742015-11-24 10:35:30 +0800172 unsigned front_seg_size = bio->bi_seg_front_size;
173 bool do_split = true;
174 struct bio *new = NULL;
Ming Leid0e5fbb2016-01-23 08:05:33 +0800175 const unsigned max_sectors = get_max_io_size(q, bio);
Kent Overstreet54efd502015-04-23 22:37:18 -0700176
Kent Overstreet54efd502015-04-23 22:37:18 -0700177 bio_for_each_segment(bv, bio, iter) {
Kent Overstreet54efd502015-04-23 22:37:18 -0700178 /*
179 * If the queue doesn't support SG gaps and adding this
180 * offset would create a gap, disallow it.
181 */
Jens Axboe5014c312015-09-02 16:46:02 -0600182 if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset))
Kent Overstreet54efd502015-04-23 22:37:18 -0700183 goto split;
184
Ming Leid0e5fbb2016-01-23 08:05:33 +0800185 if (sectors + (bv.bv_len >> 9) > max_sectors) {
Keith Busche36f6202016-01-12 15:08:39 -0700186 /*
187 * Consider this a new segment if we're splitting in
188 * the middle of this vector.
189 */
190 if (nsegs < queue_max_segments(q) &&
Ming Leid0e5fbb2016-01-23 08:05:33 +0800191 sectors < max_sectors) {
Keith Busche36f6202016-01-12 15:08:39 -0700192 nsegs++;
Ming Leid0e5fbb2016-01-23 08:05:33 +0800193 sectors = max_sectors;
Keith Busche36f6202016-01-12 15:08:39 -0700194 }
Ming Leicf8c0c62017-12-18 20:22:16 +0800195 goto split;
Keith Busche36f6202016-01-12 15:08:39 -0700196 }
197
Jens Axboe5014c312015-09-02 16:46:02 -0600198 if (bvprvp && blk_queue_cluster(q)) {
Ming Leib4b6cb62018-01-10 10:51:29 +0800199 if (seg_size + bv.bv_len > queue_max_segment_size(q))
200 goto new_segment;
Christoph Hellwig3dccdae2018-09-24 09:43:52 +0200201 if (!biovec_phys_mergeable(q, bvprvp, &bv))
Kent Overstreet54efd502015-04-23 22:37:18 -0700202 goto new_segment;
203
204 seg_size += bv.bv_len;
205 bvprv = bv;
Ming Lei578270b2015-11-24 10:35:29 +0800206 bvprvp = &bvprv;
Ming Lei52cc6ee2015-09-17 09:58:38 -0600207 sectors += bv.bv_len >> 9;
Ming Leia88d32a2015-11-30 16:05:49 +0800208
Kent Overstreet54efd502015-04-23 22:37:18 -0700209 continue;
210 }
211new_segment:
212 if (nsegs == queue_max_segments(q))
213 goto split;
214
Ming Lei6a501bf2017-12-18 20:22:14 +0800215 if (nsegs == 1 && seg_size > front_seg_size)
216 front_seg_size = seg_size;
217
Kent Overstreet54efd502015-04-23 22:37:18 -0700218 nsegs++;
219 bvprv = bv;
Ming Lei578270b2015-11-24 10:35:29 +0800220 bvprvp = &bvprv;
Kent Overstreet54efd502015-04-23 22:37:18 -0700221 seg_size = bv.bv_len;
Ming Lei52cc6ee2015-09-17 09:58:38 -0600222 sectors += bv.bv_len >> 9;
Ming Lei02e70742015-11-24 10:35:30 +0800223
Kent Overstreet54efd502015-04-23 22:37:18 -0700224 }
225
Ming Lei02e70742015-11-24 10:35:30 +0800226 do_split = false;
Kent Overstreet54efd502015-04-23 22:37:18 -0700227split:
Ming Leibdced432015-10-20 23:13:52 +0800228 *segs = nsegs;
Ming Lei02e70742015-11-24 10:35:30 +0800229
230 if (do_split) {
231 new = bio_split(bio, sectors, GFP_NOIO, bs);
232 if (new)
233 bio = new;
234 }
235
Ming Lei6a501bf2017-12-18 20:22:14 +0800236 if (nsegs == 1 && seg_size > front_seg_size)
237 front_seg_size = seg_size;
Ming Lei02e70742015-11-24 10:35:30 +0800238 bio->bi_seg_front_size = front_seg_size;
239 if (seg_size > bio->bi_seg_back_size)
240 bio->bi_seg_back_size = seg_size;
241
242 return do_split ? new : NULL;
Kent Overstreet54efd502015-04-23 22:37:18 -0700243}
244
NeilBrownaf67c312017-06-18 14:38:57 +1000245void blk_queue_split(struct request_queue *q, struct bio **bio)
Kent Overstreet54efd502015-04-23 22:37:18 -0700246{
Ming Leibdced432015-10-20 23:13:52 +0800247 struct bio *split, *res;
248 unsigned nsegs;
Kent Overstreet54efd502015-04-23 22:37:18 -0700249
Adrian Hunter7afafc82016-08-16 10:59:35 +0300250 switch (bio_op(*bio)) {
251 case REQ_OP_DISCARD:
252 case REQ_OP_SECURE_ERASE:
Kent Overstreet338aa962018-05-20 18:25:47 -0400253 split = blk_bio_discard_split(q, *bio, &q->bio_split, &nsegs);
Adrian Hunter7afafc82016-08-16 10:59:35 +0300254 break;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800255 case REQ_OP_WRITE_ZEROES:
Kent Overstreet338aa962018-05-20 18:25:47 -0400256 split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split, &nsegs);
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800257 break;
Adrian Hunter7afafc82016-08-16 10:59:35 +0300258 case REQ_OP_WRITE_SAME:
Kent Overstreet338aa962018-05-20 18:25:47 -0400259 split = blk_bio_write_same_split(q, *bio, &q->bio_split, &nsegs);
Adrian Hunter7afafc82016-08-16 10:59:35 +0300260 break;
261 default:
Kent Overstreet338aa962018-05-20 18:25:47 -0400262 split = blk_bio_segment_split(q, *bio, &q->bio_split, &nsegs);
Adrian Hunter7afafc82016-08-16 10:59:35 +0300263 break;
264 }
Ming Leibdced432015-10-20 23:13:52 +0800265
266 /* physical segments can be figured out during splitting */
267 res = split ? split : *bio;
268 res->bi_phys_segments = nsegs;
269 bio_set_flag(res, BIO_SEG_VALID);
Kent Overstreet54efd502015-04-23 22:37:18 -0700270
271 if (split) {
Ming Lei6ac45ae2015-10-20 23:13:53 +0800272 /* there isn't chance to merge the splitted bio */
Jens Axboe1eff9d32016-08-05 15:35:16 -0600273 split->bi_opf |= REQ_NOMERGE;
Ming Lei6ac45ae2015-10-20 23:13:53 +0800274
Jens Axboecd4a4ae2018-06-02 14:04:07 -0600275 /*
276 * Since we're recursing into make_request here, ensure
277 * that we mark this bio as already having entered the queue.
278 * If not, and the queue is going away, we can get stuck
279 * forever on waiting for the queue reference to drop. But
280 * that will never happen, as we're already holding a
281 * reference to it.
282 */
283 bio_set_flag(*bio, BIO_QUEUE_ENTERED);
284
Kent Overstreet54efd502015-04-23 22:37:18 -0700285 bio_chain(split, *bio);
Mike Krinkincda22642015-12-03 17:32:30 +0300286 trace_block_split(q, split, (*bio)->bi_iter.bi_sector);
Kent Overstreet54efd502015-04-23 22:37:18 -0700287 generic_make_request(*bio);
288 *bio = split;
289 }
290}
291EXPORT_SYMBOL(blk_queue_split);
292
Jens Axboe1e428072009-02-23 09:03:10 +0100293static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
Ming Lei07388542014-09-02 23:02:59 +0800294 struct bio *bio,
295 bool no_sg_merge)
Jens Axboed6d48192008-01-29 14:04:06 +0100296{
Kent Overstreet79886132013-11-23 17:19:00 -0800297 struct bio_vec bv, bvprv = { NULL };
Kent Overstreet54efd502015-04-23 22:37:18 -0700298 int cluster, prev = 0;
Jens Axboe1e428072009-02-23 09:03:10 +0100299 unsigned int seg_size, nr_phys_segs;
Jens Axboe59247ea2009-03-06 08:55:24 +0100300 struct bio *fbio, *bbio;
Kent Overstreet79886132013-11-23 17:19:00 -0800301 struct bvec_iter iter;
Jens Axboed6d48192008-01-29 14:04:06 +0100302
Jens Axboe1e428072009-02-23 09:03:10 +0100303 if (!bio)
304 return 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100305
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800306 switch (bio_op(bio)) {
307 case REQ_OP_DISCARD:
308 case REQ_OP_SECURE_ERASE:
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800309 case REQ_OP_WRITE_ZEROES:
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700310 return 0;
311 case REQ_OP_WRITE_SAME:
Kent Overstreet5cb88502014-02-07 13:53:46 -0700312 return 1;
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800313 }
Kent Overstreet5cb88502014-02-07 13:53:46 -0700314
Jens Axboe1e428072009-02-23 09:03:10 +0100315 fbio = bio;
Martin K. Petersene692cb62010-12-01 19:41:49 +0100316 cluster = blk_queue_cluster(q);
Mikulas Patocka5df97b92008-08-15 10:20:02 +0200317 seg_size = 0;
Andi Kleen2c8919d2010-06-21 11:02:47 +0200318 nr_phys_segs = 0;
Jens Axboe1e428072009-02-23 09:03:10 +0100319 for_each_bio(bio) {
Kent Overstreet79886132013-11-23 17:19:00 -0800320 bio_for_each_segment(bv, bio, iter) {
Jens Axboe1e428072009-02-23 09:03:10 +0100321 /*
Jens Axboe05f1dd52014-05-29 09:53:32 -0600322 * If SG merging is disabled, each bio vector is
323 * a segment
324 */
325 if (no_sg_merge)
326 goto new_segment;
327
Kent Overstreet54efd502015-04-23 22:37:18 -0700328 if (prev && cluster) {
Kent Overstreet79886132013-11-23 17:19:00 -0800329 if (seg_size + bv.bv_len
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400330 > queue_max_segment_size(q))
Jens Axboe1e428072009-02-23 09:03:10 +0100331 goto new_segment;
Christoph Hellwig3dccdae2018-09-24 09:43:52 +0200332 if (!biovec_phys_mergeable(q, &bvprv, &bv))
Jens Axboe1e428072009-02-23 09:03:10 +0100333 goto new_segment;
Jens Axboed6d48192008-01-29 14:04:06 +0100334
Kent Overstreet79886132013-11-23 17:19:00 -0800335 seg_size += bv.bv_len;
Jens Axboe1e428072009-02-23 09:03:10 +0100336 bvprv = bv;
337 continue;
338 }
Jens Axboed6d48192008-01-29 14:04:06 +0100339new_segment:
Jens Axboe1e428072009-02-23 09:03:10 +0100340 if (nr_phys_segs == 1 && seg_size >
341 fbio->bi_seg_front_size)
342 fbio->bi_seg_front_size = seg_size;
FUJITA Tomonori86771422008-10-13 14:19:05 +0200343
Jens Axboe1e428072009-02-23 09:03:10 +0100344 nr_phys_segs++;
345 bvprv = bv;
Kent Overstreet54efd502015-04-23 22:37:18 -0700346 prev = 1;
Kent Overstreet79886132013-11-23 17:19:00 -0800347 seg_size = bv.bv_len;
Jens Axboe1e428072009-02-23 09:03:10 +0100348 }
Jens Axboe59247ea2009-03-06 08:55:24 +0100349 bbio = bio;
Jens Axboed6d48192008-01-29 14:04:06 +0100350 }
351
Jens Axboe59247ea2009-03-06 08:55:24 +0100352 if (nr_phys_segs == 1 && seg_size > fbio->bi_seg_front_size)
353 fbio->bi_seg_front_size = seg_size;
354 if (seg_size > bbio->bi_seg_back_size)
355 bbio->bi_seg_back_size = seg_size;
Jens Axboe1e428072009-02-23 09:03:10 +0100356
357 return nr_phys_segs;
358}
359
360void blk_recalc_rq_segments(struct request *rq)
361{
Ming Lei07388542014-09-02 23:02:59 +0800362 bool no_sg_merge = !!test_bit(QUEUE_FLAG_NO_SG_MERGE,
363 &rq->q->queue_flags);
364
365 rq->nr_phys_segments = __blk_recalc_rq_segments(rq->q, rq->bio,
366 no_sg_merge);
Jens Axboed6d48192008-01-29 14:04:06 +0100367}
368
369void blk_recount_segments(struct request_queue *q, struct bio *bio)
370{
Ming Lei7f60dca2014-11-12 00:15:41 +0800371 unsigned short seg_cnt;
Ming Lei764f6122014-10-09 23:17:35 +0800372
Ming Lei7f60dca2014-11-12 00:15:41 +0800373 /* estimate segment number by bi_vcnt for non-cloned bio */
374 if (bio_flagged(bio, BIO_CLONED))
375 seg_cnt = bio_segments(bio);
376 else
377 seg_cnt = bio->bi_vcnt;
378
379 if (test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags) &&
380 (seg_cnt < queue_max_segments(q)))
381 bio->bi_phys_segments = seg_cnt;
Jens Axboe05f1dd52014-05-29 09:53:32 -0600382 else {
383 struct bio *nxt = bio->bi_next;
Jens Axboe1e428072009-02-23 09:03:10 +0100384
Jens Axboe05f1dd52014-05-29 09:53:32 -0600385 bio->bi_next = NULL;
Ming Lei7f60dca2014-11-12 00:15:41 +0800386 bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio, false);
Jens Axboe05f1dd52014-05-29 09:53:32 -0600387 bio->bi_next = nxt;
388 }
389
Jens Axboeb7c44ed2015-07-24 12:37:59 -0600390 bio_set_flag(bio, BIO_SEG_VALID);
Jens Axboed6d48192008-01-29 14:04:06 +0100391}
392EXPORT_SYMBOL(blk_recount_segments);
393
394static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio,
395 struct bio *nxt)
396{
Kent Overstreet2b8221e2013-12-03 14:29:09 -0700397 struct bio_vec end_bv = { NULL }, nxt_bv;
Kent Overstreetf619d252013-08-07 14:30:33 -0700398
Martin K. Petersene692cb62010-12-01 19:41:49 +0100399 if (!blk_queue_cluster(q))
Jens Axboed6d48192008-01-29 14:04:06 +0100400 return 0;
401
FUJITA Tomonori86771422008-10-13 14:19:05 +0200402 if (bio->bi_seg_back_size + nxt->bi_seg_front_size >
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400403 queue_max_segment_size(q))
Jens Axboed6d48192008-01-29 14:04:06 +0100404 return 0;
405
David Woodhousee17fc0a2008-08-09 16:42:20 +0100406 if (!bio_has_data(bio))
407 return 1;
408
Ming Leie8270912016-02-26 23:40:53 +0800409 bio_get_last_bvec(bio, &end_bv);
410 bio_get_first_bvec(nxt, &nxt_bv);
Kent Overstreetf619d252013-08-07 14:30:33 -0700411
Christoph Hellwig3dccdae2018-09-24 09:43:52 +0200412 return biovec_phys_mergeable(q, &end_bv, &nxt_bv);
Jens Axboed6d48192008-01-29 14:04:06 +0100413}
414
Kent Overstreet79886132013-11-23 17:19:00 -0800415static inline void
Asias He963ab9e52012-08-02 23:42:03 +0200416__blk_segment_map_sg(struct request_queue *q, struct bio_vec *bvec,
Kent Overstreet79886132013-11-23 17:19:00 -0800417 struct scatterlist *sglist, struct bio_vec *bvprv,
Asias He963ab9e52012-08-02 23:42:03 +0200418 struct scatterlist **sg, int *nsegs, int *cluster)
419{
420
421 int nbytes = bvec->bv_len;
422
Kent Overstreet79886132013-11-23 17:19:00 -0800423 if (*sg && *cluster) {
Ming Leib4b6cb62018-01-10 10:51:29 +0800424 if ((*sg)->length + nbytes > queue_max_segment_size(q))
425 goto new_segment;
Christoph Hellwig3dccdae2018-09-24 09:43:52 +0200426 if (!biovec_phys_mergeable(q, bvprv, bvec))
Asias He963ab9e52012-08-02 23:42:03 +0200427 goto new_segment;
428
429 (*sg)->length += nbytes;
430 } else {
431new_segment:
432 if (!*sg)
433 *sg = sglist;
434 else {
435 /*
436 * If the driver previously mapped a shorter
437 * list, we could see a termination bit
438 * prematurely unless it fully inits the sg
439 * table on each mapping. We KNOW that there
440 * must be more entries here or the driver
441 * would be buggy, so force clear the
442 * termination bit to avoid doing a full
443 * sg_init_table() in drivers for each command.
444 */
Paolo Bonzinic8164d82013-03-20 15:37:08 +1030445 sg_unmark_end(*sg);
Asias He963ab9e52012-08-02 23:42:03 +0200446 *sg = sg_next(*sg);
447 }
448
449 sg_set_page(*sg, bvec->bv_page, nbytes, bvec->bv_offset);
450 (*nsegs)++;
451 }
Kent Overstreet79886132013-11-23 17:19:00 -0800452 *bvprv = *bvec;
Asias He963ab9e52012-08-02 23:42:03 +0200453}
454
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700455static inline int __blk_bvec_map_sg(struct request_queue *q, struct bio_vec bv,
456 struct scatterlist *sglist, struct scatterlist **sg)
457{
458 *sg = sglist;
459 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
460 return 1;
461}
462
Kent Overstreet5cb88502014-02-07 13:53:46 -0700463static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio,
464 struct scatterlist *sglist,
465 struct scatterlist **sg)
466{
467 struct bio_vec bvec, bvprv = { NULL };
468 struct bvec_iter iter;
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700469 int cluster = blk_queue_cluster(q), nsegs = 0;
Kent Overstreet5cb88502014-02-07 13:53:46 -0700470
471 for_each_bio(bio)
472 bio_for_each_segment(bvec, bio, iter)
473 __blk_segment_map_sg(q, &bvec, sglist, &bvprv, sg,
474 &nsegs, &cluster);
475
476 return nsegs;
477}
478
Jens Axboed6d48192008-01-29 14:04:06 +0100479/*
480 * map a request to scatterlist, return number of sg entries setup. Caller
481 * must make sure sg can hold rq->nr_phys_segments entries
482 */
483int blk_rq_map_sg(struct request_queue *q, struct request *rq,
484 struct scatterlist *sglist)
485{
Kent Overstreet5cb88502014-02-07 13:53:46 -0700486 struct scatterlist *sg = NULL;
487 int nsegs = 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100488
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700489 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
490 nsegs = __blk_bvec_map_sg(q, rq->special_vec, sglist, &sg);
491 else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME)
492 nsegs = __blk_bvec_map_sg(q, bio_iovec(rq->bio), sglist, &sg);
493 else if (rq->bio)
Kent Overstreet5cb88502014-02-07 13:53:46 -0700494 nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg);
FUJITA Tomonorif18573a2008-04-11 12:56:52 +0200495
Christoph Hellwige8064022016-10-20 15:12:13 +0200496 if (unlikely(rq->rq_flags & RQF_COPY_USER) &&
Tejun Heo2e46e8b2009-05-07 22:24:41 +0900497 (blk_rq_bytes(rq) & q->dma_pad_mask)) {
498 unsigned int pad_len =
499 (q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
FUJITA Tomonorif18573a2008-04-11 12:56:52 +0200500
501 sg->length += pad_len;
502 rq->extra_len += pad_len;
503 }
504
Tejun Heo2fb98e82008-02-19 11:36:53 +0100505 if (q->dma_drain_size && q->dma_drain_needed(rq)) {
Mike Christiea8ebb052016-06-05 14:31:45 -0500506 if (op_is_write(req_op(rq)))
Tejun Heodb0a2e02008-02-19 11:36:55 +0100507 memset(q->dma_drain_buffer, 0, q->dma_drain_size);
508
Dan Williamsda81ed12015-08-07 18:15:14 +0200509 sg_unmark_end(sg);
Jens Axboed6d48192008-01-29 14:04:06 +0100510 sg = sg_next(sg);
511 sg_set_page(sg, virt_to_page(q->dma_drain_buffer),
512 q->dma_drain_size,
513 ((unsigned long)q->dma_drain_buffer) &
514 (PAGE_SIZE - 1));
515 nsegs++;
FUJITA Tomonori7a85f882008-03-04 11:17:11 +0100516 rq->extra_len += q->dma_drain_size;
Jens Axboed6d48192008-01-29 14:04:06 +0100517 }
518
519 if (sg)
520 sg_mark_end(sg);
521
Ming Lei12e57f52015-11-24 10:35:31 +0800522 /*
523 * Something must have been wrong if the figured number of
524 * segment is bigger than number of req's physical segments
525 */
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700526 WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
Ming Lei12e57f52015-11-24 10:35:31 +0800527
Jens Axboed6d48192008-01-29 14:04:06 +0100528 return nsegs;
529}
Jens Axboed6d48192008-01-29 14:04:06 +0100530EXPORT_SYMBOL(blk_rq_map_sg);
531
Jens Axboed6d48192008-01-29 14:04:06 +0100532static inline int ll_new_hw_segment(struct request_queue *q,
533 struct request *req,
534 struct bio *bio)
535{
Jens Axboed6d48192008-01-29 14:04:06 +0100536 int nr_phys_segs = bio_phys_segments(q, bio);
537
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200538 if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(q))
539 goto no_merge;
540
Martin K. Petersen4eaf99b2014-09-26 19:20:06 -0400541 if (blk_integrity_merge_bio(q, req, bio) == false)
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200542 goto no_merge;
Jens Axboed6d48192008-01-29 14:04:06 +0100543
544 /*
545 * This will form the start of a new hw segment. Bump both
546 * counters.
547 */
Jens Axboed6d48192008-01-29 14:04:06 +0100548 req->nr_phys_segments += nr_phys_segs;
549 return 1;
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200550
551no_merge:
Ritesh Harjanie0c72302016-12-01 08:36:16 -0700552 req_set_nomerge(q, req);
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200553 return 0;
Jens Axboed6d48192008-01-29 14:04:06 +0100554}
555
556int ll_back_merge_fn(struct request_queue *q, struct request *req,
557 struct bio *bio)
558{
Jens Axboe5e7c4272015-09-03 19:28:20 +0300559 if (req_gap_back_merge(req, bio))
560 return 0;
Sagi Grimberg7f39add2015-09-11 09:03:04 -0600561 if (blk_integrity_rq(req) &&
562 integrity_req_gap_back_merge(req, bio))
563 return 0;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400564 if (blk_rq_sectors(req) + bio_sectors(bio) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600565 blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
Ritesh Harjanie0c72302016-12-01 08:36:16 -0700566 req_set_nomerge(q, req);
Jens Axboed6d48192008-01-29 14:04:06 +0100567 return 0;
568 }
Jens Axboe2cdf79c2008-05-07 09:33:55 +0200569 if (!bio_flagged(req->biotail, BIO_SEG_VALID))
Jens Axboed6d48192008-01-29 14:04:06 +0100570 blk_recount_segments(q, req->biotail);
Jens Axboe2cdf79c2008-05-07 09:33:55 +0200571 if (!bio_flagged(bio, BIO_SEG_VALID))
Jens Axboed6d48192008-01-29 14:04:06 +0100572 blk_recount_segments(q, bio);
Jens Axboed6d48192008-01-29 14:04:06 +0100573
574 return ll_new_hw_segment(q, req, bio);
575}
576
Jens Axboe6728cb02008-01-31 13:03:55 +0100577int ll_front_merge_fn(struct request_queue *q, struct request *req,
Jens Axboed6d48192008-01-29 14:04:06 +0100578 struct bio *bio)
579{
Jens Axboe5e7c4272015-09-03 19:28:20 +0300580
581 if (req_gap_front_merge(req, bio))
582 return 0;
Sagi Grimberg7f39add2015-09-11 09:03:04 -0600583 if (blk_integrity_rq(req) &&
584 integrity_req_gap_front_merge(req, bio))
585 return 0;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400586 if (blk_rq_sectors(req) + bio_sectors(bio) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600587 blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
Ritesh Harjanie0c72302016-12-01 08:36:16 -0700588 req_set_nomerge(q, req);
Jens Axboed6d48192008-01-29 14:04:06 +0100589 return 0;
590 }
Jens Axboe2cdf79c2008-05-07 09:33:55 +0200591 if (!bio_flagged(bio, BIO_SEG_VALID))
Jens Axboed6d48192008-01-29 14:04:06 +0100592 blk_recount_segments(q, bio);
Jens Axboe2cdf79c2008-05-07 09:33:55 +0200593 if (!bio_flagged(req->bio, BIO_SEG_VALID))
Jens Axboed6d48192008-01-29 14:04:06 +0100594 blk_recount_segments(q, req->bio);
Jens Axboed6d48192008-01-29 14:04:06 +0100595
596 return ll_new_hw_segment(q, req, bio);
597}
598
Jens Axboe445251d2018-02-01 14:01:02 -0700599static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
600 struct request *next)
601{
602 unsigned short segments = blk_rq_nr_discard_segments(req);
603
604 if (segments >= queue_max_discard_segments(q))
605 goto no_merge;
606 if (blk_rq_sectors(req) + bio_sectors(next->bio) >
607 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
608 goto no_merge;
609
610 req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
611 return true;
612no_merge:
613 req_set_nomerge(q, req);
614 return false;
615}
616
Jens Axboed6d48192008-01-29 14:04:06 +0100617static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
618 struct request *next)
619{
620 int total_phys_segments;
FUJITA Tomonori86771422008-10-13 14:19:05 +0200621 unsigned int seg_size =
622 req->biotail->bi_seg_back_size + next->bio->bi_seg_front_size;
Jens Axboed6d48192008-01-29 14:04:06 +0100623
Jens Axboe5e7c4272015-09-03 19:28:20 +0300624 if (req_gap_back_merge(req, next->bio))
Keith Busch854fbb92015-02-11 08:20:13 -0700625 return 0;
626
Jens Axboed6d48192008-01-29 14:04:06 +0100627 /*
628 * Will it become too large?
629 */
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400630 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
Damien Le Moal17007f32016-07-20 21:40:47 -0600631 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
Jens Axboed6d48192008-01-29 14:04:06 +0100632 return 0;
633
634 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
FUJITA Tomonori86771422008-10-13 14:19:05 +0200635 if (blk_phys_contig_segment(q, req->biotail, next->bio)) {
636 if (req->nr_phys_segments == 1)
637 req->bio->bi_seg_front_size = seg_size;
638 if (next->nr_phys_segments == 1)
639 next->biotail->bi_seg_back_size = seg_size;
Jens Axboed6d48192008-01-29 14:04:06 +0100640 total_phys_segments--;
FUJITA Tomonori86771422008-10-13 14:19:05 +0200641 }
Jens Axboed6d48192008-01-29 14:04:06 +0100642
Martin K. Petersen8a783622010-02-26 00:20:39 -0500643 if (total_phys_segments > queue_max_segments(q))
Jens Axboed6d48192008-01-29 14:04:06 +0100644 return 0;
645
Martin K. Petersen4eaf99b2014-09-26 19:20:06 -0400646 if (blk_integrity_merge_rq(q, req, next) == false)
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200647 return 0;
648
Jens Axboed6d48192008-01-29 14:04:06 +0100649 /* Merge is OK... */
650 req->nr_phys_segments = total_phys_segments;
Jens Axboed6d48192008-01-29 14:04:06 +0100651 return 1;
652}
653
Tejun Heo80a761f2009-07-03 17:48:17 +0900654/**
655 * blk_rq_set_mixed_merge - mark a request as mixed merge
656 * @rq: request to mark as mixed merge
657 *
658 * Description:
659 * @rq is about to be mixed merged. Make sure the attributes
660 * which can be mixed are set in each bio and mark @rq as mixed
661 * merged.
662 */
663void blk_rq_set_mixed_merge(struct request *rq)
664{
665 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
666 struct bio *bio;
667
Christoph Hellwige8064022016-10-20 15:12:13 +0200668 if (rq->rq_flags & RQF_MIXED_MERGE)
Tejun Heo80a761f2009-07-03 17:48:17 +0900669 return;
670
671 /*
672 * @rq will no longer represent mixable attributes for all the
673 * contained bios. It will just track those of the first one.
674 * Distributes the attributs to each bio.
675 */
676 for (bio = rq->bio; bio; bio = bio->bi_next) {
Jens Axboe1eff9d32016-08-05 15:35:16 -0600677 WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
678 (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
679 bio->bi_opf |= ff;
Tejun Heo80a761f2009-07-03 17:48:17 +0900680 }
Christoph Hellwige8064022016-10-20 15:12:13 +0200681 rq->rq_flags |= RQF_MIXED_MERGE;
Tejun Heo80a761f2009-07-03 17:48:17 +0900682}
683
Jerome Marchand26308ea2009-03-27 10:31:51 +0100684static void blk_account_io_merge(struct request *req)
685{
686 if (blk_do_io_stat(req)) {
687 struct hd_struct *part;
688 int cpu;
689
690 cpu = part_stat_lock();
Jerome Marchand09e099d2011-01-05 16:57:38 +0100691 part = req->part;
Jerome Marchand26308ea2009-03-27 10:31:51 +0100692
Jens Axboed62e26b2017-06-30 21:55:08 -0600693 part_round_stats(req->q, cpu, part);
694 part_dec_in_flight(req->q, part, rq_data_dir(req));
Jerome Marchand26308ea2009-03-27 10:31:51 +0100695
Jens Axboe6c23a962011-01-07 08:43:37 +0100696 hd_struct_put(part);
Jerome Marchand26308ea2009-03-27 10:31:51 +0100697 part_stat_unlock();
698 }
699}
Jianchao Wang698404662018-10-27 19:52:14 +0800700/*
701 * Two cases of handling DISCARD merge:
702 * If max_discard_segments > 1, the driver takes every bio
703 * as a range and send them to controller together. The ranges
704 * needn't to be contiguous.
705 * Otherwise, the bios/requests will be handled as same as
706 * others which should be contiguous.
707 */
708static inline bool blk_discard_mergable(struct request *req)
709{
710 if (req_op(req) == REQ_OP_DISCARD &&
711 queue_max_discard_segments(req->q) > 1)
712 return true;
713 return false;
714}
715
Eric Biggerse96c0d82018-11-14 17:19:46 -0800716static enum elv_merge blk_try_req_merge(struct request *req,
717 struct request *next)
Jianchao Wang698404662018-10-27 19:52:14 +0800718{
719 if (blk_discard_mergable(req))
720 return ELEVATOR_DISCARD_MERGE;
721 else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
722 return ELEVATOR_BACK_MERGE;
723
724 return ELEVATOR_NO_MERGE;
725}
Jerome Marchand26308ea2009-03-27 10:31:51 +0100726
Jens Axboed6d48192008-01-29 14:04:06 +0100727/*
Jens Axboeb973cb72017-02-02 08:54:40 -0700728 * For non-mq, this has to be called with the request spinlock acquired.
729 * For mq with scheduling, the appropriate queue wide lock should be held.
Jens Axboed6d48192008-01-29 14:04:06 +0100730 */
Jens Axboeb973cb72017-02-02 08:54:40 -0700731static struct request *attempt_merge(struct request_queue *q,
732 struct request *req, struct request *next)
Jens Axboed6d48192008-01-29 14:04:06 +0100733{
734 if (!rq_mergeable(req) || !rq_mergeable(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700735 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100736
Christoph Hellwig288dab82016-06-09 16:00:36 +0200737 if (req_op(req) != req_op(next))
Jens Axboeb973cb72017-02-02 08:54:40 -0700738 return NULL;
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400739
Jens Axboed6d48192008-01-29 14:04:06 +0100740 if (rq_data_dir(req) != rq_data_dir(next)
Jens Axboe2081a562018-10-12 12:39:10 -0600741 || req->rq_disk != next->rq_disk)
Jens Axboeb973cb72017-02-02 08:54:40 -0700742 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100743
Mike Christie8fe0d472016-06-05 14:32:15 -0500744 if (req_op(req) == REQ_OP_WRITE_SAME &&
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400745 !blk_write_same_mergeable(req->bio, next->bio))
Jens Axboeb973cb72017-02-02 08:54:40 -0700746 return NULL;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400747
Jens Axboed6d48192008-01-29 14:04:06 +0100748 /*
Jens Axboecb6934f2017-06-27 09:22:02 -0600749 * Don't allow merge of different write hints, or for a hint with
750 * non-hint IO.
751 */
752 if (req->write_hint != next->write_hint)
753 return NULL;
754
Damien Le Moal668ffc02018-11-20 10:52:37 +0900755 if (req->ioprio != next->ioprio)
756 return NULL;
757
Jens Axboecb6934f2017-06-27 09:22:02 -0600758 /*
Jens Axboed6d48192008-01-29 14:04:06 +0100759 * If we are allowed to merge, then append bio list
760 * from next to rq and release next. merge_requests_fn
761 * will have updated segment counts, update sector
Jens Axboe445251d2018-02-01 14:01:02 -0700762 * counts here. Handle DISCARDs separately, as they
763 * have separate settings.
Jens Axboed6d48192008-01-29 14:04:06 +0100764 */
Jianchao Wang698404662018-10-27 19:52:14 +0800765
766 switch (blk_try_req_merge(req, next)) {
767 case ELEVATOR_DISCARD_MERGE:
Jens Axboe445251d2018-02-01 14:01:02 -0700768 if (!req_attempt_discard_merge(q, req, next))
769 return NULL;
Jianchao Wang698404662018-10-27 19:52:14 +0800770 break;
771 case ELEVATOR_BACK_MERGE:
772 if (!ll_merge_requests_fn(q, req, next))
773 return NULL;
774 break;
775 default:
Jens Axboeb973cb72017-02-02 08:54:40 -0700776 return NULL;
Jianchao Wang698404662018-10-27 19:52:14 +0800777 }
Jens Axboed6d48192008-01-29 14:04:06 +0100778
779 /*
Tejun Heo80a761f2009-07-03 17:48:17 +0900780 * If failfast settings disagree or any of the two is already
781 * a mixed merge, mark both as mixed before proceeding. This
782 * makes sure that all involved bios have mixable attributes
783 * set properly.
784 */
Christoph Hellwige8064022016-10-20 15:12:13 +0200785 if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
Tejun Heo80a761f2009-07-03 17:48:17 +0900786 (req->cmd_flags & REQ_FAILFAST_MASK) !=
787 (next->cmd_flags & REQ_FAILFAST_MASK)) {
788 blk_rq_set_mixed_merge(req);
789 blk_rq_set_mixed_merge(next);
790 }
791
792 /*
Omar Sandoval522a7772018-05-09 02:08:53 -0700793 * At this point we have either done a back merge or front merge. We
794 * need the smaller start_time_ns of the merged requests to be the
795 * current request for accounting purposes.
Jens Axboed6d48192008-01-29 14:04:06 +0100796 */
Omar Sandoval522a7772018-05-09 02:08:53 -0700797 if (next->start_time_ns < req->start_time_ns)
798 req->start_time_ns = next->start_time_ns;
Jens Axboed6d48192008-01-29 14:04:06 +0100799
800 req->biotail->bi_next = next->bio;
801 req->biotail = next->biotail;
802
Tejun Heoa2dec7b2009-05-07 22:24:44 +0900803 req->__data_len += blk_rq_bytes(next);
Jens Axboed6d48192008-01-29 14:04:06 +0100804
Jens Axboe445251d2018-02-01 14:01:02 -0700805 if (req_op(req) != REQ_OP_DISCARD)
806 elv_merge_requests(q, req, next);
Jens Axboed6d48192008-01-29 14:04:06 +0100807
Jerome Marchand42dad762009-04-22 14:01:49 +0200808 /*
809 * 'next' is going away, so update stats accordingly
810 */
811 blk_account_io_merge(next);
Jens Axboed6d48192008-01-29 14:04:06 +0100812
Jens Axboee4d750c2017-02-03 09:48:28 -0700813 /*
814 * ownership of bio passed from next to req, return 'next' for
815 * the caller to free
816 */
Boaz Harrosh1cd96c22009-03-24 12:35:07 +0100817 next->bio = NULL;
Jens Axboeb973cb72017-02-02 08:54:40 -0700818 return next;
Jens Axboed6d48192008-01-29 14:04:06 +0100819}
820
Jens Axboeb973cb72017-02-02 08:54:40 -0700821struct request *attempt_back_merge(struct request_queue *q, struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100822{
823 struct request *next = elv_latter_request(q, rq);
824
825 if (next)
826 return attempt_merge(q, rq, next);
827
Jens Axboeb973cb72017-02-02 08:54:40 -0700828 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100829}
830
Jens Axboeb973cb72017-02-02 08:54:40 -0700831struct request *attempt_front_merge(struct request_queue *q, struct request *rq)
Jens Axboed6d48192008-01-29 14:04:06 +0100832{
833 struct request *prev = elv_former_request(q, rq);
834
835 if (prev)
836 return attempt_merge(q, prev, rq);
837
Jens Axboeb973cb72017-02-02 08:54:40 -0700838 return NULL;
Jens Axboed6d48192008-01-29 14:04:06 +0100839}
Jens Axboe5e84ea32011-03-21 10:14:27 +0100840
841int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
842 struct request *next)
843{
Jens Axboee4d750c2017-02-03 09:48:28 -0700844 struct request *free;
Tahsin Erdogan72ef7992016-07-07 11:48:22 -0700845
Jens Axboee4d750c2017-02-03 09:48:28 -0700846 free = attempt_merge(q, rq, next);
847 if (free) {
Jens Axboe92bc5a22018-10-24 13:52:28 -0600848 blk_put_request(free);
Jens Axboee4d750c2017-02-03 09:48:28 -0700849 return 1;
850 }
851
852 return 0;
Jens Axboe5e84ea32011-03-21 10:14:27 +0100853}
Tejun Heo050c8ea2012-02-08 09:19:38 +0100854
855bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
856{
Martin K. Petersene2a60da2012-09-18 12:19:25 -0400857 if (!rq_mergeable(rq) || !bio_mergeable(bio))
Tejun Heo050c8ea2012-02-08 09:19:38 +0100858 return false;
859
Christoph Hellwig288dab82016-06-09 16:00:36 +0200860 if (req_op(rq) != bio_op(bio))
Martin K. Petersenf31dc1c2012-09-18 12:19:26 -0400861 return false;
862
Tejun Heo050c8ea2012-02-08 09:19:38 +0100863 /* different data direction or already started, don't merge */
864 if (bio_data_dir(bio) != rq_data_dir(rq))
865 return false;
866
Jens Axboe2081a562018-10-12 12:39:10 -0600867 /* must be same device */
868 if (rq->rq_disk != bio->bi_disk)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100869 return false;
870
871 /* only merge integrity protected bio into ditto rq */
Martin K. Petersen4eaf99b2014-09-26 19:20:06 -0400872 if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100873 return false;
874
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400875 /* must be using the same buffer */
Mike Christie8fe0d472016-06-05 14:32:15 -0500876 if (req_op(rq) == REQ_OP_WRITE_SAME &&
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400877 !blk_write_same_mergeable(rq->bio, bio))
878 return false;
879
Jens Axboecb6934f2017-06-27 09:22:02 -0600880 /*
881 * Don't allow merge of different write hints, or for a hint with
882 * non-hint IO.
883 */
884 if (rq->write_hint != bio->bi_write_hint)
885 return false;
886
Damien Le Moal668ffc02018-11-20 10:52:37 +0900887 if (rq->ioprio != bio_prio(bio))
888 return false;
889
Tejun Heo050c8ea2012-02-08 09:19:38 +0100890 return true;
891}
892
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100893enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100894{
Jianchao Wang698404662018-10-27 19:52:14 +0800895 if (blk_discard_mergable(rq))
Christoph Hellwig1e739732017-02-08 14:46:49 +0100896 return ELEVATOR_DISCARD_MERGE;
897 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100898 return ELEVATOR_BACK_MERGE;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700899 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
Tejun Heo050c8ea2012-02-08 09:19:38 +0100900 return ELEVATOR_FRONT_MERGE;
901 return ELEVATOR_NO_MERGE;
902}