blob: e9e809a63c5975183dc710452e6cac612d1a67af [file] [log] [blame]
Christoph Hellwig8c165672019-04-30 14:42:39 -04001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
Jens Axboe0fe23472006-09-04 15:41:16 +02003 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 */
5#include <linux/mm.h>
6#include <linux/swap.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>
Kent Overstreeta27bb332013-05-07 16:19:08 -07009#include <linux/uio.h>
Tejun Heo852c7882012-03-05 13:15:27 -080010#include <linux/iocontext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/slab.h>
12#include <linux/init.h>
13#include <linux/kernel.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050014#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/mempool.h>
16#include <linux/workqueue.h>
Tejun Heo852c7882012-03-05 13:15:27 -080017#include <linux/cgroup.h>
Damien Le Moalb4c58752019-07-01 14:09:15 +090018#include <linux/highmem.h>
Satya Tangiralaa892c8d2020-05-14 00:37:18 +000019#include <linux/blk-crypto.h>
Ming Lei49d1ec82021-01-11 11:05:52 +080020#include <linux/xarray.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
Li Zefan55782132009-06-09 13:43:05 +080022#include <trace/events/block.h>
Shaohua Li9e234ee2017-03-27 10:51:41 -070023#include "blk.h"
Josef Bacik67b42d02018-07-03 11:15:00 -040024#include "blk-rq-qos.h"
Ming Lei672fdcf2022-02-11 18:11:49 +080025#include "blk-cgroup.h"
Ingo Molnar0bfc2452008-11-26 11:59:56 +010026
Pavel Begunkovb99182c2022-11-02 15:18:22 +000027#define ALLOC_CACHE_THRESHOLD 16
Pavel Begunkov42b2b2f2022-11-02 15:18:23 +000028#define ALLOC_CACHE_MAX 256
Pavel Begunkovb99182c2022-11-02 15:18:22 +000029
Jens Axboebe4d2342021-03-08 11:37:47 -070030struct bio_alloc_cache {
Jens Axboefcade2c2021-12-01 16:19:18 -070031 struct bio *free_list;
Pavel Begunkovb99182c2022-11-02 15:18:22 +000032 struct bio *free_list_irq;
Jens Axboebe4d2342021-03-08 11:37:47 -070033 unsigned int nr;
Pavel Begunkovb99182c2022-11-02 15:18:22 +000034 unsigned int nr_irq;
Jens Axboebe4d2342021-03-08 11:37:47 -070035};
36
Christoph Hellwigde76fd82021-02-02 18:19:23 +010037static struct biovec_slab {
Christoph Hellwig6ac0b712021-02-02 18:19:20 +010038 int nr_vecs;
39 char *name;
40 struct kmem_cache *slab;
Christoph Hellwigde76fd82021-02-02 18:19:23 +010041} bvec_slabs[] __read_mostly = {
42 { .nr_vecs = 16, .name = "biovec-16" },
43 { .nr_vecs = 64, .name = "biovec-64" },
44 { .nr_vecs = 128, .name = "biovec-128" },
Christoph Hellwiga8affc02021-03-11 12:01:37 +010045 { .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" },
Linus Torvalds1da177e2005-04-16 15:20:36 -070046};
Christoph Hellwig6ac0b712021-02-02 18:19:20 +010047
Christoph Hellwig7a800a22021-02-02 18:19:29 +010048static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
49{
50 switch (nr_vecs) {
51 /* smaller bios use inline vecs */
52 case 5 ... 16:
53 return &bvec_slabs[0];
54 case 17 ... 64:
55 return &bvec_slabs[1];
56 case 65 ... 128:
57 return &bvec_slabs[2];
Christoph Hellwiga8affc02021-03-11 12:01:37 +010058 case 129 ... BIO_MAX_VECS:
Christoph Hellwig7a800a22021-02-02 18:19:29 +010059 return &bvec_slabs[3];
60 default:
61 BUG();
62 return NULL;
63 }
64}
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
66/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
68 * IO code that does not need private memory pools.
69 */
Kent Overstreetf4f81542018-05-08 21:33:52 -040070struct bio_set fs_bio_set;
Kent Overstreet3f86a822012-09-06 15:35:01 -070071EXPORT_SYMBOL(fs_bio_set);
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
Jens Axboebb799ca2008-12-10 15:35:05 +010073/*
74 * Our slab pool management
75 */
76struct bio_slab {
77 struct kmem_cache *slab;
78 unsigned int slab_ref;
79 unsigned int slab_size;
80 char name[8];
81};
82static DEFINE_MUTEX(bio_slab_lock);
Ming Lei49d1ec82021-01-11 11:05:52 +080083static DEFINE_XARRAY(bio_slabs);
Jens Axboebb799ca2008-12-10 15:35:05 +010084
Ming Lei49d1ec82021-01-11 11:05:52 +080085static struct bio_slab *create_bio_slab(unsigned int size)
Jens Axboebb799ca2008-12-10 15:35:05 +010086{
Ming Lei49d1ec82021-01-11 11:05:52 +080087 struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL);
88
89 if (!bslab)
90 return NULL;
91
92 snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size);
93 bslab->slab = kmem_cache_create(bslab->name, size,
Christoph Hellwig1a7e76e2021-10-12 13:12:22 +020094 ARCH_KMALLOC_MINALIGN,
95 SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL);
Ming Lei49d1ec82021-01-11 11:05:52 +080096 if (!bslab->slab)
97 goto fail_alloc_slab;
98
99 bslab->slab_ref = 1;
100 bslab->slab_size = size;
101
102 if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL)))
103 return bslab;
104
105 kmem_cache_destroy(bslab->slab);
106
107fail_alloc_slab:
108 kfree(bslab);
109 return NULL;
110}
111
112static inline unsigned int bs_bio_slab_size(struct bio_set *bs)
113{
Ming Lei9f180e32021-01-11 11:05:54 +0800114 return bs->front_pad + sizeof(struct bio) + bs->back_pad;
Ming Lei49d1ec82021-01-11 11:05:52 +0800115}
116
117static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs)
118{
119 unsigned int size = bs_bio_slab_size(bs);
120 struct bio_slab *bslab;
Jens Axboebb799ca2008-12-10 15:35:05 +0100121
122 mutex_lock(&bio_slab_lock);
Ming Lei49d1ec82021-01-11 11:05:52 +0800123 bslab = xa_load(&bio_slabs, size);
124 if (bslab)
125 bslab->slab_ref++;
126 else
127 bslab = create_bio_slab(size);
Jens Axboebb799ca2008-12-10 15:35:05 +0100128 mutex_unlock(&bio_slab_lock);
Ming Lei49d1ec82021-01-11 11:05:52 +0800129
130 if (bslab)
131 return bslab->slab;
132 return NULL;
Jens Axboebb799ca2008-12-10 15:35:05 +0100133}
134
135static void bio_put_slab(struct bio_set *bs)
136{
137 struct bio_slab *bslab = NULL;
Ming Lei49d1ec82021-01-11 11:05:52 +0800138 unsigned int slab_size = bs_bio_slab_size(bs);
Jens Axboebb799ca2008-12-10 15:35:05 +0100139
140 mutex_lock(&bio_slab_lock);
141
Ming Lei49d1ec82021-01-11 11:05:52 +0800142 bslab = xa_load(&bio_slabs, slab_size);
Jens Axboebb799ca2008-12-10 15:35:05 +0100143 if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
144 goto out;
145
Ming Lei49d1ec82021-01-11 11:05:52 +0800146 WARN_ON_ONCE(bslab->slab != bs->bio_slab);
147
Jens Axboebb799ca2008-12-10 15:35:05 +0100148 WARN_ON(!bslab->slab_ref);
149
150 if (--bslab->slab_ref)
151 goto out;
152
Ming Lei49d1ec82021-01-11 11:05:52 +0800153 xa_erase(&bio_slabs, slab_size);
154
Jens Axboebb799ca2008-12-10 15:35:05 +0100155 kmem_cache_destroy(bslab->slab);
Ming Lei49d1ec82021-01-11 11:05:52 +0800156 kfree(bslab);
Jens Axboebb799ca2008-12-10 15:35:05 +0100157
158out:
159 mutex_unlock(&bio_slab_lock);
160}
161
Christoph Hellwig7a800a22021-02-02 18:19:29 +0100162void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200163{
Christoph Hellwig9e8c0d02021-10-12 18:17:57 +0200164 BUG_ON(nr_vecs > BIO_MAX_VECS);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200165
Christoph Hellwiga8affc02021-03-11 12:01:37 +0100166 if (nr_vecs == BIO_MAX_VECS)
Kent Overstreet9f060e22012-10-12 15:29:33 -0700167 mempool_free(bv, pool);
Christoph Hellwig7a800a22021-02-02 18:19:29 +0100168 else if (nr_vecs > BIO_INLINE_VECS)
169 kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
Jens Axboebb799ca2008-12-10 15:35:05 +0100170}
171
Christoph Hellwigf2c3eb92021-02-02 18:19:21 +0100172/*
173 * Make the first allocation restricted and don't dump info on allocation
174 * failures, since we'll fall back to the mempool in case of failure.
175 */
176static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177{
Christoph Hellwigf2c3eb92021-02-02 18:19:21 +0100178 return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) |
179 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
180}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
Christoph Hellwig7a800a22021-02-02 18:19:29 +0100182struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
183 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184{
Christoph Hellwig7a800a22021-02-02 18:19:29 +0100185 struct biovec_slab *bvs = biovec_slab(*nr_vecs);
186
187 if (WARN_ON_ONCE(!bvs))
Jens Axboe7ff93452008-12-11 11:53:43 +0100188 return NULL;
Jens Axboe7ff93452008-12-11 11:53:43 +0100189
190 /*
Christoph Hellwig7a800a22021-02-02 18:19:29 +0100191 * Upgrade the nr_vecs request to take full advantage of the allocation.
192 * We also rely on this in the bvec_free path.
Jens Axboe7ff93452008-12-11 11:53:43 +0100193 */
Christoph Hellwig7a800a22021-02-02 18:19:29 +0100194 *nr_vecs = bvs->nr_vecs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 /*
Christoph Hellwigf007a3d2021-02-02 18:19:22 +0100197 * Try a slab allocation first for all smaller allocations. If that
198 * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
Christoph Hellwiga8affc02021-03-11 12:01:37 +0100199 * The mempool is sized to handle up to BIO_MAX_VECS entries.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 */
Christoph Hellwiga8affc02021-03-11 12:01:37 +0100201 if (*nr_vecs < BIO_MAX_VECS) {
Christoph Hellwigf007a3d2021-02-02 18:19:22 +0100202 struct bio_vec *bvl;
Jens Axboe7ff93452008-12-11 11:53:43 +0100203
Christoph Hellwigf2c3eb92021-02-02 18:19:21 +0100204 bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
Christoph Hellwig7a800a22021-02-02 18:19:29 +0100205 if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM))
Christoph Hellwigf007a3d2021-02-02 18:19:22 +0100206 return bvl;
Christoph Hellwiga8affc02021-03-11 12:01:37 +0100207 *nr_vecs = BIO_MAX_VECS;
Jens Axboe7ff93452008-12-11 11:53:43 +0100208 }
209
Christoph Hellwigf007a3d2021-02-02 18:19:22 +0100210 return mempool_alloc(pool, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
212
Jens Axboe9ae3b3f52017-06-28 15:30:13 -0600213void bio_uninit(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214{
Christoph Hellwigdb9819c2020-06-27 09:31:47 +0200215#ifdef CONFIG_BLK_CGROUP
216 if (bio->bi_blkg) {
217 blkg_put(bio->bi_blkg);
218 bio->bi_blkg = NULL;
219 }
220#endif
Justin Teeece841a2019-12-05 10:09:01 +0800221 if (bio_integrity(bio))
222 bio_integrity_free(bio);
Satya Tangiralaa892c8d2020-05-14 00:37:18 +0000223
224 bio_crypt_free_ctx(bio);
Kent Overstreet4254bba2012-09-06 15:35:00 -0700225}
Jens Axboe9ae3b3f52017-06-28 15:30:13 -0600226EXPORT_SYMBOL(bio_uninit);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200227
Kent Overstreet4254bba2012-09-06 15:35:00 -0700228static void bio_free(struct bio *bio)
229{
230 struct bio_set *bs = bio->bi_pool;
Christoph Hellwig066ff572022-04-06 08:12:27 +0200231 void *p = bio;
232
233 WARN_ON_ONCE(!bs);
Kent Overstreet4254bba2012-09-06 15:35:00 -0700234
Jens Axboe9ae3b3f52017-06-28 15:30:13 -0600235 bio_uninit(bio);
Christoph Hellwig066ff572022-04-06 08:12:27 +0200236 bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
237 mempool_free(p - bs->front_pad, &bs->bio_pool);
Peter Osterlund36763472005-09-06 15:16:42 -0700238}
239
Jens Axboe9ae3b3f52017-06-28 15:30:13 -0600240/*
241 * Users of this function have their own bio allocation. Subsequently,
242 * they must remember to pair any call to bio_init() with bio_uninit()
243 * when IO has completed, or when the bio is released.
244 */
Christoph Hellwig49add492022-01-24 10:11:06 +0100245void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
Bart Van Assche16458cf2022-07-14 11:06:32 -0700246 unsigned short max_vecs, blk_opf_t opf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247{
Jens Axboeda521622021-08-11 09:20:04 -0600248 bio->bi_next = NULL;
Christoph Hellwig49add492022-01-24 10:11:06 +0100249 bio->bi_bdev = bdev;
250 bio->bi_opf = opf;
Jens Axboeda521622021-08-11 09:20:04 -0600251 bio->bi_flags = 0;
252 bio->bi_ioprio = 0;
Bart Van Assche44981352024-02-02 12:39:25 -0800253 bio->bi_write_hint = 0;
Jens Axboeda521622021-08-11 09:20:04 -0600254 bio->bi_status = 0;
255 bio->bi_iter.bi_sector = 0;
256 bio->bi_iter.bi_size = 0;
257 bio->bi_iter.bi_idx = 0;
258 bio->bi_iter.bi_bvec_done = 0;
259 bio->bi_end_io = NULL;
260 bio->bi_private = NULL;
261#ifdef CONFIG_BLK_CGROUP
262 bio->bi_blkg = NULL;
263 bio->bi_issue.value = 0;
Christoph Hellwig49add492022-01-24 10:11:06 +0100264 if (bdev)
265 bio_associate_blkg(bio);
Jens Axboeda521622021-08-11 09:20:04 -0600266#ifdef CONFIG_BLK_CGROUP_IOCOST
267 bio->bi_iocost_cost = 0;
268#endif
269#endif
270#ifdef CONFIG_BLK_INLINE_ENCRYPTION
271 bio->bi_crypt_context = NULL;
272#endif
273#ifdef CONFIG_BLK_DEV_INTEGRITY
274 bio->bi_integrity = NULL;
275#endif
276 bio->bi_vcnt = 0;
277
Jens Axboec4cf5262015-04-17 16:15:18 -0600278 atomic_set(&bio->__bi_remaining, 1);
Jens Axboedac56212015-04-17 16:23:59 -0600279 atomic_set(&bio->__bi_cnt, 1);
Christoph Hellwig3e087732021-10-12 13:12:24 +0200280 bio->bi_cookie = BLK_QC_T_NONE;
Ming Lei3a83f462016-11-22 08:57:21 -0700281
Ming Lei3a83f462016-11-22 08:57:21 -0700282 bio->bi_max_vecs = max_vecs;
Jens Axboeda521622021-08-11 09:20:04 -0600283 bio->bi_io_vec = table;
284 bio->bi_pool = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285}
H Hartley Sweetena112a712009-09-26 16:19:21 +0200286EXPORT_SYMBOL(bio_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287
288/**
Kent Overstreetf44b48c72012-09-06 15:34:58 -0700289 * bio_reset - reinitialize a bio
290 * @bio: bio to reset
Christoph Hellwiga7c50c92022-01-24 10:11:07 +0100291 * @bdev: block device to use the bio for
292 * @opf: operation and flags for bio
Kent Overstreetf44b48c72012-09-06 15:34:58 -0700293 *
294 * Description:
295 * After calling bio_reset(), @bio will be in the same state as a freshly
296 * allocated bio returned bio bio_alloc_bioset() - the only fields that are
297 * preserved are the ones that are initialized by bio_alloc_bioset(). See
298 * comment in struct bio.
299 */
Bart Van Assche16458cf2022-07-14 11:06:32 -0700300void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf)
Kent Overstreetf44b48c72012-09-06 15:34:58 -0700301{
Jens Axboe9ae3b3f52017-06-28 15:30:13 -0600302 bio_uninit(bio);
Kent Overstreetf44b48c72012-09-06 15:34:58 -0700303 memset(bio, 0, BIO_RESET_BYTES);
Jens Axboec4cf5262015-04-17 16:15:18 -0600304 atomic_set(&bio->__bi_remaining, 1);
Christoph Hellwiga7c50c92022-01-24 10:11:07 +0100305 bio->bi_bdev = bdev;
Christoph Hellwig78e34372022-02-04 08:19:34 +0100306 if (bio->bi_bdev)
307 bio_associate_blkg(bio);
Christoph Hellwiga7c50c92022-01-24 10:11:07 +0100308 bio->bi_opf = opf;
Kent Overstreetf44b48c72012-09-06 15:34:58 -0700309}
310EXPORT_SYMBOL(bio_reset);
311
Christoph Hellwig38f8baa2016-03-11 17:34:51 +0100312static struct bio *__bio_chain_endio(struct bio *bio)
Kent Overstreet196d38bc2013-11-23 18:34:15 -0800313{
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200314 struct bio *parent = bio->bi_private;
315
Yufen Yu3edf5342021-03-31 07:53:59 -0400316 if (bio->bi_status && !parent->bi_status)
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200317 parent->bi_status = bio->bi_status;
Kent Overstreet196d38bc2013-11-23 18:34:15 -0800318 bio_put(bio);
Christoph Hellwig38f8baa2016-03-11 17:34:51 +0100319 return parent;
320}
321
322static void bio_chain_endio(struct bio *bio)
323{
324 bio_endio(__bio_chain_endio(bio));
Kent Overstreet196d38bc2013-11-23 18:34:15 -0800325}
326
327/**
328 * bio_chain - chain bio completions
Randy Dunlap1051a902014-04-20 16:03:31 -0700329 * @bio: the target bio
Mauro Carvalho Chehab5b874af2020-08-27 07:51:44 +0200330 * @parent: the parent bio of @bio
Kent Overstreet196d38bc2013-11-23 18:34:15 -0800331 *
332 * The caller won't have a bi_end_io called when @bio completes - instead,
333 * @parent's bi_end_io won't be called until both @parent and @bio have
334 * completed; the chained bio will also be freed when it completes.
335 *
336 * The caller must not set bi_private or bi_end_io in @bio.
337 */
338void bio_chain(struct bio *bio, struct bio *parent)
339{
340 BUG_ON(bio->bi_private || bio->bi_end_io);
341
342 bio->bi_private = parent;
343 bio->bi_end_io = bio_chain_endio;
Jens Axboec4cf5262015-04-17 16:15:18 -0600344 bio_inc_remaining(parent);
Kent Overstreet196d38bc2013-11-23 18:34:15 -0800345}
346EXPORT_SYMBOL(bio_chain);
347
Christoph Hellwig81c21682024-05-06 06:20:24 +0200348/**
349 * bio_chain_and_submit - submit a bio after chaining it to another one
350 * @prev: bio to chain and submit
351 * @new: bio to chain to
352 *
353 * If @prev is non-NULL, chain it to @new and submit it.
354 *
355 * Return: @new.
356 */
357struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new)
358{
359 if (prev) {
360 bio_chain(prev, new);
361 submit_bio(prev);
362 }
363 return new;
364}
365
Chaitanya Kulkarni0a3140e2022-01-24 10:11:02 +0100366struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
Bart Van Assche16458cf2022-07-14 11:06:32 -0700367 unsigned int nr_pages, blk_opf_t opf, gfp_t gfp)
Christoph Hellwig3b005bf2022-01-24 10:11:01 +0100368{
Christoph Hellwig81c21682024-05-06 06:20:24 +0200369 return bio_chain_and_submit(bio, bio_alloc(bdev, nr_pages, opf, gfp));
Christoph Hellwig3b005bf2022-01-24 10:11:01 +0100370}
371EXPORT_SYMBOL_GPL(blk_next_bio);
372
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700373static void bio_alloc_rescue(struct work_struct *work)
374{
375 struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
376 struct bio *bio;
377
378 while (1) {
379 spin_lock(&bs->rescue_lock);
380 bio = bio_list_pop(&bs->rescue_list);
381 spin_unlock(&bs->rescue_lock);
382
383 if (!bio)
384 break;
385
Christoph Hellwiged00aab2020-07-01 10:59:44 +0200386 submit_bio_noacct(bio);
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700387 }
388}
389
390static void punt_bios_to_rescuer(struct bio_set *bs)
391{
392 struct bio_list punt, nopunt;
393 struct bio *bio;
394
NeilBrown47e0fb42017-06-18 14:38:57 +1000395 if (WARN_ON_ONCE(!bs->rescue_workqueue))
396 return;
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700397 /*
398 * In order to guarantee forward progress we must punt only bios that
399 * were allocated from this bio_set; otherwise, if there was a bio on
400 * there for a stacking driver higher up in the stack, processing it
401 * could require allocating bios from this bio_set, and doing that from
402 * our own rescuer would be bad.
403 *
404 * Since bio lists are singly linked, pop them all instead of trying to
405 * remove from the middle of the list:
406 */
407
408 bio_list_init(&punt);
409 bio_list_init(&nopunt);
410
NeilBrownf5fe1b52017-03-10 17:00:47 +1100411 while ((bio = bio_list_pop(&current->bio_list[0])))
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700412 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
NeilBrownf5fe1b52017-03-10 17:00:47 +1100413 current->bio_list[0] = nopunt;
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700414
NeilBrownf5fe1b52017-03-10 17:00:47 +1100415 bio_list_init(&nopunt);
416 while ((bio = bio_list_pop(&current->bio_list[1])))
417 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
418 current->bio_list[1] = nopunt;
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700419
420 spin_lock(&bs->rescue_lock);
421 bio_list_merge(&bs->rescue_list, &punt);
422 spin_unlock(&bs->rescue_lock);
423
424 queue_work(bs->rescue_workqueue, &bs->rescue_work);
425}
426
Pavel Begunkovb99182c2022-11-02 15:18:22 +0000427static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache)
428{
429 unsigned long flags;
430
431 /* cache->free_list must be empty */
432 if (WARN_ON_ONCE(cache->free_list))
433 return;
434
435 local_irq_save(flags);
436 cache->free_list = cache->free_list_irq;
437 cache->free_list_irq = NULL;
438 cache->nr += cache->nr_irq;
439 cache->nr_irq = 0;
440 local_irq_restore(flags);
441}
442
Mike Snitzer0df71652022-03-24 16:35:24 -0400443static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
Bart Van Assche16458cf2022-07-14 11:06:32 -0700444 unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp,
Mike Snitzer0df71652022-03-24 16:35:24 -0400445 struct bio_set *bs)
446{
447 struct bio_alloc_cache *cache;
448 struct bio *bio;
449
450 cache = per_cpu_ptr(bs->cache, get_cpu());
451 if (!cache->free_list) {
Pavel Begunkovb99182c2022-11-02 15:18:22 +0000452 if (READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD)
453 bio_alloc_irq_cache_splice(cache);
454 if (!cache->free_list) {
455 put_cpu();
456 return NULL;
457 }
Mike Snitzer0df71652022-03-24 16:35:24 -0400458 }
459 bio = cache->free_list;
460 cache->free_list = bio->bi_next;
461 cache->nr--;
462 put_cpu();
463
464 bio_init(bio, bdev, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs, opf);
465 bio->bi_pool = bs;
466 return bio;
467}
468
Kent Overstreetf44b48c72012-09-06 15:34:58 -0700469/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 * bio_alloc_bioset - allocate a bio for I/O
Christoph Hellwig609be102022-01-24 10:11:03 +0100471 * @bdev: block device to allocate the bio for (can be %NULL)
472 * @nr_vecs: number of bvecs to pre-allocate
473 * @opf: operation and flags for bio
Randy Dunlap519c8e92017-10-16 11:01:00 -0700474 * @gfp_mask: the GFP_* mask given to the slab allocator
Jaak Ristiojadb18efa2010-01-15 12:05:07 +0200475 * @bs: the bio_set to allocate from.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 *
Christoph Hellwig31751992021-01-26 15:52:34 +0100477 * Allocate a bio from the mempools in @bs.
Kent Overstreet3f86a822012-09-06 15:35:01 -0700478 *
Christoph Hellwig31751992021-01-26 15:52:34 +0100479 * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to
480 * allocate a bio. This is due to the mempool guarantees. To make this work,
481 * callers must never allocate more than 1 bio at a time from the general pool.
482 * Callers that need to allocate more than 1 bio must always submit the
483 * previously allocated bio for IO before attempting to allocate a new one.
484 * Failure to do so can cause deadlocks under memory pressure.
Kent Overstreet3f86a822012-09-06 15:35:01 -0700485 *
Christoph Hellwig31751992021-01-26 15:52:34 +0100486 * Note that when running under submit_bio_noacct() (i.e. any block driver),
487 * bios are not submitted until after you return - see the code in
488 * submit_bio_noacct() that converts recursion into iteration, to prevent
489 * stack overflows.
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700490 *
Christoph Hellwig31751992021-01-26 15:52:34 +0100491 * This would normally mean allocating multiple bios under submit_bio_noacct()
492 * would be susceptible to deadlocks, but we have
493 * deadlock avoidance code that resubmits any blocked bios from a rescuer
494 * thread.
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700495 *
Christoph Hellwig31751992021-01-26 15:52:34 +0100496 * However, we do not guarantee forward progress for allocations from other
497 * mempools. Doing multiple allocations from the same mempool under
498 * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
499 * for per bio allocations.
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700500 *
Christoph Hellwig31751992021-01-26 15:52:34 +0100501 * Returns: Pointer to new bio on success, NULL on failure.
Kent Overstreet3f86a822012-09-06 15:35:01 -0700502 */
Christoph Hellwig609be102022-01-24 10:11:03 +0100503struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
Bart Van Assche16458cf2022-07-14 11:06:32 -0700504 blk_opf_t opf, gfp_t gfp_mask,
Dan Carpenter7a88fa12017-03-23 13:24:55 +0300505 struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506{
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700507 gfp_t saved_gfp = gfp_mask;
Tejun Heo451a9eb2009-04-15 19:50:51 +0200508 struct bio *bio;
509 void *p;
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200510
Christoph Hellwig609be102022-01-24 10:11:03 +0100511 /* should not use nobvec bioset for nr_vecs > 0 */
512 if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0))
Christoph Hellwig31751992021-01-26 15:52:34 +0100513 return NULL;
Kent Overstreet3f86a822012-09-06 15:35:01 -0700514
Mike Snitzer0df71652022-03-24 16:35:24 -0400515 if (opf & REQ_ALLOC_CACHE) {
516 if (bs->cache && nr_vecs <= BIO_INLINE_VECS) {
517 bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf,
518 gfp_mask, bs);
519 if (bio)
520 return bio;
521 /*
522 * No cached bio available, bio returned below marked with
523 * REQ_ALLOC_CACHE to particpate in per-cpu alloc cache.
524 */
525 } else {
526 opf &= ~REQ_ALLOC_CACHE;
527 }
528 }
529
Christoph Hellwig31751992021-01-26 15:52:34 +0100530 /*
531 * submit_bio_noacct() converts recursion to iteration; this means if
532 * we're running beneath it, any bios we allocate and submit will not be
533 * submitted (and thus freed) until after we return.
534 *
535 * This exposes us to a potential deadlock if we allocate multiple bios
536 * from the same bio_set() while running underneath submit_bio_noacct().
537 * If we were to allocate multiple bios (say a stacking block driver
538 * that was splitting bios), we would deadlock if we exhausted the
539 * mempool's reserve.
540 *
541 * We solve this, and guarantee forward progress, with a rescuer
542 * workqueue per bio_set. If we go to allocate and there are bios on
543 * current->bio_list, we first try the allocation without
544 * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be
545 * blocking to the rescuer workqueue before we retry with the original
546 * gfp_flags.
547 */
548 if (current->bio_list &&
549 (!bio_list_empty(&current->bio_list[0]) ||
550 !bio_list_empty(&current->bio_list[1])) &&
551 bs->rescue_workqueue)
552 gfp_mask &= ~__GFP_DIRECT_RECLAIM;
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700553
Christoph Hellwig31751992021-01-26 15:52:34 +0100554 p = mempool_alloc(&bs->bio_pool, gfp_mask);
555 if (!p && gfp_mask != saved_gfp) {
556 punt_bios_to_rescuer(bs);
557 gfp_mask = saved_gfp;
Kent Overstreet8aa6ba22018-05-08 21:33:50 -0400558 p = mempool_alloc(&bs->bio_pool, gfp_mask);
Kent Overstreet3f86a822012-09-06 15:35:01 -0700559 }
Tejun Heo451a9eb2009-04-15 19:50:51 +0200560 if (unlikely(!p))
561 return NULL;
Pavel Begunkov759aa122022-11-02 15:18:20 +0000562 if (!mempool_is_saturated(&bs->bio_pool))
563 opf &= ~REQ_ALLOC_CACHE;
Ingo Molnar34053972009-02-21 11:16:36 +0100564
Christoph Hellwig31751992021-01-26 15:52:34 +0100565 bio = p + bs->front_pad;
Christoph Hellwig609be102022-01-24 10:11:03 +0100566 if (nr_vecs > BIO_INLINE_VECS) {
Christoph Hellwig31751992021-01-26 15:52:34 +0100567 struct bio_vec *bvl = NULL;
Ingo Molnar34053972009-02-21 11:16:36 +0100568
Christoph Hellwig609be102022-01-24 10:11:03 +0100569 bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700570 if (!bvl && gfp_mask != saved_gfp) {
571 punt_bios_to_rescuer(bs);
572 gfp_mask = saved_gfp;
Christoph Hellwig609be102022-01-24 10:11:03 +0100573 bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700574 }
Ingo Molnar34053972009-02-21 11:16:36 +0100575 if (unlikely(!bvl))
576 goto err_free;
Kent Overstreeta38352e2012-05-25 13:03:11 -0700577
Christoph Hellwig49add492022-01-24 10:11:06 +0100578 bio_init(bio, bdev, bvl, nr_vecs, opf);
Christoph Hellwig609be102022-01-24 10:11:03 +0100579 } else if (nr_vecs) {
Christoph Hellwig49add492022-01-24 10:11:06 +0100580 bio_init(bio, bdev, bio->bi_inline_vecs, BIO_INLINE_VECS, opf);
Christoph Hellwig31751992021-01-26 15:52:34 +0100581 } else {
Christoph Hellwig49add492022-01-24 10:11:06 +0100582 bio_init(bio, bdev, NULL, 0, opf);
Ingo Molnar34053972009-02-21 11:16:36 +0100583 }
Kent Overstreet3f86a822012-09-06 15:35:01 -0700584
585 bio->bi_pool = bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 return bio;
Ingo Molnar34053972009-02-21 11:16:36 +0100587
588err_free:
Kent Overstreet8aa6ba22018-05-08 21:33:50 -0400589 mempool_free(p, &bs->bio_pool);
Ingo Molnar34053972009-02-21 11:16:36 +0100590 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591}
H Hartley Sweetena112a712009-09-26 16:19:21 +0200592EXPORT_SYMBOL(bio_alloc_bioset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593
Christoph Hellwig31751992021-01-26 15:52:34 +0100594/**
Christoph Hellwig066ff572022-04-06 08:12:27 +0200595 * bio_kmalloc - kmalloc a bio
596 * @nr_vecs: number of bio_vecs to allocate
Christoph Hellwig31751992021-01-26 15:52:34 +0100597 * @gfp_mask: the GFP_* mask given to the slab allocator
Christoph Hellwig31751992021-01-26 15:52:34 +0100598 *
Christoph Hellwig066ff572022-04-06 08:12:27 +0200599 * Use kmalloc to allocate a bio (including bvecs). The bio must be initialized
600 * using bio_init() before use. To free a bio returned from this function use
601 * kfree() after calling bio_uninit(). A bio returned from this function can
602 * be reused by calling bio_uninit() before calling bio_init() again.
603 *
604 * Note that unlike bio_alloc() or bio_alloc_bioset() allocations from this
Deming Wang340e1342022-10-06 04:44:50 -0400605 * function are not backed by a mempool can fail. Do not use this function
Christoph Hellwig066ff572022-04-06 08:12:27 +0200606 * for allocations in the file system I/O path.
Christoph Hellwig31751992021-01-26 15:52:34 +0100607 *
608 * Returns: Pointer to new bio on success, NULL on failure.
609 */
Christoph Hellwig066ff572022-04-06 08:12:27 +0200610struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask)
Christoph Hellwig31751992021-01-26 15:52:34 +0100611{
612 struct bio *bio;
613
Christoph Hellwig066ff572022-04-06 08:12:27 +0200614 if (nr_vecs > UIO_MAXIOV)
Christoph Hellwig31751992021-01-26 15:52:34 +0100615 return NULL;
Christoph Hellwig066ff572022-04-06 08:12:27 +0200616 return kmalloc(struct_size(bio, bi_inline_vecs, nr_vecs), gfp_mask);
Christoph Hellwig31751992021-01-26 15:52:34 +0100617}
618EXPORT_SYMBOL(bio_kmalloc);
619
Kent Overstreet649f0702023-08-13 14:26:36 -0400620void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621{
Kent Overstreet79886132013-11-23 17:19:00 -0800622 struct bio_vec bv;
623 struct bvec_iter iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624
Kent Overstreet649f0702023-08-13 14:26:36 -0400625 __bio_for_each_segment(bv, bio, iter, start)
Christoph Hellwigab6c3402021-07-27 07:56:36 +0200626 memzero_bvec(&bv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627}
Kent Overstreet649f0702023-08-13 14:26:36 -0400628EXPORT_SYMBOL(zero_fill_bio_iter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629
Ming Lei83c9c542020-01-05 09:41:14 +0800630/**
631 * bio_truncate - truncate the bio to small size of @new_size
632 * @bio: the bio to be truncated
633 * @new_size: new size for truncating the bio
634 *
635 * Description:
636 * Truncate the bio to new size of @new_size. If bio_op(bio) is
637 * REQ_OP_READ, zero the truncated part. This function should only
638 * be used for handling corner cases, such as bio eod.
639 */
Christoph Hellwig4f7ab092021-10-12 18:18:04 +0200640static void bio_truncate(struct bio *bio, unsigned new_size)
Ming Lei85a8ce62019-12-28 07:05:48 +0800641{
642 struct bio_vec bv;
643 struct bvec_iter iter;
644 unsigned int done = 0;
645 bool truncated = false;
646
647 if (new_size >= bio->bi_iter.bi_size)
648 return;
649
Ming Lei83c9c542020-01-05 09:41:14 +0800650 if (bio_op(bio) != REQ_OP_READ)
Ming Lei85a8ce62019-12-28 07:05:48 +0800651 goto exit;
652
653 bio_for_each_segment(bv, bio, iter) {
654 if (done + bv.bv_len > new_size) {
655 unsigned offset;
656
657 if (!truncated)
658 offset = new_size - done;
659 else
660 offset = 0;
OGAWA Hirofumi3ee859e2022-01-09 18:36:43 +0900661 zero_user(bv.bv_page, bv.bv_offset + offset,
662 bv.bv_len - offset);
Ming Lei85a8ce62019-12-28 07:05:48 +0800663 truncated = true;
664 }
665 done += bv.bv_len;
666 }
667
668 exit:
669 /*
670 * Don't touch bvec table here and make it really immutable, since
671 * fs bio user has to retrieve all pages via bio_for_each_segment_all
672 * in its .end_bio() callback.
673 *
674 * It is enough to truncate bio by updating .bi_size since we can make
675 * correct bvec with the updated .bi_size for drivers.
676 */
677 bio->bi_iter.bi_size = new_size;
678}
679
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680/**
Christoph Hellwig29125ed2020-03-25 16:48:40 +0100681 * guard_bio_eod - truncate a BIO to fit the block device
682 * @bio: bio to truncate
683 *
684 * This allows us to do IO even on the odd last sectors of a device, even if the
685 * block size is some multiple of the physical sector size.
686 *
687 * We'll just truncate the bio to the size of the device, and clear the end of
688 * the buffer head manually. Truly out-of-range accesses will turn into actual
689 * I/O errors, this only handles the "we need to be able to do I/O at the final
690 * sector" case.
691 */
692void guard_bio_eod(struct bio *bio)
693{
Christoph Hellwig309dca302021-01-24 11:02:34 +0100694 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
Christoph Hellwig29125ed2020-03-25 16:48:40 +0100695
696 if (!maxsector)
697 return;
698
699 /*
700 * If the *whole* IO is past the end of the device,
701 * let it through, and the IO layer will turn it into
702 * an EIO.
703 */
704 if (unlikely(bio->bi_iter.bi_sector >= maxsector))
705 return;
706
707 maxsector -= bio->bi_iter.bi_sector;
708 if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
709 return;
710
711 bio_truncate(bio, maxsector << 9);
712}
713
Pavel Begunkovb99182c2022-11-02 15:18:22 +0000714static int __bio_alloc_cache_prune(struct bio_alloc_cache *cache,
715 unsigned int nr)
Jens Axboebe4d2342021-03-08 11:37:47 -0700716{
717 unsigned int i = 0;
718 struct bio *bio;
719
Jens Axboefcade2c2021-12-01 16:19:18 -0700720 while ((bio = cache->free_list) != NULL) {
721 cache->free_list = bio->bi_next;
Jens Axboebe4d2342021-03-08 11:37:47 -0700722 cache->nr--;
723 bio_free(bio);
724 if (++i == nr)
725 break;
726 }
Pavel Begunkovb99182c2022-11-02 15:18:22 +0000727 return i;
728}
729
730static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
731 unsigned int nr)
732{
733 nr -= __bio_alloc_cache_prune(cache, nr);
734 if (!READ_ONCE(cache->free_list)) {
735 bio_alloc_irq_cache_splice(cache);
736 __bio_alloc_cache_prune(cache, nr);
737 }
Jens Axboebe4d2342021-03-08 11:37:47 -0700738}
739
740static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node)
741{
742 struct bio_set *bs;
743
744 bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead);
745 if (bs->cache) {
746 struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);
747
748 bio_alloc_cache_prune(cache, -1U);
749 }
750 return 0;
751}
752
753static void bio_alloc_cache_destroy(struct bio_set *bs)
754{
755 int cpu;
756
757 if (!bs->cache)
758 return;
759
760 cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
761 for_each_possible_cpu(cpu) {
762 struct bio_alloc_cache *cache;
763
764 cache = per_cpu_ptr(bs->cache, cpu);
765 bio_alloc_cache_prune(cache, -1U);
766 }
767 free_percpu(bs->cache);
Jens Axboe605f7412022-05-29 07:13:09 -0600768 bs->cache = NULL;
Jens Axboebe4d2342021-03-08 11:37:47 -0700769}
770
Pavel Begunkovf25cf752022-11-02 15:18:21 +0000771static inline void bio_put_percpu_cache(struct bio *bio)
772{
773 struct bio_alloc_cache *cache;
774
775 cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
Pavel Begunkove516c3f2024-02-07 14:14:29 +0000776 if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX)
777 goto out_free;
Pavel Begunkovb99182c2022-11-02 15:18:22 +0000778
Pavel Begunkovc9f5f3a2024-02-07 14:14:28 +0000779 if (in_task()) {
Pavel Begunkove516c3f2024-02-07 14:14:29 +0000780 bio_uninit(bio);
Pavel Begunkovf25cf752022-11-02 15:18:21 +0000781 bio->bi_next = cache->free_list;
Pavel Begunkovc9f5f3a2024-02-07 14:14:28 +0000782 /* Not necessary but helps not to iopoll already freed bios */
Jens Axboe11eb6952023-02-24 09:59:44 -0700783 bio->bi_bdev = NULL;
Pavel Begunkovf25cf752022-11-02 15:18:21 +0000784 cache->free_list = bio;
785 cache->nr++;
Pavel Begunkove516c3f2024-02-07 14:14:29 +0000786 } else if (in_hardirq()) {
787 lockdep_assert_irqs_disabled();
Pavel Begunkovf25cf752022-11-02 15:18:21 +0000788
Pavel Begunkove516c3f2024-02-07 14:14:29 +0000789 bio_uninit(bio);
Pavel Begunkovb99182c2022-11-02 15:18:22 +0000790 bio->bi_next = cache->free_list_irq;
791 cache->free_list_irq = bio;
792 cache->nr_irq++;
Pavel Begunkove516c3f2024-02-07 14:14:29 +0000793 } else {
794 goto out_free;
Pavel Begunkovb99182c2022-11-02 15:18:22 +0000795 }
Pavel Begunkovf25cf752022-11-02 15:18:21 +0000796 put_cpu();
Pavel Begunkove516c3f2024-02-07 14:14:29 +0000797 return;
798out_free:
799 put_cpu();
800 bio_free(bio);
Pavel Begunkovf25cf752022-11-02 15:18:21 +0000801}
802
Christoph Hellwig29125ed2020-03-25 16:48:40 +0100803/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 * bio_put - release a reference to a bio
805 * @bio: bio to release reference to
806 *
807 * Description:
808 * Put a reference to a &struct bio, either one you have gotten with
NeilBrown9b10f6a2017-06-18 14:38:59 +1000809 * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 **/
811void bio_put(struct bio *bio)
812{
Jens Axboebe4d2342021-03-08 11:37:47 -0700813 if (unlikely(bio_flagged(bio, BIO_REFFED))) {
Christoph Hellwig9e8c0d02021-10-12 18:17:57 +0200814 BUG_ON(!atomic_read(&bio->__bi_cnt));
Jens Axboebe4d2342021-03-08 11:37:47 -0700815 if (!atomic_dec_and_test(&bio->__bi_cnt))
816 return;
817 }
Pavel Begunkovf25cf752022-11-02 15:18:21 +0000818 if (bio->bi_opf & REQ_ALLOC_CACHE)
819 bio_put_percpu_cache(bio);
820 else
Jens Axboebe4d2342021-03-08 11:37:47 -0700821 bio_free(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822}
H Hartley Sweetena112a712009-09-26 16:19:21 +0200823EXPORT_SYMBOL(bio_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100825static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
Kent Overstreet59d276f2013-11-23 18:19:27 -0800826{
Jens Axboeb7c44ed2015-07-24 12:37:59 -0600827 bio_set_flag(bio, BIO_CLONED);
Hannes Reineckeca474b72018-11-12 10:35:25 -0700828 bio->bi_ioprio = bio_src->bi_ioprio;
Bart Van Assche44981352024-02-02 12:39:25 -0800829 bio->bi_write_hint = bio_src->bi_write_hint;
Kent Overstreet59d276f2013-11-23 18:19:27 -0800830 bio->bi_iter = bio_src->bi_iter;
Paolo Valente20bd7232016-07-27 07:22:05 +0200831
Christoph Hellwig7ecc56c2022-05-04 07:29:50 -0700832 if (bio->bi_bdev) {
833 if (bio->bi_bdev == bio_src->bi_bdev &&
834 bio_flagged(bio_src, BIO_REMAPPED))
835 bio_set_flag(bio, BIO_REMAPPED);
836 bio_clone_blkg_association(bio, bio_src);
837 }
Christoph Hellwig56b4b5a2022-02-02 17:01:06 +0100838
839 if (bio_crypt_clone(bio, bio_src, gfp) < 0)
840 return -ENOMEM;
841 if (bio_integrity(bio_src) &&
842 bio_integrity_clone(bio, bio_src, gfp) < 0)
843 return -ENOMEM;
844 return 0;
Kent Overstreet59d276f2013-11-23 18:19:27 -0800845}
Kent Overstreet59d276f2013-11-23 18:19:27 -0800846
847/**
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100848 * bio_alloc_clone - clone a bio that shares the original bio's biovec
849 * @bdev: block_device to clone onto
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100850 * @bio_src: bio to clone from
851 * @gfp: allocation priority
852 * @bs: bio_set to allocate from
Kent Overstreet59d276f2013-11-23 18:19:27 -0800853 *
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100854 * Allocate a new bio that is a clone of @bio_src. The caller owns the returned
855 * bio, but not the actual data it points to.
856 *
857 * The caller must ensure that the return bio is not freed before @bio_src.
Kent Overstreet59d276f2013-11-23 18:19:27 -0800858 */
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100859struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
860 gfp_t gfp, struct bio_set *bs)
Kent Overstreet59d276f2013-11-23 18:19:27 -0800861{
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100862 struct bio *bio;
Kent Overstreet59d276f2013-11-23 18:19:27 -0800863
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100864 bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs);
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100865 if (!bio)
Kent Overstreet59d276f2013-11-23 18:19:27 -0800866 return NULL;
867
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100868 if (__bio_clone(bio, bio_src, gfp) < 0) {
869 bio_put(bio);
Christoph Hellwig56b4b5a2022-02-02 17:01:06 +0100870 return NULL;
871 }
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100872 bio->bi_io_vec = bio_src->bi_io_vec;
Kent Overstreet59d276f2013-11-23 18:19:27 -0800873
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100874 return bio;
Kent Overstreet59d276f2013-11-23 18:19:27 -0800875}
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100876EXPORT_SYMBOL(bio_alloc_clone);
Kent Overstreet59d276f2013-11-23 18:19:27 -0800877
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100878/**
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100879 * bio_init_clone - clone a bio that shares the original bio's biovec
880 * @bdev: block_device to clone onto
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100881 * @bio: bio to clone into
882 * @bio_src: bio to clone from
883 * @gfp: allocation priority
884 *
885 * Initialize a new bio in caller provided memory that is a clone of @bio_src.
886 * The caller owns the returned bio, but not the actual data it points to.
887 *
888 * The caller must ensure that @bio_src is not freed before @bio.
889 */
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100890int bio_init_clone(struct block_device *bdev, struct bio *bio,
891 struct bio *bio_src, gfp_t gfp)
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100892{
893 int ret;
894
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100895 bio_init(bio, bdev, bio_src->bi_io_vec, 0, bio_src->bi_opf);
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100896 ret = __bio_clone(bio, bio_src, gfp);
897 if (ret)
898 bio_uninit(bio);
899 return ret;
900}
Christoph Hellwigabfc4262022-02-02 17:01:09 +0100901EXPORT_SYMBOL(bio_init_clone);
Christoph Hellwiga0e8de72022-02-02 17:01:08 +0100902
Christoph Hellwig9a6083b2021-10-12 18:18:01 +0200903/**
904 * bio_full - check if the bio is full
905 * @bio: bio to check
906 * @len: length of one segment to be added
907 *
908 * Return true if @bio is full and one segment with @len bytes can't be
909 * added to the bio, otherwise return false
910 */
911static inline bool bio_full(struct bio *bio, unsigned len)
912{
913 if (bio->bi_vcnt >= bio->bi_max_vecs)
914 return true;
915 if (bio->bi_iter.bi_size > UINT_MAX - len)
916 return true;
917 return false;
918}
919
Christoph Hellwig858c7082023-07-24 09:54:32 -0700920static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page,
921 unsigned int len, unsigned int off, bool *same_page)
Ming Lei59194822019-03-17 18:01:07 +0800922{
Matthew Wilcox (Oracle)d8166512020-08-17 20:52:06 +0100923 size_t bv_end = bv->bv_offset + bv->bv_len;
924 phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1;
Ming Lei59194822019-03-17 18:01:07 +0800925 phys_addr_t page_addr = page_to_phys(page);
926
927 if (vec_end_addr + 1 != page_addr + off)
928 return false;
929 if (xen_domain() && !xen_biovec_phys_mergeable(bv, page))
930 return false;
Logan Gunthorpe49580e692022-10-21 11:41:11 -0600931 if (!zone_device_pages_have_same_pgmap(bv->bv_page, page))
932 return false;
Christoph Hellwig52d52d12019-04-11 08:23:31 +0200933
Christoph Hellwigff896732019-06-17 11:14:11 +0200934 *same_page = ((vec_end_addr & PAGE_MASK) == page_addr);
Christoph Hellwig858c7082023-07-24 09:54:32 -0700935 if (!*same_page) {
936 if (IS_ENABLED(CONFIG_KMSAN))
937 return false;
938 if (bv->bv_page + bv_end / PAGE_SIZE != page + off / PAGE_SIZE)
939 return false;
940 }
Ming Lei59194822019-03-17 18:01:07 +0800941
Christoph Hellwig0eca8b62023-07-24 09:54:29 -0700942 bv->bv_len += len;
Christoph Hellwig0eca8b62023-07-24 09:54:29 -0700943 return true;
Christoph Hellwig9774b392021-10-12 18:18:02 +0200944}
945
Christoph Hellwige4581102020-05-12 17:55:46 +0900946/*
947 * Try to merge a page into a segment, while obeying the hardware segment
948 * size limit. This is not for normal read/write bios, but for passthrough
949 * or Zone Append operations that we can't split.
950 */
Jinyoung Choi7c8998f2023-08-03 11:48:27 +0900951bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv,
Christoph Hellwigae42f0b2023-07-24 09:54:33 -0700952 struct page *page, unsigned len, unsigned offset,
953 bool *same_page)
Ming Lei489fbbc2019-03-29 15:08:00 +0800954{
955 unsigned long mask = queue_segment_boundary(q);
956 phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset;
957 phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
958
959 if ((addr1 | mask) != (addr2 | mask))
960 return false;
Christoph Hellwig3f034c32023-12-04 18:34:18 +0100961 if (len > queue_max_segment_size(q) - bv->bv_len)
Ming Lei489fbbc2019-03-29 15:08:00 +0800962 return false;
Christoph Hellwig858c7082023-07-24 09:54:32 -0700963 return bvec_try_merge_page(bv, page, len, offset, same_page);
Ming Lei489fbbc2019-03-29 15:08:00 +0800964}
965
Shaohua Lif4595872017-03-24 10:34:43 -0700966/**
Christoph Hellwige4581102020-05-12 17:55:46 +0900967 * bio_add_hw_page - attempt to add a page to a bio with hw constraints
968 * @q: the target queue
969 * @bio: destination bio
970 * @page: page to add
971 * @len: vec entry length
972 * @offset: vec entry offset
973 * @max_sectors: maximum number of sectors that can be added
974 * @same_page: return if the segment has been merged inside the same page
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 *
Christoph Hellwige4581102020-05-12 17:55:46 +0900976 * Add a page to a bio while respecting the hardware max_sectors, max_segment
977 * and gap limitations.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 */
Christoph Hellwige4581102020-05-12 17:55:46 +0900979int bio_add_hw_page(struct request_queue *q, struct bio *bio,
Ming Lei19047082019-03-17 18:01:08 +0800980 struct page *page, unsigned int len, unsigned int offset,
Christoph Hellwige4581102020-05-12 17:55:46 +0900981 unsigned int max_sectors, bool *same_page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982{
Christoph Hellwig6ef02df2023-12-04 18:34:19 +0100983 unsigned int max_size = max_sectors << SECTOR_SHIFT;
984
Christoph Hellwige4581102020-05-12 17:55:46 +0900985 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 return 0;
987
Christoph Hellwig6ef02df2023-12-04 18:34:19 +0100988 len = min3(len, max_size, queue_max_segment_size(q));
989 if (len > max_size - bio->bi_iter.bi_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 return 0;
991
Jens Axboe80cfd542006-01-06 09:43:28 +0100992 if (bio->bi_vcnt > 0) {
Christoph Hellwigae42f0b2023-07-24 09:54:33 -0700993 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
994
995 if (bvec_try_merge_hw_page(q, bv, page, len, offset,
Christoph Hellwig858c7082023-07-24 09:54:32 -0700996 same_page)) {
997 bio->bi_iter.bi_size += len;
Christoph Hellwig384209cd2019-08-12 17:39:57 +0200998 return len;
Christoph Hellwig858c7082023-07-24 09:54:32 -0700999 }
Christoph Hellwig320ea862019-08-12 17:39:56 +02001000
Christoph Hellwigcd1d83e2023-07-24 09:54:26 -07001001 if (bio->bi_vcnt >=
1002 min(bio->bi_max_vecs, queue_max_segments(q)))
1003 return 0;
1004
Christoph Hellwig320ea862019-08-12 17:39:56 +02001005 /*
1006 * If the queue doesn't support SG gaps and adding this segment
1007 * would create a gap, disallow it.
1008 */
Christoph Hellwigae42f0b2023-07-24 09:54:33 -07001009 if (bvec_gap_to_prev(&q->limits, bv, offset))
Christoph Hellwig320ea862019-08-12 17:39:56 +02001010 return 0;
Jens Axboe80cfd542006-01-06 09:43:28 +01001011 }
1012
Christoph Hellwigd58cdfa2023-02-03 16:06:12 +01001013 bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, offset);
Maurizio Lombardifcbf6a02014-12-10 14:16:53 -08001014 bio->bi_vcnt++;
Christoph Hellwigdcdca752019-04-25 09:04:35 +02001015 bio->bi_iter.bi_size += len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 return len;
1017}
Ming Lei19047082019-03-17 18:01:08 +08001018
Christoph Hellwige4581102020-05-12 17:55:46 +09001019/**
1020 * bio_add_pc_page - attempt to add page to passthrough bio
1021 * @q: the target queue
1022 * @bio: destination bio
1023 * @page: page to add
1024 * @len: vec entry length
1025 * @offset: vec entry offset
1026 *
1027 * Attempt to add a page to the bio_vec maplist. This can fail for a
1028 * number of reasons, such as the bio being full or target block device
1029 * limitations. The target block device must allow bio's up to PAGE_SIZE,
1030 * so it is always possible to add a single page to an empty bio.
1031 *
1032 * This should only be used by passthrough bios.
1033 */
Ming Lei19047082019-03-17 18:01:08 +08001034int bio_add_pc_page(struct request_queue *q, struct bio *bio,
1035 struct page *page, unsigned int len, unsigned int offset)
1036{
Christoph Hellwigd1916c82019-08-12 17:39:58 +02001037 bool same_page = false;
Christoph Hellwige4581102020-05-12 17:55:46 +09001038 return bio_add_hw_page(q, bio, page, len, offset,
1039 queue_max_hw_sectors(q), &same_page);
Ming Lei19047082019-03-17 18:01:08 +08001040}
H Hartley Sweetena112a712009-09-26 16:19:21 +02001041EXPORT_SYMBOL(bio_add_pc_page);
Mike Christie6e68af62005-11-11 05:30:27 -06001042
1043/**
Johannes Thumshirnae293332021-02-04 19:21:40 +09001044 * bio_add_zone_append_page - attempt to add page to zone-append bio
1045 * @bio: destination bio
1046 * @page: page to add
1047 * @len: vec entry length
1048 * @offset: vec entry offset
1049 *
1050 * Attempt to add a page to the bio_vec maplist of a bio that will be submitted
1051 * for a zone-append request. This can fail for a number of reasons, such as the
1052 * bio being full or the target block device is not a zoned block device or
1053 * other limitations of the target block device. The target block device must
1054 * allow bio's up to PAGE_SIZE, so it is always possible to add a single page
1055 * to an empty bio.
1056 *
1057 * Returns: number of bytes added to the bio, or 0 in case of a failure.
1058 */
1059int bio_add_zone_append_page(struct bio *bio, struct page *page,
1060 unsigned int len, unsigned int offset)
1061{
Pavel Begunkov3caee4632021-10-14 15:03:28 +01001062 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
Johannes Thumshirnae293332021-02-04 19:21:40 +09001063 bool same_page = false;
1064
1065 if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND))
1066 return 0;
1067
Christoph Hellwigedd1dbc2022-07-06 09:03:37 +02001068 if (WARN_ON_ONCE(!bdev_is_zoned(bio->bi_bdev)))
Johannes Thumshirnae293332021-02-04 19:21:40 +09001069 return 0;
1070
1071 return bio_add_hw_page(q, bio, page, len, offset,
1072 queue_max_zone_append_sectors(q), &same_page);
1073}
1074EXPORT_SYMBOL_GPL(bio_add_zone_append_page);
1075
1076/**
Ming Lei551879a2019-04-23 10:51:04 +08001077 * __bio_add_page - add page(s) to a bio in a new segment
Christoph Hellwig0aa69fd2018-06-01 09:03:05 -07001078 * @bio: destination bio
Ming Lei551879a2019-04-23 10:51:04 +08001079 * @page: start page to add
1080 * @len: length of the data to add, may cross pages
1081 * @off: offset of the data relative to @page, may cross pages
Christoph Hellwig0aa69fd2018-06-01 09:03:05 -07001082 *
1083 * Add the data at @page + @off to @bio as a new bvec. The caller must ensure
1084 * that @bio has space for another bvec.
1085 */
1086void __bio_add_page(struct bio *bio, struct page *page,
1087 unsigned int len, unsigned int off)
1088{
Christoph Hellwig0aa69fd2018-06-01 09:03:05 -07001089 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
Ming Lei79d08f82019-07-01 15:14:46 +08001090 WARN_ON_ONCE(bio_full(bio, len));
Christoph Hellwig0aa69fd2018-06-01 09:03:05 -07001091
Christoph Hellwigd58cdfa2023-02-03 16:06:12 +01001092 bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off);
Christoph Hellwig0aa69fd2018-06-01 09:03:05 -07001093 bio->bi_iter.bi_size += len;
1094 bio->bi_vcnt++;
1095}
1096EXPORT_SYMBOL_GPL(__bio_add_page);
1097
1098/**
Ming Lei551879a2019-04-23 10:51:04 +08001099 * bio_add_page - attempt to add page(s) to bio
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 * @bio: destination bio
Ming Lei551879a2019-04-23 10:51:04 +08001101 * @page: start page to add
1102 * @len: vec entry length, may cross pages
1103 * @offset: vec entry offset relative to @page, may cross pages
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104 *
Ming Lei551879a2019-04-23 10:51:04 +08001105 * Attempt to add page(s) to the bio_vec maplist. This will only fail
Kent Overstreetc66a14d2013-11-23 22:30:22 -08001106 * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107 */
Kent Overstreetc66a14d2013-11-23 22:30:22 -08001108int bio_add_page(struct bio *bio, struct page *page,
1109 unsigned int len, unsigned int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110{
Christoph Hellwigff896732019-06-17 11:14:11 +02001111 bool same_page = false;
1112
Christoph Hellwig939e1a32023-07-24 09:54:28 -07001113 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1114 return 0;
Christoph Hellwig61369902023-07-24 09:54:30 -07001115 if (bio->bi_iter.bi_size > UINT_MAX - len)
1116 return 0;
Christoph Hellwig939e1a32023-07-24 09:54:28 -07001117
Christoph Hellwig0eca8b62023-07-24 09:54:29 -07001118 if (bio->bi_vcnt > 0 &&
Christoph Hellwig858c7082023-07-24 09:54:32 -07001119 bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1],
1120 page, len, offset, &same_page)) {
1121 bio->bi_iter.bi_size += len;
Christoph Hellwig0eca8b62023-07-24 09:54:29 -07001122 return len;
Christoph Hellwig858c7082023-07-24 09:54:32 -07001123 }
Christoph Hellwig0eca8b62023-07-24 09:54:29 -07001124
Christoph Hellwig80232b52023-07-24 09:54:31 -07001125 if (bio->bi_vcnt >= bio->bi_max_vecs)
Christoph Hellwig0eca8b62023-07-24 09:54:29 -07001126 return 0;
1127 __bio_add_page(bio, page, len, offset);
Kent Overstreetc66a14d2013-11-23 22:30:22 -08001128 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129}
H Hartley Sweetena112a712009-09-26 16:19:21 +02001130EXPORT_SYMBOL(bio_add_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131
Johannes Thumshirn7a150f12023-05-31 04:50:41 -07001132void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
1133 size_t off)
1134{
1135 WARN_ON_ONCE(len > UINT_MAX);
1136 WARN_ON_ONCE(off > UINT_MAX);
1137 __bio_add_page(bio, &folio->page, len, off);
1138}
Matthew Wilcox (Oracle)8fde4392024-04-25 17:37:56 +01001139EXPORT_SYMBOL_GPL(bio_add_folio_nofail);
Johannes Thumshirn7a150f12023-05-31 04:50:41 -07001140
Matthew Wilcox (Oracle)85f5a742021-04-08 01:33:45 -04001141/**
1142 * bio_add_folio - Attempt to add part of a folio to a bio.
1143 * @bio: BIO to add to.
1144 * @folio: Folio to add.
1145 * @len: How many bytes from the folio to add.
1146 * @off: First byte in this folio to add.
1147 *
1148 * Filesystems that use folios can call this function instead of calling
1149 * bio_add_page() for each page in the folio. If @off is bigger than
1150 * PAGE_SIZE, this function can create a bio_vec that starts in a page
1151 * after the bv_page. BIOs do not support folios that are 4GiB or larger.
1152 *
1153 * Return: Whether the addition was successful.
1154 */
1155bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len,
1156 size_t off)
1157{
1158 if (len > UINT_MAX || off > UINT_MAX)
Jiapeng Chong455a8442022-01-28 12:34:54 +08001159 return false;
Matthew Wilcox (Oracle)85f5a742021-04-08 01:33:45 -04001160 return bio_add_page(bio, &folio->page, len, off) > 0;
1161}
Matthew Wilcoxcd57b772023-03-24 18:01:03 +00001162EXPORT_SYMBOL(bio_add_folio);
Matthew Wilcox (Oracle)85f5a742021-04-08 01:33:45 -04001163
Pavel Begunkovc8090842021-10-19 22:24:14 +01001164void __bio_release_pages(struct bio *bio, bool mark_dirty)
Christoph Hellwig7321ecb2019-04-11 08:23:30 +02001165{
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001166 struct folio_iter fi;
Christoph Hellwig7321ecb2019-04-11 08:23:30 +02001167
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001168 bio_for_each_folio_all(fi, bio) {
1169 struct page *page;
Tony Battersby38b43532024-02-29 13:08:09 -05001170 size_t nr_pages;
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001171
1172 if (mark_dirty) {
1173 folio_lock(fi.folio);
1174 folio_mark_dirty(fi.folio);
1175 folio_unlock(fi.folio);
1176 }
1177 page = folio_page(fi.folio, fi.offset / PAGE_SIZE);
Tony Battersby38b43532024-02-29 13:08:09 -05001178 nr_pages = (fi.offset + fi.length - 1) / PAGE_SIZE -
1179 fi.offset / PAGE_SIZE + 1;
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001180 do {
1181 bio_release_page(bio, page++);
Tony Battersby38b43532024-02-29 13:08:09 -05001182 } while (--nr_pages != 0);
Christoph Hellwigd241a952019-06-26 15:49:21 +02001183 }
Christoph Hellwig7321ecb2019-04-11 08:23:30 +02001184}
Pavel Begunkovc8090842021-10-19 22:24:14 +01001185EXPORT_SYMBOL_GPL(__bio_release_pages);
Christoph Hellwig7321ecb2019-04-11 08:23:30 +02001186
Pavel Begunkov1bb6b812021-10-27 13:21:07 +01001187void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001188{
Pavel Begunkovfa5fa8e2021-10-23 17:21:33 +01001189 size_t size = iov_iter_count(iter);
1190
Christoph Hellwig7a800a22021-02-02 18:19:29 +01001191 WARN_ON_ONCE(bio->bi_max_vecs);
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001192
Pavel Begunkovfa5fa8e2021-10-23 17:21:33 +01001193 if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
1194 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1195 size_t max_sectors = queue_max_zone_append_sectors(q);
1196
1197 size = min(size, max_sectors << SECTOR_SHIFT);
1198 }
1199
Pavel Begunkovc42bca92021-01-09 16:03:03 +00001200 bio->bi_vcnt = iter->nr_segs;
Pavel Begunkovc42bca92021-01-09 16:03:03 +00001201 bio->bi_io_vec = (struct bio_vec *)iter->bvec;
1202 bio->bi_iter.bi_bvec_done = iter->iov_offset;
Pavel Begunkovfa5fa8e2021-10-23 17:21:33 +01001203 bio->bi_iter.bi_size = size;
Christoph Hellwig977be012021-02-02 18:19:27 +01001204 bio_set_flag(bio, BIO_CLONED);
Johannes Thumshirn7de55b72021-03-25 00:47:26 +09001205}
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001206
Keith Buschc58c0072022-06-10 12:58:21 -07001207static int bio_iov_add_page(struct bio *bio, struct page *page,
1208 unsigned int len, unsigned int offset)
1209{
1210 bool same_page = false;
1211
Christoph Hellwig61369902023-07-24 09:54:30 -07001212 if (WARN_ON_ONCE(bio->bi_iter.bi_size > UINT_MAX - len))
1213 return -EIO;
1214
Christoph Hellwig0eca8b62023-07-24 09:54:29 -07001215 if (bio->bi_vcnt > 0 &&
Christoph Hellwig858c7082023-07-24 09:54:32 -07001216 bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1],
1217 page, len, offset, &same_page)) {
1218 bio->bi_iter.bi_size += len;
Christoph Hellwig0eca8b62023-07-24 09:54:29 -07001219 if (same_page)
1220 bio_release_page(bio, page);
Keith Buschc58c0072022-06-10 12:58:21 -07001221 return 0;
1222 }
Christoph Hellwig0eca8b62023-07-24 09:54:29 -07001223 __bio_add_page(bio, page, len, offset);
Keith Buschc58c0072022-06-10 12:58:21 -07001224 return 0;
1225}
1226
1227static int bio_iov_add_zone_append_page(struct bio *bio, struct page *page,
1228 unsigned int len, unsigned int offset)
1229{
1230 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1231 bool same_page = false;
1232
1233 if (bio_add_hw_page(q, bio, page, len, offset,
1234 queue_max_zone_append_sectors(q), &same_page) != len)
1235 return -EINVAL;
1236 if (same_page)
David Howellsa7e689d2023-05-22 21:57:43 +01001237 bio_release_page(bio, page);
Keith Buschc58c0072022-06-10 12:58:21 -07001238 return 0;
1239}
1240
Christoph Hellwig576ed912018-09-20 08:28:21 +02001241#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
1242
Kent Overstreet2cefe4d2016-10-31 11:59:24 -06001243/**
Martin Wilck17d51b12018-07-25 23:15:09 +02001244 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
Kent Overstreet2cefe4d2016-10-31 11:59:24 -06001245 * @bio: bio to add pages to
1246 * @iter: iov iterator describing the region to be mapped
1247 *
David Howellsa7e689d2023-05-22 21:57:43 +01001248 * Extracts pages from *iter and appends them to @bio's bvec array. The pages
1249 * will have to be cleaned up in the way indicated by the BIO_PAGE_PINNED flag.
1250 * For a multi-segment *iter, this function only adds pages from the next
1251 * non-empty segment of the iov iterator.
Kent Overstreet2cefe4d2016-10-31 11:59:24 -06001252 */
Martin Wilck17d51b12018-07-25 23:15:09 +02001253static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
Kent Overstreet2cefe4d2016-10-31 11:59:24 -06001254{
David Howellsf62e52d2023-01-19 12:47:23 +00001255 iov_iter_extraction_t extraction_flags = 0;
Christoph Hellwig576ed912018-09-20 08:28:21 +02001256 unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
1257 unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
Kent Overstreet2cefe4d2016-10-31 11:59:24 -06001258 struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
1259 struct page **pages = (struct page **)bv;
Christoph Hellwig576ed912018-09-20 08:28:21 +02001260 ssize_t size, left;
Keith Busche97424f2022-07-12 08:32:56 -07001261 unsigned len, i = 0;
Kent Overstreet168145f2023-08-13 14:26:35 -04001262 size_t offset;
Keith Busch325347d2022-07-12 08:32:54 -07001263 int ret = 0;
Christoph Hellwig576ed912018-09-20 08:28:21 +02001264
1265 /*
1266 * Move page array up in the allocated memory for the bio vecs as far as
1267 * possible so that we can start filling biovecs from the beginning
1268 * without overwriting the temporary page array.
Keith Busch0512a752020-05-12 17:55:47 +09001269 */
1270 BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
1271 pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
1272
Logan Gunthorpe5e3e3f22022-10-21 11:41:13 -06001273 if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue))
David Howellsf62e52d2023-01-19 12:47:23 +00001274 extraction_flags |= ITER_ALLOW_P2PDMA;
Logan Gunthorpe5e3e3f22022-10-21 11:41:13 -06001275
Keith Buschb1a000d2022-06-10 12:58:29 -07001276 /*
1277 * Each segment in the iov is required to be a block size multiple.
1278 * However, we may not be able to get the entire segment if it spans
1279 * more pages than bi_max_vecs allows, so we have to ALIGN_DOWN the
1280 * result to ensure the bio's total size is correct. The remainder of
1281 * the iov data will be picked up in the next bio iteration.
1282 */
David Howellsa7e689d2023-05-22 21:57:43 +01001283 size = iov_iter_extract_pages(iter, &pages,
1284 UINT_MAX - bio->bi_iter.bi_size,
1285 nr_pages, extraction_flags, &offset);
Al Viro480cb842022-06-09 10:37:57 -04001286 if (unlikely(size <= 0))
1287 return size ? size : -EFAULT;
Keith Busche97424f2022-07-12 08:32:56 -07001288
Al Viro480cb842022-06-09 10:37:57 -04001289 nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE);
1290
Kent Overstreet168145f2023-08-13 14:26:35 -04001291 if (bio->bi_bdev) {
1292 size_t trim = size & (bdev_logical_block_size(bio->bi_bdev) - 1);
1293 iov_iter_revert(iter, trim);
1294 size -= trim;
1295 }
Al Viro480cb842022-06-09 10:37:57 -04001296
Al Viro480cb842022-06-09 10:37:57 -04001297 if (unlikely(!size)) {
1298 ret = -EFAULT;
Keith Busche97424f2022-07-12 08:32:56 -07001299 goto out;
1300 }
Keith Busch0512a752020-05-12 17:55:47 +09001301
1302 for (left = size, i = 0; left > 0; left -= len, i++) {
1303 struct page *page = pages[i];
Keith Busch0512a752020-05-12 17:55:47 +09001304
1305 len = min_t(size_t, PAGE_SIZE - offset, left);
Keith Busch34cdb8c2022-07-12 08:32:55 -07001306 if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
Keith Buschc58c0072022-06-10 12:58:21 -07001307 ret = bio_iov_add_zone_append_page(bio, page, len,
1308 offset);
Keith Busche97424f2022-07-12 08:32:56 -07001309 if (ret)
Keith Busch34cdb8c2022-07-12 08:32:55 -07001310 break;
Keith Busch34cdb8c2022-07-12 08:32:55 -07001311 } else
1312 bio_iov_add_page(bio, page, len, offset);
Keith Buschc58c0072022-06-10 12:58:21 -07001313
Keith Busch0512a752020-05-12 17:55:47 +09001314 offset = 0;
1315 }
1316
Al Viro480cb842022-06-09 10:37:57 -04001317 iov_iter_revert(iter, left);
Keith Busche97424f2022-07-12 08:32:56 -07001318out:
1319 while (i < nr_pages)
David Howellsa7e689d2023-05-22 21:57:43 +01001320 bio_release_page(bio, pages[i++]);
Keith Busche97424f2022-07-12 08:32:56 -07001321
Keith Busch325347d2022-07-12 08:32:54 -07001322 return ret;
Keith Busch0512a752020-05-12 17:55:47 +09001323}
1324
Martin Wilck17d51b12018-07-25 23:15:09 +02001325/**
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001326 * bio_iov_iter_get_pages - add user or kernel pages to a bio
Martin Wilck17d51b12018-07-25 23:15:09 +02001327 * @bio: bio to add pages to
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001328 * @iter: iov iterator describing the region to be added
Martin Wilck17d51b12018-07-25 23:15:09 +02001329 *
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001330 * This takes either an iterator pointing to user memory, or one pointing to
1331 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
1332 * map them into the kernel. On IO completion, the caller should put those
Pavel Begunkovc42bca92021-01-09 16:03:03 +00001333 * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided
1334 * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs
1335 * to ensure the bvecs and pages stay referenced until the submitted I/O is
1336 * completed by a call to ->ki_complete() or returns with an error other than
1337 * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF
1338 * on IO completion. If it isn't, then pages should be released.
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001339 *
Martin Wilck17d51b12018-07-25 23:15:09 +02001340 * The function tries, but does not guarantee, to pin as many pages as
Mauro Carvalho Chehab5cd3ddc2020-09-09 11:44:33 +02001341 * fit into the bio, or are requested in @iter, whatever is smaller. If
Jens Axboe6d0c48a2018-11-30 08:47:03 -07001342 * MM encounters an error pinning the requested pages, it stops. Error
1343 * is returned only if 0 pages could be pinned.
Martin Wilck17d51b12018-07-25 23:15:09 +02001344 */
1345int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
1346{
Pavel Begunkovc42bca92021-01-09 16:03:03 +00001347 int ret = 0;
Christoph Hellwig14eacf12019-04-11 08:23:29 +02001348
Christoph Hellwig939e1a32023-07-24 09:54:28 -07001349 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1350 return -EIO;
1351
Pavel Begunkovc42bca92021-01-09 16:03:03 +00001352 if (iov_iter_is_bvec(iter)) {
Pavel Begunkovfa5fa8e2021-10-23 17:21:33 +01001353 bio_iov_bvec_set(bio, iter);
1354 iov_iter_advance(iter, bio->bi_iter.bi_size);
1355 return 0;
Pavel Begunkovc42bca92021-01-09 16:03:03 +00001356 }
Martin Wilck17d51b12018-07-25 23:15:09 +02001357
David Howellsa7e689d2023-05-22 21:57:43 +01001358 if (iov_iter_extract_will_pin(iter))
1359 bio_set_flag(bio, BIO_PAGE_PINNED);
Martin Wilck17d51b12018-07-25 23:15:09 +02001360 do {
Keith Buschc58c0072022-06-10 12:58:21 -07001361 ret = __bio_iov_iter_get_pages(bio, iter);
Ming Lei79d08f82019-07-01 15:14:46 +08001362 } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
Martin Wilck17d51b12018-07-25 23:15:09 +02001363
Christoph Hellwig14eacf12019-04-11 08:23:29 +02001364 return bio->bi_vcnt ? 0 : ret;
Martin Wilck17d51b12018-07-25 23:15:09 +02001365}
Johannes Thumshirn29b2a3a2020-05-12 17:55:53 +09001366EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
Kent Overstreet2cefe4d2016-10-31 11:59:24 -06001367
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001368static void submit_bio_wait_endio(struct bio *bio)
Kent Overstreet9e882242012-09-10 14:41:12 -07001369{
Christoph Hellwig65e53aa2017-10-25 17:55:57 +09001370 complete(bio->bi_private);
Kent Overstreet9e882242012-09-10 14:41:12 -07001371}
1372
1373/**
1374 * submit_bio_wait - submit a bio, and wait until it completes
Kent Overstreet9e882242012-09-10 14:41:12 -07001375 * @bio: The &struct bio which describes the I/O
1376 *
1377 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
1378 * bio_endio() on failure.
Jan Kara3d289d62017-08-02 10:25:21 +02001379 *
1380 * WARNING: Unlike to how submit_bio() is usually used, this function does not
1381 * result in bio reference to be consumed. The caller must drop the reference
1382 * on his own.
Kent Overstreet9e882242012-09-10 14:41:12 -07001383 */
Mike Christie4e49ea42016-06-05 14:31:41 -05001384int submit_bio_wait(struct bio *bio)
Kent Overstreet9e882242012-09-10 14:41:12 -07001385{
Christoph Hellwig309dca302021-01-24 11:02:34 +01001386 DECLARE_COMPLETION_ONSTACK_MAP(done,
1387 bio->bi_bdev->bd_disk->lockdep_map);
Kent Overstreet9e882242012-09-10 14:41:12 -07001388
Christoph Hellwig65e53aa2017-10-25 17:55:57 +09001389 bio->bi_private = &done;
Kent Overstreet9e882242012-09-10 14:41:12 -07001390 bio->bi_end_io = submit_bio_wait_endio;
Jens Axboe1eff9d32016-08-05 15:35:16 -06001391 bio->bi_opf |= REQ_SYNC;
Mike Christie4e49ea42016-06-05 14:31:41 -05001392 submit_bio(bio);
Keith Busch0eb4db42024-02-23 07:59:09 -08001393 blk_wait_io(&done);
Kent Overstreet9e882242012-09-10 14:41:12 -07001394
Christoph Hellwig65e53aa2017-10-25 17:55:57 +09001395 return blk_status_to_errno(bio->bi_status);
Kent Overstreet9e882242012-09-10 14:41:12 -07001396}
1397EXPORT_SYMBOL(submit_bio_wait);
1398
Keith Busch0f8e9ec2024-05-06 06:20:26 +02001399static void bio_wait_end_io(struct bio *bio)
1400{
1401 complete(bio->bi_private);
1402 bio_put(bio);
1403}
1404
1405/*
1406 * bio_await_chain - ends @bio and waits for every chained bio to complete
1407 */
1408void bio_await_chain(struct bio *bio)
1409{
1410 DECLARE_COMPLETION_ONSTACK_MAP(done,
1411 bio->bi_bdev->bd_disk->lockdep_map);
1412
1413 bio->bi_private = &done;
1414 bio->bi_end_io = bio_wait_end_io;
1415 bio_endio(bio);
1416 blk_wait_io(&done);
1417}
1418
Jens Axboed4aa57a2021-10-13 09:01:43 -06001419void __bio_advance(struct bio *bio, unsigned bytes)
Kent Overstreet054bdf62012-09-28 13:17:55 -07001420{
1421 if (bio_integrity(bio))
1422 bio_integrity_advance(bio, bytes);
1423
Satya Tangiralaa892c8d2020-05-14 00:37:18 +00001424 bio_crypt_advance(bio, bytes);
Kent Overstreet4550dd62013-08-07 14:26:21 -07001425 bio_advance_iter(bio, &bio->bi_iter, bytes);
Kent Overstreet054bdf62012-09-28 13:17:55 -07001426}
Jens Axboed4aa57a2021-10-13 09:01:43 -06001427EXPORT_SYMBOL(__bio_advance);
Kent Overstreet054bdf62012-09-28 13:17:55 -07001428
Jens Axboeee4b4e22023-01-04 14:43:27 -07001429void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
1430 struct bio *src, struct bvec_iter *src_iter)
1431{
1432 while (src_iter->bi_size && dst_iter->bi_size) {
1433 struct bio_vec src_bv = bio_iter_iovec(src, *src_iter);
1434 struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter);
1435 unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len);
1436 void *src_buf = bvec_kmap_local(&src_bv);
1437 void *dst_buf = bvec_kmap_local(&dst_bv);
1438
1439 memcpy(dst_buf, src_buf, bytes);
1440
1441 kunmap_local(dst_buf);
1442 kunmap_local(src_buf);
1443
1444 bio_advance_iter_single(src, src_iter, bytes);
1445 bio_advance_iter_single(dst, dst_iter, bytes);
1446 }
1447}
1448EXPORT_SYMBOL(bio_copy_data_iter);
1449
Kent Overstreet45db54d2018-05-08 21:33:54 -04001450/**
1451 * bio_copy_data - copy contents of data buffers from one bio to another
1452 * @src: source bio
1453 * @dst: destination bio
1454 *
1455 * Stops when it reaches the end of either @src or @dst - that is, copies
1456 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
1457 */
1458void bio_copy_data(struct bio *dst, struct bio *src)
1459{
1460 struct bvec_iter src_iter = src->bi_iter;
1461 struct bvec_iter dst_iter = dst->bi_iter;
1462
Jens Axboeee4b4e22023-01-04 14:43:27 -07001463 bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
Kent Overstreet45db54d2018-05-08 21:33:54 -04001464}
1465EXPORT_SYMBOL(bio_copy_data);
1466
Guoqing Jiang491221f2016-09-22 03:10:01 -04001467void bio_free_pages(struct bio *bio)
Christoph Hellwig1dfa0f62015-01-18 16:16:30 +01001468{
1469 struct bio_vec *bvec;
Ming Lei6dc4f102019-02-15 19:13:19 +08001470 struct bvec_iter_all iter_all;
Christoph Hellwig1dfa0f62015-01-18 16:16:30 +01001471
Christoph Hellwig2b070cf2019-04-25 09:03:00 +02001472 bio_for_each_segment_all(bvec, bio, iter_all)
Christoph Hellwig1dfa0f62015-01-18 16:16:30 +01001473 __free_page(bvec->bv_page);
1474}
Guoqing Jiang491221f2016-09-22 03:10:01 -04001475EXPORT_SYMBOL(bio_free_pages);
Christoph Hellwig1dfa0f62015-01-18 16:16:30 +01001476
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477/*
1478 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1479 * for performing direct-IO in BIOs.
1480 *
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001481 * The problem is that we cannot run folio_mark_dirty() from interrupt context
Linus Torvalds1da177e2005-04-16 15:20:36 -07001482 * because the required locks are not interrupt-safe. So what we can do is to
1483 * mark the pages dirty _before_ performing IO. And in interrupt context,
1484 * check that the pages are still dirty. If so, fine. If not, redirty them
1485 * in process context.
1486 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 * Note that this code is very hard to test under normal circumstances because
1488 * direct-io pins the pages with get_user_pages(). This makes
1489 * is_page_cache_freeable return false, and the VM will not clean the pages.
Artem Bityutskiy0d5c3eb2012-07-25 18:12:08 +03001490 * But other code (eg, flusher threads) could clean the pages if they are mapped
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 * pagecache.
1492 *
1493 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1494 * deferred bio dirtying paths.
1495 */
1496
1497/*
1498 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1499 */
1500void bio_set_pages_dirty(struct bio *bio)
1501{
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001502 struct folio_iter fi;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001504 bio_for_each_folio_all(fi, bio) {
1505 folio_lock(fi.folio);
1506 folio_mark_dirty(fi.folio);
1507 folio_unlock(fi.folio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508 }
1509}
Kent Overstreet7ba37922023-08-13 14:26:34 -04001510EXPORT_SYMBOL_GPL(bio_set_pages_dirty);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512/*
1513 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1514 * If they are, then fine. If, however, some pages are clean then they must
1515 * have been written out during the direct-IO read. So we take another ref on
Christoph Hellwig24d54932018-07-24 14:04:12 +02001516 * the BIO and re-dirty the pages in process context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517 *
1518 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
David Howellsfd363242023-05-22 21:57:42 +01001519 * here on. It will unpin each page and will run one bio_put() against the
1520 * BIO.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 */
1522
David Howells65f27f32006-11-22 14:55:48 +00001523static void bio_dirty_fn(struct work_struct *work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524
David Howells65f27f32006-11-22 14:55:48 +00001525static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526static DEFINE_SPINLOCK(bio_dirty_lock);
1527static struct bio *bio_dirty_list;
1528
1529/*
1530 * This runs in process context
1531 */
David Howells65f27f32006-11-22 14:55:48 +00001532static void bio_dirty_fn(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001533{
Christoph Hellwig24d54932018-07-24 14:04:12 +02001534 struct bio *bio, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535
Christoph Hellwig24d54932018-07-24 14:04:12 +02001536 spin_lock_irq(&bio_dirty_lock);
1537 next = bio_dirty_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 bio_dirty_list = NULL;
Christoph Hellwig24d54932018-07-24 14:04:12 +02001539 spin_unlock_irq(&bio_dirty_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540
Christoph Hellwig24d54932018-07-24 14:04:12 +02001541 while ((bio = next) != NULL) {
1542 next = bio->bi_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543
Christoph Hellwigd241a952019-06-26 15:49:21 +02001544 bio_release_pages(bio, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 }
1547}
1548
1549void bio_check_pages_dirty(struct bio *bio)
1550{
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001551 struct folio_iter fi;
Christoph Hellwig24d54932018-07-24 14:04:12 +02001552 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553
Matthew Wilcox (Oracle)1b151e22023-08-14 15:41:00 +01001554 bio_for_each_folio_all(fi, bio) {
1555 if (!folio_test_dirty(fi.folio))
Christoph Hellwig24d54932018-07-24 14:04:12 +02001556 goto defer;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 }
1558
Christoph Hellwigd241a952019-06-26 15:49:21 +02001559 bio_release_pages(bio, false);
Christoph Hellwig24d54932018-07-24 14:04:12 +02001560 bio_put(bio);
1561 return;
1562defer:
1563 spin_lock_irqsave(&bio_dirty_lock, flags);
1564 bio->bi_private = bio_dirty_list;
1565 bio_dirty_list = bio;
1566 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1567 schedule_work(&bio_dirty_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568}
Kent Overstreet7ba37922023-08-13 14:26:34 -04001569EXPORT_SYMBOL_GPL(bio_check_pages_dirty);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570
Jens Axboec4cf5262015-04-17 16:15:18 -06001571static inline bool bio_remaining_done(struct bio *bio)
1572{
1573 /*
1574 * If we're not chaining, then ->__bi_remaining is always 1 and
1575 * we always end io on the first invocation.
1576 */
1577 if (!bio_flagged(bio, BIO_CHAIN))
1578 return true;
1579
1580 BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1581
Mike Snitzer326e1db2015-05-22 09:14:03 -04001582 if (atomic_dec_and_test(&bio->__bi_remaining)) {
Jens Axboeb7c44ed2015-07-24 12:37:59 -06001583 bio_clear_flag(bio, BIO_CHAIN);
Jens Axboec4cf5262015-04-17 16:15:18 -06001584 return true;
Mike Snitzer326e1db2015-05-22 09:14:03 -04001585 }
Jens Axboec4cf5262015-04-17 16:15:18 -06001586
1587 return false;
1588}
1589
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590/**
1591 * bio_endio - end I/O on a bio
1592 * @bio: bio
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593 *
1594 * Description:
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001595 * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1596 * way to end I/O on a bio. No one should call bi_end_io() directly on a
1597 * bio unless they own it and thus know that it has an end_io function.
NeilBrownfbbaf702017-04-07 09:40:52 -06001598 *
1599 * bio_endio() can be called several times on a bio that has been chained
1600 * using bio_chain(). The ->bi_end_io() function will only be called the
Edward Hsieh60b6a7e2021-06-24 20:30:30 +08001601 * last time.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602 **/
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001603void bio_endio(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604{
Christoph Hellwigba8c6962016-03-11 17:34:52 +01001605again:
Christoph Hellwig2b885512016-03-11 17:34:53 +01001606 if (!bio_remaining_done(bio))
Christoph Hellwigba8c6962016-03-11 17:34:52 +01001607 return;
Christoph Hellwig7c20f112017-07-03 16:58:43 -06001608 if (!bio_integrity_endio(bio))
1609 return;
Kent Overstreet196d38bc2013-11-23 18:34:15 -08001610
Damien Le Moaldd291d72024-04-08 10:41:07 +09001611 blk_zone_bio_endio(bio);
1612
Tejun Heoaa1b46d2022-03-13 21:15:02 -10001613 rq_qos_done_bio(bio);
Josef Bacik67b42d02018-07-03 11:15:00 -04001614
Edward Hsieh60b6a7e2021-06-24 20:30:30 +08001615 if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
Pavel Begunkov3caee4632021-10-14 15:03:28 +01001616 trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio);
Edward Hsieh60b6a7e2021-06-24 20:30:30 +08001617 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1618 }
1619
Christoph Hellwigba8c6962016-03-11 17:34:52 +01001620 /*
1621 * Need to have a real endio function for chained bios, otherwise
1622 * various corner cases will break (like stacking block devices that
1623 * save/restore bi_end_io) - however, we want to avoid unbounded
1624 * recursion and blowing the stack. Tail call optimization would
1625 * handle this, but compiling with frame pointers also disables
1626 * gcc's sibling call optimization.
1627 */
1628 if (bio->bi_end_io == bio_chain_endio) {
1629 bio = __bio_chain_endio(bio);
1630 goto again;
Kent Overstreet196d38bc2013-11-23 18:34:15 -08001631 }
Christoph Hellwigba8c6962016-03-11 17:34:52 +01001632
Shaohua Lib222dd22017-07-10 11:40:17 -07001633 /* release cgroup info */
1634 bio_uninit(bio);
Christoph Hellwigba8c6962016-03-11 17:34:52 +01001635 if (bio->bi_end_io)
1636 bio->bi_end_io(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637}
H Hartley Sweetena112a712009-09-26 16:19:21 +02001638EXPORT_SYMBOL(bio_endio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639
Kent Overstreet196d38bc2013-11-23 18:34:15 -08001640/**
Kent Overstreet20d01892013-11-23 18:21:01 -08001641 * bio_split - split a bio
1642 * @bio: bio to split
1643 * @sectors: number of sectors to split from the front of @bio
1644 * @gfp: gfp mask
1645 * @bs: bio set to allocate from
1646 *
1647 * Allocates and returns a new bio which represents @sectors from the start of
1648 * @bio, and updates @bio to represent the remaining sectors.
1649 *
Martin K. Petersenf3f5da62015-07-22 07:57:12 -04001650 * Unless this is a discard request the newly allocated bio will point
Bart Van Asschedad77582019-08-01 15:50:41 -07001651 * to @bio's bi_io_vec. It is the caller's responsibility to ensure that
1652 * neither @bio nor @bs are freed before the split bio.
Kent Overstreet20d01892013-11-23 18:21:01 -08001653 */
1654struct bio *bio_split(struct bio *bio, int sectors,
1655 gfp_t gfp, struct bio_set *bs)
1656{
Mikulas Patockaf341a4d2017-11-22 13:18:05 -05001657 struct bio *split;
Kent Overstreet20d01892013-11-23 18:21:01 -08001658
1659 BUG_ON(sectors <= 0);
1660 BUG_ON(sectors >= bio_sectors(bio));
1661
Keith Busch0512a752020-05-12 17:55:47 +09001662 /* Zone append commands cannot be split */
1663 if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
1664 return NULL;
1665
Christoph Hellwigabfc4262022-02-02 17:01:09 +01001666 split = bio_alloc_clone(bio->bi_bdev, bio, gfp, bs);
Kent Overstreet20d01892013-11-23 18:21:01 -08001667 if (!split)
1668 return NULL;
1669
1670 split->bi_iter.bi_size = sectors << 9;
1671
1672 if (bio_integrity(split))
Dmitry Monakhovfbd08e72017-06-29 11:31:10 -07001673 bio_integrity_trim(split);
Kent Overstreet20d01892013-11-23 18:21:01 -08001674
1675 bio_advance(bio, split->bi_iter.bi_size);
1676
NeilBrownfbbaf702017-04-07 09:40:52 -06001677 if (bio_flagged(bio, BIO_TRACE_COMPLETION))
Goldwyn Rodrigues20d59022018-01-23 09:10:19 -07001678 bio_set_flag(split, BIO_TRACE_COMPLETION);
NeilBrownfbbaf702017-04-07 09:40:52 -06001679
Kent Overstreet20d01892013-11-23 18:21:01 -08001680 return split;
1681}
1682EXPORT_SYMBOL(bio_split);
1683
Martin K. Petersenad3316b2008-10-01 22:42:53 -04001684/**
Kent Overstreet6678d832013-08-07 11:14:32 -07001685 * bio_trim - trim a bio
1686 * @bio: bio to trim
1687 * @offset: number of sectors to trim from the front of @bio
1688 * @size: size we want to trim @bio to, in sectors
Chaitanya Kulkarnie83502c2021-07-21 21:43:32 +09001689 *
1690 * This function is typically used for bios that are cloned and submitted
1691 * to the underlying device in parts.
Kent Overstreet6678d832013-08-07 11:14:32 -07001692 */
Chaitanya Kulkarnie83502c2021-07-21 21:43:32 +09001693void bio_trim(struct bio *bio, sector_t offset, sector_t size)
Kent Overstreet6678d832013-08-07 11:14:32 -07001694{
Chaitanya Kulkarnie83502c2021-07-21 21:43:32 +09001695 if (WARN_ON_ONCE(offset > BIO_MAX_SECTORS || size > BIO_MAX_SECTORS ||
Ming Lei8535c012022-04-14 16:44:43 +08001696 offset + size > bio_sectors(bio)))
Chaitanya Kulkarnie83502c2021-07-21 21:43:32 +09001697 return;
Kent Overstreet6678d832013-08-07 11:14:32 -07001698
1699 size <<= 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001700 if (offset == 0 && size == bio->bi_iter.bi_size)
Kent Overstreet6678d832013-08-07 11:14:32 -07001701 return;
1702
Kent Overstreet6678d832013-08-07 11:14:32 -07001703 bio_advance(bio, offset << 9);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001704 bio->bi_iter.bi_size = size;
Dmitry Monakhov376a78a2017-06-29 11:31:08 -07001705
1706 if (bio_integrity(bio))
Dmitry Monakhovfbd08e72017-06-29 11:31:10 -07001707 bio_integrity_trim(bio);
Kent Overstreet6678d832013-08-07 11:14:32 -07001708}
1709EXPORT_SYMBOL_GPL(bio_trim);
1710
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711/*
1712 * create memory pools for biovec's in a bio_set.
1713 * use the global biovec slabs created for general use.
1714 */
Kent Overstreet8aa6ba22018-05-08 21:33:50 -04001715int biovec_init_pool(mempool_t *pool, int pool_entries)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716{
Christoph Hellwig7a800a22021-02-02 18:19:29 +01001717 struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718
Kent Overstreet8aa6ba22018-05-08 21:33:50 -04001719 return mempool_init_slab_pool(pool, pool_entries, bp->slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001720}
1721
Kent Overstreet917a38c2018-05-08 21:33:51 -04001722/*
1723 * bioset_exit - exit a bioset initialized with bioset_init()
1724 *
1725 * May be called on a zeroed but uninitialized bioset (i.e. allocated with
1726 * kzalloc()).
1727 */
1728void bioset_exit(struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729{
Jens Axboebe4d2342021-03-08 11:37:47 -07001730 bio_alloc_cache_destroy(bs);
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -07001731 if (bs->rescue_workqueue)
1732 destroy_workqueue(bs->rescue_workqueue);
Kent Overstreet917a38c2018-05-08 21:33:51 -04001733 bs->rescue_workqueue = NULL;
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -07001734
Kent Overstreet8aa6ba22018-05-08 21:33:50 -04001735 mempool_exit(&bs->bio_pool);
1736 mempool_exit(&bs->bvec_pool);
Kent Overstreet9f060e22012-10-12 15:29:33 -07001737
Martin K. Petersen7878cba2009-06-26 15:37:49 +02001738 bioset_integrity_free(bs);
Kent Overstreet917a38c2018-05-08 21:33:51 -04001739 if (bs->bio_slab)
1740 bio_put_slab(bs);
1741 bs->bio_slab = NULL;
1742}
1743EXPORT_SYMBOL(bioset_exit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744
NeilBrown011067b2017-06-18 14:38:57 +10001745/**
Kent Overstreet917a38c2018-05-08 21:33:51 -04001746 * bioset_init - Initialize a bio_set
Kent Overstreetdad08522018-05-20 18:25:58 -04001747 * @bs: pool to initialize
Kent Overstreet917a38c2018-05-08 21:33:51 -04001748 * @pool_size: Number of bio and bio_vecs to cache in the mempool
1749 * @front_pad: Number of bytes to allocate in front of the returned bio
1750 * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS
1751 * and %BIOSET_NEED_RESCUER
1752 *
Kent Overstreetdad08522018-05-20 18:25:58 -04001753 * Description:
1754 * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1755 * to ask for a number of bytes to be allocated in front of the bio.
1756 * Front pad allocation is useful for embedding the bio inside
1757 * another structure, to avoid allocating extra data to go with the bio.
1758 * Note that the bio must be embedded at the END of that structure always,
1759 * or things will break badly.
1760 * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
Christoph Hellwigabfc4262022-02-02 17:01:09 +01001761 * for allocating iovecs. This pool is not needed e.g. for bio_init_clone().
1762 * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used
1763 * to dispatch queued requests when the mempool runs out of space.
Kent Overstreetdad08522018-05-20 18:25:58 -04001764 *
Kent Overstreet917a38c2018-05-08 21:33:51 -04001765 */
1766int bioset_init(struct bio_set *bs,
1767 unsigned int pool_size,
1768 unsigned int front_pad,
1769 int flags)
1770{
Kent Overstreet917a38c2018-05-08 21:33:51 -04001771 bs->front_pad = front_pad;
Ming Lei9f180e32021-01-11 11:05:54 +08001772 if (flags & BIOSET_NEED_BVECS)
1773 bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1774 else
1775 bs->back_pad = 0;
Kent Overstreet917a38c2018-05-08 21:33:51 -04001776
1777 spin_lock_init(&bs->rescue_lock);
1778 bio_list_init(&bs->rescue_list);
1779 INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1780
Ming Lei49d1ec82021-01-11 11:05:52 +08001781 bs->bio_slab = bio_find_or_create_slab(bs);
Kent Overstreet917a38c2018-05-08 21:33:51 -04001782 if (!bs->bio_slab)
1783 return -ENOMEM;
1784
1785 if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
1786 goto bad;
1787
1788 if ((flags & BIOSET_NEED_BVECS) &&
1789 biovec_init_pool(&bs->bvec_pool, pool_size))
1790 goto bad;
1791
Jens Axboebe4d2342021-03-08 11:37:47 -07001792 if (flags & BIOSET_NEED_RESCUER) {
1793 bs->rescue_workqueue = alloc_workqueue("bioset",
1794 WQ_MEM_RECLAIM, 0);
1795 if (!bs->rescue_workqueue)
1796 goto bad;
1797 }
1798 if (flags & BIOSET_PERCPU_CACHE) {
1799 bs->cache = alloc_percpu(struct bio_alloc_cache);
1800 if (!bs->cache)
1801 goto bad;
1802 cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
1803 }
Kent Overstreet917a38c2018-05-08 21:33:51 -04001804
1805 return 0;
1806bad:
1807 bioset_exit(bs);
1808 return -ENOMEM;
1809}
1810EXPORT_SYMBOL(bioset_init);
1811
Christoph Hellwigde76fd82021-02-02 18:19:23 +01001812static int __init init_bio(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813{
1814 int i;
1815
Jens Axboea3df2e42023-01-09 10:45:56 -07001816 BUILD_BUG_ON(BIO_FLAG_LAST > 8 * sizeof_field(struct bio, bi_flags));
1817
Martin K. Petersen7878cba2009-06-26 15:37:49 +02001818 bio_integrity_init();
Christoph Hellwigde76fd82021-02-02 18:19:23 +01001819
1820 for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
Jens Axboea7fcd372008-12-05 16:10:29 +01001821 struct biovec_slab *bvs = bvec_slabs + i;
1822
Christoph Hellwigde76fd82021-02-02 18:19:23 +01001823 bvs->slab = kmem_cache_create(bvs->name,
1824 bvs->nr_vecs * sizeof(struct bio_vec), 0,
1825 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001826 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827
Jens Axboebe4d2342021-03-08 11:37:47 -07001828 cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL,
1829 bio_cpu_dead);
1830
Jens Axboe12c5b702022-09-02 10:40:11 -06001831 if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0,
1832 BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833 panic("bio: can't allocate bios\n");
1834
Kent Overstreetf4f81542018-05-08 21:33:52 -04001835 if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
Martin K. Petersena91a2782011-03-17 11:11:05 +01001836 panic("bio: can't create integrity pool\n");
1837
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838 return 0;
1839}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840subsys_initcall(init_bio);