blob: d9f584984bc44b8280436a6ddfa044d909c626ad [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Jens Axboe8324aa92008-01-29 14:51:59 +01002#ifndef BLK_INTERNAL_H
3#define BLK_INTERNAL_H
4
Satya Tangiralaa892c8d2020-05-14 00:37:18 +00005#include <linux/blk-crypto.h>
Christoph Hellwig9bb33f22021-03-31 09:30:00 +02006#include <linux/memblock.h> /* for max_pfn/max_low_pfn */
Keith Busch0eb4db42024-02-23 07:59:09 -08007#include <linux/sched/sysctl.h>
Jens Axboe08420cf2024-01-15 14:45:07 -07008#include <linux/timekeeping.h>
Christoph Hellwigc39ae602018-09-25 13:30:08 -07009#include <xen/xen.h>
Satya Tangiralaa892c8d2020-05-14 00:37:18 +000010#include "blk-crypto-internal.h"
Tejun Heoa73f7302011-12-14 00:33:37 +010011
Christoph Hellwig2e9bc342021-09-20 14:33:23 +020012struct elevator_type;
13
Jens Axboe0d2602c2014-05-13 15:10:52 -060014/* Max future timer expiry for timeouts */
15#define BLK_MAX_TIMEOUT (5 * HZ)
16
Omar Sandoval18fbda92017-01-31 14:53:20 -080017extern struct dentry *blk_debugfs_root;
Omar Sandoval18fbda92017-01-31 14:53:20 -080018
Ming Lei7c94e1c2014-09-25 23:23:43 +080019struct blk_flush_queue {
Chengming Zhoub175c862023-07-17 12:00:57 +080020 spinlock_t mq_flush_lock;
Ming Lei7c94e1c2014-09-25 23:23:43 +080021 unsigned int flush_pending_idx:1;
22 unsigned int flush_running_idx:1;
Yufen Yu8d699662019-09-27 16:19:55 +080023 blk_status_t rq_status;
Ming Lei7c94e1c2014-09-25 23:23:43 +080024 unsigned long flush_pending_since;
25 struct list_head flush_queue[2];
Chengming Zhoub175c862023-07-17 12:00:57 +080026 unsigned long flush_data_in_flight;
Ming Lei7c94e1c2014-09-25 23:23:43 +080027 struct request *flush_rq;
Ming Lei7c94e1c2014-09-25 23:23:43 +080028};
29
Ming Leia9ed27a2021-08-18 09:09:25 +080030bool is_flush_rq(struct request *req);
Yufen Yu8d699662019-09-27 16:19:55 +080031
Guoqing Jiang754a1572020-03-09 22:41:37 +010032struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
33 gfp_t flags);
Ming Leif70ced02014-09-25 23:23:47 +080034void blk_free_flush_queue(struct blk_flush_queue *q);
Ming Leif3552652014-09-25 23:23:40 +080035
Dan Williams3ef28e82015-10-21 13:20:12 -040036void blk_freeze_queue(struct request_queue *q);
Christoph Hellwigaec89dc2021-09-29 09:12:41 +020037void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic);
Christoph Hellwig8e141f92021-09-29 09:12:40 +020038void blk_queue_start_drain(struct request_queue *q);
Jens Axboec98cb5b2021-11-04 12:45:51 -060039int __bio_queue_enter(struct request_queue *q, struct bio *bio);
Ming Lei3f98c752022-02-16 12:45:10 +080040void submit_bio_noacct_nocheck(struct bio *bio);
Jens Axboec98cb5b2021-11-04 12:45:51 -060041
42static inline bool blk_try_enter_queue(struct request_queue *q, bool pm)
43{
44 rcu_read_lock();
45 if (!percpu_ref_tryget_live_rcu(&q->q_usage_counter))
46 goto fail;
47
48 /*
49 * The code that increments the pm_only counter must ensure that the
50 * counter is globally visible before the queue is unfrozen.
51 */
52 if (blk_queue_pm_only(q) &&
53 (!pm || queue_rpm_status(q) == RPM_SUSPENDED))
54 goto fail_put;
55
56 rcu_read_unlock();
57 return true;
58
59fail_put:
60 blk_queue_exit(q);
61fail:
62 rcu_read_unlock();
63 return false;
64}
65
66static inline int bio_queue_enter(struct bio *bio)
67{
68 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
69
70 if (blk_try_enter_queue(q, false))
71 return 0;
72 return __bio_queue_enter(q, bio);
73}
Dan Williams3ef28e82015-10-21 13:20:12 -040074
Keith Busch0eb4db42024-02-23 07:59:09 -080075static inline void blk_wait_io(struct completion *done)
76{
77 /* Prevent hang_check timer from firing at us during very long I/O */
78 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;
79
80 if (timeout)
81 while (!wait_for_completion_io_timeout(done, timeout))
82 ;
83 else
84 wait_for_completion_io(done);
85}
86
Christoph Hellwigdc0b8a572021-02-02 18:19:19 +010087#define BIO_INLINE_VECS 4
Christoph Hellwig7a800a22021-02-02 18:19:29 +010088struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
89 gfp_t gfp_mask);
90void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);
Ming Leieec716a2021-01-11 11:05:56 +080091
Jinyoung Choi7c8998f2023-08-03 11:48:27 +090092bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv,
93 struct page *page, unsigned len, unsigned offset,
94 bool *same_page);
95
Christoph Hellwig3dccdae2018-09-24 09:43:52 +020096static inline bool biovec_phys_mergeable(struct request_queue *q,
97 struct bio_vec *vec1, struct bio_vec *vec2)
Christoph Hellwig6a9f5f22018-09-24 09:43:50 +020098{
Christoph Hellwig3dccdae2018-09-24 09:43:52 +020099 unsigned long mask = queue_segment_boundary(q);
Christoph Hellwig6e768462018-09-24 09:43:53 +0200100 phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset;
101 phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset;
Christoph Hellwig3dccdae2018-09-24 09:43:52 +0200102
Alexander Potapenkof630a5d2022-09-15 17:04:01 +0200103 /*
104 * Merging adjacent physical pages may not work correctly under KMSAN
105 * if their metadata pages aren't adjacent. Just disable merging.
106 */
107 if (IS_ENABLED(CONFIG_KMSAN))
108 return false;
109
Christoph Hellwig3dccdae2018-09-24 09:43:52 +0200110 if (addr1 + vec1->bv_len != addr2)
Christoph Hellwig6a9f5f22018-09-24 09:43:50 +0200111 return false;
Ming Lei0383ad42019-03-29 15:07:54 +0800112 if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page))
Christoph Hellwig6a9f5f22018-09-24 09:43:50 +0200113 return false;
Christoph Hellwig3dccdae2018-09-24 09:43:52 +0200114 if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask))
115 return false;
Christoph Hellwig6a9f5f22018-09-24 09:43:50 +0200116 return true;
117}
118
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700119static inline bool __bvec_gap_to_prev(const struct queue_limits *lim,
Christoph Hellwig27ca1d42018-09-24 09:43:49 +0200120 struct bio_vec *bprv, unsigned int offset)
121{
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400122 return (offset & lim->virt_boundary_mask) ||
123 ((bprv->bv_offset + bprv->bv_len) & lim->virt_boundary_mask);
Christoph Hellwig27ca1d42018-09-24 09:43:49 +0200124}
125
126/*
127 * Check if adding a bio_vec after bprv with offset would create a gap in
128 * the SG list. Most drivers don't care about this, but some do.
129 */
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700130static inline bool bvec_gap_to_prev(const struct queue_limits *lim,
Christoph Hellwig27ca1d42018-09-24 09:43:49 +0200131 struct bio_vec *bprv, unsigned int offset)
132{
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400133 if (!lim->virt_boundary_mask)
Christoph Hellwig27ca1d42018-09-24 09:43:49 +0200134 return false;
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400135 return __bvec_gap_to_prev(lim, bprv, offset);
Christoph Hellwig27ca1d42018-09-24 09:43:49 +0200136}
137
Christoph Hellwigbadf7f62021-09-20 14:33:26 +0200138static inline bool rq_mergeable(struct request *rq)
139{
140 if (blk_rq_is_passthrough(rq))
141 return false;
142
143 if (req_op(rq) == REQ_OP_FLUSH)
144 return false;
145
146 if (req_op(rq) == REQ_OP_WRITE_ZEROES)
147 return false;
148
149 if (req_op(rq) == REQ_OP_ZONE_APPEND)
150 return false;
151
152 if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
153 return false;
154 if (rq->rq_flags & RQF_NOMERGE_FLAGS)
155 return false;
156
157 return true;
158}
159
160/*
161 * There are two different ways to handle DISCARD merges:
162 * 1) If max_discard_segments > 1, the driver treats every bio as a range and
163 * send the bios to controller together. The ranges don't need to be
164 * contiguous.
165 * 2) Otherwise, the request will be normal read/write requests. The ranges
166 * need to be contiguous.
167 */
168static inline bool blk_discard_mergable(struct request *req)
169{
170 if (req_op(req) == REQ_OP_DISCARD &&
171 queue_max_discard_segments(req->q) > 1)
172 return true;
173 return false;
174}
175
Uday Shankar49d24392023-02-28 17:06:55 -0700176static inline unsigned int blk_rq_get_max_segments(struct request *rq)
177{
178 if (req_op(rq) == REQ_OP_DISCARD)
179 return queue_max_discard_segments(rq->q);
180 return queue_max_segments(rq->q);
181}
182
Christoph Hellwig2a9336c2022-06-14 11:09:34 +0200183static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
Bart Van Assche77e7ffd2022-07-14 11:06:28 -0700184 enum req_op op)
Christoph Hellwig2a9336c2022-06-14 11:09:34 +0200185{
186 if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
187 return min(q->limits.max_discard_sectors,
188 UINT_MAX >> SECTOR_SHIFT);
189
190 if (unlikely(op == REQ_OP_WRITE_ZEROES))
191 return q->limits.max_write_zeroes_sectors;
192
193 return q->limits.max_sectors;
194}
195
Dan Williams5a48fc12015-10-21 13:20:23 -0400196#ifdef CONFIG_BLK_DEV_INTEGRITY
197void blk_flush_integrity(void);
Christoph Hellwig7c20f112017-07-03 16:58:43 -0600198bool __bio_integrity_endio(struct bio *);
Justin Teeece841a2019-12-05 10:09:01 +0800199void bio_integrity_free(struct bio *bio);
Christoph Hellwig7c20f112017-07-03 16:58:43 -0600200static inline bool bio_integrity_endio(struct bio *bio)
201{
202 if (bio_integrity(bio))
203 return __bio_integrity_endio(bio);
204 return true;
205}
Christoph Hellwig43b729b2018-09-24 09:43:47 +0200206
Christoph Hellwig92cf2fd2020-10-06 09:07:17 +0200207bool blk_integrity_merge_rq(struct request_queue *, struct request *,
208 struct request *);
Christoph Hellwigd59da412020-10-06 09:07:18 +0200209bool blk_integrity_merge_bio(struct request_queue *, struct request *,
210 struct bio *);
Christoph Hellwig92cf2fd2020-10-06 09:07:17 +0200211
Christoph Hellwig43b729b2018-09-24 09:43:47 +0200212static inline bool integrity_req_gap_back_merge(struct request *req,
213 struct bio *next)
214{
215 struct bio_integrity_payload *bip = bio_integrity(req->bio);
216 struct bio_integrity_payload *bip_next = bio_integrity(next);
217
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400218 return bvec_gap_to_prev(&req->q->limits,
219 &bip->bip_vec[bip->bip_vcnt - 1],
Christoph Hellwig43b729b2018-09-24 09:43:47 +0200220 bip_next->bip_vec[0].bv_offset);
221}
222
223static inline bool integrity_req_gap_front_merge(struct request *req,
224 struct bio *bio)
225{
226 struct bio_integrity_payload *bip = bio_integrity(bio);
227 struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
228
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400229 return bvec_gap_to_prev(&req->q->limits,
230 &bip->bip_vec[bip->bip_vcnt - 1],
Christoph Hellwig43b729b2018-09-24 09:43:47 +0200231 bip_next->bip_vec[0].bv_offset);
232}
Christoph Hellwig581e2602020-03-25 16:48:41 +0100233
Thomas Weißschuhff53cd52023-03-18 17:36:25 +0000234extern const struct attribute_group blk_integrity_attr_group;
Christoph Hellwig43b729b2018-09-24 09:43:47 +0200235#else /* CONFIG_BLK_DEV_INTEGRITY */
Christoph Hellwig92cf2fd2020-10-06 09:07:17 +0200236static inline bool blk_integrity_merge_rq(struct request_queue *rq,
237 struct request *r1, struct request *r2)
238{
239 return true;
240}
Christoph Hellwigd59da412020-10-06 09:07:18 +0200241static inline bool blk_integrity_merge_bio(struct request_queue *rq,
242 struct request *r, struct bio *b)
243{
244 return true;
245}
Christoph Hellwig43b729b2018-09-24 09:43:47 +0200246static inline bool integrity_req_gap_back_merge(struct request *req,
247 struct bio *next)
248{
249 return false;
250}
251static inline bool integrity_req_gap_front_merge(struct request *req,
252 struct bio *bio)
253{
254 return false;
255}
256
Dan Williams5a48fc12015-10-21 13:20:23 -0400257static inline void blk_flush_integrity(void)
258{
259}
Christoph Hellwig7c20f112017-07-03 16:58:43 -0600260static inline bool bio_integrity_endio(struct bio *bio)
261{
262 return true;
263}
Justin Teeece841a2019-12-05 10:09:01 +0800264static inline void bio_integrity_free(struct bio *bio)
265{
266}
Christoph Hellwig43b729b2018-09-24 09:43:47 +0200267#endif /* CONFIG_BLK_DEV_INTEGRITY */
Jens Axboe8324aa92008-01-29 14:51:59 +0100268
Jens Axboe0d2602c2014-05-13 15:10:52 -0600269unsigned long blk_rq_timeout(unsigned long timeout);
Jens Axboe87ee7b12014-04-24 08:51:47 -0600270void blk_add_timer(struct request *req);
Jens Axboe320ae512013-10-24 09:20:05 +0100271
Jens Axboe320ae512013-10-24 09:20:05 +0100272bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
Christoph Hellwig0c5bcc92021-11-23 17:04:41 +0100273 unsigned int nr_segs);
Baolin Wangbdc6a2872020-08-28 10:52:55 +0800274bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
275 struct bio *bio, unsigned int nr_segs);
Jens Axboe320ae512013-10-24 09:20:05 +0100276
Jens Axboe242f9dc2008-09-14 05:55:09 -0700277/*
Jens Axboeba0ffdd2021-10-06 12:01:07 -0600278 * Plug flush limits
279 */
280#define BLK_MAX_REQUEST_COUNT 32
281#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
282
283/*
Tejun Heo158dbda2009-04-23 11:05:18 +0900284 * Internal elevator interface
285 */
Christoph Hellwige8064022016-10-20 15:12:13 +0200286#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
Tejun Heo158dbda2009-04-23 11:05:18 +0900287
Christoph Hellwig360f2642023-05-19 06:40:46 +0200288bool blk_insert_flush(struct request *rq);
Tejun Heodd831002010-09-03 11:56:16 +0200289
Keith Busch8237c012022-09-27 08:56:52 -0700290int elevator_switch(struct request_queue *q, struct elevator_type *new_e);
Christoph Hellwig64b36072022-10-30 11:07:14 +0100291void elevator_disable(struct request_queue *q);
Christoph Hellwig0c6cb3a2021-11-23 19:53:07 +0100292void elevator_exit(struct request_queue *q);
Ming Leicecf5d82019-08-27 19:01:48 +0800293int elv_register_queue(struct request_queue *q, bool uevent);
Bart Van Assche83d016a2018-01-17 11:48:08 -0800294void elv_unregister_queue(struct request_queue *q);
295
Christoph Hellwig3ad5cee2020-03-24 08:25:13 +0100296ssize_t part_size_show(struct device *dev, struct device_attribute *attr,
297 char *buf);
298ssize_t part_stat_show(struct device *dev, struct device_attribute *attr,
299 char *buf);
300ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr,
301 char *buf);
302ssize_t part_fail_show(struct device *dev, struct device_attribute *attr,
303 char *buf);
304ssize_t part_fail_store(struct device *dev, struct device_attribute *attr,
305 const char *buf, size_t count);
Jens Axboe581d4e22008-09-14 05:56:33 -0700306ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
307ssize_t part_timeout_store(struct device *, struct device_attribute *,
308 const char *, size_t);
Jens Axboe581d4e22008-09-14 05:56:33 -0700309
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400310static inline bool bio_may_exceed_limits(struct bio *bio,
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700311 const struct queue_limits *lim)
Jens Axboeabd45c12021-10-13 12:43:41 -0600312{
313 switch (bio_op(bio)) {
314 case REQ_OP_DISCARD:
315 case REQ_OP_SECURE_ERASE:
316 case REQ_OP_WRITE_ZEROES:
Jens Axboeabd45c12021-10-13 12:43:41 -0600317 return true; /* non-trivial splitting decisions */
318 default:
319 break;
320 }
321
322 /*
323 * All drivers must accept single-segments bios that are <= PAGE_SIZE.
324 * This is a quick and dirty check that relies on the fact that
325 * bi_io_vec[0] is always valid if a bio has data. The check might
326 * lead to occasional false negatives when bios are cloned, but compared
327 * to the performance impact of cloned bios themselves the loop below
328 * doesn't matter anyway.
329 */
Christoph Hellwigc55ddd92022-07-27 12:23:00 -0400330 return lim->chunk_sectors || bio->bi_vcnt != 1 ||
Jens Axboeabd45c12021-10-13 12:43:41 -0600331 bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE;
332}
333
Bart Van Asscheaa261f22022-10-25 12:17:54 -0700334struct bio *__bio_split_to_limits(struct bio *bio,
335 const struct queue_limits *lim,
336 unsigned int *nr_segs);
Christoph Hellwig14ccb662019-06-06 12:29:01 +0200337int ll_back_merge_fn(struct request *req, struct bio *bio,
338 unsigned int nr_segs);
Jan Karafd2ef392021-06-23 11:36:34 +0200339bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
Jens Axboe5e84ea32011-03-21 10:14:27 +0100340 struct request *next);
Christoph Hellwige9cd19c2019-06-06 12:29:02 +0200341unsigned int blk_recalc_rq_segments(struct request *rq);
Tejun Heo050c8ea2012-02-08 09:19:38 +0100342bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100343enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
Jens Axboed6d48192008-01-29 14:04:06 +0100344
Christoph Hellwigd690cb8a2024-02-13 08:34:14 +0100345int blk_set_default_limits(struct queue_limits *lim);
Adrian Bunkff889722008-03-04 11:23:45 +0100346int blk_dev_init(void);
347
Jens Axboec2553b52009-04-24 08:10:11 +0200348/*
349 * Contribute to IO statistics IFF:
350 *
351 * a) it's attached to a gendisk, and
Logan Gunthorpe48d9b0d2019-10-10 17:36:26 -0600352 * b) the queue had IO stats enabled when this request was started
Jens Axboec2553b52009-04-24 08:10:11 +0200353 */
Chengguang Xu599d0672018-08-16 22:51:40 +0800354static inline bool blk_do_io_stat(struct request *rq)
Jens Axboefb8ec182009-02-02 08:42:32 +0100355{
Christoph Hellwig41fa7222022-03-08 06:51:47 +0100356 return (rq->rq_flags & RQF_IO_STAT) && !blk_rq_is_passthrough(rq);
Pavel Begunkovbe6bfe32021-10-09 13:25:41 +0100357}
358
Christoph Hellwig450b7872021-11-17 07:14:01 +0100359void update_io_ticks(struct block_device *part, unsigned long now, bool end);
Jens Axboefb8ec182009-02-02 08:42:32 +0100360
Christoph Hellwig6cf76772017-02-08 14:46:47 +0100361static inline void req_set_nomerge(struct request_queue *q, struct request *req)
362{
363 req->cmd_flags |= REQ_NOMERGE;
364 if (req == q->last_merge)
365 q->last_merge = NULL;
366}
367
Tejun Heof2dbd762011-12-14 00:33:40 +0100368/*
369 * Internal io_context interface
370 */
Christoph Hellwig87dd1d62021-11-26 12:58:10 +0100371struct io_cq *ioc_find_get_icq(struct request_queue *q);
Christoph Hellwigeca58922021-11-26 12:58:17 +0100372struct io_cq *ioc_lookup_icq(struct request_queue *q);
Christoph Hellwig5ef16302021-12-09 07:31:31 +0100373#ifdef CONFIG_BLK_ICQ
Tejun Heo7e5a8792011-12-14 00:33:42 +0100374void ioc_clear_queue(struct request_queue *q);
Christoph Hellwig5ef16302021-12-09 07:31:31 +0100375#else
376static inline void ioc_clear_queue(struct request_queue *q)
377{
378}
379#endif /* CONFIG_BLK_ICQ */
Tejun Heof2dbd762011-12-14 00:33:40 +0100380
Shaohua Li297e3d82017-03-27 10:51:37 -0700381#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
382extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page);
383extern ssize_t blk_throtl_sample_time_store(struct request_queue *q,
384 const char *page, size_t count);
Shaohua Li9e234ee2017-03-27 10:51:41 -0700385extern void blk_throtl_bio_endio(struct bio *bio);
Shaohua Lib9147dd2017-03-27 15:19:42 -0700386extern void blk_throtl_stat_add(struct request *rq, u64 time);
Shaohua Li9e234ee2017-03-27 10:51:41 -0700387#else
388static inline void blk_throtl_bio_endio(struct bio *bio) { }
Shaohua Lib9147dd2017-03-27 15:19:42 -0700389static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
Shaohua Li297e3d82017-03-27 10:51:37 -0700390#endif
Tejun Heobc9fcbf2011-10-19 14:31:18 +0200391
Christoph Hellwig51d798c2022-07-27 12:22:56 -0400392struct bio *__blk_queue_bounce(struct bio *bio, struct request_queue *q);
Christoph Hellwig9bb33f22021-03-31 09:30:00 +0200393
394static inline bool blk_queue_may_bounce(struct request_queue *q)
395{
396 return IS_ENABLED(CONFIG_BOUNCE) &&
397 q->limits.bounce == BLK_BOUNCE_HIGH &&
398 max_low_pfn >= max_pfn;
399}
400
Christoph Hellwig51d798c2022-07-27 12:22:56 -0400401static inline struct bio *blk_queue_bounce(struct bio *bio,
402 struct request_queue *q)
Christoph Hellwig3bce0162017-06-19 09:26:21 +0200403{
Christoph Hellwig51d798c2022-07-27 12:22:56 -0400404 if (unlikely(blk_queue_may_bounce(q) && bio_has_data(bio)))
405 return __blk_queue_bounce(bio, q);
406 return bio;
Christoph Hellwig3bce0162017-06-19 09:26:21 +0200407}
Christoph Hellwig3bce0162017-06-19 09:26:21 +0200408
Damien Le Moalbf505452018-10-12 19:08:50 +0900409#ifdef CONFIG_BLK_DEV_ZONED
Christoph Hellwig5d400662022-07-06 09:03:42 +0200410void disk_free_zone_bitmaps(struct gendisk *disk);
Christoph Hellwig5e4ea8342023-06-08 13:02:54 +0200411int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd,
412 unsigned long arg);
Christoph Hellwig05bdb992023-06-08 13:02:55 +0200413int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode,
Christoph Hellwigcfb42572023-06-08 13:02:53 +0200414 unsigned int cmd, unsigned long arg);
415#else /* CONFIG_BLK_DEV_ZONED */
Christoph Hellwig5d400662022-07-06 09:03:42 +0200416static inline void disk_free_zone_bitmaps(struct gendisk *disk) {}
Christoph Hellwigcfb42572023-06-08 13:02:53 +0200417static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
Christoph Hellwig5e4ea8342023-06-08 13:02:54 +0200418 unsigned int cmd, unsigned long arg)
Christoph Hellwigcfb42572023-06-08 13:02:53 +0200419{
420 return -ENOTTY;
421}
422static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
Christoph Hellwig05bdb992023-06-08 13:02:55 +0200423 blk_mode_t mode, unsigned int cmd, unsigned long arg)
Christoph Hellwigcfb42572023-06-08 13:02:53 +0200424{
425 return -ENOTTY;
426}
427#endif /* CONFIG_BLK_DEV_ZONED */
428
429struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
430void bdev_add(struct block_device *bdev, dev_t dev);
Damien Le Moalbf505452018-10-12 19:08:50 +0900431
Christoph Hellwig7c3f8282021-05-21 07:50:51 +0200432int blk_alloc_ext_minor(void);
433void blk_free_ext_minor(unsigned int minor);
Christoph Hellwig581e2602020-03-25 16:48:41 +0100434#define ADDPART_FLAG_NONE 0
435#define ADDPART_FLAG_RAID 1
436#define ADDPART_FLAG_WHOLEDISK 2
Christoph Hellwig7f6be372021-08-10 17:45:10 +0200437int bdev_add_partition(struct gendisk *disk, int partno, sector_t start,
438 sector_t length);
Christoph Hellwig926fbb12021-08-10 17:45:11 +0200439int bdev_del_partition(struct gendisk *disk, int partno);
Christoph Hellwig3d2e7982021-08-10 17:45:12 +0200440int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start,
441 sector_t length);
Christoph Hellwigeec1be42023-06-01 11:44:50 +0200442void drop_partition(struct block_device *part);
Christoph Hellwig581e2602020-03-25 16:48:41 +0100443
Damien Le Moal83794362023-04-24 22:13:18 +0900444void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors);
445
Christoph Hellwig6f8191f2022-06-19 08:05:51 +0200446struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
447 struct lock_class_key *lkclass);
448
Christoph Hellwige4581102020-05-12 17:55:46 +0900449int bio_add_hw_page(struct request_queue *q, struct bio *bio,
Christoph Hellwig130879f2020-03-27 18:48:37 +0100450 struct page *page, unsigned int len, unsigned int offset,
Christoph Hellwige4581102020-05-12 17:55:46 +0900451 unsigned int max_sectors, bool *same_page);
Christoph Hellwig130879f2020-03-27 18:48:37 +0100452
David Howellsfd363242023-05-22 21:57:42 +0100453/*
454 * Clean up a page appropriately, where the page may be pinned, may have a
455 * ref taken on it or neither.
456 */
457static inline void bio_release_page(struct bio *bio, struct page *page)
458{
459 if (bio_flagged(bio, BIO_PAGE_PINNED))
460 unpin_user_page(page);
David Howellsfd363242023-05-22 21:57:42 +0100461}
462
Christoph Hellwigad751ba2024-02-13 08:34:18 +0100463struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id);
Ming Lei704b9142021-12-03 21:15:32 +0800464
Christoph Hellwig05bdb992023-06-08 13:02:55 +0200465int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode);
Christoph Hellwigda7ba722021-05-21 07:51:16 +0200466
Luis Chamberlain92e77552021-08-18 16:45:39 +0200467int disk_alloc_events(struct gendisk *disk);
Christoph Hellwigd5870ed2021-06-24 09:38:42 +0200468void disk_add_events(struct gendisk *disk);
469void disk_del_events(struct gendisk *disk);
470void disk_release_events(struct gendisk *disk);
Christoph Hellwig926597f2022-01-24 10:39:11 +0100471void disk_block_events(struct gendisk *disk);
472void disk_unblock_events(struct gendisk *disk);
473void disk_flush_events(struct gendisk *disk, unsigned int mask);
Christoph Hellwig2bc8cda2021-06-24 09:38:43 +0200474extern struct device_attribute dev_attr_events;
475extern struct device_attribute dev_attr_events_async;
476extern struct device_attribute dev_attr_events_poll_msecs;
Christoph Hellwigd5870ed2021-06-24 09:38:42 +0200477
Christoph Hellwigcc5c516d2022-06-28 19:18:45 +0200478extern struct attribute_group blk_trace_attr_group;
479
Christoph Hellwig05bdb992023-06-08 13:02:55 +0200480blk_mode_t file_to_blk_mode(struct file *file);
481int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode,
482 loff_t lstart, loff_t lend);
Christoph Hellwig8a709512021-10-12 12:44:50 +0200483long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
Christoph Hellwig84b85142021-10-12 12:44:49 +0200484long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
485
Christoph Hellwigcd82cca2021-09-07 16:13:02 +0200486extern const struct address_space_operations def_blk_aops;
487
Christoph Hellwig22d0c402022-06-29 08:20:13 +0200488int disk_register_independent_access_ranges(struct gendisk *disk);
Damien Le Moala2247f12021-10-27 11:22:19 +0900489void disk_unregister_independent_access_ranges(struct gendisk *disk);
490
Christoph Hellwig06c8c692021-11-17 07:13:58 +0100491#ifdef CONFIG_FAIL_MAKE_REQUEST
492bool should_fail_request(struct block_device *part, unsigned int bytes);
493#else /* CONFIG_FAIL_MAKE_REQUEST */
494static inline bool should_fail_request(struct block_device *part,
495 unsigned int bytes)
496{
497 return false;
498}
499#endif /* CONFIG_FAIL_MAKE_REQUEST */
500
Jens Axboe0a467d02021-10-14 14:39:59 -0600501/*
502 * Optimized request reference counting. Ideally we'd make timeouts be more
503 * clever, as that's the only reason we need references at all... But until
504 * this happens, this is faster than using refcount_t. Also see:
505 *
506 * abc54d634334 ("io_uring: switch to atomic_t for io_kiocb reference count")
507 */
508#define req_ref_zero_or_close_to_overflow(req) \
509 ((unsigned int) atomic_read(&(req->ref)) + 127u <= 127u)
510
511static inline bool req_ref_inc_not_zero(struct request *req)
512{
513 return atomic_inc_not_zero(&req->ref);
514}
515
516static inline bool req_ref_put_and_test(struct request *req)
517{
518 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
519 return atomic_dec_and_test(&req->ref);
520}
521
522static inline void req_ref_set(struct request *req, int value)
523{
524 atomic_set(&req->ref, value);
525}
526
527static inline int req_ref_read(struct request *req)
528{
529 return atomic_read(&req->ref);
530}
531
Jens Axboe08420cf2024-01-15 14:45:07 -0700532static inline u64 blk_time_get_ns(void)
533{
Jens Axboeda4c8c32024-01-15 14:46:03 -0700534 struct blk_plug *plug = current->plug;
535
Jens Axboeb874d4a2024-03-12 15:58:41 -0600536 if (!plug || !in_task())
Jens Axboeda4c8c32024-01-15 14:46:03 -0700537 return ktime_get_ns();
538
539 /*
540 * 0 could very well be a valid time, but rather than flag "this is
541 * a valid timestamp" separately, just accept that we'll do an extra
542 * ktime_get_ns() if we just happen to get 0 as the current time.
543 */
Jens Axboe06b23f922024-01-16 09:18:39 -0700544 if (!plug->cur_ktime) {
Jens Axboeda4c8c32024-01-15 14:46:03 -0700545 plug->cur_ktime = ktime_get_ns();
Jens Axboe06b23f922024-01-16 09:18:39 -0700546 current->flags |= PF_BLOCK_TS;
547 }
Jens Axboeda4c8c32024-01-15 14:46:03 -0700548 return plug->cur_ktime;
Jens Axboe08420cf2024-01-15 14:45:07 -0700549}
550
551static inline ktime_t blk_time_get(void)
552{
553 return ns_to_ktime(blk_time_get_ns());
554}
555
Jens Axboec4e47bbb2024-01-16 14:14:59 -0700556/*
557 * From most significant bit:
558 * 1 bit: reserved for other usage, see below
559 * 12 bits: original size of bio
560 * 51 bits: issue time of bio
561 */
562#define BIO_ISSUE_RES_BITS 1
563#define BIO_ISSUE_SIZE_BITS 12
564#define BIO_ISSUE_RES_SHIFT (64 - BIO_ISSUE_RES_BITS)
565#define BIO_ISSUE_SIZE_SHIFT (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS)
566#define BIO_ISSUE_TIME_MASK ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1)
567#define BIO_ISSUE_SIZE_MASK \
568 (((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT)
569#define BIO_ISSUE_RES_MASK (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1))
570
571/* Reserved bit for blk-throtl */
572#define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63)
573
574static inline u64 __bio_issue_time(u64 time)
575{
576 return time & BIO_ISSUE_TIME_MASK;
577}
578
579static inline u64 bio_issue_time(struct bio_issue *issue)
580{
581 return __bio_issue_time(issue->value);
582}
583
584static inline sector_t bio_issue_size(struct bio_issue *issue)
585{
586 return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT);
587}
588
589static inline void bio_issue_init(struct bio_issue *issue,
590 sector_t size)
591{
592 size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1;
593 issue->value = ((issue->value & BIO_ISSUE_RES_MASK) |
Jens Axboe08420cf2024-01-15 14:45:07 -0700594 (blk_time_get_ns() & BIO_ISSUE_TIME_MASK) |
Jens Axboec4e47bbb2024-01-16 14:14:59 -0700595 ((u64)size << BIO_ISSUE_SIZE_SHIFT));
596}
597
Christian Brauner7c09a4e2024-01-23 14:26:47 +0100598void bdev_release(struct file *bdev_file);
Christian Braunera56aefc2024-01-23 14:26:46 +0100599int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder,
600 const struct blk_holder_ops *hops, struct file *bdev_file);
601int bdev_permission(dev_t dev, blk_mode_t mode, void *holder);
Linus Torvalds1ddeeb22024-03-11 11:43:44 -0700602
Tejun Heobc9fcbf2011-10-19 14:31:18 +0200603#endif /* BLK_INTERNAL_H */