blob: 94987b1f69e15a08ed2e97b08004dc36fa139552 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Jens Axboe8324aa92008-01-29 14:51:59 +01002/*
3 * Functions related to sysfs handling
4 */
5#include <linux/kernel.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09006#include <linux/slab.h>
Jens Axboe8324aa92008-01-29 14:51:59 +01007#include <linux/module.h>
8#include <linux/bio.h>
9#include <linux/blkdev.h>
Tejun Heo66114ca2015-05-22 17:13:32 -040010#include <linux/backing-dev.h>
Jens Axboe8324aa92008-01-29 14:51:59 +010011#include <linux/blktrace_api.h>
Jens Axboe320ae512013-10-24 09:20:05 +010012#include <linux/blk-mq.h>
Tejun Heoeea8f412015-05-22 17:13:17 -040013#include <linux/blk-cgroup.h>
Jens Axboe8324aa92008-01-29 14:51:59 +010014
15#include "blk.h"
Ming Lei3edcc0c2013-12-26 21:31:38 +080016#include "blk-mq.h"
Omar Sandovald173a252017-05-04 00:31:30 -070017#include "blk-mq-debugfs.h"
Jens Axboe87760e52016-11-09 12:38:14 -070018#include "blk-wbt.h"
Jens Axboe8324aa92008-01-29 14:51:59 +010019
20struct queue_sysfs_entry {
21 struct attribute attr;
22 ssize_t (*show)(struct request_queue *, char *);
23 ssize_t (*store)(struct request_queue *, const char *, size_t);
24};
25
26static ssize_t
Xiaotian Feng9cb308c2009-07-17 15:26:26 +080027queue_var_show(unsigned long var, char *page)
Jens Axboe8324aa92008-01-29 14:51:59 +010028{
Xiaotian Feng9cb308c2009-07-17 15:26:26 +080029 return sprintf(page, "%lu\n", var);
Jens Axboe8324aa92008-01-29 14:51:59 +010030}
31
32static ssize_t
33queue_var_store(unsigned long *var, const char *page, size_t count)
34{
Dave Reisnerb1f3b642012-09-08 11:55:45 -040035 int err;
36 unsigned long v;
Jens Axboe8324aa92008-01-29 14:51:59 +010037
Jingoo Haned751e62013-09-11 14:20:08 -070038 err = kstrtoul(page, 10, &v);
Dave Reisnerb1f3b642012-09-08 11:55:45 -040039 if (err || v > UINT_MAX)
40 return -EINVAL;
41
42 *var = v;
43
Jens Axboe8324aa92008-01-29 14:51:59 +010044 return count;
45}
46
Jens Axboe80e091d2016-11-28 09:22:47 -070047static ssize_t queue_var_store64(s64 *var, const char *page)
Jens Axboe87760e52016-11-09 12:38:14 -070048{
49 int err;
Jens Axboe80e091d2016-11-28 09:22:47 -070050 s64 v;
Jens Axboe87760e52016-11-09 12:38:14 -070051
Jens Axboe80e091d2016-11-28 09:22:47 -070052 err = kstrtos64(page, 10, &v);
Jens Axboe87760e52016-11-09 12:38:14 -070053 if (err < 0)
54 return err;
55
56 *var = v;
57 return 0;
58}
59
Jens Axboe8324aa92008-01-29 14:51:59 +010060static ssize_t queue_requests_show(struct request_queue *q, char *page)
61{
62 return queue_var_show(q->nr_requests, (page));
63}
64
65static ssize_t
66queue_requests_store(struct request_queue *q, const char *page, size_t count)
67{
Jens Axboe8324aa92008-01-29 14:51:59 +010068 unsigned long nr;
Jens Axboee3a2b3f2014-05-20 11:49:02 -060069 int ret, err;
Jens Axboeb8a9ae72009-09-11 22:44:29 +020070
Jens Axboee3a2b3f2014-05-20 11:49:02 -060071 if (!q->request_fn && !q->mq_ops)
Jens Axboeb8a9ae72009-09-11 22:44:29 +020072 return -EINVAL;
73
74 ret = queue_var_store(&nr, page, count);
Dave Reisnerb1f3b642012-09-08 11:55:45 -040075 if (ret < 0)
76 return ret;
77
Jens Axboe8324aa92008-01-29 14:51:59 +010078 if (nr < BLKDEV_MIN_RQ)
79 nr = BLKDEV_MIN_RQ;
80
Jens Axboee3a2b3f2014-05-20 11:49:02 -060081 if (q->request_fn)
82 err = blk_update_nr_requests(q, nr);
83 else
84 err = blk_mq_update_nr_requests(q, nr);
Jens Axboe8324aa92008-01-29 14:51:59 +010085
Jens Axboee3a2b3f2014-05-20 11:49:02 -060086 if (err)
87 return err;
Tejun Heoa0516612012-06-26 15:05:44 -070088
Jens Axboe8324aa92008-01-29 14:51:59 +010089 return ret;
90}
91
92static ssize_t queue_ra_show(struct request_queue *q, char *page)
93{
Jan Karadc3b17c2017-02-02 15:56:50 +010094 unsigned long ra_kb = q->backing_dev_info->ra_pages <<
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +030095 (PAGE_SHIFT - 10);
Jens Axboe8324aa92008-01-29 14:51:59 +010096
97 return queue_var_show(ra_kb, (page));
98}
99
100static ssize_t
101queue_ra_store(struct request_queue *q, const char *page, size_t count)
102{
103 unsigned long ra_kb;
104 ssize_t ret = queue_var_store(&ra_kb, page, count);
105
Dave Reisnerb1f3b642012-09-08 11:55:45 -0400106 if (ret < 0)
107 return ret;
108
Jan Karadc3b17c2017-02-02 15:56:50 +0100109 q->backing_dev_info->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
Jens Axboe8324aa92008-01-29 14:51:59 +0100110
111 return ret;
112}
113
114static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
115{
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400116 int max_sectors_kb = queue_max_sectors(q) >> 1;
Jens Axboe8324aa92008-01-29 14:51:59 +0100117
118 return queue_var_show(max_sectors_kb, (page));
119}
120
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500121static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
122{
123 return queue_var_show(queue_max_segments(q), (page));
124}
125
Christoph Hellwig1e739732017-02-08 14:46:49 +0100126static ssize_t queue_max_discard_segments_show(struct request_queue *q,
127 char *page)
128{
129 return queue_var_show(queue_max_discard_segments(q), (page));
130}
131
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200132static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
133{
134 return queue_var_show(q->limits.max_integrity_segments, (page));
135}
136
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500137static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
138{
Martin K. Petersene692cb62010-12-01 19:41:49 +0100139 if (blk_queue_cluster(q))
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500140 return queue_var_show(queue_max_segment_size(q), (page));
141
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300142 return queue_var_show(PAGE_SIZE, (page));
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500143}
144
Martin K. Petersene1defc42009-05-22 17:17:49 -0400145static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
Martin K. Petersene68b9032008-01-29 19:14:08 +0100146{
Martin K. Petersene1defc42009-05-22 17:17:49 -0400147 return queue_var_show(queue_logical_block_size(q), page);
Martin K. Petersene68b9032008-01-29 19:14:08 +0100148}
149
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400150static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
151{
152 return queue_var_show(queue_physical_block_size(q), page);
153}
154
Hannes Reinecke87caf972016-10-18 15:40:30 +0900155static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
156{
157 return queue_var_show(q->limits.chunk_sectors, page);
158}
159
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400160static ssize_t queue_io_min_show(struct request_queue *q, char *page)
161{
162 return queue_var_show(queue_io_min(q), page);
163}
164
165static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
166{
167 return queue_var_show(queue_io_opt(q), page);
Jens Axboe8324aa92008-01-29 14:51:59 +0100168}
169
Martin K. Petersen86b37282009-11-10 11:50:21 +0100170static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
171{
172 return queue_var_show(q->limits.discard_granularity, page);
173}
174
Jens Axboe0034af02015-07-16 09:14:26 -0600175static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
176{
Jens Axboe0034af02015-07-16 09:14:26 -0600177
Alan18f922d02016-02-17 14:15:30 +0000178 return sprintf(page, "%llu\n",
179 (unsigned long long)q->limits.max_hw_discard_sectors << 9);
Jens Axboe0034af02015-07-16 09:14:26 -0600180}
181
Martin K. Petersen86b37282009-11-10 11:50:21 +0100182static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
183{
Martin K. Petersena934a002011-05-18 10:37:35 +0200184 return sprintf(page, "%llu\n",
185 (unsigned long long)q->limits.max_discard_sectors << 9);
Martin K. Petersen86b37282009-11-10 11:50:21 +0100186}
187
Jens Axboe0034af02015-07-16 09:14:26 -0600188static ssize_t queue_discard_max_store(struct request_queue *q,
189 const char *page, size_t count)
190{
191 unsigned long max_discard;
192 ssize_t ret = queue_var_store(&max_discard, page, count);
193
194 if (ret < 0)
195 return ret;
196
197 if (max_discard & (q->limits.discard_granularity - 1))
198 return -EINVAL;
199
200 max_discard >>= 9;
201 if (max_discard > UINT_MAX)
202 return -EINVAL;
203
204 if (max_discard > q->limits.max_hw_discard_sectors)
205 max_discard = q->limits.max_hw_discard_sectors;
206
207 q->limits.max_discard_sectors = max_discard;
208 return ret;
209}
210
Martin K. Petersen98262f22009-12-03 09:24:48 +0100211static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
212{
Christoph Hellwig48920ff2017-04-05 19:21:23 +0200213 return queue_var_show(0, page);
Martin K. Petersen98262f22009-12-03 09:24:48 +0100214}
215
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400216static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
217{
218 return sprintf(page, "%llu\n",
219 (unsigned long long)q->limits.max_write_same_sectors << 9);
220}
221
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800222static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
223{
224 return sprintf(page, "%llu\n",
225 (unsigned long long)q->limits.max_write_zeroes_sectors << 9);
226}
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400227
Jens Axboe8324aa92008-01-29 14:51:59 +0100228static ssize_t
229queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
230{
231 unsigned long max_sectors_kb,
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400232 max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300233 page_kb = 1 << (PAGE_SHIFT - 10);
Jens Axboe8324aa92008-01-29 14:51:59 +0100234 ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
235
Dave Reisnerb1f3b642012-09-08 11:55:45 -0400236 if (ret < 0)
237 return ret;
238
Martin K. Petersenca369d52015-11-13 16:46:48 -0500239 max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
240 q->limits.max_dev_sectors >> 1);
241
Jens Axboe8324aa92008-01-29 14:51:59 +0100242 if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
243 return -EINVAL;
Wu Fengguang7c239512008-11-25 09:08:39 +0100244
Jens Axboe8324aa92008-01-29 14:51:59 +0100245 spin_lock_irq(q->queue_lock);
Nikanth Karthikesanc295fc02009-09-01 22:40:15 +0200246 q->limits.max_sectors = max_sectors_kb << 1;
Jan Karadc3b17c2017-02-02 15:56:50 +0100247 q->backing_dev_info->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);
Jens Axboe8324aa92008-01-29 14:51:59 +0100248 spin_unlock_irq(q->queue_lock);
249
250 return ret;
251}
252
253static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
254{
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400255 int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
Jens Axboe8324aa92008-01-29 14:51:59 +0100256
257 return queue_var_show(max_hw_sectors_kb, (page));
258}
259
Jens Axboe956bcb72010-08-07 18:13:50 +0200260#define QUEUE_SYSFS_BIT_FNS(name, flag, neg) \
261static ssize_t \
262queue_show_##name(struct request_queue *q, char *page) \
263{ \
264 int bit; \
265 bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags); \
266 return queue_var_show(neg ? !bit : bit, page); \
267} \
268static ssize_t \
269queue_store_##name(struct request_queue *q, const char *page, size_t count) \
270{ \
271 unsigned long val; \
272 ssize_t ret; \
273 ret = queue_var_store(&val, page, count); \
Arnd Bergmannc678ef52013-04-03 21:53:57 +0200274 if (ret < 0) \
275 return ret; \
Jens Axboe956bcb72010-08-07 18:13:50 +0200276 if (neg) \
277 val = !val; \
278 \
Jens Axboe956bcb72010-08-07 18:13:50 +0200279 if (val) \
Bart Van Assche8814ce82018-03-07 17:10:04 -0800280 blk_queue_flag_set(QUEUE_FLAG_##flag, q); \
Jens Axboe956bcb72010-08-07 18:13:50 +0200281 else \
Bart Van Assche8814ce82018-03-07 17:10:04 -0800282 blk_queue_flag_clear(QUEUE_FLAG_##flag, q); \
Jens Axboe956bcb72010-08-07 18:13:50 +0200283 return ret; \
Bartlomiej Zolnierkiewicz1308835f2009-01-07 12:22:39 +0100284}
285
Jens Axboe956bcb72010-08-07 18:13:50 +0200286QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
287QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
288QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
289#undef QUEUE_SYSFS_BIT_FNS
Bartlomiej Zolnierkiewicz1308835f2009-01-07 12:22:39 +0100290
Damien Le Moal797476b2016-10-18 15:40:29 +0900291static ssize_t queue_zoned_show(struct request_queue *q, char *page)
292{
293 switch (blk_queue_zoned_model(q)) {
294 case BLK_ZONED_HA:
295 return sprintf(page, "host-aware\n");
296 case BLK_ZONED_HM:
297 return sprintf(page, "host-managed\n");
298 default:
299 return sprintf(page, "none\n");
300 }
301}
302
Alan D. Brunelleac9fafa2008-04-29 14:44:19 +0200303static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
304{
Alan D. Brunelle488991e2010-01-29 09:04:08 +0100305 return queue_var_show((blk_queue_nomerges(q) << 1) |
306 blk_queue_noxmerges(q), page);
Alan D. Brunelleac9fafa2008-04-29 14:44:19 +0200307}
308
309static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
310 size_t count)
311{
312 unsigned long nm;
313 ssize_t ret = queue_var_store(&nm, page, count);
314
Dave Reisnerb1f3b642012-09-08 11:55:45 -0400315 if (ret < 0)
316 return ret;
317
Jens Axboebf0f9702008-05-07 09:09:39 +0200318 spin_lock_irq(q->queue_lock);
Alan D. Brunelle488991e2010-01-29 09:04:08 +0100319 queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
320 queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
321 if (nm == 2)
Jens Axboebf0f9702008-05-07 09:09:39 +0200322 queue_flag_set(QUEUE_FLAG_NOMERGES, q);
Alan D. Brunelle488991e2010-01-29 09:04:08 +0100323 else if (nm)
324 queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
Jens Axboebf0f9702008-05-07 09:09:39 +0200325 spin_unlock_irq(q->queue_lock);
Bartlomiej Zolnierkiewicz1308835f2009-01-07 12:22:39 +0100326
Alan D. Brunelleac9fafa2008-04-29 14:44:19 +0200327 return ret;
328}
329
Jens Axboec7c22e42008-09-13 20:26:01 +0200330static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
331{
Xiaotian Feng9cb308c2009-07-17 15:26:26 +0800332 bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
Dan Williams5757a6d2011-07-23 20:44:25 +0200333 bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
Jens Axboec7c22e42008-09-13 20:26:01 +0200334
Dan Williams5757a6d2011-07-23 20:44:25 +0200335 return queue_var_show(set << force, page);
Jens Axboec7c22e42008-09-13 20:26:01 +0200336}
337
338static ssize_t
339queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
340{
341 ssize_t ret = -EINVAL;
Christoph Hellwig0a06ff02013-11-14 14:32:07 -0800342#ifdef CONFIG_SMP
Jens Axboec7c22e42008-09-13 20:26:01 +0200343 unsigned long val;
344
345 ret = queue_var_store(&val, page, count);
Dave Reisnerb1f3b642012-09-08 11:55:45 -0400346 if (ret < 0)
347 return ret;
348
Jens Axboec7c22e42008-09-13 20:26:01 +0200349 spin_lock_irq(q->queue_lock);
Eric Seppanene8037d42011-08-23 21:25:12 +0200350 if (val == 2) {
Jens Axboec7c22e42008-09-13 20:26:01 +0200351 queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
Eric Seppanene8037d42011-08-23 21:25:12 +0200352 queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
353 } else if (val == 1) {
354 queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
355 queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
356 } else if (val == 0) {
Dan Williams5757a6d2011-07-23 20:44:25 +0200357 queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
358 queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
359 }
Jens Axboec7c22e42008-09-13 20:26:01 +0200360 spin_unlock_irq(q->queue_lock);
361#endif
362 return ret;
363}
Jens Axboe8324aa92008-01-29 14:51:59 +0100364
Jens Axboe06426ad2016-11-14 13:01:59 -0700365static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
366{
Jens Axboe64f1c212016-11-14 13:03:03 -0700367 int val;
368
369 if (q->poll_nsec == -1)
370 val = -1;
371 else
372 val = q->poll_nsec / 1000;
373
374 return sprintf(page, "%d\n", val);
Jens Axboe06426ad2016-11-14 13:01:59 -0700375}
376
377static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
378 size_t count)
379{
Jens Axboe64f1c212016-11-14 13:03:03 -0700380 int err, val;
Jens Axboe06426ad2016-11-14 13:01:59 -0700381
382 if (!q->mq_ops || !q->mq_ops->poll)
383 return -EINVAL;
384
Jens Axboe64f1c212016-11-14 13:03:03 -0700385 err = kstrtoint(page, 10, &val);
386 if (err < 0)
387 return err;
Jens Axboe06426ad2016-11-14 13:01:59 -0700388
Jens Axboe64f1c212016-11-14 13:03:03 -0700389 if (val == -1)
390 q->poll_nsec = -1;
391 else
392 q->poll_nsec = val * 1000;
393
394 return count;
Jens Axboe06426ad2016-11-14 13:01:59 -0700395}
396
Jens Axboe05229beed2015-11-05 10:44:55 -0700397static ssize_t queue_poll_show(struct request_queue *q, char *page)
398{
399 return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
400}
401
402static ssize_t queue_poll_store(struct request_queue *q, const char *page,
403 size_t count)
404{
405 unsigned long poll_on;
406 ssize_t ret;
407
408 if (!q->mq_ops || !q->mq_ops->poll)
409 return -EINVAL;
410
411 ret = queue_var_store(&poll_on, page, count);
412 if (ret < 0)
413 return ret;
414
Jens Axboe05229beed2015-11-05 10:44:55 -0700415 if (poll_on)
Bart Van Assche8814ce82018-03-07 17:10:04 -0800416 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
Jens Axboe05229beed2015-11-05 10:44:55 -0700417 else
Bart Van Assche8814ce82018-03-07 17:10:04 -0800418 blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
Jens Axboe05229beed2015-11-05 10:44:55 -0700419
420 return ret;
421}
422
Jens Axboe87760e52016-11-09 12:38:14 -0700423static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
424{
425 if (!q->rq_wb)
426 return -EINVAL;
427
428 return sprintf(page, "%llu\n", div_u64(q->rq_wb->min_lat_nsec, 1000));
429}
430
431static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
432 size_t count)
433{
Jens Axboe80e091d2016-11-28 09:22:47 -0700434 struct rq_wb *rwb;
Jens Axboe87760e52016-11-09 12:38:14 -0700435 ssize_t ret;
Jens Axboe80e091d2016-11-28 09:22:47 -0700436 s64 val;
Jens Axboe87760e52016-11-09 12:38:14 -0700437
Jens Axboe87760e52016-11-09 12:38:14 -0700438 ret = queue_var_store64(&val, page);
439 if (ret < 0)
440 return ret;
Jens Axboed62118b2016-11-28 09:40:34 -0700441 if (val < -1)
442 return -EINVAL;
443
444 rwb = q->rq_wb;
445 if (!rwb) {
446 ret = wbt_init(q);
447 if (ret)
448 return ret;
Jens Axboed62118b2016-11-28 09:40:34 -0700449 }
Jens Axboe87760e52016-11-09 12:38:14 -0700450
weiping zhangf6804742017-11-23 21:39:40 +0800451 rwb = q->rq_wb;
Jens Axboe80e091d2016-11-28 09:22:47 -0700452 if (val == -1)
453 rwb->min_lat_nsec = wbt_default_latency_nsec(q);
454 else if (val >= 0)
455 rwb->min_lat_nsec = val * 1000ULL;
Jens Axboed62118b2016-11-28 09:40:34 -0700456
457 if (rwb->enable_state == WBT_STATE_ON_DEFAULT)
458 rwb->enable_state = WBT_STATE_ON_MANUAL;
Jens Axboe80e091d2016-11-28 09:22:47 -0700459
460 wbt_update_limits(rwb);
Jens Axboe87760e52016-11-09 12:38:14 -0700461 return count;
462}
463
Jens Axboe93e9d8e2016-04-12 12:32:46 -0600464static ssize_t queue_wc_show(struct request_queue *q, char *page)
465{
466 if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
467 return sprintf(page, "write back\n");
468
469 return sprintf(page, "write through\n");
470}
471
472static ssize_t queue_wc_store(struct request_queue *q, const char *page,
473 size_t count)
474{
475 int set = -1;
476
477 if (!strncmp(page, "write back", 10))
478 set = 1;
479 else if (!strncmp(page, "write through", 13) ||
480 !strncmp(page, "none", 4))
481 set = 0;
482
483 if (set == -1)
484 return -EINVAL;
485
Jens Axboe93e9d8e2016-04-12 12:32:46 -0600486 if (set)
Bart Van Assche8814ce82018-03-07 17:10:04 -0800487 blk_queue_flag_set(QUEUE_FLAG_WC, q);
Jens Axboe93e9d8e2016-04-12 12:32:46 -0600488 else
Bart Van Assche8814ce82018-03-07 17:10:04 -0800489 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
Jens Axboe93e9d8e2016-04-12 12:32:46 -0600490
491 return count;
492}
493
Kent Overstreet6fcefbe2018-05-08 21:33:58 -0400494static ssize_t queue_fua_show(struct request_queue *q, char *page)
495{
496 return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));
497}
498
Yigal Kormanea6ca602016-06-23 17:05:51 -0400499static ssize_t queue_dax_show(struct request_queue *q, char *page)
500{
501 return queue_var_show(blk_queue_dax(q), page);
502}
503
Jens Axboe8324aa92008-01-29 14:51:59 +0100504static struct queue_sysfs_entry queue_requests_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600505 .attr = {.name = "nr_requests", .mode = 0644 },
Jens Axboe8324aa92008-01-29 14:51:59 +0100506 .show = queue_requests_show,
507 .store = queue_requests_store,
508};
509
510static struct queue_sysfs_entry queue_ra_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600511 .attr = {.name = "read_ahead_kb", .mode = 0644 },
Jens Axboe8324aa92008-01-29 14:51:59 +0100512 .show = queue_ra_show,
513 .store = queue_ra_store,
514};
515
516static struct queue_sysfs_entry queue_max_sectors_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600517 .attr = {.name = "max_sectors_kb", .mode = 0644 },
Jens Axboe8324aa92008-01-29 14:51:59 +0100518 .show = queue_max_sectors_show,
519 .store = queue_max_sectors_store,
520};
521
522static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600523 .attr = {.name = "max_hw_sectors_kb", .mode = 0444 },
Jens Axboe8324aa92008-01-29 14:51:59 +0100524 .show = queue_max_hw_sectors_show,
525};
526
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500527static struct queue_sysfs_entry queue_max_segments_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600528 .attr = {.name = "max_segments", .mode = 0444 },
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500529 .show = queue_max_segments_show,
530};
531
Christoph Hellwig1e739732017-02-08 14:46:49 +0100532static struct queue_sysfs_entry queue_max_discard_segments_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600533 .attr = {.name = "max_discard_segments", .mode = 0444 },
Christoph Hellwig1e739732017-02-08 14:46:49 +0100534 .show = queue_max_discard_segments_show,
535};
536
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200537static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600538 .attr = {.name = "max_integrity_segments", .mode = 0444 },
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200539 .show = queue_max_integrity_segments_show,
540};
541
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500542static struct queue_sysfs_entry queue_max_segment_size_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600543 .attr = {.name = "max_segment_size", .mode = 0444 },
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500544 .show = queue_max_segment_size_show,
545};
546
Jens Axboe8324aa92008-01-29 14:51:59 +0100547static struct queue_sysfs_entry queue_iosched_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600548 .attr = {.name = "scheduler", .mode = 0644 },
Jens Axboe8324aa92008-01-29 14:51:59 +0100549 .show = elv_iosched_show,
550 .store = elv_iosched_store,
551};
552
Martin K. Petersene68b9032008-01-29 19:14:08 +0100553static struct queue_sysfs_entry queue_hw_sector_size_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600554 .attr = {.name = "hw_sector_size", .mode = 0444 },
Martin K. Petersene1defc42009-05-22 17:17:49 -0400555 .show = queue_logical_block_size_show,
556};
557
558static struct queue_sysfs_entry queue_logical_block_size_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600559 .attr = {.name = "logical_block_size", .mode = 0444 },
Martin K. Petersene1defc42009-05-22 17:17:49 -0400560 .show = queue_logical_block_size_show,
Martin K. Petersene68b9032008-01-29 19:14:08 +0100561};
562
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400563static struct queue_sysfs_entry queue_physical_block_size_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600564 .attr = {.name = "physical_block_size", .mode = 0444 },
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400565 .show = queue_physical_block_size_show,
566};
567
Hannes Reinecke87caf972016-10-18 15:40:30 +0900568static struct queue_sysfs_entry queue_chunk_sectors_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600569 .attr = {.name = "chunk_sectors", .mode = 0444 },
Hannes Reinecke87caf972016-10-18 15:40:30 +0900570 .show = queue_chunk_sectors_show,
571};
572
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400573static struct queue_sysfs_entry queue_io_min_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600574 .attr = {.name = "minimum_io_size", .mode = 0444 },
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400575 .show = queue_io_min_show,
576};
577
578static struct queue_sysfs_entry queue_io_opt_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600579 .attr = {.name = "optimal_io_size", .mode = 0444 },
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400580 .show = queue_io_opt_show,
Jens Axboe8324aa92008-01-29 14:51:59 +0100581};
582
Martin K. Petersen86b37282009-11-10 11:50:21 +0100583static struct queue_sysfs_entry queue_discard_granularity_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600584 .attr = {.name = "discard_granularity", .mode = 0444 },
Martin K. Petersen86b37282009-11-10 11:50:21 +0100585 .show = queue_discard_granularity_show,
586};
587
Jens Axboe0034af02015-07-16 09:14:26 -0600588static struct queue_sysfs_entry queue_discard_max_hw_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600589 .attr = {.name = "discard_max_hw_bytes", .mode = 0444 },
Jens Axboe0034af02015-07-16 09:14:26 -0600590 .show = queue_discard_max_hw_show,
591};
592
Martin K. Petersen86b37282009-11-10 11:50:21 +0100593static struct queue_sysfs_entry queue_discard_max_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600594 .attr = {.name = "discard_max_bytes", .mode = 0644 },
Martin K. Petersen86b37282009-11-10 11:50:21 +0100595 .show = queue_discard_max_show,
Jens Axboe0034af02015-07-16 09:14:26 -0600596 .store = queue_discard_max_store,
Martin K. Petersen86b37282009-11-10 11:50:21 +0100597};
598
Martin K. Petersen98262f22009-12-03 09:24:48 +0100599static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600600 .attr = {.name = "discard_zeroes_data", .mode = 0444 },
Martin K. Petersen98262f22009-12-03 09:24:48 +0100601 .show = queue_discard_zeroes_data_show,
602};
603
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400604static struct queue_sysfs_entry queue_write_same_max_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600605 .attr = {.name = "write_same_max_bytes", .mode = 0444 },
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400606 .show = queue_write_same_max_show,
607};
608
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800609static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600610 .attr = {.name = "write_zeroes_max_bytes", .mode = 0444 },
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800611 .show = queue_write_zeroes_max_show,
612};
613
Bartlomiej Zolnierkiewicz1308835f2009-01-07 12:22:39 +0100614static struct queue_sysfs_entry queue_nonrot_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600615 .attr = {.name = "rotational", .mode = 0644 },
Jens Axboe956bcb72010-08-07 18:13:50 +0200616 .show = queue_show_nonrot,
617 .store = queue_store_nonrot,
Bartlomiej Zolnierkiewicz1308835f2009-01-07 12:22:39 +0100618};
619
Damien Le Moal797476b2016-10-18 15:40:29 +0900620static struct queue_sysfs_entry queue_zoned_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600621 .attr = {.name = "zoned", .mode = 0444 },
Damien Le Moal797476b2016-10-18 15:40:29 +0900622 .show = queue_zoned_show,
623};
624
Alan D. Brunelleac9fafa2008-04-29 14:44:19 +0200625static struct queue_sysfs_entry queue_nomerges_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600626 .attr = {.name = "nomerges", .mode = 0644 },
Alan D. Brunelleac9fafa2008-04-29 14:44:19 +0200627 .show = queue_nomerges_show,
628 .store = queue_nomerges_store,
629};
630
Jens Axboec7c22e42008-09-13 20:26:01 +0200631static struct queue_sysfs_entry queue_rq_affinity_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600632 .attr = {.name = "rq_affinity", .mode = 0644 },
Jens Axboec7c22e42008-09-13 20:26:01 +0200633 .show = queue_rq_affinity_show,
634 .store = queue_rq_affinity_store,
635};
636
Jens Axboebc58ba92009-01-23 10:54:44 +0100637static struct queue_sysfs_entry queue_iostats_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600638 .attr = {.name = "iostats", .mode = 0644 },
Jens Axboe956bcb72010-08-07 18:13:50 +0200639 .show = queue_show_iostats,
640 .store = queue_store_iostats,
Jens Axboebc58ba92009-01-23 10:54:44 +0100641};
642
Jens Axboee2e1a142010-06-09 10:42:09 +0200643static struct queue_sysfs_entry queue_random_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600644 .attr = {.name = "add_random", .mode = 0644 },
Jens Axboe956bcb72010-08-07 18:13:50 +0200645 .show = queue_show_random,
646 .store = queue_store_random,
Jens Axboee2e1a142010-06-09 10:42:09 +0200647};
648
Jens Axboe05229beed2015-11-05 10:44:55 -0700649static struct queue_sysfs_entry queue_poll_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600650 .attr = {.name = "io_poll", .mode = 0644 },
Jens Axboe05229beed2015-11-05 10:44:55 -0700651 .show = queue_poll_show,
652 .store = queue_poll_store,
653};
654
Jens Axboe06426ad2016-11-14 13:01:59 -0700655static struct queue_sysfs_entry queue_poll_delay_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600656 .attr = {.name = "io_poll_delay", .mode = 0644 },
Jens Axboe06426ad2016-11-14 13:01:59 -0700657 .show = queue_poll_delay_show,
658 .store = queue_poll_delay_store,
659};
660
Jens Axboe93e9d8e2016-04-12 12:32:46 -0600661static struct queue_sysfs_entry queue_wc_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600662 .attr = {.name = "write_cache", .mode = 0644 },
Jens Axboe93e9d8e2016-04-12 12:32:46 -0600663 .show = queue_wc_show,
664 .store = queue_wc_store,
665};
666
Kent Overstreet6fcefbe2018-05-08 21:33:58 -0400667static struct queue_sysfs_entry queue_fua_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600668 .attr = {.name = "fua", .mode = 0444 },
Kent Overstreet6fcefbe2018-05-08 21:33:58 -0400669 .show = queue_fua_show,
670};
671
Yigal Kormanea6ca602016-06-23 17:05:51 -0400672static struct queue_sysfs_entry queue_dax_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600673 .attr = {.name = "dax", .mode = 0444 },
Yigal Kormanea6ca602016-06-23 17:05:51 -0400674 .show = queue_dax_show,
675};
676
Jens Axboe87760e52016-11-09 12:38:14 -0700677static struct queue_sysfs_entry queue_wb_lat_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600678 .attr = {.name = "wbt_lat_usec", .mode = 0644 },
Jens Axboe87760e52016-11-09 12:38:14 -0700679 .show = queue_wb_lat_show,
680 .store = queue_wb_lat_store,
681};
682
Shaohua Li297e3d82017-03-27 10:51:37 -0700683#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
684static struct queue_sysfs_entry throtl_sample_time_entry = {
Joe Perches5657a812018-05-24 13:38:59 -0600685 .attr = {.name = "throttle_sample_time", .mode = 0644 },
Shaohua Li297e3d82017-03-27 10:51:37 -0700686 .show = blk_throtl_sample_time_show,
687 .store = blk_throtl_sample_time_store,
688};
689#endif
690
Jens Axboe8324aa92008-01-29 14:51:59 +0100691static struct attribute *default_attrs[] = {
692 &queue_requests_entry.attr,
693 &queue_ra_entry.attr,
694 &queue_max_hw_sectors_entry.attr,
695 &queue_max_sectors_entry.attr,
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500696 &queue_max_segments_entry.attr,
Christoph Hellwig1e739732017-02-08 14:46:49 +0100697 &queue_max_discard_segments_entry.attr,
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200698 &queue_max_integrity_segments_entry.attr,
Martin K. Petersenc77a5712010-03-10 00:48:33 -0500699 &queue_max_segment_size_entry.attr,
Jens Axboe8324aa92008-01-29 14:51:59 +0100700 &queue_iosched_entry.attr,
Martin K. Petersene68b9032008-01-29 19:14:08 +0100701 &queue_hw_sector_size_entry.attr,
Martin K. Petersene1defc42009-05-22 17:17:49 -0400702 &queue_logical_block_size_entry.attr,
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400703 &queue_physical_block_size_entry.attr,
Hannes Reinecke87caf972016-10-18 15:40:30 +0900704 &queue_chunk_sectors_entry.attr,
Martin K. Petersenc72758f2009-05-22 17:17:53 -0400705 &queue_io_min_entry.attr,
706 &queue_io_opt_entry.attr,
Martin K. Petersen86b37282009-11-10 11:50:21 +0100707 &queue_discard_granularity_entry.attr,
708 &queue_discard_max_entry.attr,
Jens Axboe0034af02015-07-16 09:14:26 -0600709 &queue_discard_max_hw_entry.attr,
Martin K. Petersen98262f22009-12-03 09:24:48 +0100710 &queue_discard_zeroes_data_entry.attr,
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400711 &queue_write_same_max_entry.attr,
Chaitanya Kulkarnia6f07882016-11-30 12:28:59 -0800712 &queue_write_zeroes_max_entry.attr,
Bartlomiej Zolnierkiewicz1308835f2009-01-07 12:22:39 +0100713 &queue_nonrot_entry.attr,
Damien Le Moal797476b2016-10-18 15:40:29 +0900714 &queue_zoned_entry.attr,
Alan D. Brunelleac9fafa2008-04-29 14:44:19 +0200715 &queue_nomerges_entry.attr,
Jens Axboec7c22e42008-09-13 20:26:01 +0200716 &queue_rq_affinity_entry.attr,
Jens Axboebc58ba92009-01-23 10:54:44 +0100717 &queue_iostats_entry.attr,
Jens Axboee2e1a142010-06-09 10:42:09 +0200718 &queue_random_entry.attr,
Jens Axboe05229beed2015-11-05 10:44:55 -0700719 &queue_poll_entry.attr,
Jens Axboe93e9d8e2016-04-12 12:32:46 -0600720 &queue_wc_entry.attr,
Kent Overstreet6fcefbe2018-05-08 21:33:58 -0400721 &queue_fua_entry.attr,
Yigal Kormanea6ca602016-06-23 17:05:51 -0400722 &queue_dax_entry.attr,
Jens Axboe87760e52016-11-09 12:38:14 -0700723 &queue_wb_lat_entry.attr,
Jens Axboe06426ad2016-11-14 13:01:59 -0700724 &queue_poll_delay_entry.attr,
Shaohua Li297e3d82017-03-27 10:51:37 -0700725#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
726 &throtl_sample_time_entry.attr,
727#endif
Jens Axboe8324aa92008-01-29 14:51:59 +0100728 NULL,
729};
730
731#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
732
733static ssize_t
734queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
735{
736 struct queue_sysfs_entry *entry = to_queue(attr);
737 struct request_queue *q =
738 container_of(kobj, struct request_queue, kobj);
739 ssize_t res;
740
741 if (!entry->show)
742 return -EIO;
743 mutex_lock(&q->sysfs_lock);
Bart Van Assche3f3299d2012-11-28 13:42:38 +0100744 if (blk_queue_dying(q)) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100745 mutex_unlock(&q->sysfs_lock);
746 return -ENOENT;
747 }
748 res = entry->show(q, page);
749 mutex_unlock(&q->sysfs_lock);
750 return res;
751}
752
753static ssize_t
754queue_attr_store(struct kobject *kobj, struct attribute *attr,
755 const char *page, size_t length)
756{
757 struct queue_sysfs_entry *entry = to_queue(attr);
Jens Axboe6728cb02008-01-31 13:03:55 +0100758 struct request_queue *q;
Jens Axboe8324aa92008-01-29 14:51:59 +0100759 ssize_t res;
760
761 if (!entry->store)
762 return -EIO;
Jens Axboe6728cb02008-01-31 13:03:55 +0100763
764 q = container_of(kobj, struct request_queue, kobj);
Jens Axboe8324aa92008-01-29 14:51:59 +0100765 mutex_lock(&q->sysfs_lock);
Bart Van Assche3f3299d2012-11-28 13:42:38 +0100766 if (blk_queue_dying(q)) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100767 mutex_unlock(&q->sysfs_lock);
768 return -ENOENT;
769 }
770 res = entry->store(q, page, length);
771 mutex_unlock(&q->sysfs_lock);
772 return res;
773}
774
Tejun Heo548bc8e2013-01-09 08:05:13 -0800775static void blk_free_queue_rcu(struct rcu_head *rcu_head)
776{
777 struct request_queue *q = container_of(rcu_head, struct request_queue,
778 rcu_head);
779 kmem_cache_free(blk_requestq_cachep, q);
780}
781
Jens Axboe8324aa92008-01-29 14:51:59 +0100782/**
Bart Van Asschedc9edc42017-06-14 13:27:50 -0600783 * __blk_release_queue - release a request queue when it is no longer needed
784 * @work: pointer to the release_work member of the request queue to be released
Jens Axboe8324aa92008-01-29 14:51:59 +0100785 *
786 * Description:
Bart Van Asschedc9edc42017-06-14 13:27:50 -0600787 * blk_release_queue is the counterpart of blk_init_queue(). It should be
788 * called when a request queue is being released; typically when a block
789 * device is being de-registered. Its primary task it to free the queue
790 * itself.
Jens Axboe8324aa92008-01-29 14:51:59 +0100791 *
Bart Van Asschedc9edc42017-06-14 13:27:50 -0600792 * Notes:
Bart Van Assche45a9c9d2014-12-09 16:57:48 +0100793 * The low level driver must have finished any outstanding requests first
794 * via blk_cleanup_queue().
Bart Van Asschedc9edc42017-06-14 13:27:50 -0600795 *
796 * Although blk_release_queue() may be called with preemption disabled,
797 * __blk_release_queue() may sleep.
798 */
799static void __blk_release_queue(struct work_struct *work)
Jens Axboe8324aa92008-01-29 14:51:59 +0100800{
Bart Van Asschedc9edc42017-06-14 13:27:50 -0600801 struct request_queue *q = container_of(work, typeof(*q), release_work);
Jens Axboe8324aa92008-01-29 14:51:59 +0100802
Omar Sandoval34dbad52017-03-21 08:56:08 -0700803 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
804 blk_stat_remove_callback(q, q->poll_cb);
805 blk_stat_free_callback(q->poll_cb);
Hannes Reinecke777eb1b2011-09-28 08:07:01 -0600806
Omar Sandoval34dbad52017-03-21 08:56:08 -0700807 blk_free_queue_stats(q->stats);
808
Bart Van Asscheb425e502017-05-31 14:43:45 -0700809 blk_exit_rl(q, &q->root_rl);
Jens Axboe8324aa92008-01-29 14:51:59 +0100810
811 if (q->queue_tags)
812 __blk_queue_free_tags(q);
813
Christoph Hellwig6d247d72017-01-27 09:51:45 -0700814 if (!q->mq_ops) {
815 if (q->exit_rq_fn)
816 q->exit_rq_fn(q, q->fq->flush_rq);
Ming Leif70ced02014-09-25 23:23:47 +0800817 blk_free_flush_queue(q->fq);
Christoph Hellwig6d247d72017-01-27 09:51:45 -0700818 } else {
Ming Leie09aae72015-01-29 20:17:27 +0800819 blk_mq_release(q);
Christoph Hellwig6d247d72017-01-27 09:51:45 -0700820 }
Christoph Hellwig18741982014-02-10 09:29:00 -0700821
Jens Axboe8324aa92008-01-29 14:51:59 +0100822 blk_trace_shutdown(q);
823
Omar Sandoval62ebce162017-01-31 14:53:21 -0800824 if (q->mq_ops)
825 blk_mq_debugfs_unregister(q);
826
Kent Overstreet338aa962018-05-20 18:25:47 -0400827 bioset_exit(&q->bio_split);
Kent Overstreet54efd502015-04-23 22:37:18 -0700828
Tejun Heoa73f7302011-12-14 00:33:37 +0100829 ida_simple_remove(&blk_queue_ida, q->id);
Tejun Heo548bc8e2013-01-09 08:05:13 -0800830 call_rcu(&q->rcu_head, blk_free_queue_rcu);
Jens Axboe8324aa92008-01-29 14:51:59 +0100831}
832
Bart Van Asschedc9edc42017-06-14 13:27:50 -0600833static void blk_release_queue(struct kobject *kobj)
834{
835 struct request_queue *q =
836 container_of(kobj, struct request_queue, kobj);
837
838 INIT_WORK(&q->release_work, __blk_release_queue);
839 schedule_work(&q->release_work);
840}
841
Emese Revfy52cf25d2010-01-19 02:58:23 +0100842static const struct sysfs_ops queue_sysfs_ops = {
Jens Axboe8324aa92008-01-29 14:51:59 +0100843 .show = queue_attr_show,
844 .store = queue_attr_store,
845};
846
847struct kobj_type blk_queue_ktype = {
848 .sysfs_ops = &queue_sysfs_ops,
849 .default_attrs = default_attrs,
850 .release = blk_release_queue,
851};
852
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800853/**
854 * blk_register_queue - register a block layer queue with sysfs
855 * @disk: Disk of which the request queue should be registered with sysfs.
856 */
Jens Axboe8324aa92008-01-29 14:51:59 +0100857int blk_register_queue(struct gendisk *disk)
858{
859 int ret;
Li Zefan1d54ad62009-04-14 14:00:05 +0800860 struct device *dev = disk_to_dev(disk);
Jens Axboe8324aa92008-01-29 14:51:59 +0100861 struct request_queue *q = disk->queue;
862
Akinobu Mitafb199742008-04-21 09:51:06 +0200863 if (WARN_ON(!q))
Jens Axboe8324aa92008-01-29 14:51:59 +0100864 return -ENXIO;
865
Omar Sandoval334335d2017-03-28 16:12:15 -0700866 WARN_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags),
867 "%s is registering an already registered queue\n",
868 kobject_name(&dev->kobj));
869 queue_flag_set_unlocked(QUEUE_FLAG_REGISTERED, q);
870
Tejun Heo749fefe2012-09-20 14:08:52 -0700871 /*
Tejun Heo17497ac2014-09-24 13:31:50 -0400872 * SCSI probing may synchronously create and destroy a lot of
873 * request_queues for non-existent devices. Shutting down a fully
874 * functional queue takes measureable wallclock time as RCU grace
875 * periods are involved. To avoid excessive latency in these
876 * cases, a request_queue starts out in a degraded mode which is
877 * faster to shut down and is made fully functional here as
878 * request_queues for non-existent devices never get registered.
Tejun Heo749fefe2012-09-20 14:08:52 -0700879 */
Alan Sterndf35c7c2014-09-09 11:50:58 -0400880 if (!blk_queue_init_done(q)) {
881 queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q);
Dan Williams3ef28e82015-10-21 13:20:12 -0400882 percpu_ref_switch_to_percpu(&q->q_usage_counter);
Alan Sterndf35c7c2014-09-09 11:50:58 -0400883 blk_queue_bypass_end(q);
884 }
Tejun Heo749fefe2012-09-20 14:08:52 -0700885
Li Zefan1d54ad62009-04-14 14:00:05 +0800886 ret = blk_trace_init_sysfs(dev);
887 if (ret)
888 return ret;
889
Tahsin Erdoganb410aff2017-02-14 19:27:38 -0800890 /* Prevent changes through sysfs until registration is completed. */
891 mutex_lock(&q->sysfs_lock);
892
Linus Torvaldsc9059592009-06-11 10:52:27 -0700893 ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
Liu Yuaned5302d2011-04-19 13:47:58 +0200894 if (ret < 0) {
895 blk_trace_remove_sysfs(dev);
Tahsin Erdoganb410aff2017-02-14 19:27:38 -0800896 goto unlock;
Liu Yuaned5302d2011-04-19 13:47:58 +0200897 }
Jens Axboe8324aa92008-01-29 14:51:59 +0100898
Bart Van Asschea8ecdd72017-05-25 16:38:06 -0700899 if (q->mq_ops) {
Bart Van Assche2d0364c2017-04-26 13:47:48 -0700900 __blk_mq_register_dev(dev, q);
Bart Van Asschea8ecdd72017-05-25 16:38:06 -0700901 blk_mq_debugfs_register(q);
902 }
Omar Sandoval9c1051a2017-05-04 08:17:21 -0600903
Jens Axboe8324aa92008-01-29 14:51:59 +0100904 kobject_uevent(&q->kobj, KOBJ_ADD);
905
Jan Kara8330cdb2017-04-19 11:33:27 +0200906 wbt_enable_default(q);
Jens Axboe87760e52016-11-09 12:38:14 -0700907
Shaohua Lid61fcfa2017-03-27 10:51:38 -0700908 blk_throtl_register_queue(q);
909
Omar Sandoval80c6b152017-02-06 12:52:24 -0800910 if (q->request_fn || (q->mq_ops && q->elevator)) {
911 ret = elv_register_queue(q);
912 if (ret) {
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800913 mutex_unlock(&q->sysfs_lock);
Omar Sandoval80c6b152017-02-06 12:52:24 -0800914 kobject_uevent(&q->kobj, KOBJ_REMOVE);
915 kobject_del(&q->kobj);
916 blk_trace_remove_sysfs(dev);
917 kobject_put(&dev->kobj);
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800918 return ret;
Omar Sandoval80c6b152017-02-06 12:52:24 -0800919 }
Jens Axboe8324aa92008-01-29 14:51:59 +0100920 }
Tahsin Erdoganb410aff2017-02-14 19:27:38 -0800921 ret = 0;
922unlock:
923 mutex_unlock(&q->sysfs_lock);
924 return ret;
Jens Axboe8324aa92008-01-29 14:51:59 +0100925}
Mike Snitzerfa70d2e2018-01-08 22:01:13 -0500926EXPORT_SYMBOL_GPL(blk_register_queue);
Jens Axboe8324aa92008-01-29 14:51:59 +0100927
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800928/**
929 * blk_unregister_queue - counterpart of blk_register_queue()
930 * @disk: Disk of which the request queue should be unregistered from sysfs.
931 *
932 * Note: the caller is responsible for guaranteeing that this function is called
933 * after blk_register_queue() has finished.
934 */
Jens Axboe8324aa92008-01-29 14:51:59 +0100935void blk_unregister_queue(struct gendisk *disk)
936{
937 struct request_queue *q = disk->queue;
938
Akinobu Mitafb199742008-04-21 09:51:06 +0200939 if (WARN_ON(!q))
940 return;
941
Mike Snitzerfa70d2e2018-01-08 22:01:13 -0500942 /* Return early if disk->queue was never registered. */
943 if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
944 return;
945
Mike Snitzer667257e2018-01-11 14:11:01 -0500946 /*
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800947 * Since sysfs_remove_dir() prevents adding new directory entries
948 * before removal of existing entries starts, protect against
949 * concurrent elv_iosched_store() calls.
Mike Snitzer667257e2018-01-11 14:11:01 -0500950 */
David Jefferye9a823f2017-08-28 10:52:44 -0600951 mutex_lock(&q->sysfs_lock);
Mike Snitzer667257e2018-01-11 14:11:01 -0500952
Bart Van Assche8814ce82018-03-07 17:10:04 -0800953 blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
Omar Sandoval334335d2017-03-28 16:12:15 -0700954
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800955 /*
956 * Remove the sysfs attributes before unregistering the queue data
957 * structures that can be modified through sysfs.
958 */
Jens Axboe320ae512013-10-24 09:20:05 +0100959 if (q->mq_ops)
Matias Bjørlingb21d5b32016-09-16 14:25:06 +0200960 blk_mq_unregister_dev(disk_to_dev(disk), q);
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800961 mutex_unlock(&q->sysfs_lock);
Jens Axboe8324aa92008-01-29 14:51:59 +0100962
Zdenek Kabelac48c0d4d2009-09-25 06:19:26 +0200963 kobject_uevent(&q->kobj, KOBJ_REMOVE);
964 kobject_del(&q->kobj);
965 blk_trace_remove_sysfs(disk_to_dev(disk));
Mike Snitzer667257e2018-01-11 14:11:01 -0500966
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800967 wbt_exit(q);
968
969 mutex_lock(&q->sysfs_lock);
970 if (q->request_fn || (q->mq_ops && q->elevator))
971 elv_unregister_queue(q);
Mike Snitzer667257e2018-01-11 14:11:01 -0500972 mutex_unlock(&q->sysfs_lock);
Bart Van Assche2c2086a2018-01-17 11:48:10 -0800973
974 kobject_put(&disk_to_dev(disk)->kobj);
Jens Axboe8324aa92008-01-29 14:51:59 +0100975}