blob: 567b9ed1056d7b77e26918d2c60b3a21e4cc3f56 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Milan Broz784aae72009-01-06 03:05:12 +00003 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
Mike Snitzer4cc96132016-05-12 16:28:10 -04008#include "dm-core.h"
9#include "dm-rq.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +010010#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12#include <linux/init.h>
13#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080014#include <linux/mutex.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010015#include <linux/sched/signal.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/blkpg.h>
17#include <linux/bio.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/mempool.h>
Dan Williamsf26c5712017-04-12 12:35:44 -070019#include <linux/dax.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/slab.h>
21#include <linux/idr.h>
Dan Williams7e026c82017-05-29 12:57:56 -070022#include <linux/uio.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080023#include <linux/hdreg.h>
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +010024#include <linux/delay.h>
Mike Snitzerffcc3932014-10-28 18:34:52 -040025#include <linux/wait.h>
Christoph Hellwig71cdb692015-10-15 14:10:51 +020026#include <linux/pr.h>
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +030027#include <linux/refcount.h>
Li Zefan55782132009-06-09 13:43:05 +080028
Alasdair G Kergon72d94862006-06-26 00:27:35 -070029#define DM_MSG_PREFIX "core"
30
Milan Broz60935eb2009-06-22 10:12:30 +010031/*
32 * Cookies are numeric values sent with CHANGE and REMOVE
33 * uevents while resuming, removing or renaming the device.
34 */
35#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
36#define DM_COOKIE_LENGTH 24
37
Linus Torvalds1da177e2005-04-16 15:20:36 -070038static const char *_name = DM_NAME;
39
40static unsigned int major = 0;
41static unsigned int _major = 0;
42
Alasdair G Kergond15b7742011-08-02 12:32:01 +010043static DEFINE_IDR(_minor_idr);
44
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070045static DEFINE_SPINLOCK(_minor_lock);
Mikulas Patocka2c140a22013-11-01 18:27:41 -040046
47static void do_deferred_remove(struct work_struct *w);
48
49static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
50
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -040051static struct workqueue_struct *deferred_remove_workqueue;
52
Mikulas Patocka93e64422017-01-16 16:05:59 -050053atomic_t dm_global_event_nr = ATOMIC_INIT(0);
54DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
55
Mikulas Patocka62e08242017-09-20 07:29:49 -040056void dm_issue_global_event(void)
57{
58 atomic_inc(&dm_global_event_nr);
59 wake_up(&dm_global_eventq);
60}
61
Linus Torvalds1da177e2005-04-16 15:20:36 -070062/*
63 * One of these is allocated per bio.
64 */
65struct dm_io {
66 struct mapped_device *md;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +020067 blk_status_t status;
Linus Torvalds1da177e2005-04-16 15:20:36 -070068 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010069 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080070 unsigned long start_time;
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +010071 spinlock_t endio_lock;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -040072 struct dm_stats_aux stats_aux;
Linus Torvalds1da177e2005-04-16 15:20:36 -070073};
74
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070075#define MINOR_ALLOCED ((void *)-1)
76
Linus Torvalds1da177e2005-04-16 15:20:36 -070077/*
78 * Bits for the md->flags field.
79 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +010080#define DMF_BLOCK_IO_FOR_SUSPEND 0
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080082#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070083#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070084#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080085#define DMF_NOFLUSH_SUSPENDING 5
Kent Overstreet8ae12662015-04-27 23:48:34 -070086#define DMF_DEFERRED_REMOVE 6
87#define DMF_SUSPENDED_INTERNALLY 7
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Mike Snitzer115485e2016-02-22 12:16:21 -050089#define DM_NUMA_NODE NUMA_NO_NODE
Mike Snitzer115485e2016-02-22 12:16:21 -050090static int dm_numa_node = DM_NUMA_NODE;
Mike Snitzerfaad87d2016-01-28 16:52:56 -050091
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +010092/*
93 * For mempools pre-allocation at the table loading time.
94 */
95struct dm_md_mempools {
96 mempool_t *io_pool;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +010097 struct bio_set *bs;
98};
99
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500100struct table_device {
101 struct list_head list;
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300102 refcount_t count;
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500103 struct dm_dev dm_dev;
104};
105
Christoph Lametere18b8902006-12-06 20:33:20 -0800106static struct kmem_cache *_io_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000107static struct kmem_cache *_rq_tio_cache;
Mike Snitzer1ae49ea2014-12-05 17:11:05 -0500108static struct kmem_cache *_rq_cache;
Kent Overstreet94818742012-09-07 13:44:01 -0700109
Mike Snitzerf4790822013-09-12 18:06:12 -0400110/*
Mike Snitzere8603132013-09-12 18:06:12 -0400111 * Bio-based DM's mempools' reserved IOs set by the user.
112 */
Mike Snitzer4cc96132016-05-12 16:28:10 -0400113#define RESERVED_BIO_BASED_IOS 16
Mike Snitzere8603132013-09-12 18:06:12 -0400114static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
115
Mike Snitzer115485e2016-02-22 12:16:21 -0500116static int __dm_get_module_param_int(int *module_param, int min, int max)
117{
118 int param = ACCESS_ONCE(*module_param);
119 int modified_param = 0;
120 bool modified = true;
121
122 if (param < min)
123 modified_param = min;
124 else if (param > max)
125 modified_param = max;
126 else
127 modified = false;
128
129 if (modified) {
130 (void)cmpxchg(module_param, param, modified_param);
131 param = modified_param;
132 }
133
134 return param;
135}
136
Mike Snitzer4cc96132016-05-12 16:28:10 -0400137unsigned __dm_get_module_param(unsigned *module_param,
138 unsigned def, unsigned max)
Mike Snitzerf4790822013-09-12 18:06:12 -0400139{
Mike Snitzer09c2d532015-02-27 22:25:26 -0500140 unsigned param = ACCESS_ONCE(*module_param);
141 unsigned modified_param = 0;
Mike Snitzerf4790822013-09-12 18:06:12 -0400142
Mike Snitzer09c2d532015-02-27 22:25:26 -0500143 if (!param)
144 modified_param = def;
145 else if (param > max)
146 modified_param = max;
Mike Snitzerf4790822013-09-12 18:06:12 -0400147
Mike Snitzer09c2d532015-02-27 22:25:26 -0500148 if (modified_param) {
149 (void)cmpxchg(module_param, param, modified_param);
150 param = modified_param;
Mike Snitzerf4790822013-09-12 18:06:12 -0400151 }
152
Mike Snitzer09c2d532015-02-27 22:25:26 -0500153 return param;
Mike Snitzerf4790822013-09-12 18:06:12 -0400154}
155
Mike Snitzere8603132013-09-12 18:06:12 -0400156unsigned dm_get_reserved_bio_based_ios(void)
157{
Mike Snitzer09c2d532015-02-27 22:25:26 -0500158 return __dm_get_module_param(&reserved_bio_based_ios,
Mike Snitzer4cc96132016-05-12 16:28:10 -0400159 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
Mike Snitzere8603132013-09-12 18:06:12 -0400160}
161EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
162
Mike Snitzer115485e2016-02-22 12:16:21 -0500163static unsigned dm_get_numa_node(void)
164{
165 return __dm_get_module_param_int(&dm_numa_node,
166 DM_NUMA_NODE, num_online_nodes() - 1);
167}
168
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169static int __init local_init(void)
170{
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100171 int r = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100174 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 if (!_io_cache)
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100176 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000178 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
179 if (!_rq_tio_cache)
Mikulas Patockadba14162012-10-12 21:02:15 +0100180 goto out_free_io_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000181
Mike Snitzereca7ee62016-02-20 13:45:38 -0500182 _rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
Mike Snitzer1ae49ea2014-12-05 17:11:05 -0500183 __alignof__(struct request), 0, NULL);
184 if (!_rq_cache)
185 goto out_free_rq_tio_cache;
186
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100187 r = dm_uevent_init();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100188 if (r)
Mike Snitzer1ae49ea2014-12-05 17:11:05 -0500189 goto out_free_rq_cache;
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100190
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400191 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
192 if (!deferred_remove_workqueue) {
193 r = -ENOMEM;
194 goto out_uevent_exit;
195 }
196
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 _major = major;
198 r = register_blkdev(_major, _name);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100199 if (r < 0)
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400200 goto out_free_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201
202 if (!_major)
203 _major = r;
204
205 return 0;
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100206
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400207out_free_workqueue:
208 destroy_workqueue(deferred_remove_workqueue);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100209out_uevent_exit:
210 dm_uevent_exit();
Mike Snitzer1ae49ea2014-12-05 17:11:05 -0500211out_free_rq_cache:
212 kmem_cache_destroy(_rq_cache);
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000213out_free_rq_tio_cache:
214 kmem_cache_destroy(_rq_tio_cache);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100215out_free_io_cache:
216 kmem_cache_destroy(_io_cache);
217
218 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219}
220
221static void local_exit(void)
222{
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400223 flush_scheduled_work();
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400224 destroy_workqueue(deferred_remove_workqueue);
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400225
Mike Snitzer1ae49ea2014-12-05 17:11:05 -0500226 kmem_cache_destroy(_rq_cache);
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000227 kmem_cache_destroy(_rq_tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700229 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100230 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231
232 _major = 0;
233
234 DMINFO("cleaned up");
235}
236
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000237static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 local_init,
239 dm_target_init,
240 dm_linear_init,
241 dm_stripe_init,
Mikulas Patocka952b3552009-12-10 23:51:57 +0000242 dm_io_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100243 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 dm_interface_init,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400245 dm_statistics_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246};
247
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000248static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 local_exit,
250 dm_target_exit,
251 dm_linear_exit,
252 dm_stripe_exit,
Mikulas Patocka952b3552009-12-10 23:51:57 +0000253 dm_io_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100254 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 dm_interface_exit,
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400256 dm_statistics_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257};
258
259static int __init dm_init(void)
260{
261 const int count = ARRAY_SIZE(_inits);
262
263 int r, i;
264
265 for (i = 0; i < count; i++) {
266 r = _inits[i]();
267 if (r)
268 goto bad;
269 }
270
271 return 0;
272
273 bad:
274 while (i--)
275 _exits[i]();
276
277 return r;
278}
279
280static void __exit dm_exit(void)
281{
282 int i = ARRAY_SIZE(_exits);
283
284 while (i--)
285 _exits[i]();
Alasdair G Kergond15b7742011-08-02 12:32:01 +0100286
287 /*
288 * Should be empty by this point.
289 */
Alasdair G Kergond15b7742011-08-02 12:32:01 +0100290 idr_destroy(&_minor_idr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291}
292
293/*
294 * Block device functions
295 */
Mike Anderson432a2122009-12-10 23:52:20 +0000296int dm_deleting_md(struct mapped_device *md)
297{
298 return test_bit(DMF_DELETING, &md->flags);
299}
300
Al Virofe5f9f22008-03-02 10:29:31 -0500301static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302{
303 struct mapped_device *md;
304
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700305 spin_lock(&_minor_lock);
306
Al Virofe5f9f22008-03-02 10:29:31 -0500307 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700308 if (!md)
309 goto out;
310
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700311 if (test_bit(DMF_FREEING, &md->flags) ||
Mike Anderson432a2122009-12-10 23:52:20 +0000312 dm_deleting_md(md)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700313 md = NULL;
314 goto out;
315 }
316
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700318 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700319out:
320 spin_unlock(&_minor_lock);
321
322 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323}
324
Al Virodb2a1442013-05-05 21:52:57 -0400325static void dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326{
Mike Snitzer63a4f062015-03-23 17:01:43 -0400327 struct mapped_device *md;
Arnd Bergmann6e9624b2010-08-07 18:25:34 +0200328
Milan Broz4a1aeb92011-01-13 19:59:48 +0000329 spin_lock(&_minor_lock);
330
Mike Snitzer63a4f062015-03-23 17:01:43 -0400331 md = disk->private_data;
332 if (WARN_ON(!md))
333 goto out;
334
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400335 if (atomic_dec_and_test(&md->open_count) &&
336 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
Mikulas Patockaacfe0ad2014-06-14 13:44:31 -0400337 queue_work(deferred_remove_workqueue, &deferred_remove_work);
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400338
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 dm_put(md);
Mike Snitzer63a4f062015-03-23 17:01:43 -0400340out:
Milan Broz4a1aeb92011-01-13 19:59:48 +0000341 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342}
343
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700344int dm_open_count(struct mapped_device *md)
345{
346 return atomic_read(&md->open_count);
347}
348
349/*
350 * Guarantees nothing is using the device before it's deleted.
351 */
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400352int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700353{
354 int r = 0;
355
356 spin_lock(&_minor_lock);
357
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400358 if (dm_open_count(md)) {
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700359 r = -EBUSY;
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400360 if (mark_deferred)
361 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
362 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
363 r = -EEXIST;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700364 else
365 set_bit(DMF_DELETING, &md->flags);
366
367 spin_unlock(&_minor_lock);
368
369 return r;
370}
371
Mikulas Patocka2c140a22013-11-01 18:27:41 -0400372int dm_cancel_deferred_remove(struct mapped_device *md)
373{
374 int r = 0;
375
376 spin_lock(&_minor_lock);
377
378 if (test_bit(DMF_DELETING, &md->flags))
379 r = -EBUSY;
380 else
381 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
382
383 spin_unlock(&_minor_lock);
384
385 return r;
386}
387
388static void do_deferred_remove(struct work_struct *w)
389{
390 dm_deferred_remove();
391}
392
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400393sector_t dm_get_size(struct mapped_device *md)
394{
395 return get_capacity(md->disk);
396}
397
Mike Snitzer9974fa22014-02-28 15:33:43 +0100398struct request_queue *dm_get_md_queue(struct mapped_device *md)
399{
400 return md->queue;
401}
402
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400403struct dm_stats *dm_get_stats(struct mapped_device *md)
404{
405 return &md->stats;
406}
407
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800408static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
409{
410 struct mapped_device *md = bdev->bd_disk->private_data;
411
412 return dm_get_geometry(md, geo);
413}
414
Mike Snitzer956a4022016-02-18 16:13:51 -0500415static int dm_grab_bdev_for_ioctl(struct mapped_device *md,
416 struct block_device **bdev,
417 fmode_t *mode)
Milan Brozaa129a22006-10-03 01:15:15 -0700418{
Mike Snitzer66482022016-02-18 15:44:39 -0500419 struct dm_target *tgt;
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100420 struct dm_table *map;
Mike Snitzer956a4022016-02-18 16:13:51 -0500421 int srcu_idx, r;
Milan Brozaa129a22006-10-03 01:15:15 -0700422
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100423retry:
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200424 r = -ENOTTY;
Mike Snitzer956a4022016-02-18 16:13:51 -0500425 map = dm_get_live_table(md, &srcu_idx);
Milan Brozaa129a22006-10-03 01:15:15 -0700426 if (!map || !dm_table_get_size(map))
427 goto out;
428
429 /* We only support devices that have a single target */
430 if (dm_table_get_num_targets(map) != 1)
431 goto out;
432
Mike Snitzer66482022016-02-18 15:44:39 -0500433 tgt = dm_table_get_target(map, 0);
434 if (!tgt->type->prepare_ioctl)
Mike Snitzer4d341d82014-11-16 14:21:47 -0500435 goto out;
Milan Brozaa129a22006-10-03 01:15:15 -0700436
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +0000437 if (dm_suspended_md(md)) {
Milan Brozaa129a22006-10-03 01:15:15 -0700438 r = -EAGAIN;
439 goto out;
440 }
441
Mike Snitzer66482022016-02-18 15:44:39 -0500442 r = tgt->type->prepare_ioctl(tgt, bdev, mode);
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200443 if (r < 0)
444 goto out;
445
Mike Snitzer956a4022016-02-18 16:13:51 -0500446 bdgrab(*bdev);
447 dm_put_live_table(md, srcu_idx);
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200448 return r;
Milan Brozaa129a22006-10-03 01:15:15 -0700449
450out:
Mike Snitzer956a4022016-02-18 16:13:51 -0500451 dm_put_live_table(md, srcu_idx);
Junichi Nomura5bbbfdf2015-11-17 09:39:26 +0000452 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100453 msleep(10);
454 goto retry;
455 }
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200456 return r;
457}
Hannes Reinecke6c182cd2013-07-10 23:41:15 +0100458
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200459static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
460 unsigned int cmd, unsigned long arg)
461{
462 struct mapped_device *md = bdev->bd_disk->private_data;
Mike Snitzer956a4022016-02-18 16:13:51 -0500463 int r;
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200464
Mike Snitzer956a4022016-02-18 16:13:51 -0500465 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200466 if (r < 0)
467 return r;
468
469 if (r > 0) {
470 /*
Christoph Hellwige980f622017-02-04 10:45:03 +0100471 * Target determined this ioctl is being issued against a
472 * subset of the parent bdev; require extra privileges.
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200473 */
Christoph Hellwige980f622017-02-04 10:45:03 +0100474 if (!capable(CAP_SYS_RAWIO)) {
475 DMWARN_LIMIT(
476 "%s: sending ioctl %x to DM device without required privilege.",
477 current->comm, cmd);
478 r = -ENOIOCTLCMD;
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200479 goto out;
Christoph Hellwige980f622017-02-04 10:45:03 +0100480 }
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200481 }
482
Mike Snitzer66482022016-02-18 15:44:39 -0500483 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg);
Christoph Hellwige56f81e2015-10-15 14:10:50 +0200484out:
Mike Snitzer956a4022016-02-18 16:13:51 -0500485 bdput(bdev);
Milan Brozaa129a22006-10-03 01:15:15 -0700486 return r;
487}
488
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100489static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490{
491 return mempool_alloc(md->io_pool, GFP_NOIO);
492}
493
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100494static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495{
496 mempool_free(io, md->io_pool);
497}
498
Mike Snitzercfae7522016-04-11 12:05:38 -0400499static void free_tio(struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500{
Mikulas Patockadba14162012-10-12 21:02:15 +0100501 bio_put(&tio->clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502}
503
Mike Snitzer4cc96132016-05-12 16:28:10 -0400504int md_in_flight(struct mapped_device *md)
Kiyoshi Ueda90abb8c2009-12-10 23:52:13 +0000505{
506 return atomic_read(&md->pending[READ]) +
507 atomic_read(&md->pending[WRITE]);
508}
509
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800510static void start_io_acct(struct dm_io *io)
511{
512 struct mapped_device *md = io->md;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400513 struct bio *bio = io->bio;
Tejun Heoc9959052008-08-25 19:47:21 +0900514 int cpu;
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400515 int rw = bio_data_dir(bio);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800516
517 io->start_time = jiffies;
518
Tejun Heo074a7ac2008-08-25 19:56:14 +0900519 cpu = part_stat_lock();
Jens Axboed62e26b2017-06-30 21:55:08 -0600520 part_round_stats(md->queue, cpu, &dm_disk(md)->part0);
Tejun Heo074a7ac2008-08-25 19:56:14 +0900521 part_stat_unlock();
Shaohua Li1e9bb882011-03-22 08:35:35 +0100522 atomic_set(&dm_disk(md)->part0.in_flight[rw],
523 atomic_inc_return(&md->pending[rw]));
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400524
525 if (unlikely(dm_stats_used(&md->stats)))
Mike Christie528ec5a2016-06-05 14:32:03 -0500526 dm_stats_account_io(&md->stats, bio_data_dir(bio),
527 bio->bi_iter.bi_sector, bio_sectors(bio),
528 false, 0, &io->stats_aux);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800529}
530
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000531static void end_io_acct(struct dm_io *io)
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800532{
533 struct mapped_device *md = io->md;
534 struct bio *bio = io->bio;
535 unsigned long duration = jiffies - io->start_time;
Gu Zheng18c0b222014-11-24 11:05:26 +0800536 int pending;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800537 int rw = bio_data_dir(bio);
538
Jens Axboed62e26b2017-06-30 21:55:08 -0600539 generic_end_io_acct(md->queue, rw, &dm_disk(md)->part0, io->start_time);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800540
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400541 if (unlikely(dm_stats_used(&md->stats)))
Mike Christie528ec5a2016-06-05 14:32:03 -0500542 dm_stats_account_io(&md->stats, bio_data_dir(bio),
543 bio->bi_iter.bi_sector, bio_sectors(bio),
544 true, duration, &io->stats_aux);
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -0400545
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100546 /*
547 * After this is decremented the bio must not be touched if it is
Tejun Heod87f4c12010-09-03 11:56:19 +0200548 * a flush.
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100549 */
Shaohua Li1e9bb882011-03-22 08:35:35 +0100550 pending = atomic_dec_return(&md->pending[rw]);
551 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
Nikanth Karthikesan316d3152009-10-06 20:16:55 +0200552 pending += atomic_read(&md->pending[rw^0x1]);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800553
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000554 /* nudge anyone waiting on suspend queue */
555 if (!pending)
556 wake_up(&md->wait);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800557}
558
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559/*
560 * Add the bio to the list of deferred io.
561 */
Mikulas Patocka92c63902009-04-09 00:27:15 +0100562static void queue_io(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563{
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200564 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200566 spin_lock_irqsave(&md->deferred_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 bio_list_add(&md->deferred, bio);
Kiyoshi Ueda054474202010-09-08 18:07:01 +0200568 spin_unlock_irqrestore(&md->deferred_lock, flags);
Tejun Heo6a8736d2010-09-08 18:07:00 +0200569 queue_work(md->wq, &md->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570}
571
572/*
573 * Everyone (including functions in this file), should use this
574 * function to access the md->map field, and make sure they call
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100575 * dm_put_live_table() when finished.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 */
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100577struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578{
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100579 *srcu_idx = srcu_read_lock(&md->io_barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100581 return srcu_dereference(md->map, &md->io_barrier);
582}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +0100584void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
585{
586 srcu_read_unlock(&md->io_barrier, srcu_idx);
587}
588
589void dm_sync_table(struct mapped_device *md)
590{
591 synchronize_srcu(&md->io_barrier);
592 synchronize_rcu_expedited();
593}
594
595/*
596 * A fast alternative to dm_get_live_table/dm_put_live_table.
597 * The caller must not block between these two functions.
598 */
599static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
600{
601 rcu_read_lock();
602 return rcu_dereference(md->map);
603}
604
605static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
606{
607 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608}
609
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800610/*
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500611 * Open a table device so we can use it as a map destination.
612 */
613static int open_table_device(struct table_device *td, dev_t dev,
614 struct mapped_device *md)
615{
616 static char *_claim_ptr = "I belong to device-mapper";
617 struct block_device *bdev;
618
619 int r;
620
621 BUG_ON(td->dm_dev.bdev);
622
623 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
624 if (IS_ERR(bdev))
625 return PTR_ERR(bdev);
626
627 r = bd_link_disk_holder(bdev, dm_disk(md));
628 if (r) {
629 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
630 return r;
631 }
632
633 td->dm_dev.bdev = bdev;
Dan Williams817bf402017-04-12 13:37:44 -0700634 td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500635 return 0;
636}
637
638/*
639 * Close a table device that we've been using.
640 */
641static void close_table_device(struct table_device *td, struct mapped_device *md)
642{
643 if (!td->dm_dev.bdev)
644 return;
645
646 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
647 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
Dan Williams817bf402017-04-12 13:37:44 -0700648 put_dax(td->dm_dev.dax_dev);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500649 td->dm_dev.bdev = NULL;
Dan Williams817bf402017-04-12 13:37:44 -0700650 td->dm_dev.dax_dev = NULL;
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500651}
652
653static struct table_device *find_table_device(struct list_head *l, dev_t dev,
654 fmode_t mode) {
655 struct table_device *td;
656
657 list_for_each_entry(td, l, list)
658 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
659 return td;
660
661 return NULL;
662}
663
664int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
665 struct dm_dev **result) {
666 int r;
667 struct table_device *td;
668
669 mutex_lock(&md->table_devices_lock);
670 td = find_table_device(&md->table_devices, dev, mode);
671 if (!td) {
Mike Snitzer115485e2016-02-22 12:16:21 -0500672 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500673 if (!td) {
674 mutex_unlock(&md->table_devices_lock);
675 return -ENOMEM;
676 }
677
678 td->dm_dev.mode = mode;
679 td->dm_dev.bdev = NULL;
680
681 if ((r = open_table_device(td, dev, md))) {
682 mutex_unlock(&md->table_devices_lock);
683 kfree(td);
684 return r;
685 }
686
687 format_dev_t(td->dm_dev.name, dev);
688
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300689 refcount_set(&td->count, 1);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500690 list_add(&td->list, &md->table_devices);
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300691 } else {
692 refcount_inc(&td->count);
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500693 }
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500694 mutex_unlock(&md->table_devices_lock);
695
696 *result = &td->dm_dev;
697 return 0;
698}
699EXPORT_SYMBOL_GPL(dm_get_table_device);
700
701void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
702{
703 struct table_device *td = container_of(d, struct table_device, dm_dev);
704
705 mutex_lock(&md->table_devices_lock);
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300706 if (refcount_dec_and_test(&td->count)) {
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500707 close_table_device(td, md);
708 list_del(&td->list);
709 kfree(td);
710 }
711 mutex_unlock(&md->table_devices_lock);
712}
713EXPORT_SYMBOL(dm_put_table_device);
714
715static void free_table_devices(struct list_head *devices)
716{
717 struct list_head *tmp, *next;
718
719 list_for_each_safe(tmp, next, devices) {
720 struct table_device *td = list_entry(tmp, struct table_device, list);
721
722 DMWARN("dm_destroy: %s still exists with %d references",
Elena Reshetovab0b4d7c2017-10-20 10:37:39 +0300723 td->dm_dev.name, refcount_read(&td->count));
Benjamin Marzinski86f11522014-08-13 13:53:43 -0500724 kfree(td);
725 }
726}
727
728/*
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800729 * Get the geometry associated with a dm device
730 */
731int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
732{
733 *geo = md->geometry;
734
735 return 0;
736}
737
738/*
739 * Set the geometry of a device.
740 */
741int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
742{
743 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
744
745 if (geo->start > sz) {
746 DMWARN("Start sector is beyond the geometry limits.");
747 return -EINVAL;
748 }
749
750 md->geometry = *geo;
751
752 return 0;
753}
754
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755/*-----------------------------------------------------------------
756 * CRUD START:
757 * A more elegant soln is in the works that uses the queue
758 * merge fn, unfortunately there are a couple of changes to
759 * the block layer that I want to make for this. So in the
760 * interests of getting something for people to use I give
761 * you this clearly demarcated crap.
762 *---------------------------------------------------------------*/
763
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800764static int __noflush_suspending(struct mapped_device *md)
765{
766 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
767}
768
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769/*
770 * Decrements the number of outstanding ios that a bio has been
771 * cloned into, completing the original io if necc.
772 */
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200773static void dec_pending(struct dm_io *io, blk_status_t error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800775 unsigned long flags;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200776 blk_status_t io_error;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000777 struct bio *bio;
778 struct mapped_device *md = io->md;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800779
780 /* Push-back supersedes any I/O errors */
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +0100781 if (unlikely(error)) {
782 spin_lock_irqsave(&io->endio_lock, flags);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200783 if (!(io->status == BLK_STS_DM_REQUEUE &&
784 __noflush_suspending(md)))
785 io->status = error;
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +0100786 spin_unlock_irqrestore(&io->endio_lock, flags);
787 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788
789 if (atomic_dec_and_test(&io->io_count)) {
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200790 if (io->status == BLK_STS_DM_REQUEUE) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800791 /*
792 * Target requested pushing back the I/O.
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800793 */
Mikulas Patocka022c2612009-04-02 19:55:39 +0100794 spin_lock_irqsave(&md->deferred_lock, flags);
Tejun Heo6a8736d2010-09-08 18:07:00 +0200795 if (__noflush_suspending(md))
796 bio_list_add_head(&md->deferred, io->bio);
797 else
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800798 /* noflush suspend was interrupted. */
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200799 io->status = BLK_STS_IOERR;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100800 spin_unlock_irqrestore(&md->deferred_lock, flags);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800801 }
802
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200803 io_error = io->status;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000804 bio = io->bio;
Tejun Heo6a8736d2010-09-08 18:07:00 +0200805 end_io_acct(io);
806 free_io(md, io);
Jens Axboe2056a782006-03-23 20:00:26 +0100807
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200808 if (io_error == BLK_STS_DM_REQUEUE)
Tejun Heo6a8736d2010-09-08 18:07:00 +0200809 return;
810
Jens Axboe1eff9d32016-08-05 15:35:16 -0600811 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100812 /*
Tejun Heo6a8736d2010-09-08 18:07:00 +0200813 * Preflush done for flush with data, reissue
Mike Christie28a8f0d2016-06-05 14:32:25 -0500814 * without REQ_PREFLUSH.
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100815 */
Jens Axboe1eff9d32016-08-05 15:35:16 -0600816 bio->bi_opf &= ~REQ_PREFLUSH;
Tejun Heo6a8736d2010-09-08 18:07:00 +0200817 queue_io(md, bio);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100818 } else {
Mike Snitzerb372d362010-09-08 18:07:01 +0200819 /* done with normal IO or empty flush */
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200820 bio->bi_status = io_error;
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200821 bio_endio(bio);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800822 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 }
824}
825
Mike Snitzer4cc96132016-05-12 16:28:10 -0400826void disable_write_same(struct mapped_device *md)
Mike Snitzer7eee4ae2014-06-02 15:50:06 -0400827{
828 struct queue_limits *limits = dm_get_queue_limits(md);
829
830 /* device doesn't really support WRITE SAME, disable it */
831 limits->max_write_same_sectors = 0;
832}
833
Christoph Hellwigac62d622017-04-05 19:21:05 +0200834void disable_write_zeroes(struct mapped_device *md)
835{
836 struct queue_limits *limits = dm_get_queue_limits(md);
837
838 /* device doesn't really support WRITE ZEROES, disable it */
839 limits->max_write_zeroes_sectors = 0;
840}
841
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200842static void clone_endio(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843{
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200844 blk_status_t error = bio->bi_status;
Mikulas Patockabfc6d412014-03-04 18:24:49 -0500845 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000846 struct dm_io *io = tio->io;
Stefan Bader9faf4002006-10-03 01:15:41 -0700847 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 dm_endio_fn endio = tio->ti->type->end_io;
849
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200850 if (unlikely(error == BLK_STS_TARGET)) {
Christoph Hellwigac62d622017-04-05 19:21:05 +0200851 if (bio_op(bio) == REQ_OP_WRITE_SAME &&
Christoph Hellwig74d46992017-08-23 19:10:32 +0200852 !bio->bi_disk->queue->limits.max_write_same_sectors)
Christoph Hellwigac62d622017-04-05 19:21:05 +0200853 disable_write_same(md);
854 if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
Christoph Hellwig74d46992017-08-23 19:10:32 +0200855 !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
Christoph Hellwigac62d622017-04-05 19:21:05 +0200856 disable_write_zeroes(md);
857 }
Mike Snitzer7eee4ae2014-06-02 15:50:06 -0400858
Christoph Hellwig1be56902017-06-03 09:38:03 +0200859 if (endio) {
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200860 int r = endio(tio->ti, bio, &error);
Christoph Hellwig1be56902017-06-03 09:38:03 +0200861 switch (r) {
862 case DM_ENDIO_REQUEUE:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200863 error = BLK_STS_DM_REQUEUE;
Christoph Hellwig1be56902017-06-03 09:38:03 +0200864 /*FALLTHRU*/
865 case DM_ENDIO_DONE:
866 break;
867 case DM_ENDIO_INCOMPLETE:
868 /* The target will handle the io */
869 return;
870 default:
871 DMWARN("unimplemented target endio return value: %d", r);
872 BUG();
873 }
874 }
875
Mike Snitzercfae7522016-04-11 12:05:38 -0400876 free_tio(tio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000877 dec_pending(io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878}
879
Mike Snitzer78d8e582015-06-26 10:01:13 -0400880/*
Mike Snitzer56a67df2010-08-12 04:14:10 +0100881 * Return maximum size of I/O possible at the supplied sector up to the current
882 * target boundary.
883 */
884static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885{
Mike Snitzer56a67df2010-08-12 04:14:10 +0100886 sector_t target_offset = dm_target_offset(ti, sector);
887
888 return ti->len - target_offset;
889}
890
891static sector_t max_io_len(sector_t sector, struct dm_target *ti)
892{
893 sector_t len = max_io_len_target_boundary(sector, ti);
Mike Snitzer542f9032012-07-27 15:08:00 +0100894 sector_t offset, max_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895
896 /*
Mike Snitzer542f9032012-07-27 15:08:00 +0100897 * Does the target need to split even further?
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898 */
Mike Snitzer542f9032012-07-27 15:08:00 +0100899 if (ti->max_io_len) {
900 offset = dm_target_offset(ti, sector);
901 if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
902 max_len = sector_div(offset, ti->max_io_len);
903 else
904 max_len = offset & (ti->max_io_len - 1);
905 max_len = ti->max_io_len - max_len;
906
907 if (len > max_len)
908 len = max_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 }
910
911 return len;
912}
913
Mike Snitzer542f9032012-07-27 15:08:00 +0100914int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
915{
916 if (len > UINT_MAX) {
917 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
918 (unsigned long long)len, UINT_MAX);
919 ti->error = "Maximum size of target IO is too large";
920 return -EINVAL;
921 }
922
923 ti->max_io_len = (uint32_t) len;
924
925 return 0;
926}
927EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
928
Dan Williamsf26c5712017-04-12 12:35:44 -0700929static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
930 sector_t sector, int *srcu_idx)
Toshi Kani545ed202016-06-22 17:54:53 -0600931{
Toshi Kani545ed202016-06-22 17:54:53 -0600932 struct dm_table *map;
933 struct dm_target *ti;
Toshi Kani545ed202016-06-22 17:54:53 -0600934
Dan Williamsf26c5712017-04-12 12:35:44 -0700935 map = dm_get_live_table(md, srcu_idx);
Toshi Kani545ed202016-06-22 17:54:53 -0600936 if (!map)
Dan Williamsf26c5712017-04-12 12:35:44 -0700937 return NULL;
Toshi Kani545ed202016-06-22 17:54:53 -0600938
939 ti = dm_table_find_target(map, sector);
940 if (!dm_target_is_valid(ti))
Dan Williamsf26c5712017-04-12 12:35:44 -0700941 return NULL;
942
943 return ti;
944}
945
946static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
947 long nr_pages, void **kaddr, pfn_t *pfn)
948{
949 struct mapped_device *md = dax_get_private(dax_dev);
950 sector_t sector = pgoff * PAGE_SECTORS;
951 struct dm_target *ti;
952 long len, ret = -EIO;
953 int srcu_idx;
954
955 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
956
957 if (!ti)
Toshi Kani545ed202016-06-22 17:54:53 -0600958 goto out;
Dan Williamsf26c5712017-04-12 12:35:44 -0700959 if (!ti->type->direct_access)
960 goto out;
961 len = max_io_len(sector, ti) / PAGE_SECTORS;
962 if (len < 1)
963 goto out;
964 nr_pages = min(len, nr_pages);
Toshi Kani545ed202016-06-22 17:54:53 -0600965 if (ti->type->direct_access)
Dan Williams817bf402017-04-12 13:37:44 -0700966 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
967
Dan Williamsf26c5712017-04-12 12:35:44 -0700968 out:
Toshi Kani545ed202016-06-22 17:54:53 -0600969 dm_put_live_table(md, srcu_idx);
Dan Williamsf26c5712017-04-12 12:35:44 -0700970
971 return ret;
Toshi Kani545ed202016-06-22 17:54:53 -0600972}
973
Dan Williams7e026c82017-05-29 12:57:56 -0700974static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
975 void *addr, size_t bytes, struct iov_iter *i)
976{
977 struct mapped_device *md = dax_get_private(dax_dev);
978 sector_t sector = pgoff * PAGE_SECTORS;
979 struct dm_target *ti;
980 long ret = 0;
981 int srcu_idx;
982
983 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
984
985 if (!ti)
986 goto out;
987 if (!ti->type->dax_copy_from_iter) {
988 ret = copy_from_iter(addr, bytes, i);
989 goto out;
990 }
991 ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
992 out:
993 dm_put_live_table(md, srcu_idx);
994
995 return ret;
996}
997
Mikulas Patocka1dd40c32014-03-14 18:41:24 -0400998/*
999 * A target may call dm_accept_partial_bio only from the map routine. It is
Mike Christie28a8f0d2016-06-05 14:32:25 -05001000 * allowed for all bio types except REQ_PREFLUSH.
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001001 *
1002 * dm_accept_partial_bio informs the dm that the target only wants to process
1003 * additional n_sectors sectors of the bio and the rest of the data should be
1004 * sent in a next bio.
1005 *
1006 * A diagram that explains the arithmetics:
1007 * +--------------------+---------------+-------+
1008 * | 1 | 2 | 3 |
1009 * +--------------------+---------------+-------+
1010 *
1011 * <-------------- *tio->len_ptr --------------->
1012 * <------- bi_size ------->
1013 * <-- n_sectors -->
1014 *
1015 * Region 1 was already iterated over with bio_advance or similar function.
1016 * (it may be empty if the target doesn't use bio_advance)
1017 * Region 2 is the remaining bio size that the target wants to process.
1018 * (it may be empty if region 1 is non-empty, although there is no reason
1019 * to make it empty)
1020 * The target requires that region 3 is to be sent in the next bio.
1021 *
1022 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1023 * the partially processed part (the sum of regions 1+2) must be the same for all
1024 * copies of the bio.
1025 */
1026void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1027{
1028 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1029 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
Jens Axboe1eff9d32016-08-05 15:35:16 -06001030 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001031 BUG_ON(bi_size > *tio->len_ptr);
1032 BUG_ON(n_sectors > bi_size);
1033 *tio->len_ptr -= bi_size - n_sectors;
1034 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1035}
1036EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1037
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001038/*
Damien Le Moal10999302017-05-08 16:40:48 -07001039 * The zone descriptors obtained with a zone report indicate
1040 * zone positions within the target device. The zone descriptors
1041 * must be remapped to match their position within the dm device.
1042 * A target may call dm_remap_zone_report after completion of a
1043 * REQ_OP_ZONE_REPORT bio to remap the zone descriptors obtained
1044 * from the target device mapping to the dm device.
1045 */
1046void dm_remap_zone_report(struct dm_target *ti, struct bio *bio, sector_t start)
1047{
1048#ifdef CONFIG_BLK_DEV_ZONED
1049 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1050 struct bio *report_bio = tio->io->bio;
1051 struct blk_zone_report_hdr *hdr = NULL;
1052 struct blk_zone *zone;
1053 unsigned int nr_rep = 0;
1054 unsigned int ofst;
1055 struct bio_vec bvec;
1056 struct bvec_iter iter;
1057 void *addr;
1058
1059 if (bio->bi_status)
1060 return;
1061
1062 /*
1063 * Remap the start sector of the reported zones. For sequential zones,
1064 * also remap the write pointer position.
1065 */
1066 bio_for_each_segment(bvec, report_bio, iter) {
1067 addr = kmap_atomic(bvec.bv_page);
1068
1069 /* Remember the report header in the first page */
1070 if (!hdr) {
1071 hdr = addr;
1072 ofst = sizeof(struct blk_zone_report_hdr);
1073 } else
1074 ofst = 0;
1075
1076 /* Set zones start sector */
1077 while (hdr->nr_zones && ofst < bvec.bv_len) {
1078 zone = addr + ofst;
1079 if (zone->start >= start + ti->len) {
1080 hdr->nr_zones = 0;
1081 break;
1082 }
1083 zone->start = zone->start + ti->begin - start;
1084 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
1085 if (zone->cond == BLK_ZONE_COND_FULL)
1086 zone->wp = zone->start + zone->len;
1087 else if (zone->cond == BLK_ZONE_COND_EMPTY)
1088 zone->wp = zone->start;
1089 else
1090 zone->wp = zone->wp + ti->begin - start;
1091 }
1092 ofst += sizeof(struct blk_zone);
1093 hdr->nr_zones--;
1094 nr_rep++;
1095 }
1096
1097 if (addr != hdr)
1098 kunmap_atomic(addr);
1099
1100 if (!hdr->nr_zones)
1101 break;
1102 }
1103
1104 if (hdr) {
1105 hdr->nr_zones = nr_rep;
1106 kunmap_atomic(hdr);
1107 }
1108
1109 bio_advance(report_bio, report_bio->bi_iter.bi_size);
1110
1111#else /* !CONFIG_BLK_DEV_ZONED */
1112 bio->bi_status = BLK_STS_NOTSUPP;
1113#endif
1114}
1115EXPORT_SYMBOL_GPL(dm_remap_zone_report);
1116
1117/*
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001118 * Flush current->bio_list when the target map method blocks.
1119 * This fixes deadlocks in snapshot and possibly in other targets.
1120 */
1121struct dm_offload {
1122 struct blk_plug plug;
1123 struct blk_plug_cb cb;
1124};
1125
1126static void flush_current_bio_list(struct blk_plug_cb *cb, bool from_schedule)
1127{
1128 struct dm_offload *o = container_of(cb, struct dm_offload, cb);
1129 struct bio_list list;
1130 struct bio *bio;
NeilBrownf5fe1b52017-03-10 17:00:47 +11001131 int i;
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001132
1133 INIT_LIST_HEAD(&o->cb.list);
1134
1135 if (unlikely(!current->bio_list))
1136 return;
1137
NeilBrownf5fe1b52017-03-10 17:00:47 +11001138 for (i = 0; i < 2; i++) {
1139 list = current->bio_list[i];
1140 bio_list_init(&current->bio_list[i]);
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001141
NeilBrownf5fe1b52017-03-10 17:00:47 +11001142 while ((bio = bio_list_pop(&list))) {
1143 struct bio_set *bs = bio->bi_pool;
NeilBrown47e0fb42017-06-18 14:38:57 +10001144 if (unlikely(!bs) || bs == fs_bio_set ||
1145 !bs->rescue_workqueue) {
NeilBrownf5fe1b52017-03-10 17:00:47 +11001146 bio_list_add(&current->bio_list[i], bio);
1147 continue;
1148 }
1149
1150 spin_lock(&bs->rescue_lock);
1151 bio_list_add(&bs->rescue_list, bio);
1152 queue_work(bs->rescue_workqueue, &bs->rescue_work);
1153 spin_unlock(&bs->rescue_lock);
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001154 }
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001155 }
1156}
1157
1158static void dm_offload_start(struct dm_offload *o)
1159{
1160 blk_start_plug(&o->plug);
1161 o->cb.callback = flush_current_bio_list;
1162 list_add(&o->cb.list, &current->plug->cb_list);
1163}
1164
1165static void dm_offload_end(struct dm_offload *o)
1166{
1167 list_del(&o->cb.list);
1168 blk_finish_plug(&o->plug);
1169}
1170
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001171static void __map_bio(struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172{
1173 int r;
Jens Axboe2056a782006-03-23 20:00:26 +01001174 sector_t sector;
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001175 struct dm_offload o;
Mikulas Patockadba14162012-10-12 21:02:15 +01001176 struct bio *clone = &tio->clone;
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001177 struct dm_target *ti = tio->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179 clone->bi_end_io = clone_endio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001180
1181 /*
1182 * Map the clone. If r == 0 we don't need to do
1183 * anything, the target has assumed ownership of
1184 * this io.
1185 */
1186 atomic_inc(&tio->io->io_count);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001187 sector = clone->bi_iter.bi_sector;
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001188
1189 dm_offload_start(&o);
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00001190 r = ti->type->map(ti, clone);
Mikulas Patockad67a5f42017-02-15 11:26:10 -05001191 dm_offload_end(&o);
1192
Christoph Hellwig846785e2017-06-03 09:38:02 +02001193 switch (r) {
1194 case DM_MAPIO_SUBMITTED:
1195 break;
1196 case DM_MAPIO_REMAPPED:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 /* the bio has been remapped so dispatch it */
Christoph Hellwig74d46992017-08-23 19:10:32 +02001198 trace_block_bio_remap(clone->bi_disk->queue, clone,
1199 bio_dev(tio->io->bio), sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200 generic_make_request(clone);
Christoph Hellwig846785e2017-06-03 09:38:02 +02001201 break;
1202 case DM_MAPIO_KILL:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02001203 dec_pending(tio->io, BLK_STS_IOERR);
1204 free_tio(tio);
1205 break;
Christoph Hellwig846785e2017-06-03 09:38:02 +02001206 case DM_MAPIO_REQUEUE:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02001207 dec_pending(tio->io, BLK_STS_DM_REQUEUE);
Mike Snitzercfae7522016-04-11 12:05:38 -04001208 free_tio(tio);
Christoph Hellwig846785e2017-06-03 09:38:02 +02001209 break;
1210 default:
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -08001211 DMWARN("unimplemented target map return value: %d", r);
1212 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 }
1214}
1215
1216struct clone_info {
1217 struct mapped_device *md;
1218 struct dm_table *map;
1219 struct bio *bio;
1220 struct dm_io *io;
1221 sector_t sector;
Mikulas Patockae0d66092014-03-14 18:40:39 -04001222 unsigned sector_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223};
1224
Mikulas Patockae0d66092014-03-14 18:40:39 -04001225static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001226{
Kent Overstreet4f024f32013-10-11 15:44:27 -07001227 bio->bi_iter.bi_sector = sector;
1228 bio->bi_iter.bi_size = to_bytes(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001229}
1230
1231/*
1232 * Creates a bio that consists of range of complete bvecs.
1233 */
Mike Snitzerc80914e2016-03-02 12:33:03 -05001234static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1235 sector_t sector, unsigned len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236{
Mikulas Patockadba14162012-10-12 21:02:15 +01001237 struct bio *clone = &tio->clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001239 __bio_clone_fast(clone, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240
Mikulas Patockae2460f22017-04-18 16:51:48 -04001241 if (unlikely(bio_integrity(bio) != NULL)) {
1242 int r;
1243
1244 if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
1245 !dm_target_passes_integrity(tio->ti->type))) {
1246 DMWARN("%s: the target %s doesn't support integrity data.",
1247 dm_device_name(tio->io->md),
1248 tio->ti->type->name);
1249 return -EIO;
1250 }
1251
1252 r = bio_integrity_clone(clone, bio, GFP_NOIO);
Mike Snitzerc80914e2016-03-02 12:33:03 -05001253 if (r < 0)
1254 return r;
1255 }
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001256
Damien Le Moal264c8692017-05-08 16:40:47 -07001257 if (bio_op(bio) != REQ_OP_ZONE_REPORT)
1258 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001259 clone->bi_iter.bi_size = to_bytes(len);
1260
Mikulas Patockae2460f22017-04-18 16:51:48 -04001261 if (unlikely(bio_integrity(bio) != NULL))
Dmitry Monakhovfbd08e72017-06-29 11:31:10 -07001262 bio_integrity_trim(clone);
Mike Snitzerc80914e2016-03-02 12:33:03 -05001263
1264 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265}
1266
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001267static struct dm_target_io *alloc_tio(struct clone_info *ci,
Junichi Nomura99778272014-10-03 11:55:16 +00001268 struct dm_target *ti,
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001269 unsigned target_bio_nr)
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001270{
Mikulas Patockadba14162012-10-12 21:02:15 +01001271 struct dm_target_io *tio;
1272 struct bio *clone;
1273
Junichi Nomura99778272014-10-03 11:55:16 +00001274 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
Mikulas Patockadba14162012-10-12 21:02:15 +01001275 tio = container_of(clone, struct dm_target_io, clone);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001276
1277 tio->io = ci->io;
1278 tio->ti = ti;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001279 tio->target_bio_nr = target_bio_nr;
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001280
1281 return tio;
1282}
1283
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001284static void __clone_and_map_simple_bio(struct clone_info *ci,
1285 struct dm_target *ti,
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001286 unsigned target_bio_nr, unsigned *len)
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001287{
Junichi Nomura99778272014-10-03 11:55:16 +00001288 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
Mikulas Patockadba14162012-10-12 21:02:15 +01001289 struct bio *clone = &tio->clone;
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001290
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001291 tio->len_ptr = len;
1292
Junichi Nomura99778272014-10-03 11:55:16 +00001293 __bio_clone_fast(clone, ci->bio);
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001294 if (len)
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001295 bio_setup_sector(clone, ci->sector, *len);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001296
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001297 __map_bio(tio);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001298}
1299
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001300static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001301 unsigned num_bios, unsigned *len)
Mike Snitzer06a426c2010-08-12 04:14:09 +01001302{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001303 unsigned target_bio_nr;
Mike Snitzer06a426c2010-08-12 04:14:09 +01001304
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001305 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001306 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
Mike Snitzer06a426c2010-08-12 04:14:09 +01001307}
1308
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001309static int __send_empty_flush(struct clone_info *ci)
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001310{
Mike Snitzer06a426c2010-08-12 04:14:09 +01001311 unsigned target_nr = 0;
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001312 struct dm_target *ti;
1313
Mike Snitzerb372d362010-09-08 18:07:01 +02001314 BUG_ON(bio_has_data(ci->bio));
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001315 while ((ti = dm_table_get_target(ci->map, target_nr++)))
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001316 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001317
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001318 return 0;
1319}
1320
Mike Snitzerc80914e2016-03-02 12:33:03 -05001321static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001322 sector_t sector, unsigned *len)
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001323{
Mikulas Patockadba14162012-10-12 21:02:15 +01001324 struct bio *bio = ci->bio;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001325 struct dm_target_io *tio;
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001326 unsigned target_bio_nr;
1327 unsigned num_target_bios = 1;
Mike Snitzerc80914e2016-03-02 12:33:03 -05001328 int r = 0;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001329
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001330 /*
1331 * Does the target want to receive duplicate copies of the bio?
1332 */
1333 if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1334 num_target_bios = ti->num_write_bios(ti, bio);
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001335
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001336 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
Junichi Nomura99778272014-10-03 11:55:16 +00001337 tio = alloc_tio(ci, ti, target_bio_nr);
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001338 tio->len_ptr = len;
Mike Snitzerc80914e2016-03-02 12:33:03 -05001339 r = clone_bio(tio, bio, sector, *len);
Mikulas Patocka072623d2016-04-09 12:48:18 -04001340 if (r < 0) {
Mike Snitzercfae7522016-04-11 12:05:38 -04001341 free_tio(tio);
Mike Snitzerc80914e2016-03-02 12:33:03 -05001342 break;
Mikulas Patocka072623d2016-04-09 12:48:18 -04001343 }
Alasdair G Kergonb0d8ed42013-03-01 22:45:49 +00001344 __map_bio(tio);
1345 }
Mike Snitzerc80914e2016-03-02 12:33:03 -05001346
1347 return r;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001348}
1349
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001350typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
Mike Snitzer23508a92012-12-21 20:23:37 +00001351
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001352static unsigned get_num_discard_bios(struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001353{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001354 return ti->num_discard_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001355}
1356
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001357static unsigned get_num_write_same_bios(struct dm_target *ti)
Mike Snitzer23508a92012-12-21 20:23:37 +00001358{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001359 return ti->num_write_same_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001360}
1361
Christoph Hellwigac62d622017-04-05 19:21:05 +02001362static unsigned get_num_write_zeroes_bios(struct dm_target *ti)
1363{
1364 return ti->num_write_zeroes_bios;
1365}
1366
Mike Snitzer23508a92012-12-21 20:23:37 +00001367typedef bool (*is_split_required_fn)(struct dm_target *ti);
1368
1369static bool is_split_required_for_discard(struct dm_target *ti)
1370{
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001371 return ti->split_discard_bios;
Mike Snitzer23508a92012-12-21 20:23:37 +00001372}
1373
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001374static int __send_changing_extent_only(struct clone_info *ci,
1375 get_num_bios_fn get_num_bios,
1376 is_split_required_fn is_split_required)
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001377{
1378 struct dm_target *ti;
Mikulas Patockae0d66092014-03-14 18:40:39 -04001379 unsigned len;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001380 unsigned num_bios;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001381
Mike Snitzera79245b2010-08-12 04:14:24 +01001382 do {
1383 ti = dm_table_find_target(ci->map, ci->sector);
1384 if (!dm_target_is_valid(ti))
1385 return -EIO;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001386
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001387 /*
Mike Snitzer23508a92012-12-21 20:23:37 +00001388 * Even though the device advertised support for this type of
1389 * request, that does not mean every target supports it, and
Mike Snitzer936688d2011-08-02 12:32:01 +01001390 * reconfiguration might also have changed that since the
Mike Snitzera79245b2010-08-12 04:14:24 +01001391 * check was performed.
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001392 */
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001393 num_bios = get_num_bios ? get_num_bios(ti) : 0;
1394 if (!num_bios)
Mike Snitzera79245b2010-08-12 04:14:24 +01001395 return -EOPNOTSUPP;
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001396
Mike Snitzer23508a92012-12-21 20:23:37 +00001397 if (is_split_required && !is_split_required(ti))
Mikulas Patockae0d66092014-03-14 18:40:39 -04001398 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
Mikulas Patocka7acf0272012-07-27 15:08:03 +01001399 else
Mikulas Patockae0d66092014-03-14 18:40:39 -04001400 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
Mike Snitzer06a426c2010-08-12 04:14:09 +01001401
Mikulas Patocka1dd40c32014-03-14 18:41:24 -04001402 __send_duplicate_bios(ci, ti, num_bios, &len);
Mike Snitzera79245b2010-08-12 04:14:24 +01001403
1404 ci->sector += len;
1405 } while (ci->sector_count -= len);
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001406
1407 return 0;
1408}
1409
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001410static int __send_discard(struct clone_info *ci)
Mike Snitzer23508a92012-12-21 20:23:37 +00001411{
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001412 return __send_changing_extent_only(ci, get_num_discard_bios,
1413 is_split_required_for_discard);
Mike Snitzer23508a92012-12-21 20:23:37 +00001414}
1415
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001416static int __send_write_same(struct clone_info *ci)
Mike Snitzer23508a92012-12-21 20:23:37 +00001417{
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001418 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
Mike Snitzer23508a92012-12-21 20:23:37 +00001419}
1420
Christoph Hellwigac62d622017-04-05 19:21:05 +02001421static int __send_write_zeroes(struct clone_info *ci)
1422{
1423 return __send_changing_extent_only(ci, get_num_write_zeroes_bios, NULL);
1424}
1425
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001426/*
Alasdair G Kergone4c93812013-03-01 22:45:47 +00001427 * Select the correct strategy for processing a non-flush bio.
1428 */
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001429static int __split_and_process_non_flush(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430{
Mikulas Patockadba14162012-10-12 21:02:15 +01001431 struct bio *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001432 struct dm_target *ti;
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001433 unsigned len;
Mike Snitzerc80914e2016-03-02 12:33:03 -05001434 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435
Mike Christiee6047142016-06-05 14:32:04 -05001436 if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001437 return __send_discard(ci);
Mike Christiee6047142016-06-05 14:32:04 -05001438 else if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001439 return __send_write_same(ci);
Christoph Hellwigac62d622017-04-05 19:21:05 +02001440 else if (unlikely(bio_op(bio) == REQ_OP_WRITE_ZEROES))
1441 return __send_write_zeroes(ci);
Mike Snitzer5ae89a82010-08-12 04:14:08 +01001442
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001443 ti = dm_table_find_target(ci->map, ci->sector);
1444 if (!dm_target_is_valid(ti))
1445 return -EIO;
1446
Damien Le Moal264c8692017-05-08 16:40:47 -07001447 if (bio_op(bio) == REQ_OP_ZONE_REPORT)
1448 len = ci->sector_count;
1449 else
1450 len = min_t(sector_t, max_io_len(ci->sector, ti),
1451 ci->sector_count);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001452
Mike Snitzerc80914e2016-03-02 12:33:03 -05001453 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1454 if (r < 0)
1455 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001457 ci->sector += len;
1458 ci->sector_count -= len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459
Kent Overstreet1c3b13e2013-10-29 17:17:49 -07001460 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461}
1462
1463/*
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001464 * Entry point to split a bio into clones and submit them to the targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465 */
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001466static void __split_and_process_bio(struct mapped_device *md,
1467 struct dm_table *map, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468{
1469 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001470 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001472 if (unlikely(!map)) {
Tejun Heo6a8736d2010-09-08 18:07:00 +02001473 bio_io_error(bio);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +01001474 return;
1475 }
Mikulas Patocka692d0eb2009-04-09 00:27:13 +01001476
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001477 ci.map = map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478 ci.md = md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001479 ci.io = alloc_io(md);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02001480 ci.io->status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481 atomic_set(&ci.io->io_count, 1);
1482 ci.io->bio = bio;
1483 ci.io->md = md;
Kiyoshi Uedaf88fb982009-10-16 23:18:15 +01001484 spin_lock_init(&ci.io->endio_lock);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001485 ci.sector = bio->bi_iter.bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -08001487 start_io_acct(ci.io);
Alasdair G Kergonbd2a49b2013-03-01 22:45:46 +00001488
Jens Axboe1eff9d32016-08-05 15:35:16 -06001489 if (bio->bi_opf & REQ_PREFLUSH) {
Mike Snitzerb372d362010-09-08 18:07:01 +02001490 ci.bio = &ci.md->flush_bio;
1491 ci.sector_count = 0;
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001492 error = __send_empty_flush(&ci);
Mike Snitzerb372d362010-09-08 18:07:01 +02001493 /* dec_pending submits any data associated with flush */
Damien Le Moala4aa5e52017-05-08 16:40:46 -07001494 } else if (bio_op(bio) == REQ_OP_ZONE_RESET) {
1495 ci.bio = bio;
1496 ci.sector_count = 0;
1497 error = __split_and_process_non_flush(&ci);
Mike Snitzerb372d362010-09-08 18:07:01 +02001498 } else {
Tejun Heo6a8736d2010-09-08 18:07:00 +02001499 ci.bio = bio;
Tejun Heod87f4c12010-09-03 11:56:19 +02001500 ci.sector_count = bio_sectors(bio);
Mike Snitzerb372d362010-09-08 18:07:01 +02001501 while (ci.sector_count && !error)
Alasdair G Kergon14fe5942013-03-01 22:45:47 +00001502 error = __split_and_process_non_flush(&ci);
Tejun Heod87f4c12010-09-03 11:56:19 +02001503 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504
1505 /* drop the extra reference count */
Bart Van Assche54385bf2017-08-09 11:32:10 -07001506 dec_pending(ci.io, errno_to_blk_status(error));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001507}
1508/*-----------------------------------------------------------------
1509 * CRUD END
1510 *---------------------------------------------------------------*/
1511
1512/*
1513 * The request function that just remaps the bio built up by
1514 * dm_merge_bvec.
1515 */
Jens Axboedece1632015-11-05 10:41:16 -07001516static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517{
Kevin Corry12f03a42006-02-01 03:04:52 -08001518 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519 struct mapped_device *md = q->queuedata;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001520 int srcu_idx;
1521 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001523 map = dm_get_live_table(md, &srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524
Jens Axboed62e26b2017-06-30 21:55:08 -06001525 generic_start_io_acct(q, rw, bio_sectors(bio), &dm_disk(md)->part0);
Kevin Corry12f03a42006-02-01 03:04:52 -08001526
Tejun Heo6a8736d2010-09-08 18:07:00 +02001527 /* if we're suspended, we have to queue this io for later */
1528 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001529 dm_put_live_table(md, srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530
Jens Axboe1eff9d32016-08-05 15:35:16 -06001531 if (!(bio->bi_opf & REQ_RAHEAD))
Tejun Heo6a8736d2010-09-08 18:07:00 +02001532 queue_io(md, bio);
1533 else
Alasdair G Kergon54d9a1b2009-04-09 00:27:14 +01001534 bio_io_error(bio);
Jens Axboedece1632015-11-05 10:41:16 -07001535 return BLK_QC_T_NONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001536 }
1537
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001538 __split_and_process_bio(md, map, bio);
1539 dm_put_live_table(md, srcu_idx);
Jens Axboedece1632015-11-05 10:41:16 -07001540 return BLK_QC_T_NONE;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001541}
1542
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543static int dm_any_congested(void *congested_data, int bdi_bits)
1544{
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001545 int r = bdi_bits;
1546 struct mapped_device *md = congested_data;
1547 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001549 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Mike Snitzere522c032016-02-02 22:35:06 -05001550 if (dm_request_based(md)) {
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001551 /*
Mike Snitzere522c032016-02-02 22:35:06 -05001552 * With request-based DM we only need to check the
1553 * top-level queue for congestion.
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001554 */
Jan Karadc3b17c2017-02-02 15:56:50 +01001555 r = md->queue->backing_dev_info->wb.state & bdi_bits;
Mike Snitzere522c032016-02-02 22:35:06 -05001556 } else {
1557 map = dm_get_live_table_fast(md);
1558 if (map)
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001559 r = dm_table_any_congested(map, bdi_bits);
Mike Snitzere522c032016-02-02 22:35:06 -05001560 dm_put_live_table_fast(md);
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001561 }
1562 }
1563
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564 return r;
1565}
1566
1567/*-----------------------------------------------------------------
1568 * An IDR is used to keep track of allocated minor numbers.
1569 *---------------------------------------------------------------*/
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001570static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001572 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001574 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575}
1576
1577/*
1578 * See if the device with a specific minor # is free.
1579 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001580static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581{
Tejun Heoc9d76be2013-02-27 17:04:26 -08001582 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583
1584 if (minor >= (1 << MINORBITS))
1585 return -EINVAL;
1586
Tejun Heoc9d76be2013-02-27 17:04:26 -08001587 idr_preload(GFP_KERNEL);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001588 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589
Tejun Heoc9d76be2013-02-27 17:04:26 -08001590 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001592 spin_unlock(&_minor_lock);
Tejun Heoc9d76be2013-02-27 17:04:26 -08001593 idr_preload_end();
1594 if (r < 0)
1595 return r == -ENOSPC ? -EBUSY : r;
1596 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597}
1598
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001599static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001600{
Tejun Heoc9d76be2013-02-27 17:04:26 -08001601 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602
Tejun Heoc9d76be2013-02-27 17:04:26 -08001603 idr_preload(GFP_KERNEL);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001604 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605
Tejun Heoc9d76be2013-02-27 17:04:26 -08001606 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001608 spin_unlock(&_minor_lock);
Tejun Heoc9d76be2013-02-27 17:04:26 -08001609 idr_preload_end();
1610 if (r < 0)
1611 return r;
1612 *minor = r;
1613 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614}
1615
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07001616static const struct block_device_operations dm_blk_dops;
Dan Williamsf26c5712017-04-12 12:35:44 -07001617static const struct dax_operations dm_dax_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618
Mikulas Patocka53d59142009-04-02 19:55:37 +01001619static void dm_wq_work(struct work_struct *work);
1620
Mike Snitzer4cc96132016-05-12 16:28:10 -04001621void dm_init_md_queue(struct mapped_device *md)
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01001622{
1623 /*
1624 * Request-based dm devices cannot be stacked on top of bio-based dm
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05001625 * devices. The type of this dm device may not have been decided yet.
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01001626 * The type is decided at the first table loading time.
1627 * To prevent problematic device stacking, clear the queue flag
1628 * for request stacking support until then.
1629 *
1630 * This queue is new, so no concurrency on the queue_flags.
1631 */
1632 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
Mikulas Patockaad5f4982015-10-27 19:06:55 -04001633
1634 /*
1635 * Initialize data that will only be used by a non-blk-mq DM queue
1636 * - must do so here (in alloc_dev callchain) before queue is used
1637 */
1638 md->queue->queuedata = md;
Jan Karadc3b17c2017-02-02 15:56:50 +01001639 md->queue->backing_dev_info->congested_data = md;
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05001640}
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01001641
Mike Snitzer4cc96132016-05-12 16:28:10 -04001642void dm_init_normal_md_queue(struct mapped_device *md)
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05001643{
Mike Snitzer17e149b2015-03-11 15:01:09 -04001644 md->use_blk_mq = false;
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05001645 dm_init_md_queue(md);
1646
1647 /*
1648 * Initialize aspects of queue that aren't relevant for blk-mq
1649 */
Jan Karadc3b17c2017-02-02 15:56:50 +01001650 md->queue->backing_dev_info->congested_fn = dm_any_congested;
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01001651}
1652
Mike Snitzer0f209722015-04-28 11:50:29 -04001653static void cleanup_mapped_device(struct mapped_device *md)
1654{
Mike Snitzer0f209722015-04-28 11:50:29 -04001655 if (md->wq)
1656 destroy_workqueue(md->wq);
1657 if (md->kworker_task)
1658 kthread_stop(md->kworker_task);
Julia Lawall6f659852015-09-13 14:15:05 +02001659 mempool_destroy(md->io_pool);
Mike Snitzer0f209722015-04-28 11:50:29 -04001660 if (md->bs)
1661 bioset_free(md->bs);
1662
Dan Williamsf26c5712017-04-12 12:35:44 -07001663 if (md->dax_dev) {
1664 kill_dax(md->dax_dev);
1665 put_dax(md->dax_dev);
1666 md->dax_dev = NULL;
1667 }
1668
Mike Snitzer0f209722015-04-28 11:50:29 -04001669 if (md->disk) {
1670 spin_lock(&_minor_lock);
1671 md->disk->private_data = NULL;
1672 spin_unlock(&_minor_lock);
Mike Snitzer0f209722015-04-28 11:50:29 -04001673 del_gendisk(md->disk);
1674 put_disk(md->disk);
1675 }
1676
1677 if (md->queue)
1678 blk_cleanup_queue(md->queue);
1679
Tahsin Erdogand09960b2016-10-10 05:35:19 -07001680 cleanup_srcu_struct(&md->io_barrier);
1681
Mike Snitzer0f209722015-04-28 11:50:29 -04001682 if (md->bdev) {
1683 bdput(md->bdev);
1684 md->bdev = NULL;
1685 }
Mike Snitzer4cc96132016-05-12 16:28:10 -04001686
1687 dm_mq_cleanup_mapped_device(md);
Mike Snitzer0f209722015-04-28 11:50:29 -04001688}
1689
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690/*
1691 * Allocate and initialise a blank device with a given minor.
1692 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001693static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694{
Mike Snitzer115485e2016-02-22 12:16:21 -05001695 int r, numa_node_id = dm_get_numa_node();
Dan Williamsf26c5712017-04-12 12:35:44 -07001696 struct dax_device *dax_dev;
Mike Snitzer115485e2016-02-22 12:16:21 -05001697 struct mapped_device *md;
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001698 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699
Mikulas Patocka856eb092017-10-31 19:33:02 -04001700 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701 if (!md) {
1702 DMWARN("unable to allocate device, out of memory.");
1703 return NULL;
1704 }
1705
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001706 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001707 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001708
Linus Torvalds1da177e2005-04-16 15:20:36 -07001709 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001710 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001711 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001712 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001713 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001715 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001717 r = init_srcu_struct(&md->io_barrier);
1718 if (r < 0)
1719 goto bad_io_barrier;
1720
Mike Snitzer115485e2016-02-22 12:16:21 -05001721 md->numa_node_id = numa_node_id;
Mike Snitzer4cc96132016-05-12 16:28:10 -04001722 md->use_blk_mq = dm_use_blk_mq_default();
Mike Snitzer591ddcfc2016-01-31 12:05:42 -05001723 md->init_tio_pdu = false;
Mike Snitzera5664da2010-08-12 04:14:01 +01001724 md->type = DM_TYPE_NONE;
Daniel Walkere61290a2008-02-08 02:10:08 +00001725 mutex_init(&md->suspend_lock);
Mike Snitzera5664da2010-08-12 04:14:01 +01001726 mutex_init(&md->type_lock);
Benjamin Marzinski86f11522014-08-13 13:53:43 -05001727 mutex_init(&md->table_devices_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001728 spin_lock_init(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001730 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001732 atomic_set(&md->uevent_seq, 0);
1733 INIT_LIST_HEAD(&md->uevent_list);
Benjamin Marzinski86f11522014-08-13 13:53:43 -05001734 INIT_LIST_HEAD(&md->table_devices);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001735 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736
Mike Snitzer115485e2016-02-22 12:16:21 -05001737 md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738 if (!md->queue)
Mike Snitzer0f209722015-04-28 11:50:29 -04001739 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01001741 dm_init_md_queue(md);
Stefan Bader9faf4002006-10-03 01:15:41 -07001742
Mike Snitzer115485e2016-02-22 12:16:21 -05001743 md->disk = alloc_disk_node(1, numa_node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744 if (!md->disk)
Mike Snitzer0f209722015-04-28 11:50:29 -04001745 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001746
Nikanth Karthikesan316d3152009-10-06 20:16:55 +02001747 atomic_set(&md->pending[0], 0);
1748 atomic_set(&md->pending[1], 0);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001749 init_waitqueue_head(&md->wait);
Mikulas Patocka53d59142009-04-02 19:55:37 +01001750 INIT_WORK(&md->work, dm_wq_work);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001751 init_waitqueue_head(&md->eventq);
Mikulas Patocka2995fa72014-01-13 19:37:54 -05001752 init_completion(&md->kobj_holder.completion);
Keith Busch2eb6e1e2014-10-17 17:46:36 -06001753 md->kworker_task = NULL;
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001754
Linus Torvalds1da177e2005-04-16 15:20:36 -07001755 md->disk->major = _major;
1756 md->disk->first_minor = minor;
1757 md->disk->fops = &dm_blk_dops;
1758 md->disk->queue = md->queue;
1759 md->disk->private_data = md;
1760 sprintf(md->disk->disk_name, "dm-%d", minor);
Dan Williamsf26c5712017-04-12 12:35:44 -07001761
1762 dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops);
1763 if (!dax_dev)
1764 goto bad;
1765 md->dax_dev = dax_dev;
1766
Linus Torvalds1da177e2005-04-16 15:20:36 -07001767 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001768 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769
Tejun Heo670368a2013-07-30 08:40:21 -04001770 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
Milan Broz304f3f62008-02-08 02:11:17 +00001771 if (!md->wq)
Mike Snitzer0f209722015-04-28 11:50:29 -04001772 goto bad;
Milan Broz304f3f62008-02-08 02:11:17 +00001773
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001774 md->bdev = bdget_disk(md->disk, 0);
1775 if (!md->bdev)
Mike Snitzer0f209722015-04-28 11:50:29 -04001776 goto bad;
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001777
Ming Lei3a83f462016-11-22 08:57:21 -07001778 bio_init(&md->flush_bio, NULL, 0);
Christoph Hellwig74d46992017-08-23 19:10:32 +02001779 bio_set_dev(&md->flush_bio, md->bdev);
Jan Karaff0361b2017-05-31 09:44:32 +02001780 md->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
Tejun Heo6a8736d2010-09-08 18:07:00 +02001781
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04001782 dm_stats_init(&md->stats);
1783
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001784 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001785 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001786 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001787 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001788
1789 BUG_ON(old_md != MINOR_ALLOCED);
1790
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 return md;
1792
Mike Snitzer0f209722015-04-28 11:50:29 -04001793bad:
1794 cleanup_mapped_device(md);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001795bad_io_barrier:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001797bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001798 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001799bad_module_get:
Mikulas Patocka856eb092017-10-31 19:33:02 -04001800 kvfree(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801 return NULL;
1802}
1803
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001804static void unlock_fs(struct mapped_device *md);
1805
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806static void free_dev(struct mapped_device *md)
1807{
Tejun Heof331c022008-09-03 09:01:48 +02001808 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001809
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001810 unlock_fs(md);
Keith Busch2eb6e1e2014-10-17 17:46:36 -06001811
Mike Snitzer0f209722015-04-28 11:50:29 -04001812 cleanup_mapped_device(md);
Mike Snitzer0f209722015-04-28 11:50:29 -04001813
1814 free_table_devices(&md->table_devices);
1815 dm_stats_cleanup(&md->stats);
Mike Snitzer63a4f062015-03-23 17:01:43 -04001816 free_minor(minor);
1817
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001818 module_put(THIS_MODULE);
Mikulas Patocka856eb092017-10-31 19:33:02 -04001819 kvfree(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820}
1821
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001822static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
1823{
Mikulas Patockac0820cf2012-12-21 20:23:38 +00001824 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001825
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04001826 if (md->bs) {
1827 /* The md already has necessary mempools. */
Toshi Kani545ed202016-06-22 17:54:53 -06001828 if (dm_table_bio_based(t)) {
Jun'ichi Nomura16245bd2013-03-01 22:45:44 +00001829 /*
1830 * Reload bioset because front_pad may have changed
1831 * because a different table was loaded.
1832 */
1833 bioset_free(md->bs);
1834 md->bs = p->bs;
1835 p->bs = NULL;
Jun'ichi Nomura16245bd2013-03-01 22:45:44 +00001836 }
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04001837 /*
1838 * There's no need to reload with request-based dm
1839 * because the size of front_pad doesn't change.
1840 * Note for future: If you are to reload bioset,
1841 * prep-ed requests in the queue may refer
1842 * to bio from the old bioset, so you must walk
1843 * through the queue to unprep.
1844 */
1845 goto out;
Mikulas Patockac0820cf2012-12-21 20:23:38 +00001846 }
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001847
Christoph Hellwigeb8db832017-01-22 18:32:46 +01001848 BUG_ON(!p || md->io_pool || md->bs);
Mike Snitzercbc4e3c2015-04-27 16:37:50 -04001849
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001850 md->io_pool = p->io_pool;
1851 p->io_pool = NULL;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001852 md->bs = p->bs;
1853 p->bs = NULL;
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04001854
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001855out:
Mike Snitzer02233342015-03-10 23:49:26 -04001856 /* mempool bind completed, no longer need any mempools in the table */
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001857 dm_table_free_md_mempools(t);
1858}
1859
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860/*
1861 * Bind a table to the device.
1862 */
1863static void event_callback(void *context)
1864{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001865 unsigned long flags;
1866 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001867 struct mapped_device *md = (struct mapped_device *) context;
1868
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001869 spin_lock_irqsave(&md->uevent_lock, flags);
1870 list_splice_init(&md->uevent_list, &uevents);
1871 spin_unlock_irqrestore(&md->uevent_lock, flags);
1872
Tejun Heoed9e1982008-08-25 19:56:05 +09001873 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001874
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875 atomic_inc(&md->event_nr);
1876 wake_up(&md->eventq);
Mikulas Patocka62e08242017-09-20 07:29:49 -04001877 dm_issue_global_event();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001878}
1879
Mike Snitzerc2176492011-01-13 19:53:46 +00001880/*
1881 * Protected by md->suspend_lock obtained by dm_swap_table().
1882 */
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001883static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884{
Bart Van Assche1ea0654e2017-04-27 10:11:21 -07001885 lockdep_assert_held(&md->suspend_lock);
1886
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001887 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001888
Mikulas Patockadb8fef42009-06-22 10:12:15 +01001889 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001890}
1891
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00001892/*
1893 * Returns old map, which caller must destroy.
1894 */
1895static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1896 struct queue_limits *limits)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897{
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00001898 struct dm_table *old_map;
Jens Axboe165125e2007-07-24 09:28:11 +02001899 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900 sector_t size;
1901
Bart Van Assche5a8f1f82016-08-31 15:17:04 -07001902 lockdep_assert_held(&md->suspend_lock);
1903
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001905
1906 /*
1907 * Wipe any geometry if the size of the table changed.
1908 */
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04001909 if (size != dm_get_size(md))
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001910 memset(&md->geometry, 0, sizeof(md->geometry));
1911
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001912 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001914 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001915
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001916 /*
1917 * The queue hasn't been stopped yet, if the old table type wasn't
1918 * for request-based during suspension. So stop it to prevent
1919 * I/O mapping before resume.
1920 * This must be done before setting the queue restrictions,
1921 * because request-based dm may be run just after the setting.
1922 */
Mike Snitzer16f12262016-01-31 17:22:27 -05001923 if (dm_table_request_based(t)) {
Mike Snitzereca7ee62016-02-20 13:45:38 -05001924 dm_stop_queue(q);
Mike Snitzer16f12262016-01-31 17:22:27 -05001925 /*
1926 * Leverage the fact that request-based DM targets are
1927 * immutable singletons and establish md->immutable_target
1928 * - used to optimize both dm_request_fn and dm_mq_queue_rq
1929 */
1930 md->immutable_target = dm_table_get_immutable_target(t);
1931 }
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001932
1933 __bind_mempools(md, t);
1934
Eric Dumazeta12f5d42014-11-23 09:34:29 -08001935 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzer1d3aa6f2016-02-22 14:14:24 -05001936 rcu_assign_pointer(md->map, (void *)t);
Alasdair G Kergon36a04562011-10-31 20:19:04 +00001937 md->immutable_target_type = dm_table_get_immutable_target_type(t);
1938
Mike Snitzer754c5fc2009-06-22 10:12:34 +01001939 dm_table_set_restrictions(t, q, limits);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01001940 if (old_map)
1941 dm_sync_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001942
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00001943 return old_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001944}
1945
Alasdair G Kergona7940152009-12-10 23:52:23 +00001946/*
1947 * Returns unbound table for the caller to free.
1948 */
1949static struct dm_table *__unbind(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950{
Eric Dumazeta12f5d42014-11-23 09:34:29 -08001951 struct dm_table *map = rcu_dereference_protected(md->map, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952
1953 if (!map)
Alasdair G Kergona7940152009-12-10 23:52:23 +00001954 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955
1956 dm_table_event_callback(map, NULL, NULL);
Monam Agarwal9cdb8522014-03-23 23:58:27 +05301957 RCU_INIT_POINTER(md->map, NULL);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01001958 dm_sync_table(md);
Alasdair G Kergona7940152009-12-10 23:52:23 +00001959
1960 return map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961}
1962
1963/*
1964 * Constructor for a new device.
1965 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001966int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967{
1968 struct mapped_device *md;
1969
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001970 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971 if (!md)
1972 return -ENXIO;
1973
Milan Broz784aae72009-01-06 03:05:12 +00001974 dm_sysfs_init(md);
1975
Linus Torvalds1da177e2005-04-16 15:20:36 -07001976 *result = md;
1977 return 0;
1978}
1979
Mike Snitzera5664da2010-08-12 04:14:01 +01001980/*
1981 * Functions to manage md->type.
1982 * All are required to hold md->type_lock.
1983 */
1984void dm_lock_md_type(struct mapped_device *md)
1985{
1986 mutex_lock(&md->type_lock);
1987}
1988
1989void dm_unlock_md_type(struct mapped_device *md)
1990{
1991 mutex_unlock(&md->type_lock);
1992}
1993
Bart Van Assche7e0d5742017-04-27 10:11:23 -07001994void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
Mike Snitzera5664da2010-08-12 04:14:01 +01001995{
Mike Snitzer00c4fc32013-08-27 18:57:03 -04001996 BUG_ON(!mutex_is_locked(&md->type_lock));
Mike Snitzera5664da2010-08-12 04:14:01 +01001997 md->type = type;
1998}
1999
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002000enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
Mike Snitzera5664da2010-08-12 04:14:01 +01002001{
2002 return md->type;
2003}
2004
Alasdair G Kergon36a04562011-10-31 20:19:04 +00002005struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2006{
2007 return md->immutable_target_type;
2008}
2009
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002010/*
Mike Snitzerf84cb8a2013-09-19 12:13:58 -04002011 * The queue_limits are only valid as long as you have a reference
2012 * count on 'md'.
2013 */
2014struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2015{
2016 BUG_ON(!atomic_read(&md->holders));
2017 return &md->queue->limits;
2018}
2019EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2020
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002021/*
2022 * Setup the DM device's queue based on md's type
2023 */
Mike Snitzer591ddcfc2016-01-31 12:05:42 -05002024int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002025{
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002026 int r;
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002027 enum dm_queue_mode type = dm_get_md_type(md);
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002028
Toshi Kani545ed202016-06-22 17:54:53 -06002029 switch (type) {
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002030 case DM_TYPE_REQUEST_BASED:
Christoph Hellwigeb8db832017-01-22 18:32:46 +01002031 r = dm_old_init_request_queue(md, t);
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002032 if (r) {
Mike Snitzereca7ee62016-02-20 13:45:38 -05002033 DMERR("Cannot initialize queue for request-based mapped device");
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002034 return r;
Mike Snitzerff36ab32015-02-23 17:56:37 -05002035 }
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002036 break;
2037 case DM_TYPE_MQ_REQUEST_BASED:
Mike Snitzere83068a2016-05-24 21:16:51 -04002038 r = dm_mq_init_request_queue(md, t);
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002039 if (r) {
Mike Snitzereca7ee62016-02-20 13:45:38 -05002040 DMERR("Cannot initialize queue for request-based dm-mq mapped device");
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002041 return r;
2042 }
2043 break;
2044 case DM_TYPE_BIO_BASED:
Toshi Kani545ed202016-06-22 17:54:53 -06002045 case DM_TYPE_DAX_BIO_BASED:
Mike Snitzereca7ee62016-02-20 13:45:38 -05002046 dm_init_normal_md_queue(md);
Mike Snitzerff36ab32015-02-23 17:56:37 -05002047 blk_queue_make_request(md->queue, dm_make_request);
Mikulas Patockadbba42d2015-10-21 16:34:20 -04002048 /*
2049 * DM handles splitting bios as needed. Free the bio_split bioset
2050 * since it won't be used (saves 1 process per bio-based DM device).
2051 */
2052 bioset_free(md->queue->bio_split);
2053 md->queue->bio_split = NULL;
Toshi Kani545ed202016-06-22 17:54:53 -06002054
2055 if (type == DM_TYPE_DAX_BIO_BASED)
2056 queue_flag_set_unlocked(QUEUE_FLAG_DAX, md->queue);
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05002057 break;
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002058 case DM_TYPE_NONE:
2059 WARN_ON_ONCE(true);
2060 break;
Mike Snitzer4a0b4dd2010-08-12 04:14:02 +01002061 }
2062
2063 return 0;
2064}
2065
Mikulas Patocka2bec1f42015-02-17 14:30:53 -05002066struct mapped_device *dm_get_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067{
2068 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069 unsigned minor = MINOR(dev);
2070
2071 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2072 return NULL;
2073
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002074 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075
2076 md = idr_find(&_minor_idr, minor);
Mike Snitzer49de5762017-11-06 16:40:10 -05002077 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2078 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2079 md = NULL;
2080 goto out;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002081 }
Mike Snitzer49de5762017-11-06 16:40:10 -05002082 dm_get(md);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002083out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002084 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002085
David Teigland637842c2006-01-06 00:20:00 -08002086 return md;
2087}
Alasdair G Kergon3cf2e4b2011-10-31 20:19:06 +00002088EXPORT_SYMBOL_GPL(dm_get_md);
David Teiglandd229a952006-01-06 00:20:01 -08002089
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002090void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08002091{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002092 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002093}
2094
2095void dm_set_mdptr(struct mapped_device *md, void *ptr)
2096{
2097 md->interface_ptr = ptr;
2098}
2099
2100void dm_get(struct mapped_device *md)
2101{
2102 atomic_inc(&md->holders);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002103 BUG_ON(test_bit(DMF_FREEING, &md->flags));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002104}
2105
Mikulas Patocka09ee96b2015-02-26 11:41:28 -05002106int dm_hold(struct mapped_device *md)
2107{
2108 spin_lock(&_minor_lock);
2109 if (test_bit(DMF_FREEING, &md->flags)) {
2110 spin_unlock(&_minor_lock);
2111 return -EBUSY;
2112 }
2113 dm_get(md);
2114 spin_unlock(&_minor_lock);
2115 return 0;
2116}
2117EXPORT_SYMBOL_GPL(dm_hold);
2118
Alasdair G Kergon72d94862006-06-26 00:27:35 -07002119const char *dm_device_name(struct mapped_device *md)
2120{
2121 return md->name;
2122}
2123EXPORT_SYMBOL_GPL(dm_device_name);
2124
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002125static void __dm_destroy(struct mapped_device *md, bool wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002126{
Bart Van Assche3b785fb2016-08-31 15:17:49 -07002127 struct request_queue *q = dm_get_md_queue(md);
Mike Anderson1134e5ae2006-03-27 01:17:54 -08002128 struct dm_table *map;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002129 int srcu_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002130
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002131 might_sleep();
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002132
Mike Snitzer63a4f062015-03-23 17:01:43 -04002133 spin_lock(&_minor_lock);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002134 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2135 set_bit(DMF_FREEING, &md->flags);
2136 spin_unlock(&_minor_lock);
2137
Bart Van Assche2e91c362016-11-18 14:26:47 -08002138 blk_set_queue_dying(q);
Bart Van Assche3b785fb2016-08-31 15:17:49 -07002139
Mike Snitzer02233342015-03-10 23:49:26 -04002140 if (dm_request_based(md) && md->kworker_task)
Petr Mladek39891442016-10-11 13:55:20 -07002141 kthread_flush_worker(&md->kworker);
Keith Busch2eb6e1e2014-10-17 17:46:36 -06002142
Mikulas Patockaab7c7bb2015-02-27 14:04:27 -05002143 /*
2144 * Take suspend_lock so that presuspend and postsuspend methods
2145 * do not race with internal suspend.
2146 */
2147 mutex_lock(&md->suspend_lock);
Junichi Nomura2a708cf2015-10-01 08:31:51 +00002148 map = dm_get_live_table(md, &srcu_idx);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002149 if (!dm_suspended_md(md)) {
2150 dm_table_presuspend_targets(map);
2151 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002152 }
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002153 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2154 dm_put_live_table(md, srcu_idx);
Junichi Nomura2a708cf2015-10-01 08:31:51 +00002155 mutex_unlock(&md->suspend_lock);
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002156
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002157 /*
2158 * Rare, but there may be I/O requests still going to complete,
2159 * for example. Wait for all references to disappear.
2160 * No one should increment the reference count of the mapped_device,
2161 * after the mapped_device state becomes DMF_FREEING.
2162 */
2163 if (wait)
2164 while (atomic_read(&md->holders))
2165 msleep(1);
2166 else if (atomic_read(&md->holders))
2167 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2168 dm_device_name(md), atomic_read(&md->holders));
2169
2170 dm_sysfs_exit(md);
Kiyoshi Ueda3f77316d2010-08-12 04:13:56 +01002171 dm_table_destroy(__unbind(md));
2172 free_dev(md);
2173}
2174
2175void dm_destroy(struct mapped_device *md)
2176{
2177 __dm_destroy(md, true);
2178}
2179
2180void dm_destroy_immediate(struct mapped_device *md)
2181{
2182 __dm_destroy(md, false);
2183}
2184
2185void dm_put(struct mapped_device *md)
2186{
2187 atomic_dec(&md->holders);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002188}
Edward Goggin79eb8852007-05-09 02:32:56 -07002189EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002190
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002191static int dm_wait_for_completion(struct mapped_device *md, long task_state)
Milan Broz46125c12008-02-08 02:10:30 +00002192{
2193 int r = 0;
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07002194 DEFINE_WAIT(wait);
Milan Broz46125c12008-02-08 02:10:30 +00002195
2196 while (1) {
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07002197 prepare_to_wait(&md->wait, &wait, task_state);
Milan Broz46125c12008-02-08 02:10:30 +00002198
Kiyoshi Uedab4324fe2009-12-10 23:52:16 +00002199 if (!md_in_flight(md))
Milan Broz46125c12008-02-08 02:10:30 +00002200 break;
2201
Bart Van Asschee3fabdf2016-08-31 15:16:22 -07002202 if (signal_pending_state(task_state, current)) {
Milan Broz46125c12008-02-08 02:10:30 +00002203 r = -EINTR;
2204 break;
2205 }
2206
2207 io_schedule();
2208 }
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07002209 finish_wait(&md->wait, &wait);
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002210
Milan Broz46125c12008-02-08 02:10:30 +00002211 return r;
2212}
2213
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214/*
2215 * Process the deferred bios
2216 */
Mikulas Patockaef208582009-04-02 19:55:38 +01002217static void dm_wq_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002218{
Mikulas Patockaef208582009-04-02 19:55:38 +01002219 struct mapped_device *md = container_of(work, struct mapped_device,
2220 work);
Milan Broz6d6f10d2008-02-08 02:10:22 +00002221 struct bio *c;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002222 int srcu_idx;
2223 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002225 map = dm_get_live_table(md, &srcu_idx);
Mikulas Patockaef208582009-04-02 19:55:38 +01002226
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002227 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002228 spin_lock_irq(&md->deferred_lock);
2229 c = bio_list_pop(&md->deferred);
2230 spin_unlock_irq(&md->deferred_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002231
Tejun Heo6a8736d2010-09-08 18:07:00 +02002232 if (!c)
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002233 break;
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002234
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002235 if (dm_request_based(md))
2236 generic_make_request(c);
Tejun Heo6a8736d2010-09-08 18:07:00 +02002237 else
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002238 __split_and_process_bio(md, map, c);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002239 }
Milan Broz73d410c2008-02-08 02:10:25 +00002240
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002241 dm_put_live_table(md, srcu_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002242}
2243
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002244static void dm_queue_flush(struct mapped_device *md)
Milan Broz304f3f62008-02-08 02:11:17 +00002245{
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002246 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01002247 smp_mb__after_atomic();
Mikulas Patocka53d59142009-04-02 19:55:37 +01002248 queue_work(md->wq, &md->work);
Milan Broz304f3f62008-02-08 02:11:17 +00002249}
2250
Linus Torvalds1da177e2005-04-16 15:20:36 -07002251/*
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002252 * Swap in a new table, returning the old one for the caller to destroy.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002253 */
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002254struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255{
Mike Christie87eb5b22013-03-01 22:45:48 +00002256 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002257 struct queue_limits limits;
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002258 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002259
Daniel Walkere61290a2008-02-08 02:10:08 +00002260 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002261
2262 /* device must be suspended */
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +00002263 if (!dm_suspended_md(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002264 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002265
Mike Snitzer3ae70652012-09-26 23:45:45 +01002266 /*
2267 * If the new table has no data devices, retain the existing limits.
2268 * This helps multipath with queue_if_no_path if all paths disappear,
2269 * then new I/O is queued based on these limits, and then some paths
2270 * reappear.
2271 */
2272 if (dm_table_has_no_data_devices(table)) {
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002273 live_map = dm_get_live_table_fast(md);
Mike Snitzer3ae70652012-09-26 23:45:45 +01002274 if (live_map)
2275 limits = md->queue->limits;
Mikulas Patocka83d5e5b2013-07-10 23:41:18 +01002276 dm_put_live_table_fast(md);
Mike Snitzer3ae70652012-09-26 23:45:45 +01002277 }
2278
Mike Christie87eb5b22013-03-01 22:45:48 +00002279 if (!live_map) {
2280 r = dm_calculate_queue_limits(table, &limits);
2281 if (r) {
2282 map = ERR_PTR(r);
2283 goto out;
2284 }
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002285 }
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002286
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002287 map = __bind(md, table, &limits);
Mikulas Patocka62e08242017-09-20 07:29:49 -04002288 dm_issue_global_event();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002289
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002290out:
Daniel Walkere61290a2008-02-08 02:10:08 +00002291 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon042d2a92009-12-10 23:52:24 +00002292 return map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002293}
2294
2295/*
2296 * Functions to lock and unlock any filesystem running on the
2297 * device.
2298 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002299static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002300{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002301 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302
2303 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002304
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002305 md->frozen_sb = freeze_bdev(md->bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002306 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002307 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002308 md->frozen_sb = NULL;
2309 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002310 }
2311
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002312 set_bit(DMF_FROZEN, &md->flags);
2313
Linus Torvalds1da177e2005-04-16 15:20:36 -07002314 return 0;
2315}
2316
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002317static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002318{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002319 if (!test_bit(DMF_FROZEN, &md->flags))
2320 return;
2321
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002322 thaw_bdev(md->bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002323 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002324 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002325}
2326
2327/*
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002328 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2329 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2330 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2331 *
Mike Snitzerffcc3932014-10-28 18:34:52 -04002332 * If __dm_suspend returns 0, the device is completely quiescent
2333 * now. There is no request-processing activity. All new requests
2334 * are being added to md->deferred list.
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002335 */
Mike Snitzerffcc3932014-10-28 18:34:52 -04002336static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002337 unsigned suspend_flags, long task_state,
Mike Snitzereaf9a732016-08-02 13:07:20 -04002338 int dmf_suspended_flag)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002339{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002340 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2341 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2342 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002343
Bart Van Assche5a8f1f82016-08-31 15:17:04 -07002344 lockdep_assert_held(&md->suspend_lock);
2345
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002346 /*
2347 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2348 * This flag is cleared before dm_suspend returns.
2349 */
2350 if (noflush)
2351 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Bart Van Assche86331f32017-04-27 10:11:26 -07002352 else
2353 pr_debug("%s: suspending with flush\n", dm_device_name(md));
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002354
Mike Snitzerd67ee2132014-10-28 20:13:31 -04002355 /*
2356 * This gets reverted if there's an error later and the targets
2357 * provide the .presuspend_undo hook.
2358 */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002359 dm_table_presuspend_targets(map);
2360
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002361 /*
Kiyoshi Ueda9f518b22009-12-10 23:52:16 +00002362 * Flush I/O to the device.
2363 * Any I/O submitted after lock_fs() may not be flushed.
2364 * noflush takes precedence over do_lockfs.
2365 * (lock_fs() flushes I/Os and waits for them to complete.)
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002366 */
2367 if (!noflush && do_lockfs) {
2368 r = lock_fs(md);
Mike Snitzerd67ee2132014-10-28 20:13:31 -04002369 if (r) {
2370 dm_table_presuspend_undo_targets(map);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002371 return r;
Mike Snitzerd67ee2132014-10-28 20:13:31 -04002372 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002373 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002374
2375 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002376 * Here we must make sure that no processes are submitting requests
2377 * to target drivers i.e. no one may be executing
2378 * __split_and_process_bio. This is called from dm_request and
2379 * dm_wq_work.
2380 *
2381 * To get all processes out of __split_and_process_bio in dm_request,
2382 * we take the write lock. To prevent any process from reentering
Tejun Heo6a8736d2010-09-08 18:07:00 +02002383 * __split_and_process_bio from dm_request and quiesce the thread
2384 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2385 * flush_workqueue(md->wq).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002386 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01002387 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002388 if (map)
2389 synchronize_srcu(&md->io_barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002390
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002391 /*
Tejun Heo29e40132010-09-08 18:07:00 +02002392 * Stop md->queue before flushing md->wq in case request-based
2393 * dm defers requests to md->wq from md->queue.
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002394 */
Keith Busch2eb6e1e2014-10-17 17:46:36 -06002395 if (dm_request_based(md)) {
Mike Snitzereca7ee62016-02-20 13:45:38 -05002396 dm_stop_queue(md->queue);
Mike Snitzer02233342015-03-10 23:49:26 -04002397 if (md->kworker_task)
Petr Mladek39891442016-10-11 13:55:20 -07002398 kthread_flush_worker(&md->kworker);
Keith Busch2eb6e1e2014-10-17 17:46:36 -06002399 }
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002400
Kiyoshi Uedad0bcb872009-12-10 23:52:18 +00002401 flush_workqueue(md->wq);
2402
Linus Torvalds1da177e2005-04-16 15:20:36 -07002403 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002404 * At this point no more requests are entering target request routines.
2405 * We call dm_wait_for_completion to wait for all existing requests
2406 * to finish.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002407 */
Bart Van Asscheb48633f2016-08-31 15:16:02 -07002408 r = dm_wait_for_completion(md, task_state);
Mike Snitzereaf9a732016-08-02 13:07:20 -04002409 if (!r)
2410 set_bit(dmf_suspended_flag, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002411
Milan Broz6d6f10d2008-02-08 02:10:22 +00002412 if (noflush)
Mikulas Patocka022c2612009-04-02 19:55:39 +01002413 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Hannes Reinecke41abc4e2014-11-05 14:35:50 +01002414 if (map)
2415 synchronize_srcu(&md->io_barrier);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002416
Linus Torvalds1da177e2005-04-16 15:20:36 -07002417 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00002418 if (r < 0) {
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002419 dm_queue_flush(md);
Milan Broz73d410c2008-02-08 02:10:25 +00002420
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002421 if (dm_request_based(md))
Mike Snitzereca7ee62016-02-20 13:45:38 -05002422 dm_start_queue(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002423
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002424 unlock_fs(md);
Mike Snitzerd67ee2132014-10-28 20:13:31 -04002425 dm_table_presuspend_undo_targets(map);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002426 /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002427 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002428
Mike Snitzerffcc3932014-10-28 18:34:52 -04002429 return r;
2430}
2431
2432/*
2433 * We need to be able to change a mapping table under a mounted
2434 * filesystem. For example we might want to move some data in
2435 * the background. Before the table can be swapped with
2436 * dm_bind_table, dm_suspend must be called to flush any in
2437 * flight bios and ensure that any further io gets deferred.
2438 */
2439/*
2440 * Suspend mechanism in request-based dm.
2441 *
2442 * 1. Flush all I/Os by lock_fs() if needed.
2443 * 2. Stop dispatching any I/O by stopping the request_queue.
2444 * 3. Wait for all in-flight I/Os to be completed or requeued.
2445 *
2446 * To abort suspend, start the request_queue.
2447 */
2448int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2449{
2450 struct dm_table *map = NULL;
2451 int r = 0;
2452
2453retry:
2454 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2455
2456 if (dm_suspended_md(md)) {
2457 r = -EINVAL;
2458 goto out_unlock;
2459 }
2460
2461 if (dm_suspended_internally_md(md)) {
2462 /* already internally suspended, wait for internal resume */
2463 mutex_unlock(&md->suspend_lock);
2464 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2465 if (r)
2466 return r;
2467 goto retry;
2468 }
2469
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002470 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002471
Mike Snitzereaf9a732016-08-02 13:07:20 -04002472 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002473 if (r)
2474 goto out_unlock;
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002475
Kiyoshi Ueda4d4471c2009-12-10 23:52:26 +00002476 dm_table_postsuspend_targets(map);
2477
Alasdair G Kergond2874832006-11-08 17:44:43 -08002478out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00002479 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002480 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002481}
2482
Mike Snitzerffcc3932014-10-28 18:34:52 -04002483static int __dm_resume(struct mapped_device *md, struct dm_table *map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002484{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002485 if (map) {
2486 int r = dm_table_resume_targets(map);
2487 if (r)
2488 return r;
2489 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002490
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002491 dm_queue_flush(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002492
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002493 /*
2494 * Flushing deferred I/Os must be done after targets are resumed
2495 * so that mapping of targets can work correctly.
2496 * Request-based dm is queueing the deferred I/Os in its request_queue.
2497 */
2498 if (dm_request_based(md))
Mike Snitzereca7ee62016-02-20 13:45:38 -05002499 dm_start_queue(md->queue);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002500
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002501 unlock_fs(md);
2502
Mike Snitzerffcc3932014-10-28 18:34:52 -04002503 return 0;
2504}
2505
2506int dm_resume(struct mapped_device *md)
2507{
Minfei Huang8dc23652016-09-06 16:00:29 +08002508 int r;
Mike Snitzerffcc3932014-10-28 18:34:52 -04002509 struct dm_table *map = NULL;
2510
2511retry:
Minfei Huang8dc23652016-09-06 16:00:29 +08002512 r = -EINVAL;
Mike Snitzerffcc3932014-10-28 18:34:52 -04002513 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2514
2515 if (!dm_suspended_md(md))
2516 goto out;
2517
2518 if (dm_suspended_internally_md(md)) {
2519 /* already internally suspended, wait for internal resume */
2520 mutex_unlock(&md->suspend_lock);
2521 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2522 if (r)
2523 return r;
2524 goto retry;
2525 }
2526
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002527 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002528 if (!map || !dm_table_get_size(map))
2529 goto out;
2530
2531 r = __dm_resume(md, map);
2532 if (r)
2533 goto out;
2534
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002535 clear_bit(DMF_SUSPENDED, &md->flags);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002536out:
Daniel Walkere61290a2008-02-08 02:10:08 +00002537 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002538
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002539 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002540}
2541
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002542/*
2543 * Internal suspend/resume works like userspace-driven suspend. It waits
2544 * until all bios finish and prevents issuing new bios to the target drivers.
2545 * It may be used only from the kernel.
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002546 */
2547
Mike Snitzerffcc3932014-10-28 18:34:52 -04002548static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2549{
2550 struct dm_table *map = NULL;
2551
Bart Van Assche1ea0654e2017-04-27 10:11:21 -07002552 lockdep_assert_held(&md->suspend_lock);
2553
Mikulas Patocka96b26c82015-01-08 18:52:26 -05002554 if (md->internal_suspend_count++)
Mike Snitzerffcc3932014-10-28 18:34:52 -04002555 return; /* nested internal suspend */
2556
2557 if (dm_suspended_md(md)) {
2558 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2559 return; /* nest suspend */
2560 }
2561
Eric Dumazeta12f5d42014-11-23 09:34:29 -08002562 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
Mike Snitzerffcc3932014-10-28 18:34:52 -04002563
2564 /*
2565 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2566 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2567 * would require changing .presuspend to return an error -- avoid this
2568 * until there is a need for more elaborate variants of internal suspend.
2569 */
Mike Snitzereaf9a732016-08-02 13:07:20 -04002570 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2571 DMF_SUSPENDED_INTERNALLY);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002572
2573 dm_table_postsuspend_targets(map);
2574}
2575
2576static void __dm_internal_resume(struct mapped_device *md)
2577{
Mikulas Patocka96b26c82015-01-08 18:52:26 -05002578 BUG_ON(!md->internal_suspend_count);
2579
2580 if (--md->internal_suspend_count)
Mike Snitzerffcc3932014-10-28 18:34:52 -04002581 return; /* resume from nested internal suspend */
2582
2583 if (dm_suspended_md(md))
2584 goto done; /* resume from nested suspend */
2585
2586 /*
2587 * NOTE: existing callers don't need to call dm_table_resume_targets
2588 * (which may fail -- so best to avoid it for now by passing NULL map)
2589 */
2590 (void) __dm_resume(md, NULL);
2591
2592done:
2593 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2594 smp_mb__after_atomic();
2595 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2596}
2597
2598void dm_internal_suspend_noflush(struct mapped_device *md)
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002599{
2600 mutex_lock(&md->suspend_lock);
Mike Snitzerffcc3932014-10-28 18:34:52 -04002601 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2602 mutex_unlock(&md->suspend_lock);
2603}
2604EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2605
2606void dm_internal_resume(struct mapped_device *md)
2607{
2608 mutex_lock(&md->suspend_lock);
2609 __dm_internal_resume(md);
2610 mutex_unlock(&md->suspend_lock);
2611}
2612EXPORT_SYMBOL_GPL(dm_internal_resume);
2613
2614/*
2615 * Fast variants of internal suspend/resume hold md->suspend_lock,
2616 * which prevents interaction with userspace-driven suspend.
2617 */
2618
2619void dm_internal_suspend_fast(struct mapped_device *md)
2620{
2621 mutex_lock(&md->suspend_lock);
2622 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002623 return;
2624
2625 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2626 synchronize_srcu(&md->io_barrier);
2627 flush_workqueue(md->wq);
2628 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2629}
Mikulas Patockab735fed2015-02-26 11:40:35 -05002630EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002631
Mike Snitzerffcc3932014-10-28 18:34:52 -04002632void dm_internal_resume_fast(struct mapped_device *md)
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002633{
Mike Snitzerffcc3932014-10-28 18:34:52 -04002634 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002635 goto done;
2636
2637 dm_queue_flush(md);
2638
2639done:
2640 mutex_unlock(&md->suspend_lock);
2641}
Mikulas Patockab735fed2015-02-26 11:40:35 -05002642EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
Mikulas Patockafd2ed4d2013-08-16 10:54:23 -04002643
Linus Torvalds1da177e2005-04-16 15:20:36 -07002644/*-----------------------------------------------------------------
2645 * Event notification.
2646 *---------------------------------------------------------------*/
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00002647int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
Milan Broz60935eb2009-06-22 10:12:30 +01002648 unsigned cookie)
Alasdair G Kergon69267a32007-12-13 14:15:57 +00002649{
Milan Broz60935eb2009-06-22 10:12:30 +01002650 char udev_cookie[DM_COOKIE_LENGTH];
2651 char *envp[] = { udev_cookie, NULL };
2652
2653 if (!cookie)
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00002654 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
Milan Broz60935eb2009-06-22 10:12:30 +01002655 else {
2656 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2657 DM_COOKIE_ENV_VAR_NAME, cookie);
Peter Rajnoha3abf85b2010-03-06 02:32:31 +00002658 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2659 action, envp);
Milan Broz60935eb2009-06-22 10:12:30 +01002660 }
Alasdair G Kergon69267a32007-12-13 14:15:57 +00002661}
2662
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002663uint32_t dm_next_uevent_seq(struct mapped_device *md)
2664{
2665 return atomic_add_return(1, &md->uevent_seq);
2666}
2667
Linus Torvalds1da177e2005-04-16 15:20:36 -07002668uint32_t dm_get_event_nr(struct mapped_device *md)
2669{
2670 return atomic_read(&md->event_nr);
2671}
2672
2673int dm_wait_event(struct mapped_device *md, int event_nr)
2674{
2675 return wait_event_interruptible(md->eventq,
2676 (event_nr != atomic_read(&md->event_nr)));
2677}
2678
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002679void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2680{
2681 unsigned long flags;
2682
2683 spin_lock_irqsave(&md->uevent_lock, flags);
2684 list_add(elist, &md->uevent_list);
2685 spin_unlock_irqrestore(&md->uevent_lock, flags);
2686}
2687
Linus Torvalds1da177e2005-04-16 15:20:36 -07002688/*
2689 * The gendisk is only valid as long as you have a reference
2690 * count on 'md'.
2691 */
2692struct gendisk *dm_disk(struct mapped_device *md)
2693{
2694 return md->disk;
2695}
Sami Tolvanen65ff5b72015-03-18 15:52:14 +00002696EXPORT_SYMBOL_GPL(dm_disk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002697
Milan Broz784aae72009-01-06 03:05:12 +00002698struct kobject *dm_kobject(struct mapped_device *md)
2699{
Mikulas Patocka2995fa72014-01-13 19:37:54 -05002700 return &md->kobj_holder.kobj;
Milan Broz784aae72009-01-06 03:05:12 +00002701}
2702
Milan Broz784aae72009-01-06 03:05:12 +00002703struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2704{
2705 struct mapped_device *md;
2706
Mikulas Patocka2995fa72014-01-13 19:37:54 -05002707 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
Milan Broz784aae72009-01-06 03:05:12 +00002708
Hou Taob9a41d22017-11-01 15:42:36 +08002709 spin_lock(&_minor_lock);
2710 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2711 md = NULL;
2712 goto out;
2713 }
Milan Broz784aae72009-01-06 03:05:12 +00002714 dm_get(md);
Hou Taob9a41d22017-11-01 15:42:36 +08002715out:
2716 spin_unlock(&_minor_lock);
2717
Milan Broz784aae72009-01-06 03:05:12 +00002718 return md;
2719}
2720
Kiyoshi Ueda4f186f82009-12-10 23:52:26 +00002721int dm_suspended_md(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002722{
2723 return test_bit(DMF_SUSPENDED, &md->flags);
2724}
2725
Mike Snitzerffcc3932014-10-28 18:34:52 -04002726int dm_suspended_internally_md(struct mapped_device *md)
2727{
2728 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2729}
2730
Mikulas Patocka2c140a22013-11-01 18:27:41 -04002731int dm_test_deferred_remove_flag(struct mapped_device *md)
2732{
2733 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2734}
2735
Kiyoshi Ueda64dbce52009-12-10 23:52:27 +00002736int dm_suspended(struct dm_target *ti)
2737{
Kiyoshi Uedaecdb2e22010-03-06 02:29:52 +00002738 return dm_suspended_md(dm_table_get_md(ti->table));
Kiyoshi Ueda64dbce52009-12-10 23:52:27 +00002739}
2740EXPORT_SYMBOL_GPL(dm_suspended);
2741
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002742int dm_noflush_suspending(struct dm_target *ti)
2743{
Kiyoshi Uedaecdb2e22010-03-06 02:29:52 +00002744 return __noflush_suspending(dm_table_get_md(ti->table));
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002745}
2746EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2747
Bart Van Assche7e0d5742017-04-27 10:11:23 -07002748struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
Mike Snitzer30187e12016-01-31 13:28:26 -05002749 unsigned integrity, unsigned per_io_data_size)
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002750{
Mike Snitzer115485e2016-02-22 12:16:21 -05002751 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
Mike Snitzer78d8e582015-06-26 10:01:13 -04002752 unsigned int pool_size = 0;
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00002753 unsigned int front_pad;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002754
2755 if (!pools)
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04002756 return NULL;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002757
Mike Snitzer78d8e582015-06-26 10:01:13 -04002758 switch (type) {
2759 case DM_TYPE_BIO_BASED:
Toshi Kani545ed202016-06-22 17:54:53 -06002760 case DM_TYPE_DAX_BIO_BASED:
Mike Snitzer78d8e582015-06-26 10:01:13 -04002761 pool_size = dm_get_reserved_bio_based_ios();
Mike Snitzer30187e12016-01-31 13:28:26 -05002762 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
Christoph Hellwigeb8db832017-01-22 18:32:46 +01002763
2764 pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
2765 if (!pools->io_pool)
2766 goto out;
Mike Snitzer78d8e582015-06-26 10:01:13 -04002767 break;
2768 case DM_TYPE_REQUEST_BASED:
Mike Snitzer78d8e582015-06-26 10:01:13 -04002769 case DM_TYPE_MQ_REQUEST_BASED:
Christoph Hellwigeb8db832017-01-22 18:32:46 +01002770 pool_size = dm_get_reserved_rq_based_ios();
Mike Snitzer78d8e582015-06-26 10:01:13 -04002771 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
Mike Snitzer591ddcfc2016-01-31 12:05:42 -05002772 /* per_io_data_size is used for blk-mq pdu at queue allocation */
Mike Snitzer78d8e582015-06-26 10:01:13 -04002773 break;
2774 default:
2775 BUG();
2776 }
2777
NeilBrown47e0fb42017-06-18 14:38:57 +10002778 pools->bs = bioset_create(pool_size, front_pad, BIOSET_NEED_RESCUER);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002779 if (!pools->bs)
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00002780 goto out;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002781
Martin K. Petersena91a2782011-03-17 11:11:05 +01002782 if (integrity && bioset_integrity_create(pools->bs, pool_size))
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00002783 goto out;
Martin K. Petersena91a2782011-03-17 11:11:05 +01002784
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002785 return pools;
Mike Snitzer78d8e582015-06-26 10:01:13 -04002786
Jun'ichi Nomura5f015202013-03-01 22:45:48 +00002787out:
2788 dm_free_md_mempools(pools);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002789
Mike Snitzer4e6e36c2015-06-26 09:42:57 -04002790 return NULL;
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002791}
2792
2793void dm_free_md_mempools(struct dm_md_mempools *pools)
2794{
2795 if (!pools)
2796 return;
2797
Julia Lawall6f659852015-09-13 14:15:05 +02002798 mempool_destroy(pools->io_pool);
Mike Snitzer1ae49ea2014-12-05 17:11:05 -05002799
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002800 if (pools->bs)
2801 bioset_free(pools->bs);
2802
2803 kfree(pools);
2804}
2805
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09002806struct dm_pr {
2807 u64 old_key;
2808 u64 new_key;
2809 u32 flags;
2810 bool fail_early;
2811};
2812
2813static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2814 void *data)
2815{
2816 struct mapped_device *md = bdev->bd_disk->private_data;
2817 struct dm_table *table;
2818 struct dm_target *ti;
2819 int ret = -ENOTTY, srcu_idx;
2820
2821 table = dm_get_live_table(md, &srcu_idx);
2822 if (!table || !dm_table_get_size(table))
2823 goto out;
2824
2825 /* We only support devices that have a single target */
2826 if (dm_table_get_num_targets(table) != 1)
2827 goto out;
2828 ti = dm_table_get_target(table, 0);
2829
2830 ret = -EINVAL;
2831 if (!ti->type->iterate_devices)
2832 goto out;
2833
2834 ret = ti->type->iterate_devices(ti, fn, data);
2835out:
2836 dm_put_live_table(md, srcu_idx);
2837 return ret;
2838}
2839
2840/*
2841 * For register / unregister we need to manually call out to every path.
2842 */
2843static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
2844 sector_t start, sector_t len, void *data)
2845{
2846 struct dm_pr *pr = data;
2847 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
2848
2849 if (!ops || !ops->pr_register)
2850 return -EOPNOTSUPP;
2851 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
2852}
2853
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002854static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
Mike Snitzer956a4022016-02-18 16:13:51 -05002855 u32 flags)
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002856{
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09002857 struct dm_pr pr = {
2858 .old_key = old_key,
2859 .new_key = new_key,
2860 .flags = flags,
2861 .fail_early = true,
2862 };
2863 int ret;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002864
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09002865 ret = dm_call_pr(bdev, __dm_pr_register, &pr);
2866 if (ret && new_key) {
2867 /* unregister all paths if we failed to register any path */
2868 pr.old_key = new_key;
2869 pr.new_key = 0;
2870 pr.flags = 0;
2871 pr.fail_early = false;
2872 dm_call_pr(bdev, __dm_pr_register, &pr);
2873 }
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002874
Christoph Hellwig9c72bad2016-07-08 21:23:51 +09002875 return ret;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002876}
2877
2878static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
Mike Snitzer956a4022016-02-18 16:13:51 -05002879 u32 flags)
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002880{
2881 struct mapped_device *md = bdev->bd_disk->private_data;
2882 const struct pr_ops *ops;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002883 fmode_t mode;
Mike Snitzer956a4022016-02-18 16:13:51 -05002884 int r;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002885
Mike Snitzer956a4022016-02-18 16:13:51 -05002886 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002887 if (r < 0)
2888 return r;
2889
2890 ops = bdev->bd_disk->fops->pr_ops;
2891 if (ops && ops->pr_reserve)
2892 r = ops->pr_reserve(bdev, key, type, flags);
2893 else
2894 r = -EOPNOTSUPP;
2895
Mike Snitzer956a4022016-02-18 16:13:51 -05002896 bdput(bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002897 return r;
2898}
2899
2900static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2901{
2902 struct mapped_device *md = bdev->bd_disk->private_data;
2903 const struct pr_ops *ops;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002904 fmode_t mode;
Mike Snitzer956a4022016-02-18 16:13:51 -05002905 int r;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002906
Mike Snitzer956a4022016-02-18 16:13:51 -05002907 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002908 if (r < 0)
2909 return r;
2910
2911 ops = bdev->bd_disk->fops->pr_ops;
2912 if (ops && ops->pr_release)
2913 r = ops->pr_release(bdev, key, type);
2914 else
2915 r = -EOPNOTSUPP;
2916
Mike Snitzer956a4022016-02-18 16:13:51 -05002917 bdput(bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002918 return r;
2919}
2920
2921static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
Mike Snitzer956a4022016-02-18 16:13:51 -05002922 enum pr_type type, bool abort)
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002923{
2924 struct mapped_device *md = bdev->bd_disk->private_data;
2925 const struct pr_ops *ops;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002926 fmode_t mode;
Mike Snitzer956a4022016-02-18 16:13:51 -05002927 int r;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002928
Mike Snitzer956a4022016-02-18 16:13:51 -05002929 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002930 if (r < 0)
2931 return r;
2932
2933 ops = bdev->bd_disk->fops->pr_ops;
2934 if (ops && ops->pr_preempt)
2935 r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
2936 else
2937 r = -EOPNOTSUPP;
2938
Mike Snitzer956a4022016-02-18 16:13:51 -05002939 bdput(bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002940 return r;
2941}
2942
2943static int dm_pr_clear(struct block_device *bdev, u64 key)
2944{
2945 struct mapped_device *md = bdev->bd_disk->private_data;
2946 const struct pr_ops *ops;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002947 fmode_t mode;
Mike Snitzer956a4022016-02-18 16:13:51 -05002948 int r;
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002949
Mike Snitzer956a4022016-02-18 16:13:51 -05002950 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002951 if (r < 0)
2952 return r;
2953
2954 ops = bdev->bd_disk->fops->pr_ops;
2955 if (ops && ops->pr_clear)
2956 r = ops->pr_clear(bdev, key);
2957 else
2958 r = -EOPNOTSUPP;
2959
Mike Snitzer956a4022016-02-18 16:13:51 -05002960 bdput(bdev);
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002961 return r;
2962}
2963
2964static const struct pr_ops dm_pr_ops = {
2965 .pr_register = dm_pr_register,
2966 .pr_reserve = dm_pr_reserve,
2967 .pr_release = dm_pr_release,
2968 .pr_preempt = dm_pr_preempt,
2969 .pr_clear = dm_pr_clear,
2970};
2971
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07002972static const struct block_device_operations dm_blk_dops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002973 .open = dm_blk_open,
2974 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07002975 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08002976 .getgeo = dm_blk_getgeo,
Christoph Hellwig71cdb692015-10-15 14:10:51 +02002977 .pr_ops = &dm_pr_ops,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002978 .owner = THIS_MODULE
2979};
2980
Dan Williamsf26c5712017-04-12 12:35:44 -07002981static const struct dax_operations dm_dax_ops = {
2982 .direct_access = dm_dax_direct_access,
Dan Williams7e026c82017-05-29 12:57:56 -07002983 .copy_from_iter = dm_dax_copy_from_iter,
Dan Williamsf26c5712017-04-12 12:35:44 -07002984};
2985
Linus Torvalds1da177e2005-04-16 15:20:36 -07002986/*
2987 * module hooks
2988 */
2989module_init(dm_init);
2990module_exit(dm_exit);
2991
2992module_param(major, uint, 0);
2993MODULE_PARM_DESC(major, "The major number of the device mapper");
Mike Snitzerf4790822013-09-12 18:06:12 -04002994
Mike Snitzere8603132013-09-12 18:06:12 -04002995module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
2996MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
2997
Mike Snitzer115485e2016-02-22 12:16:21 -05002998module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
2999MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3000
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001MODULE_DESCRIPTION(DM_NAME " driver");
3002MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3003MODULE_LICENSE("GPL");