Christoph Hellwig | 8c16567 | 2019-04-30 14:42:39 -0400 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 2 | /* |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 3 | * Functions to sequence PREFLUSH and FUA writes. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 4 | * |
| 5 | * Copyright (C) 2011 Max Planck Institute for Gravitational Physics |
| 6 | * Copyright (C) 2011 Tejun Heo <tj@kernel.org> |
| 7 | * |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 8 | * REQ_{PREFLUSH|FUA} requests are decomposed to sequences consisted of three |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 9 | * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request |
| 10 | * properties and hardware capability. |
| 11 | * |
Mike Christie | 28a8f0d | 2016-06-05 14:32:25 -0500 | [diff] [blame] | 12 | * If a request doesn't have data, only REQ_PREFLUSH makes sense, which |
| 13 | * indicates a simple flush request. If there is data, REQ_PREFLUSH indicates |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 14 | * that the device cache should be flushed before the data is executed, and |
| 15 | * REQ_FUA means that the data must be on non-volatile media on request |
| 16 | * completion. |
| 17 | * |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 18 | * If the device doesn't have writeback cache, PREFLUSH and FUA don't make any |
| 19 | * difference. The requests are either completed immediately if there's no data |
| 20 | * or executed as normal requests otherwise. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 21 | * |
Mike Christie | 28a8f0d | 2016-06-05 14:32:25 -0500 | [diff] [blame] | 22 | * If the device has writeback cache and supports FUA, REQ_PREFLUSH is |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 23 | * translated to PREFLUSH but REQ_FUA is passed down directly with DATA. |
| 24 | * |
Mike Christie | 28a8f0d | 2016-06-05 14:32:25 -0500 | [diff] [blame] | 25 | * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH |
| 26 | * is translated to PREFLUSH and REQ_FUA to POSTFLUSH. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 27 | * |
| 28 | * The actual execution of flush is double buffered. Whenever a request |
| 29 | * needs to execute PRE or POSTFLUSH, it queues at |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 30 | * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a |
Mike Christie | 3a5e02c | 2016-06-05 14:32:23 -0500 | [diff] [blame] | 31 | * REQ_OP_FLUSH is issued and the pending_idx is toggled. When the flush |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 32 | * completes, all the requests which were pending are proceeded to the next |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 33 | * step. This allows arbitrary merging of different types of PREFLUSH/FUA |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 34 | * requests. |
| 35 | * |
| 36 | * Currently, the following conditions are used to determine when to issue |
| 37 | * flush. |
| 38 | * |
| 39 | * C1. At any given time, only one flush shall be in progress. This makes |
| 40 | * double buffering sufficient. |
| 41 | * |
| 42 | * C2. Flush is deferred if any request is executing DATA of its sequence. |
| 43 | * This avoids issuing separate POSTFLUSHes for requests which shared |
| 44 | * PREFLUSH. |
| 45 | * |
| 46 | * C3. The second condition is ignored if there is a request which has |
| 47 | * waited longer than FLUSH_PENDING_TIMEOUT. This is to avoid |
| 48 | * starvation in the unlikely case where there are continuous stream of |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 49 | * FUA (without PREFLUSH) requests. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 50 | * |
| 51 | * For devices which support FUA, it isn't clear whether C2 (and thus C3) |
| 52 | * is beneficial. |
| 53 | * |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 54 | * Note that a sequenced PREFLUSH/FUA request with DATA is completed twice. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 55 | * Once while executing DATA and again after the whole sequence is |
| 56 | * complete. The first completion updates the contained bio but doesn't |
| 57 | * finish it so that the bio submitter is notified only after the whole |
Christoph Hellwig | e806402 | 2016-10-20 15:12:13 +0200 | [diff] [blame] | 58 | * sequence is complete. This is implemented by testing RQF_FLUSH_SEQ in |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 59 | * req_bio_endio(). |
| 60 | * |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 61 | * The above peculiarity requires that each PREFLUSH/FUA request has only one |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 62 | * bio attached to it, which is guaranteed as they aren't allowed to be |
| 63 | * merged in the usual way. |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 64 | */ |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 65 | |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 66 | #include <linux/kernel.h> |
| 67 | #include <linux/module.h> |
| 68 | #include <linux/bio.h> |
| 69 | #include <linux/blkdev.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 70 | #include <linux/gfp.h> |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 71 | #include <linux/blk-mq.h> |
Bart Van Assche | b3c6a59 | 2019-12-17 16:24:35 -0800 | [diff] [blame] | 72 | #include <linux/lockdep.h> |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 73 | |
| 74 | #include "blk.h" |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 75 | #include "blk-mq.h" |
Ming Lei | 0048b48 | 2015-08-09 03:41:51 -0400 | [diff] [blame] | 76 | #include "blk-mq-tag.h" |
Jens Axboe | bd166ef | 2017-01-17 06:03:22 -0700 | [diff] [blame] | 77 | #include "blk-mq-sched.h" |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 78 | |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 79 | /* PREFLUSH/FUA sequences */ |
Tejun Heo | 4fed947 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 80 | enum { |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 81 | REQ_FSEQ_PREFLUSH = (1 << 0), /* pre-flushing in progress */ |
| 82 | REQ_FSEQ_DATA = (1 << 1), /* data write in progress */ |
| 83 | REQ_FSEQ_POSTFLUSH = (1 << 2), /* post-flushing in progress */ |
| 84 | REQ_FSEQ_DONE = (1 << 3), |
Tejun Heo | 4fed947 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 85 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 86 | REQ_FSEQ_ACTIONS = REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA | |
| 87 | REQ_FSEQ_POSTFLUSH, |
Tejun Heo | 47f70d5 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 88 | |
| 89 | /* |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 90 | * If flush has been pending longer than the following timeout, |
| 91 | * it's issued even if flush_data requests are still in flight. |
Tejun Heo | 47f70d5 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 92 | */ |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 93 | FLUSH_PENDING_TIMEOUT = 5 * HZ, |
| 94 | }; |
| 95 | |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 96 | static void blk_kick_flush(struct request_queue *q, |
Hannes Reinecke | 84fca1b | 2018-06-06 16:21:40 +0200 | [diff] [blame] | 97 | struct blk_flush_queue *fq, unsigned int flags); |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 98 | |
Jens Axboe | c888a8f | 2016-04-13 13:33:19 -0600 | [diff] [blame] | 99 | static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq) |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 100 | { |
| 101 | unsigned int policy = 0; |
| 102 | |
Jeff Moyer | fa1bf42 | 2011-08-09 20:32:09 +0200 | [diff] [blame] | 103 | if (blk_rq_sectors(rq)) |
| 104 | policy |= REQ_FSEQ_DATA; |
| 105 | |
Jens Axboe | c888a8f | 2016-04-13 13:33:19 -0600 | [diff] [blame] | 106 | if (fflags & (1UL << QUEUE_FLAG_WC)) { |
Mike Christie | 28a8f0d | 2016-06-05 14:32:25 -0500 | [diff] [blame] | 107 | if (rq->cmd_flags & REQ_PREFLUSH) |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 108 | policy |= REQ_FSEQ_PREFLUSH; |
Jens Axboe | c888a8f | 2016-04-13 13:33:19 -0600 | [diff] [blame] | 109 | if (!(fflags & (1UL << QUEUE_FLAG_FUA)) && |
| 110 | (rq->cmd_flags & REQ_FUA)) |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 111 | policy |= REQ_FSEQ_POSTFLUSH; |
| 112 | } |
| 113 | return policy; |
Tejun Heo | 47f70d5 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 114 | } |
| 115 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 116 | static unsigned int blk_flush_cur_seq(struct request *rq) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 117 | { |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 118 | return 1 << ffz(rq->flush.seq); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 119 | } |
| 120 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 121 | static void blk_flush_restore_request(struct request *rq) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 122 | { |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 123 | /* |
| 124 | * After flush data completion, @rq->bio is %NULL but we need to |
| 125 | * complete the bio again. @rq->biotail is guaranteed to equal the |
| 126 | * original @rq->bio. Restore it. |
| 127 | */ |
| 128 | rq->bio = rq->biotail; |
| 129 | |
| 130 | /* make @rq a normal request */ |
Christoph Hellwig | e806402 | 2016-10-20 15:12:13 +0200 | [diff] [blame] | 131 | rq->rq_flags &= ~RQF_FLUSH_SEQ; |
Jeff Moyer | 4853aba | 2011-08-15 21:37:25 +0200 | [diff] [blame] | 132 | rq->end_io = rq->flush.saved_end_io; |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 133 | } |
| 134 | |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 135 | static void blk_flush_queue_rq(struct request *rq, bool add_front) |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 136 | { |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 137 | blk_mq_add_to_requeue_list(rq, add_front, true); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 138 | } |
| 139 | |
Konstantin Khlebnikov | b686631 | 2019-11-21 13:40:26 +0300 | [diff] [blame] | 140 | static void blk_account_io_flush(struct request *rq) |
| 141 | { |
| 142 | struct hd_struct *part = &rq->rq_disk->part0; |
| 143 | |
| 144 | part_stat_lock(); |
| 145 | part_stat_inc(part, ios[STAT_FLUSH]); |
| 146 | part_stat_add(part, nsecs[STAT_FLUSH], |
| 147 | ktime_get_ns() - rq->start_time_ns); |
| 148 | part_stat_unlock(); |
| 149 | } |
| 150 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 151 | /** |
| 152 | * blk_flush_complete_seq - complete flush sequence |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 153 | * @rq: PREFLUSH/FUA request being sequenced |
Ming Lei | 0bae352 | 2014-09-25 23:23:45 +0800 | [diff] [blame] | 154 | * @fq: flush queue |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 155 | * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero) |
| 156 | * @error: whether an error occurred |
| 157 | * |
| 158 | * @rq just completed @seq part of its flush sequence, record the |
| 159 | * completion and trigger the next step. |
| 160 | * |
| 161 | * CONTEXT: |
Christoph Hellwig | 9809b4e | 2018-11-14 17:02:10 +0100 | [diff] [blame] | 162 | * spin_lock_irq(fq->mq_flush_lock) |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 163 | */ |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 164 | static void blk_flush_complete_seq(struct request *rq, |
Ming Lei | 0bae352 | 2014-09-25 23:23:45 +0800 | [diff] [blame] | 165 | struct blk_flush_queue *fq, |
Christoph Hellwig | 2a842ac | 2017-06-03 09:38:04 +0200 | [diff] [blame] | 166 | unsigned int seq, blk_status_t error) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 167 | { |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 168 | struct request_queue *q = rq->q; |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 169 | struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; |
Jens Axboe | 190b02e | 2018-06-09 06:37:14 -0600 | [diff] [blame] | 170 | unsigned int cmd_flags; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 171 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 172 | BUG_ON(rq->flush.seq & seq); |
| 173 | rq->flush.seq |= seq; |
Jens Axboe | 190b02e | 2018-06-09 06:37:14 -0600 | [diff] [blame] | 174 | cmd_flags = rq->cmd_flags; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 175 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 176 | if (likely(!error)) |
| 177 | seq = blk_flush_cur_seq(rq); |
| 178 | else |
| 179 | seq = REQ_FSEQ_DONE; |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 180 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 181 | switch (seq) { |
| 182 | case REQ_FSEQ_PREFLUSH: |
| 183 | case REQ_FSEQ_POSTFLUSH: |
| 184 | /* queue for flush */ |
| 185 | if (list_empty(pending)) |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 186 | fq->flush_pending_since = jiffies; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 187 | list_move_tail(&rq->flush.list, pending); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 188 | break; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 189 | |
| 190 | case REQ_FSEQ_DATA: |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 191 | list_move_tail(&rq->flush.list, &fq->flush_data_in_flight); |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 192 | blk_flush_queue_rq(rq, true); |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 193 | break; |
| 194 | |
| 195 | case REQ_FSEQ_DONE: |
Tejun Heo | 09d60c7 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 196 | /* |
Konstantin Khlebnikov | b686631 | 2019-11-21 13:40:26 +0300 | [diff] [blame] | 197 | * @rq was previously adjusted by blk_insert_flush() for |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 198 | * flush sequencing and may already have gone through the |
| 199 | * flush data request completion path. Restore @rq for |
| 200 | * normal completion and end it. |
Tejun Heo | 09d60c7 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 201 | */ |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 202 | BUG_ON(!list_empty(&rq->queuelist)); |
| 203 | list_del_init(&rq->flush.list); |
| 204 | blk_flush_restore_request(rq); |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 205 | blk_mq_end_request(rq, error); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 206 | break; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 207 | |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 208 | default: |
| 209 | BUG(); |
| 210 | } |
Christoph Hellwig | cde4c40 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 211 | |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 212 | blk_kick_flush(q, fq, cmd_flags); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 213 | } |
| 214 | |
Christoph Hellwig | 2a842ac | 2017-06-03 09:38:04 +0200 | [diff] [blame] | 215 | static void flush_end_io(struct request *flush_rq, blk_status_t error) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 216 | { |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 217 | struct request_queue *q = flush_rq->q; |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 218 | struct list_head *running; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 219 | struct request *rq, *n; |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 220 | unsigned long flags = 0; |
Ming Lei | e97c293 | 2014-09-25 23:23:46 +0800 | [diff] [blame] | 221 | struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); |
Tejun Heo | 8f11b3e | 2008-11-28 13:32:05 +0900 | [diff] [blame] | 222 | |
Konstantin Khlebnikov | b686631 | 2019-11-21 13:40:26 +0300 | [diff] [blame] | 223 | blk_account_io_flush(flush_rq); |
| 224 | |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 225 | /* release the tag's ownership to the req cloned from */ |
| 226 | spin_lock_irqsave(&fq->mq_flush_lock, flags); |
Yufen Yu | 8d69966 | 2019-09-27 16:19:55 +0800 | [diff] [blame] | 227 | |
| 228 | if (!refcount_dec_and_test(&flush_rq->ref)) { |
| 229 | fq->rq_status = error; |
| 230 | spin_unlock_irqrestore(&fq->mq_flush_lock, flags); |
| 231 | return; |
| 232 | } |
| 233 | |
| 234 | if (fq->rq_status != BLK_STS_OK) |
| 235 | error = fq->rq_status; |
| 236 | |
Jens Axboe | 4e2f62e5 | 2020-07-01 22:58:32 -0600 | [diff] [blame] | 237 | if (!q->elevator) { |
Ming Lei | 568f270 | 2020-07-06 22:41:11 +0800 | [diff] [blame] | 238 | flush_rq->tag = BLK_MQ_NO_TAG; |
Jens Axboe | 4e2f62e5 | 2020-07-01 22:58:32 -0600 | [diff] [blame] | 239 | } else { |
| 240 | blk_mq_put_driver_tag(flush_rq); |
Ming Lei | 568f270 | 2020-07-06 22:41:11 +0800 | [diff] [blame] | 241 | flush_rq->internal_tag = BLK_MQ_NO_TAG; |
Jens Axboe | 4e2f62e5 | 2020-07-01 22:58:32 -0600 | [diff] [blame] | 242 | } |
Christoph Hellwig | 1874198 | 2014-02-10 09:29:00 -0700 | [diff] [blame] | 243 | |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 244 | running = &fq->flush_queue[fq->flush_running_idx]; |
| 245 | BUG_ON(fq->flush_pending_idx == fq->flush_running_idx); |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 246 | |
| 247 | /* account completion of the flush request */ |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 248 | fq->flush_running_idx ^= 1; |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 249 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 250 | /* and push the waiting requests to the next stage */ |
| 251 | list_for_each_entry_safe(rq, n, running, flush.list) { |
| 252 | unsigned int seq = blk_flush_cur_seq(rq); |
| 253 | |
| 254 | BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH); |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 255 | blk_flush_complete_seq(rq, fq, seq, error); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 256 | } |
| 257 | |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 258 | spin_unlock_irqrestore(&fq->mq_flush_lock, flags); |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 259 | } |
| 260 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 261 | /** |
| 262 | * blk_kick_flush - consider issuing flush request |
| 263 | * @q: request_queue being kicked |
Ming Lei | 0bae352 | 2014-09-25 23:23:45 +0800 | [diff] [blame] | 264 | * @fq: flush queue |
Hannes Reinecke | 84fca1b | 2018-06-06 16:21:40 +0200 | [diff] [blame] | 265 | * @flags: cmd_flags of the original request |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 266 | * |
| 267 | * Flush related states of @q have changed, consider issuing flush request. |
| 268 | * Please read the comment at the top of this file for more info. |
| 269 | * |
| 270 | * CONTEXT: |
Christoph Hellwig | 9809b4e | 2018-11-14 17:02:10 +0100 | [diff] [blame] | 271 | * spin_lock_irq(fq->mq_flush_lock) |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 272 | * |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 273 | */ |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 274 | static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, |
Hannes Reinecke | 84fca1b | 2018-06-06 16:21:40 +0200 | [diff] [blame] | 275 | unsigned int flags) |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 276 | { |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 277 | struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 278 | struct request *first_rq = |
| 279 | list_first_entry(pending, struct request, flush.list); |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 280 | struct request *flush_rq = fq->flush_rq; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 281 | |
| 282 | /* C1 described at the top of this file */ |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 283 | if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending)) |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 284 | return; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 285 | |
Yufen Yu | b5718d6 | 2020-07-16 02:52:01 -0400 | [diff] [blame] | 286 | /* C2 and C3 */ |
| 287 | if (!list_empty(&fq->flush_data_in_flight) && |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 288 | time_before(jiffies, |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 289 | fq->flush_pending_since + FLUSH_PENDING_TIMEOUT)) |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 290 | return; |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 291 | |
| 292 | /* |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 293 | * Issue flush and toggle pending_idx. This makes pending_idx |
| 294 | * different from running_idx, which means flush is in flight. |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 295 | */ |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 296 | fq->flush_pending_idx ^= 1; |
Christoph Hellwig | 1874198 | 2014-02-10 09:29:00 -0700 | [diff] [blame] | 297 | |
Ming Lei | 7ddab5d | 2014-09-25 23:23:42 +0800 | [diff] [blame] | 298 | blk_rq_init(q, flush_rq); |
Ming Lei | f70ced0 | 2014-09-25 23:23:47 +0800 | [diff] [blame] | 299 | |
| 300 | /* |
Ming Lei | 923218f | 2017-11-02 23:24:38 +0800 | [diff] [blame] | 301 | * In case of none scheduler, borrow tag from the first request |
| 302 | * since they can't be in flight at the same time. And acquire |
| 303 | * the tag's ownership for flush req. |
| 304 | * |
| 305 | * In case of IO scheduler, flush rq need to borrow scheduler tag |
| 306 | * just for cheating put/get driver tag. |
Ming Lei | f70ced0 | 2014-09-25 23:23:47 +0800 | [diff] [blame] | 307 | */ |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 308 | flush_rq->mq_ctx = first_rq->mq_ctx; |
Jens Axboe | ea4f995 | 2018-10-29 15:06:13 -0600 | [diff] [blame] | 309 | flush_rq->mq_hctx = first_rq->mq_hctx; |
Ming Lei | 0048b48 | 2015-08-09 03:41:51 -0400 | [diff] [blame] | 310 | |
Ming Lei | 568f270 | 2020-07-06 22:41:11 +0800 | [diff] [blame] | 311 | if (!q->elevator) |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 312 | flush_rq->tag = first_rq->tag; |
Ming Lei | 568f270 | 2020-07-06 22:41:11 +0800 | [diff] [blame] | 313 | else |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 314 | flush_rq->internal_tag = first_rq->internal_tag; |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 315 | |
Christoph Hellwig | 70fd761 | 2016-11-01 07:40:10 -0600 | [diff] [blame] | 316 | flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH; |
Hannes Reinecke | 84fca1b | 2018-06-06 16:21:40 +0200 | [diff] [blame] | 317 | flush_rq->cmd_flags |= (flags & REQ_DRV) | (flags & REQ_FAILFAST_MASK); |
Christoph Hellwig | e806402 | 2016-10-20 15:12:13 +0200 | [diff] [blame] | 318 | flush_rq->rq_flags |= RQF_FLUSH_SEQ; |
Ming Lei | 7ddab5d | 2014-09-25 23:23:42 +0800 | [diff] [blame] | 319 | flush_rq->rq_disk = first_rq->rq_disk; |
| 320 | flush_rq->end_io = flush_end_io; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 321 | |
Jens Axboe | 404b8f5 | 2018-10-31 12:43:24 -0600 | [diff] [blame] | 322 | blk_flush_queue_rq(flush_rq, false); |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 323 | } |
| 324 | |
Christoph Hellwig | 2a842ac | 2017-06-03 09:38:04 +0200 | [diff] [blame] | 325 | static void mq_flush_data_end_io(struct request *rq, blk_status_t error) |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 326 | { |
| 327 | struct request_queue *q = rq->q; |
Jens Axboe | ea4f995 | 2018-10-29 15:06:13 -0600 | [diff] [blame] | 328 | struct blk_mq_hw_ctx *hctx = rq->mq_hctx; |
Ming Lei | e97c293 | 2014-09-25 23:23:46 +0800 | [diff] [blame] | 329 | struct blk_mq_ctx *ctx = rq->mq_ctx; |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 330 | unsigned long flags; |
Ming Lei | e97c293 | 2014-09-25 23:23:46 +0800 | [diff] [blame] | 331 | struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx); |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 332 | |
Jens Axboe | 4e2f62e5 | 2020-07-01 22:58:32 -0600 | [diff] [blame] | 333 | if (q->elevator) { |
| 334 | WARN_ON(rq->tag < 0); |
| 335 | blk_mq_put_driver_tag(rq); |
| 336 | } |
| 337 | |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 338 | /* |
| 339 | * After populating an empty queue, kick it to avoid stall. Read |
| 340 | * the comment in flush_end_io(). |
| 341 | */ |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 342 | spin_lock_irqsave(&fq->mq_flush_lock, flags); |
Jens Axboe | bd166ef | 2017-01-17 06:03:22 -0700 | [diff] [blame] | 343 | blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error); |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 344 | spin_unlock_irqrestore(&fq->mq_flush_lock, flags); |
Jens Axboe | bd166ef | 2017-01-17 06:03:22 -0700 | [diff] [blame] | 345 | |
Jianchao Wang | 85bd6e6 | 2019-01-30 17:01:56 +0800 | [diff] [blame] | 346 | blk_mq_sched_restart(hctx); |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 347 | } |
| 348 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 349 | /** |
Omar Sandoval | 3140c3c | 2017-08-24 11:09:25 -0700 | [diff] [blame] | 350 | * blk_insert_flush - insert a new PREFLUSH/FUA request |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 351 | * @rq: request to insert |
| 352 | * |
Jens Axboe | b710a48 | 2011-03-30 09:52:30 +0200 | [diff] [blame] | 353 | * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions. |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 354 | * or __blk_mq_run_hw_queue() to dispatch request. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 355 | * @rq is being submitted. Analyze what needs to be done and put it on the |
| 356 | * right queue. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 357 | */ |
| 358 | void blk_insert_flush(struct request *rq) |
| 359 | { |
| 360 | struct request_queue *q = rq->q; |
Jens Axboe | c888a8f | 2016-04-13 13:33:19 -0600 | [diff] [blame] | 361 | unsigned long fflags = q->queue_flags; /* may change, cache */ |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 362 | unsigned int policy = blk_flush_policy(fflags, rq); |
Ming Lei | e97c293 | 2014-09-25 23:23:46 +0800 | [diff] [blame] | 363 | struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx); |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 364 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 365 | /* |
| 366 | * @policy now records what operations need to be done. Adjust |
Mike Christie | 28a8f0d | 2016-06-05 14:32:25 -0500 | [diff] [blame] | 367 | * REQ_PREFLUSH and FUA for the driver. |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 368 | */ |
Mike Christie | 28a8f0d | 2016-06-05 14:32:25 -0500 | [diff] [blame] | 369 | rq->cmd_flags &= ~REQ_PREFLUSH; |
Jens Axboe | c888a8f | 2016-04-13 13:33:19 -0600 | [diff] [blame] | 370 | if (!(fflags & (1UL << QUEUE_FLAG_FUA))) |
Tejun Heo | 4fed947 | 2010-09-03 11:56:17 +0200 | [diff] [blame] | 371 | rq->cmd_flags &= ~REQ_FUA; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 372 | |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 373 | /* |
Jens Axboe | ae5b2ec | 2016-11-08 19:39:28 -0700 | [diff] [blame] | 374 | * REQ_PREFLUSH|REQ_FUA implies REQ_SYNC, so if we clear any |
| 375 | * of those flags, we have to set REQ_SYNC to avoid skewing |
| 376 | * the request accounting. |
| 377 | */ |
| 378 | rq->cmd_flags |= REQ_SYNC; |
| 379 | |
| 380 | /* |
Jeff Moyer | 4853aba | 2011-08-15 21:37:25 +0200 | [diff] [blame] | 381 | * An empty flush handed down from a stacking driver may |
| 382 | * translate into nothing if the underlying device does not |
| 383 | * advertise a write-back cache. In this case, simply |
| 384 | * complete the request. |
| 385 | */ |
| 386 | if (!policy) { |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 387 | blk_mq_end_request(rq, 0); |
Jeff Moyer | 4853aba | 2011-08-15 21:37:25 +0200 | [diff] [blame] | 388 | return; |
| 389 | } |
| 390 | |
Jeff Moyer | 834f9f6 | 2011-10-17 12:57:22 +0200 | [diff] [blame] | 391 | BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */ |
Jeff Moyer | 4853aba | 2011-08-15 21:37:25 +0200 | [diff] [blame] | 392 | |
| 393 | /* |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 394 | * If there's data but flush is not necessary, the request can be |
| 395 | * processed directly without going through flush machinery. Queue |
| 396 | * for normal execution. |
| 397 | */ |
| 398 | if ((policy & REQ_FSEQ_DATA) && |
| 399 | !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) { |
Ming Lei | 01e99ae | 2020-02-25 09:04:32 +0800 | [diff] [blame] | 400 | blk_mq_request_bypass_insert(rq, false, false); |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 401 | return; |
| 402 | } |
| 403 | |
| 404 | /* |
| 405 | * @rq should go through flush machinery. Mark it part of flush |
| 406 | * sequence and submit for further processing. |
| 407 | */ |
| 408 | memset(&rq->flush, 0, sizeof(rq->flush)); |
| 409 | INIT_LIST_HEAD(&rq->flush.list); |
Christoph Hellwig | e806402 | 2016-10-20 15:12:13 +0200 | [diff] [blame] | 410 | rq->rq_flags |= RQF_FLUSH_SEQ; |
Jeff Moyer | 4853aba | 2011-08-15 21:37:25 +0200 | [diff] [blame] | 411 | rq->flush.saved_end_io = rq->end_io; /* Usually NULL */ |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 412 | |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 413 | rq->end_io = mq_flush_data_end_io; |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 414 | |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 415 | spin_lock_irq(&fq->mq_flush_lock); |
Ming Lei | 0bae352 | 2014-09-25 23:23:45 +0800 | [diff] [blame] | 416 | blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 417 | spin_unlock_irq(&fq->mq_flush_lock); |
Tejun Heo | ae1b153 | 2011-01-25 12:43:54 +0100 | [diff] [blame] | 418 | } |
| 419 | |
| 420 | /** |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 421 | * blkdev_issue_flush - queue a flush |
| 422 | * @bdev: blockdev to issue flush for |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 423 | * @gfp_mask: memory allocation flags (for bio_alloc) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 424 | * |
| 425 | * Description: |
Christoph Hellwig | 9398554 | 2020-05-13 14:36:00 +0200 | [diff] [blame] | 426 | * Issue a flush for the block device in question. |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 427 | */ |
Christoph Hellwig | 9398554 | 2020-05-13 14:36:00 +0200 | [diff] [blame] | 428 | int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 429 | { |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 430 | struct bio *bio; |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 431 | int ret = 0; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 432 | |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 433 | bio = bio_alloc(gfp_mask, 0); |
Christoph Hellwig | 74d4699 | 2017-08-23 19:10:32 +0200 | [diff] [blame] | 434 | bio_set_dev(bio, bdev); |
Christoph Hellwig | 70fd761 | 2016-11-01 07:40:10 -0600 | [diff] [blame] | 435 | bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; |
Dmitry Monakhov | f17e232 | 2010-04-28 17:55:07 +0400 | [diff] [blame] | 436 | |
Mike Christie | 4e49ea4 | 2016-06-05 14:31:41 -0500 | [diff] [blame] | 437 | ret = submit_bio_wait(bio); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 438 | bio_put(bio); |
| 439 | return ret; |
| 440 | } |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 441 | EXPORT_SYMBOL(blkdev_issue_flush); |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 442 | |
Guoqing Jiang | 754a157 | 2020-03-09 22:41:37 +0100 | [diff] [blame] | 443 | struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size, |
| 444 | gfp_t flags) |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 445 | { |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 446 | struct blk_flush_queue *fq; |
| 447 | int rq_sz = sizeof(struct request); |
Ming Lei | 1bcb1ea | 2014-09-25 23:23:39 +0800 | [diff] [blame] | 448 | |
Jianchao Wang | 5b20285 | 2018-10-12 18:07:26 +0800 | [diff] [blame] | 449 | fq = kzalloc_node(sizeof(*fq), flags, node); |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 450 | if (!fq) |
| 451 | goto fail; |
Ming Lei | 1bcb1ea | 2014-09-25 23:23:39 +0800 | [diff] [blame] | 452 | |
Jens Axboe | 7e992f8 | 2018-10-13 15:44:12 -0600 | [diff] [blame] | 453 | spin_lock_init(&fq->mq_flush_lock); |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 454 | |
Christoph Hellwig | 6d247d7 | 2017-01-27 09:51:45 -0700 | [diff] [blame] | 455 | rq_sz = round_up(rq_sz + cmd_size, cache_line_size()); |
Jianchao Wang | 5b20285 | 2018-10-12 18:07:26 +0800 | [diff] [blame] | 456 | fq->flush_rq = kzalloc_node(rq_sz, flags, node); |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 457 | if (!fq->flush_rq) |
| 458 | goto fail_rq; |
| 459 | |
| 460 | INIT_LIST_HEAD(&fq->flush_queue[0]); |
| 461 | INIT_LIST_HEAD(&fq->flush_queue[1]); |
| 462 | INIT_LIST_HEAD(&fq->flush_data_in_flight); |
| 463 | |
Bart Van Assche | b3c6a59 | 2019-12-17 16:24:35 -0800 | [diff] [blame] | 464 | lockdep_register_key(&fq->key); |
| 465 | lockdep_set_class(&fq->mq_flush_lock, &fq->key); |
| 466 | |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 467 | return fq; |
| 468 | |
| 469 | fail_rq: |
| 470 | kfree(fq); |
| 471 | fail: |
| 472 | return NULL; |
| 473 | } |
| 474 | |
Ming Lei | ba48338 | 2014-09-25 23:23:44 +0800 | [diff] [blame] | 475 | void blk_free_flush_queue(struct blk_flush_queue *fq) |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 476 | { |
| 477 | /* bio based request queue hasn't flush queue */ |
| 478 | if (!fq) |
| 479 | return; |
| 480 | |
Bart Van Assche | b3c6a59 | 2019-12-17 16:24:35 -0800 | [diff] [blame] | 481 | lockdep_unregister_key(&fq->key); |
Ming Lei | 7c94e1c | 2014-09-25 23:23:43 +0800 | [diff] [blame] | 482 | kfree(fq->flush_rq); |
| 483 | kfree(fq); |
Jens Axboe | 320ae51 | 2013-10-24 09:20:05 +0100 | [diff] [blame] | 484 | } |