Christoph Hellwig | 3dcf60bc | 2019-04-30 14:42:43 -0400 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 2 | /* |
| 3 | * Block stat tracking code |
| 4 | * |
| 5 | * Copyright (C) 2016 Jens Axboe |
| 6 | */ |
| 7 | #include <linux/kernel.h> |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 8 | #include <linux/rculist.h> |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 9 | |
| 10 | #include "blk-stat.h" |
| 11 | #include "blk-mq.h" |
Shaohua Li | b9147dd | 2017-03-27 15:19:42 -0700 | [diff] [blame] | 12 | #include "blk.h" |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 13 | |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 14 | struct blk_queue_stats { |
| 15 | struct list_head callbacks; |
| 16 | spinlock_t lock; |
Jens Axboe | 6849709 | 2021-12-14 17:23:05 -0700 | [diff] [blame] | 17 | int accounting; |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 18 | }; |
| 19 | |
Josef Bacik | 2ecbf45 | 2018-07-03 11:14:57 -0400 | [diff] [blame] | 20 | void blk_rq_stat_init(struct blk_rq_stat *stat) |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 21 | { |
| 22 | stat->min = -1ULL; |
| 23 | stat->max = stat->nr_samples = stat->mean = 0; |
Shaohua Li | eca8b53 | 2017-10-06 17:55:59 -0700 | [diff] [blame] | 24 | stat->batch = 0; |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 25 | } |
| 26 | |
Shaohua Li | eca8b53 | 2017-10-06 17:55:59 -0700 | [diff] [blame] | 27 | /* src is a per-cpu stat, mean isn't initialized */ |
Josef Bacik | 2ecbf45 | 2018-07-03 11:14:57 -0400 | [diff] [blame] | 28 | void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 29 | { |
Roman Smirnov | 93f52fb | 2024-03-05 16:45:09 +0300 | [diff] [blame] | 30 | if (dst->nr_samples + src->nr_samples <= dst->nr_samples) |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 31 | return; |
| 32 | |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 33 | dst->min = min(dst->min, src->min); |
| 34 | dst->max = max(dst->max, src->max); |
| 35 | |
Shaohua Li | eca8b53 | 2017-10-06 17:55:59 -0700 | [diff] [blame] | 36 | dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples, |
| 37 | dst->nr_samples + src->nr_samples); |
| 38 | |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 39 | dst->nr_samples += src->nr_samples; |
| 40 | } |
| 41 | |
Josef Bacik | 2ecbf45 | 2018-07-03 11:14:57 -0400 | [diff] [blame] | 42 | void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value) |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 43 | { |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 44 | stat->min = min(stat->min, value); |
| 45 | stat->max = max(stat->max, value); |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 46 | stat->batch += value; |
Shaohua Li | eca8b53 | 2017-10-06 17:55:59 -0700 | [diff] [blame] | 47 | stat->nr_samples++; |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 48 | } |
| 49 | |
Omar Sandoval | 522a777 | 2018-05-09 02:08:53 -0700 | [diff] [blame] | 50 | void blk_stat_add(struct request *rq, u64 now) |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 51 | { |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 52 | struct request_queue *q = rq->q; |
| 53 | struct blk_stat_callback *cb; |
| 54 | struct blk_rq_stat *stat; |
Pavel Begunkov | 8148f0b | 2019-10-08 00:16:51 +0300 | [diff] [blame] | 55 | int bucket, cpu; |
Omar Sandoval | 522a777 | 2018-05-09 02:08:53 -0700 | [diff] [blame] | 56 | u64 value; |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 57 | |
Omar Sandoval | 544ccc8d | 2018-05-09 02:08:50 -0700 | [diff] [blame] | 58 | value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0; |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 59 | |
| 60 | rcu_read_lock(); |
Pavel Begunkov | 8148f0b | 2019-10-08 00:16:51 +0300 | [diff] [blame] | 61 | cpu = get_cpu(); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 62 | list_for_each_entry_rcu(cb, &q->stats->callbacks, list) { |
Jens Axboe | d373812 | 2017-05-09 11:39:56 -0600 | [diff] [blame] | 63 | if (!blk_stat_is_active(cb)) |
| 64 | continue; |
| 65 | |
| 66 | bucket = cb->bucket_fn(rq); |
| 67 | if (bucket < 0) |
| 68 | continue; |
| 69 | |
Pavel Begunkov | 8148f0b | 2019-10-08 00:16:51 +0300 | [diff] [blame] | 70 | stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket]; |
Josef Bacik | 2ecbf45 | 2018-07-03 11:14:57 -0400 | [diff] [blame] | 71 | blk_rq_stat_add(stat, value); |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 72 | } |
Pavel Begunkov | 8148f0b | 2019-10-08 00:16:51 +0300 | [diff] [blame] | 73 | put_cpu(); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 74 | rcu_read_unlock(); |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 75 | } |
| 76 | |
Kees Cook | e99e88a | 2017-10-16 14:43:17 -0700 | [diff] [blame] | 77 | static void blk_stat_timer_fn(struct timer_list *t) |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 78 | { |
Kees Cook | e99e88a | 2017-10-16 14:43:17 -0700 | [diff] [blame] | 79 | struct blk_stat_callback *cb = from_timer(cb, t, timer); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 80 | unsigned int bucket; |
| 81 | int cpu; |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 82 | |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 83 | for (bucket = 0; bucket < cb->buckets; bucket++) |
Josef Bacik | 2ecbf45 | 2018-07-03 11:14:57 -0400 | [diff] [blame] | 84 | blk_rq_stat_init(&cb->stat[bucket]); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 85 | |
| 86 | for_each_online_cpu(cpu) { |
| 87 | struct blk_rq_stat *cpu_stat; |
| 88 | |
| 89 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
| 90 | for (bucket = 0; bucket < cb->buckets; bucket++) { |
Josef Bacik | 2ecbf45 | 2018-07-03 11:14:57 -0400 | [diff] [blame] | 91 | blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]); |
| 92 | blk_rq_stat_init(&cpu_stat[bucket]); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 93 | } |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 94 | } |
| 95 | |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 96 | cb->timer_fn(cb); |
| 97 | } |
| 98 | |
| 99 | struct blk_stat_callback * |
| 100 | blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), |
Stephen Bates | a37244e | 2017-04-20 15:29:16 -0600 | [diff] [blame] | 101 | int (*bucket_fn)(const struct request *), |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 102 | unsigned int buckets, void *data) |
| 103 | { |
| 104 | struct blk_stat_callback *cb; |
| 105 | |
| 106 | cb = kmalloc(sizeof(*cb), GFP_KERNEL); |
| 107 | if (!cb) |
| 108 | return NULL; |
| 109 | |
| 110 | cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat), |
| 111 | GFP_KERNEL); |
| 112 | if (!cb->stat) { |
| 113 | kfree(cb); |
| 114 | return NULL; |
| 115 | } |
| 116 | cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat), |
| 117 | __alignof__(struct blk_rq_stat)); |
| 118 | if (!cb->cpu_stat) { |
| 119 | kfree(cb->stat); |
| 120 | kfree(cb); |
| 121 | return NULL; |
| 122 | } |
| 123 | |
| 124 | cb->timer_fn = timer_fn; |
| 125 | cb->bucket_fn = bucket_fn; |
| 126 | cb->data = data; |
| 127 | cb->buckets = buckets; |
Kees Cook | e99e88a | 2017-10-16 14:43:17 -0700 | [diff] [blame] | 128 | timer_setup(&cb->timer, blk_stat_timer_fn, 0); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 129 | |
| 130 | return cb; |
| 131 | } |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 132 | |
| 133 | void blk_stat_add_callback(struct request_queue *q, |
| 134 | struct blk_stat_callback *cb) |
| 135 | { |
| 136 | unsigned int bucket; |
Tejun Heo | e11d80a | 2020-09-01 14:52:32 -0400 | [diff] [blame] | 137 | unsigned long flags; |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 138 | int cpu; |
| 139 | |
| 140 | for_each_possible_cpu(cpu) { |
| 141 | struct blk_rq_stat *cpu_stat; |
| 142 | |
| 143 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
| 144 | for (bucket = 0; bucket < cb->buckets; bucket++) |
Josef Bacik | 2ecbf45 | 2018-07-03 11:14:57 -0400 | [diff] [blame] | 145 | blk_rq_stat_init(&cpu_stat[bucket]); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 146 | } |
| 147 | |
Tejun Heo | e11d80a | 2020-09-01 14:52:32 -0400 | [diff] [blame] | 148 | spin_lock_irqsave(&q->stats->lock, flags); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 149 | list_add_tail_rcu(&cb->list, &q->stats->callbacks); |
Bart Van Assche | 7dfdbc7 | 2018-03-07 17:10:05 -0800 | [diff] [blame] | 150 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
Tejun Heo | e11d80a | 2020-09-01 14:52:32 -0400 | [diff] [blame] | 151 | spin_unlock_irqrestore(&q->stats->lock, flags); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 152 | } |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 153 | |
| 154 | void blk_stat_remove_callback(struct request_queue *q, |
| 155 | struct blk_stat_callback *cb) |
| 156 | { |
Tejun Heo | e11d80a | 2020-09-01 14:52:32 -0400 | [diff] [blame] | 157 | unsigned long flags; |
| 158 | |
| 159 | spin_lock_irqsave(&q->stats->lock, flags); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 160 | list_del_rcu(&cb->list); |
Jens Axboe | 6849709 | 2021-12-14 17:23:05 -0700 | [diff] [blame] | 161 | if (list_empty(&q->stats->callbacks) && !q->stats->accounting) |
Bart Van Assche | 7dfdbc7 | 2018-03-07 17:10:05 -0800 | [diff] [blame] | 162 | blk_queue_flag_clear(QUEUE_FLAG_STATS, q); |
Tejun Heo | e11d80a | 2020-09-01 14:52:32 -0400 | [diff] [blame] | 163 | spin_unlock_irqrestore(&q->stats->lock, flags); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 164 | |
| 165 | del_timer_sync(&cb->timer); |
| 166 | } |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 167 | |
| 168 | static void blk_stat_free_callback_rcu(struct rcu_head *head) |
| 169 | { |
| 170 | struct blk_stat_callback *cb; |
| 171 | |
| 172 | cb = container_of(head, struct blk_stat_callback, rcu); |
| 173 | free_percpu(cb->cpu_stat); |
| 174 | kfree(cb->stat); |
| 175 | kfree(cb); |
| 176 | } |
| 177 | |
| 178 | void blk_stat_free_callback(struct blk_stat_callback *cb) |
| 179 | { |
Jens Axboe | a83b576 | 2017-03-21 17:20:01 -0600 | [diff] [blame] | 180 | if (cb) |
| 181 | call_rcu(&cb->rcu, blk_stat_free_callback_rcu); |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 182 | } |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 183 | |
Jens Axboe | 6849709 | 2021-12-14 17:23:05 -0700 | [diff] [blame] | 184 | void blk_stat_disable_accounting(struct request_queue *q) |
| 185 | { |
| 186 | unsigned long flags; |
| 187 | |
| 188 | spin_lock_irqsave(&q->stats->lock, flags); |
Chengming Zhou | 20de765 | 2023-04-13 14:28:04 +0800 | [diff] [blame] | 189 | if (!--q->stats->accounting && list_empty(&q->stats->callbacks)) |
Jens Axboe | 6849709 | 2021-12-14 17:23:05 -0700 | [diff] [blame] | 190 | blk_queue_flag_clear(QUEUE_FLAG_STATS, q); |
| 191 | spin_unlock_irqrestore(&q->stats->lock, flags); |
| 192 | } |
| 193 | EXPORT_SYMBOL_GPL(blk_stat_disable_accounting); |
| 194 | |
Shaohua Li | b9147dd | 2017-03-27 15:19:42 -0700 | [diff] [blame] | 195 | void blk_stat_enable_accounting(struct request_queue *q) |
| 196 | { |
Tejun Heo | e11d80a | 2020-09-01 14:52:32 -0400 | [diff] [blame] | 197 | unsigned long flags; |
| 198 | |
| 199 | spin_lock_irqsave(&q->stats->lock, flags); |
Chengming Zhou | 20de765 | 2023-04-13 14:28:04 +0800 | [diff] [blame] | 200 | if (!q->stats->accounting++ && list_empty(&q->stats->callbacks)) |
Jens Axboe | 6849709 | 2021-12-14 17:23:05 -0700 | [diff] [blame] | 201 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
Tejun Heo | e11d80a | 2020-09-01 14:52:32 -0400 | [diff] [blame] | 202 | spin_unlock_irqrestore(&q->stats->lock, flags); |
Shaohua Li | b9147dd | 2017-03-27 15:19:42 -0700 | [diff] [blame] | 203 | } |
Omar Sandoval | f8232f2 | 2018-09-27 15:55:52 -0700 | [diff] [blame] | 204 | EXPORT_SYMBOL_GPL(blk_stat_enable_accounting); |
Shaohua Li | b9147dd | 2017-03-27 15:19:42 -0700 | [diff] [blame] | 205 | |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 206 | struct blk_queue_stats *blk_alloc_queue_stats(void) |
| 207 | { |
| 208 | struct blk_queue_stats *stats; |
| 209 | |
| 210 | stats = kmalloc(sizeof(*stats), GFP_KERNEL); |
| 211 | if (!stats) |
| 212 | return NULL; |
| 213 | |
| 214 | INIT_LIST_HEAD(&stats->callbacks); |
| 215 | spin_lock_init(&stats->lock); |
Jens Axboe | 6849709 | 2021-12-14 17:23:05 -0700 | [diff] [blame] | 216 | stats->accounting = 0; |
Omar Sandoval | 34dbad5 | 2017-03-21 08:56:08 -0700 | [diff] [blame] | 217 | |
| 218 | return stats; |
| 219 | } |
| 220 | |
| 221 | void blk_free_queue_stats(struct blk_queue_stats *stats) |
| 222 | { |
| 223 | if (!stats) |
| 224 | return; |
| 225 | |
| 226 | WARN_ON(!list_empty(&stats->callbacks)); |
| 227 | |
| 228 | kfree(stats); |
Jens Axboe | cf43e6b | 2016-11-07 21:32:37 -0700 | [diff] [blame] | 229 | } |