Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* |
| 3 | * Block rq-qos policy for assigning an I/O priority class to requests. |
| 4 | * |
| 5 | * Using an rq-qos policy for assigning I/O priority class has two advantages |
| 6 | * over using the ioprio_set() system call: |
| 7 | * |
| 8 | * - This policy is cgroup based so it has all the advantages of cgroups. |
| 9 | * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos |
| 10 | * controller affects page cache writeback I/O for filesystems that support |
| 11 | * assiociating a cgroup with writeback I/O. See also |
| 12 | * Documentation/admin-guide/cgroup-v2.rst. |
| 13 | */ |
| 14 | |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 15 | #include <linux/blk-mq.h> |
| 16 | #include <linux/blk_types.h> |
| 17 | #include <linux/kernel.h> |
| 18 | #include <linux/module.h> |
Ming Lei | 672fdcf | 2022-02-11 18:11:49 +0800 | [diff] [blame] | 19 | #include "blk-cgroup.h" |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 20 | #include "blk-ioprio.h" |
| 21 | #include "blk-rq-qos.h" |
| 22 | |
| 23 | /** |
| 24 | * enum prio_policy - I/O priority class policy. |
| 25 | * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class. |
| 26 | * @POLICY_NONE_TO_RT: modify IOPRIO_CLASS_NONE into IOPRIO_CLASS_RT. |
| 27 | * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into |
| 28 | * IOPRIO_CLASS_BE. |
| 29 | * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE. |
| 30 | * |
| 31 | * See also <linux/ioprio.h>. |
| 32 | */ |
| 33 | enum prio_policy { |
| 34 | POLICY_NO_CHANGE = 0, |
| 35 | POLICY_NONE_TO_RT = 1, |
| 36 | POLICY_RESTRICT_TO_BE = 2, |
| 37 | POLICY_ALL_TO_IDLE = 3, |
| 38 | }; |
| 39 | |
| 40 | static const char *policy_name[] = { |
| 41 | [POLICY_NO_CHANGE] = "no-change", |
| 42 | [POLICY_NONE_TO_RT] = "none-to-rt", |
| 43 | [POLICY_RESTRICT_TO_BE] = "restrict-to-be", |
| 44 | [POLICY_ALL_TO_IDLE] = "idle", |
| 45 | }; |
| 46 | |
| 47 | static struct blkcg_policy ioprio_policy; |
| 48 | |
| 49 | /** |
| 50 | * struct ioprio_blkg - Per (cgroup, request queue) data. |
| 51 | * @pd: blkg_policy_data structure. |
| 52 | */ |
| 53 | struct ioprio_blkg { |
| 54 | struct blkg_policy_data pd; |
| 55 | }; |
| 56 | |
| 57 | /** |
| 58 | * struct ioprio_blkcg - Per cgroup data. |
| 59 | * @cpd: blkcg_policy_data structure. |
| 60 | * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>. |
| 61 | */ |
| 62 | struct ioprio_blkcg { |
| 63 | struct blkcg_policy_data cpd; |
| 64 | enum prio_policy prio_policy; |
| 65 | }; |
| 66 | |
| 67 | static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd) |
| 68 | { |
| 69 | return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL; |
| 70 | } |
| 71 | |
| 72 | static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg) |
| 73 | { |
| 74 | return container_of(blkcg_to_cpd(blkcg, &ioprio_policy), |
| 75 | struct ioprio_blkcg, cpd); |
| 76 | } |
| 77 | |
| 78 | static struct ioprio_blkcg * |
| 79 | ioprio_blkcg_from_css(struct cgroup_subsys_state *css) |
| 80 | { |
| 81 | return blkcg_to_ioprio_blkcg(css_to_blkcg(css)); |
| 82 | } |
| 83 | |
| 84 | static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio) |
| 85 | { |
| 86 | struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy); |
| 87 | |
| 88 | if (!pd) |
| 89 | return NULL; |
| 90 | |
| 91 | return blkcg_to_ioprio_blkcg(pd->blkg->blkcg); |
| 92 | } |
| 93 | |
| 94 | static int ioprio_show_prio_policy(struct seq_file *sf, void *v) |
| 95 | { |
| 96 | struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf)); |
| 97 | |
| 98 | seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]); |
| 99 | return 0; |
| 100 | } |
| 101 | |
| 102 | static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, |
| 103 | size_t nbytes, loff_t off) |
| 104 | { |
| 105 | struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of)); |
| 106 | int ret; |
| 107 | |
| 108 | if (off != 0) |
| 109 | return -EIO; |
| 110 | /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */ |
| 111 | ret = sysfs_match_string(policy_name, buf); |
| 112 | if (ret < 0) |
| 113 | return ret; |
| 114 | blkcg->prio_policy = ret; |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 115 | return nbytes; |
| 116 | } |
| 117 | |
| 118 | static struct blkg_policy_data * |
Christoph Hellwig | 0a0b4f7 | 2023-02-03 16:03:58 +0100 | [diff] [blame] | 119 | ioprio_alloc_pd(struct gendisk *disk, struct blkcg *blkcg, gfp_t gfp) |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 120 | { |
| 121 | struct ioprio_blkg *ioprio_blkg; |
| 122 | |
| 123 | ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp); |
| 124 | if (!ioprio_blkg) |
| 125 | return NULL; |
| 126 | |
| 127 | return &ioprio_blkg->pd; |
| 128 | } |
| 129 | |
| 130 | static void ioprio_free_pd(struct blkg_policy_data *pd) |
| 131 | { |
| 132 | struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd); |
| 133 | |
| 134 | kfree(ioprio_blkg); |
| 135 | } |
| 136 | |
| 137 | static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp) |
| 138 | { |
| 139 | struct ioprio_blkcg *blkcg; |
| 140 | |
| 141 | blkcg = kzalloc(sizeof(*blkcg), gfp); |
| 142 | if (!blkcg) |
| 143 | return NULL; |
| 144 | blkcg->prio_policy = POLICY_NO_CHANGE; |
| 145 | return &blkcg->cpd; |
| 146 | } |
| 147 | |
| 148 | static void ioprio_free_cpd(struct blkcg_policy_data *cpd) |
| 149 | { |
| 150 | struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd); |
| 151 | |
| 152 | kfree(blkcg); |
| 153 | } |
| 154 | |
| 155 | #define IOPRIO_ATTRS \ |
| 156 | { \ |
| 157 | .name = "prio.class", \ |
| 158 | .seq_show = ioprio_show_prio_policy, \ |
| 159 | .write = ioprio_set_prio_policy, \ |
| 160 | }, \ |
| 161 | { } /* sentinel */ |
| 162 | |
| 163 | /* cgroup v2 attributes */ |
| 164 | static struct cftype ioprio_files[] = { |
| 165 | IOPRIO_ATTRS |
| 166 | }; |
| 167 | |
| 168 | /* cgroup v1 attributes */ |
| 169 | static struct cftype ioprio_legacy_files[] = { |
| 170 | IOPRIO_ATTRS |
| 171 | }; |
| 172 | |
| 173 | static struct blkcg_policy ioprio_policy = { |
| 174 | .dfl_cftypes = ioprio_files, |
| 175 | .legacy_cftypes = ioprio_legacy_files, |
| 176 | |
| 177 | .cpd_alloc_fn = ioprio_alloc_cpd, |
| 178 | .cpd_free_fn = ioprio_free_cpd, |
| 179 | |
| 180 | .pd_alloc_fn = ioprio_alloc_pd, |
| 181 | .pd_free_fn = ioprio_free_pd, |
| 182 | }; |
| 183 | |
Jan Kara | 82b74ca | 2022-06-23 09:48:32 +0200 | [diff] [blame] | 184 | void blkcg_set_ioprio(struct bio *bio) |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 185 | { |
| 186 | struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio); |
Jens Axboe | 25c4b5e | 2021-11-13 13:37:38 -0700 | [diff] [blame] | 187 | u16 prio; |
| 188 | |
Jan Kara | 82b74ca | 2022-06-23 09:48:32 +0200 | [diff] [blame] | 189 | if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE) |
Jens Axboe | 25c4b5e | 2021-11-13 13:37:38 -0700 | [diff] [blame] | 190 | return; |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 191 | |
| 192 | /* |
| 193 | * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers |
| 194 | * correspond to a lower priority. Hence, the max_t() below selects |
| 195 | * the lower priority of bi_ioprio and the cgroup I/O priority class. |
Jan Kara | f258654 | 2022-06-23 09:48:31 +0200 | [diff] [blame] | 196 | * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O |
| 197 | * priority is assigned to the bio. |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 198 | */ |
Jens Axboe | 25c4b5e | 2021-11-13 13:37:38 -0700 | [diff] [blame] | 199 | prio = max_t(u16, bio->bi_ioprio, |
| 200 | IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0)); |
| 201 | if (prio > bio->bi_ioprio) |
| 202 | bio->bi_ioprio = prio; |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 203 | } |
| 204 | |
Christoph Hellwig | b0dde3f | 2022-09-21 20:04:51 +0200 | [diff] [blame] | 205 | void blk_ioprio_exit(struct gendisk *disk) |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 206 | { |
Christoph Hellwig | 40e4996 | 2023-02-03 16:03:57 +0100 | [diff] [blame] | 207 | blkcg_deactivate_policy(disk, &ioprio_policy); |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 208 | } |
| 209 | |
Christoph Hellwig | b0dde3f | 2022-09-21 20:04:51 +0200 | [diff] [blame] | 210 | int blk_ioprio_init(struct gendisk *disk) |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 211 | { |
Christoph Hellwig | 40e4996 | 2023-02-03 16:03:57 +0100 | [diff] [blame] | 212 | return blkcg_activate_policy(disk, &ioprio_policy); |
Bart Van Assche | 556910e | 2021-06-17 17:44:44 -0700 | [diff] [blame] | 213 | } |
| 214 | |
| 215 | static int __init ioprio_init(void) |
| 216 | { |
| 217 | return blkcg_policy_register(&ioprio_policy); |
| 218 | } |
| 219 | |
| 220 | static void __exit ioprio_exit(void) |
| 221 | { |
| 222 | blkcg_policy_unregister(&ioprio_policy); |
| 223 | } |
| 224 | |
| 225 | module_init(ioprio_init); |
| 226 | module_exit(ioprio_exit); |