blob: 869af9d72bcf8cb4c9d036d9e3a25a77421f6307 [file] [log] [blame]
Christoph Hellwig3dcf60bc2019-04-30 14:42:43 -04001// SPDX-License-Identifier: GPL-2.0
Vivek Goyal31e4c282009-12-03 12:59:42 -05002/*
3 * Common Block IO controller cgroup interface
4 *
5 * Based on ideas and code from CFQ, CFS and BFQ:
6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
7 *
8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
9 * Paolo Valente <paolo.valente@unimore.it>
10 *
11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
12 * Nauman Rafique <nauman@google.com>
Arianna Avanzinie48453c2015-06-05 23:38:42 +020013 *
14 * For policy-specific per-blkcg data:
15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
16 * Arianna Avanzini <avanzini.arianna@gmail.com>
Vivek Goyal31e4c282009-12-03 12:59:42 -050017 */
18#include <linux/ioprio.h>
Vivek Goyal22084192009-12-03 12:59:49 -050019#include <linux/kdev_t.h>
Vivek Goyal9d6a9862009-12-04 10:36:41 -050020#include <linux/module.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010021#include <linux/sched/signal.h>
Stephen Rothwellaccee782009-12-07 19:29:39 +110022#include <linux/err.h>
Divyesh Shah91952912010-04-01 15:01:41 -070023#include <linux/blkdev.h>
Tejun Heo52ebea72015-05-22 17:13:37 -040024#include <linux/backing-dev.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090025#include <linux/slab.h>
Tejun Heo72e06c22012-03-05 13:15:00 -080026#include <linux/delay.h>
Tejun Heo9a9e8a22012-03-19 15:10:56 -070027#include <linux/atomic.h>
Tejun Heo36aa9e52015-08-18 14:55:31 -070028#include <linux/ctype.h>
Eric W. Biederman03248ad2022-02-09 12:20:45 -060029#include <linux/resume_user_mode.h>
Josef Bacikfd112c72019-07-09 14:41:29 -070030#include <linux/psi.h>
Christoph Hellwig82d981d2021-11-23 19:53:12 +010031#include <linux/part_stat.h>
Tejun Heo5efd6112012-03-05 13:15:12 -080032#include "blk.h"
Ming Lei672fdcf2022-02-11 18:11:49 +080033#include "blk-cgroup.h"
Bart Van Assche556910e2021-06-17 17:44:44 -070034#include "blk-ioprio.h"
Jens Axboea7b36ee2021-10-05 09:11:56 -060035#include "blk-throttle.h"
Vivek Goyal3e252062009-12-04 10:36:42 -050036
Tejun Heo838f13b2015-07-09 16:39:47 -040037/*
38 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
39 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
40 * policy [un]register operations including cgroup file additions /
41 * removals. Putting cgroup file registration outside blkcg_pol_mutex
42 * allows grabbing it from cgroup callbacks.
43 */
44static DEFINE_MUTEX(blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -070045static DEFINE_MUTEX(blkcg_pol_mutex);
Tejun Heo923adde2012-03-05 13:15:13 -080046
Arianna Avanzinie48453c2015-06-05 23:38:42 +020047struct blkcg blkcg_root;
Tejun Heo3c798392012-04-16 13:57:25 -070048EXPORT_SYMBOL_GPL(blkcg_root);
Vivek Goyal9d6a9862009-12-04 10:36:41 -050049
Tejun Heo496d5e72015-05-22 17:13:21 -040050struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
Tejun Heo9b0eb692019-06-27 13:39:48 -070051EXPORT_SYMBOL_GPL(blkcg_root_css);
Tejun Heo496d5e72015-05-22 17:13:21 -040052
Tejun Heo3c798392012-04-16 13:57:25 -070053static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
Tejun Heo035d10b2012-03-05 13:15:04 -080054
Tejun Heo7876f932015-07-09 16:39:49 -040055static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
56
Tejun Heo07b0fde2019-07-16 07:58:31 -070057bool blkcg_debug_stats = false;
Tejun Heod3f77df2019-06-27 13:39:52 -070058static struct workqueue_struct *blkcg_punt_bio_wq;
Josef Bacik903d23f2018-07-03 11:14:52 -040059
Yu Kuaia7317632021-07-07 09:56:49 +080060#define BLKG_DESTROY_BATCH_SIZE 64
61
Christoph Hellwigbc5fee92022-04-20 06:27:20 +020062/**
63 * blkcg_css - find the current css
64 *
65 * Find the css associated with either the kthread or the current task.
66 * This may return a dying css, so it is up to the caller to use tryget logic
67 * to confirm it is alive and well.
68 */
69static struct cgroup_subsys_state *blkcg_css(void)
70{
71 struct cgroup_subsys_state *css;
72
73 css = kthread_blkcg();
74 if (css)
75 return css;
76 return task_css(current, io_cgrp_id);
77}
78
Tejun Heoa2b16932012-04-13 13:11:33 -070079static bool blkcg_policy_enabled(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -070080 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -070081{
82 return pol && test_bit(pol->plid, q->blkcg_pols);
83}
84
Ming Leid578c772022-03-23 09:13:08 +080085static void blkg_free_workfn(struct work_struct *work)
Tejun Heo03814112012-03-05 13:15:14 -080086{
Ming Leid578c772022-03-23 09:13:08 +080087 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
88 free_work);
Tejun Heoe8989fa2012-03-05 13:15:20 -080089 int i;
Tejun Heo549d3aa2012-03-05 13:15:16 -080090
Tejun Heodb613672013-05-14 13:52:31 -070091 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo001bea72015-08-18 14:55:11 -070092 if (blkg->pd[i])
93 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
Tejun Heoe8989fa2012-03-05 13:15:20 -080094
Ming Lei0a9a25c2022-03-18 21:01:43 +080095 if (blkg->q)
96 blk_put_queue(blkg->q);
Tejun Heof7331642019-11-07 11:18:03 -080097 free_percpu(blkg->iostat_cpu);
Tejun Heoef069b92019-06-13 15:30:39 -070098 percpu_ref_exit(&blkg->refcnt);
Tejun Heo549d3aa2012-03-05 13:15:16 -080099 kfree(blkg);
Tejun Heo03814112012-03-05 13:15:14 -0800100}
101
Ming Leid578c772022-03-23 09:13:08 +0800102/**
103 * blkg_free - free a blkg
104 * @blkg: blkg to free
105 *
106 * Free @blkg which may be partially allocated.
107 */
108static void blkg_free(struct blkcg_gq *blkg)
109{
110 if (!blkg)
111 return;
112
113 /*
114 * Both ->pd_free_fn() and request queue's release handler may
115 * sleep, so free us by scheduling one work func
116 */
117 INIT_WORK(&blkg->free_work, blkg_free_workfn);
118 schedule_work(&blkg->free_work);
119}
120
Dennis Zhou7fcf2b02018-12-05 12:10:38 -0500121static void __blkg_release(struct rcu_head *rcu)
122{
123 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
124
Tejun Heod3f77df2019-06-27 13:39:52 -0700125 WARN_ON(!bio_list_empty(&blkg->async_bios));
126
Dennis Zhou7fcf2b02018-12-05 12:10:38 -0500127 /* release the blkcg and parent blkg refs this blkg has been holding */
128 css_put(&blkg->blkcg->css);
129 if (blkg->parent)
130 blkg_put(blkg->parent);
Dennis Zhou7fcf2b02018-12-05 12:10:38 -0500131 blkg_free(blkg);
132}
133
134/*
135 * A group is RCU protected, but having an rcu lock does not mean that one
136 * can access all the fields of blkg and assume these are valid. For
137 * example, don't try to follow throtl_data and request queue links.
138 *
139 * Having a reference to blkg under an rcu allows accesses to only values
140 * local to groups like group stats and group rate limits.
141 */
142static void blkg_release(struct percpu_ref *ref)
143{
144 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
145
146 call_rcu(&blkg->rcu_head, __blkg_release);
147}
148
Tejun Heod3f77df2019-06-27 13:39:52 -0700149static void blkg_async_bio_workfn(struct work_struct *work)
150{
151 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
152 async_bio_work);
153 struct bio_list bios = BIO_EMPTY_LIST;
154 struct bio *bio;
Xianting Tian192f1c62020-09-10 14:15:06 +0800155 struct blk_plug plug;
156 bool need_plug = false;
Tejun Heod3f77df2019-06-27 13:39:52 -0700157
158 /* as long as there are pending bios, @blkg can't go away */
159 spin_lock_bh(&blkg->async_bio_lock);
160 bio_list_merge(&bios, &blkg->async_bios);
161 bio_list_init(&blkg->async_bios);
162 spin_unlock_bh(&blkg->async_bio_lock);
163
Xianting Tian192f1c62020-09-10 14:15:06 +0800164 /* start plug only when bio_list contains at least 2 bios */
165 if (bios.head && bios.head->bi_next) {
166 need_plug = true;
167 blk_start_plug(&plug);
168 }
Tejun Heod3f77df2019-06-27 13:39:52 -0700169 while ((bio = bio_list_pop(&bios)))
170 submit_bio(bio);
Xianting Tian192f1c62020-09-10 14:15:06 +0800171 if (need_plug)
172 blk_finish_plug(&plug);
Tejun Heod3f77df2019-06-27 13:39:52 -0700173}
174
Tejun Heo03814112012-03-05 13:15:14 -0800175/**
Christoph Hellwigbbb1ebe2022-04-20 06:27:17 +0200176 * bio_blkcg_css - return the blkcg CSS associated with a bio
177 * @bio: target bio
178 *
179 * This returns the CSS for the blkcg associated with a bio, or %NULL if not
180 * associated. Callers are expected to either handle %NULL or know association
181 * has been done prior to calling this.
182 */
183struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio)
184{
185 if (!bio || !bio->bi_blkg)
186 return NULL;
187 return &bio->bi_blkg->blkcg->css;
188}
189EXPORT_SYMBOL_GPL(bio_blkcg_css);
190
191/**
Christoph Hellwig397c9f42022-04-20 06:27:14 +0200192 * blkcg_parent - get the parent of a blkcg
193 * @blkcg: blkcg of interest
194 *
195 * Return the parent blkcg of @blkcg. Can be called anytime.
196 */
197static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
198{
199 return css_to_blkcg(blkcg->css.parent);
200}
201
202/**
Tejun Heo03814112012-03-05 13:15:14 -0800203 * blkg_alloc - allocate a blkg
204 * @blkcg: block cgroup the new blkg is associated with
205 * @q: request_queue the new blkg is associated with
Tejun Heo15974992012-06-04 20:40:52 -0700206 * @gfp_mask: allocation mask to use
Tejun Heo03814112012-03-05 13:15:14 -0800207 *
Tejun Heoe8989fa2012-03-05 13:15:20 -0800208 * Allocate a new blkg assocating @blkcg and @q.
Tejun Heo03814112012-03-05 13:15:14 -0800209 */
Tejun Heo15974992012-06-04 20:40:52 -0700210static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
211 gfp_t gfp_mask)
Tejun Heo03814112012-03-05 13:15:14 -0800212{
Tejun Heo3c798392012-04-16 13:57:25 -0700213 struct blkcg_gq *blkg;
Tejun Heof7331642019-11-07 11:18:03 -0800214 int i, cpu;
Tejun Heo03814112012-03-05 13:15:14 -0800215
216 /* alloc and init base part */
Tejun Heo15974992012-06-04 20:40:52 -0700217 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
Tejun Heo03814112012-03-05 13:15:14 -0800218 if (!blkg)
219 return NULL;
220
Tejun Heoef069b92019-06-13 15:30:39 -0700221 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
222 goto err_free;
223
Tejun Heof7331642019-11-07 11:18:03 -0800224 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
225 if (!blkg->iostat_cpu)
Tejun Heo77ea7332015-08-18 14:55:24 -0700226 goto err_free;
227
Ming Lei0a9a25c2022-03-18 21:01:43 +0800228 if (!blk_get_queue(q))
229 goto err_free;
230
Tejun Heoc875f4d2012-03-05 13:15:22 -0800231 blkg->q = q;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800232 INIT_LIST_HEAD(&blkg->q_node);
Tejun Heod3f77df2019-06-27 13:39:52 -0700233 spin_lock_init(&blkg->async_bio_lock);
234 bio_list_init(&blkg->async_bios);
235 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
Tejun Heo03814112012-03-05 13:15:14 -0800236 blkg->blkcg = blkcg;
Tejun Heo03814112012-03-05 13:15:14 -0800237
Tejun Heof7331642019-11-07 11:18:03 -0800238 u64_stats_init(&blkg->iostat.sync);
239 for_each_possible_cpu(cpu)
240 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
241
Tejun Heo8bd435b2012-04-13 13:11:28 -0700242 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700243 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heoe8989fa2012-03-05 13:15:20 -0800244 struct blkg_policy_data *pd;
Tejun Heo03814112012-03-05 13:15:14 -0800245
Tejun Heoa2b16932012-04-13 13:11:33 -0700246 if (!blkcg_policy_enabled(q, pol))
Tejun Heoe8989fa2012-03-05 13:15:20 -0800247 continue;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800248
Tejun Heoe8989fa2012-03-05 13:15:20 -0800249 /* alloc per-policy data and attach it to blkg */
Tejun Heocf09a8e2019-08-28 15:05:51 -0700250 pd = pol->pd_alloc_fn(gfp_mask, q, blkcg);
Tejun Heoa0516612012-06-26 15:05:44 -0700251 if (!pd)
252 goto err_free;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800253
Tejun Heoe8989fa2012-03-05 13:15:20 -0800254 blkg->pd[i] = pd;
255 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -0800256 pd->plid = i;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800257 }
258
Tejun Heo03814112012-03-05 13:15:14 -0800259 return blkg;
Tejun Heoa0516612012-06-26 15:05:44 -0700260
261err_free:
262 blkg_free(blkg);
263 return NULL;
Tejun Heo03814112012-03-05 13:15:14 -0800264}
265
Tejun Heo24f29042015-08-18 14:55:17 -0700266struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
267 struct request_queue *q, bool update_hint)
Tejun Heo80fd9972012-04-13 14:50:53 -0700268{
Tejun Heo3c798392012-04-16 13:57:25 -0700269 struct blkcg_gq *blkg;
Tejun Heo80fd9972012-04-13 14:50:53 -0700270
Tejun Heoa6371202012-04-19 16:29:24 -0700271 /*
Tejun Heo86cde6b2013-01-09 08:05:10 -0800272 * Hint didn't match. Look up from the radix tree. Note that the
273 * hint can only be updated under queue_lock as otherwise @blkg
274 * could have already been removed from blkg_tree. The caller is
275 * responsible for grabbing queue_lock if @update_hint.
Tejun Heoa6371202012-04-19 16:29:24 -0700276 */
277 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800278 if (blkg && blkg->q == q) {
279 if (update_hint) {
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700280 lockdep_assert_held(&q->queue_lock);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800281 rcu_assign_pointer(blkcg->blkg_hint, blkg);
282 }
Tejun Heoa6371202012-04-19 16:29:24 -0700283 return blkg;
Tejun Heo86cde6b2013-01-09 08:05:10 -0800284 }
Tejun Heoa6371202012-04-19 16:29:24 -0700285
Tejun Heo80fd9972012-04-13 14:50:53 -0700286 return NULL;
287}
Tejun Heoae118892015-08-18 14:55:20 -0700288EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
Tejun Heo80fd9972012-04-13 14:50:53 -0700289
Tejun Heo15974992012-06-04 20:40:52 -0700290/*
Jens Axboed708f0d2017-03-29 11:25:48 -0600291 * If @new_blkg is %NULL, this function tries to allocate a new one as
292 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
Tejun Heo15974992012-06-04 20:40:52 -0700293 */
Tejun Heo86cde6b2013-01-09 08:05:10 -0800294static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
Jens Axboed708f0d2017-03-29 11:25:48 -0600295 struct request_queue *q,
296 struct blkcg_gq *new_blkg)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400297{
Jens Axboed708f0d2017-03-29 11:25:48 -0600298 struct blkcg_gq *blkg;
Tejun Heof427d902013-01-09 08:05:12 -0800299 int i, ret;
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400300
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700301 lockdep_assert_held(&q->queue_lock);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500302
Dennis Zhou0273ac32018-12-11 18:03:08 -0500303 /* request_queue is dying, do not create/recreate a blkg */
304 if (blk_queue_dying(q)) {
305 ret = -ENODEV;
306 goto err_free_blkg;
307 }
308
Tejun Heo7ee9c562012-03-05 13:15:11 -0800309 /* blkg holds a reference to blkcg */
Tejun Heoec903c02014-05-13 12:11:01 -0400310 if (!css_tryget_online(&blkcg->css)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700311 ret = -ENODEV;
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800312 goto err_free_blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700313 }
Tejun Heocd1604f2012-03-05 13:15:06 -0800314
Jens Axboed708f0d2017-03-29 11:25:48 -0600315 /* allocate */
316 if (!new_blkg) {
317 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
318 if (unlikely(!new_blkg)) {
319 ret = -ENOMEM;
Christoph Hellwig8c911f32020-07-01 11:06:21 +0200320 goto err_put_css;
Jens Axboed708f0d2017-03-29 11:25:48 -0600321 }
322 }
323 blkg = new_blkg;
Tahsin Erdogan7fc6b872017-03-09 00:05:31 -0800324
Tejun Heodb613672013-05-14 13:52:31 -0700325 /* link parent */
Tejun Heo3c547862013-01-09 08:05:10 -0800326 if (blkcg_parent(blkcg)) {
327 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
328 if (WARN_ON_ONCE(!blkg->parent)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700329 ret = -ENODEV;
Christoph Hellwig8c911f32020-07-01 11:06:21 +0200330 goto err_put_css;
Tejun Heo3c547862013-01-09 08:05:10 -0800331 }
332 blkg_get(blkg->parent);
333 }
334
Tejun Heodb613672013-05-14 13:52:31 -0700335 /* invoke per-policy init */
336 for (i = 0; i < BLKCG_MAX_POLS; i++) {
337 struct blkcg_policy *pol = blkcg_policy[i];
338
339 if (blkg->pd[i] && pol->pd_init_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700340 pol->pd_init_fn(blkg->pd[i]);
Tejun Heodb613672013-05-14 13:52:31 -0700341 }
342
343 /* insert */
Tejun Heoa6371202012-04-19 16:29:24 -0700344 spin_lock(&blkcg->lock);
345 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
346 if (likely(!ret)) {
347 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
348 list_add(&blkg->q_node, &q->blkg_list);
Tejun Heof427d902013-01-09 08:05:12 -0800349
350 for (i = 0; i < BLKCG_MAX_POLS; i++) {
351 struct blkcg_policy *pol = blkcg_policy[i];
352
353 if (blkg->pd[i] && pol->pd_online_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700354 pol->pd_online_fn(blkg->pd[i]);
Tejun Heof427d902013-01-09 08:05:12 -0800355 }
Tejun Heoa6371202012-04-19 16:29:24 -0700356 }
Tejun Heof427d902013-01-09 08:05:12 -0800357 blkg->online = true;
Tejun Heoa6371202012-04-19 16:29:24 -0700358 spin_unlock(&blkcg->lock);
359
Tejun Heoec13b1d2015-05-22 17:13:19 -0400360 if (!ret)
Tejun Heoa6371202012-04-19 16:29:24 -0700361 return blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700362
Tejun Heo3c547862013-01-09 08:05:10 -0800363 /* @blkg failed fully initialized, use the usual release path */
364 blkg_put(blkg);
365 return ERR_PTR(ret);
366
Jens Axboed708f0d2017-03-29 11:25:48 -0600367err_put_css:
Tejun Heo496fb782012-04-19 16:29:23 -0700368 css_put(&blkcg->css);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800369err_free_blkg:
Jens Axboed708f0d2017-03-29 11:25:48 -0600370 blkg_free(new_blkg);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800371 return ERR_PTR(ret);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500372}
Tejun Heo3c96cb32012-04-13 13:11:34 -0700373
Tejun Heo86cde6b2013-01-09 08:05:10 -0800374/**
Christoph Hellwig8c546282020-06-27 09:31:52 +0200375 * blkg_lookup_create - lookup blkg, try to create one if not there
Tejun Heo86cde6b2013-01-09 08:05:10 -0800376 * @blkcg: blkcg of interest
377 * @q: request_queue of interest
378 *
379 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
Tejun Heo3c547862013-01-09 08:05:10 -0800380 * create one. blkg creation is performed recursively from blkcg_root such
381 * that all non-root blkg's have access to the parent blkg. This function
Christoph Hellwig8c546282020-06-27 09:31:52 +0200382 * should be called under RCU read lock and takes @q->queue_lock.
Tejun Heo86cde6b2013-01-09 08:05:10 -0800383 *
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500384 * Returns the blkg or the closest blkg if blkg_create() fails as it walks
385 * down from root.
Tejun Heo86cde6b2013-01-09 08:05:10 -0800386 */
Christoph Hellwig8c546282020-06-27 09:31:52 +0200387static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
388 struct request_queue *q)
Tejun Heo3c96cb32012-04-13 13:11:34 -0700389{
Tejun Heo86cde6b2013-01-09 08:05:10 -0800390 struct blkcg_gq *blkg;
Christoph Hellwig8c546282020-06-27 09:31:52 +0200391 unsigned long flags;
Tejun Heo86cde6b2013-01-09 08:05:10 -0800392
393 WARN_ON_ONCE(!rcu_read_lock_held());
Tejun Heo86cde6b2013-01-09 08:05:10 -0800394
Christoph Hellwig8c546282020-06-27 09:31:52 +0200395 blkg = blkg_lookup(blkcg, q);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800396 if (blkg)
397 return blkg;
398
Christoph Hellwig8c546282020-06-27 09:31:52 +0200399 spin_lock_irqsave(&q->queue_lock, flags);
400 blkg = __blkg_lookup(blkcg, q, true);
401 if (blkg)
402 goto found;
403
Tejun Heo3c547862013-01-09 08:05:10 -0800404 /*
405 * Create blkgs walking down from blkcg_root to @blkcg, so that all
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500406 * non-root blkgs have access to their parents. Returns the closest
407 * blkg to the intended blkg should blkg_create() fail.
Tejun Heo3c547862013-01-09 08:05:10 -0800408 */
409 while (true) {
410 struct blkcg *pos = blkcg;
411 struct blkcg *parent = blkcg_parent(blkcg);
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500412 struct blkcg_gq *ret_blkg = q->root_blkg;
Tejun Heo3c547862013-01-09 08:05:10 -0800413
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500414 while (parent) {
415 blkg = __blkg_lookup(parent, q, false);
416 if (blkg) {
417 /* remember closest blkg */
418 ret_blkg = blkg;
419 break;
420 }
Tejun Heo3c547862013-01-09 08:05:10 -0800421 pos = parent;
422 parent = blkcg_parent(parent);
423 }
424
Jens Axboed708f0d2017-03-29 11:25:48 -0600425 blkg = blkg_create(pos, q, NULL);
Christoph Hellwig8c546282020-06-27 09:31:52 +0200426 if (IS_ERR(blkg)) {
427 blkg = ret_blkg;
428 break;
429 }
Dennis Zhoubeea9da2018-12-05 12:10:28 -0500430 if (pos == blkcg)
Christoph Hellwig8c546282020-06-27 09:31:52 +0200431 break;
Dennis Zhoub9789622018-12-05 12:10:27 -0500432 }
433
Christoph Hellwig8c546282020-06-27 09:31:52 +0200434found:
435 spin_unlock_irqrestore(&q->queue_lock, flags);
Dennis Zhoub9789622018-12-05 12:10:27 -0500436 return blkg;
437}
438
Tejun Heo3c798392012-04-16 13:57:25 -0700439static void blkg_destroy(struct blkcg_gq *blkg)
Tejun Heo72e06c22012-03-05 13:15:00 -0800440{
Tejun Heo3c798392012-04-16 13:57:25 -0700441 struct blkcg *blkcg = blkg->blkcg;
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -0400442 int i;
Tejun Heo03aa2642012-03-05 13:15:19 -0800443
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700444 lockdep_assert_held(&blkg->q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800445 lockdep_assert_held(&blkcg->lock);
Tejun Heo03aa2642012-03-05 13:15:19 -0800446
447 /* Something wrong if we are trying to remove same group twice */
Tejun Heoe8989fa2012-03-05 13:15:20 -0800448 WARN_ON_ONCE(list_empty(&blkg->q_node));
Tejun Heo9f13ef62012-03-05 13:15:21 -0800449 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
Tejun Heoa6371202012-04-19 16:29:24 -0700450
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -0400451 for (i = 0; i < BLKCG_MAX_POLS; i++) {
452 struct blkcg_policy *pol = blkcg_policy[i];
453
454 if (blkg->pd[i] && pol->pd_offline_fn)
455 pol->pd_offline_fn(blkg->pd[i]);
456 }
457
Tejun Heof427d902013-01-09 08:05:12 -0800458 blkg->online = false;
459
Tejun Heoa6371202012-04-19 16:29:24 -0700460 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
Tejun Heoe8989fa2012-03-05 13:15:20 -0800461 list_del_init(&blkg->q_node);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800462 hlist_del_init_rcu(&blkg->blkcg_node);
Tejun Heo03aa2642012-03-05 13:15:19 -0800463
Tejun Heo03aa2642012-03-05 13:15:19 -0800464 /*
Tejun Heoa6371202012-04-19 16:29:24 -0700465 * Both setting lookup hint to and clearing it from @blkg are done
466 * under queue_lock. If it's not pointing to @blkg now, it never
467 * will. Hint assignment itself can race safely.
468 */
Paul E. McKenneyec6c676a2014-02-17 13:35:57 -0800469 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
Tejun Heoa6371202012-04-19 16:29:24 -0700470 rcu_assign_pointer(blkcg->blkg_hint, NULL);
471
472 /*
Tejun Heo03aa2642012-03-05 13:15:19 -0800473 * Put the reference taken at the time of creation so that when all
474 * queues are gone, group can be destroyed.
475 */
Dennis Zhou7fcf2b02018-12-05 12:10:38 -0500476 percpu_ref_kill(&blkg->refcnt);
Tejun Heo03aa2642012-03-05 13:15:19 -0800477}
478
Tejun Heo9f13ef62012-03-05 13:15:21 -0800479/**
480 * blkg_destroy_all - destroy all blkgs associated with a request_queue
481 * @q: request_queue of interest
Tejun Heo9f13ef62012-03-05 13:15:21 -0800482 *
Tejun Heo3c96cb32012-04-13 13:11:34 -0700483 * Destroy all blkgs associated with @q.
Tejun Heo9f13ef62012-03-05 13:15:21 -0800484 */
Tejun Heo3c96cb32012-04-13 13:11:34 -0700485static void blkg_destroy_all(struct request_queue *q)
Tejun Heo03aa2642012-03-05 13:15:19 -0800486{
Tejun Heo3c798392012-04-16 13:57:25 -0700487 struct blkcg_gq *blkg, *n;
Yu Kuaia7317632021-07-07 09:56:49 +0800488 int count = BLKG_DESTROY_BATCH_SIZE;
Tejun Heo72e06c22012-03-05 13:15:00 -0800489
Yu Kuaia7317632021-07-07 09:56:49 +0800490restart:
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700491 spin_lock_irq(&q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800492 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
Tejun Heo3c798392012-04-16 13:57:25 -0700493 struct blkcg *blkcg = blkg->blkcg;
Tejun Heo72e06c22012-03-05 13:15:00 -0800494
Tejun Heo9f13ef62012-03-05 13:15:21 -0800495 spin_lock(&blkcg->lock);
496 blkg_destroy(blkg);
497 spin_unlock(&blkcg->lock);
Yu Kuaia7317632021-07-07 09:56:49 +0800498
499 /*
500 * in order to avoid holding the spin lock for too long, release
501 * it when a batch of blkgs are destroyed.
502 */
503 if (!(--count)) {
504 count = BLKG_DESTROY_BATCH_SIZE;
505 spin_unlock_irq(&q->queue_lock);
506 cond_resched();
507 goto restart;
508 }
Tejun Heo72e06c22012-03-05 13:15:00 -0800509 }
Tejun Heo6fe810b2015-09-05 15:47:36 -0400510
511 q->root_blkg = NULL;
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700512 spin_unlock_irq(&q->queue_lock);
Tejun Heo72e06c22012-03-05 13:15:00 -0800513}
514
Tejun Heo182446d2013-08-08 20:11:24 -0400515static int blkcg_reset_stats(struct cgroup_subsys_state *css,
516 struct cftype *cftype, u64 val)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700517{
Tejun Heo182446d2013-08-08 20:11:24 -0400518 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heo3c798392012-04-16 13:57:25 -0700519 struct blkcg_gq *blkg;
Tejun Heof7331642019-11-07 11:18:03 -0800520 int i, cpu;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700521
Tejun Heo838f13b2015-07-09 16:39:47 -0400522 mutex_lock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700523 spin_lock_irq(&blkcg->lock);
Tejun Heo997a0262012-03-08 10:53:58 -0800524
525 /*
526 * Note that stat reset is racy - it doesn't synchronize against
527 * stat updates. This is a debug feature which shouldn't exist
528 * anyway. If you get hit by a race, retry.
529 */
Sasha Levinb67bfe02013-02-27 17:06:00 -0800530 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heof7331642019-11-07 11:18:03 -0800531 for_each_possible_cpu(cpu) {
532 struct blkg_iostat_set *bis =
533 per_cpu_ptr(blkg->iostat_cpu, cpu);
534 memset(bis, 0, sizeof(*bis));
535 }
536 memset(&blkg->iostat, 0, sizeof(blkg->iostat));
Tejun Heo77ea7332015-08-18 14:55:24 -0700537
Tejun Heo8bd435b2012-04-13 13:11:28 -0700538 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700539 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heo549d3aa2012-03-05 13:15:16 -0800540
Tejun Heoa9520cd2015-08-18 14:55:14 -0700541 if (blkg->pd[i] && pol->pd_reset_stats_fn)
542 pol->pd_reset_stats_fn(blkg->pd[i]);
Tejun Heobc0d6502012-04-13 13:11:26 -0700543 }
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700544 }
Vivek Goyalf0bdc8c2011-05-19 15:38:30 -0400545
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700546 spin_unlock_irq(&blkcg->lock);
Tejun Heobc0d6502012-04-13 13:11:26 -0700547 mutex_unlock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700548 return 0;
549}
550
Tejun Heodd165eb2015-08-18 14:55:33 -0700551const char *blkg_dev_name(struct blkcg_gq *blkg)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700552{
Christoph Hellwigd152c682021-08-16 15:46:24 +0200553 if (!blkg->q->disk || !blkg->q->disk->bdi->dev)
Christoph Hellwigedb08722021-08-09 16:17:43 +0200554 return NULL;
Christoph Hellwigd152c682021-08-16 15:46:24 +0200555 return bdi_dev_name(blkg->q->disk->bdi);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700556}
557
Tejun Heod3d32e62012-04-01 14:38:42 -0700558/**
559 * blkcg_print_blkgs - helper for printing per-blkg data
560 * @sf: seq_file to print to
561 * @blkcg: blkcg of interest
562 * @prfill: fill function to print out a blkg
563 * @pol: policy in question
564 * @data: data to be passed to @prfill
565 * @show_total: to print out sum of prfill return values or not
566 *
567 * This function invokes @prfill on each blkg of @blkcg if pd for the
568 * policy specified by @pol exists. @prfill is invoked with @sf, the
Tejun Heo810ecfa2013-01-09 08:05:13 -0800569 * policy data and @data and the matching queue lock held. If @show_total
570 * is %true, the sum of the return values from @prfill is printed with
571 * "Total" label at the end.
Tejun Heod3d32e62012-04-01 14:38:42 -0700572 *
573 * This is to be used to construct print functions for
574 * cftype->read_seq_string method.
575 */
Tejun Heo3c798392012-04-16 13:57:25 -0700576void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
Tejun Heof95a04a2012-04-16 13:57:26 -0700577 u64 (*prfill)(struct seq_file *,
578 struct blkg_policy_data *, int),
Tejun Heo3c798392012-04-16 13:57:25 -0700579 const struct blkcg_policy *pol, int data,
Tejun Heoec399342012-04-13 13:11:27 -0700580 bool show_total)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400581{
Tejun Heo3c798392012-04-16 13:57:25 -0700582 struct blkcg_gq *blkg;
Tejun Heod3d32e62012-04-01 14:38:42 -0700583 u64 total = 0;
584
Tejun Heo810ecfa2013-01-09 08:05:13 -0800585 rcu_read_lock();
Linus Torvaldsee89f812013-02-28 12:52:24 -0800586 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700587 spin_lock_irq(&blkg->q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -0700588 if (blkcg_policy_enabled(blkg->q, pol))
Tejun Heof95a04a2012-04-16 13:57:26 -0700589 total += prfill(sf, blkg->pd[pol->plid], data);
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700590 spin_unlock_irq(&blkg->q->queue_lock);
Tejun Heo810ecfa2013-01-09 08:05:13 -0800591 }
592 rcu_read_unlock();
Tejun Heod3d32e62012-04-01 14:38:42 -0700593
594 if (show_total)
595 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
596}
Tejun Heo829fdb52012-04-01 14:38:43 -0700597EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
Tejun Heod3d32e62012-04-01 14:38:42 -0700598
599/**
600 * __blkg_prfill_u64 - prfill helper for a single u64 value
601 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700602 * @pd: policy private data of interest
Tejun Heod3d32e62012-04-01 14:38:42 -0700603 * @v: value to print
604 *
Tejun Heof95a04a2012-04-16 13:57:26 -0700605 * Print @v to @sf for the device assocaited with @pd.
Tejun Heod3d32e62012-04-01 14:38:42 -0700606 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700607u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
Tejun Heod3d32e62012-04-01 14:38:42 -0700608{
Tejun Heof95a04a2012-04-16 13:57:26 -0700609 const char *dname = blkg_dev_name(pd->blkg);
Tejun Heod3d32e62012-04-01 14:38:42 -0700610
611 if (!dname)
612 return 0;
613
614 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
615 return v;
616}
Tejun Heo829fdb52012-04-01 14:38:43 -0700617EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
Tejun Heod3d32e62012-04-01 14:38:42 -0700618
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600619/* Performs queue bypass and policy enabled checks then looks up blkg. */
620static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
621 const struct blkcg_policy *pol,
622 struct request_queue *q)
623{
624 WARN_ON_ONCE(!rcu_read_lock_held());
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700625 lockdep_assert_held(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600626
627 if (!blkcg_policy_enabled(q, pol))
628 return ERR_PTR(-EOPNOTSUPP);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600629 return __blkg_lookup(blkcg, q, true /* update_hint */);
630}
631
Tejun Heo16b3de62013-01-09 08:05:12 -0800632/**
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100633 * blkcg_conf_open_bdev - parse and open bdev for per-blkg config update
Tejun Heo015d2542019-08-28 15:05:53 -0700634 * @inputp: input string pointer
635 *
636 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100637 * from @input and get and return the matching bdev. *@inputp is
Tejun Heo015d2542019-08-28 15:05:53 -0700638 * updated to point past the device node prefix. Returns an ERR_PTR()
639 * value on error.
640 *
641 * Use this function iff blkg_conf_prep() can't be used for some reason.
642 */
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100643struct block_device *blkcg_conf_open_bdev(char **inputp)
Tejun Heo015d2542019-08-28 15:05:53 -0700644{
645 char *input = *inputp;
646 unsigned int major, minor;
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100647 struct block_device *bdev;
648 int key_len;
Tejun Heo015d2542019-08-28 15:05:53 -0700649
650 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
651 return ERR_PTR(-EINVAL);
652
653 input += key_len;
654 if (!isspace(*input))
655 return ERR_PTR(-EINVAL);
656 input = skip_spaces(input);
657
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100658 bdev = blkdev_get_no_open(MKDEV(major, minor));
659 if (!bdev)
Tejun Heo015d2542019-08-28 15:05:53 -0700660 return ERR_PTR(-ENODEV);
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100661 if (bdev_is_partition(bdev)) {
662 blkdev_put_no_open(bdev);
Tejun Heo015d2542019-08-28 15:05:53 -0700663 return ERR_PTR(-ENODEV);
664 }
665
666 *inputp = input;
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100667 return bdev;
Tejun Heo015d2542019-08-28 15:05:53 -0700668}
669
670/**
671 * blkg_conf_prep - parse and prepare for per-blkg config update
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700672 * @blkcg: target block cgroup
Tejun Heoda8b0662012-04-13 13:11:29 -0700673 * @pol: target policy
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700674 * @input: input string
675 * @ctx: blkg_conf_ctx to be filled
676 *
677 * Parse per-blkg config update from @input and initialize @ctx with the
Tejun Heo36aa9e52015-08-18 14:55:31 -0700678 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
679 * part of @input following MAJ:MIN. This function returns with RCU read
680 * lock and queue lock held and must be paired with blkg_conf_finish().
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700681 */
Tejun Heo3c798392012-04-16 13:57:25 -0700682int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
Tejun Heo36aa9e52015-08-18 14:55:31 -0700683 char *input, struct blkg_conf_ctx *ctx)
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100684 __acquires(rcu) __acquires(&bdev->bd_queue->queue_lock)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800685{
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100686 struct block_device *bdev;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600687 struct request_queue *q;
Tejun Heo3c798392012-04-16 13:57:25 -0700688 struct blkcg_gq *blkg;
Tejun Heo015d2542019-08-28 15:05:53 -0700689 int ret;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800690
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100691 bdev = blkcg_conf_open_bdev(&input);
692 if (IS_ERR(bdev))
693 return PTR_ERR(bdev);
Tejun Heoe56da7e2012-03-05 13:15:07 -0800694
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100695 q = bdev_get_queue(bdev);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600696
Yu Kuai0c9d3382021-10-20 09:40:36 +0800697 /*
698 * blkcg_deactivate_policy() requires queue to be frozen, we can grab
699 * q_usage_counter to prevent concurrent with blkcg_deactivate_policy().
700 */
701 ret = blk_queue_enter(q, 0);
702 if (ret)
Yu Kuai15c30102021-11-02 10:07:05 +0800703 goto fail;
Yu Kuai0c9d3382021-10-20 09:40:36 +0800704
Tejun Heoe56da7e2012-03-05 13:15:07 -0800705 rcu_read_lock();
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700706 spin_lock_irq(&q->queue_lock);
Tejun Heoda8b0662012-04-13 13:11:29 -0700707
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600708 blkg = blkg_lookup_check(blkcg, pol, q);
Tejun Heo4bfd4822012-03-05 13:15:08 -0800709 if (IS_ERR(blkg)) {
710 ret = PTR_ERR(blkg);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600711 goto fail_unlock;
Vivek Goyal062a6442010-09-15 17:06:33 -0400712 }
Tejun Heoe56da7e2012-03-05 13:15:07 -0800713
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600714 if (blkg)
715 goto success;
716
717 /*
718 * Create blkgs walking down from blkcg_root to @blkcg, so that all
719 * non-root blkgs have access to their parents.
720 */
721 while (true) {
722 struct blkcg *pos = blkcg;
723 struct blkcg *parent;
724 struct blkcg_gq *new_blkg;
725
726 parent = blkcg_parent(blkcg);
727 while (parent && !__blkg_lookup(parent, q, false)) {
728 pos = parent;
729 parent = blkcg_parent(parent);
730 }
731
732 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700733 spin_unlock_irq(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600734 rcu_read_unlock();
735
736 new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
737 if (unlikely(!new_blkg)) {
738 ret = -ENOMEM;
Yu Kuai15c30102021-11-02 10:07:05 +0800739 goto fail_exit_queue;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600740 }
741
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400742 if (radix_tree_preload(GFP_KERNEL)) {
743 blkg_free(new_blkg);
744 ret = -ENOMEM;
Yu Kuai15c30102021-11-02 10:07:05 +0800745 goto fail_exit_queue;
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400746 }
747
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600748 rcu_read_lock();
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700749 spin_lock_irq(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600750
751 blkg = blkg_lookup_check(pos, pol, q);
752 if (IS_ERR(blkg)) {
753 ret = PTR_ERR(blkg);
Gabriel Krisman Bertazi52abfcb2020-10-22 16:58:41 -0400754 blkg_free(new_blkg);
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400755 goto fail_preloaded;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600756 }
757
758 if (blkg) {
759 blkg_free(new_blkg);
760 } else {
761 blkg = blkg_create(pos, q, new_blkg);
Kefeng Wang98d669b2019-06-05 22:24:27 +0800762 if (IS_ERR(blkg)) {
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600763 ret = PTR_ERR(blkg);
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400764 goto fail_preloaded;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600765 }
766 }
767
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400768 radix_tree_preload_end();
769
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600770 if (pos == blkcg)
771 goto success;
772 }
773success:
Yu Kuai0c9d3382021-10-20 09:40:36 +0800774 blk_queue_exit(q);
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100775 ctx->bdev = bdev;
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700776 ctx->blkg = blkg;
Tejun Heo015d2542019-08-28 15:05:53 -0700777 ctx->body = input;
Tejun Heo726fa6942012-04-01 14:38:43 -0700778 return 0;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600779
Gabriel Krisman Bertazif255c192020-10-22 16:58:42 -0400780fail_preloaded:
781 radix_tree_preload_end();
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600782fail_unlock:
Christoph Hellwig0d945c12018-11-15 12:17:28 -0700783 spin_unlock_irq(&q->queue_lock);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600784 rcu_read_unlock();
Yu Kuai15c30102021-11-02 10:07:05 +0800785fail_exit_queue:
786 blk_queue_exit(q);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600787fail:
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100788 blkdev_put_no_open(bdev);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600789 /*
790 * If queue was bypassing, we should retry. Do so after a
791 * short msleep(). It isn't strictly necessary but queue
792 * can be bypassing for some time and it's always nice to
793 * avoid busy looping.
794 */
795 if (ret == -EBUSY) {
796 msleep(10);
797 ret = restart_syscall();
798 }
799 return ret;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800800}
Pavel Begunkov89f3b6d2019-09-14 20:31:50 +0300801EXPORT_SYMBOL_GPL(blkg_conf_prep);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800802
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700803/**
804 * blkg_conf_finish - finish up per-blkg config update
805 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
806 *
807 * Finish up after per-blkg config update. This function must be paired
808 * with blkg_conf_prep().
809 */
Tejun Heo829fdb52012-04-01 14:38:43 -0700810void blkg_conf_finish(struct blkg_conf_ctx *ctx)
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100811 __releases(&ctx->bdev->bd_queue->queue_lock) __releases(rcu)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800812{
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100813 spin_unlock_irq(&bdev_get_queue(ctx->bdev)->queue_lock);
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700814 rcu_read_unlock();
Christoph Hellwig22ae8ce2020-11-26 09:23:26 +0100815 blkdev_put_no_open(ctx->bdev);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800816}
Pavel Begunkov89f3b6d2019-09-14 20:31:50 +0300817EXPORT_SYMBOL_GPL(blkg_conf_finish);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800818
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700819static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
820{
821 int i;
822
823 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
824 dst->bytes[i] = src->bytes[i];
825 dst->ios[i] = src->ios[i];
826 }
827}
828
829static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)
830{
831 int i;
832
833 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
834 dst->bytes[i] += src->bytes[i];
835 dst->ios[i] += src->ios[i];
836 }
837}
838
839static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)
840{
841 int i;
842
843 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
844 dst->bytes[i] -= src->bytes[i];
845 dst->ios[i] -= src->ios[i];
846 }
847}
848
Jason Yan362b8c12022-06-29 15:09:16 +0800849static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
850 struct blkg_iostat *last)
851{
852 struct blkg_iostat delta;
853 unsigned long flags;
854
855 /* propagate percpu delta to global */
856 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
857 blkg_iostat_set(&delta, cur);
858 blkg_iostat_sub(&delta, last);
859 blkg_iostat_add(&blkg->iostat.cur, &delta);
860 blkg_iostat_add(last, &delta);
861 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
862}
863
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700864static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
865{
866 struct blkcg *blkcg = css_to_blkcg(css);
867 struct blkcg_gq *blkg;
868
Johannes Weinerdc265322021-04-29 22:56:23 -0700869 /* Root-level stats are sourced from system-wide IO stats */
870 if (!cgroup_parent(css->cgroup))
871 return;
872
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700873 rcu_read_lock();
874
875 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
876 struct blkcg_gq *parent = blkg->parent;
877 struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu);
Jason Yan362b8c12022-06-29 15:09:16 +0800878 struct blkg_iostat cur;
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700879 unsigned int seq;
880
881 /* fetch the current per-cpu values */
882 do {
883 seq = u64_stats_fetch_begin(&bisc->sync);
884 blkg_iostat_set(&cur, &bisc->cur);
885 } while (u64_stats_fetch_retry(&bisc->sync, seq));
886
Jason Yan362b8c12022-06-29 15:09:16 +0800887 blkcg_iostat_update(blkg, &cur, &bisc->last);
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700888
Johannes Weinerdc265322021-04-29 22:56:23 -0700889 /* propagate global delta to parent (unless that's root) */
Jason Yan362b8c12022-06-29 15:09:16 +0800890 if (parent && parent->parent)
891 blkcg_iostat_update(parent, &blkg->iostat.cur,
892 &blkg->iostat.last);
Boris Burkovcd1fc4b2020-06-01 13:11:43 -0700893 }
894
895 rcu_read_unlock();
896}
897
Boris Burkovef45fe42020-06-01 13:12:05 -0700898/*
Johannes Weinerdc265322021-04-29 22:56:23 -0700899 * We source root cgroup stats from the system-wide stats to avoid
900 * tracking the same information twice and incurring overhead when no
901 * cgroups are defined. For that reason, cgroup_rstat_flush in
902 * blkcg_print_stat does not actually fill out the iostat in the root
903 * cgroup's blkcg_gq.
Boris Burkovef45fe42020-06-01 13:12:05 -0700904 *
905 * However, we would like to re-use the printing code between the root and
906 * non-root cgroups to the extent possible. For that reason, we simulate
907 * flushing the root cgroup's stats by explicitly filling in the iostat
908 * with disk level statistics.
909 */
910static void blkcg_fill_root_iostats(void)
911{
912 struct class_dev_iter iter;
913 struct device *dev;
914
915 class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
916 while ((dev = class_dev_iter_next(&iter))) {
Christoph Hellwig0d021292020-11-27 16:43:51 +0100917 struct block_device *bdev = dev_to_bdev(dev);
918 struct blkcg_gq *blkg =
Pavel Begunkoved6cdde2021-10-14 15:03:30 +0100919 blk_queue_root_blkg(bdev_get_queue(bdev));
Boris Burkovef45fe42020-06-01 13:12:05 -0700920 struct blkg_iostat tmp;
921 int cpu;
Chengming Zhouf122d102022-02-13 16:59:02 +0800922 unsigned long flags;
Boris Burkovef45fe42020-06-01 13:12:05 -0700923
924 memset(&tmp, 0, sizeof(tmp));
925 for_each_possible_cpu(cpu) {
926 struct disk_stats *cpu_dkstats;
927
Christoph Hellwig0d021292020-11-27 16:43:51 +0100928 cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);
Boris Burkovef45fe42020-06-01 13:12:05 -0700929 tmp.ios[BLKG_IOSTAT_READ] +=
930 cpu_dkstats->ios[STAT_READ];
931 tmp.ios[BLKG_IOSTAT_WRITE] +=
932 cpu_dkstats->ios[STAT_WRITE];
933 tmp.ios[BLKG_IOSTAT_DISCARD] +=
934 cpu_dkstats->ios[STAT_DISCARD];
935 // convert sectors to bytes
936 tmp.bytes[BLKG_IOSTAT_READ] +=
937 cpu_dkstats->sectors[STAT_READ] << 9;
938 tmp.bytes[BLKG_IOSTAT_WRITE] +=
939 cpu_dkstats->sectors[STAT_WRITE] << 9;
940 tmp.bytes[BLKG_IOSTAT_DISCARD] +=
941 cpu_dkstats->sectors[STAT_DISCARD] << 9;
Boris Burkovef45fe42020-06-01 13:12:05 -0700942 }
Chengming Zhouf122d102022-02-13 16:59:02 +0800943
944 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
945 blkg_iostat_set(&blkg->iostat.cur, &tmp);
946 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
Boris Burkovef45fe42020-06-01 13:12:05 -0700947 }
948}
949
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200950static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
951{
952 struct blkg_iostat_set *bis = &blkg->iostat;
953 u64 rbytes, wbytes, rios, wios, dbytes, dios;
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200954 const char *dname;
955 unsigned seq;
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200956 int i;
957
958 if (!blkg->online)
959 return;
960
961 dname = blkg_dev_name(blkg);
962 if (!dname)
963 return;
964
Christoph Hellwig252c6512021-08-10 17:26:23 +0200965 seq_printf(s, "%s ", dname);
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200966
967 do {
968 seq = u64_stats_fetch_begin(&bis->sync);
969
970 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
971 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
972 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
973 rios = bis->cur.ios[BLKG_IOSTAT_READ];
974 wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
975 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
976 } while (u64_stats_fetch_retry(&bis->sync, seq));
977
978 if (rbytes || wbytes || rios || wios) {
Christoph Hellwig252c6512021-08-10 17:26:23 +0200979 seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200980 rbytes, wbytes, rios, wios,
981 dbytes, dios);
982 }
983
984 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
Christoph Hellwig252c6512021-08-10 17:26:23 +0200985 seq_printf(s, " use_delay=%d delay_nsec=%llu",
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200986 atomic_read(&blkg->use_delay),
987 atomic64_read(&blkg->delay_nsec));
988 }
989
990 for (i = 0; i < BLKCG_MAX_POLS; i++) {
991 struct blkcg_policy *pol = blkcg_policy[i];
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200992
993 if (!blkg->pd[i] || !pol->pd_stat_fn)
994 continue;
995
Wolfgang Bumiller36078492022-01-11 09:31:59 +0100996 pol->pd_stat_fn(blkg->pd[i], s);
Christoph Hellwig49cb5162021-08-10 17:26:22 +0200997 }
998
Wolfgang Bumiller36078492022-01-11 09:31:59 +0100999 seq_puts(s, "\n");
Christoph Hellwig49cb5162021-08-10 17:26:22 +02001000}
1001
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001002static int blkcg_print_stat(struct seq_file *sf, void *v)
1003{
1004 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
1005 struct blkcg_gq *blkg;
1006
Boris Burkovef45fe42020-06-01 13:12:05 -07001007 if (!seq_css(sf)->parent)
1008 blkcg_fill_root_iostats();
1009 else
1010 cgroup_rstat_flush(blkcg->css.cgroup);
1011
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001012 rcu_read_lock();
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001013 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heob0814362019-11-05 08:09:51 -08001014 spin_lock_irq(&blkg->q->queue_lock);
Christoph Hellwig49cb5162021-08-10 17:26:22 +02001015 blkcg_print_one_stat(blkg, sf);
Tejun Heob0814362019-11-05 08:09:51 -08001016 spin_unlock_irq(&blkg->q->queue_lock);
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001017 }
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001018 rcu_read_unlock();
1019 return 0;
1020}
1021
Bart Van Asschee1f3b942016-06-14 17:04:32 +02001022static struct cftype blkcg_files[] = {
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001023 {
1024 .name = "stat",
1025 .seq_show = blkcg_print_stat,
1026 },
1027 { } /* terminate */
1028};
1029
Bart Van Asschee1f3b942016-06-14 17:04:32 +02001030static struct cftype blkcg_legacy_files[] = {
Vivek Goyal31e4c282009-12-03 12:59:42 -05001031 {
Divyesh Shah84c124d2010-04-09 08:31:19 +02001032 .name = "reset_stats",
Tejun Heo3c798392012-04-16 13:57:25 -07001033 .write_u64 = blkcg_reset_stats,
Vivek Goyal22084192009-12-03 12:59:49 -05001034 },
Tejun Heo4baf6e32012-04-01 12:09:55 -07001035 { } /* terminate */
Vivek Goyal31e4c282009-12-03 12:59:42 -05001036};
1037
Christoph Hellwigdec223c2022-04-20 06:27:15 +02001038#ifdef CONFIG_CGROUP_WRITEBACK
1039struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css)
1040{
1041 return &css_to_blkcg(css)->cgwb_list;
1042}
1043#endif
1044
Dennis Zhou (Facebook)59b57712018-08-31 16:22:43 -04001045/*
1046 * blkcg destruction is a three-stage process.
1047 *
1048 * 1. Destruction starts. The blkcg_css_offline() callback is invoked
1049 * which offlines writeback. Here we tie the next stage of blkg destruction
1050 * to the completion of writeback associated with the blkcg. This lets us
1051 * avoid punting potentially large amounts of outstanding writeback to root
1052 * while maintaining any ongoing policies. The next stage is triggered when
1053 * the nr_cgwbs count goes to zero.
1054 *
1055 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
1056 * and handles the destruction of blkgs. Here the css reference held by
1057 * the blkg is put back eventually allowing blkcg_css_free() to be called.
1058 * This work may occur in cgwb_release_workfn() on the cgwb_release
1059 * workqueue. Any submitted ios that fail to get the blkg ref will be
1060 * punted to the root_blkg.
1061 *
1062 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
1063 * This finally frees the blkcg.
1064 */
1065
Tejun Heo9f13ef62012-03-05 13:15:21 -08001066/**
Dennis Zhou (Facebook)59b57712018-08-31 16:22:43 -04001067 * blkcg_destroy_blkgs - responsible for shooting down blkgs
1068 * @blkcg: blkcg of interest
1069 *
1070 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1071 * is nested inside q lock, this function performs reverse double lock dancing.
1072 * Destroying the blkgs releases the reference held on the blkcg's css allowing
1073 * blkcg_css_free to eventually be called.
1074 *
1075 * This is the blkcg counterpart of ioc_release_fn().
1076 */
Christoph Hellwig397c9f42022-04-20 06:27:14 +02001077static void blkcg_destroy_blkgs(struct blkcg *blkcg)
Dennis Zhou (Facebook)59b57712018-08-31 16:22:43 -04001078{
Baolin Wang6c635ca2021-01-28 13:58:15 +08001079 might_sleep();
1080
Tejun Heo9f13ef62012-03-05 13:15:21 -08001081 spin_lock_irq(&blkcg->lock);
Tejun Heo7ee9c562012-03-05 13:15:11 -08001082
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -04001083 while (!hlist_empty(&blkcg->blkg_list)) {
1084 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1085 struct blkcg_gq, blkcg_node);
Tejun Heoc875f4d2012-03-05 13:15:22 -08001086 struct request_queue *q = blkg->q;
Vivek Goyalb1c35762009-12-03 12:59:47 -05001087
Baolin Wang6c635ca2021-01-28 13:58:15 +08001088 if (need_resched() || !spin_trylock(&q->queue_lock)) {
1089 /*
1090 * Given that the system can accumulate a huge number
1091 * of blkgs in pathological cases, check to see if we
1092 * need to rescheduling to avoid softlockup.
1093 */
Tejun Heo9f13ef62012-03-05 13:15:21 -08001094 spin_unlock_irq(&blkcg->lock);
Baolin Wang6c635ca2021-01-28 13:58:15 +08001095 cond_resched();
Dan Carpentera5567932012-03-29 20:57:08 +02001096 spin_lock_irq(&blkcg->lock);
Baolin Wang6c635ca2021-01-28 13:58:15 +08001097 continue;
Jens Axboe0f3942a2010-05-03 14:28:55 +02001098 }
Baolin Wang6c635ca2021-01-28 13:58:15 +08001099
1100 blkg_destroy(blkg);
1101 spin_unlock(&q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -08001102 }
Jens Axboe0f3942a2010-05-03 14:28:55 +02001103
Tejun Heo9f13ef62012-03-05 13:15:21 -08001104 spin_unlock_irq(&blkcg->lock);
Tejun Heo7ee9c562012-03-05 13:15:11 -08001105}
1106
Christoph Hellwig397c9f42022-04-20 06:27:14 +02001107/**
1108 * blkcg_pin_online - pin online state
1109 * @blkcg_css: blkcg of interest
1110 *
1111 * While pinned, a blkcg is kept online. This is primarily used to
1112 * impedance-match blkg and cgwb lifetimes so that blkg doesn't go offline
1113 * while an associated cgwb is still active.
1114 */
1115void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css)
1116{
1117 refcount_inc(&css_to_blkcg(blkcg_css)->online_pin);
1118}
1119
1120/**
1121 * blkcg_unpin_online - unpin online state
1122 * @blkcg_css: blkcg of interest
1123 *
1124 * This is primarily used to impedance-match blkg and cgwb lifetimes so
1125 * that blkg doesn't go offline while an associated cgwb is still active.
1126 * When this count goes to zero, all active cgwbs have finished so the
1127 * blkcg can continue destruction by calling blkcg_destroy_blkgs().
1128 */
1129void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css)
1130{
1131 struct blkcg *blkcg = css_to_blkcg(blkcg_css);
1132
1133 do {
1134 if (!refcount_dec_and_test(&blkcg->online_pin))
1135 break;
1136 blkcg_destroy_blkgs(blkcg);
1137 blkcg = blkcg_parent(blkcg);
1138 } while (blkcg);
1139}
1140
1141/**
1142 * blkcg_css_offline - cgroup css_offline callback
1143 * @css: css of interest
1144 *
1145 * This function is called when @css is about to go away. Here the cgwbs are
1146 * offlined first and only once writeback associated with the blkcg has
1147 * finished do we start step 2 (see above).
1148 */
1149static void blkcg_css_offline(struct cgroup_subsys_state *css)
1150{
1151 /* this prevents anyone from attaching or migrating to this blkcg */
Christoph Hellwigdec223c2022-04-20 06:27:15 +02001152 wb_blkcg_offline(css);
Christoph Hellwig397c9f42022-04-20 06:27:14 +02001153
1154 /* put the base online pin allowing step 2 to be triggered */
1155 blkcg_unpin_online(css);
1156}
1157
Tejun Heoeb954192013-08-08 20:11:23 -04001158static void blkcg_css_free(struct cgroup_subsys_state *css)
Tejun Heo7ee9c562012-03-05 13:15:11 -08001159{
Tejun Heoeb954192013-08-08 20:11:23 -04001160 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heobc915e62015-08-18 14:55:08 -07001161 int i;
Tejun Heo7ee9c562012-03-05 13:15:11 -08001162
Tejun Heo7876f932015-07-09 16:39:49 -04001163 mutex_lock(&blkcg_pol_mutex);
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001164
Tejun Heo7876f932015-07-09 16:39:49 -04001165 list_del(&blkcg->all_blkcgs_node);
Tejun Heo7876f932015-07-09 16:39:49 -04001166
Tejun Heobc915e62015-08-18 14:55:08 -07001167 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001168 if (blkcg->cpd[i])
1169 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1170
1171 mutex_unlock(&blkcg_pol_mutex);
1172
Tejun Heobc915e62015-08-18 14:55:08 -07001173 kfree(blkcg);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001174}
1175
Tejun Heoeb954192013-08-08 20:11:23 -04001176static struct cgroup_subsys_state *
1177blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001178{
Tejun Heo3c798392012-04-16 13:57:25 -07001179 struct blkcg *blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001180 struct cgroup_subsys_state *ret;
1181 int i;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001182
Tejun Heo7876f932015-07-09 16:39:49 -04001183 mutex_lock(&blkcg_pol_mutex);
1184
Tejun Heoeb954192013-08-08 20:11:23 -04001185 if (!parent_css) {
Tejun Heo3c798392012-04-16 13:57:25 -07001186 blkcg = &blkcg_root;
Tejun Heobc915e62015-08-18 14:55:08 -07001187 } else {
1188 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1189 if (!blkcg) {
1190 ret = ERR_PTR(-ENOMEM);
weiping zhang4c18c9e2017-08-25 23:49:32 +08001191 goto unlock;
Tejun Heobc915e62015-08-18 14:55:08 -07001192 }
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001193 }
Vivek Goyal31e4c282009-12-03 12:59:42 -05001194
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001195 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1196 struct blkcg_policy *pol = blkcg_policy[i];
1197 struct blkcg_policy_data *cpd;
1198
1199 /*
1200 * If the policy hasn't been attached yet, wait for it
1201 * to be attached before doing anything else. Otherwise,
1202 * check if the policy requires any specific per-cgroup
1203 * data: if it does, allocate and initialize it.
1204 */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001205 if (!pol || !pol->cpd_alloc_fn)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001206 continue;
1207
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001208 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001209 if (!cpd) {
1210 ret = ERR_PTR(-ENOMEM);
1211 goto free_pd_blkcg;
1212 }
Tejun Heo81437642015-08-18 14:55:15 -07001213 blkcg->cpd[i] = cpd;
1214 cpd->blkcg = blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001215 cpd->plid = i;
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001216 if (pol->cpd_init_fn)
1217 pol->cpd_init_fn(cpd);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001218 }
1219
Vivek Goyal31e4c282009-12-03 12:59:42 -05001220 spin_lock_init(&blkcg->lock);
Tejun Heod866dbf2019-07-24 10:37:22 -07001221 refcount_set(&blkcg->online_pin, 1);
Tejun Heoe00f4f42016-11-21 18:03:32 -05001222 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001223 INIT_HLIST_HEAD(&blkcg->blkg_list);
Tejun Heo52ebea72015-05-22 17:13:37 -04001224#ifdef CONFIG_CGROUP_WRITEBACK
1225 INIT_LIST_HEAD(&blkcg->cgwb_list);
1226#endif
Tejun Heo7876f932015-07-09 16:39:49 -04001227 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1228
1229 mutex_unlock(&blkcg_pol_mutex);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001230 return &blkcg->css;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001231
1232free_pd_blkcg:
1233 for (i--; i >= 0; i--)
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001234 if (blkcg->cpd[i])
1235 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
weiping zhang4c18c9e2017-08-25 23:49:32 +08001236
1237 if (blkcg != &blkcg_root)
1238 kfree(blkcg);
1239unlock:
Tejun Heo7876f932015-07-09 16:39:49 -04001240 mutex_unlock(&blkcg_pol_mutex);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001241 return ret;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001242}
1243
Tejun Heo4308a432019-07-24 10:37:55 -07001244static int blkcg_css_online(struct cgroup_subsys_state *css)
1245{
Christoph Hellwig397c9f42022-04-20 06:27:14 +02001246 struct blkcg *parent = blkcg_parent(css_to_blkcg(css));
Tejun Heo4308a432019-07-24 10:37:55 -07001247
1248 /*
1249 * blkcg_pin_online() is used to delay blkcg offline so that blkgs
1250 * don't go offline while cgwbs are still active on them. Pin the
1251 * parent so that offline always happens towards the root.
1252 */
1253 if (parent)
Christoph Hellwig397c9f42022-04-20 06:27:14 +02001254 blkcg_pin_online(css);
Tejun Heo4308a432019-07-24 10:37:55 -07001255 return 0;
1256}
1257
Tejun Heo5efd6112012-03-05 13:15:12 -08001258/**
1259 * blkcg_init_queue - initialize blkcg part of request queue
1260 * @q: request_queue to initialize
1261 *
Christoph Hellwigc62b37d2020-07-01 10:59:43 +02001262 * Called from blk_alloc_queue(). Responsible for initializing blkcg
Tejun Heo5efd6112012-03-05 13:15:12 -08001263 * part of new request_queue @q.
1264 *
1265 * RETURNS:
1266 * 0 on success, -errno on failure.
1267 */
1268int blkcg_init_queue(struct request_queue *q)
1269{
Jens Axboed708f0d2017-03-29 11:25:48 -06001270 struct blkcg_gq *new_blkg, *blkg;
1271 bool preloaded;
Tejun Heoec13b1d2015-05-22 17:13:19 -04001272 int ret;
Tejun Heo5efd6112012-03-05 13:15:12 -08001273
Ming Lei472e4312022-02-11 18:11:48 +08001274 INIT_LIST_HEAD(&q->blkg_list);
1275
Jens Axboed708f0d2017-03-29 11:25:48 -06001276 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1277 if (!new_blkg)
1278 return -ENOMEM;
1279
1280 preloaded = !radix_tree_preload(GFP_KERNEL);
1281
Jiang Biaobea54882018-04-19 12:04:26 +08001282 /* Make sure the root blkg exists. */
Fanjun Kong77c570a2022-05-17 01:39:30 +08001283 /* spin_lock_irq can serve as RCU read-side critical section. */
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001284 spin_lock_irq(&q->queue_lock);
Jens Axboed708f0d2017-03-29 11:25:48 -06001285 blkg = blkg_create(&blkcg_root, q, new_blkg);
Jiang Biao901932a2018-04-19 12:06:09 +08001286 if (IS_ERR(blkg))
1287 goto err_unlock;
1288 q->root_blkg = blkg;
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001289 spin_unlock_irq(&q->queue_lock);
Tejun Heoec13b1d2015-05-22 17:13:19 -04001290
Jens Axboed708f0d2017-03-29 11:25:48 -06001291 if (preloaded)
1292 radix_tree_preload_end();
1293
Bart Van Assche556910e2021-06-17 17:44:44 -07001294 ret = blk_ioprio_init(q);
1295 if (ret)
1296 goto err_destroy_all;
1297
Tejun Heoec13b1d2015-05-22 17:13:19 -04001298 ret = blk_throtl_init(q);
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001299 if (ret)
1300 goto err_destroy_all;
Yufen Yu27029b42020-08-10 22:21:16 -04001301
Yanfei Xu6f5ddde2021-09-15 15:24:26 +08001302 ret = blk_iolatency_init(q);
1303 if (ret) {
1304 blk_throtl_exit(q);
Jan Kara82b74ca2022-06-23 09:48:32 +02001305 blk_ioprio_exit(q);
Yanfei Xu6f5ddde2021-09-15 15:24:26 +08001306 goto err_destroy_all;
1307 }
1308
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001309 return 0;
Jiang Biao901932a2018-04-19 12:06:09 +08001310
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001311err_destroy_all:
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001312 blkg_destroy_all(q);
Christoph Hellwig04be60b2018-11-14 17:02:12 +01001313 return ret;
Jiang Biao901932a2018-04-19 12:06:09 +08001314err_unlock:
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001315 spin_unlock_irq(&q->queue_lock);
Jiang Biao901932a2018-04-19 12:06:09 +08001316 if (preloaded)
1317 radix_tree_preload_end();
1318 return PTR_ERR(blkg);
Tejun Heo5efd6112012-03-05 13:15:12 -08001319}
1320
1321/**
Tejun Heo5efd6112012-03-05 13:15:12 -08001322 * blkcg_exit_queue - exit and release blkcg part of request_queue
1323 * @q: request_queue being released
1324 *
Marcos Paulo de Souza7585d502019-01-25 00:01:42 -02001325 * Called from blk_exit_queue(). Responsible for exiting blkcg part.
Tejun Heo5efd6112012-03-05 13:15:12 -08001326 */
1327void blkcg_exit_queue(struct request_queue *q)
1328{
Tejun Heo3c96cb32012-04-13 13:11:34 -07001329 blkg_destroy_all(q);
Tejun Heo5efd6112012-03-05 13:15:12 -08001330 blk_throtl_exit(q);
1331}
1332
Tejun Heo69d7fde2015-08-18 14:55:36 -07001333static void blkcg_bind(struct cgroup_subsys_state *root_css)
1334{
1335 int i;
1336
1337 mutex_lock(&blkcg_pol_mutex);
1338
1339 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1340 struct blkcg_policy *pol = blkcg_policy[i];
1341 struct blkcg *blkcg;
1342
1343 if (!pol || !pol->cpd_bind_fn)
1344 continue;
1345
1346 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1347 if (blkcg->cpd[pol->plid])
1348 pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1349 }
1350 mutex_unlock(&blkcg_pol_mutex);
1351}
1352
Josef Bacikd09d8df2018-07-03 11:14:55 -04001353static void blkcg_exit(struct task_struct *tsk)
1354{
1355 if (tsk->throttle_queue)
1356 blk_put_queue(tsk->throttle_queue);
1357 tsk->throttle_queue = NULL;
1358}
1359
Tejun Heoc165b3e2015-08-18 14:55:29 -07001360struct cgroup_subsys io_cgrp_subsys = {
Tejun Heo92fb9742012-11-19 08:13:38 -08001361 .css_alloc = blkcg_css_alloc,
Tejun Heo4308a432019-07-24 10:37:55 -07001362 .css_online = blkcg_css_online,
Tejun Heo92fb9742012-11-19 08:13:38 -08001363 .css_offline = blkcg_css_offline,
1364 .css_free = blkcg_css_free,
Tejun Heof7331642019-11-07 11:18:03 -08001365 .css_rstat_flush = blkcg_rstat_flush,
Tejun Heo69d7fde2015-08-18 14:55:36 -07001366 .bind = blkcg_bind,
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001367 .dfl_cftypes = blkcg_files,
Tejun Heo880f50e22015-08-18 14:55:30 -07001368 .legacy_cftypes = blkcg_legacy_files,
Tejun Heoc165b3e2015-08-18 14:55:29 -07001369 .legacy_name = "blkio",
Josef Bacikd09d8df2018-07-03 11:14:55 -04001370 .exit = blkcg_exit,
Tejun Heo1ced953b2014-07-08 18:02:57 -04001371#ifdef CONFIG_MEMCG
1372 /*
1373 * This ensures that, if available, memcg is automatically enabled
1374 * together on the default hierarchy so that the owner cgroup can
1375 * be retrieved from writeback pages.
1376 */
1377 .depends_on = 1 << memory_cgrp_id,
1378#endif
Tejun Heo676f7c82012-04-01 12:09:55 -07001379};
Tejun Heoc165b3e2015-08-18 14:55:29 -07001380EXPORT_SYMBOL_GPL(io_cgrp_subsys);
Tejun Heo676f7c82012-04-01 12:09:55 -07001381
Tejun Heo8bd435b2012-04-13 13:11:28 -07001382/**
Tejun Heoa2b16932012-04-13 13:11:33 -07001383 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1384 * @q: request_queue of interest
1385 * @pol: blkcg policy to activate
1386 *
1387 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1388 * bypass mode to populate its blkgs with policy_data for @pol.
1389 *
1390 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1391 * from IO path. Update of each blkg is protected by both queue and blkcg
1392 * locks so that holding either lock and testing blkcg_policy_enabled() is
1393 * always enough for dereferencing policy data.
1394 *
1395 * The caller is responsible for synchronizing [de]activations and policy
1396 * [un]registerations. Returns 0 on success, -errno on failure.
1397 */
1398int blkcg_activate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001399 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001400{
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001401 struct blkg_policy_data *pd_prealloc = NULL;
Tejun Heo9d179b82019-10-15 09:03:47 -07001402 struct blkcg_gq *blkg, *pinned_blkg = NULL;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001403 int ret;
Tejun Heoa2b16932012-04-13 13:11:33 -07001404
1405 if (blkcg_policy_enabled(q, pol))
1406 return 0;
1407
Jens Axboe344e9ff2018-11-15 12:22:51 -07001408 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001409 blk_mq_freeze_queue(q);
Tejun Heo9d179b82019-10-15 09:03:47 -07001410retry:
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001411 spin_lock_irq(&q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -07001412
Tejun Heo9d179b82019-10-15 09:03:47 -07001413 /* blkg_list is pushed at the head, reverse walk to allocate parents first */
Tejun Heo71c81402019-06-13 15:30:40 -07001414 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001415 struct blkg_policy_data *pd;
Tejun Heoa2b16932012-04-13 13:11:33 -07001416
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001417 if (blkg->pd[pol->plid])
1418 continue;
1419
Tejun Heo9d179b82019-10-15 09:03:47 -07001420 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */
1421 if (blkg == pinned_blkg) {
1422 pd = pd_prealloc;
1423 pd_prealloc = NULL;
1424 } else {
1425 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q,
1426 blkg->blkcg);
1427 }
1428
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001429 if (!pd) {
Tejun Heo9d179b82019-10-15 09:03:47 -07001430 /*
1431 * GFP_NOWAIT failed. Free the existing one and
1432 * prealloc for @blkg w/ GFP_KERNEL.
1433 */
1434 if (pinned_blkg)
1435 blkg_put(pinned_blkg);
1436 blkg_get(blkg);
1437 pinned_blkg = blkg;
1438
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001439 spin_unlock_irq(&q->queue_lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001440
1441 if (pd_prealloc)
1442 pol->pd_free_fn(pd_prealloc);
1443 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q,
1444 blkg->blkcg);
1445 if (pd_prealloc)
1446 goto retry;
1447 else
1448 goto enomem;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001449 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001450
1451 blkg->pd[pol->plid] = pd;
1452 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -08001453 pd->plid = pol->plid;
Tejun Heoa2b16932012-04-13 13:11:33 -07001454 }
1455
Tejun Heo9d179b82019-10-15 09:03:47 -07001456 /* all allocated, init in the same order */
1457 if (pol->pd_init_fn)
1458 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node)
1459 pol->pd_init_fn(blkg->pd[pol->plid]);
1460
Tejun Heoa2b16932012-04-13 13:11:33 -07001461 __set_bit(pol->plid, q->blkcg_pols);
1462 ret = 0;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001463
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001464 spin_unlock_irq(&q->queue_lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001465out:
Jens Axboe344e9ff2018-11-15 12:22:51 -07001466 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001467 blk_mq_unfreeze_queue(q);
Tejun Heo9d179b82019-10-15 09:03:47 -07001468 if (pinned_blkg)
1469 blkg_put(pinned_blkg);
Tejun Heo001bea72015-08-18 14:55:11 -07001470 if (pd_prealloc)
1471 pol->pd_free_fn(pd_prealloc);
Tejun Heoa2b16932012-04-13 13:11:33 -07001472 return ret;
Tejun Heo9d179b82019-10-15 09:03:47 -07001473
1474enomem:
1475 /* alloc failed, nothing's initialized yet, free everything */
1476 spin_lock_irq(&q->queue_lock);
1477 list_for_each_entry(blkg, &q->blkg_list, q_node) {
Li Jinlin858560b2021-09-14 12:26:05 +08001478 struct blkcg *blkcg = blkg->blkcg;
1479
1480 spin_lock(&blkcg->lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001481 if (blkg->pd[pol->plid]) {
1482 pol->pd_free_fn(blkg->pd[pol->plid]);
1483 blkg->pd[pol->plid] = NULL;
1484 }
Li Jinlin858560b2021-09-14 12:26:05 +08001485 spin_unlock(&blkcg->lock);
Tejun Heo9d179b82019-10-15 09:03:47 -07001486 }
1487 spin_unlock_irq(&q->queue_lock);
1488 ret = -ENOMEM;
1489 goto out;
Tejun Heoa2b16932012-04-13 13:11:33 -07001490}
1491EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1492
1493/**
1494 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1495 * @q: request_queue of interest
1496 * @pol: blkcg policy to deactivate
1497 *
1498 * Deactivate @pol on @q. Follows the same synchronization rules as
1499 * blkcg_activate_policy().
1500 */
1501void blkcg_deactivate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001502 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001503{
Tejun Heo3c798392012-04-16 13:57:25 -07001504 struct blkcg_gq *blkg;
Tejun Heoa2b16932012-04-13 13:11:33 -07001505
1506 if (!blkcg_policy_enabled(q, pol))
1507 return;
1508
Jens Axboe344e9ff2018-11-15 12:22:51 -07001509 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001510 blk_mq_freeze_queue(q);
Jens Axboebd166ef2017-01-17 06:03:22 -07001511
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001512 spin_lock_irq(&q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -07001513
1514 __clear_bit(pol->plid, q->blkcg_pols);
1515
1516 list_for_each_entry(blkg, &q->blkg_list, q_node) {
Li Jinlin858560b2021-09-14 12:26:05 +08001517 struct blkcg *blkcg = blkg->blkcg;
1518
1519 spin_lock(&blkcg->lock);
Tejun Heo001bea72015-08-18 14:55:11 -07001520 if (blkg->pd[pol->plid]) {
Dennis Zhou (Facebook)6b065462018-08-31 16:22:42 -04001521 if (pol->pd_offline_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -07001522 pol->pd_offline_fn(blkg->pd[pol->plid]);
Tejun Heo001bea72015-08-18 14:55:11 -07001523 pol->pd_free_fn(blkg->pd[pol->plid]);
1524 blkg->pd[pol->plid] = NULL;
1525 }
Li Jinlin858560b2021-09-14 12:26:05 +08001526 spin_unlock(&blkcg->lock);
Tejun Heoa2b16932012-04-13 13:11:33 -07001527 }
1528
Christoph Hellwig0d945c12018-11-15 12:17:28 -07001529 spin_unlock_irq(&q->queue_lock);
Jens Axboebd166ef2017-01-17 06:03:22 -07001530
Jens Axboe344e9ff2018-11-15 12:22:51 -07001531 if (queue_is_mq(q))
Jens Axboebd166ef2017-01-17 06:03:22 -07001532 blk_mq_unfreeze_queue(q);
Tejun Heoa2b16932012-04-13 13:11:33 -07001533}
1534EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1535
Jason Yane55cf792022-06-29 15:09:17 +08001536static void blkcg_free_all_cpd(struct blkcg_policy *pol)
1537{
1538 struct blkcg *blkcg;
1539
1540 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1541 if (blkcg->cpd[pol->plid]) {
1542 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1543 blkcg->cpd[pol->plid] = NULL;
1544 }
1545 }
1546}
1547
Tejun Heoa2b16932012-04-13 13:11:33 -07001548/**
Tejun Heo3c798392012-04-16 13:57:25 -07001549 * blkcg_policy_register - register a blkcg policy
1550 * @pol: blkcg policy to register
Tejun Heo8bd435b2012-04-13 13:11:28 -07001551 *
Tejun Heo3c798392012-04-16 13:57:25 -07001552 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1553 * successful registration. Returns 0 on success and -errno on failure.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001554 */
Jens Axboed5bf0292014-06-22 16:31:56 -06001555int blkcg_policy_register(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001556{
Tejun Heo06b285b2015-07-09 16:39:50 -04001557 struct blkcg *blkcg;
Tejun Heo8bd435b2012-04-13 13:11:28 -07001558 int i, ret;
Tejun Heoe8989fa2012-03-05 13:15:20 -08001559
Tejun Heo838f13b2015-07-09 16:39:47 -04001560 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001561 mutex_lock(&blkcg_pol_mutex);
1562
Tejun Heo8bd435b2012-04-13 13:11:28 -07001563 /* find an empty slot */
1564 ret = -ENOSPC;
1565 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo3c798392012-04-16 13:57:25 -07001566 if (!blkcg_policy[i])
Tejun Heo8bd435b2012-04-13 13:11:28 -07001567 break;
Jens Axboe01c5f852018-09-11 10:59:53 -06001568 if (i >= BLKCG_MAX_POLS) {
1569 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
Tejun Heo838f13b2015-07-09 16:39:47 -04001570 goto err_unlock;
Jens Axboe01c5f852018-09-11 10:59:53 -06001571 }
Tejun Heo035d10b2012-03-05 13:15:04 -08001572
weiping zhange8401072017-10-17 23:56:21 +08001573 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1574 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1575 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1576 goto err_unlock;
1577
Tejun Heo06b285b2015-07-09 16:39:50 -04001578 /* register @pol */
Tejun Heo3c798392012-04-16 13:57:25 -07001579 pol->plid = i;
Tejun Heo06b285b2015-07-09 16:39:50 -04001580 blkcg_policy[pol->plid] = pol;
1581
1582 /* allocate and install cpd's */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001583 if (pol->cpd_alloc_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001584 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1585 struct blkcg_policy_data *cpd;
1586
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001587 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Bart Van Asschebbb427e2016-09-29 08:33:30 -07001588 if (!cpd)
Tejun Heo06b285b2015-07-09 16:39:50 -04001589 goto err_free_cpds;
Tejun Heo06b285b2015-07-09 16:39:50 -04001590
Tejun Heo81437642015-08-18 14:55:15 -07001591 blkcg->cpd[pol->plid] = cpd;
1592 cpd->blkcg = blkcg;
Tejun Heo06b285b2015-07-09 16:39:50 -04001593 cpd->plid = pol->plid;
Tejun Heo86a5bba2019-08-28 15:05:52 -07001594 if (pol->cpd_init_fn)
1595 pol->cpd_init_fn(cpd);
Tejun Heo06b285b2015-07-09 16:39:50 -04001596 }
1597 }
1598
Tejun Heo838f13b2015-07-09 16:39:47 -04001599 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001600
Tejun Heo8bd435b2012-04-13 13:11:28 -07001601 /* everything is in place, add intf files for the new policy */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001602 if (pol->dfl_cftypes)
1603 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1604 pol->dfl_cftypes));
Tejun Heo880f50e22015-08-18 14:55:30 -07001605 if (pol->legacy_cftypes)
Tejun Heoc165b3e2015-08-18 14:55:29 -07001606 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
Tejun Heo880f50e22015-08-18 14:55:30 -07001607 pol->legacy_cftypes));
Tejun Heo838f13b2015-07-09 16:39:47 -04001608 mutex_unlock(&blkcg_pol_register_mutex);
1609 return 0;
1610
Tejun Heo06b285b2015-07-09 16:39:50 -04001611err_free_cpds:
Jason Yane55cf792022-06-29 15:09:17 +08001612 if (pol->cpd_free_fn)
1613 blkcg_free_all_cpd(pol);
1614
Tejun Heo06b285b2015-07-09 16:39:50 -04001615 blkcg_policy[pol->plid] = NULL;
Tejun Heo838f13b2015-07-09 16:39:47 -04001616err_unlock:
Tejun Heobc0d6502012-04-13 13:11:26 -07001617 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001618 mutex_unlock(&blkcg_pol_register_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001619 return ret;
Vivek Goyal3e252062009-12-04 10:36:42 -05001620}
Tejun Heo3c798392012-04-16 13:57:25 -07001621EXPORT_SYMBOL_GPL(blkcg_policy_register);
Vivek Goyal3e252062009-12-04 10:36:42 -05001622
Tejun Heo8bd435b2012-04-13 13:11:28 -07001623/**
Tejun Heo3c798392012-04-16 13:57:25 -07001624 * blkcg_policy_unregister - unregister a blkcg policy
1625 * @pol: blkcg policy to unregister
Tejun Heo8bd435b2012-04-13 13:11:28 -07001626 *
Tejun Heo3c798392012-04-16 13:57:25 -07001627 * Undo blkcg_policy_register(@pol). Might sleep.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001628 */
Tejun Heo3c798392012-04-16 13:57:25 -07001629void blkcg_policy_unregister(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001630{
Tejun Heo838f13b2015-07-09 16:39:47 -04001631 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001632
Tejun Heo3c798392012-04-16 13:57:25 -07001633 if (WARN_ON(blkcg_policy[pol->plid] != pol))
Tejun Heo8bd435b2012-04-13 13:11:28 -07001634 goto out_unlock;
1635
1636 /* kill the intf files first */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001637 if (pol->dfl_cftypes)
1638 cgroup_rm_cftypes(pol->dfl_cftypes);
Tejun Heo880f50e22015-08-18 14:55:30 -07001639 if (pol->legacy_cftypes)
1640 cgroup_rm_cftypes(pol->legacy_cftypes);
Tejun Heo44ea53d2012-04-01 14:38:43 -07001641
Tejun Heo06b285b2015-07-09 16:39:50 -04001642 /* remove cpds and unregister */
Tejun Heo838f13b2015-07-09 16:39:47 -04001643 mutex_lock(&blkcg_pol_mutex);
Tejun Heo06b285b2015-07-09 16:39:50 -04001644
Jason Yane55cf792022-06-29 15:09:17 +08001645 if (pol->cpd_free_fn)
1646 blkcg_free_all_cpd(pol);
1647
Tejun Heo3c798392012-04-16 13:57:25 -07001648 blkcg_policy[pol->plid] = NULL;
Tejun Heo06b285b2015-07-09 16:39:50 -04001649
Tejun Heobc0d6502012-04-13 13:11:26 -07001650 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001651out_unlock:
1652 mutex_unlock(&blkcg_pol_register_mutex);
Vivek Goyal3e252062009-12-04 10:36:42 -05001653}
Tejun Heo3c798392012-04-16 13:57:25 -07001654EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
Josef Bacik903d23f2018-07-03 11:14:52 -04001655
Tejun Heod3f77df2019-06-27 13:39:52 -07001656bool __blkcg_punt_bio_submit(struct bio *bio)
1657{
1658 struct blkcg_gq *blkg = bio->bi_blkg;
1659
1660 /* consume the flag first */
1661 bio->bi_opf &= ~REQ_CGROUP_PUNT;
1662
1663 /* never bounce for the root cgroup */
1664 if (!blkg->parent)
1665 return false;
1666
1667 spin_lock_bh(&blkg->async_bio_lock);
1668 bio_list_add(&blkg->async_bios, bio);
1669 spin_unlock_bh(&blkg->async_bio_lock);
1670
1671 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
1672 return true;
1673}
1674
Josef Bacikd09d8df2018-07-03 11:14:55 -04001675/*
1676 * Scale the accumulated delay based on how long it has been since we updated
1677 * the delay. We only call this when we are adding delay, in case it's been a
1678 * while since we added delay, and when we are checking to see if we need to
1679 * delay a task, to account for any delays that may have occurred.
1680 */
1681static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
1682{
1683 u64 old = atomic64_read(&blkg->delay_start);
1684
Tejun Heo54c52e12020-04-13 12:27:55 -04001685 /* negative use_delay means no scaling, see blkcg_set_delay() */
1686 if (atomic_read(&blkg->use_delay) < 0)
1687 return;
1688
Josef Bacikd09d8df2018-07-03 11:14:55 -04001689 /*
1690 * We only want to scale down every second. The idea here is that we
1691 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1692 * time window. We only want to throttle tasks for recent delay that
1693 * has occurred, in 1 second time windows since that's the maximum
1694 * things can be throttled. We save the current delay window in
1695 * blkg->last_delay so we know what amount is still left to be charged
1696 * to the blkg from this point onward. blkg->last_use keeps track of
1697 * the use_delay counter. The idea is if we're unthrottling the blkg we
1698 * are ok with whatever is happening now, and we can take away more of
1699 * the accumulated delay as we've already throttled enough that
1700 * everybody is happy with their IO latencies.
1701 */
1702 if (time_before64(old + NSEC_PER_SEC, now) &&
Uros Bizjak96388f52022-07-12 17:44:55 +02001703 atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) {
Josef Bacikd09d8df2018-07-03 11:14:55 -04001704 u64 cur = atomic64_read(&blkg->delay_nsec);
1705 u64 sub = min_t(u64, blkg->last_delay, now - old);
1706 int cur_use = atomic_read(&blkg->use_delay);
1707
1708 /*
1709 * We've been unthrottled, subtract a larger chunk of our
1710 * accumulated delay.
1711 */
1712 if (cur_use < blkg->last_use)
1713 sub = max_t(u64, sub, blkg->last_delay >> 1);
1714
1715 /*
1716 * This shouldn't happen, but handle it anyway. Our delay_nsec
1717 * should only ever be growing except here where we subtract out
1718 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1719 * rather not end up with negative numbers.
1720 */
1721 if (unlikely(cur < sub)) {
1722 atomic64_set(&blkg->delay_nsec, 0);
1723 blkg->last_delay = 0;
1724 } else {
1725 atomic64_sub(sub, &blkg->delay_nsec);
1726 blkg->last_delay = cur - sub;
1727 }
1728 blkg->last_use = cur_use;
1729 }
1730}
1731
1732/*
1733 * This is called when we want to actually walk up the hierarchy and check to
1734 * see if we need to throttle, and then actually throttle if there is some
1735 * accumulated delay. This should only be called upon return to user space so
1736 * we're not holding some lock that would induce a priority inversion.
1737 */
1738static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
1739{
Josef Bacikfd112c72019-07-09 14:41:29 -07001740 unsigned long pflags;
Tejun Heo5160a5a2020-09-01 14:52:52 -04001741 bool clamp;
Josef Bacikd09d8df2018-07-03 11:14:55 -04001742 u64 now = ktime_to_ns(ktime_get());
1743 u64 exp;
1744 u64 delay_nsec = 0;
1745 int tok;
1746
1747 while (blkg->parent) {
Tejun Heo5160a5a2020-09-01 14:52:52 -04001748 int use_delay = atomic_read(&blkg->use_delay);
1749
1750 if (use_delay) {
1751 u64 this_delay;
1752
Josef Bacikd09d8df2018-07-03 11:14:55 -04001753 blkcg_scale_delay(blkg, now);
Tejun Heo5160a5a2020-09-01 14:52:52 -04001754 this_delay = atomic64_read(&blkg->delay_nsec);
1755 if (this_delay > delay_nsec) {
1756 delay_nsec = this_delay;
1757 clamp = use_delay > 0;
1758 }
Josef Bacikd09d8df2018-07-03 11:14:55 -04001759 }
1760 blkg = blkg->parent;
1761 }
1762
1763 if (!delay_nsec)
1764 return;
1765
1766 /*
1767 * Let's not sleep for all eternity if we've amassed a huge delay.
1768 * Swapping or metadata IO can accumulate 10's of seconds worth of
1769 * delay, and we want userspace to be able to do _something_ so cap the
Tejun Heo5160a5a2020-09-01 14:52:52 -04001770 * delays at 0.25s. If there's 10's of seconds worth of delay then the
1771 * tasks will be delayed for 0.25 second for every syscall. If
1772 * blkcg_set_delay() was used as indicated by negative use_delay, the
1773 * caller is responsible for regulating the range.
Josef Bacikd09d8df2018-07-03 11:14:55 -04001774 */
Tejun Heo5160a5a2020-09-01 14:52:52 -04001775 if (clamp)
1776 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001777
Josef Bacikfd112c72019-07-09 14:41:29 -07001778 if (use_memdelay)
1779 psi_memstall_enter(&pflags);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001780
1781 exp = ktime_add_ns(now, delay_nsec);
1782 tok = io_schedule_prepare();
1783 do {
1784 __set_current_state(TASK_KILLABLE);
1785 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
1786 break;
1787 } while (!fatal_signal_pending(current));
1788 io_schedule_finish(tok);
Josef Bacikfd112c72019-07-09 14:41:29 -07001789
1790 if (use_memdelay)
1791 psi_memstall_leave(&pflags);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001792}
1793
1794/**
1795 * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1796 *
1797 * This is only called if we've been marked with set_notify_resume(). Obviously
1798 * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1799 * check to see if current->throttle_queue is set and if not this doesn't do
1800 * anything. This should only ever be called by the resume code, it's not meant
1801 * to be called by people willy-nilly as it will actually do the work to
1802 * throttle the task if it is setup for throttling.
1803 */
1804void blkcg_maybe_throttle_current(void)
1805{
1806 struct request_queue *q = current->throttle_queue;
Josef Bacikd09d8df2018-07-03 11:14:55 -04001807 struct blkcg *blkcg;
1808 struct blkcg_gq *blkg;
1809 bool use_memdelay = current->use_memdelay;
1810
1811 if (!q)
1812 return;
1813
1814 current->throttle_queue = NULL;
1815 current->use_memdelay = false;
1816
1817 rcu_read_lock();
Christoph Hellwig82778252022-04-20 06:27:22 +02001818 blkcg = css_to_blkcg(blkcg_css());
Josef Bacikd09d8df2018-07-03 11:14:55 -04001819 if (!blkcg)
1820 goto out;
1821 blkg = blkg_lookup(blkcg, q);
1822 if (!blkg)
1823 goto out;
Dennis Zhou7754f662018-12-05 12:10:39 -05001824 if (!blkg_tryget(blkg))
Josef Bacikd09d8df2018-07-03 11:14:55 -04001825 goto out;
1826 rcu_read_unlock();
Josef Bacikd09d8df2018-07-03 11:14:55 -04001827
1828 blkcg_maybe_throttle_blkg(blkg, use_memdelay);
1829 blkg_put(blkg);
Josef Bacikcc7ecc252018-07-31 12:39:03 -04001830 blk_put_queue(q);
Josef Bacikd09d8df2018-07-03 11:14:55 -04001831 return;
1832out:
1833 rcu_read_unlock();
1834 blk_put_queue(q);
1835}
Josef Bacikd09d8df2018-07-03 11:14:55 -04001836
1837/**
1838 * blkcg_schedule_throttle - this task needs to check for throttling
Bart Van Assche537d71b2019-03-20 13:18:45 -07001839 * @q: the request queue IO was submitted on
1840 * @use_memdelay: do we charge this to memory delay for PSI
Josef Bacikd09d8df2018-07-03 11:14:55 -04001841 *
1842 * This is called by the IO controller when we know there's delay accumulated
1843 * for the blkg for this task. We do not pass the blkg because there are places
1844 * we call this that may not have that information, the swapping code for
1845 * instance will only have a request_queue at that point. This set's the
1846 * notify_resume for the task to check and see if it requires throttling before
1847 * returning to user space.
1848 *
1849 * We will only schedule once per syscall. You can call this over and over
1850 * again and it will only do the check once upon return to user space, and only
1851 * throttle once. If the task needs to be throttled again it'll need to be
1852 * re-set at the next time we see the task.
1853 */
1854void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay)
1855{
1856 if (unlikely(current->flags & PF_KTHREAD))
1857 return;
1858
Chunguang Xu49d18222021-01-25 13:05:28 +08001859 if (current->throttle_queue != q) {
1860 if (!blk_get_queue(q))
1861 return;
Josef Bacikd09d8df2018-07-03 11:14:55 -04001862
Chunguang Xu49d18222021-01-25 13:05:28 +08001863 if (current->throttle_queue)
1864 blk_put_queue(current->throttle_queue);
1865 current->throttle_queue = q;
1866 }
1867
Josef Bacikd09d8df2018-07-03 11:14:55 -04001868 if (use_memdelay)
1869 current->use_memdelay = use_memdelay;
1870 set_notify_resume(current);
1871}
Josef Bacikd09d8df2018-07-03 11:14:55 -04001872
1873/**
1874 * blkcg_add_delay - add delay to this blkg
Bart Van Assche537d71b2019-03-20 13:18:45 -07001875 * @blkg: blkg of interest
1876 * @now: the current time in nanoseconds
1877 * @delta: how many nanoseconds of delay to add
Josef Bacikd09d8df2018-07-03 11:14:55 -04001878 *
1879 * Charge @delta to the blkg's current delay accumulation. This is used to
1880 * throttle tasks if an IO controller thinks we need more throttling.
1881 */
1882void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
1883{
Tejun Heo54c52e12020-04-13 12:27:55 -04001884 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
1885 return;
Josef Bacikd09d8df2018-07-03 11:14:55 -04001886 blkcg_scale_delay(blkg, now);
1887 atomic64_add(delta, &blkg->delay_nsec);
1888}
Josef Bacikd09d8df2018-07-03 11:14:55 -04001889
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001890/**
1891 * blkg_tryget_closest - try and get a blkg ref on the closet blkg
Christoph Hellwig13c78632020-06-27 09:31:54 +02001892 * @bio: target bio
1893 * @css: target css
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001894 *
Christoph Hellwig13c78632020-06-27 09:31:54 +02001895 * As the failure mode here is to walk up the blkg tree, this ensure that the
1896 * blkg->parent pointers are always valid. This returns the blkg that it ended
1897 * up taking a reference on or %NULL if no reference was taken.
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001898 */
Christoph Hellwig13c78632020-06-27 09:31:54 +02001899static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
1900 struct cgroup_subsys_state *css)
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001901{
Christoph Hellwig13c78632020-06-27 09:31:54 +02001902 struct blkcg_gq *blkg, *ret_blkg = NULL;
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001903
Christoph Hellwig13c78632020-06-27 09:31:54 +02001904 rcu_read_lock();
Christoph Hellwig309dca302021-01-24 11:02:34 +01001905 blkg = blkg_lookup_create(css_to_blkcg(css),
Pavel Begunkoved6cdde2021-10-14 15:03:30 +01001906 bdev_get_queue(bio->bi_bdev));
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001907 while (blkg) {
1908 if (blkg_tryget(blkg)) {
1909 ret_blkg = blkg;
1910 break;
1911 }
1912 blkg = blkg->parent;
1913 }
Christoph Hellwig13c78632020-06-27 09:31:54 +02001914 rcu_read_unlock();
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001915
1916 return ret_blkg;
1917}
1918
1919/**
1920 * bio_associate_blkg_from_css - associate a bio with a specified css
1921 * @bio: target bio
1922 * @css: target css
1923 *
1924 * Associate @bio with the blkg found by combining the css's blkg and the
1925 * request_queue of the @bio. An association failure is handled by walking up
1926 * the blkg tree. Therefore, the blkg associated can be anything between @blkg
1927 * and q->root_blkg. This situation only happens when a cgroup is dying and
1928 * then the remaining bios will spill to the closest alive blkg.
1929 *
1930 * A reference will be taken on the blkg and will be released when @bio is
1931 * freed.
1932 */
1933void bio_associate_blkg_from_css(struct bio *bio,
1934 struct cgroup_subsys_state *css)
1935{
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001936 if (bio->bi_blkg)
1937 blkg_put(bio->bi_blkg);
1938
Christoph Hellwiga5b97522020-06-27 09:31:53 +02001939 if (css && css->parent) {
Christoph Hellwig13c78632020-06-27 09:31:54 +02001940 bio->bi_blkg = blkg_tryget_closest(bio, css);
Christoph Hellwiga5b97522020-06-27 09:31:53 +02001941 } else {
Pavel Begunkoved6cdde2021-10-14 15:03:30 +01001942 blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg);
1943 bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg;
Christoph Hellwiga5b97522020-06-27 09:31:53 +02001944 }
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001945}
1946EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
1947
1948/**
1949 * bio_associate_blkg - associate a bio with a blkg
1950 * @bio: target bio
1951 *
1952 * Associate @bio with the blkg found from the bio's css and request_queue.
1953 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
1954 * already associated, the css is reused and association redone as the
1955 * request_queue may have changed.
1956 */
1957void bio_associate_blkg(struct bio *bio)
1958{
1959 struct cgroup_subsys_state *css;
1960
1961 rcu_read_lock();
1962
1963 if (bio->bi_blkg)
Christoph Hellwigbbb1ebe2022-04-20 06:27:17 +02001964 css = bio_blkcg_css(bio);
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001965 else
1966 css = blkcg_css();
1967
1968 bio_associate_blkg_from_css(bio, css);
1969
1970 rcu_read_unlock();
1971}
1972EXPORT_SYMBOL_GPL(bio_associate_blkg);
1973
1974/**
1975 * bio_clone_blkg_association - clone blkg association from src to dst bio
1976 * @dst: destination bio
1977 * @src: source bio
1978 */
1979void bio_clone_blkg_association(struct bio *dst, struct bio *src)
1980{
Jan Kara22b106e2022-06-02 10:12:42 +02001981 if (src->bi_blkg)
1982 bio_associate_blkg_from_css(dst, bio_blkcg_css(src));
Christoph Hellwig28fc5912020-06-27 09:31:51 +02001983}
1984EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
1985
Christoph Hellwigdb18a532020-06-27 09:31:58 +02001986static int blk_cgroup_io_type(struct bio *bio)
1987{
1988 if (op_is_discard(bio->bi_opf))
1989 return BLKG_IOSTAT_DISCARD;
1990 if (op_is_write(bio->bi_opf))
1991 return BLKG_IOSTAT_WRITE;
1992 return BLKG_IOSTAT_READ;
1993}
1994
1995void blk_cgroup_bio_start(struct bio *bio)
1996{
1997 int rwd = blk_cgroup_io_type(bio), cpu;
1998 struct blkg_iostat_set *bis;
Tejun Heo3c08b092021-10-14 13:20:22 -10001999 unsigned long flags;
Christoph Hellwigdb18a532020-06-27 09:31:58 +02002000
2001 cpu = get_cpu();
2002 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
Tejun Heo3c08b092021-10-14 13:20:22 -10002003 flags = u64_stats_update_begin_irqsave(&bis->sync);
Christoph Hellwigdb18a532020-06-27 09:31:58 +02002004
2005 /*
2006 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
2007 * bio and we would have already accounted for the size of the bio.
2008 */
2009 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
2010 bio_set_flag(bio, BIO_CGROUP_ACCT);
Colin Ian King0b8cc252020-06-30 16:54:41 +01002011 bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
Christoph Hellwigdb18a532020-06-27 09:31:58 +02002012 }
2013 bis->cur.ios[rwd]++;
2014
Tejun Heo3c08b092021-10-14 13:20:22 -10002015 u64_stats_update_end_irqrestore(&bis->sync, flags);
Christoph Hellwigdb18a532020-06-27 09:31:58 +02002016 if (cgroup_subsys_on_dfl(io_cgrp_subsys))
2017 cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu);
2018 put_cpu();
2019}
2020
Christoph Hellwig216889a2022-04-20 06:27:13 +02002021bool blk_cgroup_congested(void)
2022{
2023 struct cgroup_subsys_state *css;
2024 bool ret = false;
2025
2026 rcu_read_lock();
Christoph Hellwigd200ca12022-04-20 06:27:21 +02002027 for (css = blkcg_css(); css; css = css->parent) {
Christoph Hellwig216889a2022-04-20 06:27:13 +02002028 if (atomic_read(&css->cgroup->congestion_count)) {
2029 ret = true;
2030 break;
2031 }
Christoph Hellwig216889a2022-04-20 06:27:13 +02002032 }
2033 rcu_read_unlock();
2034 return ret;
2035}
2036
Tejun Heod3f77df2019-06-27 13:39:52 -07002037static int __init blkcg_init(void)
2038{
2039 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
2040 WQ_MEM_RECLAIM | WQ_FREEZABLE |
2041 WQ_UNBOUND | WQ_SYSFS, 0);
2042 if (!blkcg_punt_bio_wq)
2043 return -ENOMEM;
2044 return 0;
2045}
2046subsys_initcall(blkcg_init);
2047
Josef Bacik903d23f2018-07-03 11:14:52 -04002048module_param(blkcg_debug_stats, bool, 0644);
2049MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");