blob: 4117524ca45bc76cceb163c42f752ffd8d7e1659 [file] [log] [blame]
Vivek Goyal31e4c282009-12-03 12:59:42 -05001/*
2 * Common Block IO controller cgroup interface
3 *
4 * Based on ideas and code from CFQ, CFS and BFQ:
5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
6 *
7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8 * Paolo Valente <paolo.valente@unimore.it>
9 *
10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11 * Nauman Rafique <nauman@google.com>
Arianna Avanzinie48453c2015-06-05 23:38:42 +020012 *
13 * For policy-specific per-blkcg data:
14 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
15 * Arianna Avanzini <avanzini.arianna@gmail.com>
Vivek Goyal31e4c282009-12-03 12:59:42 -050016 */
17#include <linux/ioprio.h>
Vivek Goyal22084192009-12-03 12:59:49 -050018#include <linux/kdev_t.h>
Vivek Goyal9d6a9862009-12-04 10:36:41 -050019#include <linux/module.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010020#include <linux/sched/signal.h>
Stephen Rothwellaccee782009-12-07 19:29:39 +110021#include <linux/err.h>
Divyesh Shah91952912010-04-01 15:01:41 -070022#include <linux/blkdev.h>
Tejun Heo52ebea72015-05-22 17:13:37 -040023#include <linux/backing-dev.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090024#include <linux/slab.h>
Gui Jianfeng34d0f172010-04-13 16:05:49 +080025#include <linux/genhd.h>
Tejun Heo72e06c22012-03-05 13:15:00 -080026#include <linux/delay.h>
Tejun Heo9a9e8a22012-03-19 15:10:56 -070027#include <linux/atomic.h>
Tejun Heo36aa9e52015-08-18 14:55:31 -070028#include <linux/ctype.h>
Tejun Heoeea8f412015-05-22 17:13:17 -040029#include <linux/blk-cgroup.h>
Tejun Heo5efd6112012-03-05 13:15:12 -080030#include "blk.h"
Vivek Goyal3e252062009-12-04 10:36:42 -050031
Divyesh Shah84c124d2010-04-09 08:31:19 +020032#define MAX_KEY_LEN 100
33
Tejun Heo838f13b2015-07-09 16:39:47 -040034/*
35 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
36 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
37 * policy [un]register operations including cgroup file additions /
38 * removals. Putting cgroup file registration outside blkcg_pol_mutex
39 * allows grabbing it from cgroup callbacks.
40 */
41static DEFINE_MUTEX(blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -070042static DEFINE_MUTEX(blkcg_pol_mutex);
Tejun Heo923adde2012-03-05 13:15:13 -080043
Arianna Avanzinie48453c2015-06-05 23:38:42 +020044struct blkcg blkcg_root;
Tejun Heo3c798392012-04-16 13:57:25 -070045EXPORT_SYMBOL_GPL(blkcg_root);
Vivek Goyal9d6a9862009-12-04 10:36:41 -050046
Tejun Heo496d5e72015-05-22 17:13:21 -040047struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
48
Tejun Heo3c798392012-04-16 13:57:25 -070049static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
Tejun Heo035d10b2012-03-05 13:15:04 -080050
Tejun Heo7876f932015-07-09 16:39:49 -040051static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
52
Tejun Heoa2b16932012-04-13 13:11:33 -070053static bool blkcg_policy_enabled(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -070054 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -070055{
56 return pol && test_bit(pol->plid, q->blkcg_pols);
57}
58
Tejun Heo03814112012-03-05 13:15:14 -080059/**
60 * blkg_free - free a blkg
61 * @blkg: blkg to free
62 *
63 * Free @blkg which may be partially allocated.
64 */
Tejun Heo3c798392012-04-16 13:57:25 -070065static void blkg_free(struct blkcg_gq *blkg)
Tejun Heo03814112012-03-05 13:15:14 -080066{
Tejun Heoe8989fa2012-03-05 13:15:20 -080067 int i;
Tejun Heo549d3aa2012-03-05 13:15:16 -080068
69 if (!blkg)
70 return;
71
Tejun Heodb613672013-05-14 13:52:31 -070072 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo001bea72015-08-18 14:55:11 -070073 if (blkg->pd[i])
74 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
Tejun Heoe8989fa2012-03-05 13:15:20 -080075
Tejun Heo994b7832015-08-18 14:55:07 -070076 if (blkg->blkcg != &blkcg_root)
Bart Van Asscheb425e502017-05-31 14:43:45 -070077 blk_exit_rl(blkg->q, &blkg->rl);
Tejun Heo77ea7332015-08-18 14:55:24 -070078
79 blkg_rwstat_exit(&blkg->stat_ios);
80 blkg_rwstat_exit(&blkg->stat_bytes);
Tejun Heo549d3aa2012-03-05 13:15:16 -080081 kfree(blkg);
Tejun Heo03814112012-03-05 13:15:14 -080082}
83
84/**
85 * blkg_alloc - allocate a blkg
86 * @blkcg: block cgroup the new blkg is associated with
87 * @q: request_queue the new blkg is associated with
Tejun Heo15974992012-06-04 20:40:52 -070088 * @gfp_mask: allocation mask to use
Tejun Heo03814112012-03-05 13:15:14 -080089 *
Tejun Heoe8989fa2012-03-05 13:15:20 -080090 * Allocate a new blkg assocating @blkcg and @q.
Tejun Heo03814112012-03-05 13:15:14 -080091 */
Tejun Heo15974992012-06-04 20:40:52 -070092static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
93 gfp_t gfp_mask)
Tejun Heo03814112012-03-05 13:15:14 -080094{
Tejun Heo3c798392012-04-16 13:57:25 -070095 struct blkcg_gq *blkg;
Tejun Heoe8989fa2012-03-05 13:15:20 -080096 int i;
Tejun Heo03814112012-03-05 13:15:14 -080097
98 /* alloc and init base part */
Tejun Heo15974992012-06-04 20:40:52 -070099 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
Tejun Heo03814112012-03-05 13:15:14 -0800100 if (!blkg)
101 return NULL;
102
Tejun Heo77ea7332015-08-18 14:55:24 -0700103 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
104 blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
105 goto err_free;
106
Tejun Heoc875f4d2012-03-05 13:15:22 -0800107 blkg->q = q;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800108 INIT_LIST_HEAD(&blkg->q_node);
Tejun Heo03814112012-03-05 13:15:14 -0800109 blkg->blkcg = blkcg;
Tejun Heoa5049a82014-06-19 17:42:57 -0400110 atomic_set(&blkg->refcnt, 1);
Tejun Heo03814112012-03-05 13:15:14 -0800111
Tejun Heoa0516612012-06-26 15:05:44 -0700112 /* root blkg uses @q->root_rl, init rl only for !root blkgs */
113 if (blkcg != &blkcg_root) {
114 if (blk_init_rl(&blkg->rl, q, gfp_mask))
115 goto err_free;
116 blkg->rl.blkg = blkg;
117 }
118
Tejun Heo8bd435b2012-04-13 13:11:28 -0700119 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700120 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heoe8989fa2012-03-05 13:15:20 -0800121 struct blkg_policy_data *pd;
Tejun Heo03814112012-03-05 13:15:14 -0800122
Tejun Heoa2b16932012-04-13 13:11:33 -0700123 if (!blkcg_policy_enabled(q, pol))
Tejun Heoe8989fa2012-03-05 13:15:20 -0800124 continue;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800125
Tejun Heoe8989fa2012-03-05 13:15:20 -0800126 /* alloc per-policy data and attach it to blkg */
Tejun Heo001bea72015-08-18 14:55:11 -0700127 pd = pol->pd_alloc_fn(gfp_mask, q->node);
Tejun Heoa0516612012-06-26 15:05:44 -0700128 if (!pd)
129 goto err_free;
Tejun Heo549d3aa2012-03-05 13:15:16 -0800130
Tejun Heoe8989fa2012-03-05 13:15:20 -0800131 blkg->pd[i] = pd;
132 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -0800133 pd->plid = i;
Tejun Heoe8989fa2012-03-05 13:15:20 -0800134 }
135
Tejun Heo03814112012-03-05 13:15:14 -0800136 return blkg;
Tejun Heoa0516612012-06-26 15:05:44 -0700137
138err_free:
139 blkg_free(blkg);
140 return NULL;
Tejun Heo03814112012-03-05 13:15:14 -0800141}
142
Tejun Heo24f29042015-08-18 14:55:17 -0700143struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
144 struct request_queue *q, bool update_hint)
Tejun Heo80fd9972012-04-13 14:50:53 -0700145{
Tejun Heo3c798392012-04-16 13:57:25 -0700146 struct blkcg_gq *blkg;
Tejun Heo80fd9972012-04-13 14:50:53 -0700147
Tejun Heoa6371202012-04-19 16:29:24 -0700148 /*
Tejun Heo86cde6b2013-01-09 08:05:10 -0800149 * Hint didn't match. Look up from the radix tree. Note that the
150 * hint can only be updated under queue_lock as otherwise @blkg
151 * could have already been removed from blkg_tree. The caller is
152 * responsible for grabbing queue_lock if @update_hint.
Tejun Heoa6371202012-04-19 16:29:24 -0700153 */
154 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
Tejun Heo86cde6b2013-01-09 08:05:10 -0800155 if (blkg && blkg->q == q) {
156 if (update_hint) {
157 lockdep_assert_held(q->queue_lock);
158 rcu_assign_pointer(blkcg->blkg_hint, blkg);
159 }
Tejun Heoa6371202012-04-19 16:29:24 -0700160 return blkg;
Tejun Heo86cde6b2013-01-09 08:05:10 -0800161 }
Tejun Heoa6371202012-04-19 16:29:24 -0700162
Tejun Heo80fd9972012-04-13 14:50:53 -0700163 return NULL;
164}
Tejun Heoae118892015-08-18 14:55:20 -0700165EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
Tejun Heo80fd9972012-04-13 14:50:53 -0700166
Tejun Heo15974992012-06-04 20:40:52 -0700167/*
Jens Axboed708f0d2017-03-29 11:25:48 -0600168 * If @new_blkg is %NULL, this function tries to allocate a new one as
169 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
Tejun Heo15974992012-06-04 20:40:52 -0700170 */
Tejun Heo86cde6b2013-01-09 08:05:10 -0800171static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
Jens Axboed708f0d2017-03-29 11:25:48 -0600172 struct request_queue *q,
173 struct blkcg_gq *new_blkg)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400174{
Jens Axboed708f0d2017-03-29 11:25:48 -0600175 struct blkcg_gq *blkg;
Tejun Heoce7acfe2015-05-22 17:13:38 -0400176 struct bdi_writeback_congested *wb_congested;
Tejun Heof427d902013-01-09 08:05:12 -0800177 int i, ret;
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400178
Tejun Heocd1604f2012-03-05 13:15:06 -0800179 WARN_ON_ONCE(!rcu_read_lock_held());
180 lockdep_assert_held(q->queue_lock);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500181
Tejun Heo7ee9c562012-03-05 13:15:11 -0800182 /* blkg holds a reference to blkcg */
Tejun Heoec903c02014-05-13 12:11:01 -0400183 if (!css_tryget_online(&blkcg->css)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700184 ret = -ENODEV;
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800185 goto err_free_blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700186 }
Tejun Heocd1604f2012-03-05 13:15:06 -0800187
Tahsin Erdogan7fc6b872017-03-09 00:05:31 -0800188 wb_congested = wb_congested_get_create(q->backing_dev_info,
Jens Axboed708f0d2017-03-29 11:25:48 -0600189 blkcg->css.id,
190 GFP_NOWAIT | __GFP_NOWARN);
191 if (!wb_congested) {
Tahsin Erdogan7fc6b872017-03-09 00:05:31 -0800192 ret = -ENOMEM;
Jens Axboed708f0d2017-03-29 11:25:48 -0600193 goto err_put_css;
Tahsin Erdogan7fc6b872017-03-09 00:05:31 -0800194 }
195
Jens Axboed708f0d2017-03-29 11:25:48 -0600196 /* allocate */
197 if (!new_blkg) {
198 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
199 if (unlikely(!new_blkg)) {
200 ret = -ENOMEM;
201 goto err_put_congested;
202 }
203 }
204 blkg = new_blkg;
Tahsin Erdogan7fc6b872017-03-09 00:05:31 -0800205 blkg->wb_congested = wb_congested;
206
Tejun Heodb613672013-05-14 13:52:31 -0700207 /* link parent */
Tejun Heo3c547862013-01-09 08:05:10 -0800208 if (blkcg_parent(blkcg)) {
209 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
210 if (WARN_ON_ONCE(!blkg->parent)) {
Tejun Heo20386ce2015-08-18 14:55:28 -0700211 ret = -ENODEV;
Jens Axboed708f0d2017-03-29 11:25:48 -0600212 goto err_put_congested;
Tejun Heo3c547862013-01-09 08:05:10 -0800213 }
214 blkg_get(blkg->parent);
215 }
216
Tejun Heodb613672013-05-14 13:52:31 -0700217 /* invoke per-policy init */
218 for (i = 0; i < BLKCG_MAX_POLS; i++) {
219 struct blkcg_policy *pol = blkcg_policy[i];
220
221 if (blkg->pd[i] && pol->pd_init_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700222 pol->pd_init_fn(blkg->pd[i]);
Tejun Heodb613672013-05-14 13:52:31 -0700223 }
224
225 /* insert */
Tejun Heoa6371202012-04-19 16:29:24 -0700226 spin_lock(&blkcg->lock);
227 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
228 if (likely(!ret)) {
229 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
230 list_add(&blkg->q_node, &q->blkg_list);
Tejun Heof427d902013-01-09 08:05:12 -0800231
232 for (i = 0; i < BLKCG_MAX_POLS; i++) {
233 struct blkcg_policy *pol = blkcg_policy[i];
234
235 if (blkg->pd[i] && pol->pd_online_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700236 pol->pd_online_fn(blkg->pd[i]);
Tejun Heof427d902013-01-09 08:05:12 -0800237 }
Tejun Heoa6371202012-04-19 16:29:24 -0700238 }
Tejun Heof427d902013-01-09 08:05:12 -0800239 blkg->online = true;
Tejun Heoa6371202012-04-19 16:29:24 -0700240 spin_unlock(&blkcg->lock);
241
Tejun Heoec13b1d2015-05-22 17:13:19 -0400242 if (!ret)
Tejun Heoa6371202012-04-19 16:29:24 -0700243 return blkg;
Tejun Heo15974992012-06-04 20:40:52 -0700244
Tejun Heo3c547862013-01-09 08:05:10 -0800245 /* @blkg failed fully initialized, use the usual release path */
246 blkg_put(blkg);
247 return ERR_PTR(ret);
248
Jens Axboed708f0d2017-03-29 11:25:48 -0600249err_put_congested:
250 wb_congested_put(wb_congested);
251err_put_css:
Tejun Heo496fb782012-04-19 16:29:23 -0700252 css_put(&blkcg->css);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800253err_free_blkg:
Jens Axboed708f0d2017-03-29 11:25:48 -0600254 blkg_free(new_blkg);
Tejun Heo93e6d5d2013-01-09 08:05:10 -0800255 return ERR_PTR(ret);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500256}
Tejun Heo3c96cb32012-04-13 13:11:34 -0700257
Tejun Heo86cde6b2013-01-09 08:05:10 -0800258/**
Jens Axboed708f0d2017-03-29 11:25:48 -0600259 * blkg_lookup_create - lookup blkg, try to create one if not there
Tejun Heo86cde6b2013-01-09 08:05:10 -0800260 * @blkcg: blkcg of interest
261 * @q: request_queue of interest
262 *
263 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
Tejun Heo3c547862013-01-09 08:05:10 -0800264 * create one. blkg creation is performed recursively from blkcg_root such
265 * that all non-root blkg's have access to the parent blkg. This function
266 * should be called under RCU read lock and @q->queue_lock.
Tejun Heo86cde6b2013-01-09 08:05:10 -0800267 *
268 * Returns pointer to the looked up or created blkg on success, ERR_PTR()
269 * value on error. If @q is dead, returns ERR_PTR(-EINVAL). If @q is not
270 * dead and bypassing, returns ERR_PTR(-EBUSY).
271 */
Jens Axboed708f0d2017-03-29 11:25:48 -0600272struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
273 struct request_queue *q)
Tejun Heo3c96cb32012-04-13 13:11:34 -0700274{
Tejun Heo86cde6b2013-01-09 08:05:10 -0800275 struct blkcg_gq *blkg;
276
277 WARN_ON_ONCE(!rcu_read_lock_held());
278 lockdep_assert_held(q->queue_lock);
279
Jens Axboed708f0d2017-03-29 11:25:48 -0600280 /*
281 * This could be the first entry point of blkcg implementation and
282 * we shouldn't allow anything to go through for a bypassing queue.
283 */
284 if (unlikely(blk_queue_bypass(q)))
285 return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
286
Tejun Heo86cde6b2013-01-09 08:05:10 -0800287 blkg = __blkg_lookup(blkcg, q, true);
288 if (blkg)
289 return blkg;
290
Tejun Heo3c547862013-01-09 08:05:10 -0800291 /*
292 * Create blkgs walking down from blkcg_root to @blkcg, so that all
293 * non-root blkgs have access to their parents.
294 */
295 while (true) {
296 struct blkcg *pos = blkcg;
297 struct blkcg *parent = blkcg_parent(blkcg);
298
299 while (parent && !__blkg_lookup(parent, q, false)) {
300 pos = parent;
301 parent = blkcg_parent(parent);
302 }
303
Jens Axboed708f0d2017-03-29 11:25:48 -0600304 blkg = blkg_create(pos, q, NULL);
Tejun Heo3c547862013-01-09 08:05:10 -0800305 if (pos == blkcg || IS_ERR(blkg))
306 return blkg;
307 }
Tejun Heo3c96cb32012-04-13 13:11:34 -0700308}
Vivek Goyal31e4c282009-12-03 12:59:42 -0500309
Tejun Heo3c798392012-04-16 13:57:25 -0700310static void blkg_destroy(struct blkcg_gq *blkg)
Tejun Heo72e06c22012-03-05 13:15:00 -0800311{
Tejun Heo3c798392012-04-16 13:57:25 -0700312 struct blkcg *blkcg = blkg->blkcg;
Tejun Heo77ea7332015-08-18 14:55:24 -0700313 struct blkcg_gq *parent = blkg->parent;
Tejun Heof427d902013-01-09 08:05:12 -0800314 int i;
Tejun Heo03aa2642012-03-05 13:15:19 -0800315
Tejun Heo27e1f9d2012-06-05 13:36:44 +0200316 lockdep_assert_held(blkg->q->queue_lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800317 lockdep_assert_held(&blkcg->lock);
Tejun Heo03aa2642012-03-05 13:15:19 -0800318
319 /* Something wrong if we are trying to remove same group twice */
Tejun Heoe8989fa2012-03-05 13:15:20 -0800320 WARN_ON_ONCE(list_empty(&blkg->q_node));
Tejun Heo9f13ef62012-03-05 13:15:21 -0800321 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
Tejun Heoa6371202012-04-19 16:29:24 -0700322
Tejun Heof427d902013-01-09 08:05:12 -0800323 for (i = 0; i < BLKCG_MAX_POLS; i++) {
324 struct blkcg_policy *pol = blkcg_policy[i];
325
326 if (blkg->pd[i] && pol->pd_offline_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -0700327 pol->pd_offline_fn(blkg->pd[i]);
Tejun Heof427d902013-01-09 08:05:12 -0800328 }
Tejun Heo77ea7332015-08-18 14:55:24 -0700329
330 if (parent) {
331 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
332 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
333 }
334
Tejun Heof427d902013-01-09 08:05:12 -0800335 blkg->online = false;
336
Tejun Heoa6371202012-04-19 16:29:24 -0700337 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
Tejun Heoe8989fa2012-03-05 13:15:20 -0800338 list_del_init(&blkg->q_node);
Tejun Heo9f13ef62012-03-05 13:15:21 -0800339 hlist_del_init_rcu(&blkg->blkcg_node);
Tejun Heo03aa2642012-03-05 13:15:19 -0800340
Tejun Heo03aa2642012-03-05 13:15:19 -0800341 /*
Tejun Heoa6371202012-04-19 16:29:24 -0700342 * Both setting lookup hint to and clearing it from @blkg are done
343 * under queue_lock. If it's not pointing to @blkg now, it never
344 * will. Hint assignment itself can race safely.
345 */
Paul E. McKenneyec6c676a2014-02-17 13:35:57 -0800346 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
Tejun Heoa6371202012-04-19 16:29:24 -0700347 rcu_assign_pointer(blkcg->blkg_hint, NULL);
348
349 /*
Tejun Heo03aa2642012-03-05 13:15:19 -0800350 * Put the reference taken at the time of creation so that when all
351 * queues are gone, group can be destroyed.
352 */
353 blkg_put(blkg);
354}
355
Tejun Heo9f13ef62012-03-05 13:15:21 -0800356/**
357 * blkg_destroy_all - destroy all blkgs associated with a request_queue
358 * @q: request_queue of interest
Tejun Heo9f13ef62012-03-05 13:15:21 -0800359 *
Tejun Heo3c96cb32012-04-13 13:11:34 -0700360 * Destroy all blkgs associated with @q.
Tejun Heo9f13ef62012-03-05 13:15:21 -0800361 */
Tejun Heo3c96cb32012-04-13 13:11:34 -0700362static void blkg_destroy_all(struct request_queue *q)
Tejun Heo03aa2642012-03-05 13:15:19 -0800363{
Tejun Heo3c798392012-04-16 13:57:25 -0700364 struct blkcg_gq *blkg, *n;
Tejun Heo72e06c22012-03-05 13:15:00 -0800365
Tejun Heo6d18b002012-04-13 13:11:35 -0700366 lockdep_assert_held(q->queue_lock);
Tejun Heo72e06c22012-03-05 13:15:00 -0800367
Tejun Heo9f13ef62012-03-05 13:15:21 -0800368 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
Tejun Heo3c798392012-04-16 13:57:25 -0700369 struct blkcg *blkcg = blkg->blkcg;
Tejun Heo72e06c22012-03-05 13:15:00 -0800370
Tejun Heo9f13ef62012-03-05 13:15:21 -0800371 spin_lock(&blkcg->lock);
372 blkg_destroy(blkg);
373 spin_unlock(&blkcg->lock);
Tejun Heo72e06c22012-03-05 13:15:00 -0800374 }
Tejun Heo6fe810b2015-09-05 15:47:36 -0400375
376 q->root_blkg = NULL;
377 q->root_rl.blkg = NULL;
Tejun Heo72e06c22012-03-05 13:15:00 -0800378}
379
Tejun Heo2a4fd072013-05-14 13:52:31 -0700380/*
381 * A group is RCU protected, but having an rcu lock does not mean that one
382 * can access all the fields of blkg and assume these are valid. For
383 * example, don't try to follow throtl_data and request queue links.
384 *
385 * Having a reference to blkg under an rcu allows accesses to only values
386 * local to groups like group stats and group rate limits.
387 */
388void __blkg_release_rcu(struct rcu_head *rcu_head)
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800389{
Tejun Heo2a4fd072013-05-14 13:52:31 -0700390 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
Tejun Heodb613672013-05-14 13:52:31 -0700391
Tejun Heo3c547862013-01-09 08:05:10 -0800392 /* release the blkcg and parent blkg refs this blkg has been holding */
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800393 css_put(&blkg->blkcg->css);
Tejun Heoa5049a82014-06-19 17:42:57 -0400394 if (blkg->parent)
Tejun Heo3c547862013-01-09 08:05:10 -0800395 blkg_put(blkg->parent);
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800396
Tejun Heoce7acfe2015-05-22 17:13:38 -0400397 wb_congested_put(blkg->wb_congested);
398
Tejun Heo2a4fd072013-05-14 13:52:31 -0700399 blkg_free(blkg);
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800400}
Tejun Heo2a4fd072013-05-14 13:52:31 -0700401EXPORT_SYMBOL_GPL(__blkg_release_rcu);
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800402
Tejun Heoa0516612012-06-26 15:05:44 -0700403/*
404 * The next function used by blk_queue_for_each_rl(). It's a bit tricky
405 * because the root blkg uses @q->root_rl instead of its own rl.
406 */
407struct request_list *__blk_queue_next_rl(struct request_list *rl,
408 struct request_queue *q)
409{
410 struct list_head *ent;
411 struct blkcg_gq *blkg;
412
413 /*
414 * Determine the current blkg list_head. The first entry is
415 * root_rl which is off @q->blkg_list and mapped to the head.
416 */
417 if (rl == &q->root_rl) {
418 ent = &q->blkg_list;
Jun'ichi Nomura65c77fd2012-10-22 10:15:37 +0900419 /* There are no more block groups, hence no request lists */
420 if (list_empty(ent))
421 return NULL;
Tejun Heoa0516612012-06-26 15:05:44 -0700422 } else {
423 blkg = container_of(rl, struct blkcg_gq, rl);
424 ent = &blkg->q_node;
425 }
426
427 /* walk to the next list_head, skip root blkcg */
428 ent = ent->next;
429 if (ent == &q->root_blkg->q_node)
430 ent = ent->next;
431 if (ent == &q->blkg_list)
432 return NULL;
433
434 blkg = container_of(ent, struct blkcg_gq, q_node);
435 return &blkg->rl;
436}
437
Tejun Heo182446d2013-08-08 20:11:24 -0400438static int blkcg_reset_stats(struct cgroup_subsys_state *css,
439 struct cftype *cftype, u64 val)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700440{
Tejun Heo182446d2013-08-08 20:11:24 -0400441 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heo3c798392012-04-16 13:57:25 -0700442 struct blkcg_gq *blkg;
Tejun Heobc0d6502012-04-13 13:11:26 -0700443 int i;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700444
Tejun Heo838f13b2015-07-09 16:39:47 -0400445 mutex_lock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700446 spin_lock_irq(&blkcg->lock);
Tejun Heo997a0262012-03-08 10:53:58 -0800447
448 /*
449 * Note that stat reset is racy - it doesn't synchronize against
450 * stat updates. This is a debug feature which shouldn't exist
451 * anyway. If you get hit by a race, retry.
452 */
Sasha Levinb67bfe02013-02-27 17:06:00 -0800453 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heo77ea7332015-08-18 14:55:24 -0700454 blkg_rwstat_reset(&blkg->stat_bytes);
455 blkg_rwstat_reset(&blkg->stat_ios);
456
Tejun Heo8bd435b2012-04-13 13:11:28 -0700457 for (i = 0; i < BLKCG_MAX_POLS; i++) {
Tejun Heo3c798392012-04-16 13:57:25 -0700458 struct blkcg_policy *pol = blkcg_policy[i];
Tejun Heo549d3aa2012-03-05 13:15:16 -0800459
Tejun Heoa9520cd2015-08-18 14:55:14 -0700460 if (blkg->pd[i] && pol->pd_reset_stats_fn)
461 pol->pd_reset_stats_fn(blkg->pd[i]);
Tejun Heobc0d6502012-04-13 13:11:26 -0700462 }
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700463 }
Vivek Goyalf0bdc8c2011-05-19 15:38:30 -0400464
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700465 spin_unlock_irq(&blkcg->lock);
Tejun Heobc0d6502012-04-13 13:11:26 -0700466 mutex_unlock(&blkcg_pol_mutex);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700467 return 0;
468}
469
Tejun Heodd165eb2015-08-18 14:55:33 -0700470const char *blkg_dev_name(struct blkcg_gq *blkg)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700471{
Tejun Heod3d32e62012-04-01 14:38:42 -0700472 /* some drivers (floppy) instantiate a queue w/o disk registered */
Jan Karadc3b17c2017-02-02 15:56:50 +0100473 if (blkg->q->backing_dev_info->dev)
474 return dev_name(blkg->q->backing_dev_info->dev);
Tejun Heod3d32e62012-04-01 14:38:42 -0700475 return NULL;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700476}
Tejun Heodd165eb2015-08-18 14:55:33 -0700477EXPORT_SYMBOL_GPL(blkg_dev_name);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700478
Tejun Heod3d32e62012-04-01 14:38:42 -0700479/**
480 * blkcg_print_blkgs - helper for printing per-blkg data
481 * @sf: seq_file to print to
482 * @blkcg: blkcg of interest
483 * @prfill: fill function to print out a blkg
484 * @pol: policy in question
485 * @data: data to be passed to @prfill
486 * @show_total: to print out sum of prfill return values or not
487 *
488 * This function invokes @prfill on each blkg of @blkcg if pd for the
489 * policy specified by @pol exists. @prfill is invoked with @sf, the
Tejun Heo810ecfa2013-01-09 08:05:13 -0800490 * policy data and @data and the matching queue lock held. If @show_total
491 * is %true, the sum of the return values from @prfill is printed with
492 * "Total" label at the end.
Tejun Heod3d32e62012-04-01 14:38:42 -0700493 *
494 * This is to be used to construct print functions for
495 * cftype->read_seq_string method.
496 */
Tejun Heo3c798392012-04-16 13:57:25 -0700497void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
Tejun Heof95a04a2012-04-16 13:57:26 -0700498 u64 (*prfill)(struct seq_file *,
499 struct blkg_policy_data *, int),
Tejun Heo3c798392012-04-16 13:57:25 -0700500 const struct blkcg_policy *pol, int data,
Tejun Heoec399342012-04-13 13:11:27 -0700501 bool show_total)
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400502{
Tejun Heo3c798392012-04-16 13:57:25 -0700503 struct blkcg_gq *blkg;
Tejun Heod3d32e62012-04-01 14:38:42 -0700504 u64 total = 0;
505
Tejun Heo810ecfa2013-01-09 08:05:13 -0800506 rcu_read_lock();
Linus Torvaldsee89f812013-02-28 12:52:24 -0800507 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heo810ecfa2013-01-09 08:05:13 -0800508 spin_lock_irq(blkg->q->queue_lock);
Tejun Heoa2b16932012-04-13 13:11:33 -0700509 if (blkcg_policy_enabled(blkg->q, pol))
Tejun Heof95a04a2012-04-16 13:57:26 -0700510 total += prfill(sf, blkg->pd[pol->plid], data);
Tejun Heo810ecfa2013-01-09 08:05:13 -0800511 spin_unlock_irq(blkg->q->queue_lock);
512 }
513 rcu_read_unlock();
Tejun Heod3d32e62012-04-01 14:38:42 -0700514
515 if (show_total)
516 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
517}
Tejun Heo829fdb52012-04-01 14:38:43 -0700518EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
Tejun Heod3d32e62012-04-01 14:38:42 -0700519
520/**
521 * __blkg_prfill_u64 - prfill helper for a single u64 value
522 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700523 * @pd: policy private data of interest
Tejun Heod3d32e62012-04-01 14:38:42 -0700524 * @v: value to print
525 *
Tejun Heof95a04a2012-04-16 13:57:26 -0700526 * Print @v to @sf for the device assocaited with @pd.
Tejun Heod3d32e62012-04-01 14:38:42 -0700527 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700528u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
Tejun Heod3d32e62012-04-01 14:38:42 -0700529{
Tejun Heof95a04a2012-04-16 13:57:26 -0700530 const char *dname = blkg_dev_name(pd->blkg);
Tejun Heod3d32e62012-04-01 14:38:42 -0700531
532 if (!dname)
533 return 0;
534
535 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
536 return v;
537}
Tejun Heo829fdb52012-04-01 14:38:43 -0700538EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
Tejun Heod3d32e62012-04-01 14:38:42 -0700539
540/**
541 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
542 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700543 * @pd: policy private data of interest
Tejun Heod3d32e62012-04-01 14:38:42 -0700544 * @rwstat: rwstat to print
545 *
Tejun Heof95a04a2012-04-16 13:57:26 -0700546 * Print @rwstat to @sf for the device assocaited with @pd.
Tejun Heod3d32e62012-04-01 14:38:42 -0700547 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700548u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
Tejun Heo829fdb52012-04-01 14:38:43 -0700549 const struct blkg_rwstat *rwstat)
Tejun Heod3d32e62012-04-01 14:38:42 -0700550{
551 static const char *rwstr[] = {
552 [BLKG_RWSTAT_READ] = "Read",
553 [BLKG_RWSTAT_WRITE] = "Write",
554 [BLKG_RWSTAT_SYNC] = "Sync",
555 [BLKG_RWSTAT_ASYNC] = "Async",
556 };
Tejun Heof95a04a2012-04-16 13:57:26 -0700557 const char *dname = blkg_dev_name(pd->blkg);
Tejun Heod3d32e62012-04-01 14:38:42 -0700558 u64 v;
559 int i;
560
561 if (!dname)
562 return 0;
563
564 for (i = 0; i < BLKG_RWSTAT_NR; i++)
565 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
Tejun Heo24bdb8e2015-08-18 14:55:22 -0700566 (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
Tejun Heod3d32e62012-04-01 14:38:42 -0700567
Tejun Heo24bdb8e2015-08-18 14:55:22 -0700568 v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
569 atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
Tejun Heod3d32e62012-04-01 14:38:42 -0700570 seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
571 return v;
572}
Tejun Heob50da392013-01-09 08:05:12 -0800573EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700574
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700575/**
576 * blkg_prfill_stat - prfill callback for blkg_stat
577 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700578 * @pd: policy private data of interest
579 * @off: offset to the blkg_stat in @pd
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700580 *
581 * prfill callback for printing a blkg_stat.
582 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700583u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
Tejun Heod3d32e62012-04-01 14:38:42 -0700584{
Tejun Heof95a04a2012-04-16 13:57:26 -0700585 return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
Tejun Heod3d32e62012-04-01 14:38:42 -0700586}
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700587EXPORT_SYMBOL_GPL(blkg_prfill_stat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700588
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700589/**
590 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
591 * @sf: seq_file to print to
Tejun Heof95a04a2012-04-16 13:57:26 -0700592 * @pd: policy private data of interest
593 * @off: offset to the blkg_rwstat in @pd
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700594 *
595 * prfill callback for printing a blkg_rwstat.
596 */
Tejun Heof95a04a2012-04-16 13:57:26 -0700597u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
598 int off)
Tejun Heod3d32e62012-04-01 14:38:42 -0700599{
Tejun Heof95a04a2012-04-16 13:57:26 -0700600 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
Tejun Heod3d32e62012-04-01 14:38:42 -0700601
Tejun Heof95a04a2012-04-16 13:57:26 -0700602 return __blkg_prfill_rwstat(sf, pd, &rwstat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700603}
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700604EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
Tejun Heod3d32e62012-04-01 14:38:42 -0700605
Tejun Heo77ea7332015-08-18 14:55:24 -0700606static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
607 struct blkg_policy_data *pd, int off)
608{
609 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);
610
611 return __blkg_prfill_rwstat(sf, pd, &rwstat);
612}
613
614/**
615 * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
616 * @sf: seq_file to print to
617 * @v: unused
618 *
619 * To be used as cftype->seq_show to print blkg->stat_bytes.
620 * cftype->private must be set to the blkcg_policy.
621 */
622int blkg_print_stat_bytes(struct seq_file *sf, void *v)
623{
624 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
625 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
626 offsetof(struct blkcg_gq, stat_bytes), true);
627 return 0;
628}
629EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);
630
631/**
632 * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
633 * @sf: seq_file to print to
634 * @v: unused
635 *
636 * To be used as cftype->seq_show to print blkg->stat_ios. cftype->private
637 * must be set to the blkcg_policy.
638 */
639int blkg_print_stat_ios(struct seq_file *sf, void *v)
640{
641 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
642 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
643 offsetof(struct blkcg_gq, stat_ios), true);
644 return 0;
645}
646EXPORT_SYMBOL_GPL(blkg_print_stat_ios);
647
648static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
649 struct blkg_policy_data *pd,
650 int off)
651{
652 struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
653 NULL, off);
654 return __blkg_prfill_rwstat(sf, pd, &rwstat);
655}
656
657/**
658 * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
659 * @sf: seq_file to print to
660 * @v: unused
661 */
662int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
663{
664 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
665 blkg_prfill_rwstat_field_recursive,
666 (void *)seq_cft(sf)->private,
667 offsetof(struct blkcg_gq, stat_bytes), true);
668 return 0;
669}
670EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);
671
672/**
673 * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
674 * @sf: seq_file to print to
675 * @v: unused
676 */
677int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
678{
679 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
680 blkg_prfill_rwstat_field_recursive,
681 (void *)seq_cft(sf)->private,
682 offsetof(struct blkcg_gq, stat_ios), true);
683 return 0;
684}
685EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
686
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700687/**
Tejun Heo16b3de62013-01-09 08:05:12 -0800688 * blkg_stat_recursive_sum - collect hierarchical blkg_stat
Tejun Heof12c74c2015-08-18 14:55:23 -0700689 * @blkg: blkg of interest
690 * @pol: blkcg_policy which contains the blkg_stat
691 * @off: offset to the blkg_stat in blkg_policy_data or @blkg
Tejun Heo16b3de62013-01-09 08:05:12 -0800692 *
Tejun Heof12c74c2015-08-18 14:55:23 -0700693 * Collect the blkg_stat specified by @blkg, @pol and @off and all its
694 * online descendants and their aux counts. The caller must be holding the
695 * queue lock for online tests.
696 *
697 * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
698 * at @off bytes into @blkg's blkg_policy_data of the policy.
Tejun Heo16b3de62013-01-09 08:05:12 -0800699 */
Tejun Heof12c74c2015-08-18 14:55:23 -0700700u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
701 struct blkcg_policy *pol, int off)
Tejun Heo16b3de62013-01-09 08:05:12 -0800702{
Tejun Heo16b3de62013-01-09 08:05:12 -0800703 struct blkcg_gq *pos_blkg;
Tejun Heo492eb212013-08-08 20:11:25 -0400704 struct cgroup_subsys_state *pos_css;
Tejun Heobd8815a2013-08-08 20:11:27 -0400705 u64 sum = 0;
Tejun Heo16b3de62013-01-09 08:05:12 -0800706
Tejun Heof12c74c2015-08-18 14:55:23 -0700707 lockdep_assert_held(blkg->q->queue_lock);
Tejun Heo16b3de62013-01-09 08:05:12 -0800708
Tejun Heo16b3de62013-01-09 08:05:12 -0800709 rcu_read_lock();
Tejun Heof12c74c2015-08-18 14:55:23 -0700710 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
711 struct blkg_stat *stat;
Tejun Heo16b3de62013-01-09 08:05:12 -0800712
Tejun Heof12c74c2015-08-18 14:55:23 -0700713 if (!pos_blkg->online)
714 continue;
715
716 if (pol)
717 stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
718 else
719 stat = (void *)blkg + off;
720
721 sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
Tejun Heo16b3de62013-01-09 08:05:12 -0800722 }
723 rcu_read_unlock();
724
725 return sum;
726}
727EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
728
729/**
730 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
Tejun Heof12c74c2015-08-18 14:55:23 -0700731 * @blkg: blkg of interest
732 * @pol: blkcg_policy which contains the blkg_rwstat
733 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
Tejun Heo16b3de62013-01-09 08:05:12 -0800734 *
Tejun Heof12c74c2015-08-18 14:55:23 -0700735 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
736 * online descendants and their aux counts. The caller must be holding the
737 * queue lock for online tests.
738 *
739 * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
740 * is at @off bytes into @blkg's blkg_policy_data of the policy.
Tejun Heo16b3de62013-01-09 08:05:12 -0800741 */
Tejun Heof12c74c2015-08-18 14:55:23 -0700742struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
743 struct blkcg_policy *pol, int off)
Tejun Heo16b3de62013-01-09 08:05:12 -0800744{
Tejun Heo16b3de62013-01-09 08:05:12 -0800745 struct blkcg_gq *pos_blkg;
Tejun Heo492eb212013-08-08 20:11:25 -0400746 struct cgroup_subsys_state *pos_css;
Tejun Heobd8815a2013-08-08 20:11:27 -0400747 struct blkg_rwstat sum = { };
Tejun Heo16b3de62013-01-09 08:05:12 -0800748 int i;
749
Tejun Heof12c74c2015-08-18 14:55:23 -0700750 lockdep_assert_held(blkg->q->queue_lock);
Tejun Heo16b3de62013-01-09 08:05:12 -0800751
Tejun Heo16b3de62013-01-09 08:05:12 -0800752 rcu_read_lock();
Tejun Heof12c74c2015-08-18 14:55:23 -0700753 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
Tejun Heo3a7faea2015-08-18 14:55:26 -0700754 struct blkg_rwstat *rwstat;
Tejun Heo16b3de62013-01-09 08:05:12 -0800755
756 if (!pos_blkg->online)
757 continue;
758
Tejun Heof12c74c2015-08-18 14:55:23 -0700759 if (pol)
760 rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
761 else
762 rwstat = (void *)pos_blkg + off;
763
Tejun Heo16b3de62013-01-09 08:05:12 -0800764 for (i = 0; i < BLKG_RWSTAT_NR; i++)
Tejun Heo3a7faea2015-08-18 14:55:26 -0700765 atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
766 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
767 &sum.aux_cnt[i]);
Tejun Heo16b3de62013-01-09 08:05:12 -0800768 }
769 rcu_read_unlock();
770
771 return sum;
772}
773EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
774
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600775/* Performs queue bypass and policy enabled checks then looks up blkg. */
776static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
777 const struct blkcg_policy *pol,
778 struct request_queue *q)
779{
780 WARN_ON_ONCE(!rcu_read_lock_held());
781 lockdep_assert_held(q->queue_lock);
782
783 if (!blkcg_policy_enabled(q, pol))
784 return ERR_PTR(-EOPNOTSUPP);
785
786 /*
787 * This could be the first entry point of blkcg implementation and
788 * we shouldn't allow anything to go through for a bypassing queue.
789 */
790 if (unlikely(blk_queue_bypass(q)))
791 return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
792
793 return __blkg_lookup(blkcg, q, true /* update_hint */);
794}
795
Tejun Heo16b3de62013-01-09 08:05:12 -0800796/**
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700797 * blkg_conf_prep - parse and prepare for per-blkg config update
798 * @blkcg: target block cgroup
Tejun Heoda8b0662012-04-13 13:11:29 -0700799 * @pol: target policy
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700800 * @input: input string
801 * @ctx: blkg_conf_ctx to be filled
802 *
803 * Parse per-blkg config update from @input and initialize @ctx with the
Tejun Heo36aa9e52015-08-18 14:55:31 -0700804 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
805 * part of @input following MAJ:MIN. This function returns with RCU read
806 * lock and queue lock held and must be paired with blkg_conf_finish().
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700807 */
Tejun Heo3c798392012-04-16 13:57:25 -0700808int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
Tejun Heo36aa9e52015-08-18 14:55:31 -0700809 char *input, struct blkg_conf_ctx *ctx)
Tejun Heoda8b0662012-04-13 13:11:29 -0700810 __acquires(rcu) __acquires(disk->queue->queue_lock)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800811{
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700812 struct gendisk *disk;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600813 struct request_queue *q;
Tejun Heo3c798392012-04-16 13:57:25 -0700814 struct blkcg_gq *blkg;
Roman Pen39a169b2016-02-09 12:33:35 -0700815 struct module *owner;
Tejun Heo726fa6942012-04-01 14:38:43 -0700816 unsigned int major, minor;
Tejun Heo36aa9e52015-08-18 14:55:31 -0700817 int key_len, part, ret;
818 char *body;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800819
Tejun Heo36aa9e52015-08-18 14:55:31 -0700820 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
Tejun Heo726fa6942012-04-01 14:38:43 -0700821 return -EINVAL;
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700822
Tejun Heo36aa9e52015-08-18 14:55:31 -0700823 body = input + key_len;
824 if (!isspace(*body))
825 return -EINVAL;
826 body = skip_spaces(body);
827
Tejun Heo726fa6942012-04-01 14:38:43 -0700828 disk = get_gendisk(MKDEV(major, minor), &part);
Tejun Heo5f6c2d22015-07-22 18:05:53 -0400829 if (!disk)
Tejun Heo20386ce2015-08-18 14:55:28 -0700830 return -ENODEV;
Tejun Heo5f6c2d22015-07-22 18:05:53 -0400831 if (part) {
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600832 ret = -ENODEV;
833 goto fail;
Tejun Heo5f6c2d22015-07-22 18:05:53 -0400834 }
Tejun Heoe56da7e2012-03-05 13:15:07 -0800835
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600836 q = disk->queue;
837
Tejun Heoe56da7e2012-03-05 13:15:07 -0800838 rcu_read_lock();
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600839 spin_lock_irq(q->queue_lock);
Tejun Heoda8b0662012-04-13 13:11:29 -0700840
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600841 blkg = blkg_lookup_check(blkcg, pol, q);
Tejun Heo4bfd4822012-03-05 13:15:08 -0800842 if (IS_ERR(blkg)) {
843 ret = PTR_ERR(blkg);
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600844 goto fail_unlock;
Vivek Goyal062a6442010-09-15 17:06:33 -0400845 }
Tejun Heoe56da7e2012-03-05 13:15:07 -0800846
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600847 if (blkg)
848 goto success;
849
850 /*
851 * Create blkgs walking down from blkcg_root to @blkcg, so that all
852 * non-root blkgs have access to their parents.
853 */
854 while (true) {
855 struct blkcg *pos = blkcg;
856 struct blkcg *parent;
857 struct blkcg_gq *new_blkg;
858
859 parent = blkcg_parent(blkcg);
860 while (parent && !__blkg_lookup(parent, q, false)) {
861 pos = parent;
862 parent = blkcg_parent(parent);
863 }
864
865 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
866 spin_unlock_irq(q->queue_lock);
867 rcu_read_unlock();
868
869 new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
870 if (unlikely(!new_blkg)) {
871 ret = -ENOMEM;
872 goto fail;
873 }
874
875 rcu_read_lock();
876 spin_lock_irq(q->queue_lock);
877
878 blkg = blkg_lookup_check(pos, pol, q);
879 if (IS_ERR(blkg)) {
880 ret = PTR_ERR(blkg);
881 goto fail_unlock;
882 }
883
884 if (blkg) {
885 blkg_free(new_blkg);
886 } else {
887 blkg = blkg_create(pos, q, new_blkg);
888 if (unlikely(IS_ERR(blkg))) {
889 ret = PTR_ERR(blkg);
890 goto fail_unlock;
891 }
892 }
893
894 if (pos == blkcg)
895 goto success;
896 }
897success:
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700898 ctx->disk = disk;
899 ctx->blkg = blkg;
Tejun Heo36aa9e52015-08-18 14:55:31 -0700900 ctx->body = body;
Tejun Heo726fa6942012-04-01 14:38:43 -0700901 return 0;
Tahsin Erdogan457e490f2017-03-29 11:27:19 -0600902
903fail_unlock:
904 spin_unlock_irq(q->queue_lock);
905 rcu_read_unlock();
906fail:
907 owner = disk->fops->owner;
908 put_disk(disk);
909 module_put(owner);
910 /*
911 * If queue was bypassing, we should retry. Do so after a
912 * short msleep(). It isn't strictly necessary but queue
913 * can be bypassing for some time and it's always nice to
914 * avoid busy looping.
915 */
916 if (ret == -EBUSY) {
917 msleep(10);
918 ret = restart_syscall();
919 }
920 return ret;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800921}
Tejun Heo829fdb52012-04-01 14:38:43 -0700922EXPORT_SYMBOL_GPL(blkg_conf_prep);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800923
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700924/**
925 * blkg_conf_finish - finish up per-blkg config update
926 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
927 *
928 * Finish up after per-blkg config update. This function must be paired
929 * with blkg_conf_prep().
930 */
Tejun Heo829fdb52012-04-01 14:38:43 -0700931void blkg_conf_finish(struct blkg_conf_ctx *ctx)
Tejun Heoda8b0662012-04-13 13:11:29 -0700932 __releases(ctx->disk->queue->queue_lock) __releases(rcu)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800933{
Roman Pen39a169b2016-02-09 12:33:35 -0700934 struct module *owner;
935
Tejun Heoda8b0662012-04-13 13:11:29 -0700936 spin_unlock_irq(ctx->disk->queue->queue_lock);
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700937 rcu_read_unlock();
Roman Pen39a169b2016-02-09 12:33:35 -0700938 owner = ctx->disk->fops->owner;
Tejun Heo3a8b31d2012-04-01 14:38:43 -0700939 put_disk(ctx->disk);
Roman Pen39a169b2016-02-09 12:33:35 -0700940 module_put(owner);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800941}
Tejun Heo829fdb52012-04-01 14:38:43 -0700942EXPORT_SYMBOL_GPL(blkg_conf_finish);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800943
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700944static int blkcg_print_stat(struct seq_file *sf, void *v)
945{
946 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
947 struct blkcg_gq *blkg;
948
949 rcu_read_lock();
950
951 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
952 const char *dname;
953 struct blkg_rwstat rwstat;
954 u64 rbytes, wbytes, rios, wios;
955
956 dname = blkg_dev_name(blkg);
957 if (!dname)
958 continue;
959
960 spin_lock_irq(blkg->q->queue_lock);
961
962 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
963 offsetof(struct blkcg_gq, stat_bytes));
964 rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
965 wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
966
967 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
968 offsetof(struct blkcg_gq, stat_ios));
969 rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
970 wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
971
972 spin_unlock_irq(blkg->q->queue_lock);
973
974 if (rbytes || wbytes || rios || wios)
975 seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n",
976 dname, rbytes, wbytes, rios, wios);
977 }
978
979 rcu_read_unlock();
980 return 0;
981}
982
Bart Van Asschee1f3b942016-06-14 17:04:32 +0200983static struct cftype blkcg_files[] = {
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700984 {
985 .name = "stat",
Tejun Heoca0752c2015-10-22 09:48:37 +0900986 .flags = CFTYPE_NOT_ON_ROOT,
Tejun Heo2ee867dc2015-08-18 14:55:34 -0700987 .seq_show = blkcg_print_stat,
988 },
989 { } /* terminate */
990};
991
Bart Van Asschee1f3b942016-06-14 17:04:32 +0200992static struct cftype blkcg_legacy_files[] = {
Vivek Goyal31e4c282009-12-03 12:59:42 -0500993 {
Divyesh Shah84c124d2010-04-09 08:31:19 +0200994 .name = "reset_stats",
Tejun Heo3c798392012-04-16 13:57:25 -0700995 .write_u64 = blkcg_reset_stats,
Vivek Goyal22084192009-12-03 12:59:49 -0500996 },
Tejun Heo4baf6e32012-04-01 12:09:55 -0700997 { } /* terminate */
Vivek Goyal31e4c282009-12-03 12:59:42 -0500998};
999
Tejun Heo9f13ef62012-03-05 13:15:21 -08001000/**
Tejun Heo92fb9742012-11-19 08:13:38 -08001001 * blkcg_css_offline - cgroup css_offline callback
Tejun Heoeb954192013-08-08 20:11:23 -04001002 * @css: css of interest
Tejun Heo9f13ef62012-03-05 13:15:21 -08001003 *
Tejun Heoeb954192013-08-08 20:11:23 -04001004 * This function is called when @css is about to go away and responsible
1005 * for shooting down all blkgs associated with @css. blkgs should be
Tejun Heo9f13ef62012-03-05 13:15:21 -08001006 * removed while holding both q and blkcg locks. As blkcg lock is nested
1007 * inside q lock, this function performs reverse double lock dancing.
1008 *
1009 * This is the blkcg counterpart of ioc_release_fn().
1010 */
Tejun Heoeb954192013-08-08 20:11:23 -04001011static void blkcg_css_offline(struct cgroup_subsys_state *css)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001012{
Tejun Heoeb954192013-08-08 20:11:23 -04001013 struct blkcg *blkcg = css_to_blkcg(css);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001014
Tejun Heo9f13ef62012-03-05 13:15:21 -08001015 spin_lock_irq(&blkcg->lock);
Tejun Heo7ee9c562012-03-05 13:15:11 -08001016
Tejun Heo9f13ef62012-03-05 13:15:21 -08001017 while (!hlist_empty(&blkcg->blkg_list)) {
Tejun Heo3c798392012-04-16 13:57:25 -07001018 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1019 struct blkcg_gq, blkcg_node);
Tejun Heoc875f4d2012-03-05 13:15:22 -08001020 struct request_queue *q = blkg->q;
Vivek Goyalb1c35762009-12-03 12:59:47 -05001021
Tejun Heo9f13ef62012-03-05 13:15:21 -08001022 if (spin_trylock(q->queue_lock)) {
1023 blkg_destroy(blkg);
1024 spin_unlock(q->queue_lock);
1025 } else {
1026 spin_unlock_irq(&blkcg->lock);
Tejun Heo9f13ef62012-03-05 13:15:21 -08001027 cpu_relax();
Dan Carpentera5567932012-03-29 20:57:08 +02001028 spin_lock_irq(&blkcg->lock);
Jens Axboe0f3942a2010-05-03 14:28:55 +02001029 }
Tejun Heo9f13ef62012-03-05 13:15:21 -08001030 }
Jens Axboe0f3942a2010-05-03 14:28:55 +02001031
Tejun Heo9f13ef62012-03-05 13:15:21 -08001032 spin_unlock_irq(&blkcg->lock);
Tejun Heo52ebea72015-05-22 17:13:37 -04001033
1034 wb_blkcg_offline(blkcg);
Tejun Heo7ee9c562012-03-05 13:15:11 -08001035}
1036
Tejun Heoeb954192013-08-08 20:11:23 -04001037static void blkcg_css_free(struct cgroup_subsys_state *css)
Tejun Heo7ee9c562012-03-05 13:15:11 -08001038{
Tejun Heoeb954192013-08-08 20:11:23 -04001039 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heobc915e62015-08-18 14:55:08 -07001040 int i;
Tejun Heo7ee9c562012-03-05 13:15:11 -08001041
Tejun Heo7876f932015-07-09 16:39:49 -04001042 mutex_lock(&blkcg_pol_mutex);
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001043
Tejun Heo7876f932015-07-09 16:39:49 -04001044 list_del(&blkcg->all_blkcgs_node);
Tejun Heo7876f932015-07-09 16:39:49 -04001045
Tejun Heobc915e62015-08-18 14:55:08 -07001046 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001047 if (blkcg->cpd[i])
1048 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1049
1050 mutex_unlock(&blkcg_pol_mutex);
1051
Tejun Heobc915e62015-08-18 14:55:08 -07001052 kfree(blkcg);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001053}
1054
Tejun Heoeb954192013-08-08 20:11:23 -04001055static struct cgroup_subsys_state *
1056blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001057{
Tejun Heo3c798392012-04-16 13:57:25 -07001058 struct blkcg *blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001059 struct cgroup_subsys_state *ret;
1060 int i;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001061
Tejun Heo7876f932015-07-09 16:39:49 -04001062 mutex_lock(&blkcg_pol_mutex);
1063
Tejun Heoeb954192013-08-08 20:11:23 -04001064 if (!parent_css) {
Tejun Heo3c798392012-04-16 13:57:25 -07001065 blkcg = &blkcg_root;
Tejun Heobc915e62015-08-18 14:55:08 -07001066 } else {
1067 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1068 if (!blkcg) {
1069 ret = ERR_PTR(-ENOMEM);
weiping zhang4c18c9e2017-08-25 23:49:32 +08001070 goto unlock;
Tejun Heobc915e62015-08-18 14:55:08 -07001071 }
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001072 }
Vivek Goyal31e4c282009-12-03 12:59:42 -05001073
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001074 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1075 struct blkcg_policy *pol = blkcg_policy[i];
1076 struct blkcg_policy_data *cpd;
1077
1078 /*
1079 * If the policy hasn't been attached yet, wait for it
1080 * to be attached before doing anything else. Otherwise,
1081 * check if the policy requires any specific per-cgroup
1082 * data: if it does, allocate and initialize it.
1083 */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001084 if (!pol || !pol->cpd_alloc_fn)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001085 continue;
1086
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001087 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001088 if (!cpd) {
1089 ret = ERR_PTR(-ENOMEM);
1090 goto free_pd_blkcg;
1091 }
Tejun Heo81437642015-08-18 14:55:15 -07001092 blkcg->cpd[i] = cpd;
1093 cpd->blkcg = blkcg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001094 cpd->plid = i;
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001095 if (pol->cpd_init_fn)
1096 pol->cpd_init_fn(cpd);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001097 }
1098
Vivek Goyal31e4c282009-12-03 12:59:42 -05001099 spin_lock_init(&blkcg->lock);
Tejun Heoe00f4f42016-11-21 18:03:32 -05001100 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001101 INIT_HLIST_HEAD(&blkcg->blkg_list);
Tejun Heo52ebea72015-05-22 17:13:37 -04001102#ifdef CONFIG_CGROUP_WRITEBACK
1103 INIT_LIST_HEAD(&blkcg->cgwb_list);
1104#endif
Tejun Heo7876f932015-07-09 16:39:49 -04001105 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1106
1107 mutex_unlock(&blkcg_pol_mutex);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001108 return &blkcg->css;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001109
1110free_pd_blkcg:
1111 for (i--; i >= 0; i--)
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001112 if (blkcg->cpd[i])
1113 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
weiping zhang4c18c9e2017-08-25 23:49:32 +08001114
1115 if (blkcg != &blkcg_root)
1116 kfree(blkcg);
1117unlock:
Tejun Heo7876f932015-07-09 16:39:49 -04001118 mutex_unlock(&blkcg_pol_mutex);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001119 return ret;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001120}
1121
Tejun Heo5efd6112012-03-05 13:15:12 -08001122/**
1123 * blkcg_init_queue - initialize blkcg part of request queue
1124 * @q: request_queue to initialize
1125 *
1126 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
1127 * part of new request_queue @q.
1128 *
1129 * RETURNS:
1130 * 0 on success, -errno on failure.
1131 */
1132int blkcg_init_queue(struct request_queue *q)
1133{
Jens Axboed708f0d2017-03-29 11:25:48 -06001134 struct blkcg_gq *new_blkg, *blkg;
1135 bool preloaded;
Tejun Heoec13b1d2015-05-22 17:13:19 -04001136 int ret;
Tejun Heo5efd6112012-03-05 13:15:12 -08001137
Jens Axboed708f0d2017-03-29 11:25:48 -06001138 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1139 if (!new_blkg)
1140 return -ENOMEM;
1141
1142 preloaded = !radix_tree_preload(GFP_KERNEL);
1143
1144 /*
1145 * Make sure the root blkg exists and count the existing blkgs. As
1146 * @q is bypassing at this point, blkg_lookup_create() can't be
1147 * used. Open code insertion.
1148 */
Tejun Heoec13b1d2015-05-22 17:13:19 -04001149 rcu_read_lock();
1150 spin_lock_irq(q->queue_lock);
Jens Axboed708f0d2017-03-29 11:25:48 -06001151 blkg = blkg_create(&blkcg_root, q, new_blkg);
Tejun Heoec13b1d2015-05-22 17:13:19 -04001152 spin_unlock_irq(q->queue_lock);
1153 rcu_read_unlock();
1154
Jens Axboed708f0d2017-03-29 11:25:48 -06001155 if (preloaded)
1156 radix_tree_preload_end();
1157
Hou Tao9b54d812017-02-03 17:19:07 +08001158 if (IS_ERR(blkg))
Tejun Heoec13b1d2015-05-22 17:13:19 -04001159 return PTR_ERR(blkg);
Tejun Heoec13b1d2015-05-22 17:13:19 -04001160
1161 q->root_blkg = blkg;
1162 q->root_rl.blkg = blkg;
1163
1164 ret = blk_throtl_init(q);
1165 if (ret) {
1166 spin_lock_irq(q->queue_lock);
1167 blkg_destroy_all(q);
1168 spin_unlock_irq(q->queue_lock);
1169 }
1170 return ret;
Tejun Heo5efd6112012-03-05 13:15:12 -08001171}
1172
1173/**
1174 * blkcg_drain_queue - drain blkcg part of request_queue
1175 * @q: request_queue to drain
1176 *
1177 * Called from blk_drain_queue(). Responsible for draining blkcg part.
1178 */
1179void blkcg_drain_queue(struct request_queue *q)
1180{
1181 lockdep_assert_held(q->queue_lock);
1182
Tejun Heo0b462c82014-07-05 18:43:21 -04001183 /*
1184 * @q could be exiting and already have destroyed all blkgs as
1185 * indicated by NULL root_blkg. If so, don't confuse policies.
1186 */
1187 if (!q->root_blkg)
1188 return;
1189
Tejun Heo5efd6112012-03-05 13:15:12 -08001190 blk_throtl_drain(q);
1191}
1192
1193/**
1194 * blkcg_exit_queue - exit and release blkcg part of request_queue
1195 * @q: request_queue being released
1196 *
1197 * Called from blk_release_queue(). Responsible for exiting blkcg part.
1198 */
1199void blkcg_exit_queue(struct request_queue *q)
1200{
Tejun Heo6d18b002012-04-13 13:11:35 -07001201 spin_lock_irq(q->queue_lock);
Tejun Heo3c96cb32012-04-13 13:11:34 -07001202 blkg_destroy_all(q);
Tejun Heo6d18b002012-04-13 13:11:35 -07001203 spin_unlock_irq(q->queue_lock);
1204
Tejun Heo5efd6112012-03-05 13:15:12 -08001205 blk_throtl_exit(q);
1206}
1207
Vivek Goyal31e4c282009-12-03 12:59:42 -05001208/*
1209 * We cannot support shared io contexts, as we have no mean to support
1210 * two tasks with the same ioc in two different groups without major rework
1211 * of the main cic data structures. For now we allow a task to change
1212 * its cgroup only if it's the only owner of its ioc.
1213 */
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05001214static int blkcg_can_attach(struct cgroup_taskset *tset)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001215{
Tejun Heobb9d97b2011-12-12 18:12:21 -08001216 struct task_struct *task;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05001217 struct cgroup_subsys_state *dst_css;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001218 struct io_context *ioc;
1219 int ret = 0;
1220
1221 /* task_lock() is needed to avoid races with exit_io_context() */
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05001222 cgroup_taskset_for_each(task, dst_css, tset) {
Tejun Heobb9d97b2011-12-12 18:12:21 -08001223 task_lock(task);
1224 ioc = task->io_context;
1225 if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1226 ret = -EINVAL;
1227 task_unlock(task);
1228 if (ret)
1229 break;
1230 }
Vivek Goyal31e4c282009-12-03 12:59:42 -05001231 return ret;
1232}
1233
Tejun Heo69d7fde2015-08-18 14:55:36 -07001234static void blkcg_bind(struct cgroup_subsys_state *root_css)
1235{
1236 int i;
1237
1238 mutex_lock(&blkcg_pol_mutex);
1239
1240 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1241 struct blkcg_policy *pol = blkcg_policy[i];
1242 struct blkcg *blkcg;
1243
1244 if (!pol || !pol->cpd_bind_fn)
1245 continue;
1246
1247 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1248 if (blkcg->cpd[pol->plid])
1249 pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1250 }
1251 mutex_unlock(&blkcg_pol_mutex);
1252}
1253
Tejun Heoc165b3e2015-08-18 14:55:29 -07001254struct cgroup_subsys io_cgrp_subsys = {
Tejun Heo92fb9742012-11-19 08:13:38 -08001255 .css_alloc = blkcg_css_alloc,
1256 .css_offline = blkcg_css_offline,
1257 .css_free = blkcg_css_free,
Tejun Heo3c798392012-04-16 13:57:25 -07001258 .can_attach = blkcg_can_attach,
Tejun Heo69d7fde2015-08-18 14:55:36 -07001259 .bind = blkcg_bind,
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001260 .dfl_cftypes = blkcg_files,
Tejun Heo880f50e22015-08-18 14:55:30 -07001261 .legacy_cftypes = blkcg_legacy_files,
Tejun Heoc165b3e2015-08-18 14:55:29 -07001262 .legacy_name = "blkio",
Tejun Heo1ced953b2014-07-08 18:02:57 -04001263#ifdef CONFIG_MEMCG
1264 /*
1265 * This ensures that, if available, memcg is automatically enabled
1266 * together on the default hierarchy so that the owner cgroup can
1267 * be retrieved from writeback pages.
1268 */
1269 .depends_on = 1 << memory_cgrp_id,
1270#endif
Tejun Heo676f7c82012-04-01 12:09:55 -07001271};
Tejun Heoc165b3e2015-08-18 14:55:29 -07001272EXPORT_SYMBOL_GPL(io_cgrp_subsys);
Tejun Heo676f7c82012-04-01 12:09:55 -07001273
Tejun Heo8bd435b2012-04-13 13:11:28 -07001274/**
Tejun Heoa2b16932012-04-13 13:11:33 -07001275 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1276 * @q: request_queue of interest
1277 * @pol: blkcg policy to activate
1278 *
1279 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1280 * bypass mode to populate its blkgs with policy_data for @pol.
1281 *
1282 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1283 * from IO path. Update of each blkg is protected by both queue and blkcg
1284 * locks so that holding either lock and testing blkcg_policy_enabled() is
1285 * always enough for dereferencing policy data.
1286 *
1287 * The caller is responsible for synchronizing [de]activations and policy
1288 * [un]registerations. Returns 0 on success, -errno on failure.
1289 */
1290int blkcg_activate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001291 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001292{
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001293 struct blkg_policy_data *pd_prealloc = NULL;
Tejun Heoec13b1d2015-05-22 17:13:19 -04001294 struct blkcg_gq *blkg;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001295 int ret;
Tejun Heoa2b16932012-04-13 13:11:33 -07001296
1297 if (blkcg_policy_enabled(q, pol))
1298 return 0;
1299
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001300 if (q->mq_ops)
Jens Axboebd166ef2017-01-17 06:03:22 -07001301 blk_mq_freeze_queue(q);
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001302 else
Jens Axboebd166ef2017-01-17 06:03:22 -07001303 blk_queue_bypass_start(q);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001304pd_prealloc:
1305 if (!pd_prealloc) {
Tejun Heo001bea72015-08-18 14:55:11 -07001306 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001307 if (!pd_prealloc) {
Tejun Heoa2b16932012-04-13 13:11:33 -07001308 ret = -ENOMEM;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001309 goto out_bypass_end;
Tejun Heoa2b16932012-04-13 13:11:33 -07001310 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001311 }
1312
Tejun Heoa2b16932012-04-13 13:11:33 -07001313 spin_lock_irq(q->queue_lock);
1314
1315 list_for_each_entry(blkg, &q->blkg_list, q_node) {
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001316 struct blkg_policy_data *pd;
Tejun Heoa2b16932012-04-13 13:11:33 -07001317
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001318 if (blkg->pd[pol->plid])
1319 continue;
1320
Tejun Heoe00f4f42016-11-21 18:03:32 -05001321 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q->node);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001322 if (!pd)
1323 swap(pd, pd_prealloc);
1324 if (!pd) {
1325 spin_unlock_irq(q->queue_lock);
1326 goto pd_prealloc;
1327 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001328
1329 blkg->pd[pol->plid] = pd;
1330 pd->blkg = blkg;
Tejun Heob276a872013-01-09 08:05:12 -08001331 pd->plid = pol->plid;
Tejun Heo3e418712015-08-18 14:55:10 -07001332 if (pol->pd_init_fn)
Tejun Heoa9520cd2015-08-18 14:55:14 -07001333 pol->pd_init_fn(pd);
Tejun Heoa2b16932012-04-13 13:11:33 -07001334 }
1335
1336 __set_bit(pol->plid, q->blkcg_pols);
1337 ret = 0;
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001338
Tejun Heoa2b16932012-04-13 13:11:33 -07001339 spin_unlock_irq(q->queue_lock);
Tejun Heo4c55f4f2015-08-18 14:55:09 -07001340out_bypass_end:
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001341 if (q->mq_ops)
Jens Axboebd166ef2017-01-17 06:03:22 -07001342 blk_mq_unfreeze_queue(q);
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001343 else
Jens Axboebd166ef2017-01-17 06:03:22 -07001344 blk_queue_bypass_end(q);
Tejun Heo001bea72015-08-18 14:55:11 -07001345 if (pd_prealloc)
1346 pol->pd_free_fn(pd_prealloc);
Tejun Heoa2b16932012-04-13 13:11:33 -07001347 return ret;
1348}
1349EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1350
1351/**
1352 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1353 * @q: request_queue of interest
1354 * @pol: blkcg policy to deactivate
1355 *
1356 * Deactivate @pol on @q. Follows the same synchronization rules as
1357 * blkcg_activate_policy().
1358 */
1359void blkcg_deactivate_policy(struct request_queue *q,
Tejun Heo3c798392012-04-16 13:57:25 -07001360 const struct blkcg_policy *pol)
Tejun Heoa2b16932012-04-13 13:11:33 -07001361{
Tejun Heo3c798392012-04-16 13:57:25 -07001362 struct blkcg_gq *blkg;
Tejun Heoa2b16932012-04-13 13:11:33 -07001363
1364 if (!blkcg_policy_enabled(q, pol))
1365 return;
1366
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001367 if (q->mq_ops)
Jens Axboebd166ef2017-01-17 06:03:22 -07001368 blk_mq_freeze_queue(q);
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001369 else
Jens Axboebd166ef2017-01-17 06:03:22 -07001370 blk_queue_bypass_start(q);
1371
Tejun Heoa2b16932012-04-13 13:11:33 -07001372 spin_lock_irq(q->queue_lock);
1373
1374 __clear_bit(pol->plid, q->blkcg_pols);
1375
1376 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1377 /* grab blkcg lock too while removing @pd from @blkg */
1378 spin_lock(&blkg->blkcg->lock);
1379
Tejun Heo001bea72015-08-18 14:55:11 -07001380 if (blkg->pd[pol->plid]) {
Tejun Heoa9520cd2015-08-18 14:55:14 -07001381 if (pol->pd_offline_fn)
1382 pol->pd_offline_fn(blkg->pd[pol->plid]);
Tejun Heo001bea72015-08-18 14:55:11 -07001383 pol->pd_free_fn(blkg->pd[pol->plid]);
1384 blkg->pd[pol->plid] = NULL;
1385 }
Tejun Heoa2b16932012-04-13 13:11:33 -07001386
1387 spin_unlock(&blkg->blkcg->lock);
1388 }
1389
1390 spin_unlock_irq(q->queue_lock);
Jens Axboebd166ef2017-01-17 06:03:22 -07001391
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001392 if (q->mq_ops)
Jens Axboebd166ef2017-01-17 06:03:22 -07001393 blk_mq_unfreeze_queue(q);
Jens Axboe38dbb7d2017-01-18 15:37:27 -07001394 else
Jens Axboebd166ef2017-01-17 06:03:22 -07001395 blk_queue_bypass_end(q);
Tejun Heoa2b16932012-04-13 13:11:33 -07001396}
1397EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1398
1399/**
Tejun Heo3c798392012-04-16 13:57:25 -07001400 * blkcg_policy_register - register a blkcg policy
1401 * @pol: blkcg policy to register
Tejun Heo8bd435b2012-04-13 13:11:28 -07001402 *
Tejun Heo3c798392012-04-16 13:57:25 -07001403 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1404 * successful registration. Returns 0 on success and -errno on failure.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001405 */
Jens Axboed5bf0292014-06-22 16:31:56 -06001406int blkcg_policy_register(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001407{
Tejun Heo06b285b2015-07-09 16:39:50 -04001408 struct blkcg *blkcg;
Tejun Heo8bd435b2012-04-13 13:11:28 -07001409 int i, ret;
Tejun Heoe8989fa2012-03-05 13:15:20 -08001410
Tejun Heo838f13b2015-07-09 16:39:47 -04001411 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001412 mutex_lock(&blkcg_pol_mutex);
1413
Tejun Heo8bd435b2012-04-13 13:11:28 -07001414 /* find an empty slot */
1415 ret = -ENOSPC;
1416 for (i = 0; i < BLKCG_MAX_POLS; i++)
Tejun Heo3c798392012-04-16 13:57:25 -07001417 if (!blkcg_policy[i])
Tejun Heo8bd435b2012-04-13 13:11:28 -07001418 break;
1419 if (i >= BLKCG_MAX_POLS)
Tejun Heo838f13b2015-07-09 16:39:47 -04001420 goto err_unlock;
Tejun Heo035d10b2012-03-05 13:15:04 -08001421
weiping zhange8401072017-10-17 23:56:21 +08001422 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1423 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1424 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1425 goto err_unlock;
1426
Tejun Heo06b285b2015-07-09 16:39:50 -04001427 /* register @pol */
Tejun Heo3c798392012-04-16 13:57:25 -07001428 pol->plid = i;
Tejun Heo06b285b2015-07-09 16:39:50 -04001429 blkcg_policy[pol->plid] = pol;
1430
1431 /* allocate and install cpd's */
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001432 if (pol->cpd_alloc_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001433 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1434 struct blkcg_policy_data *cpd;
1435
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001436 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
Bart Van Asschebbb427e2016-09-29 08:33:30 -07001437 if (!cpd)
Tejun Heo06b285b2015-07-09 16:39:50 -04001438 goto err_free_cpds;
Tejun Heo06b285b2015-07-09 16:39:50 -04001439
Tejun Heo81437642015-08-18 14:55:15 -07001440 blkcg->cpd[pol->plid] = cpd;
1441 cpd->blkcg = blkcg;
Tejun Heo06b285b2015-07-09 16:39:50 -04001442 cpd->plid = pol->plid;
Tejun Heo81437642015-08-18 14:55:15 -07001443 pol->cpd_init_fn(cpd);
Tejun Heo06b285b2015-07-09 16:39:50 -04001444 }
1445 }
1446
Tejun Heo838f13b2015-07-09 16:39:47 -04001447 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001448
Tejun Heo8bd435b2012-04-13 13:11:28 -07001449 /* everything is in place, add intf files for the new policy */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001450 if (pol->dfl_cftypes)
1451 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1452 pol->dfl_cftypes));
Tejun Heo880f50e22015-08-18 14:55:30 -07001453 if (pol->legacy_cftypes)
Tejun Heoc165b3e2015-08-18 14:55:29 -07001454 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
Tejun Heo880f50e22015-08-18 14:55:30 -07001455 pol->legacy_cftypes));
Tejun Heo838f13b2015-07-09 16:39:47 -04001456 mutex_unlock(&blkcg_pol_register_mutex);
1457 return 0;
1458
Tejun Heo06b285b2015-07-09 16:39:50 -04001459err_free_cpds:
weiping zhang58a9edc2017-10-10 22:53:46 +08001460 if (pol->cpd_free_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001461 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001462 if (blkcg->cpd[pol->plid]) {
1463 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1464 blkcg->cpd[pol->plid] = NULL;
1465 }
Tejun Heo06b285b2015-07-09 16:39:50 -04001466 }
1467 }
1468 blkcg_policy[pol->plid] = NULL;
Tejun Heo838f13b2015-07-09 16:39:47 -04001469err_unlock:
Tejun Heobc0d6502012-04-13 13:11:26 -07001470 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001471 mutex_unlock(&blkcg_pol_register_mutex);
Tejun Heo8bd435b2012-04-13 13:11:28 -07001472 return ret;
Vivek Goyal3e252062009-12-04 10:36:42 -05001473}
Tejun Heo3c798392012-04-16 13:57:25 -07001474EXPORT_SYMBOL_GPL(blkcg_policy_register);
Vivek Goyal3e252062009-12-04 10:36:42 -05001475
Tejun Heo8bd435b2012-04-13 13:11:28 -07001476/**
Tejun Heo3c798392012-04-16 13:57:25 -07001477 * blkcg_policy_unregister - unregister a blkcg policy
1478 * @pol: blkcg policy to unregister
Tejun Heo8bd435b2012-04-13 13:11:28 -07001479 *
Tejun Heo3c798392012-04-16 13:57:25 -07001480 * Undo blkcg_policy_register(@pol). Might sleep.
Tejun Heo8bd435b2012-04-13 13:11:28 -07001481 */
Tejun Heo3c798392012-04-16 13:57:25 -07001482void blkcg_policy_unregister(struct blkcg_policy *pol)
Vivek Goyal3e252062009-12-04 10:36:42 -05001483{
Tejun Heo06b285b2015-07-09 16:39:50 -04001484 struct blkcg *blkcg;
1485
Tejun Heo838f13b2015-07-09 16:39:47 -04001486 mutex_lock(&blkcg_pol_register_mutex);
Tejun Heobc0d6502012-04-13 13:11:26 -07001487
Tejun Heo3c798392012-04-16 13:57:25 -07001488 if (WARN_ON(blkcg_policy[pol->plid] != pol))
Tejun Heo8bd435b2012-04-13 13:11:28 -07001489 goto out_unlock;
1490
1491 /* kill the intf files first */
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001492 if (pol->dfl_cftypes)
1493 cgroup_rm_cftypes(pol->dfl_cftypes);
Tejun Heo880f50e22015-08-18 14:55:30 -07001494 if (pol->legacy_cftypes)
1495 cgroup_rm_cftypes(pol->legacy_cftypes);
Tejun Heo44ea53d2012-04-01 14:38:43 -07001496
Tejun Heo06b285b2015-07-09 16:39:50 -04001497 /* remove cpds and unregister */
Tejun Heo838f13b2015-07-09 16:39:47 -04001498 mutex_lock(&blkcg_pol_mutex);
Tejun Heo06b285b2015-07-09 16:39:50 -04001499
weiping zhang58a9edc2017-10-10 22:53:46 +08001500 if (pol->cpd_free_fn) {
Tejun Heo06b285b2015-07-09 16:39:50 -04001501 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001502 if (blkcg->cpd[pol->plid]) {
1503 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1504 blkcg->cpd[pol->plid] = NULL;
1505 }
Tejun Heo06b285b2015-07-09 16:39:50 -04001506 }
1507 }
Tejun Heo3c798392012-04-16 13:57:25 -07001508 blkcg_policy[pol->plid] = NULL;
Tejun Heo06b285b2015-07-09 16:39:50 -04001509
Tejun Heobc0d6502012-04-13 13:11:26 -07001510 mutex_unlock(&blkcg_pol_mutex);
Tejun Heo838f13b2015-07-09 16:39:47 -04001511out_unlock:
1512 mutex_unlock(&blkcg_pol_register_mutex);
Vivek Goyal3e252062009-12-04 10:36:42 -05001513}
Tejun Heo3c798392012-04-16 13:57:25 -07001514EXPORT_SYMBOL_GPL(blkcg_policy_unregister);