blob: 0b38b6ad547d1cf22fa9021a53f8c5b451710c6b [file] [log] [blame]
Thomas Gleixnerc942fdd2019-05-27 08:55:06 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Balbir Singh8cdea7c2008-02-07 00:13:50 -08002/* memcontrol.c - Memory Controller
3 *
4 * Copyright IBM Corporation, 2007
5 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08007 * Copyright 2007 OpenVZ SWsoft Inc
8 * Author: Pavel Emelianov <xemul@openvz.org>
9 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080010 * Memory thresholds
11 * Copyright (C) 2009 Nokia Corporation
12 * Author: Kirill A. Shutemov
13 *
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080014 * Kernel Memory Controller
15 * Copyright (C) 2012 Parallels Inc. and Google Inc.
16 * Authors: Glauber Costa and Suleiman Souhlal
17 *
Johannes Weiner1575e682015-04-14 15:44:51 -070018 * Native page reclaim
19 * Charge lifetime sanitation
20 * Lockless page tracking & accounting
21 * Unified hierarchy configuration model
22 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
Balbir Singh8cdea7c2008-02-07 00:13:50 -080023 */
24
Johannes Weiner3e32cb22014-12-10 15:42:31 -080025#include <linux/page_counter.h>
Balbir Singh8cdea7c2008-02-07 00:13:50 -080026#include <linux/memcontrol.h>
27#include <linux/cgroup.h>
Christoph Hellwiga5201102019-08-28 16:19:53 +020028#include <linux/pagewalk.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010029#include <linux/sched/mm.h>
Hugh Dickins3a4f8a02017-02-24 14:59:36 -080030#include <linux/shmem_fs.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080031#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080032#include <linux/pagemap.h>
Chris Down1ff9e6e2019-03-05 15:48:09 -080033#include <linux/vm_event_item.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080034#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080035#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080036#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080037#include <linux/bit_spinlock.h>
38#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070039#include <linux/limits.h>
Paul Gortmakerb9e15ba2011-05-26 16:00:52 -040040#include <linux/export.h>
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -080041#include <linux/mutex.h>
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -070042#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070043#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080044#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080045#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080046#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080047#include <linux/eventfd.h>
Tejun Heo79bd9812013-11-22 18:20:42 -050048#include <linux/poll.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080049#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080050#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080051#include <linux/seq_file.h>
Anton Vorontsov70ddf632013-04-29 15:08:31 -070052#include <linux/vmpressure.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070053#include <linux/mm_inline.h>
Johannes Weiner5d1ea48b2014-12-10 15:44:55 -080054#include <linux/swap_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080055#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070056#include <linux/oom.h>
Johannes Weiner0056f4e2013-10-31 16:34:14 -070057#include <linux/lockdep.h>
Tejun Heo79bd9812013-11-22 18:20:42 -050058#include <linux/file.h>
Tejun Heob23afb92015-11-05 18:46:11 -080059#include <linux/tracehook.h>
Chris Down0e4b01d2019-09-23 15:34:55 -070060#include <linux/psi.h>
Johannes Weinerc8713d02019-07-11 20:55:59 -070061#include <linux/seq_buf.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080062#include "internal.h"
Glauber Costad1a4c0b2011-12-11 21:47:04 +000063#include <net/sock.h>
Michal Hocko4bd2c1e2012-10-08 16:33:10 -070064#include <net/ip.h>
Qiang Huangf35c3a82013-11-12 15:08:22 -080065#include "slab.h"
Balbir Singh8cdea7c2008-02-07 00:13:50 -080066
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080067#include <linux/uaccess.h>
Balbir Singh8697d332008-02-07 00:13:59 -080068
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070069#include <trace/events/vmscan.h>
70
Tejun Heo073219e2014-02-08 10:36:58 -050071struct cgroup_subsys memory_cgrp_subsys __read_mostly;
72EXPORT_SYMBOL(memory_cgrp_subsys);
David Rientjes68ae5642012-12-12 13:51:57 -080073
Johannes Weiner7d828602016-01-14 15:20:56 -080074struct mem_cgroup *root_mem_cgroup __read_mostly;
75
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070076#define MEM_CGROUP_RECLAIM_RETRIES 5
Balbir Singh8cdea7c2008-02-07 00:13:50 -080077
Johannes Weinerf7e1cb62016-01-14 15:21:29 -080078/* Socket memory accounting disabled? */
79static bool cgroup_memory_nosocket;
80
Vladimir Davydov04823c82016-01-20 15:02:38 -080081/* Kernel memory accounting disabled? */
82static bool cgroup_memory_nokmem;
83
Johannes Weiner21afa382015-02-11 15:26:36 -080084/* Whether the swap controller is active */
Andrew Mortonc255a452012-07-31 16:43:02 -070085#ifdef CONFIG_MEMCG_SWAP
Johannes Weinereccb52e2020-06-03 16:02:11 -070086bool cgroup_memory_noswap __read_mostly;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080087#else
Johannes Weinereccb52e2020-06-03 16:02:11 -070088#define cgroup_memory_noswap 1
Johannes Weiner2d1c4982020-06-03 16:02:14 -070089#endif
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080090
Tejun Heo97b27822019-08-26 09:06:56 -070091#ifdef CONFIG_CGROUP_WRITEBACK
92static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
93#endif
94
Johannes Weiner7941d212016-01-14 15:21:23 -080095/* Whether legacy memory+swap accounting is active */
96static bool do_memsw_account(void)
97{
Johannes Weinereccb52e2020-06-03 16:02:11 -070098 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
Johannes Weiner7941d212016-01-14 15:21:23 -080099}
100
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700101#define THRESHOLDS_EVENTS_TARGET 128
102#define SOFTLIMIT_EVENTS_TARGET 1024
Johannes Weinere9f89742011-03-23 16:42:37 -0700103
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700104/*
105 * Cgroups above their limits are maintained in a RB-Tree, independent of
106 * their hierarchy representation
107 */
108
Mel Gormanef8f2322016-07-28 15:46:05 -0700109struct mem_cgroup_tree_per_node {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700110 struct rb_root rb_root;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700111 struct rb_node *rb_rightmost;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700112 spinlock_t lock;
113};
114
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700115struct mem_cgroup_tree {
116 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
117};
118
119static struct mem_cgroup_tree soft_limit_tree __read_mostly;
120
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700121/* for OOM */
122struct mem_cgroup_eventfd_list {
123 struct list_head list;
124 struct eventfd_ctx *eventfd;
125};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800126
Tejun Heo79bd9812013-11-22 18:20:42 -0500127/*
128 * cgroup_event represents events which userspace want to receive.
129 */
Tejun Heo3bc942f2013-11-22 18:20:44 -0500130struct mem_cgroup_event {
Tejun Heo79bd9812013-11-22 18:20:42 -0500131 /*
Tejun Heo59b6f872013-11-22 18:20:43 -0500132 * memcg which the event belongs to.
Tejun Heo79bd9812013-11-22 18:20:42 -0500133 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500134 struct mem_cgroup *memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -0500135 /*
Tejun Heo79bd9812013-11-22 18:20:42 -0500136 * eventfd to signal userspace about the event.
137 */
138 struct eventfd_ctx *eventfd;
139 /*
140 * Each of these stored in a list by the cgroup.
141 */
142 struct list_head list;
143 /*
Tejun Heofba94802013-11-22 18:20:43 -0500144 * register_event() callback will be used to add new userspace
145 * waiter for changes related to this event. Use eventfd_signal()
146 * on eventfd to send notification to userspace.
147 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500148 int (*register_event)(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -0500149 struct eventfd_ctx *eventfd, const char *args);
Tejun Heofba94802013-11-22 18:20:43 -0500150 /*
151 * unregister_event() callback will be called when userspace closes
152 * the eventfd or on cgroup removing. This callback must be set,
153 * if you want provide notification functionality.
154 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500155 void (*unregister_event)(struct mem_cgroup *memcg,
Tejun Heofba94802013-11-22 18:20:43 -0500156 struct eventfd_ctx *eventfd);
157 /*
Tejun Heo79bd9812013-11-22 18:20:42 -0500158 * All fields below needed to unregister event when
159 * userspace closes eventfd.
160 */
161 poll_table pt;
162 wait_queue_head_t *wqh;
Ingo Molnarac6424b2017-06-20 12:06:13 +0200163 wait_queue_entry_t wait;
Tejun Heo79bd9812013-11-22 18:20:42 -0500164 struct work_struct remove;
165};
166
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700167static void mem_cgroup_threshold(struct mem_cgroup *memcg);
168static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800169
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800170/* Stuffs for move charges at task migration. */
171/*
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800172 * Types of charges to be moved.
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800173 */
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800174#define MOVE_ANON 0x1U
175#define MOVE_FILE 0x2U
176#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800177
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800178/* "mc" and its members are protected by cgroup_mutex */
179static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800180 spinlock_t lock; /* for from, to */
Tejun Heo264a0ae2016-04-21 19:09:02 -0400181 struct mm_struct *mm;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800182 struct mem_cgroup *from;
183 struct mem_cgroup *to;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800184 unsigned long flags;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800185 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800186 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800187 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800188 struct task_struct *moving_task; /* a task moving charges */
189 wait_queue_head_t waitq; /* a waitq for other context */
190} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700191 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800192 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
193};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800194
Balbir Singh4e416952009-09-23 15:56:39 -0700195/*
196 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
197 * limit reclaim to prevent infinite loops, if they ever occur.
198 */
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700199#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700200#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
Balbir Singh4e416952009-09-23 15:56:39 -0700201
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800202enum charge_type {
203 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -0700204 MEM_CGROUP_CHARGE_TYPE_ANON,
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800205 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -0700206 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700207 NR_CHARGE_TYPE,
208};
209
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800210/* for encoding cft->private value on file */
Glauber Costa86ae53e2012-12-18 14:21:45 -0800211enum res_type {
212 _MEM,
213 _MEMSWAP,
214 _OOM_TYPE,
Glauber Costa510fc4e2012-12-18 14:21:47 -0800215 _KMEM,
Vladimir Davydovd55f90b2016-01-20 15:02:44 -0800216 _TCP,
Glauber Costa86ae53e2012-12-18 14:21:45 -0800217};
218
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700219#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
220#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800221#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700222/* Used for OOM nofiier */
223#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800224
Kirill Tkhaib05706f2018-08-17 15:47:33 -0700225/*
226 * Iteration constructs for visiting all cgroups (under a tree). If
227 * loops are exited prematurely (break), mem_cgroup_iter_break() must
228 * be used for reference counting.
229 */
230#define for_each_mem_cgroup_tree(iter, root) \
231 for (iter = mem_cgroup_iter(root, NULL, NULL); \
232 iter != NULL; \
233 iter = mem_cgroup_iter(root, iter, NULL))
234
235#define for_each_mem_cgroup(iter) \
236 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
237 iter != NULL; \
238 iter = mem_cgroup_iter(NULL, iter, NULL))
239
Tetsuo Handa7775fac2019-03-05 15:46:47 -0800240static inline bool should_force_charge(void)
241{
242 return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
243 (current->flags & PF_EXITING);
244}
245
Anton Vorontsov70ddf632013-04-29 15:08:31 -0700246/* Some nice accessors for the vmpressure. */
247struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
248{
249 if (!memcg)
250 memcg = root_mem_cgroup;
251 return &memcg->vmpressure;
252}
253
254struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
255{
256 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
257}
258
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700259#ifdef CONFIG_MEMCG_KMEM
Glauber Costa55007d82012-12-18 14:22:38 -0800260/*
Vladimir Davydovf7ce3192015-02-12 14:59:20 -0800261 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
Li Zefanb8627832013-09-23 16:56:47 +0800262 * The main reason for not using cgroup id for this:
263 * this works better in sparse environments, where we have a lot of memcgs,
264 * but only a few kmem-limited. Or also, if we have, for instance, 200
265 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
266 * 200 entry array for that.
Glauber Costa55007d82012-12-18 14:22:38 -0800267 *
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800268 * The current size of the caches array is stored in memcg_nr_cache_ids. It
269 * will double each time we have to increase it.
Glauber Costa55007d82012-12-18 14:22:38 -0800270 */
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800271static DEFINE_IDA(memcg_cache_ida);
272int memcg_nr_cache_ids;
Glauber Costa749c5412012-12-18 14:23:01 -0800273
Vladimir Davydov05257a12015-02-12 14:59:01 -0800274/* Protects memcg_nr_cache_ids */
275static DECLARE_RWSEM(memcg_cache_ids_sem);
276
277void memcg_get_cache_ids(void)
278{
279 down_read(&memcg_cache_ids_sem);
280}
281
282void memcg_put_cache_ids(void)
283{
284 up_read(&memcg_cache_ids_sem);
285}
286
Glauber Costa55007d82012-12-18 14:22:38 -0800287/*
288 * MIN_SIZE is different than 1, because we would like to avoid going through
289 * the alloc/free process all the time. In a small machine, 4 kmem-limited
290 * cgroups is a reasonable guess. In the future, it could be a parameter or
291 * tunable, but that is strictly not necessary.
292 *
Li Zefanb8627832013-09-23 16:56:47 +0800293 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
Glauber Costa55007d82012-12-18 14:22:38 -0800294 * this constant directly from cgroup, but it is understandable that this is
295 * better kept as an internal representation in cgroup.c. In any case, the
Li Zefanb8627832013-09-23 16:56:47 +0800296 * cgrp_id space is not getting any smaller, and we don't have to necessarily
Glauber Costa55007d82012-12-18 14:22:38 -0800297 * increase ours as well if it increases.
298 */
299#define MEMCG_CACHES_MIN_SIZE 4
Li Zefanb8627832013-09-23 16:56:47 +0800300#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
Glauber Costa55007d82012-12-18 14:22:38 -0800301
Glauber Costad7f25f82012-12-18 14:22:40 -0800302/*
303 * A lot of the calls to the cache allocation functions are expected to be
304 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
305 * conditional to this static branch, we'll have to allow modules that does
306 * kmem_cache_alloc and the such to see this symbol as well
307 */
Johannes Weineref129472016-01-14 15:21:34 -0800308DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
Glauber Costad7f25f82012-12-18 14:22:40 -0800309EXPORT_SYMBOL(memcg_kmem_enabled_key);
Glauber Costaa8964b92012-12-18 14:22:09 -0800310
Tejun Heo17cc4df2017-02-22 15:41:36 -0800311struct workqueue_struct *memcg_kmem_cache_wq;
Yang Shi0a432dcb2019-09-23 15:38:12 -0700312#endif
Tejun Heo17cc4df2017-02-22 15:41:36 -0800313
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700314static int memcg_shrinker_map_size;
315static DEFINE_MUTEX(memcg_shrinker_map_mutex);
316
317static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
318{
319 kvfree(container_of(head, struct memcg_shrinker_map, rcu));
320}
321
322static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
323 int size, int old_size)
324{
325 struct memcg_shrinker_map *new, *old;
326 int nid;
327
328 lockdep_assert_held(&memcg_shrinker_map_mutex);
329
330 for_each_node(nid) {
331 old = rcu_dereference_protected(
332 mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
333 /* Not yet online memcg */
334 if (!old)
335 return 0;
336
Kirill Tkhai86daf942020-04-01 21:06:33 -0700337 new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700338 if (!new)
339 return -ENOMEM;
340
341 /* Set all old bits, clear all new bits */
342 memset(new->map, (int)0xff, old_size);
343 memset((void *)new->map + old_size, 0, size - old_size);
344
345 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
346 call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
347 }
348
349 return 0;
350}
351
352static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
353{
354 struct mem_cgroup_per_node *pn;
355 struct memcg_shrinker_map *map;
356 int nid;
357
358 if (mem_cgroup_is_root(memcg))
359 return;
360
361 for_each_node(nid) {
362 pn = mem_cgroup_nodeinfo(memcg, nid);
363 map = rcu_dereference_protected(pn->shrinker_map, true);
364 if (map)
365 kvfree(map);
366 rcu_assign_pointer(pn->shrinker_map, NULL);
367 }
368}
369
370static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
371{
372 struct memcg_shrinker_map *map;
373 int nid, size, ret = 0;
374
375 if (mem_cgroup_is_root(memcg))
376 return 0;
377
378 mutex_lock(&memcg_shrinker_map_mutex);
379 size = memcg_shrinker_map_size;
380 for_each_node(nid) {
Kirill Tkhai86daf942020-04-01 21:06:33 -0700381 map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700382 if (!map) {
383 memcg_free_shrinker_maps(memcg);
384 ret = -ENOMEM;
385 break;
386 }
387 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
388 }
389 mutex_unlock(&memcg_shrinker_map_mutex);
390
391 return ret;
392}
393
394int memcg_expand_shrinker_maps(int new_id)
395{
396 int size, old_size, ret = 0;
397 struct mem_cgroup *memcg;
398
399 size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
400 old_size = memcg_shrinker_map_size;
401 if (size <= old_size)
402 return 0;
403
404 mutex_lock(&memcg_shrinker_map_mutex);
405 if (!root_mem_cgroup)
406 goto unlock;
407
408 for_each_mem_cgroup(memcg) {
409 if (mem_cgroup_is_root(memcg))
410 continue;
411 ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
Vasily Averin75866af62b2020-02-20 20:04:18 -0800412 if (ret) {
413 mem_cgroup_iter_break(NULL, memcg);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700414 goto unlock;
Vasily Averin75866af62b2020-02-20 20:04:18 -0800415 }
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700416 }
417unlock:
418 if (!ret)
419 memcg_shrinker_map_size = size;
420 mutex_unlock(&memcg_shrinker_map_mutex);
421 return ret;
422}
Kirill Tkhaifae91d62018-08-17 15:48:10 -0700423
424void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
425{
426 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
427 struct memcg_shrinker_map *map;
428
429 rcu_read_lock();
430 map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
Kirill Tkhaif90280d2018-08-17 15:48:25 -0700431 /* Pairs with smp mb in shrink_slab() */
432 smp_mb__before_atomic();
Kirill Tkhaifae91d62018-08-17 15:48:10 -0700433 set_bit(shrinker_id, map->map);
434 rcu_read_unlock();
435 }
436}
437
Tejun Heoad7fa852015-05-27 20:00:02 -0400438/**
439 * mem_cgroup_css_from_page - css of the memcg associated with a page
440 * @page: page of interest
441 *
442 * If memcg is bound to the default hierarchy, css of the memcg associated
443 * with @page is returned. The returned css remains associated with @page
444 * until it is released.
445 *
446 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
447 * is returned.
Tejun Heoad7fa852015-05-27 20:00:02 -0400448 */
449struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
450{
451 struct mem_cgroup *memcg;
452
Tejun Heoad7fa852015-05-27 20:00:02 -0400453 memcg = page->mem_cgroup;
454
Tejun Heo9e10a132015-09-18 11:56:28 -0400455 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Tejun Heoad7fa852015-05-27 20:00:02 -0400456 memcg = root_mem_cgroup;
457
Tejun Heoad7fa852015-05-27 20:00:02 -0400458 return &memcg->css;
459}
460
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700461/**
462 * page_cgroup_ino - return inode number of the memcg a page is charged to
463 * @page: the page
464 *
465 * Look up the closest online ancestor of the memory cgroup @page is charged to
466 * and return its inode number or 0 if @page is not charged to any cgroup. It
467 * is safe to call this function without holding a reference to @page.
468 *
469 * Note, this function is inherently racy, because there is nothing to prevent
470 * the cgroup inode from getting torn down and potentially reallocated a moment
471 * after page_cgroup_ino() returns, so it only should be used by callers that
472 * do not care (such as procfs interfaces).
473 */
474ino_t page_cgroup_ino(struct page *page)
475{
476 struct mem_cgroup *memcg;
477 unsigned long ino = 0;
478
479 rcu_read_lock();
Roman Gushchin221ec5c2019-11-05 21:17:03 -0800480 if (PageSlab(page) && !PageTail(page))
Roman Gushchin4d96ba32019-07-11 20:56:31 -0700481 memcg = memcg_from_slab_page(page);
482 else
483 memcg = READ_ONCE(page->mem_cgroup);
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700484 while (memcg && !(memcg->css.flags & CSS_ONLINE))
485 memcg = parent_mem_cgroup(memcg);
486 if (memcg)
487 ino = cgroup_ino(memcg->css.cgroup);
488 rcu_read_unlock();
489 return ino;
490}
491
Mel Gormanef8f2322016-07-28 15:46:05 -0700492static struct mem_cgroup_per_node *
493mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700494{
Johannes Weiner97a6c372011-03-23 16:42:27 -0700495 int nid = page_to_nid(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700496
Mel Gormanef8f2322016-07-28 15:46:05 -0700497 return memcg->nodeinfo[nid];
Balbir Singhf64c3f52009-09-23 15:56:37 -0700498}
499
Mel Gormanef8f2322016-07-28 15:46:05 -0700500static struct mem_cgroup_tree_per_node *
501soft_limit_tree_node(int nid)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700502{
Mel Gormanef8f2322016-07-28 15:46:05 -0700503 return soft_limit_tree.rb_tree_per_node[nid];
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700504}
505
Mel Gormanef8f2322016-07-28 15:46:05 -0700506static struct mem_cgroup_tree_per_node *
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700507soft_limit_tree_from_page(struct page *page)
508{
509 int nid = page_to_nid(page);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700510
Mel Gormanef8f2322016-07-28 15:46:05 -0700511 return soft_limit_tree.rb_tree_per_node[nid];
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700512}
513
Mel Gormanef8f2322016-07-28 15:46:05 -0700514static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
515 struct mem_cgroup_tree_per_node *mctz,
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800516 unsigned long new_usage_in_excess)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700517{
518 struct rb_node **p = &mctz->rb_root.rb_node;
519 struct rb_node *parent = NULL;
Mel Gormanef8f2322016-07-28 15:46:05 -0700520 struct mem_cgroup_per_node *mz_node;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700521 bool rightmost = true;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700522
523 if (mz->on_tree)
524 return;
525
526 mz->usage_in_excess = new_usage_in_excess;
527 if (!mz->usage_in_excess)
528 return;
529 while (*p) {
530 parent = *p;
Mel Gormanef8f2322016-07-28 15:46:05 -0700531 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700532 tree_node);
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700533 if (mz->usage_in_excess < mz_node->usage_in_excess) {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700534 p = &(*p)->rb_left;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700535 rightmost = false;
536 }
537
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700538 /*
539 * We can't avoid mem cgroups that are over their soft
540 * limit by the same amount
541 */
542 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
543 p = &(*p)->rb_right;
544 }
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700545
546 if (rightmost)
547 mctz->rb_rightmost = &mz->tree_node;
548
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700549 rb_link_node(&mz->tree_node, parent, p);
550 rb_insert_color(&mz->tree_node, &mctz->rb_root);
551 mz->on_tree = true;
552}
553
Mel Gormanef8f2322016-07-28 15:46:05 -0700554static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
555 struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700556{
557 if (!mz->on_tree)
558 return;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700559
560 if (&mz->tree_node == mctz->rb_rightmost)
561 mctz->rb_rightmost = rb_prev(&mz->tree_node);
562
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700563 rb_erase(&mz->tree_node, &mctz->rb_root);
564 mz->on_tree = false;
565}
566
Mel Gormanef8f2322016-07-28 15:46:05 -0700567static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
568 struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700569{
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700570 unsigned long flags;
571
572 spin_lock_irqsave(&mctz->lock, flags);
Johannes Weinercf2c8122014-06-06 14:38:21 -0700573 __mem_cgroup_remove_exceeded(mz, mctz);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700574 spin_unlock_irqrestore(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700575}
576
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800577static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
578{
579 unsigned long nr_pages = page_counter_read(&memcg->memory);
Jason Low4db0c3c2015-04-15 16:14:08 -0700580 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800581 unsigned long excess = 0;
582
583 if (nr_pages > soft_limit)
584 excess = nr_pages - soft_limit;
585
586 return excess;
587}
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700588
589static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
590{
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800591 unsigned long excess;
Mel Gormanef8f2322016-07-28 15:46:05 -0700592 struct mem_cgroup_per_node *mz;
593 struct mem_cgroup_tree_per_node *mctz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700594
Jianyu Zhane2318752014-06-06 14:38:20 -0700595 mctz = soft_limit_tree_from_page(page);
Laurent Dufourbfc72282017-03-09 16:17:06 -0800596 if (!mctz)
597 return;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700598 /*
599 * Necessary to update all ancestors when hierarchy is used.
600 * because their event counter is not touched.
601 */
602 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
Mel Gormanef8f2322016-07-28 15:46:05 -0700603 mz = mem_cgroup_page_nodeinfo(memcg, page);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800604 excess = soft_limit_excess(memcg);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700605 /*
606 * We have to update the tree if mz is on RB-tree or
607 * mem is over its softlimit.
608 */
609 if (excess || mz->on_tree) {
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700610 unsigned long flags;
611
612 spin_lock_irqsave(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700613 /* if on-tree, remove it */
614 if (mz->on_tree)
Johannes Weinercf2c8122014-06-06 14:38:21 -0700615 __mem_cgroup_remove_exceeded(mz, mctz);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700616 /*
617 * Insert again. mz->usage_in_excess will be updated.
618 * If excess is 0, no tree ops.
619 */
Johannes Weinercf2c8122014-06-06 14:38:21 -0700620 __mem_cgroup_insert_exceeded(mz, mctz, excess);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700621 spin_unlock_irqrestore(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700622 }
623 }
624}
625
626static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
627{
Mel Gormanef8f2322016-07-28 15:46:05 -0700628 struct mem_cgroup_tree_per_node *mctz;
629 struct mem_cgroup_per_node *mz;
630 int nid;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700631
Jianyu Zhane2318752014-06-06 14:38:20 -0700632 for_each_node(nid) {
Mel Gormanef8f2322016-07-28 15:46:05 -0700633 mz = mem_cgroup_nodeinfo(memcg, nid);
634 mctz = soft_limit_tree_node(nid);
Laurent Dufourbfc72282017-03-09 16:17:06 -0800635 if (mctz)
636 mem_cgroup_remove_exceeded(mz, mctz);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700637 }
638}
639
Mel Gormanef8f2322016-07-28 15:46:05 -0700640static struct mem_cgroup_per_node *
641__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700642{
Mel Gormanef8f2322016-07-28 15:46:05 -0700643 struct mem_cgroup_per_node *mz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700644
645retry:
646 mz = NULL;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700647 if (!mctz->rb_rightmost)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700648 goto done; /* Nothing to reclaim from */
649
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700650 mz = rb_entry(mctz->rb_rightmost,
651 struct mem_cgroup_per_node, tree_node);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700652 /*
653 * Remove the node now but someone else can add it back,
654 * we will to add it back at the end of reclaim to its correct
655 * position in the tree.
656 */
Johannes Weinercf2c8122014-06-06 14:38:21 -0700657 __mem_cgroup_remove_exceeded(mz, mctz);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800658 if (!soft_limit_excess(mz->memcg) ||
Shakeel Butt8965aa22020-04-01 21:07:10 -0700659 !css_tryget(&mz->memcg->css))
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700660 goto retry;
661done:
662 return mz;
663}
664
Mel Gormanef8f2322016-07-28 15:46:05 -0700665static struct mem_cgroup_per_node *
666mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700667{
Mel Gormanef8f2322016-07-28 15:46:05 -0700668 struct mem_cgroup_per_node *mz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700669
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700670 spin_lock_irq(&mctz->lock);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700671 mz = __mem_cgroup_largest_soft_limit_node(mctz);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700672 spin_unlock_irq(&mctz->lock);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700673 return mz;
674}
675
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700676/**
677 * __mod_memcg_state - update cgroup memory statistics
678 * @memcg: the memory cgroup
679 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
680 * @val: delta to add to the counter, can be negative
681 */
682void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
683{
684 long x;
685
686 if (mem_cgroup_disabled())
687 return;
688
689 x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
690 if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
Johannes Weiner42a30032019-05-14 15:47:12 -0700691 struct mem_cgroup *mi;
692
Yafang Shao766a4c12019-07-16 16:26:06 -0700693 /*
694 * Batch local counters to keep them in sync with
695 * the hierarchical ones.
696 */
697 __this_cpu_add(memcg->vmstats_local->stat[idx], x);
Johannes Weiner42a30032019-05-14 15:47:12 -0700698 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
699 atomic_long_add(x, &mi->vmstats[idx]);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700700 x = 0;
701 }
702 __this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
703}
704
Johannes Weiner42a30032019-05-14 15:47:12 -0700705static struct mem_cgroup_per_node *
706parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
707{
708 struct mem_cgroup *parent;
709
710 parent = parent_mem_cgroup(pn->memcg);
711 if (!parent)
712 return NULL;
713 return mem_cgroup_nodeinfo(parent, nid);
714}
715
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700716/**
717 * __mod_lruvec_state - update lruvec memory statistics
718 * @lruvec: the lruvec
719 * @idx: the stat item
720 * @val: delta to add to the counter, can be negative
721 *
722 * The lruvec is the intersection of the NUMA node and a cgroup. This
723 * function updates the all three counters that are affected by a
724 * change of state at this level: per-node, per-cgroup, per-lruvec.
725 */
726void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
727 int val)
728{
Johannes Weiner42a30032019-05-14 15:47:12 -0700729 pg_data_t *pgdat = lruvec_pgdat(lruvec);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700730 struct mem_cgroup_per_node *pn;
Johannes Weiner42a30032019-05-14 15:47:12 -0700731 struct mem_cgroup *memcg;
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700732 long x;
733
734 /* Update node */
Johannes Weiner42a30032019-05-14 15:47:12 -0700735 __mod_node_page_state(pgdat, idx, val);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700736
737 if (mem_cgroup_disabled())
738 return;
739
740 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Johannes Weiner42a30032019-05-14 15:47:12 -0700741 memcg = pn->memcg;
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700742
743 /* Update memcg */
Johannes Weiner42a30032019-05-14 15:47:12 -0700744 __mod_memcg_state(memcg, idx, val);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700745
Roman Gushchinb4c46482019-08-30 16:04:39 -0700746 /* Update lruvec */
747 __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
748
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700749 x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
750 if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
Johannes Weiner42a30032019-05-14 15:47:12 -0700751 struct mem_cgroup_per_node *pi;
752
Johannes Weiner42a30032019-05-14 15:47:12 -0700753 for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
754 atomic_long_add(x, &pi->lruvec_stat[idx]);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700755 x = 0;
756 }
757 __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
758}
759
Roman Gushchinec9f0232019-08-13 15:37:41 -0700760void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
761{
Roman Gushchin4f103c62020-04-01 21:06:36 -0700762 pg_data_t *pgdat = page_pgdat(virt_to_page(p));
Roman Gushchinec9f0232019-08-13 15:37:41 -0700763 struct mem_cgroup *memcg;
764 struct lruvec *lruvec;
765
766 rcu_read_lock();
Roman Gushchin4f103c62020-04-01 21:06:36 -0700767 memcg = mem_cgroup_from_obj(p);
Roman Gushchinec9f0232019-08-13 15:37:41 -0700768
769 /* Untracked pages have no memcg, no lruvec. Update only the node */
770 if (!memcg || memcg == root_mem_cgroup) {
771 __mod_node_page_state(pgdat, idx, val);
772 } else {
Johannes Weiner867e5e12019-11-30 17:55:34 -0800773 lruvec = mem_cgroup_lruvec(memcg, pgdat);
Roman Gushchinec9f0232019-08-13 15:37:41 -0700774 __mod_lruvec_state(lruvec, idx, val);
775 }
776 rcu_read_unlock();
777}
778
Roman Gushchin8380ce42020-03-28 19:17:25 -0700779void mod_memcg_obj_state(void *p, int idx, int val)
780{
781 struct mem_cgroup *memcg;
782
783 rcu_read_lock();
784 memcg = mem_cgroup_from_obj(p);
785 if (memcg)
786 mod_memcg_state(memcg, idx, val);
787 rcu_read_unlock();
788}
789
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700790/**
791 * __count_memcg_events - account VM events in a cgroup
792 * @memcg: the memory cgroup
793 * @idx: the event item
794 * @count: the number of events that occured
795 */
796void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
797 unsigned long count)
798{
799 unsigned long x;
800
801 if (mem_cgroup_disabled())
802 return;
803
804 x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
805 if (unlikely(x > MEMCG_CHARGE_BATCH)) {
Johannes Weiner42a30032019-05-14 15:47:12 -0700806 struct mem_cgroup *mi;
807
Yafang Shao766a4c12019-07-16 16:26:06 -0700808 /*
809 * Batch local counters to keep them in sync with
810 * the hierarchical ones.
811 */
812 __this_cpu_add(memcg->vmstats_local->events[idx], x);
Johannes Weiner42a30032019-05-14 15:47:12 -0700813 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
814 atomic_long_add(x, &mi->vmevents[idx]);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700815 x = 0;
816 }
817 __this_cpu_write(memcg->vmstats_percpu->events[idx], x);
818}
819
Johannes Weiner42a30032019-05-14 15:47:12 -0700820static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
Johannes Weinere9f89742011-03-23 16:42:37 -0700821{
Chris Down871789d2019-05-14 15:46:57 -0700822 return atomic_long_read(&memcg->vmevents[event]);
Johannes Weinere9f89742011-03-23 16:42:37 -0700823}
824
Johannes Weiner42a30032019-05-14 15:47:12 -0700825static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
826{
Johannes Weiner815744d2019-06-13 15:55:46 -0700827 long x = 0;
828 int cpu;
829
830 for_each_possible_cpu(cpu)
831 x += per_cpu(memcg->vmstats_local->events[event], cpu);
832 return x;
Johannes Weiner42a30032019-05-14 15:47:12 -0700833}
834
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700835static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
David Rientjesb070e652013-05-07 16:18:09 -0700836 struct page *page,
Johannes Weiner3fba69a2020-06-03 16:01:31 -0700837 int nr_pages)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800838{
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800839 /* pagein of a big page is an event. So, ignore page size */
840 if (nr_pages > 0)
Johannes Weinerc9019e92018-01-31 16:16:37 -0800841 __count_memcg_events(memcg, PGPGIN, 1);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800842 else {
Johannes Weinerc9019e92018-01-31 16:16:37 -0800843 __count_memcg_events(memcg, PGPGOUT, 1);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800844 nr_pages = -nr_pages; /* for event */
845 }
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800846
Chris Down871789d2019-05-14 15:46:57 -0700847 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800848}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800849
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800850static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
851 enum mem_cgroup_events_target target)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800852{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700853 unsigned long val, next;
854
Chris Down871789d2019-05-14 15:46:57 -0700855 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
856 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700857 /* from time_after() in jiffies.h */
Michal Hocko6a1a8b82017-07-10 15:48:53 -0700858 if ((long)(next - val) < 0) {
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800859 switch (target) {
860 case MEM_CGROUP_TARGET_THRESH:
861 next = val + THRESHOLDS_EVENTS_TARGET;
862 break;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700863 case MEM_CGROUP_TARGET_SOFTLIMIT:
864 next = val + SOFTLIMIT_EVENTS_TARGET;
865 break;
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800866 default:
867 break;
868 }
Chris Down871789d2019-05-14 15:46:57 -0700869 __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800870 return true;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700871 }
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800872 return false;
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800873}
874
875/*
876 * Check events in order.
877 *
878 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700879static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800880{
881 /* threshold event is triggered in finer grain than soft limit */
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800882 if (unlikely(mem_cgroup_event_ratelimit(memcg,
883 MEM_CGROUP_TARGET_THRESH))) {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700884 bool do_softlimit;
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800885
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700886 do_softlimit = mem_cgroup_event_ratelimit(memcg,
887 MEM_CGROUP_TARGET_SOFTLIMIT);
Johannes Weinerf53d7ce32012-01-12 17:18:23 -0800888 mem_cgroup_threshold(memcg);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700889 if (unlikely(do_softlimit))
890 mem_cgroup_update_tree(memcg, page);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700891 }
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800892}
893
Balbir Singhcf475ad2008-04-29 01:00:16 -0700894struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800895{
Balbir Singh31a78f22008-09-28 23:09:31 +0100896 /*
897 * mm_update_next_owner() may clear mm->owner to NULL
898 * if it races with swapoff, page migration, etc.
899 * So this can be called with p == NULL.
900 */
901 if (unlikely(!p))
902 return NULL;
903
Tejun Heo073219e2014-02-08 10:36:58 -0500904 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800905}
Michal Hocko33398cf2015-09-08 15:01:02 -0700906EXPORT_SYMBOL(mem_cgroup_from_task);
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800907
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700908/**
909 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
910 * @mm: mm from which memcg should be extracted. It can be NULL.
911 *
912 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
913 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
914 * returned.
915 */
916struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800917{
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700918 struct mem_cgroup *memcg;
919
920 if (mem_cgroup_disabled())
921 return NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700922
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800923 rcu_read_lock();
924 do {
Michal Hocko6f6acb02014-05-22 11:54:19 -0700925 /*
926 * Page cache insertions can happen withou an
927 * actual mm context, e.g. during disk probing
928 * on boot, loopback IO, acct() writes etc.
929 */
930 if (unlikely(!mm))
Johannes Weinerdf381972014-04-07 15:37:43 -0700931 memcg = root_mem_cgroup;
Michal Hocko6f6acb02014-05-22 11:54:19 -0700932 else {
933 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
934 if (unlikely(!memcg))
935 memcg = root_mem_cgroup;
936 }
Roman Gushchin00d484f2019-11-15 17:34:43 -0800937 } while (!css_tryget(&memcg->css));
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800938 rcu_read_unlock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700939 return memcg;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800940}
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700941EXPORT_SYMBOL(get_mem_cgroup_from_mm);
942
943/**
Shakeel Buttf745c6f2018-08-17 15:46:44 -0700944 * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
945 * @page: page from which memcg should be extracted.
946 *
947 * Obtain a reference on page->memcg and returns it if successful. Otherwise
948 * root_mem_cgroup is returned.
949 */
950struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
951{
952 struct mem_cgroup *memcg = page->mem_cgroup;
953
954 if (mem_cgroup_disabled())
955 return NULL;
956
957 rcu_read_lock();
Shakeel Butt8965aa22020-04-01 21:07:10 -0700958 /* Page should not get uncharged and freed memcg under us. */
959 if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css)))
Shakeel Buttf745c6f2018-08-17 15:46:44 -0700960 memcg = root_mem_cgroup;
961 rcu_read_unlock();
962 return memcg;
963}
964EXPORT_SYMBOL(get_mem_cgroup_from_page);
965
966/**
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700967 * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
968 */
969static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
970{
971 if (unlikely(current->active_memcg)) {
Shakeel Butt8965aa22020-04-01 21:07:10 -0700972 struct mem_cgroup *memcg;
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700973
974 rcu_read_lock();
Shakeel Butt8965aa22020-04-01 21:07:10 -0700975 /* current->active_memcg must hold a ref. */
976 if (WARN_ON_ONCE(!css_tryget(&current->active_memcg->css)))
977 memcg = root_mem_cgroup;
978 else
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700979 memcg = current->active_memcg;
980 rcu_read_unlock();
981 return memcg;
982 }
983 return get_mem_cgroup_from_mm(current->mm);
984}
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800985
Johannes Weiner56600482012-01-12 17:17:59 -0800986/**
987 * mem_cgroup_iter - iterate over memory cgroup hierarchy
988 * @root: hierarchy root
989 * @prev: previously returned memcg, NULL on first invocation
990 * @reclaim: cookie for shared reclaim walks, NULL for full walks
991 *
992 * Returns references to children of the hierarchy below @root, or
993 * @root itself, or %NULL after a full round-trip.
994 *
995 * Caller must pass the return value in @prev on subsequent
996 * invocations for reference counting, or use mem_cgroup_iter_break()
997 * to cancel a hierarchy walk before the round-trip is complete.
998 *
Honglei Wangb213b542018-03-28 16:01:12 -0700999 * Reclaimers can specify a node and a priority level in @reclaim to
Johannes Weiner56600482012-01-12 17:17:59 -08001000 * divide up the memcgs in the hierarchy among all concurrent
Honglei Wangb213b542018-03-28 16:01:12 -07001001 * reclaimers operating on the same node and priority.
Johannes Weiner56600482012-01-12 17:17:59 -08001002 */
Andrew Morton694fbc02013-09-24 15:27:37 -07001003struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
Johannes Weiner56600482012-01-12 17:17:59 -08001004 struct mem_cgroup *prev,
Andrew Morton694fbc02013-09-24 15:27:37 -07001005 struct mem_cgroup_reclaim_cookie *reclaim)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07001006{
Michal Hocko33398cf2015-09-08 15:01:02 -07001007 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001008 struct cgroup_subsys_state *css = NULL;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001009 struct mem_cgroup *memcg = NULL;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001010 struct mem_cgroup *pos = NULL;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001011
Andrew Morton694fbc02013-09-24 15:27:37 -07001012 if (mem_cgroup_disabled())
1013 return NULL;
Johannes Weiner56600482012-01-12 17:17:59 -08001014
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001015 if (!root)
1016 root = root_mem_cgroup;
1017
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001018 if (prev && !reclaim)
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001019 pos = prev;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001020
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001021 if (!root->use_hierarchy && root != root_mem_cgroup) {
1022 if (prev)
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001023 goto out;
Andrew Morton694fbc02013-09-24 15:27:37 -07001024 return root;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001025 }
1026
Michal Hocko542f85f2013-04-29 15:07:15 -07001027 rcu_read_lock();
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001028
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001029 if (reclaim) {
Mel Gormanef8f2322016-07-28 15:46:05 -07001030 struct mem_cgroup_per_node *mz;
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001031
Mel Gormanef8f2322016-07-28 15:46:05 -07001032 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
Yafang Shao9da83f32019-11-30 17:50:03 -08001033 iter = &mz->iter;
Michal Hocko5f578162013-04-29 15:07:17 -07001034
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001035 if (prev && reclaim->generation != iter->generation)
Michal Hocko542f85f2013-04-29 15:07:15 -07001036 goto out_unlock;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001037
Vladimir Davydov6df38682015-12-29 14:54:10 -08001038 while (1) {
Jason Low4db0c3c2015-04-15 16:14:08 -07001039 pos = READ_ONCE(iter->position);
Vladimir Davydov6df38682015-12-29 14:54:10 -08001040 if (!pos || css_tryget(&pos->css))
1041 break;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001042 /*
Vladimir Davydov6df38682015-12-29 14:54:10 -08001043 * css reference reached zero, so iter->position will
1044 * be cleared by ->css_released. However, we should not
1045 * rely on this happening soon, because ->css_released
1046 * is called from a work queue, and by busy-waiting we
1047 * might block it. So we clear iter->position right
1048 * away.
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001049 */
Vladimir Davydov6df38682015-12-29 14:54:10 -08001050 (void)cmpxchg(&iter->position, pos, NULL);
1051 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001052 }
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001053
1054 if (pos)
1055 css = &pos->css;
1056
1057 for (;;) {
1058 css = css_next_descendant_pre(css, &root->css);
1059 if (!css) {
1060 /*
1061 * Reclaimers share the hierarchy walk, and a
1062 * new one might jump in right at the end of
1063 * the hierarchy - make sure they see at least
1064 * one group and restart from the beginning.
1065 */
1066 if (!prev)
1067 continue;
1068 break;
1069 }
1070
1071 /*
1072 * Verify the css and acquire a reference. The root
1073 * is provided by the caller, so we know it's alive
1074 * and kicking, and don't take an extra reference.
1075 */
1076 memcg = mem_cgroup_from_css(css);
1077
1078 if (css == &root->css)
1079 break;
1080
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08001081 if (css_tryget(css))
1082 break;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001083
1084 memcg = NULL;
1085 }
1086
1087 if (reclaim) {
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001088 /*
Vladimir Davydov6df38682015-12-29 14:54:10 -08001089 * The position could have already been updated by a competing
1090 * thread, so check that the value hasn't changed since we read
1091 * it to avoid reclaiming from the same cgroup twice.
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001092 */
Vladimir Davydov6df38682015-12-29 14:54:10 -08001093 (void)cmpxchg(&iter->position, pos, memcg);
1094
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001095 if (pos)
1096 css_put(&pos->css);
1097
1098 if (!memcg)
1099 iter->generation++;
1100 else if (!prev)
1101 reclaim->generation = iter->generation;
1102 }
1103
Michal Hocko542f85f2013-04-29 15:07:15 -07001104out_unlock:
1105 rcu_read_unlock();
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001106out:
Michal Hockoc40046f2013-04-29 15:07:14 -07001107 if (prev && prev != root)
1108 css_put(&prev->css);
1109
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001110 return memcg;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001111}
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001112
Johannes Weiner56600482012-01-12 17:17:59 -08001113/**
1114 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1115 * @root: hierarchy root
1116 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1117 */
1118void mem_cgroup_iter_break(struct mem_cgroup *root,
1119 struct mem_cgroup *prev)
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001120{
1121 if (!root)
1122 root = root_mem_cgroup;
1123 if (prev && prev != root)
1124 css_put(&prev->css);
1125}
1126
Miles Chen54a83d62019-08-13 15:37:28 -07001127static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1128 struct mem_cgroup *dead_memcg)
Vladimir Davydov6df38682015-12-29 14:54:10 -08001129{
Vladimir Davydov6df38682015-12-29 14:54:10 -08001130 struct mem_cgroup_reclaim_iter *iter;
Mel Gormanef8f2322016-07-28 15:46:05 -07001131 struct mem_cgroup_per_node *mz;
1132 int nid;
Vladimir Davydov6df38682015-12-29 14:54:10 -08001133
Miles Chen54a83d62019-08-13 15:37:28 -07001134 for_each_node(nid) {
1135 mz = mem_cgroup_nodeinfo(from, nid);
Yafang Shao9da83f32019-11-30 17:50:03 -08001136 iter = &mz->iter;
1137 cmpxchg(&iter->position, dead_memcg, NULL);
Vladimir Davydov6df38682015-12-29 14:54:10 -08001138 }
1139}
1140
Miles Chen54a83d62019-08-13 15:37:28 -07001141static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1142{
1143 struct mem_cgroup *memcg = dead_memcg;
1144 struct mem_cgroup *last;
1145
1146 do {
1147 __invalidate_reclaim_iterators(memcg, dead_memcg);
1148 last = memcg;
1149 } while ((memcg = parent_mem_cgroup(memcg)));
1150
1151 /*
1152 * When cgruop1 non-hierarchy mode is used,
1153 * parent_mem_cgroup() does not walk all the way up to the
1154 * cgroup root (root_mem_cgroup). So we have to handle
1155 * dead_memcg from cgroup root separately.
1156 */
1157 if (last != root_mem_cgroup)
1158 __invalidate_reclaim_iterators(root_mem_cgroup,
1159 dead_memcg);
1160}
1161
Johannes Weiner925b7672012-01-12 17:18:15 -08001162/**
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001163 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1164 * @memcg: hierarchy root
1165 * @fn: function to call for each task
1166 * @arg: argument passed to @fn
1167 *
1168 * This function iterates over tasks attached to @memcg or to any of its
1169 * descendants and calls @fn for each task. If @fn returns a non-zero
1170 * value, the function breaks the iteration loop and returns the value.
1171 * Otherwise, it will iterate over all tasks and return 0.
1172 *
1173 * This function must not be called for the root memory cgroup.
1174 */
1175int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1176 int (*fn)(struct task_struct *, void *), void *arg)
1177{
1178 struct mem_cgroup *iter;
1179 int ret = 0;
1180
1181 BUG_ON(memcg == root_mem_cgroup);
1182
1183 for_each_mem_cgroup_tree(iter, memcg) {
1184 struct css_task_iter it;
1185 struct task_struct *task;
1186
Tetsuo Handaf168a9a2019-07-11 21:00:20 -07001187 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001188 while (!ret && (task = css_task_iter_next(&it)))
1189 ret = fn(task, arg);
1190 css_task_iter_end(&it);
1191 if (ret) {
1192 mem_cgroup_iter_break(memcg, iter);
1193 break;
1194 }
1195 }
1196 return ret;
1197}
1198
1199/**
Johannes Weinerdfe0e772014-12-10 15:43:43 -08001200 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
Johannes Weiner925b7672012-01-12 17:18:15 -08001201 * @page: the page
Mike Rapoportf144c392018-02-06 15:42:16 -08001202 * @pgdat: pgdat of the page
Johannes Weinerdfe0e772014-12-10 15:43:43 -08001203 *
Johannes Weinera0b5b412020-06-03 16:02:27 -07001204 * This function relies on page->mem_cgroup being stable - see the
1205 * access rules in commit_charge().
Minchan Kim3f58a822011-03-22 16:32:53 -07001206 */
Mel Gorman599d0c92016-07-28 15:45:31 -07001207struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
Minchan Kim3f58a822011-03-22 16:32:53 -07001208{
Mel Gormanef8f2322016-07-28 15:46:05 -07001209 struct mem_cgroup_per_node *mz;
Johannes Weiner925b7672012-01-12 17:18:15 -08001210 struct mem_cgroup *memcg;
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001211 struct lruvec *lruvec;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001212
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001213 if (mem_cgroup_disabled()) {
Johannes Weiner867e5e12019-11-30 17:55:34 -08001214 lruvec = &pgdat->__lruvec;
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001215 goto out;
1216 }
Christoph Lameterb69408e2008-10-18 20:26:14 -07001217
Johannes Weiner1306a852014-12-10 15:44:52 -08001218 memcg = page->mem_cgroup;
Hugh Dickins75121022012-03-05 14:59:18 -08001219 /*
Johannes Weinerdfe0e772014-12-10 15:43:43 -08001220 * Swapcache readahead pages are added to the LRU - and
Johannes Weiner29833312014-12-10 15:44:02 -08001221 * possibly migrated - before they are charged.
Hugh Dickins75121022012-03-05 14:59:18 -08001222 */
Johannes Weiner29833312014-12-10 15:44:02 -08001223 if (!memcg)
1224 memcg = root_mem_cgroup;
Hugh Dickins75121022012-03-05 14:59:18 -08001225
Mel Gormanef8f2322016-07-28 15:46:05 -07001226 mz = mem_cgroup_page_nodeinfo(memcg, page);
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001227 lruvec = &mz->lruvec;
1228out:
1229 /*
1230 * Since a node can be onlined after the mem_cgroup was created,
1231 * we have to be prepared to initialize lruvec->zone here;
1232 * and if offlined then reonlined, we need to reinitialize it.
1233 */
Mel Gorman599d0c92016-07-28 15:45:31 -07001234 if (unlikely(lruvec->pgdat != pgdat))
1235 lruvec->pgdat = pgdat;
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001236 return lruvec;
Johannes Weiner925b7672012-01-12 17:18:15 -08001237}
1238
1239/**
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001240 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1241 * @lruvec: mem_cgroup per zone lru vector
1242 * @lru: index of lru list the page is sitting on
Michal Hockob4536f0c82017-01-10 16:58:04 -08001243 * @zid: zone id of the accounted pages
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001244 * @nr_pages: positive when adding or negative when removing
Johannes Weiner925b7672012-01-12 17:18:15 -08001245 *
Hugh Dickinsca707232016-05-19 17:12:35 -07001246 * This function must be called under lru_lock, just before a page is added
1247 * to or just after a page is removed from an lru list (that ordering being
1248 * so as to allow it to check that lru_size 0 is consistent with list_empty).
Johannes Weiner925b7672012-01-12 17:18:15 -08001249 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001250void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
Michal Hockob4536f0c82017-01-10 16:58:04 -08001251 int zid, int nr_pages)
Johannes Weiner925b7672012-01-12 17:18:15 -08001252{
Mel Gormanef8f2322016-07-28 15:46:05 -07001253 struct mem_cgroup_per_node *mz;
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001254 unsigned long *lru_size;
Hugh Dickinsca707232016-05-19 17:12:35 -07001255 long size;
Johannes Weiner925b7672012-01-12 17:18:15 -08001256
1257 if (mem_cgroup_disabled())
1258 return;
1259
Mel Gormanef8f2322016-07-28 15:46:05 -07001260 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Michal Hockob4536f0c82017-01-10 16:58:04 -08001261 lru_size = &mz->lru_zone_size[zid][lru];
Hugh Dickinsca707232016-05-19 17:12:35 -07001262
1263 if (nr_pages < 0)
1264 *lru_size += nr_pages;
1265
1266 size = *lru_size;
Michal Hockob4536f0c82017-01-10 16:58:04 -08001267 if (WARN_ONCE(size < 0,
1268 "%s(%p, %d, %d): lru_size %ld\n",
1269 __func__, lruvec, lru, nr_pages, size)) {
Hugh Dickinsca707232016-05-19 17:12:35 -07001270 VM_BUG_ON(1);
1271 *lru_size = 0;
1272 }
1273
1274 if (nr_pages > 0)
1275 *lru_size += nr_pages;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001276}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001277
Johannes Weiner19942822011-02-01 15:52:43 -08001278/**
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001279 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
Wanpeng Lidad75572012-06-20 12:53:01 -07001280 * @memcg: the memory cgroup
Johannes Weiner19942822011-02-01 15:52:43 -08001281 *
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001282 * Returns the maximum amount of memory @mem can be charged with, in
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001283 * pages.
Johannes Weiner19942822011-02-01 15:52:43 -08001284 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001285static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
Johannes Weiner19942822011-02-01 15:52:43 -08001286{
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001287 unsigned long margin = 0;
1288 unsigned long count;
1289 unsigned long limit;
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001290
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001291 count = page_counter_read(&memcg->memory);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001292 limit = READ_ONCE(memcg->memory.max);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001293 if (count < limit)
1294 margin = limit - count;
1295
Johannes Weiner7941d212016-01-14 15:21:23 -08001296 if (do_memsw_account()) {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001297 count = page_counter_read(&memcg->memsw);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001298 limit = READ_ONCE(memcg->memsw.max);
Kaixu Xia1c4448e2020-06-01 21:49:36 -07001299 if (count < limit)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001300 margin = min(margin, limit - count);
Li RongQingcbedbac2016-05-27 14:27:43 -07001301 else
1302 margin = 0;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001303 }
1304
1305 return margin;
Johannes Weiner19942822011-02-01 15:52:43 -08001306}
1307
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001308/*
Qiang Huangbdcbb652014-06-04 16:08:21 -07001309 * A routine for checking "mem" is under move_account() or not.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001310 *
Qiang Huangbdcbb652014-06-04 16:08:21 -07001311 * Checking a cgroup is mc.from or mc.to or under hierarchy of
1312 * moving cgroups. This is for waiting at high-memory pressure
1313 * caused by "move".
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001314 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001315static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001316{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001317 struct mem_cgroup *from;
1318 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001319 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001320 /*
1321 * Unlike task_move routines, we access mc.to, mc.from not under
1322 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1323 */
1324 spin_lock(&mc.lock);
1325 from = mc.from;
1326 to = mc.to;
1327 if (!from)
1328 goto unlock;
Michal Hocko3e920412011-07-26 16:08:29 -07001329
Johannes Weiner2314b422014-12-10 15:44:33 -08001330 ret = mem_cgroup_is_descendant(from, memcg) ||
1331 mem_cgroup_is_descendant(to, memcg);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001332unlock:
1333 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001334 return ret;
1335}
1336
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001337static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001338{
1339 if (mc.moving_task && current != mc.moving_task) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001340 if (mem_cgroup_under_move(memcg)) {
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001341 DEFINE_WAIT(wait);
1342 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1343 /* moving charge context might have finished. */
1344 if (mc.moving_task)
1345 schedule();
1346 finish_wait(&mc.waitq, &wait);
1347 return true;
1348 }
1349 }
1350 return false;
1351}
1352
Johannes Weinerc8713d02019-07-11 20:55:59 -07001353static char *memory_stat_format(struct mem_cgroup *memcg)
1354{
1355 struct seq_buf s;
1356 int i;
Johannes Weiner71cd3112017-05-03 14:55:13 -07001357
Johannes Weinerc8713d02019-07-11 20:55:59 -07001358 seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
1359 if (!s.buffer)
1360 return NULL;
1361
1362 /*
1363 * Provide statistics on the state of the memory subsystem as
1364 * well as cumulative event counters that show past behavior.
1365 *
1366 * This list is ordered following a combination of these gradients:
1367 * 1) generic big picture -> specifics and details
1368 * 2) reflecting userspace activity -> reflecting kernel heuristics
1369 *
1370 * Current memory state:
1371 */
1372
1373 seq_buf_printf(&s, "anon %llu\n",
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07001374 (u64)memcg_page_state(memcg, NR_ANON_MAPPED) *
Johannes Weinerc8713d02019-07-11 20:55:59 -07001375 PAGE_SIZE);
1376 seq_buf_printf(&s, "file %llu\n",
Johannes Weiner0d1c2072020-06-03 16:01:54 -07001377 (u64)memcg_page_state(memcg, NR_FILE_PAGES) *
Johannes Weinerc8713d02019-07-11 20:55:59 -07001378 PAGE_SIZE);
1379 seq_buf_printf(&s, "kernel_stack %llu\n",
1380 (u64)memcg_page_state(memcg, MEMCG_KERNEL_STACK_KB) *
1381 1024);
1382 seq_buf_printf(&s, "slab %llu\n",
1383 (u64)(memcg_page_state(memcg, NR_SLAB_RECLAIMABLE) +
1384 memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE)) *
1385 PAGE_SIZE);
1386 seq_buf_printf(&s, "sock %llu\n",
1387 (u64)memcg_page_state(memcg, MEMCG_SOCK) *
1388 PAGE_SIZE);
1389
1390 seq_buf_printf(&s, "shmem %llu\n",
1391 (u64)memcg_page_state(memcg, NR_SHMEM) *
1392 PAGE_SIZE);
1393 seq_buf_printf(&s, "file_mapped %llu\n",
1394 (u64)memcg_page_state(memcg, NR_FILE_MAPPED) *
1395 PAGE_SIZE);
1396 seq_buf_printf(&s, "file_dirty %llu\n",
1397 (u64)memcg_page_state(memcg, NR_FILE_DIRTY) *
1398 PAGE_SIZE);
1399 seq_buf_printf(&s, "file_writeback %llu\n",
1400 (u64)memcg_page_state(memcg, NR_WRITEBACK) *
1401 PAGE_SIZE);
1402
Johannes Weiner468c3982020-06-03 16:02:01 -07001403#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Johannes Weinerc8713d02019-07-11 20:55:59 -07001404 seq_buf_printf(&s, "anon_thp %llu\n",
Johannes Weiner468c3982020-06-03 16:02:01 -07001405 (u64)memcg_page_state(memcg, NR_ANON_THPS) *
1406 HPAGE_PMD_SIZE);
1407#endif
Johannes Weinerc8713d02019-07-11 20:55:59 -07001408
1409 for (i = 0; i < NR_LRU_LISTS; i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001410 seq_buf_printf(&s, "%s %llu\n", lru_list_name(i),
Johannes Weinerc8713d02019-07-11 20:55:59 -07001411 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
1412 PAGE_SIZE);
1413
1414 seq_buf_printf(&s, "slab_reclaimable %llu\n",
1415 (u64)memcg_page_state(memcg, NR_SLAB_RECLAIMABLE) *
1416 PAGE_SIZE);
1417 seq_buf_printf(&s, "slab_unreclaimable %llu\n",
1418 (u64)memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE) *
1419 PAGE_SIZE);
1420
1421 /* Accumulated memory events */
1422
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001423 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT),
1424 memcg_events(memcg, PGFAULT));
1425 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT),
1426 memcg_events(memcg, PGMAJFAULT));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001427
1428 seq_buf_printf(&s, "workingset_refault %lu\n",
1429 memcg_page_state(memcg, WORKINGSET_REFAULT));
1430 seq_buf_printf(&s, "workingset_activate %lu\n",
1431 memcg_page_state(memcg, WORKINGSET_ACTIVATE));
Yafang Shaoa6f55762020-06-01 21:49:32 -07001432 seq_buf_printf(&s, "workingset_restore %lu\n",
1433 memcg_page_state(memcg, WORKINGSET_RESTORE));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001434 seq_buf_printf(&s, "workingset_nodereclaim %lu\n",
1435 memcg_page_state(memcg, WORKINGSET_NODERECLAIM));
1436
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001437 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGREFILL),
1438 memcg_events(memcg, PGREFILL));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001439 seq_buf_printf(&s, "pgscan %lu\n",
1440 memcg_events(memcg, PGSCAN_KSWAPD) +
1441 memcg_events(memcg, PGSCAN_DIRECT));
1442 seq_buf_printf(&s, "pgsteal %lu\n",
1443 memcg_events(memcg, PGSTEAL_KSWAPD) +
1444 memcg_events(memcg, PGSTEAL_DIRECT));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001445 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE),
1446 memcg_events(memcg, PGACTIVATE));
1447 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE),
1448 memcg_events(memcg, PGDEACTIVATE));
1449 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE),
1450 memcg_events(memcg, PGLAZYFREE));
1451 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED),
1452 memcg_events(memcg, PGLAZYFREED));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001453
1454#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001455 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC),
Johannes Weinerc8713d02019-07-11 20:55:59 -07001456 memcg_events(memcg, THP_FAULT_ALLOC));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001457 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC),
Johannes Weinerc8713d02019-07-11 20:55:59 -07001458 memcg_events(memcg, THP_COLLAPSE_ALLOC));
1459#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1460
1461 /* The above should easily fit into one page */
1462 WARN_ON_ONCE(seq_buf_has_overflowed(&s));
1463
1464 return s.buffer;
1465}
Johannes Weiner71cd3112017-05-03 14:55:13 -07001466
Sha Zhengju58cf1882013-02-22 16:32:05 -08001467#define K(x) ((x) << (PAGE_SHIFT-10))
Balbir Singhe2224322009-04-02 16:57:39 -07001468/**
yuzhoujianf0c867d2018-12-28 00:36:10 -08001469 * mem_cgroup_print_oom_context: Print OOM information relevant to
1470 * memory controller.
Balbir Singhe2224322009-04-02 16:57:39 -07001471 * @memcg: The memory cgroup that went over limit
1472 * @p: Task that is going to be killed
1473 *
1474 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1475 * enabled
1476 */
yuzhoujianf0c867d2018-12-28 00:36:10 -08001477void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1478{
1479 rcu_read_lock();
1480
1481 if (memcg) {
1482 pr_cont(",oom_memcg=");
1483 pr_cont_cgroup_path(memcg->css.cgroup);
1484 } else
1485 pr_cont(",global_oom");
1486 if (p) {
1487 pr_cont(",task_memcg=");
1488 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1489 }
1490 rcu_read_unlock();
1491}
1492
1493/**
1494 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1495 * memory controller.
1496 * @memcg: The memory cgroup that went over limit
1497 */
1498void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
Balbir Singhe2224322009-04-02 16:57:39 -07001499{
Johannes Weinerc8713d02019-07-11 20:55:59 -07001500 char *buf;
Balbir Singhe2224322009-04-02 16:57:39 -07001501
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001502 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1503 K((u64)page_counter_read(&memcg->memory)),
Chris Down15b42562020-04-01 21:07:20 -07001504 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001505 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1506 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1507 K((u64)page_counter_read(&memcg->swap)),
Chris Down32d087c2020-04-01 21:07:30 -07001508 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001509 else {
1510 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1511 K((u64)page_counter_read(&memcg->memsw)),
1512 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1513 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1514 K((u64)page_counter_read(&memcg->kmem)),
1515 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
Sha Zhengju58cf1882013-02-22 16:32:05 -08001516 }
Johannes Weinerc8713d02019-07-11 20:55:59 -07001517
1518 pr_info("Memory cgroup stats for ");
1519 pr_cont_cgroup_path(memcg->css.cgroup);
1520 pr_cont(":");
1521 buf = memory_stat_format(memcg);
1522 if (!buf)
1523 return;
1524 pr_info("%s", buf);
1525 kfree(buf);
Balbir Singhe2224322009-04-02 16:57:39 -07001526}
1527
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001528/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001529 * Return the memory (and swap, if configured) limit for a memcg.
1530 */
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001531unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
David Rientjesa63d83f2010-08-09 17:19:46 -07001532{
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001533 unsigned long max;
David Rientjesa63d83f2010-08-09 17:19:46 -07001534
Chris Down15b42562020-04-01 21:07:20 -07001535 max = READ_ONCE(memcg->memory.max);
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001536 if (mem_cgroup_swappiness(memcg)) {
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001537 unsigned long memsw_max;
1538 unsigned long swap_max;
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001539
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001540 memsw_max = memcg->memsw.max;
Chris Down32d087c2020-04-01 21:07:30 -07001541 swap_max = READ_ONCE(memcg->swap.max);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001542 swap_max = min(swap_max, (unsigned long)total_swap_pages);
1543 max = min(max + swap_max, memsw_max);
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001544 }
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001545 return max;
David Rientjesa63d83f2010-08-09 17:19:46 -07001546}
1547
Chris Down9783aa92019-10-06 17:58:32 -07001548unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1549{
1550 return page_counter_read(&memcg->memory);
1551}
1552
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07001553static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
David Rientjes19965462012-12-11 16:00:26 -08001554 int order)
David Rientjes9cbb78b2012-07-31 16:43:44 -07001555{
David Rientjes6e0fc462015-09-08 15:00:36 -07001556 struct oom_control oc = {
1557 .zonelist = NULL,
1558 .nodemask = NULL,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07001559 .memcg = memcg,
David Rientjes6e0fc462015-09-08 15:00:36 -07001560 .gfp_mask = gfp_mask,
1561 .order = order,
David Rientjes6e0fc462015-09-08 15:00:36 -07001562 };
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001563 bool ret;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001564
Tetsuo Handa7775fac2019-03-05 15:46:47 -08001565 if (mutex_lock_killable(&oom_lock))
1566 return true;
1567 /*
1568 * A few threads which were not waiting at mutex_lock_killable() can
1569 * fail to bail out. Therefore, check again after holding oom_lock.
1570 */
1571 ret = should_force_charge() || out_of_memory(&oc);
Johannes Weinerdc564012015-06-24 16:57:19 -07001572 mutex_unlock(&oom_lock);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001573 return ret;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001574}
1575
Andrew Morton0608f432013-09-24 15:27:41 -07001576static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
Mel Gormanef8f2322016-07-28 15:46:05 -07001577 pg_data_t *pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07001578 gfp_t gfp_mask,
1579 unsigned long *total_scanned)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001580{
Andrew Morton0608f432013-09-24 15:27:41 -07001581 struct mem_cgroup *victim = NULL;
1582 int total = 0;
1583 int loop = 0;
1584 unsigned long excess;
1585 unsigned long nr_scanned;
1586 struct mem_cgroup_reclaim_cookie reclaim = {
Mel Gormanef8f2322016-07-28 15:46:05 -07001587 .pgdat = pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07001588 };
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001589
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001590 excess = soft_limit_excess(root_memcg);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001591
Andrew Morton0608f432013-09-24 15:27:41 -07001592 while (1) {
1593 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1594 if (!victim) {
1595 loop++;
1596 if (loop >= 2) {
1597 /*
1598 * If we have not been able to reclaim
1599 * anything, it might because there are
1600 * no reclaimable pages under this hierarchy
1601 */
1602 if (!total)
1603 break;
1604 /*
1605 * We want to do more targeted reclaim.
1606 * excess >> 2 is not to excessive so as to
1607 * reclaim too much, nor too less that we keep
1608 * coming back to reclaim from this cgroup
1609 */
1610 if (total >= (excess >> 2) ||
1611 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1612 break;
1613 }
1614 continue;
1615 }
Mel Gormana9dd0a82016-07-28 15:46:02 -07001616 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
Mel Gormanef8f2322016-07-28 15:46:05 -07001617 pgdat, &nr_scanned);
Andrew Morton0608f432013-09-24 15:27:41 -07001618 *total_scanned += nr_scanned;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001619 if (!soft_limit_excess(root_memcg))
Andrew Morton0608f432013-09-24 15:27:41 -07001620 break;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001621 }
Andrew Morton0608f432013-09-24 15:27:41 -07001622 mem_cgroup_iter_break(root_memcg, victim);
1623 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001624}
1625
Johannes Weiner0056f4e2013-10-31 16:34:14 -07001626#ifdef CONFIG_LOCKDEP
1627static struct lockdep_map memcg_oom_lock_dep_map = {
1628 .name = "memcg_oom_lock",
1629};
1630#endif
1631
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001632static DEFINE_SPINLOCK(memcg_oom_lock);
1633
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001634/*
1635 * Check OOM-Killer is already running under our hierarchy.
1636 * If someone is running, return false.
1637 */
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001638static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001639{
Michal Hocko79dfdac2011-07-26 16:08:23 -07001640 struct mem_cgroup *iter, *failed = NULL;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001641
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001642 spin_lock(&memcg_oom_lock);
1643
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001644 for_each_mem_cgroup_tree(iter, memcg) {
Johannes Weiner23751be2011-08-25 15:59:16 -07001645 if (iter->oom_lock) {
Michal Hocko79dfdac2011-07-26 16:08:23 -07001646 /*
1647 * this subtree of our hierarchy is already locked
1648 * so we cannot give a lock.
1649 */
Michal Hocko79dfdac2011-07-26 16:08:23 -07001650 failed = iter;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001651 mem_cgroup_iter_break(memcg, iter);
1652 break;
Johannes Weiner23751be2011-08-25 15:59:16 -07001653 } else
1654 iter->oom_lock = true;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001655 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001656
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001657 if (failed) {
1658 /*
1659 * OK, we failed to lock the whole subtree so we have
1660 * to clean up what we set up to the failing subtree
1661 */
1662 for_each_mem_cgroup_tree(iter, memcg) {
1663 if (iter == failed) {
1664 mem_cgroup_iter_break(memcg, iter);
1665 break;
1666 }
1667 iter->oom_lock = false;
Michal Hocko79dfdac2011-07-26 16:08:23 -07001668 }
Johannes Weiner0056f4e2013-10-31 16:34:14 -07001669 } else
1670 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001671
1672 spin_unlock(&memcg_oom_lock);
1673
1674 return !failed;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001675}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001676
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001677static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001678{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001679 struct mem_cgroup *iter;
1680
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001681 spin_lock(&memcg_oom_lock);
Qian Cai5facae42019-09-19 12:09:40 -04001682 mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001683 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001684 iter->oom_lock = false;
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001685 spin_unlock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07001686}
1687
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001688static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001689{
1690 struct mem_cgroup *iter;
1691
Tejun Heoc2b42d32015-06-24 16:58:23 -07001692 spin_lock(&memcg_oom_lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001693 for_each_mem_cgroup_tree(iter, memcg)
Tejun Heoc2b42d32015-06-24 16:58:23 -07001694 iter->under_oom++;
1695 spin_unlock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07001696}
1697
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001698static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001699{
1700 struct mem_cgroup *iter;
1701
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001702 /*
1703 * When a new child is created while the hierarchy is under oom,
Tejun Heoc2b42d32015-06-24 16:58:23 -07001704 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001705 */
Tejun Heoc2b42d32015-06-24 16:58:23 -07001706 spin_lock(&memcg_oom_lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001707 for_each_mem_cgroup_tree(iter, memcg)
Tejun Heoc2b42d32015-06-24 16:58:23 -07001708 if (iter->under_oom > 0)
1709 iter->under_oom--;
1710 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001711}
1712
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001713static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1714
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001715struct oom_wait_info {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001716 struct mem_cgroup *memcg;
Ingo Molnarac6424b2017-06-20 12:06:13 +02001717 wait_queue_entry_t wait;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001718};
1719
Ingo Molnarac6424b2017-06-20 12:06:13 +02001720static int memcg_oom_wake_function(wait_queue_entry_t *wait,
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001721 unsigned mode, int sync, void *arg)
1722{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001723 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1724 struct mem_cgroup *oom_wait_memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001725 struct oom_wait_info *oom_wait_info;
1726
1727 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001728 oom_wait_memcg = oom_wait_info->memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001729
Johannes Weiner2314b422014-12-10 15:44:33 -08001730 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1731 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001732 return 0;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001733 return autoremove_wake_function(wait, mode, sync, arg);
1734}
1735
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001736static void memcg_oom_recover(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001737{
Tejun Heoc2b42d32015-06-24 16:58:23 -07001738 /*
1739 * For the following lockless ->under_oom test, the only required
1740 * guarantee is that it must see the state asserted by an OOM when
1741 * this function is called as a result of userland actions
1742 * triggered by the notification of the OOM. This is trivially
1743 * achieved by invoking mem_cgroup_mark_under_oom() before
1744 * triggering notification.
1745 */
1746 if (memcg && memcg->under_oom)
Tejun Heof4b90b702015-06-24 16:58:21 -07001747 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001748}
1749
Michal Hocko29ef6802018-08-17 15:47:11 -07001750enum oom_status {
1751 OOM_SUCCESS,
1752 OOM_FAILED,
1753 OOM_ASYNC,
1754 OOM_SKIPPED
1755};
1756
1757static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001758{
Michal Hocko7056d3a2018-12-28 00:39:57 -08001759 enum oom_status ret;
1760 bool locked;
1761
Michal Hocko29ef6802018-08-17 15:47:11 -07001762 if (order > PAGE_ALLOC_COSTLY_ORDER)
1763 return OOM_SKIPPED;
1764
Roman Gushchin7a1adfd2018-10-26 15:09:48 -07001765 memcg_memory_event(memcg, MEMCG_OOM);
1766
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001767 /*
Johannes Weiner494264202013-10-16 13:46:59 -07001768 * We are in the middle of the charge context here, so we
1769 * don't want to block when potentially sitting on a callstack
1770 * that holds all kinds of filesystem and mm locks.
1771 *
Michal Hocko29ef6802018-08-17 15:47:11 -07001772 * cgroup1 allows disabling the OOM killer and waiting for outside
1773 * handling until the charge can succeed; remember the context and put
1774 * the task to sleep at the end of the page fault when all locks are
1775 * released.
Johannes Weiner494264202013-10-16 13:46:59 -07001776 *
Michal Hocko29ef6802018-08-17 15:47:11 -07001777 * On the other hand, in-kernel OOM killer allows for an async victim
1778 * memory reclaim (oom_reaper) and that means that we are not solely
1779 * relying on the oom victim to make a forward progress and we can
1780 * invoke the oom killer here.
1781 *
1782 * Please note that mem_cgroup_out_of_memory might fail to find a
1783 * victim and then we have to bail out from the charge path.
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001784 */
Michal Hocko29ef6802018-08-17 15:47:11 -07001785 if (memcg->oom_kill_disable) {
1786 if (!current->in_user_fault)
1787 return OOM_SKIPPED;
1788 css_get(&memcg->css);
1789 current->memcg_in_oom = memcg;
1790 current->memcg_oom_gfp_mask = mask;
1791 current->memcg_oom_order = order;
1792
1793 return OOM_ASYNC;
1794 }
1795
Michal Hocko7056d3a2018-12-28 00:39:57 -08001796 mem_cgroup_mark_under_oom(memcg);
Michal Hocko29ef6802018-08-17 15:47:11 -07001797
Michal Hocko7056d3a2018-12-28 00:39:57 -08001798 locked = mem_cgroup_oom_trylock(memcg);
1799
1800 if (locked)
1801 mem_cgroup_oom_notify(memcg);
1802
1803 mem_cgroup_unmark_under_oom(memcg);
1804 if (mem_cgroup_out_of_memory(memcg, mask, order))
1805 ret = OOM_SUCCESS;
1806 else
1807 ret = OOM_FAILED;
1808
1809 if (locked)
1810 mem_cgroup_oom_unlock(memcg);
1811
1812 return ret;
Johannes Weiner494264202013-10-16 13:46:59 -07001813}
1814
1815/**
1816 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1817 * @handle: actually kill/wait or just clean up the OOM state
1818 *
1819 * This has to be called at the end of a page fault if the memcg OOM
1820 * handler was enabled.
1821 *
1822 * Memcg supports userspace OOM handling where failed allocations must
1823 * sleep on a waitqueue until the userspace task resolves the
1824 * situation. Sleeping directly in the charge context with all kinds
1825 * of locks held is not a good idea, instead we remember an OOM state
1826 * in the task and mem_cgroup_oom_synchronize() has to be called at
1827 * the end of the page fault to complete the OOM handling.
1828 *
1829 * Returns %true if an ongoing memcg OOM situation was detected and
1830 * completed, %false otherwise.
1831 */
1832bool mem_cgroup_oom_synchronize(bool handle)
1833{
Tejun Heo626ebc42015-11-05 18:46:09 -08001834 struct mem_cgroup *memcg = current->memcg_in_oom;
Johannes Weiner494264202013-10-16 13:46:59 -07001835 struct oom_wait_info owait;
1836 bool locked;
1837
1838 /* OOM is global, do not handle */
1839 if (!memcg)
1840 return false;
1841
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001842 if (!handle)
Johannes Weiner494264202013-10-16 13:46:59 -07001843 goto cleanup;
1844
1845 owait.memcg = memcg;
1846 owait.wait.flags = 0;
1847 owait.wait.func = memcg_oom_wake_function;
1848 owait.wait.private = current;
Ingo Molnar2055da92017-06-20 12:06:46 +02001849 INIT_LIST_HEAD(&owait.wait.entry);
Johannes Weiner494264202013-10-16 13:46:59 -07001850
1851 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001852 mem_cgroup_mark_under_oom(memcg);
1853
1854 locked = mem_cgroup_oom_trylock(memcg);
1855
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001856 if (locked)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001857 mem_cgroup_oom_notify(memcg);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001858
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001859 if (locked && !memcg->oom_kill_disable) {
1860 mem_cgroup_unmark_under_oom(memcg);
Johannes Weiner494264202013-10-16 13:46:59 -07001861 finish_wait(&memcg_oom_waitq, &owait.wait);
Tejun Heo626ebc42015-11-05 18:46:09 -08001862 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1863 current->memcg_oom_order);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001864 } else {
Johannes Weiner3812c8c2013-09-12 15:13:44 -07001865 schedule();
Johannes Weiner494264202013-10-16 13:46:59 -07001866 mem_cgroup_unmark_under_oom(memcg);
1867 finish_wait(&memcg_oom_waitq, &owait.wait);
1868 }
1869
1870 if (locked) {
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001871 mem_cgroup_oom_unlock(memcg);
1872 /*
1873 * There is no guarantee that an OOM-lock contender
1874 * sees the wakeups triggered by the OOM kill
1875 * uncharges. Wake any sleepers explicitely.
1876 */
1877 memcg_oom_recover(memcg);
1878 }
Johannes Weiner494264202013-10-16 13:46:59 -07001879cleanup:
Tejun Heo626ebc42015-11-05 18:46:09 -08001880 current->memcg_in_oom = NULL;
Johannes Weiner3812c8c2013-09-12 15:13:44 -07001881 css_put(&memcg->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001882 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001883}
1884
Johannes Weinerd7365e72014-10-29 14:50:48 -07001885/**
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07001886 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
1887 * @victim: task to be killed by the OOM killer
1888 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
1889 *
1890 * Returns a pointer to a memory cgroup, which has to be cleaned up
1891 * by killing all belonging OOM-killable tasks.
1892 *
1893 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
1894 */
1895struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
1896 struct mem_cgroup *oom_domain)
1897{
1898 struct mem_cgroup *oom_group = NULL;
1899 struct mem_cgroup *memcg;
1900
1901 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1902 return NULL;
1903
1904 if (!oom_domain)
1905 oom_domain = root_mem_cgroup;
1906
1907 rcu_read_lock();
1908
1909 memcg = mem_cgroup_from_task(victim);
1910 if (memcg == root_mem_cgroup)
1911 goto out;
1912
1913 /*
Roman Gushchin48fe2672020-04-01 21:07:39 -07001914 * If the victim task has been asynchronously moved to a different
1915 * memory cgroup, we might end up killing tasks outside oom_domain.
1916 * In this case it's better to ignore memory.group.oom.
1917 */
1918 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
1919 goto out;
1920
1921 /*
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07001922 * Traverse the memory cgroup hierarchy from the victim task's
1923 * cgroup up to the OOMing cgroup (or root) to find the
1924 * highest-level memory cgroup with oom.group set.
1925 */
1926 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1927 if (memcg->oom_group)
1928 oom_group = memcg;
1929
1930 if (memcg == oom_domain)
1931 break;
1932 }
1933
1934 if (oom_group)
1935 css_get(&oom_group->css);
1936out:
1937 rcu_read_unlock();
1938
1939 return oom_group;
1940}
1941
1942void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1943{
1944 pr_info("Tasks in ");
1945 pr_cont_cgroup_path(memcg->css.cgroup);
1946 pr_cont(" are going to be killed due to memory.oom.group set\n");
1947}
1948
1949/**
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07001950 * lock_page_memcg - lock a page->mem_cgroup binding
1951 * @page: the page
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001952 *
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07001953 * This function protects unlocked LRU pages from being moved to
Johannes Weiner739f79f2017-08-18 15:15:48 -07001954 * another cgroup.
1955 *
1956 * It ensures lifetime of the returned memcg. Caller is responsible
1957 * for the lifetime of the page; __unlock_page_memcg() is available
1958 * when @page might get freed inside the locked section.
Balbir Singhd69b0422009-06-17 16:26:34 -07001959 */
Johannes Weiner739f79f2017-08-18 15:15:48 -07001960struct mem_cgroup *lock_page_memcg(struct page *page)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001961{
Johannes Weiner9da7b522020-06-03 16:01:51 -07001962 struct page *head = compound_head(page); /* rmap on tail pages */
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001963 struct mem_cgroup *memcg;
Johannes Weiner6de22612015-02-11 15:25:01 -08001964 unsigned long flags;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001965
Johannes Weiner6de22612015-02-11 15:25:01 -08001966 /*
1967 * The RCU lock is held throughout the transaction. The fast
1968 * path can get away without acquiring the memcg->move_lock
1969 * because page moving starts with an RCU grace period.
Johannes Weiner739f79f2017-08-18 15:15:48 -07001970 *
1971 * The RCU lock also protects the memcg from being freed when
1972 * the page state that is going to change is the only thing
1973 * preventing the page itself from being freed. E.g. writeback
1974 * doesn't hold a page reference and relies on PG_writeback to
1975 * keep off truncation, migration and so forth.
1976 */
Johannes Weinerd7365e72014-10-29 14:50:48 -07001977 rcu_read_lock();
1978
1979 if (mem_cgroup_disabled())
Johannes Weiner739f79f2017-08-18 15:15:48 -07001980 return NULL;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001981again:
Johannes Weiner9da7b522020-06-03 16:01:51 -07001982 memcg = head->mem_cgroup;
Johannes Weiner29833312014-12-10 15:44:02 -08001983 if (unlikely(!memcg))
Johannes Weiner739f79f2017-08-18 15:15:48 -07001984 return NULL;
Johannes Weinerd7365e72014-10-29 14:50:48 -07001985
Qiang Huangbdcbb652014-06-04 16:08:21 -07001986 if (atomic_read(&memcg->moving_account) <= 0)
Johannes Weiner739f79f2017-08-18 15:15:48 -07001987 return memcg;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001988
Johannes Weiner6de22612015-02-11 15:25:01 -08001989 spin_lock_irqsave(&memcg->move_lock, flags);
Johannes Weiner9da7b522020-06-03 16:01:51 -07001990 if (memcg != head->mem_cgroup) {
Johannes Weiner6de22612015-02-11 15:25:01 -08001991 spin_unlock_irqrestore(&memcg->move_lock, flags);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001992 goto again;
1993 }
Johannes Weiner6de22612015-02-11 15:25:01 -08001994
1995 /*
1996 * When charge migration first begins, we can have locked and
1997 * unlocked page stat updates happening concurrently. Track
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07001998 * the task who has the lock for unlock_page_memcg().
Johannes Weiner6de22612015-02-11 15:25:01 -08001999 */
2000 memcg->move_lock_task = current;
2001 memcg->move_lock_flags = flags;
Johannes Weinerd7365e72014-10-29 14:50:48 -07002002
Johannes Weiner739f79f2017-08-18 15:15:48 -07002003 return memcg;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002004}
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002005EXPORT_SYMBOL(lock_page_memcg);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002006
Johannes Weinerd7365e72014-10-29 14:50:48 -07002007/**
Johannes Weiner739f79f2017-08-18 15:15:48 -07002008 * __unlock_page_memcg - unlock and unpin a memcg
2009 * @memcg: the memcg
2010 *
2011 * Unlock and unpin a memcg returned by lock_page_memcg().
Johannes Weinerd7365e72014-10-29 14:50:48 -07002012 */
Johannes Weiner739f79f2017-08-18 15:15:48 -07002013void __unlock_page_memcg(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002014{
Johannes Weiner6de22612015-02-11 15:25:01 -08002015 if (memcg && memcg->move_lock_task == current) {
2016 unsigned long flags = memcg->move_lock_flags;
2017
2018 memcg->move_lock_task = NULL;
2019 memcg->move_lock_flags = 0;
2020
2021 spin_unlock_irqrestore(&memcg->move_lock, flags);
2022 }
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002023
Johannes Weinerd7365e72014-10-29 14:50:48 -07002024 rcu_read_unlock();
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002025}
Johannes Weiner739f79f2017-08-18 15:15:48 -07002026
2027/**
2028 * unlock_page_memcg - unlock a page->mem_cgroup binding
2029 * @page: the page
2030 */
2031void unlock_page_memcg(struct page *page)
2032{
Johannes Weiner9da7b522020-06-03 16:01:51 -07002033 struct page *head = compound_head(page);
2034
2035 __unlock_page_memcg(head->mem_cgroup);
Johannes Weiner739f79f2017-08-18 15:15:48 -07002036}
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002037EXPORT_SYMBOL(unlock_page_memcg);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002038
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002039struct memcg_stock_pcp {
2040 struct mem_cgroup *cached; /* this never be root cgroup */
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002041 unsigned int nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002042 struct work_struct work;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002043 unsigned long flags;
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -07002044#define FLUSHING_CACHED_CHARGE 0
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002045};
2046static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002047static DEFINE_MUTEX(percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002048
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002049/**
2050 * consume_stock: Try to consume stocked charge on this cpu.
2051 * @memcg: memcg to consume from.
2052 * @nr_pages: how many pages to charge.
2053 *
2054 * The charges will only happen if @memcg matches the current cpu's memcg
2055 * stock, and at least @nr_pages are available in that stock. Failure to
2056 * service an allocation will refill the stock.
2057 *
2058 * returns true if successful, false otherwise.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002059 */
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002060static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002061{
2062 struct memcg_stock_pcp *stock;
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002063 unsigned long flags;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002064 bool ret = false;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002065
Johannes Weinera983b5e2018-01-31 16:16:45 -08002066 if (nr_pages > MEMCG_CHARGE_BATCH)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002067 return ret;
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002068
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002069 local_irq_save(flags);
2070
2071 stock = this_cpu_ptr(&memcg_stock);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002072 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002073 stock->nr_pages -= nr_pages;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002074 ret = true;
2075 }
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002076
2077 local_irq_restore(flags);
2078
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002079 return ret;
2080}
2081
2082/*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002083 * Returns stocks cached in percpu and reset cached information.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002084 */
2085static void drain_stock(struct memcg_stock_pcp *stock)
2086{
2087 struct mem_cgroup *old = stock->cached;
2088
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002089 if (stock->nr_pages) {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002090 page_counter_uncharge(&old->memory, stock->nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002091 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002092 page_counter_uncharge(&old->memsw, stock->nr_pages);
Johannes Weinere8ea14c2014-12-10 15:42:42 -08002093 css_put_many(&old->css, stock->nr_pages);
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002094 stock->nr_pages = 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002095 }
2096 stock->cached = NULL;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002097}
2098
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002099static void drain_local_stock(struct work_struct *dummy)
2100{
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002101 struct memcg_stock_pcp *stock;
2102 unsigned long flags;
2103
Michal Hocko72f01842017-10-03 16:14:53 -07002104 /*
2105 * The only protection from memory hotplug vs. drain_stock races is
2106 * that we always operate on local CPU stock here with IRQ disabled
2107 */
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002108 local_irq_save(flags);
2109
2110 stock = this_cpu_ptr(&memcg_stock);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002111 drain_stock(stock);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002112 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002113
2114 local_irq_restore(flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002115}
2116
2117/*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002118 * Cache charges(val) to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01002119 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002120 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002121static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002122{
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002123 struct memcg_stock_pcp *stock;
2124 unsigned long flags;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002125
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002126 local_irq_save(flags);
2127
2128 stock = this_cpu_ptr(&memcg_stock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002129 if (stock->cached != memcg) { /* reset if necessary */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002130 drain_stock(stock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002131 stock->cached = memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002132 }
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002133 stock->nr_pages += nr_pages;
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002134
Johannes Weinera983b5e2018-01-31 16:16:45 -08002135 if (stock->nr_pages > MEMCG_CHARGE_BATCH)
Roman Gushchin475d0482017-09-08 16:13:09 -07002136 drain_stock(stock);
2137
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002138 local_irq_restore(flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002139}
2140
2141/*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002142 * Drains all per-CPU charge caches for given root_memcg resp. subtree
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002143 * of the hierarchy under it.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002144 */
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002145static void drain_all_stock(struct mem_cgroup *root_memcg)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002146{
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002147 int cpu, curcpu;
Michal Hockod38144b2011-07-26 16:08:28 -07002148
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002149 /* If someone's already draining, avoid adding running more workers. */
2150 if (!mutex_trylock(&percpu_charge_mutex))
2151 return;
Michal Hocko72f01842017-10-03 16:14:53 -07002152 /*
2153 * Notify other cpus that system-wide "drain" is running
2154 * We do not care about races with the cpu hotplug because cpu down
2155 * as well as workers from this path always operate on the local
2156 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2157 */
Johannes Weiner5af12d02011-08-25 15:59:07 -07002158 curcpu = get_cpu();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002159 for_each_online_cpu(cpu) {
2160 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002161 struct mem_cgroup *memcg;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002162 bool flush = false;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002163
Roman Gushchine1a366b2019-09-23 15:34:58 -07002164 rcu_read_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002165 memcg = stock->cached;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002166 if (memcg && stock->nr_pages &&
2167 mem_cgroup_is_descendant(memcg, root_memcg))
2168 flush = true;
2169 rcu_read_unlock();
2170
2171 if (flush &&
2172 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
Michal Hockod1a05b62011-07-26 16:08:27 -07002173 if (cpu == curcpu)
2174 drain_local_stock(&stock->work);
2175 else
2176 schedule_work_on(cpu, &stock->work);
2177 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002178 }
Johannes Weiner5af12d02011-08-25 15:59:07 -07002179 put_cpu();
Michal Hocko9f50fad2011-08-09 11:56:26 +02002180 mutex_unlock(&percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002181}
2182
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01002183static int memcg_hotplug_cpu_dead(unsigned int cpu)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002184{
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002185 struct memcg_stock_pcp *stock;
Johannes Weiner42a30032019-05-14 15:47:12 -07002186 struct mem_cgroup *memcg, *mi;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002187
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002188 stock = &per_cpu(memcg_stock, cpu);
2189 drain_stock(stock);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002190
2191 for_each_mem_cgroup(memcg) {
2192 int i;
2193
2194 for (i = 0; i < MEMCG_NR_STAT; i++) {
2195 int nid;
2196 long x;
2197
Chris Down871789d2019-05-14 15:46:57 -07002198 x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
Johannes Weiner815744d2019-06-13 15:55:46 -07002199 if (x)
Johannes Weiner42a30032019-05-14 15:47:12 -07002200 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
2201 atomic_long_add(x, &memcg->vmstats[i]);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002202
2203 if (i >= NR_VM_NODE_STAT_ITEMS)
2204 continue;
2205
2206 for_each_node(nid) {
2207 struct mem_cgroup_per_node *pn;
2208
2209 pn = mem_cgroup_nodeinfo(memcg, nid);
2210 x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
Johannes Weiner815744d2019-06-13 15:55:46 -07002211 if (x)
Johannes Weiner42a30032019-05-14 15:47:12 -07002212 do {
2213 atomic_long_add(x, &pn->lruvec_stat[i]);
2214 } while ((pn = parent_nodeinfo(pn, nid)));
Johannes Weinera983b5e2018-01-31 16:16:45 -08002215 }
2216 }
2217
Johannes Weinere27be242018-04-10 16:29:45 -07002218 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
Johannes Weinera983b5e2018-01-31 16:16:45 -08002219 long x;
2220
Chris Down871789d2019-05-14 15:46:57 -07002221 x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
Johannes Weiner815744d2019-06-13 15:55:46 -07002222 if (x)
Johannes Weiner42a30032019-05-14 15:47:12 -07002223 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
2224 atomic_long_add(x, &memcg->vmevents[i]);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002225 }
2226 }
2227
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01002228 return 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002229}
2230
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002231static void reclaim_high(struct mem_cgroup *memcg,
2232 unsigned int nr_pages,
2233 gfp_t gfp_mask)
2234{
2235 do {
Jakub Kicinskid1663a92020-06-01 21:49:49 -07002236 if (page_counter_read(&memcg->memory) <=
2237 READ_ONCE(memcg->memory.high))
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002238 continue;
Johannes Weinere27be242018-04-10 16:29:45 -07002239 memcg_memory_event(memcg, MEMCG_HIGH);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002240 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
Chris Down4bf17302020-04-06 20:03:30 -07002241 } while ((memcg = parent_mem_cgroup(memcg)) &&
2242 !mem_cgroup_is_root(memcg));
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002243}
2244
2245static void high_work_func(struct work_struct *work)
2246{
2247 struct mem_cgroup *memcg;
2248
2249 memcg = container_of(work, struct mem_cgroup, high_work);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002250 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002251}
2252
Tejun Heob23afb92015-11-05 18:46:11 -08002253/*
Chris Down0e4b01d2019-09-23 15:34:55 -07002254 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
2255 * enough to still cause a significant slowdown in most cases, while still
2256 * allowing diagnostics and tracing to proceed without becoming stuck.
2257 */
2258#define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
2259
2260/*
2261 * When calculating the delay, we use these either side of the exponentiation to
2262 * maintain precision and scale to a reasonable number of jiffies (see the table
2263 * below.
2264 *
2265 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
2266 * overage ratio to a delay.
2267 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down down the
2268 * proposed penalty in order to reduce to a reasonable number of jiffies, and
2269 * to produce a reasonable delay curve.
2270 *
2271 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
2272 * reasonable delay curve compared to precision-adjusted overage, not
2273 * penalising heavily at first, but still making sure that growth beyond the
2274 * limit penalises misbehaviour cgroups by slowing them down exponentially. For
2275 * example, with a high of 100 megabytes:
2276 *
2277 * +-------+------------------------+
2278 * | usage | time to allocate in ms |
2279 * +-------+------------------------+
2280 * | 100M | 0 |
2281 * | 101M | 6 |
2282 * | 102M | 25 |
2283 * | 103M | 57 |
2284 * | 104M | 102 |
2285 * | 105M | 159 |
2286 * | 106M | 230 |
2287 * | 107M | 313 |
2288 * | 108M | 409 |
2289 * | 109M | 518 |
2290 * | 110M | 639 |
2291 * | 111M | 774 |
2292 * | 112M | 921 |
2293 * | 113M | 1081 |
2294 * | 114M | 1254 |
2295 * | 115M | 1439 |
2296 * | 116M | 1638 |
2297 * | 117M | 1849 |
2298 * | 118M | 2000 |
2299 * | 119M | 2000 |
2300 * | 120M | 2000 |
2301 * +-------+------------------------+
2302 */
2303 #define MEMCG_DELAY_PRECISION_SHIFT 20
2304 #define MEMCG_DELAY_SCALING_SHIFT 14
2305
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002306static u64 calculate_overage(unsigned long usage, unsigned long high)
2307{
2308 u64 overage;
2309
2310 if (usage <= high)
2311 return 0;
2312
2313 /*
2314 * Prevent division by 0 in overage calculation by acting as if
2315 * it was a threshold of 1 page
2316 */
2317 high = max(high, 1UL);
2318
2319 overage = usage - high;
2320 overage <<= MEMCG_DELAY_PRECISION_SHIFT;
2321 return div64_u64(overage, high);
2322}
2323
2324static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2325{
2326 u64 overage, max_overage = 0;
2327
2328 do {
2329 overage = calculate_overage(page_counter_read(&memcg->memory),
Jakub Kicinskid1663a92020-06-01 21:49:49 -07002330 READ_ONCE(memcg->memory.high));
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002331 max_overage = max(overage, max_overage);
2332 } while ((memcg = parent_mem_cgroup(memcg)) &&
2333 !mem_cgroup_is_root(memcg));
2334
2335 return max_overage;
2336}
2337
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002338static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2339{
2340 u64 overage, max_overage = 0;
2341
2342 do {
2343 overage = calculate_overage(page_counter_read(&memcg->swap),
2344 READ_ONCE(memcg->swap.high));
2345 if (overage)
2346 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2347 max_overage = max(overage, max_overage);
2348 } while ((memcg = parent_mem_cgroup(memcg)) &&
2349 !mem_cgroup_is_root(memcg));
2350
2351 return max_overage;
2352}
2353
Chris Down0e4b01d2019-09-23 15:34:55 -07002354/*
Chris Downe26733e2020-03-21 18:22:23 -07002355 * Get the number of jiffies that we should penalise a mischievous cgroup which
2356 * is exceeding its memory.high by checking both it and its ancestors.
Tejun Heob23afb92015-11-05 18:46:11 -08002357 */
Chris Downe26733e2020-03-21 18:22:23 -07002358static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002359 unsigned int nr_pages,
2360 u64 max_overage)
Tejun Heob23afb92015-11-05 18:46:11 -08002361{
Chris Downe26733e2020-03-21 18:22:23 -07002362 unsigned long penalty_jiffies;
Chris Downe26733e2020-03-21 18:22:23 -07002363
2364 if (!max_overage)
2365 return 0;
Chris Down0e4b01d2019-09-23 15:34:55 -07002366
2367 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002368 * We use overage compared to memory.high to calculate the number of
2369 * jiffies to sleep (penalty_jiffies). Ideally this value should be
2370 * fairly lenient on small overages, and increasingly harsh when the
2371 * memcg in question makes it clear that it has no intention of stopping
2372 * its crazy behaviour, so we exponentially increase the delay based on
2373 * overage amount.
2374 */
Chris Downe26733e2020-03-21 18:22:23 -07002375 penalty_jiffies = max_overage * max_overage * HZ;
2376 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2377 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
Chris Down0e4b01d2019-09-23 15:34:55 -07002378
2379 /*
2380 * Factor in the task's own contribution to the overage, such that four
2381 * N-sized allocations are throttled approximately the same as one
2382 * 4N-sized allocation.
2383 *
2384 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
2385 * larger the current charge patch is than that.
2386 */
Jakub Kicinskiff144e62020-06-01 21:49:45 -07002387 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
Chris Downe26733e2020-03-21 18:22:23 -07002388}
2389
2390/*
2391 * Scheduled by try_charge() to be executed from the userland return path
2392 * and reclaims memory over the high limit.
2393 */
2394void mem_cgroup_handle_over_high(void)
2395{
2396 unsigned long penalty_jiffies;
2397 unsigned long pflags;
2398 unsigned int nr_pages = current->memcg_nr_pages_over_high;
2399 struct mem_cgroup *memcg;
2400
2401 if (likely(!nr_pages))
2402 return;
2403
2404 memcg = get_mem_cgroup_from_mm(current->mm);
2405 reclaim_high(memcg, nr_pages, GFP_KERNEL);
2406 current->memcg_nr_pages_over_high = 0;
2407
2408 /*
2409 * memory.high is breached and reclaim is unable to keep up. Throttle
2410 * allocators proactively to slow down excessive growth.
2411 */
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002412 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2413 mem_find_max_overage(memcg));
Chris Down0e4b01d2019-09-23 15:34:55 -07002414
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002415 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2416 swap_find_max_overage(memcg));
2417
Chris Down0e4b01d2019-09-23 15:34:55 -07002418 /*
Jakub Kicinskiff144e62020-06-01 21:49:45 -07002419 * Clamp the max delay per usermode return so as to still keep the
2420 * application moving forwards and also permit diagnostics, albeit
2421 * extremely slowly.
2422 */
2423 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2424
2425 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002426 * Don't sleep if the amount of jiffies this memcg owes us is so low
2427 * that it's not even worth doing, in an attempt to be nice to those who
2428 * go only a small amount over their memory.high value and maybe haven't
2429 * been aggressively reclaimed enough yet.
2430 */
2431 if (penalty_jiffies <= HZ / 100)
2432 goto out;
2433
2434 /*
2435 * If we exit early, we're guaranteed to die (since
2436 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
2437 * need to account for any ill-begotten jiffies to pay them off later.
2438 */
2439 psi_memstall_enter(&pflags);
2440 schedule_timeout_killable(penalty_jiffies);
2441 psi_memstall_leave(&pflags);
2442
2443out:
2444 css_put(&memcg->css);
Tejun Heob23afb92015-11-05 18:46:11 -08002445}
2446
Johannes Weiner00501b52014-08-08 14:19:20 -07002447static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2448 unsigned int nr_pages)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002449{
Johannes Weinera983b5e2018-01-31 16:16:45 -08002450 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
Johannes Weiner9b130612014-08-06 16:05:51 -07002451 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002452 struct mem_cgroup *mem_over_limit;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002453 struct page_counter *counter;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002454 unsigned long nr_reclaimed;
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002455 bool may_swap = true;
2456 bool drained = false;
Michal Hocko29ef6802018-08-17 15:47:11 -07002457 enum oom_status oom_status;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002458
Johannes Weinerce00a962014-09-05 08:43:57 -04002459 if (mem_cgroup_is_root(memcg))
Tejun Heo10d53c72015-11-05 18:46:17 -08002460 return 0;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002461retry:
Michal Hockob6b6cc72014-04-07 15:37:44 -07002462 if (consume_stock(memcg, nr_pages))
Tejun Heo10d53c72015-11-05 18:46:17 -08002463 return 0;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002464
Johannes Weiner7941d212016-01-14 15:21:23 -08002465 if (!do_memsw_account() ||
Johannes Weiner6071ca52015-11-05 18:50:26 -08002466 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2467 if (page_counter_try_charge(&memcg->memory, batch, &counter))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002468 goto done_restock;
Johannes Weiner7941d212016-01-14 15:21:23 -08002469 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002470 page_counter_uncharge(&memcg->memsw, batch);
2471 mem_over_limit = mem_cgroup_from_counter(counter, memory);
Johannes Weiner3fbe7242014-10-09 15:28:54 -07002472 } else {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002473 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002474 may_swap = false;
Johannes Weiner3fbe7242014-10-09 15:28:54 -07002475 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002476
Johannes Weiner6539cc02014-08-06 16:05:42 -07002477 if (batch > nr_pages) {
2478 batch = nr_pages;
2479 goto retry;
2480 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002481
Johannes Weiner06b078fc02014-08-06 16:05:44 -07002482 /*
Johannes Weiner869712f2019-11-05 21:17:13 -08002483 * Memcg doesn't have a dedicated reserve for atomic
2484 * allocations. But like the global atomic pool, we need to
2485 * put the burden of reclaim on regular allocation requests
2486 * and let these go through as privileged allocations.
2487 */
2488 if (gfp_mask & __GFP_ATOMIC)
2489 goto force;
2490
2491 /*
Johannes Weiner06b078fc02014-08-06 16:05:44 -07002492 * Unlike in global OOM situations, memcg is not in a physical
2493 * memory shortage. Allow dying and OOM-killed tasks to
2494 * bypass the last charges so that they can exit quickly and
2495 * free their memory.
2496 */
Tetsuo Handa7775fac2019-03-05 15:46:47 -08002497 if (unlikely(should_force_charge()))
Tejun Heo10d53c72015-11-05 18:46:17 -08002498 goto force;
Johannes Weiner06b078fc02014-08-06 16:05:44 -07002499
Johannes Weiner89a28482016-10-27 17:46:56 -07002500 /*
2501 * Prevent unbounded recursion when reclaim operations need to
2502 * allocate memory. This might exceed the limits temporarily,
2503 * but we prefer facilitating memory reclaim and getting back
2504 * under the limit over triggering OOM kills in these cases.
2505 */
2506 if (unlikely(current->flags & PF_MEMALLOC))
2507 goto force;
2508
Johannes Weiner06b078fc02014-08-06 16:05:44 -07002509 if (unlikely(task_in_memcg_oom(current)))
2510 goto nomem;
2511
Mel Gormand0164ad2015-11-06 16:28:21 -08002512 if (!gfpflags_allow_blocking(gfp_mask))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002513 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002514
Johannes Weinere27be242018-04-10 16:29:45 -07002515 memcg_memory_event(mem_over_limit, MEMCG_MAX);
Johannes Weiner241994ed2015-02-11 15:26:06 -08002516
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002517 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2518 gfp_mask, may_swap);
Johannes Weiner6539cc02014-08-06 16:05:42 -07002519
Johannes Weiner61e02c72014-08-06 16:08:16 -07002520 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
Johannes Weiner6539cc02014-08-06 16:05:42 -07002521 goto retry;
Johannes Weiner28c34c22014-08-06 16:05:47 -07002522
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002523 if (!drained) {
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002524 drain_all_stock(mem_over_limit);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002525 drained = true;
2526 goto retry;
2527 }
2528
Johannes Weiner28c34c22014-08-06 16:05:47 -07002529 if (gfp_mask & __GFP_NORETRY)
2530 goto nomem;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002531 /*
2532 * Even though the limit is exceeded at this point, reclaim
2533 * may have been able to free some pages. Retry the charge
2534 * before killing the task.
2535 *
2536 * Only for regular pages, though: huge pages are rather
2537 * unlikely to succeed so close to the limit, and we fall back
2538 * to regular pages anyway in case of failure.
2539 */
Johannes Weiner61e02c72014-08-06 16:08:16 -07002540 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002541 goto retry;
2542 /*
2543 * At task move, charge accounts can be doubly counted. So, it's
2544 * better to wait until the end of task_move if something is going on.
2545 */
2546 if (mem_cgroup_wait_acct_move(mem_over_limit))
2547 goto retry;
2548
Johannes Weiner9b130612014-08-06 16:05:51 -07002549 if (nr_retries--)
2550 goto retry;
2551
Shakeel Butt38d38492019-07-11 20:55:48 -07002552 if (gfp_mask & __GFP_RETRY_MAYFAIL)
Michal Hocko29ef6802018-08-17 15:47:11 -07002553 goto nomem;
2554
Johannes Weiner06b078fc02014-08-06 16:05:44 -07002555 if (gfp_mask & __GFP_NOFAIL)
Tejun Heo10d53c72015-11-05 18:46:17 -08002556 goto force;
Johannes Weiner06b078fc02014-08-06 16:05:44 -07002557
Johannes Weiner6539cc02014-08-06 16:05:42 -07002558 if (fatal_signal_pending(current))
Tejun Heo10d53c72015-11-05 18:46:17 -08002559 goto force;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002560
Michal Hocko29ef6802018-08-17 15:47:11 -07002561 /*
2562 * keep retrying as long as the memcg oom killer is able to make
2563 * a forward progress or bypass the charge if the oom killer
2564 * couldn't make any progress.
2565 */
2566 oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
Jerome Marchand3608de02015-11-05 18:47:29 -08002567 get_order(nr_pages * PAGE_SIZE));
Michal Hocko29ef6802018-08-17 15:47:11 -07002568 switch (oom_status) {
2569 case OOM_SUCCESS:
2570 nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
Michal Hocko29ef6802018-08-17 15:47:11 -07002571 goto retry;
2572 case OOM_FAILED:
2573 goto force;
2574 default:
2575 goto nomem;
2576 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002577nomem:
Johannes Weiner6d1fdc42014-04-07 15:37:45 -07002578 if (!(gfp_mask & __GFP_NOFAIL))
Johannes Weiner3168ecb2013-10-31 16:34:13 -07002579 return -ENOMEM;
Tejun Heo10d53c72015-11-05 18:46:17 -08002580force:
2581 /*
2582 * The allocation either can't fail or will lead to more memory
2583 * being freed very soon. Allow memory usage go over the limit
2584 * temporarily by force charging it.
2585 */
2586 page_counter_charge(&memcg->memory, nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002587 if (do_memsw_account())
Tejun Heo10d53c72015-11-05 18:46:17 -08002588 page_counter_charge(&memcg->memsw, nr_pages);
2589 css_get_many(&memcg->css, nr_pages);
2590
2591 return 0;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002592
2593done_restock:
Johannes Weinere8ea14c2014-12-10 15:42:42 -08002594 css_get_many(&memcg->css, batch);
Johannes Weiner6539cc02014-08-06 16:05:42 -07002595 if (batch > nr_pages)
2596 refill_stock(memcg, batch - nr_pages);
Tejun Heob23afb92015-11-05 18:46:11 -08002597
Johannes Weiner241994ed2015-02-11 15:26:06 -08002598 /*
Tejun Heob23afb92015-11-05 18:46:11 -08002599 * If the hierarchy is above the normal consumption range, schedule
2600 * reclaim on returning to userland. We can perform reclaim here
Mel Gorman71baba42015-11-06 16:28:28 -08002601 * if __GFP_RECLAIM but let's always punt for simplicity and so that
Tejun Heob23afb92015-11-05 18:46:11 -08002602 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2603 * not recorded as it most likely matches current's and won't
2604 * change in the meantime. As high limit is checked again before
2605 * reclaim, the cost of mismatch is negligible.
Johannes Weiner241994ed2015-02-11 15:26:06 -08002606 */
2607 do {
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002608 bool mem_high, swap_high;
2609
2610 mem_high = page_counter_read(&memcg->memory) >
2611 READ_ONCE(memcg->memory.high);
2612 swap_high = page_counter_read(&memcg->swap) >
2613 READ_ONCE(memcg->swap.high);
2614
2615 /* Don't bother a random interrupted task */
2616 if (in_interrupt()) {
2617 if (mem_high) {
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002618 schedule_work(&memcg->high_work);
2619 break;
2620 }
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002621 continue;
2622 }
2623
2624 if (mem_high || swap_high) {
2625 /*
2626 * The allocating tasks in this cgroup will need to do
2627 * reclaim or be throttled to prevent further growth
2628 * of the memory or swap footprints.
2629 *
2630 * Target some best-effort fairness between the tasks,
2631 * and distribute reclaim work and delay penalties
2632 * based on how much each task is actually allocating.
2633 */
Vladimir Davydov9516a182015-12-11 13:40:24 -08002634 current->memcg_nr_pages_over_high += batch;
Tejun Heob23afb92015-11-05 18:46:11 -08002635 set_notify_resume(current);
2636 break;
2637 }
Johannes Weiner241994ed2015-02-11 15:26:06 -08002638 } while ((memcg = parent_mem_cgroup(memcg)));
Tejun Heo10d53c72015-11-05 18:46:17 -08002639
2640 return 0;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002641}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002642
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07002643#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
Johannes Weiner00501b52014-08-08 14:19:20 -07002644static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002645{
Johannes Weinerce00a962014-09-05 08:43:57 -04002646 if (mem_cgroup_is_root(memcg))
2647 return;
2648
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002649 page_counter_uncharge(&memcg->memory, nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002650 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002651 page_counter_uncharge(&memcg->memsw, nr_pages);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002652
Johannes Weinere8ea14c2014-12-10 15:42:42 -08002653 css_put_many(&memcg->css, nr_pages);
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002654}
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07002655#endif
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002656
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07002657static void commit_charge(struct page *page, struct mem_cgroup *memcg)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002658{
Johannes Weiner1306a852014-12-10 15:44:52 -08002659 VM_BUG_ON_PAGE(page->mem_cgroup, page);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002660 /*
Johannes Weinera0b5b412020-06-03 16:02:27 -07002661 * Any of the following ensures page->mem_cgroup stability:
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002662 *
Johannes Weinera0b5b412020-06-03 16:02:27 -07002663 * - the page lock
2664 * - LRU isolation
2665 * - lock_page_memcg()
2666 * - exclusive reference
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002667 */
Johannes Weiner1306a852014-12-10 15:44:52 -08002668 page->mem_cgroup = memcg;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002669}
2670
Kirill Tkhai84c07d12018-08-17 15:47:25 -07002671#ifdef CONFIG_MEMCG_KMEM
Roman Gushchin8380ce42020-03-28 19:17:25 -07002672/*
2673 * Returns a pointer to the memory cgroup to which the kernel object is charged.
2674 *
2675 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2676 * cgroup_mutex, etc.
2677 */
2678struct mem_cgroup *mem_cgroup_from_obj(void *p)
2679{
2680 struct page *page;
2681
2682 if (mem_cgroup_disabled())
2683 return NULL;
2684
2685 page = virt_to_head_page(p);
2686
2687 /*
2688 * Slab pages don't have page->mem_cgroup set because corresponding
2689 * kmem caches can be reparented during the lifetime. That's why
2690 * memcg_from_slab_page() should be used instead.
2691 */
2692 if (PageSlab(page))
2693 return memcg_from_slab_page(page);
2694
2695 /* All other pages use page->mem_cgroup */
2696 return page->mem_cgroup;
2697}
2698
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002699static int memcg_alloc_cache_id(void)
Glauber Costa55007d82012-12-18 14:22:38 -08002700{
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002701 int id, size;
2702 int err;
Glauber Costa55007d82012-12-18 14:22:38 -08002703
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08002704 id = ida_simple_get(&memcg_cache_ida,
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002705 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2706 if (id < 0)
2707 return id;
2708
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08002709 if (id < memcg_nr_cache_ids)
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002710 return id;
2711
2712 /*
2713 * There's no space for the new id in memcg_caches arrays,
2714 * so we have to grow them.
2715 */
Vladimir Davydov05257a12015-02-12 14:59:01 -08002716 down_write(&memcg_cache_ids_sem);
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002717
2718 size = 2 * (id + 1);
Glauber Costa55007d82012-12-18 14:22:38 -08002719 if (size < MEMCG_CACHES_MIN_SIZE)
2720 size = MEMCG_CACHES_MIN_SIZE;
2721 else if (size > MEMCG_CACHES_MAX_SIZE)
2722 size = MEMCG_CACHES_MAX_SIZE;
2723
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002724 err = memcg_update_all_caches(size);
Vladimir Davydov05257a12015-02-12 14:59:01 -08002725 if (!err)
Vladimir Davydov60d3fd32015-02-12 14:59:10 -08002726 err = memcg_update_all_list_lrus(size);
2727 if (!err)
Vladimir Davydov05257a12015-02-12 14:59:01 -08002728 memcg_nr_cache_ids = size;
2729
2730 up_write(&memcg_cache_ids_sem);
2731
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002732 if (err) {
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08002733 ida_simple_remove(&memcg_cache_ida, id);
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002734 return err;
2735 }
2736 return id;
2737}
2738
2739static void memcg_free_cache_id(int id)
2740{
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08002741 ida_simple_remove(&memcg_cache_ida, id);
Glauber Costa55007d82012-12-18 14:22:38 -08002742}
2743
Vladimir Davydovd5b3cf72015-02-10 14:11:47 -08002744struct memcg_kmem_cache_create_work {
Vladimir Davydov5722d092014-04-07 15:39:24 -07002745 struct mem_cgroup *memcg;
2746 struct kmem_cache *cachep;
2747 struct work_struct work;
2748};
2749
Vladimir Davydovd5b3cf72015-02-10 14:11:47 -08002750static void memcg_kmem_cache_create_func(struct work_struct *w)
Glauber Costad7f25f82012-12-18 14:22:40 -08002751{
Vladimir Davydovd5b3cf72015-02-10 14:11:47 -08002752 struct memcg_kmem_cache_create_work *cw =
2753 container_of(w, struct memcg_kmem_cache_create_work, work);
Vladimir Davydov5722d092014-04-07 15:39:24 -07002754 struct mem_cgroup *memcg = cw->memcg;
2755 struct kmem_cache *cachep = cw->cachep;
Glauber Costad7f25f82012-12-18 14:22:40 -08002756
Vladimir Davydovd5b3cf72015-02-10 14:11:47 -08002757 memcg_create_kmem_cache(memcg, cachep);
Vladimir Davydovbd673142014-06-04 16:07:40 -07002758
Vladimir Davydov5722d092014-04-07 15:39:24 -07002759 css_put(&memcg->css);
Glauber Costad7f25f82012-12-18 14:22:40 -08002760 kfree(cw);
2761}
2762
2763/*
2764 * Enqueue the creation of a per-memcg kmem_cache.
Glauber Costad7f25f82012-12-18 14:22:40 -08002765 */
Shakeel Butt85cfb242018-10-26 15:07:41 -07002766static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
Vladimir Davydovd5b3cf72015-02-10 14:11:47 -08002767 struct kmem_cache *cachep)
Glauber Costad7f25f82012-12-18 14:22:40 -08002768{
Vladimir Davydovd5b3cf72015-02-10 14:11:47 -08002769 struct memcg_kmem_cache_create_work *cw;
Glauber Costad7f25f82012-12-18 14:22:40 -08002770
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002771 if (!css_tryget_online(&memcg->css))
2772 return;
2773
Minchan Kimc892fd82018-04-20 14:56:17 -07002774 cw = kmalloc(sizeof(*cw), GFP_NOWAIT | __GFP_NOWARN);
Vladimir Davydov8135be52014-12-12 16:56:38 -08002775 if (!cw)
Glauber Costad7f25f82012-12-18 14:22:40 -08002776 return;
Vladimir Davydov8135be52014-12-12 16:56:38 -08002777
Glauber Costad7f25f82012-12-18 14:22:40 -08002778 cw->memcg = memcg;
2779 cw->cachep = cachep;
Vladimir Davydovd5b3cf72015-02-10 14:11:47 -08002780 INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
Glauber Costad7f25f82012-12-18 14:22:40 -08002781
Tejun Heo17cc4df2017-02-22 15:41:36 -08002782 queue_work(memcg_kmem_cache_wq, &cw->work);
Glauber Costad7f25f82012-12-18 14:22:40 -08002783}
2784
Vladimir Davydov45264772016-07-26 15:24:21 -07002785static inline bool memcg_kmem_bypass(void)
2786{
Zefan Li50d53d72020-06-01 21:49:55 -07002787 if (in_interrupt())
2788 return true;
2789
2790 /* Allow remote memcg charging in kthread contexts. */
2791 if ((!current->mm || (current->flags & PF_KTHREAD)) &&
2792 !current->active_memcg)
Vladimir Davydov45264772016-07-26 15:24:21 -07002793 return true;
2794 return false;
2795}
2796
2797/**
2798 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2799 * @cachep: the original global kmem cache
2800 *
Glauber Costad7f25f82012-12-18 14:22:40 -08002801 * Return the kmem_cache we're supposed to use for a slab allocation.
2802 * We try to use the current memcg's version of the cache.
2803 *
Vladimir Davydov45264772016-07-26 15:24:21 -07002804 * If the cache does not exist yet, if we are the first user of it, we
2805 * create it asynchronously in a workqueue and let the current allocation
2806 * go through with the original cache.
Glauber Costad7f25f82012-12-18 14:22:40 -08002807 *
Vladimir Davydov45264772016-07-26 15:24:21 -07002808 * This function takes a reference to the cache it returns to assure it
2809 * won't get destroyed while we are working with it. Once the caller is
2810 * done with it, memcg_kmem_put_cache() must be called to release the
2811 * reference.
Glauber Costad7f25f82012-12-18 14:22:40 -08002812 */
Vladimir Davydov45264772016-07-26 15:24:21 -07002813struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
Glauber Costad7f25f82012-12-18 14:22:40 -08002814{
2815 struct mem_cgroup *memcg;
Vladimir Davydov959c8962014-01-23 15:52:59 -08002816 struct kmem_cache *memcg_cachep;
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002817 struct memcg_cache_array *arr;
Vladimir Davydov2a4db7e2015-02-12 14:59:32 -08002818 int kmemcg_id;
Glauber Costad7f25f82012-12-18 14:22:40 -08002819
Vladimir Davydovf7ce3192015-02-12 14:59:20 -08002820 VM_BUG_ON(!is_root_cache(cachep));
Glauber Costad7f25f82012-12-18 14:22:40 -08002821
Vladimir Davydov45264772016-07-26 15:24:21 -07002822 if (memcg_kmem_bypass())
Vladimir Davydov230e9fc2016-01-14 15:18:15 -08002823 return cachep;
2824
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002825 rcu_read_lock();
2826
2827 if (unlikely(current->active_memcg))
2828 memcg = current->active_memcg;
2829 else
2830 memcg = mem_cgroup_from_task(current);
2831
2832 if (!memcg || memcg == root_mem_cgroup)
2833 goto out_unlock;
2834
Jason Low4db0c3c2015-04-15 16:14:08 -07002835 kmemcg_id = READ_ONCE(memcg->kmemcg_id);
Vladimir Davydov2a4db7e2015-02-12 14:59:32 -08002836 if (kmemcg_id < 0)
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002837 goto out_unlock;
Glauber Costad7f25f82012-12-18 14:22:40 -08002838
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002839 arr = rcu_dereference(cachep->memcg_params.memcg_caches);
2840
2841 /*
2842 * Make sure we will access the up-to-date value. The code updating
2843 * memcg_caches issues a write barrier to match the data dependency
2844 * barrier inside READ_ONCE() (see memcg_create_kmem_cache()).
2845 */
2846 memcg_cachep = READ_ONCE(arr->entries[kmemcg_id]);
Li Zefanca0dde92013-04-29 15:08:57 -07002847
2848 /*
2849 * If we are in a safe context (can wait, and not in interrupt
2850 * context), we could be be predictable and return right away.
2851 * This would guarantee that the allocation being performed
2852 * already belongs in the new cache.
2853 *
2854 * However, there are some clashes that can arrive from locking.
2855 * For instance, because we acquire the slab_mutex while doing
Vladimir Davydov776ed0f2014-06-04 16:10:02 -07002856 * memcg_create_kmem_cache, this means no further allocation
2857 * could happen with the slab_mutex held. So it's better to
2858 * defer everything.
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002859 *
2860 * If the memcg is dying or memcg_cache is about to be released,
2861 * don't bother creating new kmem_caches. Because memcg_cachep
2862 * is ZEROed as the fist step of kmem offlining, we don't need
2863 * percpu_ref_tryget_live() here. css_tryget_online() check in
2864 * memcg_schedule_kmem_cache_create() will prevent us from
2865 * creation of a new kmem_cache.
Li Zefanca0dde92013-04-29 15:08:57 -07002866 */
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002867 if (unlikely(!memcg_cachep))
2868 memcg_schedule_kmem_cache_create(memcg, cachep);
2869 else if (percpu_ref_tryget(&memcg_cachep->memcg_params.refcnt))
2870 cachep = memcg_cachep;
2871out_unlock:
2872 rcu_read_unlock();
Li Zefanca0dde92013-04-29 15:08:57 -07002873 return cachep;
Glauber Costad7f25f82012-12-18 14:22:40 -08002874}
Glauber Costad7f25f82012-12-18 14:22:40 -08002875
Vladimir Davydov45264772016-07-26 15:24:21 -07002876/**
2877 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2878 * @cachep: the cache returned by memcg_kmem_get_cache
2879 */
2880void memcg_kmem_put_cache(struct kmem_cache *cachep)
Vladimir Davydov8135be52014-12-12 16:56:38 -08002881{
2882 if (!is_root_cache(cachep))
Roman Gushchinf0a3a242019-07-11 20:56:27 -07002883 percpu_ref_put(&cachep->memcg_params.refcnt);
Vladimir Davydov8135be52014-12-12 16:56:38 -08002884}
2885
Vladimir Davydov45264772016-07-26 15:24:21 -07002886/**
Roman Gushchin4b13f642020-04-01 21:06:56 -07002887 * __memcg_kmem_charge: charge a number of kernel pages to a memcg
Roman Gushchin10eaec22020-04-01 21:06:39 -07002888 * @memcg: memory cgroup to charge
Vladimir Davydov45264772016-07-26 15:24:21 -07002889 * @gfp: reclaim mode
Roman Gushchin92d0510c2020-04-01 21:06:49 -07002890 * @nr_pages: number of pages to charge
Vladimir Davydov45264772016-07-26 15:24:21 -07002891 *
2892 * Returns 0 on success, an error code on failure.
2893 */
Roman Gushchin4b13f642020-04-01 21:06:56 -07002894int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
2895 unsigned int nr_pages)
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002896{
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002897 struct page_counter *counter;
Johannes Weiner6071ca52015-11-05 18:50:26 -08002898 int ret;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002899
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002900 ret = try_charge(memcg, gfp, nr_pages);
Johannes Weiner52c29b02016-01-20 15:02:35 -08002901 if (ret)
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002902 return ret;
Johannes Weiner52c29b02016-01-20 15:02:35 -08002903
2904 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2905 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
Michal Hockoe55d9d92019-09-25 16:45:53 -07002906
2907 /*
2908 * Enforce __GFP_NOFAIL allocation because callers are not
2909 * prepared to see failures and likely do not have any failure
2910 * handling code.
2911 */
2912 if (gfp & __GFP_NOFAIL) {
2913 page_counter_charge(&memcg->kmem, nr_pages);
2914 return 0;
2915 }
Johannes Weiner52c29b02016-01-20 15:02:35 -08002916 cancel_charge(memcg, nr_pages);
2917 return -ENOMEM;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002918 }
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002919 return 0;
2920}
2921
Vladimir Davydov45264772016-07-26 15:24:21 -07002922/**
Roman Gushchin4b13f642020-04-01 21:06:56 -07002923 * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg
2924 * @memcg: memcg to uncharge
2925 * @nr_pages: number of pages to uncharge
2926 */
2927void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
2928{
2929 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2930 page_counter_uncharge(&memcg->kmem, nr_pages);
2931
2932 page_counter_uncharge(&memcg->memory, nr_pages);
2933 if (do_memsw_account())
2934 page_counter_uncharge(&memcg->memsw, nr_pages);
2935}
2936
2937/**
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07002938 * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
Vladimir Davydov45264772016-07-26 15:24:21 -07002939 * @page: page to charge
2940 * @gfp: reclaim mode
2941 * @order: allocation order
2942 *
2943 * Returns 0 on success, an error code on failure.
2944 */
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07002945int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002946{
2947 struct mem_cgroup *memcg;
Vladimir Davydovfcff7d72016-03-17 14:17:29 -07002948 int ret = 0;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002949
Shakeel Butt60cd4bc2019-03-05 15:43:13 -08002950 if (memcg_kmem_bypass())
Vladimir Davydov45264772016-07-26 15:24:21 -07002951 return 0;
2952
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07002953 memcg = get_mem_cgroup_from_current();
Vladimir Davydovc4159a72016-08-08 23:03:12 +03002954 if (!mem_cgroup_is_root(memcg)) {
Roman Gushchin4b13f642020-04-01 21:06:56 -07002955 ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
Roman Gushchin4d96ba32019-07-11 20:56:31 -07002956 if (!ret) {
2957 page->mem_cgroup = memcg;
Vladimir Davydovc4159a72016-08-08 23:03:12 +03002958 __SetPageKmemcg(page);
Roman Gushchin4d96ba32019-07-11 20:56:31 -07002959 }
Vladimir Davydovc4159a72016-08-08 23:03:12 +03002960 }
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002961 css_put(&memcg->css);
Vladimir Davydovd05e83a2015-11-05 18:48:59 -08002962 return ret;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002963}
Roman Gushchin49a18ea2019-07-11 20:56:13 -07002964
2965/**
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07002966 * __memcg_kmem_uncharge_page: uncharge a kmem page
Vladimir Davydov45264772016-07-26 15:24:21 -07002967 * @page: page to uncharge
2968 * @order: allocation order
2969 */
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07002970void __memcg_kmem_uncharge_page(struct page *page, int order)
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002971{
Johannes Weiner1306a852014-12-10 15:44:52 -08002972 struct mem_cgroup *memcg = page->mem_cgroup;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002973 unsigned int nr_pages = 1 << order;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002974
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002975 if (!memcg)
2976 return;
2977
Sasha Levin309381fea2014-01-23 15:52:54 -08002978 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
Roman Gushchin4b13f642020-04-01 21:06:56 -07002979 __memcg_kmem_uncharge(memcg, nr_pages);
Johannes Weiner1306a852014-12-10 15:44:52 -08002980 page->mem_cgroup = NULL;
Vladimir Davydovc4159a72016-08-08 23:03:12 +03002981
2982 /* slab pages do not have PageKmemcg flag set */
2983 if (PageKmemcg(page))
2984 __ClearPageKmemcg(page);
2985
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08002986 css_put_many(&memcg->css, nr_pages);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -08002987}
Kirill Tkhai84c07d12018-08-17 15:47:25 -07002988#endif /* CONFIG_MEMCG_KMEM */
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002989
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002990#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2991
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002992/*
2993 * Because tail pages are not marked as "used", set it. We're under
Andrey Ryabininf4b7e272019-03-05 15:49:39 -08002994 * pgdat->lru_lock and migration entries setup in all page mappings.
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002995 */
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08002996void mem_cgroup_split_huge_fixup(struct page *head)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002997{
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08002998 int i;
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002999
KAMEZAWA Hiroyuki3d37c4a2011-01-25 15:07:28 -08003000 if (mem_cgroup_disabled())
3001 return;
David Rientjesb070e652013-05-07 16:18:09 -07003002
Johannes Weiner29833312014-12-10 15:44:02 -08003003 for (i = 1; i < HPAGE_PMD_NR; i++)
Johannes Weiner1306a852014-12-10 15:44:52 -08003004 head[i].mem_cgroup = head->mem_cgroup;
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003005}
Hugh Dickins12d27102012-01-12 17:19:52 -08003006#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003007
Andrew Mortonc255a452012-07-31 16:43:02 -07003008#ifdef CONFIG_MEMCG_SWAP
Daisuke Nishimura02491442010-03-10 15:22:17 -08003009/**
3010 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
3011 * @entry: swap entry to be moved
3012 * @from: mem_cgroup which the entry is moved from
3013 * @to: mem_cgroup which the entry is moved to
3014 *
3015 * It succeeds only when the swap_cgroup's record for this entry is the same
3016 * as the mem_cgroup's id of @from.
3017 *
3018 * Returns 0 on success, -EINVAL on failure.
3019 *
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003020 * The caller must have charged to @to, IOW, called page_counter_charge() about
Daisuke Nishimura02491442010-03-10 15:22:17 -08003021 * both res and memsw, and called css_get().
3022 */
3023static int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07003024 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003025{
3026 unsigned short old_id, new_id;
3027
Li Zefan34c00c32013-09-23 16:56:01 +08003028 old_id = mem_cgroup_id(from);
3029 new_id = mem_cgroup_id(to);
Daisuke Nishimura02491442010-03-10 15:22:17 -08003030
3031 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Johannes Weinerc9019e92018-01-31 16:16:37 -08003032 mod_memcg_state(from, MEMCG_SWAP, -1);
3033 mod_memcg_state(to, MEMCG_SWAP, 1);
Daisuke Nishimura02491442010-03-10 15:22:17 -08003034 return 0;
3035 }
3036 return -EINVAL;
3037}
3038#else
3039static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07003040 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003041{
3042 return -EINVAL;
3043}
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003044#endif
3045
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003046static DEFINE_MUTEX(memcg_max_mutex);
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07003047
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003048static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3049 unsigned long max, bool memsw)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003050{
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003051 bool enlarge = false;
Shakeel Buttbb4a7ea2018-06-07 17:07:27 -07003052 bool drained = false;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003053 int ret;
Yu Zhaoc054a782018-01-31 16:20:02 -08003054 bool limits_invariant;
3055 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003056
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003057 do {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003058 if (signal_pending(current)) {
3059 ret = -EINTR;
3060 break;
3061 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003062
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003063 mutex_lock(&memcg_max_mutex);
Yu Zhaoc054a782018-01-31 16:20:02 -08003064 /*
3065 * Make sure that the new limit (memsw or memory limit) doesn't
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003066 * break our basic invariant rule memory.max <= memsw.max.
Yu Zhaoc054a782018-01-31 16:20:02 -08003067 */
Chris Down15b42562020-04-01 21:07:20 -07003068 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003069 max <= memcg->memsw.max;
Yu Zhaoc054a782018-01-31 16:20:02 -08003070 if (!limits_invariant) {
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003071 mutex_unlock(&memcg_max_mutex);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003072 ret = -EINVAL;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003073 break;
3074 }
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003075 if (max > counter->max)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003076 enlarge = true;
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003077 ret = page_counter_set_max(counter, max);
3078 mutex_unlock(&memcg_max_mutex);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003079
3080 if (!ret)
3081 break;
3082
Shakeel Buttbb4a7ea2018-06-07 17:07:27 -07003083 if (!drained) {
3084 drain_all_stock(memcg);
3085 drained = true;
3086 continue;
3087 }
3088
Andrey Ryabinin1ab5c052018-01-31 16:20:37 -08003089 if (!try_to_free_mem_cgroup_pages(memcg, 1,
3090 GFP_KERNEL, !memsw)) {
3091 ret = -EBUSY;
3092 break;
3093 }
3094 } while (true);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003095
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003096 if (!ret && enlarge)
3097 memcg_oom_recover(memcg);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003098
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003099 return ret;
3100}
3101
Mel Gormanef8f2322016-07-28 15:46:05 -07003102unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -07003103 gfp_t gfp_mask,
3104 unsigned long *total_scanned)
3105{
3106 unsigned long nr_reclaimed = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003107 struct mem_cgroup_per_node *mz, *next_mz = NULL;
Andrew Morton0608f432013-09-24 15:27:41 -07003108 unsigned long reclaimed;
3109 int loop = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003110 struct mem_cgroup_tree_per_node *mctz;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003111 unsigned long excess;
Andrew Morton0608f432013-09-24 15:27:41 -07003112 unsigned long nr_scanned;
3113
3114 if (order > 0)
3115 return 0;
3116
Mel Gormanef8f2322016-07-28 15:46:05 -07003117 mctz = soft_limit_tree_node(pgdat->node_id);
Michal Hockod6507ff2016-08-02 14:02:37 -07003118
3119 /*
3120 * Do not even bother to check the largest node if the root
3121 * is empty. Do it lockless to prevent lock bouncing. Races
3122 * are acceptable as soft limit is best effort anyway.
3123 */
Laurent Dufourbfc72282017-03-09 16:17:06 -08003124 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
Michal Hockod6507ff2016-08-02 14:02:37 -07003125 return 0;
3126
Andrew Morton0608f432013-09-24 15:27:41 -07003127 /*
3128 * This loop can run a while, specially if mem_cgroup's continuously
3129 * keep exceeding their soft limit and putting the system under
3130 * pressure
3131 */
3132 do {
3133 if (next_mz)
3134 mz = next_mz;
3135 else
3136 mz = mem_cgroup_largest_soft_limit_node(mctz);
3137 if (!mz)
3138 break;
3139
3140 nr_scanned = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003141 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07003142 gfp_mask, &nr_scanned);
3143 nr_reclaimed += reclaimed;
3144 *total_scanned += nr_scanned;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07003145 spin_lock_irq(&mctz->lock);
Vladimir Davydovbc2f2e72014-12-10 15:43:40 -08003146 __mem_cgroup_remove_exceeded(mz, mctz);
Andrew Morton0608f432013-09-24 15:27:41 -07003147
3148 /*
3149 * If we failed to reclaim anything from this memory cgroup
3150 * it is time to move on to the next cgroup
3151 */
3152 next_mz = NULL;
Vladimir Davydovbc2f2e72014-12-10 15:43:40 -08003153 if (!reclaimed)
3154 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3155
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003156 excess = soft_limit_excess(mz->memcg);
Andrew Morton0608f432013-09-24 15:27:41 -07003157 /*
3158 * One school of thought says that we should not add
3159 * back the node to the tree if reclaim returns 0.
3160 * But our reclaim could return 0, simply because due
3161 * to priority we are exposing a smaller subset of
3162 * memory to reclaim from. Consider this as a longer
3163 * term TODO.
3164 */
3165 /* If excess == 0, no tree ops */
Johannes Weinercf2c8122014-06-06 14:38:21 -07003166 __mem_cgroup_insert_exceeded(mz, mctz, excess);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07003167 spin_unlock_irq(&mctz->lock);
Andrew Morton0608f432013-09-24 15:27:41 -07003168 css_put(&mz->memcg->css);
3169 loop++;
3170 /*
3171 * Could not reclaim anything and there are no more
3172 * mem cgroups to try or we seem to be looping without
3173 * reclaiming anything.
3174 */
3175 if (!nr_reclaimed &&
3176 (next_mz == NULL ||
3177 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3178 break;
3179 } while (!nr_reclaimed);
3180 if (next_mz)
3181 css_put(&next_mz->memcg->css);
3182 return nr_reclaimed;
3183}
3184
Tejun Heoea280e72014-05-16 13:22:48 -04003185/*
3186 * Test whether @memcg has children, dead or alive. Note that this
3187 * function doesn't care whether @memcg has use_hierarchy enabled and
3188 * returns %true if there are child csses according to the cgroup
Ethon Paulb8f29352020-06-04 16:49:28 -07003189 * hierarchy. Testing use_hierarchy is the caller's responsibility.
Tejun Heoea280e72014-05-16 13:22:48 -04003190 */
Glauber Costab5f99b52013-02-22 16:34:53 -08003191static inline bool memcg_has_children(struct mem_cgroup *memcg)
3192{
Tejun Heoea280e72014-05-16 13:22:48 -04003193 bool ret;
3194
Tejun Heoea280e72014-05-16 13:22:48 -04003195 rcu_read_lock();
3196 ret = css_next_child(NULL, &memcg->css);
3197 rcu_read_unlock();
3198 return ret;
Glauber Costab5f99b52013-02-22 16:34:53 -08003199}
3200
3201/*
Greg Thelen51038172016-05-20 16:58:18 -07003202 * Reclaims as many pages from the given memcg as possible.
Michal Hockoc26251f2012-10-26 13:37:28 +02003203 *
3204 * Caller is responsible for holding css reference for memcg.
3205 */
3206static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3207{
3208 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
Michal Hockoc26251f2012-10-26 13:37:28 +02003209
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003210 /* we call try-to-free pages for make this cgroup empty */
3211 lru_add_drain_all();
Junaid Shahidd12c60f2018-06-07 17:07:31 -07003212
3213 drain_all_stock(memcg);
3214
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003215 /* try to free all pages in this cgroup */
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003216 while (nr_retries && page_counter_read(&memcg->memory)) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003217 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003218
Michal Hockoc26251f2012-10-26 13:37:28 +02003219 if (signal_pending(current))
3220 return -EINTR;
3221
Johannes Weinerb70a2a22014-10-09 15:28:56 -07003222 progress = try_to_free_mem_cgroup_pages(memcg, 1,
3223 GFP_KERNEL, true);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003224 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003225 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003226 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02003227 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003228 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003229
3230 }
Michal Hockoab5196c2012-10-26 13:37:32 +02003231
3232 return 0;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003233}
3234
Tejun Heo6770c642014-05-13 12:16:21 -04003235static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
3236 char *buf, size_t nbytes,
3237 loff_t off)
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003238{
Tejun Heo6770c642014-05-13 12:16:21 -04003239 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Michal Hockoc26251f2012-10-26 13:37:28 +02003240
Michal Hockod8423012012-10-26 13:37:29 +02003241 if (mem_cgroup_is_root(memcg))
3242 return -EINVAL;
Tejun Heo6770c642014-05-13 12:16:21 -04003243 return mem_cgroup_force_empty(memcg) ?: nbytes;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003244}
3245
Tejun Heo182446d2013-08-08 20:11:24 -04003246static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3247 struct cftype *cft)
Balbir Singh18f59ea2009-01-07 18:08:07 -08003248{
Tejun Heo182446d2013-08-08 20:11:24 -04003249 return mem_cgroup_from_css(css)->use_hierarchy;
Balbir Singh18f59ea2009-01-07 18:08:07 -08003250}
3251
Tejun Heo182446d2013-08-08 20:11:24 -04003252static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3253 struct cftype *cft, u64 val)
Balbir Singh18f59ea2009-01-07 18:08:07 -08003254{
3255 int retval = 0;
Tejun Heo182446d2013-08-08 20:11:24 -04003256 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo5c9d5352014-05-16 13:22:48 -04003257 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08003258
Glauber Costa567fb432012-07-31 16:43:07 -07003259 if (memcg->use_hierarchy == val)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003260 return 0;
Glauber Costa567fb432012-07-31 16:43:07 -07003261
Balbir Singh18f59ea2009-01-07 18:08:07 -08003262 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02003263 * If parent's use_hierarchy is set, we can't make any modifications
Balbir Singh18f59ea2009-01-07 18:08:07 -08003264 * in the child subtrees. If it is unset, then the change can
3265 * occur, provided the current cgroup has no children.
3266 *
3267 * For the root cgroup, parent_mem is NULL, we allow value to be
3268 * set if there are no children.
3269 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003270 if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
Balbir Singh18f59ea2009-01-07 18:08:07 -08003271 (val == 1 || val == 0)) {
Tejun Heoea280e72014-05-16 13:22:48 -04003272 if (!memcg_has_children(memcg))
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003273 memcg->use_hierarchy = val;
Balbir Singh18f59ea2009-01-07 18:08:07 -08003274 else
3275 retval = -EBUSY;
3276 } else
3277 retval = -EINVAL;
Glauber Costa567fb432012-07-31 16:43:07 -07003278
Balbir Singh18f59ea2009-01-07 18:08:07 -08003279 return retval;
3280}
3281
Andrew Morton6f646152015-11-06 16:28:58 -08003282static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
Johannes Weinerce00a962014-09-05 08:43:57 -04003283{
Johannes Weiner42a30032019-05-14 15:47:12 -07003284 unsigned long val;
Johannes Weinerce00a962014-09-05 08:43:57 -04003285
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003286 if (mem_cgroup_is_root(memcg)) {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07003287 val = memcg_page_state(memcg, NR_FILE_PAGES) +
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07003288 memcg_page_state(memcg, NR_ANON_MAPPED);
Johannes Weiner42a30032019-05-14 15:47:12 -07003289 if (swap)
3290 val += memcg_page_state(memcg, MEMCG_SWAP);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003291 } else {
Johannes Weinerce00a962014-09-05 08:43:57 -04003292 if (!swap)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003293 val = page_counter_read(&memcg->memory);
Johannes Weinerce00a962014-09-05 08:43:57 -04003294 else
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003295 val = page_counter_read(&memcg->memsw);
Johannes Weinerce00a962014-09-05 08:43:57 -04003296 }
Michal Hockoc12176d2015-11-05 18:50:29 -08003297 return val;
Johannes Weinerce00a962014-09-05 08:43:57 -04003298}
3299
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003300enum {
3301 RES_USAGE,
3302 RES_LIMIT,
3303 RES_MAX_USAGE,
3304 RES_FAILCNT,
3305 RES_SOFT_LIMIT,
3306};
Johannes Weinerce00a962014-09-05 08:43:57 -04003307
Tejun Heo791badb2013-12-05 12:28:02 -05003308static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
Johannes Weiner05b84302014-08-06 16:05:59 -07003309 struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003310{
Tejun Heo182446d2013-08-08 20:11:24 -04003311 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003312 struct page_counter *counter;
Tejun Heoaf36f902012-04-01 12:09:55 -07003313
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003314 switch (MEMFILE_TYPE(cft->private)) {
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003315 case _MEM:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003316 counter = &memcg->memory;
Glauber Costa510fc4e2012-12-18 14:21:47 -08003317 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003318 case _MEMSWAP:
3319 counter = &memcg->memsw;
3320 break;
3321 case _KMEM:
3322 counter = &memcg->kmem;
3323 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003324 case _TCP:
Johannes Weiner0db15292016-01-20 15:02:50 -08003325 counter = &memcg->tcpmem;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003326 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003327 default:
3328 BUG();
3329 }
3330
3331 switch (MEMFILE_ATTR(cft->private)) {
3332 case RES_USAGE:
3333 if (counter == &memcg->memory)
Michal Hockoc12176d2015-11-05 18:50:29 -08003334 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003335 if (counter == &memcg->memsw)
Michal Hockoc12176d2015-11-05 18:50:29 -08003336 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003337 return (u64)page_counter_read(counter) * PAGE_SIZE;
3338 case RES_LIMIT:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003339 return (u64)counter->max * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003340 case RES_MAX_USAGE:
3341 return (u64)counter->watermark * PAGE_SIZE;
3342 case RES_FAILCNT:
3343 return counter->failcnt;
3344 case RES_SOFT_LIMIT:
3345 return (u64)memcg->soft_limit * PAGE_SIZE;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003346 default:
3347 BUG();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003348 }
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003349}
Glauber Costa510fc4e2012-12-18 14:21:47 -08003350
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003351static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
Roman Gushchinc350a992019-08-24 17:54:47 -07003352{
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003353 unsigned long stat[MEMCG_NR_STAT] = {0};
Roman Gushchinc350a992019-08-24 17:54:47 -07003354 struct mem_cgroup *mi;
3355 int node, cpu, i;
Roman Gushchinc350a992019-08-24 17:54:47 -07003356
3357 for_each_online_cpu(cpu)
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003358 for (i = 0; i < MEMCG_NR_STAT; i++)
Shakeel Butt6c1c2802019-08-30 16:04:53 -07003359 stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
Roman Gushchinc350a992019-08-24 17:54:47 -07003360
3361 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003362 for (i = 0; i < MEMCG_NR_STAT; i++)
Roman Gushchinc350a992019-08-24 17:54:47 -07003363 atomic_long_add(stat[i], &mi->vmstats[i]);
3364
3365 for_each_node(node) {
3366 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
3367 struct mem_cgroup_per_node *pi;
3368
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003369 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
Roman Gushchinc350a992019-08-24 17:54:47 -07003370 stat[i] = 0;
3371
3372 for_each_online_cpu(cpu)
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003373 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
Shakeel Butt6c1c2802019-08-30 16:04:53 -07003374 stat[i] += per_cpu(
3375 pn->lruvec_stat_cpu->count[i], cpu);
Roman Gushchinc350a992019-08-24 17:54:47 -07003376
3377 for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003378 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
Roman Gushchinc350a992019-08-24 17:54:47 -07003379 atomic_long_add(stat[i], &pi->lruvec_stat[i]);
3380 }
3381}
3382
Roman Gushchinbb65f892019-08-24 17:54:50 -07003383static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
3384{
3385 unsigned long events[NR_VM_EVENT_ITEMS];
3386 struct mem_cgroup *mi;
3387 int cpu, i;
3388
3389 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3390 events[i] = 0;
3391
3392 for_each_online_cpu(cpu)
3393 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
Shakeel Butt6c1c2802019-08-30 16:04:53 -07003394 events[i] += per_cpu(memcg->vmstats_percpu->events[i],
3395 cpu);
Roman Gushchinbb65f892019-08-24 17:54:50 -07003396
3397 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3398 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3399 atomic_long_add(events[i], &mi->vmevents[i]);
3400}
3401
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003402#ifdef CONFIG_MEMCG_KMEM
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003403static int memcg_online_kmem(struct mem_cgroup *memcg)
Vladimir Davydovd6441632014-01-23 15:53:09 -08003404{
Vladimir Davydovd6441632014-01-23 15:53:09 -08003405 int memcg_id;
3406
Vladimir Davydovb313aee2016-03-17 14:18:27 -07003407 if (cgroup_memory_nokmem)
3408 return 0;
3409
Vladimir Davydov2a4db7e2015-02-12 14:59:32 -08003410 BUG_ON(memcg->kmemcg_id >= 0);
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003411 BUG_ON(memcg->kmem_state);
Vladimir Davydovd6441632014-01-23 15:53:09 -08003412
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003413 memcg_id = memcg_alloc_cache_id();
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003414 if (memcg_id < 0)
3415 return memcg_id;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003416
Johannes Weineref129472016-01-14 15:21:34 -08003417 static_branch_inc(&memcg_kmem_enabled_key);
Vladimir Davydovd6441632014-01-23 15:53:09 -08003418 /*
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003419 * A memory cgroup is considered kmem-online as soon as it gets
Vladimir Davydov900a38f2014-12-12 16:55:10 -08003420 * kmemcg_id. Setting the id after enabling static branching will
Vladimir Davydovd6441632014-01-23 15:53:09 -08003421 * guarantee no one starts accounting before all call sites are
3422 * patched.
3423 */
Vladimir Davydov900a38f2014-12-12 16:55:10 -08003424 memcg->kmemcg_id = memcg_id;
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003425 memcg->kmem_state = KMEM_ONLINE;
Tejun Heobc2791f2017-02-22 15:41:21 -08003426 INIT_LIST_HEAD(&memcg->kmem_caches);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003427
3428 return 0;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003429}
3430
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003431static void memcg_offline_kmem(struct mem_cgroup *memcg)
3432{
3433 struct cgroup_subsys_state *css;
3434 struct mem_cgroup *parent, *child;
3435 int kmemcg_id;
3436
3437 if (memcg->kmem_state != KMEM_ONLINE)
3438 return;
3439 /*
3440 * Clear the online state before clearing memcg_caches array
3441 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
3442 * guarantees that no cache will be created for this cgroup
3443 * after we are done (see memcg_create_kmem_cache()).
3444 */
3445 memcg->kmem_state = KMEM_ALLOCATED;
3446
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003447 parent = parent_mem_cgroup(memcg);
3448 if (!parent)
3449 parent = root_mem_cgroup;
3450
Roman Gushchinbee07b32019-08-30 16:04:32 -07003451 /*
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003452 * Deactivate and reparent kmem_caches.
Roman Gushchinbee07b32019-08-30 16:04:32 -07003453 */
Roman Gushchinfb2f2b02019-07-11 20:56:34 -07003454 memcg_deactivate_kmem_caches(memcg, parent);
3455
3456 kmemcg_id = memcg->kmemcg_id;
3457 BUG_ON(kmemcg_id < 0);
3458
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003459 /*
3460 * Change kmemcg_id of this cgroup and all its descendants to the
3461 * parent's id, and then move all entries from this cgroup's list_lrus
3462 * to ones of the parent. After we have finished, all list_lrus
3463 * corresponding to this cgroup are guaranteed to remain empty. The
3464 * ordering is imposed by list_lru_node->lock taken by
3465 * memcg_drain_all_list_lrus().
3466 */
Tejun Heo3a06bb72016-06-03 14:55:44 -07003467 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003468 css_for_each_descendant_pre(css, &memcg->css) {
3469 child = mem_cgroup_from_css(css);
3470 BUG_ON(child->kmemcg_id != kmemcg_id);
3471 child->kmemcg_id = parent->kmemcg_id;
3472 if (!memcg->use_hierarchy)
3473 break;
3474 }
Tejun Heo3a06bb72016-06-03 14:55:44 -07003475 rcu_read_unlock();
3476
Kirill Tkhai9bec5c32018-08-17 15:47:58 -07003477 memcg_drain_all_list_lrus(kmemcg_id, parent);
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003478
3479 memcg_free_cache_id(kmemcg_id);
3480}
3481
3482static void memcg_free_kmem(struct mem_cgroup *memcg)
3483{
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003484 /* css_alloc() failed, offlining didn't happen */
3485 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3486 memcg_offline_kmem(memcg);
3487
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003488 if (memcg->kmem_state == KMEM_ALLOCATED) {
Roman Gushchinf0a3a242019-07-11 20:56:27 -07003489 WARN_ON(!list_empty(&memcg->kmem_caches));
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003490 static_branch_dec(&memcg_kmem_enabled_key);
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003491 }
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003492}
Vladimir Davydovd6441632014-01-23 15:53:09 -08003493#else
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003494static int memcg_online_kmem(struct mem_cgroup *memcg)
Johannes Weiner127424c2016-01-20 15:02:32 -08003495{
3496 return 0;
3497}
3498static void memcg_offline_kmem(struct mem_cgroup *memcg)
3499{
3500}
3501static void memcg_free_kmem(struct mem_cgroup *memcg)
3502{
3503}
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003504#endif /* CONFIG_MEMCG_KMEM */
Johannes Weiner127424c2016-01-20 15:02:32 -08003505
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003506static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3507 unsigned long max)
Johannes Weiner127424c2016-01-20 15:02:32 -08003508{
Vladimir Davydovb313aee2016-03-17 14:18:27 -07003509 int ret;
Johannes Weiner127424c2016-01-20 15:02:32 -08003510
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003511 mutex_lock(&memcg_max_mutex);
3512 ret = page_counter_set_max(&memcg->kmem, max);
3513 mutex_unlock(&memcg_max_mutex);
Johannes Weiner127424c2016-01-20 15:02:32 -08003514 return ret;
3515}
Glauber Costa510fc4e2012-12-18 14:21:47 -08003516
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003517static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003518{
3519 int ret;
3520
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003521 mutex_lock(&memcg_max_mutex);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003522
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003523 ret = page_counter_set_max(&memcg->tcpmem, max);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003524 if (ret)
3525 goto out;
3526
Johannes Weiner0db15292016-01-20 15:02:50 -08003527 if (!memcg->tcpmem_active) {
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003528 /*
3529 * The active flag needs to be written after the static_key
3530 * update. This is what guarantees that the socket activation
Johannes Weiner2d758072016-10-07 17:00:58 -07003531 * function is the last one to run. See mem_cgroup_sk_alloc()
3532 * for details, and note that we don't mark any socket as
3533 * belonging to this memcg until that flag is up.
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003534 *
3535 * We need to do this, because static_keys will span multiple
3536 * sites, but we can't control their order. If we mark a socket
3537 * as accounted, but the accounting functions are not patched in
3538 * yet, we'll lose accounting.
3539 *
Johannes Weiner2d758072016-10-07 17:00:58 -07003540 * We never race with the readers in mem_cgroup_sk_alloc(),
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003541 * because when this value change, the code to process it is not
3542 * patched in yet.
3543 */
3544 static_branch_inc(&memcg_sockets_enabled_key);
Johannes Weiner0db15292016-01-20 15:02:50 -08003545 memcg->tcpmem_active = true;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003546 }
3547out:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003548 mutex_unlock(&memcg_max_mutex);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003549 return ret;
3550}
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003551
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003552/*
3553 * The user of this function is...
3554 * RES_LIMIT.
3555 */
Tejun Heo451af5042014-05-13 12:16:21 -04003556static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3557 char *buf, size_t nbytes, loff_t off)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003558{
Tejun Heo451af5042014-05-13 12:16:21 -04003559 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003560 unsigned long nr_pages;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003561 int ret;
3562
Tejun Heo451af5042014-05-13 12:16:21 -04003563 buf = strstrip(buf);
Johannes Weiner650c5e52015-02-11 15:26:03 -08003564 ret = page_counter_memparse(buf, "-1", &nr_pages);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003565 if (ret)
3566 return ret;
Tejun Heoaf36f902012-04-01 12:09:55 -07003567
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003568 switch (MEMFILE_ATTR(of_cft(of)->private)) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003569 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07003570 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3571 ret = -EINVAL;
3572 break;
3573 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003574 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3575 case _MEM:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003576 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003577 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003578 case _MEMSWAP:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003579 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003580 break;
3581 case _KMEM:
Michal Hocko01581152019-09-23 15:37:22 -07003582 pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
3583 "Please report your usecase to linux-mm@kvack.org if you "
3584 "depend on this functionality.\n");
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003585 ret = memcg_update_kmem_max(memcg, nr_pages);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003586 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003587 case _TCP:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003588 ret = memcg_update_tcp_max(memcg, nr_pages);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003589 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003590 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003591 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07003592 case RES_SOFT_LIMIT:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003593 memcg->soft_limit = nr_pages;
3594 ret = 0;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003595 break;
3596 }
Tejun Heo451af5042014-05-13 12:16:21 -04003597 return ret ?: nbytes;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003598}
3599
Tejun Heo6770c642014-05-13 12:16:21 -04003600static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3601 size_t nbytes, loff_t off)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003602{
Tejun Heo6770c642014-05-13 12:16:21 -04003603 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003604 struct page_counter *counter;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003605
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003606 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3607 case _MEM:
3608 counter = &memcg->memory;
3609 break;
3610 case _MEMSWAP:
3611 counter = &memcg->memsw;
3612 break;
3613 case _KMEM:
3614 counter = &memcg->kmem;
3615 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003616 case _TCP:
Johannes Weiner0db15292016-01-20 15:02:50 -08003617 counter = &memcg->tcpmem;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003618 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003619 default:
3620 BUG();
3621 }
Tejun Heoaf36f902012-04-01 12:09:55 -07003622
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003623 switch (MEMFILE_ATTR(of_cft(of)->private)) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003624 case RES_MAX_USAGE:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003625 page_counter_reset_watermark(counter);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003626 break;
3627 case RES_FAILCNT:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003628 counter->failcnt = 0;
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003629 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003630 default:
3631 BUG();
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003632 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07003633
Tejun Heo6770c642014-05-13 12:16:21 -04003634 return nbytes;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003635}
3636
Tejun Heo182446d2013-08-08 20:11:24 -04003637static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003638 struct cftype *cft)
3639{
Tejun Heo182446d2013-08-08 20:11:24 -04003640 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003641}
3642
Daisuke Nishimura02491442010-03-10 15:22:17 -08003643#ifdef CONFIG_MMU
Tejun Heo182446d2013-08-08 20:11:24 -04003644static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003645 struct cftype *cft, u64 val)
3646{
Tejun Heo182446d2013-08-08 20:11:24 -04003647 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003648
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08003649 if (val & ~MOVE_MASK)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003650 return -EINVAL;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003651
Glauber Costaee5e8472013-02-22 16:34:50 -08003652 /*
3653 * No kind of locking is needed in here, because ->can_attach() will
3654 * check this value once in the beginning of the process, and then carry
3655 * on with stale data. This means that changes to this value will only
3656 * affect task migrations starting after the change.
3657 */
3658 memcg->move_charge_at_immigrate = val;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003659 return 0;
3660}
Daisuke Nishimura02491442010-03-10 15:22:17 -08003661#else
Tejun Heo182446d2013-08-08 20:11:24 -04003662static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura02491442010-03-10 15:22:17 -08003663 struct cftype *cft, u64 val)
3664{
3665 return -ENOSYS;
3666}
3667#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003668
Ying Han406eb0c2011-05-26 16:25:37 -07003669#ifdef CONFIG_NUMA
Johannes Weiner113b7df2019-05-13 17:18:11 -07003670
3671#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
3672#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
3673#define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
3674
3675static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003676 int nid, unsigned int lru_mask, bool tree)
Johannes Weiner113b7df2019-05-13 17:18:11 -07003677{
Johannes Weiner867e5e12019-11-30 17:55:34 -08003678 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
Johannes Weiner113b7df2019-05-13 17:18:11 -07003679 unsigned long nr = 0;
3680 enum lru_list lru;
3681
3682 VM_BUG_ON((unsigned)nid >= nr_node_ids);
3683
3684 for_each_lru(lru) {
3685 if (!(BIT(lru) & lru_mask))
3686 continue;
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003687 if (tree)
3688 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
3689 else
3690 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
Johannes Weiner113b7df2019-05-13 17:18:11 -07003691 }
3692 return nr;
3693}
3694
3695static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003696 unsigned int lru_mask,
3697 bool tree)
Johannes Weiner113b7df2019-05-13 17:18:11 -07003698{
3699 unsigned long nr = 0;
3700 enum lru_list lru;
3701
3702 for_each_lru(lru) {
3703 if (!(BIT(lru) & lru_mask))
3704 continue;
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003705 if (tree)
3706 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
3707 else
3708 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
Johannes Weiner113b7df2019-05-13 17:18:11 -07003709 }
3710 return nr;
3711}
3712
Tejun Heo2da8ca82013-12-05 12:28:04 -05003713static int memcg_numa_stat_show(struct seq_file *m, void *v)
Ying Han406eb0c2011-05-26 16:25:37 -07003714{
Greg Thelen25485de2013-11-12 15:07:40 -08003715 struct numa_stat {
3716 const char *name;
3717 unsigned int lru_mask;
3718 };
3719
3720 static const struct numa_stat stats[] = {
3721 { "total", LRU_ALL },
3722 { "file", LRU_ALL_FILE },
3723 { "anon", LRU_ALL_ANON },
3724 { "unevictable", BIT(LRU_UNEVICTABLE) },
3725 };
3726 const struct numa_stat *stat;
Ying Han406eb0c2011-05-26 16:25:37 -07003727 int nid;
Chris Downaa9694b2019-03-05 15:45:52 -08003728 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Ying Han406eb0c2011-05-26 16:25:37 -07003729
Greg Thelen25485de2013-11-12 15:07:40 -08003730 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003731 seq_printf(m, "%s=%lu", stat->name,
3732 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3733 false));
3734 for_each_node_state(nid, N_MEMORY)
3735 seq_printf(m, " N%d=%lu", nid,
3736 mem_cgroup_node_nr_lru_pages(memcg, nid,
3737 stat->lru_mask, false));
Greg Thelen25485de2013-11-12 15:07:40 -08003738 seq_putc(m, '\n');
Ying Han406eb0c2011-05-26 16:25:37 -07003739 }
Ying Han406eb0c2011-05-26 16:25:37 -07003740
Ying Han071aee12013-11-12 15:07:41 -08003741 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
Ying Han406eb0c2011-05-26 16:25:37 -07003742
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003743 seq_printf(m, "hierarchical_%s=%lu", stat->name,
3744 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3745 true));
3746 for_each_node_state(nid, N_MEMORY)
3747 seq_printf(m, " N%d=%lu", nid,
3748 mem_cgroup_node_nr_lru_pages(memcg, nid,
3749 stat->lru_mask, true));
Ying Han071aee12013-11-12 15:07:41 -08003750 seq_putc(m, '\n');
Ying Han406eb0c2011-05-26 16:25:37 -07003751 }
Ying Han406eb0c2011-05-26 16:25:37 -07003752
Ying Han406eb0c2011-05-26 16:25:37 -07003753 return 0;
3754}
3755#endif /* CONFIG_NUMA */
3756
Johannes Weinerc8713d02019-07-11 20:55:59 -07003757static const unsigned int memcg1_stats[] = {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07003758 NR_FILE_PAGES,
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07003759 NR_ANON_MAPPED,
Johannes Weiner468c3982020-06-03 16:02:01 -07003760#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3761 NR_ANON_THPS,
3762#endif
Johannes Weinerc8713d02019-07-11 20:55:59 -07003763 NR_SHMEM,
3764 NR_FILE_MAPPED,
3765 NR_FILE_DIRTY,
3766 NR_WRITEBACK,
3767 MEMCG_SWAP,
3768};
3769
3770static const char *const memcg1_stat_names[] = {
3771 "cache",
3772 "rss",
Johannes Weiner468c3982020-06-03 16:02:01 -07003773#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Johannes Weinerc8713d02019-07-11 20:55:59 -07003774 "rss_huge",
Johannes Weiner468c3982020-06-03 16:02:01 -07003775#endif
Johannes Weinerc8713d02019-07-11 20:55:59 -07003776 "shmem",
3777 "mapped_file",
3778 "dirty",
3779 "writeback",
3780 "swap",
3781};
3782
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07003783/* Universal VM events cgroup1 shows, original sort order */
Greg Thelen8dd53fd2018-06-07 17:07:23 -07003784static const unsigned int memcg1_events[] = {
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07003785 PGPGIN,
3786 PGPGOUT,
3787 PGFAULT,
3788 PGMAJFAULT,
3789};
3790
Tejun Heo2da8ca82013-12-05 12:28:04 -05003791static int memcg_stat_show(struct seq_file *m, void *v)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003792{
Chris Downaa9694b2019-03-05 15:45:52 -08003793 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003794 unsigned long memory, memsw;
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003795 struct mem_cgroup *mi;
3796 unsigned int i;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003797
Johannes Weiner71cd3112017-05-03 14:55:13 -07003798 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
Rickard Strandqvist70bc0682014-12-12 16:56:41 -08003799
Johannes Weiner71cd3112017-05-03 14:55:13 -07003800 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
Johannes Weiner468c3982020-06-03 16:02:01 -07003801 unsigned long nr;
3802
Johannes Weiner71cd3112017-05-03 14:55:13 -07003803 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003804 continue;
Johannes Weiner468c3982020-06-03 16:02:01 -07003805 nr = memcg_page_state_local(memcg, memcg1_stats[i]);
3806#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3807 if (memcg1_stats[i] == NR_ANON_THPS)
3808 nr *= HPAGE_PMD_NR;
3809#endif
3810 seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003811 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003812
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07003813 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08003814 seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
Johannes Weiner205b20c2019-05-14 15:47:06 -07003815 memcg_events_local(memcg, memcg1_events[i]));
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003816
3817 for (i = 0; i < NR_LRU_LISTS; i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08003818 seq_printf(m, "%s %lu\n", lru_list_name(i),
Johannes Weiner205b20c2019-05-14 15:47:06 -07003819 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
Johannes Weiner21d89d12019-05-13 17:18:08 -07003820 PAGE_SIZE);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003821
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003822 /* Hierarchical information */
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003823 memory = memsw = PAGE_COUNTER_MAX;
3824 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
Chris Down15b42562020-04-01 21:07:20 -07003825 memory = min(memory, READ_ONCE(mi->memory.max));
3826 memsw = min(memsw, READ_ONCE(mi->memsw.max));
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003827 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003828 seq_printf(m, "hierarchical_memory_limit %llu\n",
3829 (u64)memory * PAGE_SIZE);
Johannes Weiner7941d212016-01-14 15:21:23 -08003830 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003831 seq_printf(m, "hierarchical_memsw_limit %llu\n",
3832 (u64)memsw * PAGE_SIZE);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003833
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07003834 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
Johannes Weiner71cd3112017-05-03 14:55:13 -07003835 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003836 continue;
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07003837 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
Yafang Shaodd923992019-07-11 20:52:11 -07003838 (u64)memcg_page_state(memcg, memcg1_stats[i]) *
3839 PAGE_SIZE);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003840 }
3841
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07003842 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08003843 seq_printf(m, "total_%s %llu\n",
3844 vm_event_name(memcg1_events[i]),
Yafang Shaodd923992019-07-11 20:52:11 -07003845 (u64)memcg_events(memcg, memcg1_events[i]));
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003846
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07003847 for (i = 0; i < NR_LRU_LISTS; i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08003848 seq_printf(m, "total_%s %llu\n", lru_list_name(i),
Johannes Weiner42a30032019-05-14 15:47:12 -07003849 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
3850 PAGE_SIZE);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003851
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003852#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003853 {
Mel Gormanef8f2322016-07-28 15:46:05 -07003854 pg_data_t *pgdat;
3855 struct mem_cgroup_per_node *mz;
Johannes Weiner1431d4d2020-06-03 16:02:53 -07003856 unsigned long anon_cost = 0;
3857 unsigned long file_cost = 0;
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003858
Mel Gormanef8f2322016-07-28 15:46:05 -07003859 for_each_online_pgdat(pgdat) {
3860 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003861
Johannes Weiner1431d4d2020-06-03 16:02:53 -07003862 anon_cost += mz->lruvec.anon_cost;
3863 file_cost += mz->lruvec.file_cost;
Mel Gormanef8f2322016-07-28 15:46:05 -07003864 }
Johannes Weiner1431d4d2020-06-03 16:02:53 -07003865 seq_printf(m, "anon_cost %lu\n", anon_cost);
3866 seq_printf(m, "file_cost %lu\n", file_cost);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003867 }
3868#endif
3869
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003870 return 0;
3871}
3872
Tejun Heo182446d2013-08-08 20:11:24 -04003873static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3874 struct cftype *cft)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003875{
Tejun Heo182446d2013-08-08 20:11:24 -04003876 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003877
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07003878 return mem_cgroup_swappiness(memcg);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003879}
3880
Tejun Heo182446d2013-08-08 20:11:24 -04003881static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3882 struct cftype *cft, u64 val)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003883{
Tejun Heo182446d2013-08-08 20:11:24 -04003884 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Li Zefan068b38c2009-01-15 13:51:26 -08003885
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07003886 if (val > 100)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003887 return -EINVAL;
3888
Linus Torvalds14208b02014-06-09 15:03:33 -07003889 if (css->parent)
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07003890 memcg->swappiness = val;
3891 else
3892 vm_swappiness = val;
Li Zefan068b38c2009-01-15 13:51:26 -08003893
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003894 return 0;
3895}
3896
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003897static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3898{
3899 struct mem_cgroup_threshold_ary *t;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003900 unsigned long usage;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003901 int i;
3902
3903 rcu_read_lock();
3904 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003905 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003906 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003907 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003908
3909 if (!t)
3910 goto unlock;
3911
Johannes Weinerce00a962014-09-05 08:43:57 -04003912 usage = mem_cgroup_usage(memcg, swap);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003913
3914 /*
Sha Zhengju748dad32012-05-29 15:06:57 -07003915 * current_threshold points to threshold just below or equal to usage.
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003916 * If it's not true, a threshold was crossed after last
3917 * call of __mem_cgroup_threshold().
3918 */
Phil Carmody5407a562010-05-26 14:42:42 -07003919 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003920
3921 /*
3922 * Iterate backward over array of thresholds starting from
3923 * current_threshold and check if a threshold is crossed.
3924 * If none of thresholds below usage is crossed, we read
3925 * only one element of the array here.
3926 */
3927 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3928 eventfd_signal(t->entries[i].eventfd, 1);
3929
3930 /* i = current_threshold + 1 */
3931 i++;
3932
3933 /*
3934 * Iterate forward over array of thresholds starting from
3935 * current_threshold+1 and check if a threshold is crossed.
3936 * If none of thresholds above usage is crossed, we read
3937 * only one element of the array here.
3938 */
3939 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3940 eventfd_signal(t->entries[i].eventfd, 1);
3941
3942 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07003943 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003944unlock:
3945 rcu_read_unlock();
3946}
3947
3948static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3949{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07003950 while (memcg) {
3951 __mem_cgroup_threshold(memcg, false);
Johannes Weiner7941d212016-01-14 15:21:23 -08003952 if (do_memsw_account())
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07003953 __mem_cgroup_threshold(memcg, true);
3954
3955 memcg = parent_mem_cgroup(memcg);
3956 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003957}
3958
3959static int compare_thresholds(const void *a, const void *b)
3960{
3961 const struct mem_cgroup_threshold *_a = a;
3962 const struct mem_cgroup_threshold *_b = b;
3963
Greg Thelen2bff24a2013-09-11 14:23:08 -07003964 if (_a->threshold > _b->threshold)
3965 return 1;
3966
3967 if (_a->threshold < _b->threshold)
3968 return -1;
3969
3970 return 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003971}
3972
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003973static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003974{
3975 struct mem_cgroup_eventfd_list *ev;
3976
Michal Hocko2bcf2e92014-07-30 16:08:33 -07003977 spin_lock(&memcg_oom_lock);
3978
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003979 list_for_each_entry(ev, &memcg->oom_notify, list)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003980 eventfd_signal(ev->eventfd, 1);
Michal Hocko2bcf2e92014-07-30 16:08:33 -07003981
3982 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003983 return 0;
3984}
3985
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003986static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003987{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003988 struct mem_cgroup *iter;
3989
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003990 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003991 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003992}
3993
Tejun Heo59b6f872013-11-22 18:20:43 -05003994static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05003995 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003996{
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003997 struct mem_cgroup_thresholds *thresholds;
3998 struct mem_cgroup_threshold_ary *new;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003999 unsigned long threshold;
4000 unsigned long usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004001 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004002
Johannes Weiner650c5e52015-02-11 15:26:03 -08004003 ret = page_counter_memparse(args, "-1", &threshold);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004004 if (ret)
4005 return ret;
4006
4007 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004008
Johannes Weiner05b84302014-08-06 16:05:59 -07004009 if (type == _MEM) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004010 thresholds = &memcg->thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004011 usage = mem_cgroup_usage(memcg, false);
Johannes Weiner05b84302014-08-06 16:05:59 -07004012 } else if (type == _MEMSWAP) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004013 thresholds = &memcg->memsw_thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004014 usage = mem_cgroup_usage(memcg, true);
Johannes Weiner05b84302014-08-06 16:05:59 -07004015 } else
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004016 BUG();
4017
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004018 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004019 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004020 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4021
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004022 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004023
4024 /* Allocate memory for new array of thresholds */
Gustavo A. R. Silva67b80462019-03-05 15:44:05 -08004025 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004026 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004027 ret = -ENOMEM;
4028 goto unlock;
4029 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004030 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004031
4032 /* Copy thresholds (if any) to new array */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004033 if (thresholds->primary) {
4034 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004035 sizeof(struct mem_cgroup_threshold));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004036 }
4037
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004038 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004039 new->entries[size - 1].eventfd = eventfd;
4040 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004041
4042 /* Sort thresholds. Registering of new threshold isn't time-critical */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004043 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004044 compare_thresholds, NULL);
4045
4046 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004047 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004048 for (i = 0; i < size; i++) {
Sha Zhengju748dad32012-05-29 15:06:57 -07004049 if (new->entries[i].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004050 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004051 * new->current_threshold will not be used until
4052 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004053 * it here.
4054 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004055 ++new->current_threshold;
Sha Zhengju748dad32012-05-29 15:06:57 -07004056 } else
4057 break;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004058 }
4059
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004060 /* Free old spare buffer and save old primary buffer as spare */
4061 kfree(thresholds->spare);
4062 thresholds->spare = thresholds->primary;
4063
4064 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004065
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004066 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004067 synchronize_rcu();
4068
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004069unlock:
4070 mutex_unlock(&memcg->thresholds_lock);
4071
4072 return ret;
4073}
4074
Tejun Heo59b6f872013-11-22 18:20:43 -05004075static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004076 struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004077{
Tejun Heo59b6f872013-11-22 18:20:43 -05004078 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
Tejun Heo347c4a82013-11-22 18:20:43 -05004079}
4080
Tejun Heo59b6f872013-11-22 18:20:43 -05004081static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004082 struct eventfd_ctx *eventfd, const char *args)
4083{
Tejun Heo59b6f872013-11-22 18:20:43 -05004084 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
Tejun Heo347c4a82013-11-22 18:20:43 -05004085}
4086
Tejun Heo59b6f872013-11-22 18:20:43 -05004087static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004088 struct eventfd_ctx *eventfd, enum res_type type)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004089{
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004090 struct mem_cgroup_thresholds *thresholds;
4091 struct mem_cgroup_threshold_ary *new;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004092 unsigned long usage;
Chunguang Xu7d366652020-03-21 18:22:10 -07004093 int i, j, size, entries;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004094
4095 mutex_lock(&memcg->thresholds_lock);
Johannes Weiner05b84302014-08-06 16:05:59 -07004096
4097 if (type == _MEM) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004098 thresholds = &memcg->thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004099 usage = mem_cgroup_usage(memcg, false);
Johannes Weiner05b84302014-08-06 16:05:59 -07004100 } else if (type == _MEMSWAP) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004101 thresholds = &memcg->memsw_thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004102 usage = mem_cgroup_usage(memcg, true);
Johannes Weiner05b84302014-08-06 16:05:59 -07004103 } else
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004104 BUG();
4105
Anton Vorontsov371528c2012-02-24 05:14:46 +04004106 if (!thresholds->primary)
4107 goto unlock;
4108
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004109 /* Check if a threshold crossed before removing */
4110 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4111
4112 /* Calculate new number of threshold */
Chunguang Xu7d366652020-03-21 18:22:10 -07004113 size = entries = 0;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004114 for (i = 0; i < thresholds->primary->size; i++) {
4115 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004116 size++;
Chunguang Xu7d366652020-03-21 18:22:10 -07004117 else
4118 entries++;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004119 }
4120
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004121 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004122
Chunguang Xu7d366652020-03-21 18:22:10 -07004123 /* If no items related to eventfd have been cleared, nothing to do */
4124 if (!entries)
4125 goto unlock;
4126
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004127 /* Set thresholds array to NULL if we don't have thresholds */
4128 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004129 kfree(new);
4130 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004131 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004132 }
4133
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004134 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004135
4136 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004137 new->current_threshold = -1;
4138 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4139 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004140 continue;
4141
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004142 new->entries[j] = thresholds->primary->entries[i];
Sha Zhengju748dad32012-05-29 15:06:57 -07004143 if (new->entries[j].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004144 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004145 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004146 * until rcu_assign_pointer(), so it's safe to increment
4147 * it here.
4148 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004149 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004150 }
4151 j++;
4152 }
4153
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004154swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004155 /* Swap primary and spare array */
4156 thresholds->spare = thresholds->primary;
Sha Zhengju8c757762012-05-10 13:01:45 -07004157
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004158 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004159
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004160 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004161 synchronize_rcu();
Martijn Coenen6611d8d2016-01-15 16:57:49 -08004162
4163 /* If all events are unregistered, free the spare array */
4164 if (!new) {
4165 kfree(thresholds->spare);
4166 thresholds->spare = NULL;
4167 }
Anton Vorontsov371528c2012-02-24 05:14:46 +04004168unlock:
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004169 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004170}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004171
Tejun Heo59b6f872013-11-22 18:20:43 -05004172static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004173 struct eventfd_ctx *eventfd)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004174{
Tejun Heo59b6f872013-11-22 18:20:43 -05004175 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
Tejun Heo347c4a82013-11-22 18:20:43 -05004176}
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004177
Tejun Heo59b6f872013-11-22 18:20:43 -05004178static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004179 struct eventfd_ctx *eventfd)
4180{
Tejun Heo59b6f872013-11-22 18:20:43 -05004181 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
Tejun Heo347c4a82013-11-22 18:20:43 -05004182}
4183
Tejun Heo59b6f872013-11-22 18:20:43 -05004184static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004185 struct eventfd_ctx *eventfd, const char *args)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004186{
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004187 struct mem_cgroup_eventfd_list *event;
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004188
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004189 event = kmalloc(sizeof(*event), GFP_KERNEL);
4190 if (!event)
4191 return -ENOMEM;
4192
Michal Hocko1af8efe2011-07-26 16:08:24 -07004193 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004194
4195 event->eventfd = eventfd;
4196 list_add(&event->list, &memcg->oom_notify);
4197
4198 /* already in OOM ? */
Tejun Heoc2b42d32015-06-24 16:58:23 -07004199 if (memcg->under_oom)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004200 eventfd_signal(eventfd, 1);
Michal Hocko1af8efe2011-07-26 16:08:24 -07004201 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004202
4203 return 0;
4204}
4205
Tejun Heo59b6f872013-11-22 18:20:43 -05004206static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004207 struct eventfd_ctx *eventfd)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004208{
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004209 struct mem_cgroup_eventfd_list *ev, *tmp;
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004210
Michal Hocko1af8efe2011-07-26 16:08:24 -07004211 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004212
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004213 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004214 if (ev->eventfd == eventfd) {
4215 list_del(&ev->list);
4216 kfree(ev);
4217 }
4218 }
4219
Michal Hocko1af8efe2011-07-26 16:08:24 -07004220 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004221}
4222
Tejun Heo2da8ca82013-12-05 12:28:04 -05004223static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004224{
Chris Downaa9694b2019-03-05 15:45:52 -08004225 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004226
Tejun Heo791badb2013-12-05 12:28:02 -05004227 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
Tejun Heoc2b42d32015-06-24 16:58:23 -07004228 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
Roman Gushchinfe6bdfc2018-06-14 15:28:05 -07004229 seq_printf(sf, "oom_kill %lu\n",
4230 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004231 return 0;
4232}
4233
Tejun Heo182446d2013-08-08 20:11:24 -04004234static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004235 struct cftype *cft, u64 val)
4236{
Tejun Heo182446d2013-08-08 20:11:24 -04004237 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004238
4239 /* cannot set to root cgroup and only 0 and 1 are allowed */
Linus Torvalds14208b02014-06-09 15:03:33 -07004240 if (!css->parent || !((val == 0) || (val == 1)))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004241 return -EINVAL;
4242
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004243 memcg->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07004244 if (!val)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004245 memcg_oom_recover(memcg);
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07004246
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004247 return 0;
4248}
4249
Tejun Heo52ebea72015-05-22 17:13:37 -04004250#ifdef CONFIG_CGROUP_WRITEBACK
4251
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004252#include <trace/events/writeback.h>
4253
Tejun Heo841710a2015-05-22 18:23:33 -04004254static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4255{
4256 return wb_domain_init(&memcg->cgwb_domain, gfp);
4257}
4258
4259static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4260{
4261 wb_domain_exit(&memcg->cgwb_domain);
4262}
4263
Tejun Heo2529bb32015-05-22 18:23:34 -04004264static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4265{
4266 wb_domain_size_changed(&memcg->cgwb_domain);
4267}
4268
Tejun Heo841710a2015-05-22 18:23:33 -04004269struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
4270{
4271 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4272
4273 if (!memcg->css.parent)
4274 return NULL;
4275
4276 return &memcg->cgwb_domain;
4277}
4278
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004279/*
4280 * idx can be of type enum memcg_stat_item or node_stat_item.
4281 * Keep in sync with memcg_exact_page().
4282 */
4283static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
4284{
Chris Down871789d2019-05-14 15:46:57 -07004285 long x = atomic_long_read(&memcg->vmstats[idx]);
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004286 int cpu;
4287
4288 for_each_online_cpu(cpu)
Chris Down871789d2019-05-14 15:46:57 -07004289 x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004290 if (x < 0)
4291 x = 0;
4292 return x;
4293}
4294
Tejun Heoc2aa7232015-05-22 18:23:35 -04004295/**
4296 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4297 * @wb: bdi_writeback in question
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004298 * @pfilepages: out parameter for number of file pages
4299 * @pheadroom: out parameter for number of allocatable pages according to memcg
Tejun Heoc2aa7232015-05-22 18:23:35 -04004300 * @pdirty: out parameter for number of dirty pages
4301 * @pwriteback: out parameter for number of pages under writeback
4302 *
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004303 * Determine the numbers of file, headroom, dirty, and writeback pages in
4304 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
4305 * is a bit more involved.
Tejun Heoc2aa7232015-05-22 18:23:35 -04004306 *
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004307 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
4308 * headroom is calculated as the lowest headroom of itself and the
4309 * ancestors. Note that this doesn't consider the actual amount of
4310 * available memory in the system. The caller should further cap
4311 * *@pheadroom accordingly.
Tejun Heoc2aa7232015-05-22 18:23:35 -04004312 */
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004313void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
4314 unsigned long *pheadroom, unsigned long *pdirty,
4315 unsigned long *pwriteback)
Tejun Heoc2aa7232015-05-22 18:23:35 -04004316{
4317 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4318 struct mem_cgroup *parent;
Tejun Heoc2aa7232015-05-22 18:23:35 -04004319
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004320 *pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
Tejun Heoc2aa7232015-05-22 18:23:35 -04004321
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004322 *pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
Johannes Weiner21d89d12019-05-13 17:18:08 -07004323 *pfilepages = memcg_exact_page_state(memcg, NR_INACTIVE_FILE) +
4324 memcg_exact_page_state(memcg, NR_ACTIVE_FILE);
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004325 *pheadroom = PAGE_COUNTER_MAX;
Tejun Heoc2aa7232015-05-22 18:23:35 -04004326
Tejun Heoc2aa7232015-05-22 18:23:35 -04004327 while ((parent = parent_mem_cgroup(memcg))) {
Chris Down15b42562020-04-01 21:07:20 -07004328 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
Jakub Kicinskid1663a92020-06-01 21:49:49 -07004329 READ_ONCE(memcg->memory.high));
Tejun Heoc2aa7232015-05-22 18:23:35 -04004330 unsigned long used = page_counter_read(&memcg->memory);
4331
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004332 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
Tejun Heoc2aa7232015-05-22 18:23:35 -04004333 memcg = parent;
4334 }
Tejun Heoc2aa7232015-05-22 18:23:35 -04004335}
4336
Tejun Heo97b27822019-08-26 09:06:56 -07004337/*
4338 * Foreign dirty flushing
4339 *
4340 * There's an inherent mismatch between memcg and writeback. The former
4341 * trackes ownership per-page while the latter per-inode. This was a
4342 * deliberate design decision because honoring per-page ownership in the
4343 * writeback path is complicated, may lead to higher CPU and IO overheads
4344 * and deemed unnecessary given that write-sharing an inode across
4345 * different cgroups isn't a common use-case.
4346 *
4347 * Combined with inode majority-writer ownership switching, this works well
4348 * enough in most cases but there are some pathological cases. For
4349 * example, let's say there are two cgroups A and B which keep writing to
4350 * different but confined parts of the same inode. B owns the inode and
4351 * A's memory is limited far below B's. A's dirty ratio can rise enough to
4352 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
4353 * triggering background writeback. A will be slowed down without a way to
4354 * make writeback of the dirty pages happen.
4355 *
4356 * Conditions like the above can lead to a cgroup getting repatedly and
4357 * severely throttled after making some progress after each
4358 * dirty_expire_interval while the underyling IO device is almost
4359 * completely idle.
4360 *
4361 * Solving this problem completely requires matching the ownership tracking
4362 * granularities between memcg and writeback in either direction. However,
4363 * the more egregious behaviors can be avoided by simply remembering the
4364 * most recent foreign dirtying events and initiating remote flushes on
4365 * them when local writeback isn't enough to keep the memory clean enough.
4366 *
4367 * The following two functions implement such mechanism. When a foreign
4368 * page - a page whose memcg and writeback ownerships don't match - is
4369 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
4370 * bdi_writeback on the page owning memcg. When balance_dirty_pages()
4371 * decides that the memcg needs to sleep due to high dirty ratio, it calls
4372 * mem_cgroup_flush_foreign() which queues writeback on the recorded
4373 * foreign bdi_writebacks which haven't expired. Both the numbers of
4374 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
4375 * limited to MEMCG_CGWB_FRN_CNT.
4376 *
4377 * The mechanism only remembers IDs and doesn't hold any object references.
4378 * As being wrong occasionally doesn't matter, updates and accesses to the
4379 * records are lockless and racy.
4380 */
4381void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
4382 struct bdi_writeback *wb)
4383{
4384 struct mem_cgroup *memcg = page->mem_cgroup;
4385 struct memcg_cgwb_frn *frn;
4386 u64 now = get_jiffies_64();
4387 u64 oldest_at = now;
4388 int oldest = -1;
4389 int i;
4390
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004391 trace_track_foreign_dirty(page, wb);
4392
Tejun Heo97b27822019-08-26 09:06:56 -07004393 /*
4394 * Pick the slot to use. If there is already a slot for @wb, keep
4395 * using it. If not replace the oldest one which isn't being
4396 * written out.
4397 */
4398 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4399 frn = &memcg->cgwb_frn[i];
4400 if (frn->bdi_id == wb->bdi->id &&
4401 frn->memcg_id == wb->memcg_css->id)
4402 break;
4403 if (time_before64(frn->at, oldest_at) &&
4404 atomic_read(&frn->done.cnt) == 1) {
4405 oldest = i;
4406 oldest_at = frn->at;
4407 }
4408 }
4409
4410 if (i < MEMCG_CGWB_FRN_CNT) {
4411 /*
4412 * Re-using an existing one. Update timestamp lazily to
4413 * avoid making the cacheline hot. We want them to be
4414 * reasonably up-to-date and significantly shorter than
4415 * dirty_expire_interval as that's what expires the record.
4416 * Use the shorter of 1s and dirty_expire_interval / 8.
4417 */
4418 unsigned long update_intv =
4419 min_t(unsigned long, HZ,
4420 msecs_to_jiffies(dirty_expire_interval * 10) / 8);
4421
4422 if (time_before64(frn->at, now - update_intv))
4423 frn->at = now;
4424 } else if (oldest >= 0) {
4425 /* replace the oldest free one */
4426 frn = &memcg->cgwb_frn[oldest];
4427 frn->bdi_id = wb->bdi->id;
4428 frn->memcg_id = wb->memcg_css->id;
4429 frn->at = now;
4430 }
4431}
4432
4433/* issue foreign writeback flushes for recorded foreign dirtying events */
4434void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
4435{
4436 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4437 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
4438 u64 now = jiffies_64;
4439 int i;
4440
4441 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4442 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4443
4444 /*
4445 * If the record is older than dirty_expire_interval,
4446 * writeback on it has already started. No need to kick it
4447 * off again. Also, don't start a new one if there's
4448 * already one in flight.
4449 */
4450 if (time_after64(frn->at, now - intv) &&
4451 atomic_read(&frn->done.cnt) == 1) {
4452 frn->at = 0;
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004453 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
Tejun Heo97b27822019-08-26 09:06:56 -07004454 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
4455 WB_REASON_FOREIGN_FLUSH,
4456 &frn->done);
4457 }
4458 }
4459}
4460
Tejun Heo841710a2015-05-22 18:23:33 -04004461#else /* CONFIG_CGROUP_WRITEBACK */
4462
4463static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4464{
4465 return 0;
4466}
4467
4468static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4469{
4470}
4471
Tejun Heo2529bb32015-05-22 18:23:34 -04004472static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4473{
4474}
4475
Tejun Heo52ebea72015-05-22 17:13:37 -04004476#endif /* CONFIG_CGROUP_WRITEBACK */
4477
Tejun Heo79bd9812013-11-22 18:20:42 -05004478/*
Tejun Heo3bc942f2013-11-22 18:20:44 -05004479 * DO NOT USE IN NEW FILES.
4480 *
4481 * "cgroup.event_control" implementation.
4482 *
4483 * This is way over-engineered. It tries to support fully configurable
4484 * events for each user. Such level of flexibility is completely
4485 * unnecessary especially in the light of the planned unified hierarchy.
4486 *
4487 * Please deprecate this and replace with something simpler if at all
4488 * possible.
4489 */
4490
4491/*
Tejun Heo79bd9812013-11-22 18:20:42 -05004492 * Unregister event and free resources.
4493 *
4494 * Gets called from workqueue.
4495 */
Tejun Heo3bc942f2013-11-22 18:20:44 -05004496static void memcg_event_remove(struct work_struct *work)
Tejun Heo79bd9812013-11-22 18:20:42 -05004497{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004498 struct mem_cgroup_event *event =
4499 container_of(work, struct mem_cgroup_event, remove);
Tejun Heo59b6f872013-11-22 18:20:43 -05004500 struct mem_cgroup *memcg = event->memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -05004501
4502 remove_wait_queue(event->wqh, &event->wait);
4503
Tejun Heo59b6f872013-11-22 18:20:43 -05004504 event->unregister_event(memcg, event->eventfd);
Tejun Heo79bd9812013-11-22 18:20:42 -05004505
4506 /* Notify userspace the event is going away. */
4507 eventfd_signal(event->eventfd, 1);
4508
4509 eventfd_ctx_put(event->eventfd);
4510 kfree(event);
Tejun Heo59b6f872013-11-22 18:20:43 -05004511 css_put(&memcg->css);
Tejun Heo79bd9812013-11-22 18:20:42 -05004512}
4513
4514/*
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004515 * Gets called on EPOLLHUP on eventfd when user closes it.
Tejun Heo79bd9812013-11-22 18:20:42 -05004516 *
4517 * Called with wqh->lock held and interrupts disabled.
4518 */
Ingo Molnarac6424b2017-06-20 12:06:13 +02004519static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
Tejun Heo3bc942f2013-11-22 18:20:44 -05004520 int sync, void *key)
Tejun Heo79bd9812013-11-22 18:20:42 -05004521{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004522 struct mem_cgroup_event *event =
4523 container_of(wait, struct mem_cgroup_event, wait);
Tejun Heo59b6f872013-11-22 18:20:43 -05004524 struct mem_cgroup *memcg = event->memcg;
Al Viro3ad6f932017-07-03 20:14:56 -04004525 __poll_t flags = key_to_poll(key);
Tejun Heo79bd9812013-11-22 18:20:42 -05004526
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004527 if (flags & EPOLLHUP) {
Tejun Heo79bd9812013-11-22 18:20:42 -05004528 /*
4529 * If the event has been detached at cgroup removal, we
4530 * can simply return knowing the other side will cleanup
4531 * for us.
4532 *
4533 * We can't race against event freeing since the other
4534 * side will require wqh->lock via remove_wait_queue(),
4535 * which we hold.
4536 */
Tejun Heofba94802013-11-22 18:20:43 -05004537 spin_lock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004538 if (!list_empty(&event->list)) {
4539 list_del_init(&event->list);
4540 /*
4541 * We are in atomic context, but cgroup_event_remove()
4542 * may sleep, so we have to call it in workqueue.
4543 */
4544 schedule_work(&event->remove);
4545 }
Tejun Heofba94802013-11-22 18:20:43 -05004546 spin_unlock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004547 }
4548
4549 return 0;
4550}
4551
Tejun Heo3bc942f2013-11-22 18:20:44 -05004552static void memcg_event_ptable_queue_proc(struct file *file,
Tejun Heo79bd9812013-11-22 18:20:42 -05004553 wait_queue_head_t *wqh, poll_table *pt)
4554{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004555 struct mem_cgroup_event *event =
4556 container_of(pt, struct mem_cgroup_event, pt);
Tejun Heo79bd9812013-11-22 18:20:42 -05004557
4558 event->wqh = wqh;
4559 add_wait_queue(wqh, &event->wait);
4560}
4561
4562/*
Tejun Heo3bc942f2013-11-22 18:20:44 -05004563 * DO NOT USE IN NEW FILES.
4564 *
Tejun Heo79bd9812013-11-22 18:20:42 -05004565 * Parse input and register new cgroup event handler.
4566 *
4567 * Input must be in format '<event_fd> <control_fd> <args>'.
4568 * Interpretation of args is defined by control file implementation.
4569 */
Tejun Heo451af5042014-05-13 12:16:21 -04004570static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4571 char *buf, size_t nbytes, loff_t off)
Tejun Heo79bd9812013-11-22 18:20:42 -05004572{
Tejun Heo451af5042014-05-13 12:16:21 -04004573 struct cgroup_subsys_state *css = of_css(of);
Tejun Heofba94802013-11-22 18:20:43 -05004574 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo3bc942f2013-11-22 18:20:44 -05004575 struct mem_cgroup_event *event;
Tejun Heo79bd9812013-11-22 18:20:42 -05004576 struct cgroup_subsys_state *cfile_css;
4577 unsigned int efd, cfd;
4578 struct fd efile;
4579 struct fd cfile;
Tejun Heofba94802013-11-22 18:20:43 -05004580 const char *name;
Tejun Heo79bd9812013-11-22 18:20:42 -05004581 char *endp;
4582 int ret;
4583
Tejun Heo451af5042014-05-13 12:16:21 -04004584 buf = strstrip(buf);
4585
4586 efd = simple_strtoul(buf, &endp, 10);
Tejun Heo79bd9812013-11-22 18:20:42 -05004587 if (*endp != ' ')
4588 return -EINVAL;
Tejun Heo451af5042014-05-13 12:16:21 -04004589 buf = endp + 1;
Tejun Heo79bd9812013-11-22 18:20:42 -05004590
Tejun Heo451af5042014-05-13 12:16:21 -04004591 cfd = simple_strtoul(buf, &endp, 10);
Tejun Heo79bd9812013-11-22 18:20:42 -05004592 if ((*endp != ' ') && (*endp != '\0'))
4593 return -EINVAL;
Tejun Heo451af5042014-05-13 12:16:21 -04004594 buf = endp + 1;
Tejun Heo79bd9812013-11-22 18:20:42 -05004595
4596 event = kzalloc(sizeof(*event), GFP_KERNEL);
4597 if (!event)
4598 return -ENOMEM;
4599
Tejun Heo59b6f872013-11-22 18:20:43 -05004600 event->memcg = memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -05004601 INIT_LIST_HEAD(&event->list);
Tejun Heo3bc942f2013-11-22 18:20:44 -05004602 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4603 init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4604 INIT_WORK(&event->remove, memcg_event_remove);
Tejun Heo79bd9812013-11-22 18:20:42 -05004605
4606 efile = fdget(efd);
4607 if (!efile.file) {
4608 ret = -EBADF;
4609 goto out_kfree;
4610 }
4611
4612 event->eventfd = eventfd_ctx_fileget(efile.file);
4613 if (IS_ERR(event->eventfd)) {
4614 ret = PTR_ERR(event->eventfd);
4615 goto out_put_efile;
4616 }
4617
4618 cfile = fdget(cfd);
4619 if (!cfile.file) {
4620 ret = -EBADF;
4621 goto out_put_eventfd;
4622 }
4623
4624 /* the process need read permission on control file */
4625 /* AV: shouldn't we check that it's been opened for read instead? */
4626 ret = inode_permission(file_inode(cfile.file), MAY_READ);
4627 if (ret < 0)
4628 goto out_put_cfile;
4629
Tejun Heo79bd9812013-11-22 18:20:42 -05004630 /*
Tejun Heofba94802013-11-22 18:20:43 -05004631 * Determine the event callbacks and set them in @event. This used
4632 * to be done via struct cftype but cgroup core no longer knows
4633 * about these events. The following is crude but the whole thing
4634 * is for compatibility anyway.
Tejun Heo3bc942f2013-11-22 18:20:44 -05004635 *
4636 * DO NOT ADD NEW FILES.
Tejun Heofba94802013-11-22 18:20:43 -05004637 */
Al Virob5830432014-10-31 01:22:04 -04004638 name = cfile.file->f_path.dentry->d_name.name;
Tejun Heofba94802013-11-22 18:20:43 -05004639
4640 if (!strcmp(name, "memory.usage_in_bytes")) {
4641 event->register_event = mem_cgroup_usage_register_event;
4642 event->unregister_event = mem_cgroup_usage_unregister_event;
4643 } else if (!strcmp(name, "memory.oom_control")) {
4644 event->register_event = mem_cgroup_oom_register_event;
4645 event->unregister_event = mem_cgroup_oom_unregister_event;
4646 } else if (!strcmp(name, "memory.pressure_level")) {
4647 event->register_event = vmpressure_register_event;
4648 event->unregister_event = vmpressure_unregister_event;
4649 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
Tejun Heo347c4a82013-11-22 18:20:43 -05004650 event->register_event = memsw_cgroup_usage_register_event;
4651 event->unregister_event = memsw_cgroup_usage_unregister_event;
Tejun Heofba94802013-11-22 18:20:43 -05004652 } else {
4653 ret = -EINVAL;
4654 goto out_put_cfile;
4655 }
4656
4657 /*
Tejun Heob5557c42013-11-22 18:20:42 -05004658 * Verify @cfile should belong to @css. Also, remaining events are
4659 * automatically removed on cgroup destruction but the removal is
4660 * asynchronous, so take an extra ref on @css.
Tejun Heo79bd9812013-11-22 18:20:42 -05004661 */
Al Virob5830432014-10-31 01:22:04 -04004662 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
Tejun Heoec903c02014-05-13 12:11:01 -04004663 &memory_cgrp_subsys);
Tejun Heo79bd9812013-11-22 18:20:42 -05004664 ret = -EINVAL;
Tejun Heo5a17f542014-02-11 11:52:47 -05004665 if (IS_ERR(cfile_css))
Tejun Heo79bd9812013-11-22 18:20:42 -05004666 goto out_put_cfile;
Tejun Heo5a17f542014-02-11 11:52:47 -05004667 if (cfile_css != css) {
4668 css_put(cfile_css);
4669 goto out_put_cfile;
4670 }
Tejun Heo79bd9812013-11-22 18:20:42 -05004671
Tejun Heo451af5042014-05-13 12:16:21 -04004672 ret = event->register_event(memcg, event->eventfd, buf);
Tejun Heo79bd9812013-11-22 18:20:42 -05004673 if (ret)
4674 goto out_put_css;
4675
Christoph Hellwig9965ed172018-03-05 07:26:05 -08004676 vfs_poll(efile.file, &event->pt);
Tejun Heo79bd9812013-11-22 18:20:42 -05004677
Tejun Heofba94802013-11-22 18:20:43 -05004678 spin_lock(&memcg->event_list_lock);
4679 list_add(&event->list, &memcg->event_list);
4680 spin_unlock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004681
4682 fdput(cfile);
4683 fdput(efile);
4684
Tejun Heo451af5042014-05-13 12:16:21 -04004685 return nbytes;
Tejun Heo79bd9812013-11-22 18:20:42 -05004686
4687out_put_css:
Tejun Heob5557c42013-11-22 18:20:42 -05004688 css_put(css);
Tejun Heo79bd9812013-11-22 18:20:42 -05004689out_put_cfile:
4690 fdput(cfile);
4691out_put_eventfd:
4692 eventfd_ctx_put(event->eventfd);
4693out_put_efile:
4694 fdput(efile);
4695out_kfree:
4696 kfree(event);
4697
4698 return ret;
4699}
4700
Johannes Weiner241994ed2015-02-11 15:26:06 -08004701static struct cftype mem_cgroup_legacy_files[] = {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004702 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004703 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004704 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Tejun Heo791badb2013-12-05 12:28:02 -05004705 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004706 },
4707 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004708 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004709 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Tejun Heo6770c642014-05-13 12:16:21 -04004710 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004711 .read_u64 = mem_cgroup_read_u64,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004712 },
4713 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004714 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004715 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Tejun Heo451af5042014-05-13 12:16:21 -04004716 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004717 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004718 },
4719 {
Balbir Singh296c81d2009-09-23 15:56:36 -07004720 .name = "soft_limit_in_bytes",
4721 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
Tejun Heo451af5042014-05-13 12:16:21 -04004722 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004723 .read_u64 = mem_cgroup_read_u64,
Balbir Singh296c81d2009-09-23 15:56:36 -07004724 },
4725 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004726 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004727 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Tejun Heo6770c642014-05-13 12:16:21 -04004728 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004729 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004730 },
Balbir Singh8697d332008-02-07 00:13:59 -08004731 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004732 .name = "stat",
Tejun Heo2da8ca82013-12-05 12:28:04 -05004733 .seq_show = memcg_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004734 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004735 {
4736 .name = "force_empty",
Tejun Heo6770c642014-05-13 12:16:21 -04004737 .write = mem_cgroup_force_empty_write,
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004738 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08004739 {
4740 .name = "use_hierarchy",
4741 .write_u64 = mem_cgroup_hierarchy_write,
4742 .read_u64 = mem_cgroup_hierarchy_read,
4743 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004744 {
Tejun Heo3bc942f2013-11-22 18:20:44 -05004745 .name = "cgroup.event_control", /* XXX: for compat */
Tejun Heo451af5042014-05-13 12:16:21 -04004746 .write = memcg_write_event_control,
Tejun Heo7dbdb192015-09-18 17:54:23 -04004747 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
Tejun Heo79bd9812013-11-22 18:20:42 -05004748 },
4749 {
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004750 .name = "swappiness",
4751 .read_u64 = mem_cgroup_swappiness_read,
4752 .write_u64 = mem_cgroup_swappiness_write,
4753 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004754 {
4755 .name = "move_charge_at_immigrate",
4756 .read_u64 = mem_cgroup_move_charge_read,
4757 .write_u64 = mem_cgroup_move_charge_write,
4758 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004759 {
4760 .name = "oom_control",
Tejun Heo2da8ca82013-12-05 12:28:04 -05004761 .seq_show = mem_cgroup_oom_control_read,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004762 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004763 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4764 },
Anton Vorontsov70ddf632013-04-29 15:08:31 -07004765 {
4766 .name = "pressure_level",
Anton Vorontsov70ddf632013-04-29 15:08:31 -07004767 },
Ying Han406eb0c2011-05-26 16:25:37 -07004768#ifdef CONFIG_NUMA
4769 {
4770 .name = "numa_stat",
Tejun Heo2da8ca82013-12-05 12:28:04 -05004771 .seq_show = memcg_numa_stat_show,
Ying Han406eb0c2011-05-26 16:25:37 -07004772 },
4773#endif
Glauber Costa510fc4e2012-12-18 14:21:47 -08004774 {
4775 .name = "kmem.limit_in_bytes",
4776 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
Tejun Heo451af5042014-05-13 12:16:21 -04004777 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004778 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004779 },
4780 {
4781 .name = "kmem.usage_in_bytes",
4782 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
Tejun Heo791badb2013-12-05 12:28:02 -05004783 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004784 },
4785 {
4786 .name = "kmem.failcnt",
4787 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
Tejun Heo6770c642014-05-13 12:16:21 -04004788 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004789 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004790 },
4791 {
4792 .name = "kmem.max_usage_in_bytes",
4793 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
Tejun Heo6770c642014-05-13 12:16:21 -04004794 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004795 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004796 },
Yafang Shaoa87425a2020-04-01 21:06:30 -07004797#if defined(CONFIG_MEMCG_KMEM) && \
4798 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
Glauber Costa749c5412012-12-18 14:23:01 -08004799 {
4800 .name = "kmem.slabinfo",
Tejun Heobc2791f2017-02-22 15:41:21 -08004801 .seq_start = memcg_slab_start,
4802 .seq_next = memcg_slab_next,
4803 .seq_stop = memcg_slab_stop,
Vladimir Davydovb0475012014-12-10 15:44:19 -08004804 .seq_show = memcg_slab_show,
Glauber Costa749c5412012-12-18 14:23:01 -08004805 },
4806#endif
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08004807 {
4808 .name = "kmem.tcp.limit_in_bytes",
4809 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4810 .write = mem_cgroup_write,
4811 .read_u64 = mem_cgroup_read_u64,
4812 },
4813 {
4814 .name = "kmem.tcp.usage_in_bytes",
4815 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4816 .read_u64 = mem_cgroup_read_u64,
4817 },
4818 {
4819 .name = "kmem.tcp.failcnt",
4820 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4821 .write = mem_cgroup_reset,
4822 .read_u64 = mem_cgroup_read_u64,
4823 },
4824 {
4825 .name = "kmem.tcp.max_usage_in_bytes",
4826 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4827 .write = mem_cgroup_reset,
4828 .read_u64 = mem_cgroup_read_u64,
4829 },
Tejun Heo6bc10342012-04-01 12:09:55 -07004830 { }, /* terminate */
Tejun Heoaf36f902012-04-01 12:09:55 -07004831};
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004832
Johannes Weiner73f576c2016-07-20 15:44:57 -07004833/*
4834 * Private memory cgroup IDR
4835 *
4836 * Swap-out records and page cache shadow entries need to store memcg
4837 * references in constrained space, so we maintain an ID space that is
4838 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4839 * memory-controlled cgroups to 64k.
4840 *
Ethon Paulb8f29352020-06-04 16:49:28 -07004841 * However, there usually are many references to the offline CSS after
Johannes Weiner73f576c2016-07-20 15:44:57 -07004842 * the cgroup has been destroyed, such as page cache or reclaimable
4843 * slab objects, that don't need to hang on to the ID. We want to keep
4844 * those dead CSS from occupying IDs, or we might quickly exhaust the
4845 * relatively small ID space and prevent the creation of new cgroups
4846 * even when there are much fewer than 64k cgroups - possibly none.
4847 *
4848 * Maintain a private 16-bit ID space for memcg, and allow the ID to
4849 * be freed and recycled when it's no longer needed, which is usually
4850 * when the CSS is offlined.
4851 *
4852 * The only exception to that are records of swapped out tmpfs/shmem
4853 * pages that need to be attributed to live ancestors on swapin. But
4854 * those references are manageable from userspace.
4855 */
4856
4857static DEFINE_IDR(mem_cgroup_idr);
4858
Kirill Tkhai7e97de02018-08-02 15:36:01 -07004859static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
4860{
4861 if (memcg->id.id > 0) {
4862 idr_remove(&mem_cgroup_idr, memcg->id.id);
4863 memcg->id.id = 0;
4864 }
4865}
4866
Vincenzo Frascinoc1514c02020-04-01 21:07:13 -07004867static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
4868 unsigned int n)
Johannes Weiner73f576c2016-07-20 15:44:57 -07004869{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07004870 refcount_add(n, &memcg->id.ref);
Johannes Weiner73f576c2016-07-20 15:44:57 -07004871}
4872
Vladimir Davydov615d66c2016-08-11 15:33:03 -07004873static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
Johannes Weiner73f576c2016-07-20 15:44:57 -07004874{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07004875 if (refcount_sub_and_test(n, &memcg->id.ref)) {
Kirill Tkhai7e97de02018-08-02 15:36:01 -07004876 mem_cgroup_id_remove(memcg);
Johannes Weiner73f576c2016-07-20 15:44:57 -07004877
4878 /* Memcg ID pins CSS */
4879 css_put(&memcg->css);
4880 }
4881}
4882
Vladimir Davydov615d66c2016-08-11 15:33:03 -07004883static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4884{
4885 mem_cgroup_id_put_many(memcg, 1);
4886}
4887
Johannes Weiner73f576c2016-07-20 15:44:57 -07004888/**
4889 * mem_cgroup_from_id - look up a memcg from a memcg id
4890 * @id: the memcg id to look up
4891 *
4892 * Caller must hold rcu_read_lock().
4893 */
4894struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4895{
4896 WARN_ON_ONCE(!rcu_read_lock_held());
4897 return idr_find(&mem_cgroup_idr, id);
4898}
4899
Mel Gormanef8f2322016-07-28 15:46:05 -07004900static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004901{
4902 struct mem_cgroup_per_node *pn;
Mel Gormanef8f2322016-07-28 15:46:05 -07004903 int tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004904 /*
4905 * This routine is called against possible nodes.
4906 * But it's BUG to call kmalloc() against offline node.
4907 *
4908 * TODO: this routine can waste much memory for nodes which will
4909 * never be onlined. It's better to use memory hotplug callback
4910 * function.
4911 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07004912 if (!node_state(node, N_NORMAL_MEMORY))
4913 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08004914 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004915 if (!pn)
4916 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004917
Johannes Weiner815744d2019-06-13 15:55:46 -07004918 pn->lruvec_stat_local = alloc_percpu(struct lruvec_stat);
4919 if (!pn->lruvec_stat_local) {
4920 kfree(pn);
4921 return 1;
4922 }
4923
Johannes Weinera983b5e2018-01-31 16:16:45 -08004924 pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
4925 if (!pn->lruvec_stat_cpu) {
Johannes Weiner815744d2019-06-13 15:55:46 -07004926 free_percpu(pn->lruvec_stat_local);
Johannes Weiner00f3ca22017-07-06 15:40:52 -07004927 kfree(pn);
4928 return 1;
4929 }
4930
Mel Gormanef8f2322016-07-28 15:46:05 -07004931 lruvec_init(&pn->lruvec);
4932 pn->usage_in_excess = 0;
4933 pn->on_tree = false;
4934 pn->memcg = memcg;
4935
Johannes Weiner54f72fe2013-07-08 15:59:49 -07004936 memcg->nodeinfo[node] = pn;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004937 return 0;
4938}
4939
Mel Gormanef8f2322016-07-28 15:46:05 -07004940static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004941{
Johannes Weiner00f3ca22017-07-06 15:40:52 -07004942 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
4943
Michal Hocko4eaf4312018-04-10 16:29:52 -07004944 if (!pn)
4945 return;
4946
Johannes Weinera983b5e2018-01-31 16:16:45 -08004947 free_percpu(pn->lruvec_stat_cpu);
Johannes Weiner815744d2019-06-13 15:55:46 -07004948 free_percpu(pn->lruvec_stat_local);
Johannes Weiner00f3ca22017-07-06 15:40:52 -07004949 kfree(pn);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004950}
4951
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08004952static void __mem_cgroup_free(struct mem_cgroup *memcg)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08004953{
4954 int node;
4955
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08004956 for_each_node(node)
Mel Gormanef8f2322016-07-28 15:46:05 -07004957 free_mem_cgroup_per_node_info(memcg, node);
Chris Down871789d2019-05-14 15:46:57 -07004958 free_percpu(memcg->vmstats_percpu);
Johannes Weiner815744d2019-06-13 15:55:46 -07004959 free_percpu(memcg->vmstats_local);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08004960 kfree(memcg);
4961}
4962
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08004963static void mem_cgroup_free(struct mem_cgroup *memcg)
4964{
4965 memcg_wb_domain_exit(memcg);
Shakeel Butt7961eee2019-11-05 21:16:21 -08004966 /*
4967 * Flush percpu vmstats and vmevents to guarantee the value correctness
4968 * on parent's and all ancestor levels.
4969 */
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08004970 memcg_flush_percpu_vmstats(memcg);
Shakeel Butt7961eee2019-11-05 21:16:21 -08004971 memcg_flush_percpu_vmevents(memcg);
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08004972 __mem_cgroup_free(memcg);
4973}
4974
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004975static struct mem_cgroup *mem_cgroup_alloc(void)
4976{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004977 struct mem_cgroup *memcg;
Alexey Dobriyanb9726c22019-03-05 15:48:26 -08004978 unsigned int size;
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08004979 int node;
Tejun Heo97b27822019-08-26 09:06:56 -07004980 int __maybe_unused i;
Yafang Shao11d67612020-05-07 18:35:43 -07004981 long error = -ENOMEM;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004982
Vladimir Davydov8ff69e22014-01-23 15:52:52 -08004983 size = sizeof(struct mem_cgroup);
4984 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004985
Vladimir Davydov8ff69e22014-01-23 15:52:52 -08004986 memcg = kzalloc(size, GFP_KERNEL);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004987 if (!memcg)
Yafang Shao11d67612020-05-07 18:35:43 -07004988 return ERR_PTR(error);
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07004989
Johannes Weiner73f576c2016-07-20 15:44:57 -07004990 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4991 1, MEM_CGROUP_ID_MAX,
4992 GFP_KERNEL);
Yafang Shao11d67612020-05-07 18:35:43 -07004993 if (memcg->id.id < 0) {
4994 error = memcg->id.id;
Johannes Weiner73f576c2016-07-20 15:44:57 -07004995 goto fail;
Yafang Shao11d67612020-05-07 18:35:43 -07004996 }
Johannes Weiner73f576c2016-07-20 15:44:57 -07004997
Johannes Weiner815744d2019-06-13 15:55:46 -07004998 memcg->vmstats_local = alloc_percpu(struct memcg_vmstats_percpu);
4999 if (!memcg->vmstats_local)
5000 goto fail;
5001
Chris Down871789d2019-05-14 15:46:57 -07005002 memcg->vmstats_percpu = alloc_percpu(struct memcg_vmstats_percpu);
5003 if (!memcg->vmstats_percpu)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005004 goto fail;
Pavel Emelianov78fb7462008-02-07 00:13:51 -08005005
Bob Liu3ed28fa2012-01-12 17:19:04 -08005006 for_each_node(node)
Mel Gormanef8f2322016-07-28 15:46:05 -07005007 if (alloc_mem_cgroup_per_node_info(memcg, node))
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005008 goto fail;
Balbir Singhf64c3f52009-09-23 15:56:37 -07005009
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005010 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5011 goto fail;
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005012
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005013 INIT_WORK(&memcg->high_work, high_work_func);
Glauber Costad142e3e2013-02-22 16:34:52 -08005014 INIT_LIST_HEAD(&memcg->oom_notify);
Glauber Costad142e3e2013-02-22 16:34:52 -08005015 mutex_init(&memcg->thresholds_lock);
5016 spin_lock_init(&memcg->move_lock);
Anton Vorontsov70ddf632013-04-29 15:08:31 -07005017 vmpressure_init(&memcg->vmpressure);
Tejun Heofba94802013-11-22 18:20:43 -05005018 INIT_LIST_HEAD(&memcg->event_list);
5019 spin_lock_init(&memcg->event_list_lock);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08005020 memcg->socket_pressure = jiffies;
Kirill Tkhai84c07d12018-08-17 15:47:25 -07005021#ifdef CONFIG_MEMCG_KMEM
Vladimir Davydov900a38f2014-12-12 16:55:10 -08005022 memcg->kmemcg_id = -1;
Vladimir Davydov900a38f2014-12-12 16:55:10 -08005023#endif
Tejun Heo52ebea72015-05-22 17:13:37 -04005024#ifdef CONFIG_CGROUP_WRITEBACK
5025 INIT_LIST_HEAD(&memcg->cgwb_list);
Tejun Heo97b27822019-08-26 09:06:56 -07005026 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5027 memcg->cgwb_frn[i].done =
5028 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
Tejun Heo52ebea72015-05-22 17:13:37 -04005029#endif
Yang Shi87eaceb2019-09-23 15:38:15 -07005030#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5031 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5032 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5033 memcg->deferred_split_queue.split_queue_len = 0;
5034#endif
Johannes Weiner73f576c2016-07-20 15:44:57 -07005035 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005036 return memcg;
5037fail:
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005038 mem_cgroup_id_remove(memcg);
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005039 __mem_cgroup_free(memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005040 return ERR_PTR(error);
Glauber Costad142e3e2013-02-22 16:34:52 -08005041}
5042
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005043static struct cgroup_subsys_state * __ref
5044mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
Glauber Costad142e3e2013-02-22 16:34:52 -08005045{
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005046 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
5047 struct mem_cgroup *memcg;
5048 long error = -ENOMEM;
Glauber Costad142e3e2013-02-22 16:34:52 -08005049
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005050 memcg = mem_cgroup_alloc();
Yafang Shao11d67612020-05-07 18:35:43 -07005051 if (IS_ERR(memcg))
5052 return ERR_CAST(memcg);
Li Zefan4219b2d2013-09-23 16:56:29 +08005053
Jakub Kicinskid1663a92020-06-01 21:49:49 -07005054 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005055 memcg->soft_limit = PAGE_COUNTER_MAX;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07005056 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005057 if (parent) {
5058 memcg->swappiness = mem_cgroup_swappiness(parent);
5059 memcg->oom_kill_disable = parent->oom_kill_disable;
5060 }
5061 if (parent && parent->use_hierarchy) {
5062 memcg->use_hierarchy = true;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005063 page_counter_init(&memcg->memory, &parent->memory);
Vladimir Davydov37e84352016-01-20 15:02:56 -08005064 page_counter_init(&memcg->swap, &parent->swap);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005065 page_counter_init(&memcg->memsw, &parent->memsw);
5066 page_counter_init(&memcg->kmem, &parent->kmem);
Johannes Weiner0db15292016-01-20 15:02:50 -08005067 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
Balbir Singh18f59ea2009-01-07 18:08:07 -08005068 } else {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005069 page_counter_init(&memcg->memory, NULL);
Vladimir Davydov37e84352016-01-20 15:02:56 -08005070 page_counter_init(&memcg->swap, NULL);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005071 page_counter_init(&memcg->memsw, NULL);
5072 page_counter_init(&memcg->kmem, NULL);
Johannes Weiner0db15292016-01-20 15:02:50 -08005073 page_counter_init(&memcg->tcpmem, NULL);
Tejun Heo8c7f6ed2012-09-13 12:20:58 -07005074 /*
5075 * Deeper hierachy with use_hierarchy == false doesn't make
5076 * much sense so let cgroup subsystem know about this
5077 * unfortunate state in our controller.
5078 */
Glauber Costad142e3e2013-02-22 16:34:52 -08005079 if (parent != root_mem_cgroup)
Tejun Heo073219e2014-02-08 10:36:58 -05005080 memory_cgrp_subsys.broken_hierarchy = true;
Balbir Singh18f59ea2009-01-07 18:08:07 -08005081 }
Vladimir Davydovd6441632014-01-23 15:53:09 -08005082
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005083 /* The following stuff does not apply to the root */
5084 if (!parent) {
Roman Gushchinfb2f2b02019-07-11 20:56:34 -07005085#ifdef CONFIG_MEMCG_KMEM
5086 INIT_LIST_HEAD(&memcg->kmem_caches);
5087#endif
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005088 root_mem_cgroup = memcg;
5089 return &memcg->css;
5090 }
5091
Vladimir Davydovb313aee2016-03-17 14:18:27 -07005092 error = memcg_online_kmem(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005093 if (error)
5094 goto fail;
Johannes Weiner127424c2016-01-20 15:02:32 -08005095
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005096 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
Johannes Weineref129472016-01-14 15:21:34 -08005097 static_branch_inc(&memcg_sockets_enabled_key);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005098
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005099 return &memcg->css;
5100fail:
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005101 mem_cgroup_id_remove(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005102 mem_cgroup_free(memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005103 return ERR_PTR(error);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005104}
5105
Johannes Weiner73f576c2016-07-20 15:44:57 -07005106static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005107{
Vladimir Davydov58fa2a52016-10-07 16:57:29 -07005108 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5109
Kirill Tkhai0a4465d2018-08-17 15:47:37 -07005110 /*
5111 * A memcg must be visible for memcg_expand_shrinker_maps()
5112 * by the time the maps are allocated. So, we allocate maps
5113 * here, when for_each_mem_cgroup() can't skip it.
5114 */
5115 if (memcg_alloc_shrinker_maps(memcg)) {
5116 mem_cgroup_id_remove(memcg);
5117 return -ENOMEM;
5118 }
5119
Johannes Weiner73f576c2016-07-20 15:44:57 -07005120 /* Online state pins memcg ID, memcg ID pins CSS */
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07005121 refcount_set(&memcg->id.ref, 1);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005122 css_get(css);
Johannes Weiner2f7dd7a2014-10-02 16:16:57 -07005123 return 0;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005124}
5125
Tejun Heoeb954192013-08-08 20:11:23 -04005126static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005127{
Tejun Heoeb954192013-08-08 20:11:23 -04005128 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo3bc942f2013-11-22 18:20:44 -05005129 struct mem_cgroup_event *event, *tmp;
Tejun Heo79bd9812013-11-22 18:20:42 -05005130
5131 /*
5132 * Unregister events and notify userspace.
5133 * Notify userspace about cgroup removing only after rmdir of cgroup
5134 * directory to avoid race between userspace and kernelspace.
5135 */
Tejun Heofba94802013-11-22 18:20:43 -05005136 spin_lock(&memcg->event_list_lock);
5137 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
Tejun Heo79bd9812013-11-22 18:20:42 -05005138 list_del_init(&event->list);
5139 schedule_work(&event->remove);
5140 }
Tejun Heofba94802013-11-22 18:20:43 -05005141 spin_unlock(&memcg->event_list_lock);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07005142
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005143 page_counter_set_min(&memcg->memory, 0);
Roman Gushchin23067152018-06-07 17:06:22 -07005144 page_counter_set_low(&memcg->memory, 0);
Roman Gushchin63677c742017-09-06 16:21:47 -07005145
Johannes Weiner567e9ab2016-01-20 15:02:24 -08005146 memcg_offline_kmem(memcg);
Tejun Heo52ebea72015-05-22 17:13:37 -04005147 wb_memcg_offline(memcg);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005148
Roman Gushchin591edfb2018-10-26 15:03:23 -07005149 drain_all_stock(memcg);
5150
Johannes Weiner73f576c2016-07-20 15:44:57 -07005151 mem_cgroup_id_put(memcg);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005152}
5153
Vladimir Davydov6df38682015-12-29 14:54:10 -08005154static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
5155{
5156 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5157
5158 invalidate_reclaim_iterators(memcg);
5159}
5160
Tejun Heoeb954192013-08-08 20:11:23 -04005161static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005162{
Tejun Heoeb954192013-08-08 20:11:23 -04005163 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo97b27822019-08-26 09:06:56 -07005164 int __maybe_unused i;
Daisuke Nishimurac268e992009-01-15 13:51:13 -08005165
Tejun Heo97b27822019-08-26 09:06:56 -07005166#ifdef CONFIG_CGROUP_WRITEBACK
5167 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5168 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5169#endif
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005170 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
Johannes Weineref129472016-01-14 15:21:34 -08005171 static_branch_dec(&memcg_sockets_enabled_key);
Johannes Weiner3893e302016-01-20 15:02:29 -08005172
Johannes Weiner0db15292016-01-20 15:02:50 -08005173 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08005174 static_branch_dec(&memcg_sockets_enabled_key);
Johannes Weiner3893e302016-01-20 15:02:29 -08005175
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005176 vmpressure_cleanup(&memcg->vmpressure);
5177 cancel_work_sync(&memcg->high_work);
5178 mem_cgroup_remove_from_trees(memcg);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -07005179 memcg_free_shrinker_maps(memcg);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08005180 memcg_free_kmem(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005181 mem_cgroup_free(memcg);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005182}
5183
Tejun Heo1ced953b2014-07-08 18:02:57 -04005184/**
5185 * mem_cgroup_css_reset - reset the states of a mem_cgroup
5186 * @css: the target css
5187 *
5188 * Reset the states of the mem_cgroup associated with @css. This is
5189 * invoked when the userland requests disabling on the default hierarchy
5190 * but the memcg is pinned through dependency. The memcg should stop
5191 * applying policies and should revert to the vanilla state as it may be
5192 * made visible again.
5193 *
5194 * The current implementation only resets the essential configurations.
5195 * This needs to be expanded to cover all the visible parts.
5196 */
5197static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
5198{
5199 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5200
Roman Gushchinbbec2e12018-06-07 17:06:18 -07005201 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5202 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5203 page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX);
5204 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5205 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005206 page_counter_set_min(&memcg->memory, 0);
Roman Gushchin23067152018-06-07 17:06:22 -07005207 page_counter_set_low(&memcg->memory, 0);
Jakub Kicinskid1663a92020-06-01 21:49:49 -07005208 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
Johannes Weiner24d404d2015-01-08 14:32:35 -08005209 memcg->soft_limit = PAGE_COUNTER_MAX;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07005210 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
Tejun Heo2529bb32015-05-22 18:23:34 -04005211 memcg_wb_domain_size_changed(memcg);
Tejun Heo1ced953b2014-07-08 18:02:57 -04005212}
5213
Daisuke Nishimura02491442010-03-10 15:22:17 -08005214#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005215/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005216static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005217{
Johannes Weiner05b84302014-08-06 16:05:59 -07005218 int ret;
Johannes Weiner9476db92014-08-06 16:05:55 -07005219
Mel Gormand0164ad2015-11-06 16:28:21 -08005220 /* Try a single bulk charge without reclaim first, kswapd may wake */
5221 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
Johannes Weiner9476db92014-08-06 16:05:55 -07005222 if (!ret) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005223 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005224 return ret;
5225 }
Johannes Weiner9476db92014-08-06 16:05:55 -07005226
David Rientjes36745342017-01-24 15:18:10 -08005227 /* Try charges one by one with reclaim, but do not retry */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005228 while (count--) {
David Rientjes36745342017-01-24 15:18:10 -08005229 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08005230 if (ret)
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08005231 return ret;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005232 mc.precharge++;
Johannes Weiner9476db92014-08-06 16:05:55 -07005233 cond_resched();
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005234 }
Johannes Weiner9476db92014-08-06 16:05:55 -07005235 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005236}
5237
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005238union mc_target {
5239 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005240 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005241};
5242
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005243enum mc_target_type {
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005244 MC_TARGET_NONE = 0,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005245 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08005246 MC_TARGET_SWAP,
Jérôme Glissec733a822017-09-08 16:11:54 -07005247 MC_TARGET_DEVICE,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005248};
5249
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005250static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
5251 unsigned long addr, pte_t ptent)
5252{
Christoph Hellwig25b29952019-06-13 22:50:49 +02005253 struct page *page = vm_normal_page(vma, addr, ptent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005254
5255 if (!page || !page_mapped(page))
5256 return NULL;
5257 if (PageAnon(page)) {
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005258 if (!(mc.flags & MOVE_ANON))
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005259 return NULL;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005260 } else {
5261 if (!(mc.flags & MOVE_FILE))
5262 return NULL;
5263 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005264 if (!get_page_unless_zero(page))
5265 return NULL;
5266
5267 return page;
5268}
5269
Jérôme Glissec733a822017-09-08 16:11:54 -07005270#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005271static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
Li RongQing48406ef2016-07-26 15:22:14 -07005272 pte_t ptent, swp_entry_t *entry)
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005273{
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005274 struct page *page = NULL;
5275 swp_entry_t ent = pte_to_swp_entry(ptent);
5276
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005277 if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005278 return NULL;
Jérôme Glissec733a822017-09-08 16:11:54 -07005279
5280 /*
5281 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
5282 * a device and because they are not accessible by CPU they are store
5283 * as special swap entry in the CPU page table.
5284 */
5285 if (is_device_private_entry(ent)) {
5286 page = device_private_entry_to_page(ent);
5287 /*
5288 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
5289 * a refcount of 1 when free (unlike normal page)
5290 */
5291 if (!page_ref_add_unless(page, 1, 1))
5292 return NULL;
5293 return page;
5294 }
5295
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005296 /*
5297 * Because lookup_swap_cache() updates some statistics counter,
5298 * we call find_get_page() with swapper_space directly.
5299 */
Huang Yingf6ab1f72016-10-07 17:00:21 -07005300 page = find_get_page(swap_address_space(ent), swp_offset(ent));
Johannes Weiner2d1c4982020-06-03 16:02:14 -07005301 entry->val = ent.val;
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005302
5303 return page;
5304}
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005305#else
5306static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
Li RongQing48406ef2016-07-26 15:22:14 -07005307 pte_t ptent, swp_entry_t *entry)
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005308{
5309 return NULL;
5310}
5311#endif
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005312
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005313static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
5314 unsigned long addr, pte_t ptent, swp_entry_t *entry)
5315{
5316 struct page *page = NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005317 struct address_space *mapping;
5318 pgoff_t pgoff;
5319
5320 if (!vma->vm_file) /* anonymous vma */
5321 return NULL;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005322 if (!(mc.flags & MOVE_FILE))
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005323 return NULL;
5324
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005325 mapping = vma->vm_file->f_mapping;
Kirill A. Shutemov0661a332015-02-10 14:10:04 -08005326 pgoff = linear_page_index(vma, addr);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005327
5328 /* page is moved even if it's not RSS of this task(page-faulted). */
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07005329#ifdef CONFIG_SWAP
5330 /* shmem/tmpfs may report page out on swap: account for that too. */
Johannes Weiner139b6a62014-05-06 12:50:05 -07005331 if (shmem_mapping(mapping)) {
5332 page = find_get_entry(mapping, pgoff);
Matthew Wilcox3159f942017-11-03 13:30:42 -04005333 if (xa_is_value(page)) {
Johannes Weiner139b6a62014-05-06 12:50:05 -07005334 swp_entry_t swp = radix_to_swp_entry(page);
Johannes Weiner2d1c4982020-06-03 16:02:14 -07005335 *entry = swp;
Huang Yingf6ab1f72016-10-07 17:00:21 -07005336 page = find_get_page(swap_address_space(swp),
5337 swp_offset(swp));
Johannes Weiner139b6a62014-05-06 12:50:05 -07005338 }
5339 } else
5340 page = find_get_page(mapping, pgoff);
5341#else
5342 page = find_get_page(mapping, pgoff);
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07005343#endif
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005344 return page;
5345}
5346
Chen Gangb1b0dea2015-04-14 15:47:35 -07005347/**
5348 * mem_cgroup_move_account - move account of the page
5349 * @page: the page
Li RongQing25843c22016-07-26 15:26:56 -07005350 * @compound: charge the page as compound or small page
Chen Gangb1b0dea2015-04-14 15:47:35 -07005351 * @from: mem_cgroup which the page is moved from.
5352 * @to: mem_cgroup which the page is moved to. @from != @to.
5353 *
Kirill A. Shutemov3ac808f2016-01-15 16:53:07 -08005354 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
Chen Gangb1b0dea2015-04-14 15:47:35 -07005355 *
5356 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
5357 * from old cgroup.
5358 */
5359static int mem_cgroup_move_account(struct page *page,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005360 bool compound,
Chen Gangb1b0dea2015-04-14 15:47:35 -07005361 struct mem_cgroup *from,
5362 struct mem_cgroup *to)
5363{
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005364 struct lruvec *from_vec, *to_vec;
5365 struct pglist_data *pgdat;
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005366 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
Chen Gangb1b0dea2015-04-14 15:47:35 -07005367 int ret;
5368
5369 VM_BUG_ON(from == to);
5370 VM_BUG_ON_PAGE(PageLRU(page), page);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005371 VM_BUG_ON(compound && !PageTransHuge(page));
Chen Gangb1b0dea2015-04-14 15:47:35 -07005372
5373 /*
Johannes Weiner6a93ca82016-03-15 14:57:19 -07005374 * Prevent mem_cgroup_migrate() from looking at
Hugh Dickins45637ba2015-11-05 18:49:40 -08005375 * page->mem_cgroup of its source page while we change it.
Chen Gangb1b0dea2015-04-14 15:47:35 -07005376 */
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005377 ret = -EBUSY;
Chen Gangb1b0dea2015-04-14 15:47:35 -07005378 if (!trylock_page(page))
5379 goto out;
5380
5381 ret = -EINVAL;
5382 if (page->mem_cgroup != from)
5383 goto out_unlock;
5384
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005385 pgdat = page_pgdat(page);
Johannes Weiner867e5e12019-11-30 17:55:34 -08005386 from_vec = mem_cgroup_lruvec(from, pgdat);
5387 to_vec = mem_cgroup_lruvec(to, pgdat);
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005388
Johannes Weinerabb242f2020-06-03 16:01:28 -07005389 lock_page_memcg(page);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005390
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07005391 if (PageAnon(page)) {
5392 if (page_mapped(page)) {
5393 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
5394 __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
Johannes Weiner468c3982020-06-03 16:02:01 -07005395 if (PageTransHuge(page)) {
5396 __mod_lruvec_state(from_vec, NR_ANON_THPS,
5397 -nr_pages);
5398 __mod_lruvec_state(to_vec, NR_ANON_THPS,
5399 nr_pages);
5400 }
5401
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07005402 }
5403 } else {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07005404 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
5405 __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
5406
5407 if (PageSwapBacked(page)) {
5408 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
5409 __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
5410 }
5411
Johannes Weiner49e50d22020-06-03 16:01:47 -07005412 if (page_mapped(page)) {
5413 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
5414 __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
5415 }
Chen Gangb1b0dea2015-04-14 15:47:35 -07005416
Johannes Weiner49e50d22020-06-03 16:01:47 -07005417 if (PageDirty(page)) {
5418 struct address_space *mapping = page_mapping(page);
Greg Thelenc4843a72015-05-22 17:13:16 -04005419
Johannes Weiner49e50d22020-06-03 16:01:47 -07005420 if (mapping_cap_account_dirty(mapping)) {
5421 __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
5422 -nr_pages);
5423 __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
5424 nr_pages);
5425 }
Greg Thelenc4843a72015-05-22 17:13:16 -04005426 }
5427 }
5428
Chen Gangb1b0dea2015-04-14 15:47:35 -07005429 if (PageWriteback(page)) {
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005430 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
5431 __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005432 }
5433
5434 /*
Johannes Weinerabb242f2020-06-03 16:01:28 -07005435 * All state has been migrated, let's switch to the new memcg.
5436 *
Chen Gangb1b0dea2015-04-14 15:47:35 -07005437 * It is safe to change page->mem_cgroup here because the page
Johannes Weinerabb242f2020-06-03 16:01:28 -07005438 * is referenced, charged, isolated, and locked: we can't race
5439 * with (un)charging, migration, LRU putback, or anything else
5440 * that would rely on a stable page->mem_cgroup.
5441 *
5442 * Note that lock_page_memcg is a memcg lock, not a page lock,
5443 * to save space. As soon as we switch page->mem_cgroup to a
5444 * new memcg that isn't locked, the above state can change
5445 * concurrently again. Make sure we're truly done with it.
Chen Gangb1b0dea2015-04-14 15:47:35 -07005446 */
Johannes Weinerabb242f2020-06-03 16:01:28 -07005447 smp_mb();
Chen Gangb1b0dea2015-04-14 15:47:35 -07005448
Johannes Weinerabb242f2020-06-03 16:01:28 -07005449 page->mem_cgroup = to; /* caller should have done css_get */
Yang Shi87eaceb2019-09-23 15:38:15 -07005450
Johannes Weinerabb242f2020-06-03 16:01:28 -07005451 __unlock_page_memcg(from);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005452
5453 ret = 0;
5454
5455 local_irq_disable();
Johannes Weiner3fba69a2020-06-03 16:01:31 -07005456 mem_cgroup_charge_statistics(to, page, nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005457 memcg_check_events(to, page);
Johannes Weiner3fba69a2020-06-03 16:01:31 -07005458 mem_cgroup_charge_statistics(from, page, -nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005459 memcg_check_events(from, page);
5460 local_irq_enable();
5461out_unlock:
5462 unlock_page(page);
5463out:
5464 return ret;
5465}
5466
Li RongQing7cf78062016-05-27 14:27:46 -07005467/**
5468 * get_mctgt_type - get target type of moving charge
5469 * @vma: the vma the pte to be checked belongs
5470 * @addr: the address corresponding to the pte to be checked
5471 * @ptent: the pte to be checked
5472 * @target: the pointer the target page or swap ent will be stored(can be NULL)
5473 *
5474 * Returns
5475 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
5476 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
5477 * move charge. if @target is not NULL, the page is stored in target->page
5478 * with extra refcnt got(Callers should handle it).
5479 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
5480 * target for charge migration. if @target is not NULL, the entry is stored
5481 * in target->ent.
Christoph Hellwig25b29952019-06-13 22:50:49 +02005482 * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE
5483 * (so ZONE_DEVICE page and thus not on the lru).
Jérôme Glissedf6ad692017-09-08 16:12:24 -07005484 * For now we such page is charge like a regular page would be as for all
5485 * intent and purposes it is just special memory taking the place of a
5486 * regular page.
Jérôme Glissec733a822017-09-08 16:11:54 -07005487 *
5488 * See Documentations/vm/hmm.txt and include/linux/hmm.h
Li RongQing7cf78062016-05-27 14:27:46 -07005489 *
5490 * Called with pte lock held.
5491 */
5492
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005493static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005494 unsigned long addr, pte_t ptent, union mc_target *target)
5495{
Daisuke Nishimura02491442010-03-10 15:22:17 -08005496 struct page *page = NULL;
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005497 enum mc_target_type ret = MC_TARGET_NONE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005498 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005499
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005500 if (pte_present(ptent))
5501 page = mc_handle_present_pte(vma, addr, ptent);
5502 else if (is_swap_pte(ptent))
Li RongQing48406ef2016-07-26 15:22:14 -07005503 page = mc_handle_swap_pte(vma, ptent, &ent);
Kirill A. Shutemov0661a332015-02-10 14:10:04 -08005504 else if (pte_none(ptent))
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005505 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005506
5507 if (!page && !ent.val)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005508 return ret;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005509 if (page) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005510 /*
Johannes Weiner0a31bc92014-08-08 14:19:22 -07005511 * Do only loose check w/o serialization.
Johannes Weiner1306a852014-12-10 15:44:52 -08005512 * mem_cgroup_move_account() checks the page is valid or
Johannes Weiner0a31bc92014-08-08 14:19:22 -07005513 * not under LRU exclusion.
Daisuke Nishimura02491442010-03-10 15:22:17 -08005514 */
Johannes Weiner1306a852014-12-10 15:44:52 -08005515 if (page->mem_cgroup == mc.from) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005516 ret = MC_TARGET_PAGE;
Christoph Hellwig25b29952019-06-13 22:50:49 +02005517 if (is_device_private_page(page))
Jérôme Glissec733a822017-09-08 16:11:54 -07005518 ret = MC_TARGET_DEVICE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005519 if (target)
5520 target->page = page;
5521 }
5522 if (!ret || !target)
5523 put_page(page);
5524 }
Huang Ying3e14a572017-09-06 16:22:37 -07005525 /*
5526 * There is a swap entry and a page doesn't exist or isn't charged.
5527 * But we cannot move a tail-page in a THP.
5528 */
5529 if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
Li Zefan34c00c32013-09-23 16:56:01 +08005530 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07005531 ret = MC_TARGET_SWAP;
5532 if (target)
5533 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005534 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005535 return ret;
5536}
5537
Naoya Horiguchi12724852012-03-21 16:34:28 -07005538#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5539/*
Huang Yingd6810d732017-09-06 16:22:45 -07005540 * We don't consider PMD mapped swapping or file mapped pages because THP does
5541 * not support them for now.
Naoya Horiguchi12724852012-03-21 16:34:28 -07005542 * Caller should make sure that pmd_trans_huge(pmd) is true.
5543 */
5544static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5545 unsigned long addr, pmd_t pmd, union mc_target *target)
5546{
5547 struct page *page = NULL;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005548 enum mc_target_type ret = MC_TARGET_NONE;
5549
Zi Yan84c3fc42017-09-08 16:11:01 -07005550 if (unlikely(is_swap_pmd(pmd))) {
5551 VM_BUG_ON(thp_migration_supported() &&
5552 !is_pmd_migration_entry(pmd));
5553 return ret;
5554 }
Naoya Horiguchi12724852012-03-21 16:34:28 -07005555 page = pmd_page(pmd);
Sasha Levin309381fea2014-01-23 15:52:54 -08005556 VM_BUG_ON_PAGE(!page || !PageHead(page), page);
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005557 if (!(mc.flags & MOVE_ANON))
Naoya Horiguchi12724852012-03-21 16:34:28 -07005558 return ret;
Johannes Weiner1306a852014-12-10 15:44:52 -08005559 if (page->mem_cgroup == mc.from) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07005560 ret = MC_TARGET_PAGE;
5561 if (target) {
5562 get_page(page);
5563 target->page = page;
5564 }
5565 }
5566 return ret;
5567}
5568#else
5569static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5570 unsigned long addr, pmd_t pmd, union mc_target *target)
5571{
5572 return MC_TARGET_NONE;
5573}
5574#endif
5575
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005576static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
5577 unsigned long addr, unsigned long end,
5578 struct mm_walk *walk)
5579{
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08005580 struct vm_area_struct *vma = walk->vma;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005581 pte_t *pte;
5582 spinlock_t *ptl;
5583
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08005584 ptl = pmd_trans_huge_lock(pmd, vma);
5585 if (ptl) {
Jérôme Glissec733a822017-09-08 16:11:54 -07005586 /*
5587 * Note their can not be MC_TARGET_DEVICE for now as we do not
Christoph Hellwig25b29952019-06-13 22:50:49 +02005588 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
5589 * this might change.
Jérôme Glissec733a822017-09-08 16:11:54 -07005590 */
Naoya Horiguchi12724852012-03-21 16:34:28 -07005591 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
5592 mc.precharge += HPAGE_PMD_NR;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005593 spin_unlock(ptl);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07005594 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005595 }
Dave Hansen03319322011-03-22 16:32:56 -07005596
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07005597 if (pmd_trans_unstable(pmd))
5598 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005599 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5600 for (; addr != end; pte++, addr += PAGE_SIZE)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005601 if (get_mctgt_type(vma, addr, *pte, NULL))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005602 mc.precharge++; /* increment precharge temporarily */
5603 pte_unmap_unlock(pte - 1, ptl);
5604 cond_resched();
5605
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005606 return 0;
5607}
5608
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005609static const struct mm_walk_ops precharge_walk_ops = {
5610 .pmd_entry = mem_cgroup_count_precharge_pte_range,
5611};
5612
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005613static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5614{
5615 unsigned long precharge;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005616
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005617 mmap_read_lock(mm);
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005618 walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005619 mmap_read_unlock(mm);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005620
5621 precharge = mc.precharge;
5622 mc.precharge = 0;
5623
5624 return precharge;
5625}
5626
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005627static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5628{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005629 unsigned long precharge = mem_cgroup_count_precharge(mm);
5630
5631 VM_BUG_ON(mc.moving_task);
5632 mc.moving_task = current;
5633 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005634}
5635
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005636/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
5637static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005638{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005639 struct mem_cgroup *from = mc.from;
5640 struct mem_cgroup *to = mc.to;
5641
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005642 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005643 if (mc.precharge) {
Johannes Weiner00501b52014-08-08 14:19:20 -07005644 cancel_charge(mc.to, mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005645 mc.precharge = 0;
5646 }
5647 /*
5648 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5649 * we must uncharge here.
5650 */
5651 if (mc.moved_charge) {
Johannes Weiner00501b52014-08-08 14:19:20 -07005652 cancel_charge(mc.from, mc.moved_charge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005653 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005654 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005655 /* we must fixup refcnts and charges */
5656 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005657 /* uncharge swap account from the old cgroup */
Johannes Weinerce00a962014-09-05 08:43:57 -04005658 if (!mem_cgroup_is_root(mc.from))
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005659 page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005660
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005661 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5662
Johannes Weiner05b84302014-08-06 16:05:59 -07005663 /*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005664 * we charged both to->memory and to->memsw, so we
5665 * should uncharge to->memory.
Johannes Weiner05b84302014-08-06 16:05:59 -07005666 */
Johannes Weinerce00a962014-09-05 08:43:57 -04005667 if (!mem_cgroup_is_root(mc.to))
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005668 page_counter_uncharge(&mc.to->memory, mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005669
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005670 mem_cgroup_id_get_many(mc.to, mc.moved_swap);
5671 css_put_many(&mc.to->css, mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005672
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005673 mc.moved_swap = 0;
5674 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005675 memcg_oom_recover(from);
5676 memcg_oom_recover(to);
5677 wake_up_all(&mc.waitq);
5678}
5679
5680static void mem_cgroup_clear_mc(void)
5681{
Tejun Heo264a0ae2016-04-21 19:09:02 -04005682 struct mm_struct *mm = mc.mm;
5683
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005684 /*
5685 * we must clear moving_task before waking up waiters at the end of
5686 * task migration.
5687 */
5688 mc.moving_task = NULL;
5689 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005690 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005691 mc.from = NULL;
5692 mc.to = NULL;
Tejun Heo264a0ae2016-04-21 19:09:02 -04005693 mc.mm = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005694 spin_unlock(&mc.lock);
Tejun Heo264a0ae2016-04-21 19:09:02 -04005695
5696 mmput(mm);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005697}
5698
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005699static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005700{
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005701 struct cgroup_subsys_state *css;
Ross Zwislereed67d72015-12-23 14:53:27 -07005702 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
Tejun Heo9f2115f2015-09-08 15:01:10 -07005703 struct mem_cgroup *from;
Tejun Heo4530edd2015-09-11 15:00:19 -04005704 struct task_struct *leader, *p;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005705 struct mm_struct *mm;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005706 unsigned long move_flags;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005707 int ret = 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005708
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005709 /* charge immigration isn't supported on the default hierarchy */
5710 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Tejun Heo9f2115f2015-09-08 15:01:10 -07005711 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005712
Tejun Heo4530edd2015-09-11 15:00:19 -04005713 /*
5714 * Multi-process migrations only happen on the default hierarchy
5715 * where charge immigration is not used. Perform charge
5716 * immigration if @tset contains a leader and whine if there are
5717 * multiple.
5718 */
5719 p = NULL;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005720 cgroup_taskset_for_each_leader(leader, css, tset) {
Tejun Heo4530edd2015-09-11 15:00:19 -04005721 WARN_ON_ONCE(p);
5722 p = leader;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005723 memcg = mem_cgroup_from_css(css);
Tejun Heo4530edd2015-09-11 15:00:19 -04005724 }
5725 if (!p)
5726 return 0;
5727
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005728 /*
5729 * We are now commited to this value whatever it is. Changes in this
5730 * tunable will only affect upcoming migrations, not the current one.
5731 * So we need to save it, and keep it going.
5732 */
5733 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
5734 if (!move_flags)
5735 return 0;
5736
Tejun Heo9f2115f2015-09-08 15:01:10 -07005737 from = mem_cgroup_from_task(p);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005738
Tejun Heo9f2115f2015-09-08 15:01:10 -07005739 VM_BUG_ON(from == memcg);
Johannes Weiner247b1442014-12-10 15:44:11 -08005740
Tejun Heo9f2115f2015-09-08 15:01:10 -07005741 mm = get_task_mm(p);
5742 if (!mm)
5743 return 0;
5744 /* We move charges only when we move a owner of the mm */
5745 if (mm->owner == p) {
5746 VM_BUG_ON(mc.from);
5747 VM_BUG_ON(mc.to);
5748 VM_BUG_ON(mc.precharge);
5749 VM_BUG_ON(mc.moved_charge);
5750 VM_BUG_ON(mc.moved_swap);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005751
Tejun Heo9f2115f2015-09-08 15:01:10 -07005752 spin_lock(&mc.lock);
Tejun Heo264a0ae2016-04-21 19:09:02 -04005753 mc.mm = mm;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005754 mc.from = from;
5755 mc.to = memcg;
5756 mc.flags = move_flags;
5757 spin_unlock(&mc.lock);
5758 /* We set mc.moving_task later */
5759
5760 ret = mem_cgroup_precharge_mc(mm);
5761 if (ret)
5762 mem_cgroup_clear_mc();
Tejun Heo264a0ae2016-04-21 19:09:02 -04005763 } else {
5764 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005765 }
5766 return ret;
5767}
5768
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005769static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005770{
Johannes Weiner4e2f2452014-12-10 15:44:08 -08005771 if (mc.to)
5772 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005773}
5774
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005775static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
5776 unsigned long addr, unsigned long end,
5777 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005778{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005779 int ret = 0;
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08005780 struct vm_area_struct *vma = walk->vma;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005781 pte_t *pte;
5782 spinlock_t *ptl;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005783 enum mc_target_type target_type;
5784 union mc_target target;
5785 struct page *page;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005786
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08005787 ptl = pmd_trans_huge_lock(pmd, vma);
5788 if (ptl) {
Hugh Dickins62ade862012-05-18 11:28:34 -07005789 if (mc.precharge < HPAGE_PMD_NR) {
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005790 spin_unlock(ptl);
Naoya Horiguchi12724852012-03-21 16:34:28 -07005791 return 0;
5792 }
5793 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
5794 if (target_type == MC_TARGET_PAGE) {
5795 page = target.page;
5796 if (!isolate_lru_page(page)) {
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005797 if (!mem_cgroup_move_account(page, true,
Johannes Weiner1306a852014-12-10 15:44:52 -08005798 mc.from, mc.to)) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07005799 mc.precharge -= HPAGE_PMD_NR;
5800 mc.moved_charge += HPAGE_PMD_NR;
5801 }
5802 putback_lru_page(page);
5803 }
5804 put_page(page);
Jérôme Glissec733a822017-09-08 16:11:54 -07005805 } else if (target_type == MC_TARGET_DEVICE) {
5806 page = target.page;
5807 if (!mem_cgroup_move_account(page, true,
5808 mc.from, mc.to)) {
5809 mc.precharge -= HPAGE_PMD_NR;
5810 mc.moved_charge += HPAGE_PMD_NR;
5811 }
5812 put_page(page);
Naoya Horiguchi12724852012-03-21 16:34:28 -07005813 }
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005814 spin_unlock(ptl);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07005815 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005816 }
5817
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07005818 if (pmd_trans_unstable(pmd))
5819 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005820retry:
5821 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5822 for (; addr != end; addr += PAGE_SIZE) {
5823 pte_t ptent = *(pte++);
Jérôme Glissec733a822017-09-08 16:11:54 -07005824 bool device = false;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005825 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005826
5827 if (!mc.precharge)
5828 break;
5829
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005830 switch (get_mctgt_type(vma, addr, ptent, &target)) {
Jérôme Glissec733a822017-09-08 16:11:54 -07005831 case MC_TARGET_DEVICE:
5832 device = true;
Joe Perchese4a9bc52020-04-06 20:08:39 -07005833 fallthrough;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005834 case MC_TARGET_PAGE:
5835 page = target.page;
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08005836 /*
5837 * We can have a part of the split pmd here. Moving it
5838 * can be done but it would be too convoluted so simply
5839 * ignore such a partial THP and keep it in original
5840 * memcg. There should be somebody mapping the head.
5841 */
5842 if (PageTransCompound(page))
5843 goto put;
Jérôme Glissec733a822017-09-08 16:11:54 -07005844 if (!device && isolate_lru_page(page))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005845 goto put;
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005846 if (!mem_cgroup_move_account(page, false,
5847 mc.from, mc.to)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005848 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005849 /* we uncharge from mc.from later. */
5850 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005851 }
Jérôme Glissec733a822017-09-08 16:11:54 -07005852 if (!device)
5853 putback_lru_page(page);
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005854put: /* get_mctgt_type() gets the page */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005855 put_page(page);
5856 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005857 case MC_TARGET_SWAP:
5858 ent = target.ent;
Hugh Dickinse91cbb42012-05-29 15:06:51 -07005859 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005860 mc.precharge--;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005861 /* we fixup refcnts and charges later. */
5862 mc.moved_swap++;
5863 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08005864 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005865 default:
5866 break;
5867 }
5868 }
5869 pte_unmap_unlock(pte - 1, ptl);
5870 cond_resched();
5871
5872 if (addr != end) {
5873 /*
5874 * We have consumed all precharges we got in can_attach().
5875 * We try charge one by one, but don't do any additional
5876 * charges to mc.to if we have failed in charge once in attach()
5877 * phase.
5878 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005879 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005880 if (!ret)
5881 goto retry;
5882 }
5883
5884 return ret;
5885}
5886
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005887static const struct mm_walk_ops charge_walk_ops = {
5888 .pmd_entry = mem_cgroup_move_charge_pte_range,
5889};
5890
Tejun Heo264a0ae2016-04-21 19:09:02 -04005891static void mem_cgroup_move_charge(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005892{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005893 lru_add_drain_all();
Johannes Weiner312722c2014-12-10 15:44:25 -08005894 /*
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07005895 * Signal lock_page_memcg() to take the memcg's move_lock
5896 * while we're moving its pages to another memcg. Then wait
5897 * for already started RCU-only updates to finish.
Johannes Weiner312722c2014-12-10 15:44:25 -08005898 */
5899 atomic_inc(&mc.from->moving_account);
5900 synchronize_rcu();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005901retry:
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005902 if (unlikely(!mmap_read_trylock(mc.mm))) {
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005903 /*
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -07005904 * Someone who are holding the mmap_lock might be waiting in
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005905 * waitq. So we cancel all extra charges, wake up all waiters,
5906 * and retry. Because we cancel precharges, we might not be able
5907 * to move enough charges, but moving charge is a best-effort
5908 * feature anyway, so it wouldn't be a big problem.
5909 */
5910 __mem_cgroup_clear_mc();
5911 cond_resched();
5912 goto retry;
5913 }
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08005914 /*
5915 * When we have consumed all precharges and failed in doing
5916 * additional charge, the page walk just aborts.
5917 */
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005918 walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
5919 NULL);
James Morse0247f3f2016-10-07 17:00:12 -07005920
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005921 mmap_read_unlock(mc.mm);
Johannes Weiner312722c2014-12-10 15:44:25 -08005922 atomic_dec(&mc.from->moving_account);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005923}
5924
Tejun Heo264a0ae2016-04-21 19:09:02 -04005925static void mem_cgroup_move_task(void)
Balbir Singh67e465a2008-02-07 00:13:54 -08005926{
Tejun Heo264a0ae2016-04-21 19:09:02 -04005927 if (mc.to) {
5928 mem_cgroup_move_charge();
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07005929 mem_cgroup_clear_mc();
Tejun Heo264a0ae2016-04-21 19:09:02 -04005930 }
Balbir Singh67e465a2008-02-07 00:13:54 -08005931}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005932#else /* !CONFIG_MMU */
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005933static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005934{
5935 return 0;
5936}
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005937static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005938{
5939}
Tejun Heo264a0ae2016-04-21 19:09:02 -04005940static void mem_cgroup_move_task(void)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005941{
5942}
5943#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08005944
Tejun Heof00baae2013-04-15 13:41:15 -07005945/*
5946 * Cgroup retains root cgroups across [un]mount cycles making it necessary
Tejun Heoaa6ec292014-07-09 10:08:08 -04005947 * to verify whether we're attached to the default hierarchy on each mount
5948 * attempt.
Tejun Heof00baae2013-04-15 13:41:15 -07005949 */
Tejun Heoeb954192013-08-08 20:11:23 -04005950static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
Tejun Heof00baae2013-04-15 13:41:15 -07005951{
5952 /*
Tejun Heoaa6ec292014-07-09 10:08:08 -04005953 * use_hierarchy is forced on the default hierarchy. cgroup core
Tejun Heof00baae2013-04-15 13:41:15 -07005954 * guarantees that @root doesn't have any children, so turning it
5955 * on for the root memcg is enough.
5956 */
Tejun Heo9e10a132015-09-18 11:56:28 -04005957 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydov7feee5902015-03-12 16:26:19 -07005958 root_mem_cgroup->use_hierarchy = true;
5959 else
5960 root_mem_cgroup->use_hierarchy = false;
Tejun Heof00baae2013-04-15 13:41:15 -07005961}
5962
Chris Down677dc972019-03-05 15:45:55 -08005963static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
5964{
5965 if (value == PAGE_COUNTER_MAX)
5966 seq_puts(m, "max\n");
5967 else
5968 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
5969
5970 return 0;
5971}
5972
Johannes Weiner241994ed2015-02-11 15:26:06 -08005973static u64 memory_current_read(struct cgroup_subsys_state *css,
5974 struct cftype *cft)
5975{
Johannes Weinerf5fc3c5d2015-11-05 18:50:23 -08005976 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5977
5978 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
Johannes Weiner241994ed2015-02-11 15:26:06 -08005979}
5980
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005981static int memory_min_show(struct seq_file *m, void *v)
5982{
Chris Down677dc972019-03-05 15:45:55 -08005983 return seq_puts_memcg_tunable(m,
5984 READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005985}
5986
5987static ssize_t memory_min_write(struct kernfs_open_file *of,
5988 char *buf, size_t nbytes, loff_t off)
5989{
5990 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5991 unsigned long min;
5992 int err;
5993
5994 buf = strstrip(buf);
5995 err = page_counter_memparse(buf, "max", &min);
5996 if (err)
5997 return err;
5998
5999 page_counter_set_min(&memcg->memory, min);
6000
6001 return nbytes;
6002}
6003
Johannes Weiner241994ed2015-02-11 15:26:06 -08006004static int memory_low_show(struct seq_file *m, void *v)
6005{
Chris Down677dc972019-03-05 15:45:55 -08006006 return seq_puts_memcg_tunable(m,
6007 READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006008}
6009
6010static ssize_t memory_low_write(struct kernfs_open_file *of,
6011 char *buf, size_t nbytes, loff_t off)
6012{
6013 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6014 unsigned long low;
6015 int err;
6016
6017 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006018 err = page_counter_memparse(buf, "max", &low);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006019 if (err)
6020 return err;
6021
Roman Gushchin23067152018-06-07 17:06:22 -07006022 page_counter_set_low(&memcg->memory, low);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006023
6024 return nbytes;
6025}
6026
6027static int memory_high_show(struct seq_file *m, void *v)
6028{
Jakub Kicinskid1663a92020-06-01 21:49:49 -07006029 return seq_puts_memcg_tunable(m,
6030 READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006031}
6032
6033static ssize_t memory_high_write(struct kernfs_open_file *of,
6034 char *buf, size_t nbytes, loff_t off)
6035{
6036 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006037 unsigned int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
6038 bool drained = false;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006039 unsigned long high;
6040 int err;
6041
6042 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006043 err = page_counter_memparse(buf, "max", &high);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006044 if (err)
6045 return err;
6046
Jakub Kicinskid1663a92020-06-01 21:49:49 -07006047 page_counter_set_high(&memcg->memory, high);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006048
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006049 for (;;) {
6050 unsigned long nr_pages = page_counter_read(&memcg->memory);
6051 unsigned long reclaimed;
Johannes Weiner588083b2016-03-17 14:20:25 -07006052
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006053 if (nr_pages <= high)
6054 break;
6055
6056 if (signal_pending(current))
6057 break;
6058
6059 if (!drained) {
6060 drain_all_stock(memcg);
6061 drained = true;
6062 continue;
6063 }
6064
6065 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6066 GFP_KERNEL, true);
6067
6068 if (!reclaimed && !nr_retries--)
6069 break;
6070 }
6071
Johannes Weiner241994ed2015-02-11 15:26:06 -08006072 return nbytes;
6073}
6074
6075static int memory_max_show(struct seq_file *m, void *v)
6076{
Chris Down677dc972019-03-05 15:45:55 -08006077 return seq_puts_memcg_tunable(m,
6078 READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006079}
6080
6081static ssize_t memory_max_write(struct kernfs_open_file *of,
6082 char *buf, size_t nbytes, loff_t off)
6083{
6084 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006085 unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
6086 bool drained = false;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006087 unsigned long max;
6088 int err;
6089
6090 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006091 err = page_counter_memparse(buf, "max", &max);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006092 if (err)
6093 return err;
6094
Roman Gushchinbbec2e12018-06-07 17:06:18 -07006095 xchg(&memcg->memory.max, max);
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006096
6097 for (;;) {
6098 unsigned long nr_pages = page_counter_read(&memcg->memory);
6099
6100 if (nr_pages <= max)
6101 break;
6102
Johannes Weiner7249c9f2019-11-30 17:50:06 -08006103 if (signal_pending(current))
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006104 break;
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006105
6106 if (!drained) {
6107 drain_all_stock(memcg);
6108 drained = true;
6109 continue;
6110 }
6111
6112 if (nr_reclaims) {
6113 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6114 GFP_KERNEL, true))
6115 nr_reclaims--;
6116 continue;
6117 }
6118
Johannes Weinere27be242018-04-10 16:29:45 -07006119 memcg_memory_event(memcg, MEMCG_OOM);
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006120 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6121 break;
6122 }
Johannes Weiner241994ed2015-02-11 15:26:06 -08006123
Tejun Heo2529bb32015-05-22 18:23:34 -04006124 memcg_wb_domain_size_changed(memcg);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006125 return nbytes;
6126}
6127
Shakeel Butt1e577f92019-07-11 20:55:55 -07006128static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
6129{
6130 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
6131 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
6132 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
6133 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
6134 seq_printf(m, "oom_kill %lu\n",
6135 atomic_long_read(&events[MEMCG_OOM_KILL]));
6136}
6137
Johannes Weiner241994ed2015-02-11 15:26:06 -08006138static int memory_events_show(struct seq_file *m, void *v)
6139{
Chris Downaa9694b2019-03-05 15:45:52 -08006140 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006141
Shakeel Butt1e577f92019-07-11 20:55:55 -07006142 __memory_events_show(m, memcg->memory_events);
6143 return 0;
6144}
Johannes Weiner241994ed2015-02-11 15:26:06 -08006145
Shakeel Butt1e577f92019-07-11 20:55:55 -07006146static int memory_events_local_show(struct seq_file *m, void *v)
6147{
6148 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6149
6150 __memory_events_show(m, memcg->memory_events_local);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006151 return 0;
6152}
6153
Johannes Weiner587d9f72016-01-20 15:03:19 -08006154static int memory_stat_show(struct seq_file *m, void *v)
6155{
Chris Downaa9694b2019-03-05 15:45:52 -08006156 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weinerc8713d02019-07-11 20:55:59 -07006157 char *buf;
Johannes Weiner587d9f72016-01-20 15:03:19 -08006158
Johannes Weinerc8713d02019-07-11 20:55:59 -07006159 buf = memory_stat_format(memcg);
6160 if (!buf)
6161 return -ENOMEM;
6162 seq_puts(m, buf);
6163 kfree(buf);
Johannes Weiner587d9f72016-01-20 15:03:19 -08006164 return 0;
6165}
6166
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006167static int memory_oom_group_show(struct seq_file *m, void *v)
6168{
Chris Downaa9694b2019-03-05 15:45:52 -08006169 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006170
6171 seq_printf(m, "%d\n", memcg->oom_group);
6172
6173 return 0;
6174}
6175
6176static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
6177 char *buf, size_t nbytes, loff_t off)
6178{
6179 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6180 int ret, oom_group;
6181
6182 buf = strstrip(buf);
6183 if (!buf)
6184 return -EINVAL;
6185
6186 ret = kstrtoint(buf, 0, &oom_group);
6187 if (ret)
6188 return ret;
6189
6190 if (oom_group != 0 && oom_group != 1)
6191 return -EINVAL;
6192
6193 memcg->oom_group = oom_group;
6194
6195 return nbytes;
6196}
6197
Johannes Weiner241994ed2015-02-11 15:26:06 -08006198static struct cftype memory_files[] = {
6199 {
6200 .name = "current",
Johannes Weinerf5fc3c5d2015-11-05 18:50:23 -08006201 .flags = CFTYPE_NOT_ON_ROOT,
Johannes Weiner241994ed2015-02-11 15:26:06 -08006202 .read_u64 = memory_current_read,
6203 },
6204 {
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006205 .name = "min",
6206 .flags = CFTYPE_NOT_ON_ROOT,
6207 .seq_show = memory_min_show,
6208 .write = memory_min_write,
6209 },
6210 {
Johannes Weiner241994ed2015-02-11 15:26:06 -08006211 .name = "low",
6212 .flags = CFTYPE_NOT_ON_ROOT,
6213 .seq_show = memory_low_show,
6214 .write = memory_low_write,
6215 },
6216 {
6217 .name = "high",
6218 .flags = CFTYPE_NOT_ON_ROOT,
6219 .seq_show = memory_high_show,
6220 .write = memory_high_write,
6221 },
6222 {
6223 .name = "max",
6224 .flags = CFTYPE_NOT_ON_ROOT,
6225 .seq_show = memory_max_show,
6226 .write = memory_max_write,
6227 },
6228 {
6229 .name = "events",
6230 .flags = CFTYPE_NOT_ON_ROOT,
Tejun Heo472912a2015-09-18 18:01:59 -04006231 .file_offset = offsetof(struct mem_cgroup, events_file),
Johannes Weiner241994ed2015-02-11 15:26:06 -08006232 .seq_show = memory_events_show,
6233 },
Johannes Weiner587d9f72016-01-20 15:03:19 -08006234 {
Shakeel Butt1e577f92019-07-11 20:55:55 -07006235 .name = "events.local",
6236 .flags = CFTYPE_NOT_ON_ROOT,
6237 .file_offset = offsetof(struct mem_cgroup, events_local_file),
6238 .seq_show = memory_events_local_show,
6239 },
6240 {
Johannes Weiner587d9f72016-01-20 15:03:19 -08006241 .name = "stat",
Johannes Weiner587d9f72016-01-20 15:03:19 -08006242 .seq_show = memory_stat_show,
6243 },
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006244 {
6245 .name = "oom.group",
6246 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
6247 .seq_show = memory_oom_group_show,
6248 .write = memory_oom_group_write,
6249 },
Johannes Weiner241994ed2015-02-11 15:26:06 -08006250 { } /* terminate */
6251};
6252
Tejun Heo073219e2014-02-08 10:36:58 -05006253struct cgroup_subsys memory_cgrp_subsys = {
Tejun Heo92fb9742012-11-19 08:13:38 -08006254 .css_alloc = mem_cgroup_css_alloc,
Glauber Costad142e3e2013-02-22 16:34:52 -08006255 .css_online = mem_cgroup_css_online,
Tejun Heo92fb9742012-11-19 08:13:38 -08006256 .css_offline = mem_cgroup_css_offline,
Vladimir Davydov6df38682015-12-29 14:54:10 -08006257 .css_released = mem_cgroup_css_released,
Tejun Heo92fb9742012-11-19 08:13:38 -08006258 .css_free = mem_cgroup_css_free,
Tejun Heo1ced953b2014-07-08 18:02:57 -04006259 .css_reset = mem_cgroup_css_reset,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006260 .can_attach = mem_cgroup_can_attach,
6261 .cancel_attach = mem_cgroup_cancel_attach,
Tejun Heo264a0ae2016-04-21 19:09:02 -04006262 .post_attach = mem_cgroup_move_task,
Tejun Heof00baae2013-04-15 13:41:15 -07006263 .bind = mem_cgroup_bind,
Johannes Weiner241994ed2015-02-11 15:26:06 -08006264 .dfl_cftypes = memory_files,
6265 .legacy_cftypes = mem_cgroup_legacy_files,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006266 .early_init = 0,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006267};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006268
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006269/*
6270 * This function calculates an individual cgroup's effective
6271 * protection which is derived from its own memory.min/low, its
6272 * parent's and siblings' settings, as well as the actual memory
6273 * distribution in the tree.
6274 *
6275 * The following rules apply to the effective protection values:
6276 *
6277 * 1. At the first level of reclaim, effective protection is equal to
6278 * the declared protection in memory.min and memory.low.
6279 *
6280 * 2. To enable safe delegation of the protection configuration, at
6281 * subsequent levels the effective protection is capped to the
6282 * parent's effective protection.
6283 *
6284 * 3. To make complex and dynamic subtrees easier to configure, the
6285 * user is allowed to overcommit the declared protection at a given
6286 * level. If that is the case, the parent's effective protection is
6287 * distributed to the children in proportion to how much protection
6288 * they have declared and how much of it they are utilizing.
6289 *
6290 * This makes distribution proportional, but also work-conserving:
6291 * if one cgroup claims much more protection than it uses memory,
6292 * the unused remainder is available to its siblings.
6293 *
6294 * 4. Conversely, when the declared protection is undercommitted at a
6295 * given level, the distribution of the larger parental protection
6296 * budget is NOT proportional. A cgroup's protection from a sibling
6297 * is capped to its own memory.min/low setting.
6298 *
Johannes Weiner8a931f82020-04-01 21:07:07 -07006299 * 5. However, to allow protecting recursive subtrees from each other
6300 * without having to declare each individual cgroup's fixed share
6301 * of the ancestor's claim to protection, any unutilized -
6302 * "floating" - protection from up the tree is distributed in
6303 * proportion to each cgroup's *usage*. This makes the protection
6304 * neutral wrt sibling cgroups and lets them compete freely over
6305 * the shared parental protection budget, but it protects the
6306 * subtree as a whole from neighboring subtrees.
6307 *
6308 * Note that 4. and 5. are not in conflict: 4. is about protecting
6309 * against immediate siblings whereas 5. is about protecting against
6310 * neighboring subtrees.
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006311 */
6312static unsigned long effective_protection(unsigned long usage,
Johannes Weiner8a931f82020-04-01 21:07:07 -07006313 unsigned long parent_usage,
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006314 unsigned long setting,
6315 unsigned long parent_effective,
6316 unsigned long siblings_protected)
6317{
6318 unsigned long protected;
Johannes Weiner8a931f82020-04-01 21:07:07 -07006319 unsigned long ep;
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006320
6321 protected = min(usage, setting);
6322 /*
6323 * If all cgroups at this level combined claim and use more
6324 * protection then what the parent affords them, distribute
6325 * shares in proportion to utilization.
6326 *
6327 * We are using actual utilization rather than the statically
6328 * claimed protection in order to be work-conserving: claimed
6329 * but unused protection is available to siblings that would
6330 * otherwise get a smaller chunk than what they claimed.
6331 */
6332 if (siblings_protected > parent_effective)
6333 return protected * parent_effective / siblings_protected;
6334
6335 /*
6336 * Ok, utilized protection of all children is within what the
6337 * parent affords them, so we know whatever this child claims
6338 * and utilizes is effectively protected.
6339 *
6340 * If there is unprotected usage beyond this value, reclaim
6341 * will apply pressure in proportion to that amount.
6342 *
6343 * If there is unutilized protection, the cgroup will be fully
6344 * shielded from reclaim, but we do return a smaller value for
6345 * protection than what the group could enjoy in theory. This
6346 * is okay. With the overcommit distribution above, effective
6347 * protection is always dependent on how memory is actually
6348 * consumed among the siblings anyway.
6349 */
Johannes Weiner8a931f82020-04-01 21:07:07 -07006350 ep = protected;
6351
6352 /*
6353 * If the children aren't claiming (all of) the protection
6354 * afforded to them by the parent, distribute the remainder in
6355 * proportion to the (unprotected) memory of each cgroup. That
6356 * way, cgroups that aren't explicitly prioritized wrt each
6357 * other compete freely over the allowance, but they are
6358 * collectively protected from neighboring trees.
6359 *
6360 * We're using unprotected memory for the weight so that if
6361 * some cgroups DO claim explicit protection, we don't protect
6362 * the same bytes twice.
6363 */
6364 if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
6365 return ep;
6366
6367 if (parent_effective > siblings_protected && usage > protected) {
6368 unsigned long unclaimed;
6369
6370 unclaimed = parent_effective - siblings_protected;
6371 unclaimed *= usage - protected;
6372 unclaimed /= parent_usage - siblings_protected;
6373
6374 ep += unclaimed;
6375 }
6376
6377 return ep;
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006378}
6379
Johannes Weiner241994ed2015-02-11 15:26:06 -08006380/**
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006381 * mem_cgroup_protected - check if memory consumption is in the normal range
Sean Christopherson34c81052017-07-10 15:48:05 -07006382 * @root: the top ancestor of the sub-tree being checked
Johannes Weiner241994ed2015-02-11 15:26:06 -08006383 * @memcg: the memory cgroup to check
6384 *
Roman Gushchin23067152018-06-07 17:06:22 -07006385 * WARNING: This function is not stateless! It can only be used as part
6386 * of a top-down tree iteration, not for isolated queries.
Sean Christopherson34c81052017-07-10 15:48:05 -07006387 *
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006388 * Returns one of the following:
6389 * MEMCG_PROT_NONE: cgroup memory is not protected
6390 * MEMCG_PROT_LOW: cgroup memory is protected as long there is
6391 * an unprotected supply of reclaimable memory from other cgroups.
6392 * MEMCG_PROT_MIN: cgroup memory is protected
Johannes Weiner241994ed2015-02-11 15:26:06 -08006393 */
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006394enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
6395 struct mem_cgroup *memcg)
Johannes Weiner241994ed2015-02-11 15:26:06 -08006396{
Johannes Weiner8a931f82020-04-01 21:07:07 -07006397 unsigned long usage, parent_usage;
Roman Gushchin23067152018-06-07 17:06:22 -07006398 struct mem_cgroup *parent;
6399
Johannes Weiner241994ed2015-02-11 15:26:06 -08006400 if (mem_cgroup_disabled())
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006401 return MEMCG_PROT_NONE;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006402
Sean Christopherson34c81052017-07-10 15:48:05 -07006403 if (!root)
6404 root = root_mem_cgroup;
6405 if (memcg == root)
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006406 return MEMCG_PROT_NONE;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006407
Roman Gushchin23067152018-06-07 17:06:22 -07006408 usage = page_counter_read(&memcg->memory);
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006409 if (!usage)
6410 return MEMCG_PROT_NONE;
Sean Christopherson34c81052017-07-10 15:48:05 -07006411
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006412 parent = parent_mem_cgroup(memcg);
Roman Gushchindf2a4192018-06-14 15:26:17 -07006413 /* No parent means a non-hierarchical mode on v1 memcg */
6414 if (!parent)
6415 return MEMCG_PROT_NONE;
6416
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006417 if (parent == root) {
Chris Downc3d53202020-04-01 21:07:27 -07006418 memcg->memory.emin = READ_ONCE(memcg->memory.min);
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006419 memcg->memory.elow = memcg->memory.low;
6420 goto out;
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006421 }
6422
Johannes Weiner8a931f82020-04-01 21:07:07 -07006423 parent_usage = page_counter_read(&parent->memory);
6424
Chris Downb3a78222020-04-01 21:07:33 -07006425 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
Chris Downc3d53202020-04-01 21:07:27 -07006426 READ_ONCE(memcg->memory.min),
6427 READ_ONCE(parent->memory.emin),
Chris Downb3a78222020-04-01 21:07:33 -07006428 atomic_long_read(&parent->memory.children_min_usage)));
Roman Gushchin23067152018-06-07 17:06:22 -07006429
Chris Downb3a78222020-04-01 21:07:33 -07006430 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006431 memcg->memory.low, READ_ONCE(parent->memory.elow),
Chris Downb3a78222020-04-01 21:07:33 -07006432 atomic_long_read(&parent->memory.children_low_usage)));
Roman Gushchin23067152018-06-07 17:06:22 -07006433
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006434out:
6435 if (usage <= memcg->memory.emin)
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006436 return MEMCG_PROT_MIN;
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006437 else if (usage <= memcg->memory.elow)
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006438 return MEMCG_PROT_LOW;
6439 else
6440 return MEMCG_PROT_NONE;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006441}
6442
Johannes Weiner00501b52014-08-08 14:19:20 -07006443/**
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006444 * mem_cgroup_charge - charge a newly allocated page to a cgroup
Johannes Weiner00501b52014-08-08 14:19:20 -07006445 * @page: page to charge
6446 * @mm: mm context of the victim
6447 * @gfp_mask: reclaim mode
Johannes Weiner00501b52014-08-08 14:19:20 -07006448 *
6449 * Try to charge @page to the memcg that @mm belongs to, reclaiming
6450 * pages according to @gfp_mask if necessary.
6451 *
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006452 * Returns 0 on success. Otherwise, an error code is returned.
Johannes Weiner00501b52014-08-08 14:19:20 -07006453 */
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006454int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
Johannes Weiner00501b52014-08-08 14:19:20 -07006455{
Johannes Weiner3fba69a2020-06-03 16:01:31 -07006456 unsigned int nr_pages = hpage_nr_pages(page);
Johannes Weiner00501b52014-08-08 14:19:20 -07006457 struct mem_cgroup *memcg = NULL;
Johannes Weiner00501b52014-08-08 14:19:20 -07006458 int ret = 0;
6459
6460 if (mem_cgroup_disabled())
6461 goto out;
6462
6463 if (PageSwapCache(page)) {
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006464 swp_entry_t ent = { .val = page_private(page), };
6465 unsigned short id;
6466
Johannes Weiner00501b52014-08-08 14:19:20 -07006467 /*
6468 * Every swap fault against a single page tries to charge the
6469 * page, bail as early as possible. shmem_unuse() encounters
Johannes Weinereccb52e2020-06-03 16:02:11 -07006470 * already charged pages, too. page->mem_cgroup is protected
6471 * by the page lock, which serializes swap cache removal, which
Johannes Weiner00501b52014-08-08 14:19:20 -07006472 * in turn serializes uncharging.
6473 */
Vladimir Davydove993d902015-09-09 15:35:35 -07006474 VM_BUG_ON_PAGE(!PageLocked(page), page);
Huang Yingabe28952017-09-06 16:22:41 -07006475 if (compound_head(page)->mem_cgroup)
Johannes Weiner00501b52014-08-08 14:19:20 -07006476 goto out;
Vladimir Davydove993d902015-09-09 15:35:35 -07006477
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006478 id = lookup_swap_cgroup_id(ent);
6479 rcu_read_lock();
6480 memcg = mem_cgroup_from_id(id);
6481 if (memcg && !css_tryget_online(&memcg->css))
6482 memcg = NULL;
6483 rcu_read_unlock();
Johannes Weiner00501b52014-08-08 14:19:20 -07006484 }
6485
Johannes Weiner00501b52014-08-08 14:19:20 -07006486 if (!memcg)
6487 memcg = get_mem_cgroup_from_mm(mm);
6488
6489 ret = try_charge(memcg, gfp_mask, nr_pages);
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006490 if (ret)
6491 goto out_put;
Johannes Weiner00501b52014-08-08 14:19:20 -07006492
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006493 commit_charge(page, memcg);
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006494
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006495 local_irq_disable();
Johannes Weiner3fba69a2020-06-03 16:01:31 -07006496 mem_cgroup_charge_statistics(memcg, page, nr_pages);
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006497 memcg_check_events(memcg, page);
6498 local_irq_enable();
Johannes Weiner00501b52014-08-08 14:19:20 -07006499
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006500 if (PageSwapCache(page)) {
Johannes Weiner00501b52014-08-08 14:19:20 -07006501 swp_entry_t entry = { .val = page_private(page) };
6502 /*
6503 * The swap entry might not get freed for a long time,
6504 * let's not wait for it. The page already received a
6505 * memory+swap charge, drop the swap entry duplicate.
6506 */
Huang Ying38d8b4e2017-07-06 15:37:18 -07006507 mem_cgroup_uncharge_swap(entry, nr_pages);
Johannes Weiner00501b52014-08-08 14:19:20 -07006508 }
Johannes Weiner00501b52014-08-08 14:19:20 -07006509
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006510out_put:
6511 css_put(&memcg->css);
6512out:
6513 return ret;
Johannes Weiner3fea5a42020-06-03 16:01:41 -07006514}
6515
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006516struct uncharge_gather {
6517 struct mem_cgroup *memcg;
Johannes Weiner9f762db2020-06-03 16:01:44 -07006518 unsigned long nr_pages;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006519 unsigned long pgpgout;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006520 unsigned long nr_kmem;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006521 struct page *dummy_page;
6522};
6523
6524static inline void uncharge_gather_clear(struct uncharge_gather *ug)
Johannes Weiner747db952014-08-08 14:19:24 -07006525{
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006526 memset(ug, 0, sizeof(*ug));
6527}
6528
6529static void uncharge_batch(const struct uncharge_gather *ug)
6530{
Johannes Weiner747db952014-08-08 14:19:24 -07006531 unsigned long flags;
6532
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006533 if (!mem_cgroup_is_root(ug->memcg)) {
Johannes Weiner9f762db2020-06-03 16:01:44 -07006534 page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08006535 if (do_memsw_account())
Johannes Weiner9f762db2020-06-03 16:01:44 -07006536 page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006537 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
6538 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6539 memcg_oom_recover(ug->memcg);
Johannes Weinerce00a962014-09-05 08:43:57 -04006540 }
Johannes Weiner747db952014-08-08 14:19:24 -07006541
6542 local_irq_save(flags);
Johannes Weinerc9019e92018-01-31 16:16:37 -08006543 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
Johannes Weiner9f762db2020-06-03 16:01:44 -07006544 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006545 memcg_check_events(ug->memcg, ug->dummy_page);
Johannes Weiner747db952014-08-08 14:19:24 -07006546 local_irq_restore(flags);
Johannes Weinere8ea14c2014-12-10 15:42:42 -08006547
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006548 if (!mem_cgroup_is_root(ug->memcg))
Johannes Weiner9f762db2020-06-03 16:01:44 -07006549 css_put_many(&ug->memcg->css, ug->nr_pages);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006550}
6551
6552static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6553{
Johannes Weiner9f762db2020-06-03 16:01:44 -07006554 unsigned long nr_pages;
6555
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006556 VM_BUG_ON_PAGE(PageLRU(page), page);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006557
6558 if (!page->mem_cgroup)
6559 return;
6560
6561 /*
6562 * Nobody should be changing or seriously looking at
6563 * page->mem_cgroup at this point, we have fully
6564 * exclusive access to the page.
6565 */
6566
6567 if (ug->memcg != page->mem_cgroup) {
6568 if (ug->memcg) {
6569 uncharge_batch(ug);
6570 uncharge_gather_clear(ug);
6571 }
6572 ug->memcg = page->mem_cgroup;
6573 }
6574
Johannes Weiner9f762db2020-06-03 16:01:44 -07006575 nr_pages = compound_nr(page);
6576 ug->nr_pages += nr_pages;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006577
Johannes Weiner9f762db2020-06-03 16:01:44 -07006578 if (!PageKmemcg(page)) {
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006579 ug->pgpgout++;
6580 } else {
Johannes Weiner9f762db2020-06-03 16:01:44 -07006581 ug->nr_kmem += nr_pages;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006582 __ClearPageKmemcg(page);
6583 }
6584
6585 ug->dummy_page = page;
6586 page->mem_cgroup = NULL;
Johannes Weiner747db952014-08-08 14:19:24 -07006587}
6588
6589static void uncharge_list(struct list_head *page_list)
6590{
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006591 struct uncharge_gather ug;
Johannes Weiner747db952014-08-08 14:19:24 -07006592 struct list_head *next;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006593
6594 uncharge_gather_clear(&ug);
Johannes Weiner747db952014-08-08 14:19:24 -07006595
Johannes Weiner8b592652016-03-17 14:20:31 -07006596 /*
6597 * Note that the list can be a single page->lru; hence the
6598 * do-while loop instead of a simple list_for_each_entry().
6599 */
Johannes Weiner747db952014-08-08 14:19:24 -07006600 next = page_list->next;
6601 do {
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006602 struct page *page;
6603
Johannes Weiner747db952014-08-08 14:19:24 -07006604 page = list_entry(next, struct page, lru);
6605 next = page->lru.next;
6606
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006607 uncharge_page(page, &ug);
Johannes Weiner747db952014-08-08 14:19:24 -07006608 } while (next != page_list);
6609
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006610 if (ug.memcg)
6611 uncharge_batch(&ug);
Johannes Weiner747db952014-08-08 14:19:24 -07006612}
6613
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006614/**
6615 * mem_cgroup_uncharge - uncharge a page
6616 * @page: page to uncharge
6617 *
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006618 * Uncharge a page previously charged with mem_cgroup_charge().
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006619 */
6620void mem_cgroup_uncharge(struct page *page)
6621{
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006622 struct uncharge_gather ug;
6623
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006624 if (mem_cgroup_disabled())
6625 return;
6626
Johannes Weiner747db952014-08-08 14:19:24 -07006627 /* Don't touch page->lru of any random page, pre-check: */
Johannes Weiner1306a852014-12-10 15:44:52 -08006628 if (!page->mem_cgroup)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006629 return;
6630
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006631 uncharge_gather_clear(&ug);
6632 uncharge_page(page, &ug);
6633 uncharge_batch(&ug);
Johannes Weiner747db952014-08-08 14:19:24 -07006634}
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006635
Johannes Weiner747db952014-08-08 14:19:24 -07006636/**
6637 * mem_cgroup_uncharge_list - uncharge a list of page
6638 * @page_list: list of pages to uncharge
6639 *
6640 * Uncharge a list of pages previously charged with
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006641 * mem_cgroup_charge().
Johannes Weiner747db952014-08-08 14:19:24 -07006642 */
6643void mem_cgroup_uncharge_list(struct list_head *page_list)
6644{
6645 if (mem_cgroup_disabled())
6646 return;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006647
Johannes Weiner747db952014-08-08 14:19:24 -07006648 if (!list_empty(page_list))
6649 uncharge_list(page_list);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006650}
6651
6652/**
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006653 * mem_cgroup_migrate - charge a page's replacement
6654 * @oldpage: currently circulating page
6655 * @newpage: replacement page
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006656 *
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006657 * Charge @newpage as a replacement page for @oldpage. @oldpage will
6658 * be uncharged upon free.
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006659 *
6660 * Both pages must be locked, @newpage->mapping must be set up.
6661 */
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006662void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006663{
Johannes Weiner29833312014-12-10 15:44:02 -08006664 struct mem_cgroup *memcg;
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006665 unsigned int nr_pages;
Tejun Heod93c4132016-06-24 14:49:54 -07006666 unsigned long flags;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006667
6668 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
6669 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006670 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006671 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
6672 newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006673
6674 if (mem_cgroup_disabled())
6675 return;
6676
6677 /* Page cache replacement: new page already charged? */
Johannes Weiner1306a852014-12-10 15:44:52 -08006678 if (newpage->mem_cgroup)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006679 return;
6680
Hugh Dickins45637ba2015-11-05 18:49:40 -08006681 /* Swapcache readahead pages can get replaced before being charged */
Johannes Weiner1306a852014-12-10 15:44:52 -08006682 memcg = oldpage->mem_cgroup;
Johannes Weiner29833312014-12-10 15:44:02 -08006683 if (!memcg)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006684 return;
6685
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006686 /* Force-charge the new page. The old one will be freed soon */
Kaitao Cheng928552702020-01-30 22:13:42 -08006687 nr_pages = hpage_nr_pages(newpage);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006688
6689 page_counter_charge(&memcg->memory, nr_pages);
6690 if (do_memsw_account())
6691 page_counter_charge(&memcg->memsw, nr_pages);
6692 css_get_many(&memcg->css, nr_pages);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006693
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006694 commit_charge(newpage, memcg);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006695
Tejun Heod93c4132016-06-24 14:49:54 -07006696 local_irq_save(flags);
Johannes Weiner3fba69a2020-06-03 16:01:31 -07006697 mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006698 memcg_check_events(memcg, newpage);
Tejun Heod93c4132016-06-24 14:49:54 -07006699 local_irq_restore(flags);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006700}
6701
Johannes Weineref129472016-01-14 15:21:34 -08006702DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
Johannes Weiner11092082016-01-14 15:21:26 -08006703EXPORT_SYMBOL(memcg_sockets_enabled_key);
6704
Johannes Weiner2d758072016-10-07 17:00:58 -07006705void mem_cgroup_sk_alloc(struct sock *sk)
Johannes Weiner11092082016-01-14 15:21:26 -08006706{
6707 struct mem_cgroup *memcg;
6708
Johannes Weiner2d758072016-10-07 17:00:58 -07006709 if (!mem_cgroup_sockets_enabled)
6710 return;
6711
Shakeel Butte876ecc2020-03-09 22:16:05 -07006712 /* Do not associate the sock with unrelated interrupted task's memcg. */
6713 if (in_interrupt())
6714 return;
6715
Johannes Weiner11092082016-01-14 15:21:26 -08006716 rcu_read_lock();
6717 memcg = mem_cgroup_from_task(current);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006718 if (memcg == root_mem_cgroup)
6719 goto out;
Johannes Weiner0db15292016-01-20 15:02:50 -08006720 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006721 goto out;
Shakeel Butt8965aa22020-04-01 21:07:10 -07006722 if (css_tryget(&memcg->css))
Johannes Weiner11092082016-01-14 15:21:26 -08006723 sk->sk_memcg = memcg;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006724out:
Johannes Weiner11092082016-01-14 15:21:26 -08006725 rcu_read_unlock();
6726}
Johannes Weiner11092082016-01-14 15:21:26 -08006727
Johannes Weiner2d758072016-10-07 17:00:58 -07006728void mem_cgroup_sk_free(struct sock *sk)
Johannes Weiner11092082016-01-14 15:21:26 -08006729{
Johannes Weiner2d758072016-10-07 17:00:58 -07006730 if (sk->sk_memcg)
6731 css_put(&sk->sk_memcg->css);
Johannes Weiner11092082016-01-14 15:21:26 -08006732}
6733
6734/**
6735 * mem_cgroup_charge_skmem - charge socket memory
6736 * @memcg: memcg to charge
6737 * @nr_pages: number of pages to charge
6738 *
6739 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
6740 * @memcg's configured limit, %false if the charge had to be forced.
6741 */
6742bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6743{
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006744 gfp_t gfp_mask = GFP_KERNEL;
Johannes Weiner11092082016-01-14 15:21:26 -08006745
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006746 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
Johannes Weiner0db15292016-01-20 15:02:50 -08006747 struct page_counter *fail;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006748
Johannes Weiner0db15292016-01-20 15:02:50 -08006749 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
6750 memcg->tcpmem_pressure = 0;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006751 return true;
6752 }
Johannes Weiner0db15292016-01-20 15:02:50 -08006753 page_counter_charge(&memcg->tcpmem, nr_pages);
6754 memcg->tcpmem_pressure = 1;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006755 return false;
Johannes Weiner11092082016-01-14 15:21:26 -08006756 }
Johannes Weinerd886f4e2016-01-20 15:02:47 -08006757
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006758 /* Don't block in the packet receive path */
6759 if (in_softirq())
6760 gfp_mask = GFP_NOWAIT;
6761
Johannes Weinerc9019e92018-01-31 16:16:37 -08006762 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
Johannes Weinerb2807f02016-01-20 15:03:22 -08006763
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006764 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
6765 return true;
6766
6767 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
Johannes Weiner11092082016-01-14 15:21:26 -08006768 return false;
6769}
6770
6771/**
6772 * mem_cgroup_uncharge_skmem - uncharge socket memory
Mike Rapoportb7701a52018-02-06 15:42:13 -08006773 * @memcg: memcg to uncharge
6774 * @nr_pages: number of pages to uncharge
Johannes Weiner11092082016-01-14 15:21:26 -08006775 */
6776void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6777{
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006778 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
Johannes Weiner0db15292016-01-20 15:02:50 -08006779 page_counter_uncharge(&memcg->tcpmem, nr_pages);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006780 return;
6781 }
Johannes Weinerd886f4e2016-01-20 15:02:47 -08006782
Johannes Weinerc9019e92018-01-31 16:16:37 -08006783 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
Johannes Weinerb2807f02016-01-20 15:03:22 -08006784
Roman Gushchin475d0482017-09-08 16:13:09 -07006785 refill_stock(memcg, nr_pages);
Johannes Weiner11092082016-01-14 15:21:26 -08006786}
6787
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006788static int __init cgroup_memory(char *s)
6789{
6790 char *token;
6791
6792 while ((token = strsep(&s, ",")) != NULL) {
6793 if (!*token)
6794 continue;
6795 if (!strcmp(token, "nosocket"))
6796 cgroup_memory_nosocket = true;
Vladimir Davydov04823c82016-01-20 15:02:38 -08006797 if (!strcmp(token, "nokmem"))
6798 cgroup_memory_nokmem = true;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006799 }
6800 return 0;
6801}
6802__setup("cgroup.memory=", cgroup_memory);
Johannes Weiner11092082016-01-14 15:21:26 -08006803
Michal Hocko2d110852013-02-22 16:34:43 -08006804/*
Michal Hocko10813122013-02-22 16:35:41 -08006805 * subsys_initcall() for memory controller.
6806 *
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01006807 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
6808 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
6809 * basically everything that doesn't depend on a specific mem_cgroup structure
6810 * should be initialized from here.
Michal Hocko2d110852013-02-22 16:34:43 -08006811 */
6812static int __init mem_cgroup_init(void)
6813{
Johannes Weiner95a045f2015-02-11 15:26:33 -08006814 int cpu, node;
6815
Kirill Tkhai84c07d12018-08-17 15:47:25 -07006816#ifdef CONFIG_MEMCG_KMEM
Vladimir Davydov13583c32016-12-12 16:41:29 -08006817 /*
6818 * Kmem cache creation is mostly done with the slab_mutex held,
Tejun Heo17cc4df2017-02-22 15:41:36 -08006819 * so use a workqueue with limited concurrency to avoid stalling
6820 * all worker threads in case lots of cgroups are created and
6821 * destroyed simultaneously.
Vladimir Davydov13583c32016-12-12 16:41:29 -08006822 */
Tejun Heo17cc4df2017-02-22 15:41:36 -08006823 memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
6824 BUG_ON(!memcg_kmem_cache_wq);
Vladimir Davydov13583c32016-12-12 16:41:29 -08006825#endif
6826
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01006827 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
6828 memcg_hotplug_cpu_dead);
Johannes Weiner95a045f2015-02-11 15:26:33 -08006829
6830 for_each_possible_cpu(cpu)
6831 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
6832 drain_local_stock);
6833
6834 for_each_node(node) {
6835 struct mem_cgroup_tree_per_node *rtpn;
Johannes Weiner95a045f2015-02-11 15:26:33 -08006836
6837 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
6838 node_online(node) ? node : NUMA_NO_NODE);
6839
Mel Gormanef8f2322016-07-28 15:46:05 -07006840 rtpn->rb_root = RB_ROOT;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -07006841 rtpn->rb_rightmost = NULL;
Mel Gormanef8f2322016-07-28 15:46:05 -07006842 spin_lock_init(&rtpn->lock);
Johannes Weiner95a045f2015-02-11 15:26:33 -08006843 soft_limit_tree.rb_tree_per_node[node] = rtpn;
6844 }
6845
Michal Hocko2d110852013-02-22 16:34:43 -08006846 return 0;
6847}
6848subsys_initcall(mem_cgroup_init);
Johannes Weiner21afa382015-02-11 15:26:36 -08006849
6850#ifdef CONFIG_MEMCG_SWAP
Arnd Bergmann358c07f2016-08-25 15:17:08 -07006851static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
6852{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07006853 while (!refcount_inc_not_zero(&memcg->id.ref)) {
Arnd Bergmann358c07f2016-08-25 15:17:08 -07006854 /*
6855 * The root cgroup cannot be destroyed, so it's refcount must
6856 * always be >= 1.
6857 */
6858 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
6859 VM_BUG_ON(1);
6860 break;
6861 }
6862 memcg = parent_mem_cgroup(memcg);
6863 if (!memcg)
6864 memcg = root_mem_cgroup;
6865 }
6866 return memcg;
6867}
6868
Johannes Weiner21afa382015-02-11 15:26:36 -08006869/**
6870 * mem_cgroup_swapout - transfer a memsw charge to swap
6871 * @page: page whose memsw charge to transfer
6872 * @entry: swap entry to move the charge to
6873 *
6874 * Transfer the memsw charge of @page to @entry.
6875 */
6876void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
6877{
Vladimir Davydov1f47b612016-08-11 15:33:00 -07006878 struct mem_cgroup *memcg, *swap_memcg;
Huang Yingd6810d732017-09-06 16:22:45 -07006879 unsigned int nr_entries;
Johannes Weiner21afa382015-02-11 15:26:36 -08006880 unsigned short oldid;
6881
6882 VM_BUG_ON_PAGE(PageLRU(page), page);
6883 VM_BUG_ON_PAGE(page_count(page), page);
6884
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006885 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Johannes Weiner21afa382015-02-11 15:26:36 -08006886 return;
6887
6888 memcg = page->mem_cgroup;
6889
6890 /* Readahead page, never charged */
6891 if (!memcg)
6892 return;
6893
Vladimir Davydov1f47b612016-08-11 15:33:00 -07006894 /*
6895 * In case the memcg owning these pages has been offlined and doesn't
6896 * have an ID allocated to it anymore, charge the closest online
6897 * ancestor for the swap instead and transfer the memory+swap charge.
6898 */
6899 swap_memcg = mem_cgroup_id_get_online(memcg);
Huang Yingd6810d732017-09-06 16:22:45 -07006900 nr_entries = hpage_nr_pages(page);
6901 /* Get references for the tail pages, too */
6902 if (nr_entries > 1)
6903 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
6904 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
6905 nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08006906 VM_BUG_ON_PAGE(oldid, page);
Johannes Weinerc9019e92018-01-31 16:16:37 -08006907 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08006908
6909 page->mem_cgroup = NULL;
6910
6911 if (!mem_cgroup_is_root(memcg))
Huang Yingd6810d732017-09-06 16:22:45 -07006912 page_counter_uncharge(&memcg->memory, nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08006913
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006914 if (!cgroup_memory_noswap && memcg != swap_memcg) {
Vladimir Davydov1f47b612016-08-11 15:33:00 -07006915 if (!mem_cgroup_is_root(swap_memcg))
Huang Yingd6810d732017-09-06 16:22:45 -07006916 page_counter_charge(&swap_memcg->memsw, nr_entries);
6917 page_counter_uncharge(&memcg->memsw, nr_entries);
Vladimir Davydov1f47b612016-08-11 15:33:00 -07006918 }
6919
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07006920 /*
6921 * Interrupts should be disabled here because the caller holds the
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07006922 * i_pages lock which is taken with interrupts-off. It is
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07006923 * important here to have the interrupts disabled because it is the
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07006924 * only synchronisation we have for updating the per-CPU variables.
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07006925 */
6926 VM_BUG_ON(!irqs_disabled());
Johannes Weiner3fba69a2020-06-03 16:01:31 -07006927 mem_cgroup_charge_statistics(memcg, page, -nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08006928 memcg_check_events(memcg, page);
Johannes Weiner73f576c2016-07-20 15:44:57 -07006929
6930 if (!mem_cgroup_is_root(memcg))
Shakeel Buttd08afa142017-11-29 16:11:15 -08006931 css_put_many(&memcg->css, nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08006932}
6933
Huang Ying38d8b4e2017-07-06 15:37:18 -07006934/**
6935 * mem_cgroup_try_charge_swap - try charging swap space for a page
Vladimir Davydov37e84352016-01-20 15:02:56 -08006936 * @page: page being added to swap
6937 * @entry: swap entry to charge
6938 *
Huang Ying38d8b4e2017-07-06 15:37:18 -07006939 * Try to charge @page's memcg for the swap space at @entry.
Vladimir Davydov37e84352016-01-20 15:02:56 -08006940 *
6941 * Returns 0 on success, -ENOMEM on failure.
6942 */
6943int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
6944{
Huang Ying38d8b4e2017-07-06 15:37:18 -07006945 unsigned int nr_pages = hpage_nr_pages(page);
Vladimir Davydov37e84352016-01-20 15:02:56 -08006946 struct page_counter *counter;
Huang Ying38d8b4e2017-07-06 15:37:18 -07006947 struct mem_cgroup *memcg;
Vladimir Davydov37e84352016-01-20 15:02:56 -08006948 unsigned short oldid;
6949
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006950 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydov37e84352016-01-20 15:02:56 -08006951 return 0;
6952
6953 memcg = page->mem_cgroup;
6954
6955 /* Readahead page, never charged */
6956 if (!memcg)
6957 return 0;
6958
Tejun Heof3a53a32018-06-07 17:05:35 -07006959 if (!entry.val) {
6960 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
Tejun Heobb98f2c2018-06-07 17:05:31 -07006961 return 0;
Tejun Heof3a53a32018-06-07 17:05:35 -07006962 }
Tejun Heobb98f2c2018-06-07 17:05:31 -07006963
Vladimir Davydov1f47b612016-08-11 15:33:00 -07006964 memcg = mem_cgroup_id_get_online(memcg);
Vladimir Davydov37e84352016-01-20 15:02:56 -08006965
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006966 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
Huang Ying38d8b4e2017-07-06 15:37:18 -07006967 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
Tejun Heof3a53a32018-06-07 17:05:35 -07006968 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
6969 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
Vladimir Davydov1f47b612016-08-11 15:33:00 -07006970 mem_cgroup_id_put(memcg);
6971 return -ENOMEM;
6972 }
6973
Huang Ying38d8b4e2017-07-06 15:37:18 -07006974 /* Get references for the tail pages, too */
6975 if (nr_pages > 1)
6976 mem_cgroup_id_get_many(memcg, nr_pages - 1);
6977 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08006978 VM_BUG_ON_PAGE(oldid, page);
Johannes Weinerc9019e92018-01-31 16:16:37 -08006979 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08006980
Vladimir Davydov37e84352016-01-20 15:02:56 -08006981 return 0;
6982}
6983
Johannes Weiner21afa382015-02-11 15:26:36 -08006984/**
Huang Ying38d8b4e2017-07-06 15:37:18 -07006985 * mem_cgroup_uncharge_swap - uncharge swap space
Johannes Weiner21afa382015-02-11 15:26:36 -08006986 * @entry: swap entry to uncharge
Huang Ying38d8b4e2017-07-06 15:37:18 -07006987 * @nr_pages: the amount of swap space to uncharge
Johannes Weiner21afa382015-02-11 15:26:36 -08006988 */
Huang Ying38d8b4e2017-07-06 15:37:18 -07006989void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
Johannes Weiner21afa382015-02-11 15:26:36 -08006990{
6991 struct mem_cgroup *memcg;
6992 unsigned short id;
6993
Huang Ying38d8b4e2017-07-06 15:37:18 -07006994 id = swap_cgroup_record(entry, 0, nr_pages);
Johannes Weiner21afa382015-02-11 15:26:36 -08006995 rcu_read_lock();
Vladimir Davydovadbe4272015-04-15 16:13:00 -07006996 memcg = mem_cgroup_from_id(id);
Johannes Weiner21afa382015-02-11 15:26:36 -08006997 if (memcg) {
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006998 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
Vladimir Davydov37e84352016-01-20 15:02:56 -08006999 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Huang Ying38d8b4e2017-07-06 15:37:18 -07007000 page_counter_uncharge(&memcg->swap, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007001 else
Huang Ying38d8b4e2017-07-06 15:37:18 -07007002 page_counter_uncharge(&memcg->memsw, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007003 }
Johannes Weinerc9019e92018-01-31 16:16:37 -08007004 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
Huang Ying38d8b4e2017-07-06 15:37:18 -07007005 mem_cgroup_id_put_many(memcg, nr_pages);
Johannes Weiner21afa382015-02-11 15:26:36 -08007006 }
7007 rcu_read_unlock();
7008}
7009
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007010long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7011{
7012 long nr_swap_pages = get_nr_swap_pages();
7013
Johannes Weinereccb52e2020-06-03 16:02:11 -07007014 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007015 return nr_swap_pages;
7016 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7017 nr_swap_pages = min_t(long, nr_swap_pages,
Roman Gushchinbbec2e12018-06-07 17:06:18 -07007018 READ_ONCE(memcg->swap.max) -
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007019 page_counter_read(&memcg->swap));
7020 return nr_swap_pages;
7021}
7022
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007023bool mem_cgroup_swap_full(struct page *page)
7024{
7025 struct mem_cgroup *memcg;
7026
7027 VM_BUG_ON_PAGE(!PageLocked(page), page);
7028
7029 if (vm_swap_full())
7030 return true;
Johannes Weinereccb52e2020-06-03 16:02:11 -07007031 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007032 return false;
7033
7034 memcg = page->mem_cgroup;
7035 if (!memcg)
7036 return false;
7037
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007038 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
7039 unsigned long usage = page_counter_read(&memcg->swap);
7040
7041 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7042 usage * 2 >= READ_ONCE(memcg->swap.max))
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007043 return true;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007044 }
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007045
7046 return false;
7047}
7048
Johannes Weinereccb52e2020-06-03 16:02:11 -07007049static int __init setup_swap_account(char *s)
Johannes Weiner21afa382015-02-11 15:26:36 -08007050{
7051 if (!strcmp(s, "1"))
Johannes Weinereccb52e2020-06-03 16:02:11 -07007052 cgroup_memory_noswap = 0;
Johannes Weiner21afa382015-02-11 15:26:36 -08007053 else if (!strcmp(s, "0"))
Johannes Weinereccb52e2020-06-03 16:02:11 -07007054 cgroup_memory_noswap = 1;
Johannes Weiner21afa382015-02-11 15:26:36 -08007055 return 1;
7056}
Johannes Weinereccb52e2020-06-03 16:02:11 -07007057__setup("swapaccount=", setup_swap_account);
Johannes Weiner21afa382015-02-11 15:26:36 -08007058
Vladimir Davydov37e84352016-01-20 15:02:56 -08007059static u64 swap_current_read(struct cgroup_subsys_state *css,
7060 struct cftype *cft)
7061{
7062 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7063
7064 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7065}
7066
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007067static int swap_high_show(struct seq_file *m, void *v)
7068{
7069 return seq_puts_memcg_tunable(m,
7070 READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
7071}
7072
7073static ssize_t swap_high_write(struct kernfs_open_file *of,
7074 char *buf, size_t nbytes, loff_t off)
7075{
7076 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7077 unsigned long high;
7078 int err;
7079
7080 buf = strstrip(buf);
7081 err = page_counter_memparse(buf, "max", &high);
7082 if (err)
7083 return err;
7084
7085 page_counter_set_high(&memcg->swap, high);
7086
7087 return nbytes;
7088}
7089
Vladimir Davydov37e84352016-01-20 15:02:56 -08007090static int swap_max_show(struct seq_file *m, void *v)
7091{
Chris Down677dc972019-03-05 15:45:55 -08007092 return seq_puts_memcg_tunable(m,
7093 READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
Vladimir Davydov37e84352016-01-20 15:02:56 -08007094}
7095
7096static ssize_t swap_max_write(struct kernfs_open_file *of,
7097 char *buf, size_t nbytes, loff_t off)
7098{
7099 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7100 unsigned long max;
7101 int err;
7102
7103 buf = strstrip(buf);
7104 err = page_counter_memparse(buf, "max", &max);
7105 if (err)
7106 return err;
7107
Tejun Heobe091022018-06-07 17:09:21 -07007108 xchg(&memcg->swap.max, max);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007109
7110 return nbytes;
7111}
7112
Tejun Heof3a53a32018-06-07 17:05:35 -07007113static int swap_events_show(struct seq_file *m, void *v)
7114{
Chris Downaa9694b2019-03-05 15:45:52 -08007115 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Tejun Heof3a53a32018-06-07 17:05:35 -07007116
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007117 seq_printf(m, "high %lu\n",
7118 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
Tejun Heof3a53a32018-06-07 17:05:35 -07007119 seq_printf(m, "max %lu\n",
7120 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7121 seq_printf(m, "fail %lu\n",
7122 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7123
7124 return 0;
7125}
7126
Vladimir Davydov37e84352016-01-20 15:02:56 -08007127static struct cftype swap_files[] = {
7128 {
7129 .name = "swap.current",
7130 .flags = CFTYPE_NOT_ON_ROOT,
7131 .read_u64 = swap_current_read,
7132 },
7133 {
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007134 .name = "swap.high",
7135 .flags = CFTYPE_NOT_ON_ROOT,
7136 .seq_show = swap_high_show,
7137 .write = swap_high_write,
7138 },
7139 {
Vladimir Davydov37e84352016-01-20 15:02:56 -08007140 .name = "swap.max",
7141 .flags = CFTYPE_NOT_ON_ROOT,
7142 .seq_show = swap_max_show,
7143 .write = swap_max_write,
7144 },
Tejun Heof3a53a32018-06-07 17:05:35 -07007145 {
7146 .name = "swap.events",
7147 .flags = CFTYPE_NOT_ON_ROOT,
7148 .file_offset = offsetof(struct mem_cgroup, swap_events_file),
7149 .seq_show = swap_events_show,
7150 },
Vladimir Davydov37e84352016-01-20 15:02:56 -08007151 { } /* terminate */
7152};
7153
Johannes Weinereccb52e2020-06-03 16:02:11 -07007154static struct cftype memsw_files[] = {
Johannes Weiner21afa382015-02-11 15:26:36 -08007155 {
7156 .name = "memsw.usage_in_bytes",
7157 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
7158 .read_u64 = mem_cgroup_read_u64,
7159 },
7160 {
7161 .name = "memsw.max_usage_in_bytes",
7162 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
7163 .write = mem_cgroup_reset,
7164 .read_u64 = mem_cgroup_read_u64,
7165 },
7166 {
7167 .name = "memsw.limit_in_bytes",
7168 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
7169 .write = mem_cgroup_write,
7170 .read_u64 = mem_cgroup_read_u64,
7171 },
7172 {
7173 .name = "memsw.failcnt",
7174 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
7175 .write = mem_cgroup_reset,
7176 .read_u64 = mem_cgroup_read_u64,
7177 },
7178 { }, /* terminate */
7179};
7180
7181static int __init mem_cgroup_swap_init(void)
7182{
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007183 /* No memory control -> no swap control */
7184 if (mem_cgroup_disabled())
7185 cgroup_memory_noswap = true;
7186
7187 if (cgroup_memory_noswap)
Johannes Weinereccb52e2020-06-03 16:02:11 -07007188 return 0;
7189
7190 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
7191 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
7192
Johannes Weiner21afa382015-02-11 15:26:36 -08007193 return 0;
7194}
7195subsys_initcall(mem_cgroup_swap_init);
7196
7197#endif /* CONFIG_MEMCG_SWAP */