blob: db9fb5c8dae73c89b45b29a9392e86eadbf1614e [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Christoph Lameter97d06602012-07-06 15:25:11 -05002#ifndef MM_SLAB_H
3#define MM_SLAB_H
4/*
5 * Internal slab definitions
6 */
7
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +01008/* Reuses the bits in struct page */
9struct slab {
10 unsigned long __page_flags;
Vlastimil Babka401fb122021-11-04 11:30:58 +010011
12#if defined(CONFIG_SLAB)
13
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010014 union {
15 struct list_head slab_list;
Vlastimil Babka401fb122021-11-04 11:30:58 +010016 struct rcu_head rcu_head;
17 };
18 struct kmem_cache *slab_cache;
19 void *freelist; /* array of free object indexes */
20 void *s_mem; /* first object */
21 unsigned int active;
22
23#elif defined(CONFIG_SLUB)
24
25 union {
26 struct list_head slab_list;
27 struct rcu_head rcu_head;
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +010028#ifdef CONFIG_SLUB_CPU_PARTIAL
Vlastimil Babka401fb122021-11-04 11:30:58 +010029 struct {
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010030 struct slab *next;
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010031 int slabs; /* Nr of slabs left */
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010032 };
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +010033#endif
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010034 };
Vlastimil Babka401fb122021-11-04 11:30:58 +010035 struct kmem_cache *slab_cache;
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010036 /* Double-word boundary */
37 void *freelist; /* first free object */
38 union {
Vlastimil Babka401fb122021-11-04 11:30:58 +010039 unsigned long counters;
40 struct {
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010041 unsigned inuse:16;
42 unsigned objects:15;
43 unsigned frozen:1;
44 };
45 };
Vlastimil Babka401fb122021-11-04 11:30:58 +010046 unsigned int __unused;
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010047
Vlastimil Babka401fb122021-11-04 11:30:58 +010048#elif defined(CONFIG_SLOB)
49
50 struct list_head slab_list;
51 void *__unused_1;
52 void *freelist; /* first free block */
Hyeonggon Yoob01af5c2021-12-12 06:52:41 +000053 long units;
54 unsigned int __unused_2;
Vlastimil Babka401fb122021-11-04 11:30:58 +010055
56#else
57#error "Unexpected slab allocator configured"
58#endif
59
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010060 atomic_t __page_refcount;
61#ifdef CONFIG_MEMCG
62 unsigned long memcg_data;
63#endif
64};
65
66#define SLAB_MATCH(pg, sl) \
67 static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))
68SLAB_MATCH(flags, __page_flags);
69SLAB_MATCH(compound_head, slab_list); /* Ensure bit 0 is clear */
Vlastimil Babka401fb122021-11-04 11:30:58 +010070#ifndef CONFIG_SLOB
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010071SLAB_MATCH(rcu_head, rcu_head);
Vlastimil Babka401fb122021-11-04 11:30:58 +010072#endif
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010073SLAB_MATCH(_refcount, __page_refcount);
74#ifdef CONFIG_MEMCG
75SLAB_MATCH(memcg_data, memcg_data);
76#endif
77#undef SLAB_MATCH
78static_assert(sizeof(struct slab) <= sizeof(struct page));
79
80/**
81 * folio_slab - Converts from folio to slab.
82 * @folio: The folio.
83 *
84 * Currently struct slab is a different representation of a folio where
85 * folio_test_slab() is true.
86 *
87 * Return: The slab which contains this folio.
88 */
89#define folio_slab(folio) (_Generic((folio), \
90 const struct folio *: (const struct slab *)(folio), \
91 struct folio *: (struct slab *)(folio)))
92
93/**
94 * slab_folio - The folio allocated for a slab
95 * @slab: The slab.
96 *
97 * Slabs are allocated as folios that contain the individual objects and are
98 * using some fields in the first struct page of the folio - those fields are
99 * now accessed by struct slab. It is occasionally necessary to convert back to
100 * a folio in order to communicate with the rest of the mm. Please use this
101 * helper function instead of casting yourself, as the implementation may change
102 * in the future.
103 */
104#define slab_folio(s) (_Generic((s), \
105 const struct slab *: (const struct folio *)s, \
106 struct slab *: (struct folio *)s))
107
108/**
109 * page_slab - Converts from first struct page to slab.
110 * @p: The first (either head of compound or single) page of slab.
111 *
112 * A temporary wrapper to convert struct page to struct slab in situations where
113 * we know the page is the compound head, or single order-0 page.
114 *
115 * Long-term ideally everything would work with struct slab directly or go
116 * through folio to struct slab.
117 *
118 * Return: The slab which contains this page
119 */
120#define page_slab(p) (_Generic((p), \
121 const struct page *: (const struct slab *)(p), \
122 struct page *: (struct slab *)(p)))
123
124/**
125 * slab_page - The first struct page allocated for a slab
126 * @slab: The slab.
127 *
128 * A convenience wrapper for converting slab to the first struct page of the
129 * underlying folio, to communicate with code not yet converted to folio or
130 * struct slab.
131 */
132#define slab_page(s) folio_page(slab_folio(s), 0)
133
134/*
135 * If network-based swap is enabled, sl*b must keep track of whether pages
136 * were allocated from pfmemalloc reserves.
137 */
138static inline bool slab_test_pfmemalloc(const struct slab *slab)
139{
140 return folio_test_active((struct folio *)slab_folio(slab));
141}
142
143static inline void slab_set_pfmemalloc(struct slab *slab)
144{
145 folio_set_active(slab_folio(slab));
146}
147
148static inline void slab_clear_pfmemalloc(struct slab *slab)
149{
150 folio_clear_active(slab_folio(slab));
151}
152
153static inline void __slab_clear_pfmemalloc(struct slab *slab)
154{
155 __folio_clear_active(slab_folio(slab));
156}
157
158static inline void *slab_address(const struct slab *slab)
159{
160 return folio_address(slab_folio(slab));
161}
162
163static inline int slab_nid(const struct slab *slab)
164{
165 return folio_nid(slab_folio(slab));
166}
167
168static inline pg_data_t *slab_pgdat(const struct slab *slab)
169{
170 return folio_pgdat(slab_folio(slab));
171}
172
173static inline struct slab *virt_to_slab(const void *addr)
174{
175 struct folio *folio = virt_to_folio(addr);
176
177 if (!folio_test_slab(folio))
178 return NULL;
179
180 return folio_slab(folio);
181}
182
183static inline int slab_order(const struct slab *slab)
184{
185 return folio_order((struct folio *)slab_folio(slab));
186}
187
188static inline size_t slab_size(const struct slab *slab)
189{
190 return PAGE_SIZE << slab_order(slab);
191}
192
Joonsoo Kim07f361b2014-10-09 15:26:00 -0700193#ifdef CONFIG_SLOB
194/*
195 * Common fields provided in kmem_cache by all slab allocators
196 * This struct is either used directly by the allocator (SLOB)
197 * or the allocator must include definitions for all fields
198 * provided in kmem_cache_common in their definition of kmem_cache.
199 *
200 * Once we can do anonymous structs (C11 standard) we could put a
201 * anonymous struct definition in these allocators so that the
202 * separate allocations in the kmem_cache structure of SLAB and
203 * SLUB is no longer needed.
204 */
205struct kmem_cache {
206 unsigned int object_size;/* The original size of the object */
207 unsigned int size; /* The aligned/padded/added on size */
208 unsigned int align; /* Alignment as calculated */
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800209 slab_flags_t flags; /* Active flags on the slab */
Alexey Dobriyan7bbdb812018-04-05 16:21:31 -0700210 unsigned int useroffset;/* Usercopy region offset */
211 unsigned int usersize; /* Usercopy region size */
Joonsoo Kim07f361b2014-10-09 15:26:00 -0700212 const char *name; /* Slab name for sysfs */
213 int refcount; /* Use counter */
214 void (*ctor)(void *); /* Called on object slot creation */
215 struct list_head list; /* List of all slab caches on the system */
216};
217
218#endif /* CONFIG_SLOB */
219
220#ifdef CONFIG_SLAB
221#include <linux/slab_def.h>
222#endif
223
224#ifdef CONFIG_SLUB
225#include <linux/slub_def.h>
226#endif
227
228#include <linux/memcontrol.h>
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700229#include <linux/fault-inject.h>
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700230#include <linux/kasan.h>
231#include <linux/kmemleak.h>
Thomas Garnier7c00fce2016-07-26 15:21:56 -0700232#include <linux/random.h>
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +0100233#include <linux/sched/mm.h>
Muchun Song88f2ef72022-03-22 14:40:56 -0700234#include <linux/list_lru.h>
Joonsoo Kim07f361b2014-10-09 15:26:00 -0700235
Christoph Lameter97d06602012-07-06 15:25:11 -0500236/*
237 * State of the slab allocator.
238 *
239 * This is used to describe the states of the allocator during bootup.
240 * Allocators use this to gradually bootstrap themselves. Most allocators
241 * have the problem that the structures used for managing slab caches are
242 * allocated from slab caches themselves.
243 */
244enum slab_state {
245 DOWN, /* No slab functionality yet */
246 PARTIAL, /* SLUB: kmem_cache_node available */
Christoph Lameterce8eb6c2013-01-10 19:14:19 +0000247 PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */
Christoph Lameter97d06602012-07-06 15:25:11 -0500248 UP, /* Slab caches usable but not all extras yet */
249 FULL /* Everything is working */
250};
251
252extern enum slab_state slab_state;
253
Christoph Lameter18004c52012-07-06 15:25:12 -0500254/* The slab cache mutex protects the management structures during changes */
255extern struct mutex slab_mutex;
Christoph Lameter9b030cb2012-09-05 00:20:33 +0000256
257/* The list of all slab caches on the system */
Christoph Lameter18004c52012-07-06 15:25:12 -0500258extern struct list_head slab_caches;
259
Christoph Lameter9b030cb2012-09-05 00:20:33 +0000260/* The slab cache that manages slab cache information */
261extern struct kmem_cache *kmem_cache;
262
Vlastimil Babkaaf3b5f82017-02-22 15:41:05 -0800263/* A table of kmalloc cache names and sizes */
264extern const struct kmalloc_info_struct {
Pengfei Licb5d9fb2019-11-30 17:49:21 -0800265 const char *name[NR_KMALLOC_TYPES];
Alexey Dobriyan55de8b92018-04-05 16:20:29 -0700266 unsigned int size;
Vlastimil Babkaaf3b5f82017-02-22 15:41:05 -0800267} kmalloc_info[];
268
Christoph Lameterf97d5f632013-01-10 19:12:17 +0000269#ifndef CONFIG_SLOB
270/* Kmalloc array related functions */
Daniel Sanders34cc6992015-06-24 16:55:57 -0700271void setup_kmalloc_cache_index_table(void);
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800272void create_kmalloc_caches(slab_flags_t);
Christoph Lameter2c59dd62013-01-10 19:14:19 +0000273
274/* Find the kmalloc slab corresponding for a certain size */
275struct kmem_cache *kmalloc_slab(size_t, gfp_t);
Christoph Lameterf97d5f632013-01-10 19:12:17 +0000276#endif
277
Long Li44405092020-08-06 23:18:28 -0700278gfp_t kmalloc_fix_flags(gfp_t flags);
Christoph Lameterf97d5f632013-01-10 19:12:17 +0000279
Christoph Lameter9b030cb2012-09-05 00:20:33 +0000280/* Functions provided by the slab allocators */
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800281int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags);
Christoph Lameter97d06602012-07-06 15:25:11 -0500282
Alexey Dobriyan55de8b92018-04-05 16:20:29 -0700283struct kmem_cache *create_kmalloc_cache(const char *name, unsigned int size,
284 slab_flags_t flags, unsigned int useroffset,
285 unsigned int usersize);
Christoph Lameter45530c42012-11-28 16:23:07 +0000286extern void create_boot_cache(struct kmem_cache *, const char *name,
Alexey Dobriyan361d5752018-04-05 16:20:33 -0700287 unsigned int size, slab_flags_t flags,
288 unsigned int useroffset, unsigned int usersize);
Christoph Lameter45530c42012-11-28 16:23:07 +0000289
Joonsoo Kim423c9292014-10-09 15:26:22 -0700290int slab_unmergeable(struct kmem_cache *s);
Alexey Dobriyanf4957d52018-04-05 16:20:37 -0700291struct kmem_cache *find_mergeable(unsigned size, unsigned align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800292 slab_flags_t flags, const char *name, void (*ctor)(void *));
Joonsoo Kim12220de2014-10-09 15:26:24 -0700293#ifndef CONFIG_SLOB
Glauber Costa2633d7a2012-12-18 14:22:34 -0800294struct kmem_cache *
Alexey Dobriyanf4957d52018-04-05 16:20:37 -0700295__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800296 slab_flags_t flags, void (*ctor)(void *));
Joonsoo Kim423c9292014-10-09 15:26:22 -0700297
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -0700298slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -0800299 slab_flags_t flags, const char *name);
Christoph Lametercbb79692012-09-05 00:18:32 +0000300#else
Glauber Costa2633d7a2012-12-18 14:22:34 -0800301static inline struct kmem_cache *
Alexey Dobriyanf4957d52018-04-05 16:20:37 -0700302__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800303 slab_flags_t flags, void (*ctor)(void *))
Christoph Lametercbb79692012-09-05 00:18:32 +0000304{ return NULL; }
Joonsoo Kim423c9292014-10-09 15:26:22 -0700305
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -0700306static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -0800307 slab_flags_t flags, const char *name)
Joonsoo Kim423c9292014-10-09 15:26:22 -0700308{
309 return flags;
310}
Christoph Lametercbb79692012-09-05 00:18:32 +0000311#endif
312
313
Glauber Costad8843922012-10-17 15:36:51 +0400314/* Legal flag mask for kmem_cache_create(), for various configurations */
Nicolas Boichat6d6ea1e2019-03-28 20:43:42 -0700315#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
316 SLAB_CACHE_DMA32 | SLAB_PANIC | \
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -0800317 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS )
Glauber Costad8843922012-10-17 15:36:51 +0400318
319#if defined(CONFIG_DEBUG_SLAB)
320#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
321#elif defined(CONFIG_SLUB_DEBUG)
322#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
Laura Abbottbecfda62016-03-15 14:55:06 -0700323 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
Glauber Costad8843922012-10-17 15:36:51 +0400324#else
325#define SLAB_DEBUG_FLAGS (0)
326#endif
327
328#if defined(CONFIG_SLAB)
329#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
Vladimir Davydov230e9fc2016-01-14 15:18:15 -0800330 SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -0800331 SLAB_ACCOUNT)
Glauber Costad8843922012-10-17 15:36:51 +0400332#elif defined(CONFIG_SLUB)
333#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
Hyeonggon Yooa2859092022-04-06 15:00:03 +0900334 SLAB_TEMPORARY | SLAB_ACCOUNT | SLAB_NO_USER_FLAGS)
Glauber Costad8843922012-10-17 15:36:51 +0400335#else
Rustam Kovhaev34dbc3aa2021-11-19 16:43:37 -0800336#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE)
Glauber Costad8843922012-10-17 15:36:51 +0400337#endif
338
Thomas Garniere70954f2016-12-12 16:41:38 -0800339/* Common flags available with current configuration */
Glauber Costad8843922012-10-17 15:36:51 +0400340#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
341
Thomas Garniere70954f2016-12-12 16:41:38 -0800342/* Common flags permitted for kmem_cache_create */
343#define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \
344 SLAB_RED_ZONE | \
345 SLAB_POISON | \
346 SLAB_STORE_USER | \
347 SLAB_TRACE | \
348 SLAB_CONSISTENCY_CHECKS | \
349 SLAB_MEM_SPREAD | \
350 SLAB_NOLEAKTRACE | \
351 SLAB_RECLAIM_ACCOUNT | \
352 SLAB_TEMPORARY | \
Hyeonggon Yooa2859092022-04-06 15:00:03 +0900353 SLAB_ACCOUNT | \
354 SLAB_NO_USER_FLAGS)
Thomas Garniere70954f2016-12-12 16:41:38 -0800355
Shakeel Buttf9e13c02018-04-05 16:21:57 -0700356bool __kmem_cache_empty(struct kmem_cache *);
Christoph Lameter945cf2b2012-09-04 23:18:33 +0000357int __kmem_cache_shutdown(struct kmem_cache *);
Dmitry Safonov52b4b952016-02-17 13:11:37 -0800358void __kmem_cache_release(struct kmem_cache *);
Tejun Heoc9fc5862017-02-22 15:41:27 -0800359int __kmem_cache_shrink(struct kmem_cache *);
Christoph Lameter41a21282014-05-06 12:50:08 -0700360void slab_kmem_cache_release(struct kmem_cache *);
Christoph Lameter945cf2b2012-09-04 23:18:33 +0000361
Glauber Costab7454ad2012-10-19 18:20:25 +0400362struct seq_file;
363struct file;
Glauber Costab7454ad2012-10-19 18:20:25 +0400364
Glauber Costa0d7561c2012-10-19 18:20:27 +0400365struct slabinfo {
366 unsigned long active_objs;
367 unsigned long num_objs;
368 unsigned long active_slabs;
369 unsigned long num_slabs;
370 unsigned long shared_avail;
371 unsigned int limit;
372 unsigned int batchcount;
373 unsigned int shared;
374 unsigned int objects_per_slab;
375 unsigned int cache_order;
376};
377
378void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
379void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
Glauber Costab7454ad2012-10-19 18:20:25 +0400380ssize_t slabinfo_write(struct file *file, const char __user *buffer,
381 size_t count, loff_t *ppos);
Glauber Costaba6c4962012-12-18 14:22:27 -0800382
Christoph Lameter484748f2015-09-04 15:45:34 -0700383/*
384 * Generic implementation of bulk operations
385 * These are useful for situations in which the allocator cannot
Jesper Dangaard Brouer9f706d62016-03-15 14:54:03 -0700386 * perform optimizations. In that case segments of the object listed
Christoph Lameter484748f2015-09-04 15:45:34 -0700387 * may be allocated or freed using these operations.
388 */
389void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -0800390int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
Christoph Lameter484748f2015-09-04 15:45:34 -0700391
Muchun Song1a984c42020-12-14 19:06:24 -0800392static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700393{
394 return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
Roman Gushchind42f3242020-08-06 23:20:39 -0700395 NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700396}
397
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700398#ifdef CONFIG_SLUB_DEBUG
399#ifdef CONFIG_SLUB_DEBUG_ON
400DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
401#else
402DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
403#endif
404extern void print_tracking(struct kmem_cache *s, void *object);
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700405long validate_slab_cache(struct kmem_cache *s);
Marco Elver0d4a0622021-07-14 21:26:34 -0700406static inline bool __slub_debug_enabled(void)
407{
408 return static_branch_unlikely(&slub_debug_enabled);
409}
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700410#else
411static inline void print_tracking(struct kmem_cache *s, void *object)
412{
413}
Marco Elver0d4a0622021-07-14 21:26:34 -0700414static inline bool __slub_debug_enabled(void)
415{
416 return false;
417}
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700418#endif
419
420/*
421 * Returns true if any of the specified slub_debug flags is enabled for the
422 * cache. Use only for flags parsed by setup_slub_debug() as it also enables
423 * the static key.
424 */
425static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
426{
Marco Elver0d4a0622021-07-14 21:26:34 -0700427 if (IS_ENABLED(CONFIG_SLUB_DEBUG))
428 VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
429 if (__slub_debug_enabled())
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700430 return s->flags & flags;
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700431 return false;
432}
433
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700434#ifdef CONFIG_MEMCG_KMEM
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100435/*
436 * slab_objcgs - get the object cgroups vector associated with a slab
437 * @slab: a pointer to the slab struct
438 *
439 * Returns a pointer to the object cgroups vector associated with the slab,
440 * or NULL if no such vector has been associated yet.
441 */
442static inline struct obj_cgroup **slab_objcgs(struct slab *slab)
443{
444 unsigned long memcg_data = READ_ONCE(slab->memcg_data);
445
446 VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS),
447 slab_page(slab));
448 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab));
449
450 return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
451}
452
453int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s,
454 gfp_t gfp, bool new_slab);
Waiman Longfdbcb2a2021-06-28 19:37:19 -0700455void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
456 enum node_stat_item idx, int nr);
Roman Gushchin286e04b2020-08-06 23:20:52 -0700457
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100458static inline void memcg_free_slab_cgroups(struct slab *slab)
Roman Gushchin286e04b2020-08-06 23:20:52 -0700459{
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100460 kfree(slab_objcgs(slab));
461 slab->memcg_data = 0;
Roman Gushchin286e04b2020-08-06 23:20:52 -0700462}
463
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700464static inline size_t obj_full_size(struct kmem_cache *s)
465{
466 /*
467 * For each accounted object there is an extra space which is used
468 * to store obj_cgroup membership. Charge it too.
469 */
470 return s->size + sizeof(struct obj_cgroup *);
471}
472
Roman Gushchinbecaba62020-12-05 22:14:45 -0800473/*
474 * Returns false if the allocation should fail.
475 */
476static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
Muchun Song88f2ef72022-03-22 14:40:56 -0700477 struct list_lru *lru,
Roman Gushchinbecaba62020-12-05 22:14:45 -0800478 struct obj_cgroup **objcgp,
479 size_t objects, gfp_t flags)
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700480{
Roman Gushchin98556092020-08-06 23:21:10 -0700481 struct obj_cgroup *objcg;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700482
Roman Gushchinbecaba62020-12-05 22:14:45 -0800483 if (!memcg_kmem_enabled())
484 return true;
485
486 if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))
487 return true;
488
Roman Gushchin98556092020-08-06 23:21:10 -0700489 objcg = get_obj_cgroup_from_current();
490 if (!objcg)
Roman Gushchinbecaba62020-12-05 22:14:45 -0800491 return true;
Roman Gushchin98556092020-08-06 23:21:10 -0700492
Muchun Song88f2ef72022-03-22 14:40:56 -0700493 if (lru) {
494 int ret;
495 struct mem_cgroup *memcg;
496
497 memcg = get_mem_cgroup_from_objcg(objcg);
498 ret = memcg_list_lru_alloc(memcg, lru, flags);
499 css_put(&memcg->css);
500
501 if (ret)
502 goto out;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700503 }
504
Muchun Song88f2ef72022-03-22 14:40:56 -0700505 if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s)))
506 goto out;
507
Roman Gushchinbecaba62020-12-05 22:14:45 -0800508 *objcgp = objcg;
509 return true;
Muchun Song88f2ef72022-03-22 14:40:56 -0700510out:
511 obj_cgroup_put(objcg);
512 return false;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700513}
514
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700515static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
516 struct obj_cgroup *objcg,
Roman Gushchin10befea2020-08-06 23:21:27 -0700517 gfp_t flags, size_t size,
518 void **p)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700519{
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100520 struct slab *slab;
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700521 unsigned long off;
522 size_t i;
523
Roman Gushchinbecaba62020-12-05 22:14:45 -0800524 if (!memcg_kmem_enabled() || !objcg)
Roman Gushchin10befea2020-08-06 23:21:27 -0700525 return;
526
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700527 for (i = 0; i < size; i++) {
528 if (likely(p[i])) {
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100529 slab = virt_to_slab(p[i]);
Roman Gushchin10befea2020-08-06 23:21:27 -0700530
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100531 if (!slab_objcgs(slab) &&
532 memcg_alloc_slab_cgroups(slab, s, flags,
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800533 false)) {
Roman Gushchin10befea2020-08-06 23:21:27 -0700534 obj_cgroup_uncharge(objcg, obj_full_size(s));
535 continue;
536 }
537
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100538 off = obj_to_index(s, slab, p[i]);
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700539 obj_cgroup_get(objcg);
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100540 slab_objcgs(slab)[off] = objcg;
541 mod_objcg_state(objcg, slab_pgdat(slab),
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700542 cache_vmstat_idx(s), obj_full_size(s));
543 } else {
544 obj_cgroup_uncharge(objcg, obj_full_size(s));
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700545 }
546 }
547 obj_cgroup_put(objcg);
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700548}
549
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700550static inline void memcg_slab_free_hook(struct kmem_cache *s_orig,
551 void **p, int objects)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700552{
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700553 struct kmem_cache *s;
Roman Gushchin270c6a72020-12-01 13:58:28 -0800554 struct obj_cgroup **objcgs;
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700555 struct obj_cgroup *objcg;
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100556 struct slab *slab;
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700557 unsigned int off;
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700558 int i;
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700559
Roman Gushchin10befea2020-08-06 23:21:27 -0700560 if (!memcg_kmem_enabled())
561 return;
562
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700563 for (i = 0; i < objects; i++) {
564 if (unlikely(!p[i]))
565 continue;
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700566
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100567 slab = virt_to_slab(p[i]);
568 /* we could be given a kmalloc_large() object, skip those */
569 if (!slab)
570 continue;
571
572 objcgs = slab_objcgs(slab);
Roman Gushchin270c6a72020-12-01 13:58:28 -0800573 if (!objcgs)
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700574 continue;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700575
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700576 if (!s_orig)
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100577 s = slab->slab_cache;
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700578 else
579 s = s_orig;
Roman Gushchin10befea2020-08-06 23:21:27 -0700580
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100581 off = obj_to_index(s, slab, p[i]);
Roman Gushchin270c6a72020-12-01 13:58:28 -0800582 objcg = objcgs[off];
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700583 if (!objcg)
584 continue;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700585
Roman Gushchin270c6a72020-12-01 13:58:28 -0800586 objcgs[off] = NULL;
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700587 obj_cgroup_uncharge(objcg, obj_full_size(s));
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100588 mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s),
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700589 -obj_full_size(s));
590 obj_cgroup_put(objcg);
591 }
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700592}
593
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700594#else /* CONFIG_MEMCG_KMEM */
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100595static inline struct obj_cgroup **slab_objcgs(struct slab *slab)
596{
597 return NULL;
598}
599
Roman Gushchin98556092020-08-06 23:21:10 -0700600static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
Roman Gushchin4d96ba32019-07-11 20:56:31 -0700601{
602 return NULL;
603}
604
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100605static inline int memcg_alloc_slab_cgroups(struct slab *slab,
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800606 struct kmem_cache *s, gfp_t gfp,
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100607 bool new_slab)
Roman Gushchin286e04b2020-08-06 23:20:52 -0700608{
609 return 0;
610}
611
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100612static inline void memcg_free_slab_cgroups(struct slab *slab)
Roman Gushchin286e04b2020-08-06 23:20:52 -0700613{
614}
615
Roman Gushchinbecaba62020-12-05 22:14:45 -0800616static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
Muchun Song88f2ef72022-03-22 14:40:56 -0700617 struct list_lru *lru,
Roman Gushchinbecaba62020-12-05 22:14:45 -0800618 struct obj_cgroup **objcgp,
619 size_t objects, gfp_t flags)
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700620{
Roman Gushchinbecaba62020-12-05 22:14:45 -0800621 return true;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700622}
623
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700624static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
625 struct obj_cgroup *objcg,
Roman Gushchin10befea2020-08-06 23:21:27 -0700626 gfp_t flags, size_t size,
627 void **p)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700628{
629}
630
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700631static inline void memcg_slab_free_hook(struct kmem_cache *s,
632 void **p, int objects)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700633{
634}
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700635#endif /* CONFIG_MEMCG_KMEM */
Glauber Costab9ce5ef2012-12-18 14:22:46 -0800636
Vlastimil Babka401fb122021-11-04 11:30:58 +0100637#ifndef CONFIG_SLOB
Kees Cooka64b5372019-07-11 20:53:26 -0700638static inline struct kmem_cache *virt_to_cache(const void *obj)
639{
Matthew Wilcox (Oracle)82c17752021-10-04 14:45:53 +0100640 struct slab *slab;
Kees Cooka64b5372019-07-11 20:53:26 -0700641
Matthew Wilcox (Oracle)82c17752021-10-04 14:45:53 +0100642 slab = virt_to_slab(obj);
643 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n",
Kees Cooka64b5372019-07-11 20:53:26 -0700644 __func__))
645 return NULL;
Matthew Wilcox (Oracle)82c17752021-10-04 14:45:53 +0100646 return slab->slab_cache;
Kees Cooka64b5372019-07-11 20:53:26 -0700647}
648
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100649static __always_inline void account_slab(struct slab *slab, int order,
650 struct kmem_cache *s, gfp_t gfp)
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700651{
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800652 if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100653 memcg_alloc_slab_cgroups(slab, s, gfp, true);
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800654
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100655 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700656 PAGE_SIZE << order);
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700657}
658
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100659static __always_inline void unaccount_slab(struct slab *slab, int order,
660 struct kmem_cache *s)
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700661{
Roman Gushchin10befea2020-08-06 23:21:27 -0700662 if (memcg_kmem_enabled())
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100663 memcg_free_slab_cgroups(slab);
Roman Gushchin98556092020-08-06 23:21:10 -0700664
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100665 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700666 -(PAGE_SIZE << order));
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700667}
668
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700669static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
670{
671 struct kmem_cache *cachep;
672
673 if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700674 !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
675 return s;
676
677 cachep = virt_to_cache(x);
Roman Gushchin10befea2020-08-06 23:21:27 -0700678 if (WARN(cachep && cachep != s,
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700679 "%s: Wrong slab cache. %s but object is from %s\n",
680 __func__, s->name, cachep->name))
681 print_tracking(cachep, x);
682 return cachep;
683}
Vlastimil Babka401fb122021-11-04 11:30:58 +0100684#endif /* CONFIG_SLOB */
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700685
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700686static inline size_t slab_ksize(const struct kmem_cache *s)
687{
688#ifndef CONFIG_SLUB
689 return s->object_size;
690
691#else /* CONFIG_SLUB */
692# ifdef CONFIG_SLUB_DEBUG
693 /*
694 * Debugging requires use of the padding between object
695 * and whatever may come after it.
696 */
697 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
698 return s->object_size;
699# endif
Alexander Potapenko80a92012016-07-28 15:49:07 -0700700 if (s->flags & SLAB_KASAN)
701 return s->object_size;
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700702 /*
703 * If we have the need to store the freelist pointer
704 * back there or track user information then we can
705 * only use the space before that information.
706 */
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -0800707 if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700708 return s->inuse;
709 /*
710 * Else we can use all the padding etc for the allocation
711 */
712 return s->size;
713#endif
714}
715
716static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
Muchun Song88f2ef72022-03-22 14:40:56 -0700717 struct list_lru *lru,
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700718 struct obj_cgroup **objcgp,
719 size_t size, gfp_t flags)
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700720{
721 flags &= gfp_allowed_mask;
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +0100722
Daniel Vetter95d6c702020-12-14 19:08:34 -0800723 might_alloc(flags);
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700724
Jesper Dangaard Brouerfab99632016-03-15 14:53:38 -0700725 if (should_failslab(s, flags))
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700726 return NULL;
727
Muchun Song88f2ef72022-03-22 14:40:56 -0700728 if (!memcg_slab_pre_alloc_hook(s, lru, objcgp, size, flags))
Roman Gushchinbecaba62020-12-05 22:14:45 -0800729 return NULL;
Vladimir Davydov45264772016-07-26 15:24:21 -0700730
731 return s;
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700732}
733
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700734static inline void slab_post_alloc_hook(struct kmem_cache *s,
Andrey Konovalovda844b72021-04-29 23:00:06 -0700735 struct obj_cgroup *objcg, gfp_t flags,
736 size_t size, void **p, bool init)
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700737{
738 size_t i;
739
740 flags &= gfp_allowed_mask;
Andrey Konovalovda844b72021-04-29 23:00:06 -0700741
742 /*
743 * As memory initialization might be integrated into KASAN,
744 * kasan_slab_alloc and initialization memset must be
745 * kept together to avoid discrepancies in behavior.
746 *
747 * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
748 */
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700749 for (i = 0; i < size; i++) {
Andrey Konovalovda844b72021-04-29 23:00:06 -0700750 p[i] = kasan_slab_alloc(s, p[i], flags, init);
751 if (p[i] && init && !kasan_has_integrated_init())
752 memset(p[i], 0, s->object_size);
Andrey Konovalov53128242019-02-20 22:19:11 -0800753 kmemleak_alloc_recursive(p[i], s->object_size, 1,
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700754 s->flags, flags);
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700755 }
Vladimir Davydov45264772016-07-26 15:24:21 -0700756
Roman Gushchinbecaba62020-12-05 22:14:45 -0800757 memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700758}
759
Christoph Lameter44c53562014-08-06 16:04:07 -0700760#ifndef CONFIG_SLOB
Christoph Lameterca349562013-01-10 19:14:19 +0000761/*
762 * The slab lists for all objects.
763 */
764struct kmem_cache_node {
765 spinlock_t list_lock;
766
767#ifdef CONFIG_SLAB
768 struct list_head slabs_partial; /* partial list first, better asm code */
769 struct list_head slabs_full;
770 struct list_head slabs_free;
David Rientjesbf00bd32016-12-12 16:41:44 -0800771 unsigned long total_slabs; /* length of all slab lists */
772 unsigned long free_slabs; /* length of free slab list only */
Christoph Lameterca349562013-01-10 19:14:19 +0000773 unsigned long free_objects;
774 unsigned int free_limit;
775 unsigned int colour_next; /* Per-node cache coloring */
776 struct array_cache *shared; /* shared per node */
Joonsoo Kimc8522a32014-08-06 16:04:29 -0700777 struct alien_cache **alien; /* on other nodes */
Christoph Lameterca349562013-01-10 19:14:19 +0000778 unsigned long next_reap; /* updated without locking */
779 int free_touched; /* updated without locking */
780#endif
781
782#ifdef CONFIG_SLUB
783 unsigned long nr_partial;
784 struct list_head partial;
785#ifdef CONFIG_SLUB_DEBUG
786 atomic_long_t nr_slabs;
787 atomic_long_t total_objects;
788 struct list_head full;
789#endif
790#endif
791
792};
Wanpeng Lie25839f2013-07-04 08:33:23 +0800793
Christoph Lameter44c53562014-08-06 16:04:07 -0700794static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
795{
796 return s->node[node];
797}
798
799/*
800 * Iterator over all nodes. The body will be executed for each node that has
801 * a kmem_cache_node structure allocated (which is true for all online nodes)
802 */
803#define for_each_kmem_cache_node(__s, __node, __n) \
Mikulas Patocka91635822014-10-09 15:26:20 -0700804 for (__node = 0; __node < nr_node_ids; __node++) \
805 if ((__n = get_node(__s, __node)))
Christoph Lameter44c53562014-08-06 16:04:07 -0700806
807#endif
808
Yang Shi852d8be2017-11-15 17:32:07 -0800809#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
810void dump_unreclaimable_slab(void);
811#else
812static inline void dump_unreclaimable_slab(void)
813{
814}
815#endif
816
Alexander Potapenko55834c52016-05-20 16:59:11 -0700817void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);
818
Thomas Garnier7c00fce2016-07-26 15:21:56 -0700819#ifdef CONFIG_SLAB_FREELIST_RANDOM
820int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
821 gfp_t gfp);
822void cache_random_seq_destroy(struct kmem_cache *cachep);
823#else
824static inline int cache_random_seq_create(struct kmem_cache *cachep,
825 unsigned int count, gfp_t gfp)
826{
827 return 0;
828}
829static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
830#endif /* CONFIG_SLAB_FREELIST_RANDOM */
831
Alexander Potapenko64713842019-07-11 20:59:19 -0700832static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c)
833{
Kees Cook51cba1e2021-04-01 16:23:43 -0700834 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
835 &init_on_alloc)) {
Alexander Potapenko64713842019-07-11 20:59:19 -0700836 if (c->ctor)
837 return false;
838 if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))
839 return flags & __GFP_ZERO;
840 return true;
841 }
842 return flags & __GFP_ZERO;
843}
844
845static inline bool slab_want_init_on_free(struct kmem_cache *c)
846{
Kees Cook51cba1e2021-04-01 16:23:43 -0700847 if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
848 &init_on_free))
Alexander Potapenko64713842019-07-11 20:59:19 -0700849 return !(c->ctor ||
850 (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)));
851 return false;
852}
853
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -0700854#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
855void debugfs_slab_release(struct kmem_cache *);
856#else
857static inline void debugfs_slab_release(struct kmem_cache *s) { }
858#endif
859
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -0800860#ifdef CONFIG_PRINTK
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800861#define KS_ADDRS_COUNT 16
862struct kmem_obj_info {
863 void *kp_ptr;
Matthew Wilcox (Oracle)72132302021-10-04 14:45:55 +0100864 struct slab *kp_slab;
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800865 void *kp_objp;
866 unsigned long kp_data_offset;
867 struct kmem_cache *kp_slab_cache;
868 void *kp_ret;
869 void *kp_stack[KS_ADDRS_COUNT];
Maninder Singhe548eaa2021-03-16 16:07:11 +0530870 void *kp_free_stack[KS_ADDRS_COUNT];
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800871};
Marco Elver2dfe63e2022-04-14 19:13:40 -0700872void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -0800873#endif
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800874
Matthew Wilcox (Oracle)0b3eb0912021-10-04 14:45:56 +0100875#ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR
876void __check_heap_object(const void *ptr, unsigned long n,
877 const struct slab *slab, bool to_user);
878#else
879static inline
880void __check_heap_object(const void *ptr, unsigned long n,
881 const struct slab *slab, bool to_user)
882{
883}
884#endif
885
Andrey Ryabinin5240ab42014-08-06 16:04:14 -0700886#endif /* MM_SLAB_H */