blob: 4d8330d575735a36df73cac1c90c9dcd970bf6c0 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Christoph Lameter97d06602012-07-06 15:25:11 -05002#ifndef MM_SLAB_H
3#define MM_SLAB_H
4/*
5 * Internal slab definitions
6 */
7
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +01008/* Reuses the bits in struct page */
9struct slab {
10 unsigned long __page_flags;
Vlastimil Babka401fb122021-11-04 11:30:58 +010011
12#if defined(CONFIG_SLAB)
13
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010014 union {
15 struct list_head slab_list;
Vlastimil Babka401fb122021-11-04 11:30:58 +010016 struct rcu_head rcu_head;
17 };
18 struct kmem_cache *slab_cache;
19 void *freelist; /* array of free object indexes */
20 void *s_mem; /* first object */
21 unsigned int active;
22
23#elif defined(CONFIG_SLUB)
24
25 union {
26 struct list_head slab_list;
27 struct rcu_head rcu_head;
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +010028#ifdef CONFIG_SLUB_CPU_PARTIAL
Vlastimil Babka401fb122021-11-04 11:30:58 +010029 struct {
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010030 struct slab *next;
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010031 int slabs; /* Nr of slabs left */
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010032 };
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +010033#endif
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010034 };
Vlastimil Babka401fb122021-11-04 11:30:58 +010035 struct kmem_cache *slab_cache;
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010036 /* Double-word boundary */
37 void *freelist; /* first free object */
38 union {
Vlastimil Babka401fb122021-11-04 11:30:58 +010039 unsigned long counters;
40 struct {
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010041 unsigned inuse:16;
42 unsigned objects:15;
43 unsigned frozen:1;
44 };
45 };
Vlastimil Babka401fb122021-11-04 11:30:58 +010046 unsigned int __unused;
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010047
Vlastimil Babka401fb122021-11-04 11:30:58 +010048#elif defined(CONFIG_SLOB)
49
50 struct list_head slab_list;
51 void *__unused_1;
52 void *freelist; /* first free block */
Hyeonggon Yoob01af5c2021-12-12 06:52:41 +000053 long units;
54 unsigned int __unused_2;
Vlastimil Babka401fb122021-11-04 11:30:58 +010055
56#else
57#error "Unexpected slab allocator configured"
58#endif
59
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010060 atomic_t __page_refcount;
61#ifdef CONFIG_MEMCG
62 unsigned long memcg_data;
63#endif
64};
65
66#define SLAB_MATCH(pg, sl) \
67 static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))
68SLAB_MATCH(flags, __page_flags);
69SLAB_MATCH(compound_head, slab_list); /* Ensure bit 0 is clear */
Vlastimil Babka401fb122021-11-04 11:30:58 +010070#ifndef CONFIG_SLOB
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010071SLAB_MATCH(rcu_head, rcu_head);
Vlastimil Babka401fb122021-11-04 11:30:58 +010072#endif
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +010073SLAB_MATCH(_refcount, __page_refcount);
74#ifdef CONFIG_MEMCG
75SLAB_MATCH(memcg_data, memcg_data);
76#endif
77#undef SLAB_MATCH
78static_assert(sizeof(struct slab) <= sizeof(struct page));
79
80/**
81 * folio_slab - Converts from folio to slab.
82 * @folio: The folio.
83 *
84 * Currently struct slab is a different representation of a folio where
85 * folio_test_slab() is true.
86 *
87 * Return: The slab which contains this folio.
88 */
89#define folio_slab(folio) (_Generic((folio), \
90 const struct folio *: (const struct slab *)(folio), \
91 struct folio *: (struct slab *)(folio)))
92
93/**
94 * slab_folio - The folio allocated for a slab
95 * @slab: The slab.
96 *
97 * Slabs are allocated as folios that contain the individual objects and are
98 * using some fields in the first struct page of the folio - those fields are
99 * now accessed by struct slab. It is occasionally necessary to convert back to
100 * a folio in order to communicate with the rest of the mm. Please use this
101 * helper function instead of casting yourself, as the implementation may change
102 * in the future.
103 */
104#define slab_folio(s) (_Generic((s), \
105 const struct slab *: (const struct folio *)s, \
106 struct slab *: (struct folio *)s))
107
108/**
109 * page_slab - Converts from first struct page to slab.
110 * @p: The first (either head of compound or single) page of slab.
111 *
112 * A temporary wrapper to convert struct page to struct slab in situations where
113 * we know the page is the compound head, or single order-0 page.
114 *
115 * Long-term ideally everything would work with struct slab directly or go
116 * through folio to struct slab.
117 *
118 * Return: The slab which contains this page
119 */
120#define page_slab(p) (_Generic((p), \
121 const struct page *: (const struct slab *)(p), \
122 struct page *: (struct slab *)(p)))
123
124/**
125 * slab_page - The first struct page allocated for a slab
126 * @slab: The slab.
127 *
128 * A convenience wrapper for converting slab to the first struct page of the
129 * underlying folio, to communicate with code not yet converted to folio or
130 * struct slab.
131 */
132#define slab_page(s) folio_page(slab_folio(s), 0)
133
134/*
135 * If network-based swap is enabled, sl*b must keep track of whether pages
136 * were allocated from pfmemalloc reserves.
137 */
138static inline bool slab_test_pfmemalloc(const struct slab *slab)
139{
140 return folio_test_active((struct folio *)slab_folio(slab));
141}
142
143static inline void slab_set_pfmemalloc(struct slab *slab)
144{
145 folio_set_active(slab_folio(slab));
146}
147
148static inline void slab_clear_pfmemalloc(struct slab *slab)
149{
150 folio_clear_active(slab_folio(slab));
151}
152
153static inline void __slab_clear_pfmemalloc(struct slab *slab)
154{
155 __folio_clear_active(slab_folio(slab));
156}
157
158static inline void *slab_address(const struct slab *slab)
159{
160 return folio_address(slab_folio(slab));
161}
162
163static inline int slab_nid(const struct slab *slab)
164{
165 return folio_nid(slab_folio(slab));
166}
167
168static inline pg_data_t *slab_pgdat(const struct slab *slab)
169{
170 return folio_pgdat(slab_folio(slab));
171}
172
173static inline struct slab *virt_to_slab(const void *addr)
174{
175 struct folio *folio = virt_to_folio(addr);
176
177 if (!folio_test_slab(folio))
178 return NULL;
179
180 return folio_slab(folio);
181}
182
183static inline int slab_order(const struct slab *slab)
184{
185 return folio_order((struct folio *)slab_folio(slab));
186}
187
188static inline size_t slab_size(const struct slab *slab)
189{
190 return PAGE_SIZE << slab_order(slab);
191}
192
Joonsoo Kim07f361b2014-10-09 15:26:00 -0700193#ifdef CONFIG_SLOB
194/*
195 * Common fields provided in kmem_cache by all slab allocators
196 * This struct is either used directly by the allocator (SLOB)
197 * or the allocator must include definitions for all fields
198 * provided in kmem_cache_common in their definition of kmem_cache.
199 *
200 * Once we can do anonymous structs (C11 standard) we could put a
201 * anonymous struct definition in these allocators so that the
202 * separate allocations in the kmem_cache structure of SLAB and
203 * SLUB is no longer needed.
204 */
205struct kmem_cache {
206 unsigned int object_size;/* The original size of the object */
207 unsigned int size; /* The aligned/padded/added on size */
208 unsigned int align; /* Alignment as calculated */
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800209 slab_flags_t flags; /* Active flags on the slab */
Alexey Dobriyan7bbdb812018-04-05 16:21:31 -0700210 unsigned int useroffset;/* Usercopy region offset */
211 unsigned int usersize; /* Usercopy region size */
Joonsoo Kim07f361b2014-10-09 15:26:00 -0700212 const char *name; /* Slab name for sysfs */
213 int refcount; /* Use counter */
214 void (*ctor)(void *); /* Called on object slot creation */
215 struct list_head list; /* List of all slab caches on the system */
216};
217
218#endif /* CONFIG_SLOB */
219
220#ifdef CONFIG_SLAB
221#include <linux/slab_def.h>
222#endif
223
224#ifdef CONFIG_SLUB
225#include <linux/slub_def.h>
226#endif
227
228#include <linux/memcontrol.h>
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700229#include <linux/fault-inject.h>
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700230#include <linux/kasan.h>
231#include <linux/kmemleak.h>
Thomas Garnier7c00fce2016-07-26 15:21:56 -0700232#include <linux/random.h>
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +0100233#include <linux/sched/mm.h>
Muchun Song88f2ef72022-03-22 14:40:56 -0700234#include <linux/list_lru.h>
Joonsoo Kim07f361b2014-10-09 15:26:00 -0700235
Christoph Lameter97d06602012-07-06 15:25:11 -0500236/*
237 * State of the slab allocator.
238 *
239 * This is used to describe the states of the allocator during bootup.
240 * Allocators use this to gradually bootstrap themselves. Most allocators
241 * have the problem that the structures used for managing slab caches are
242 * allocated from slab caches themselves.
243 */
244enum slab_state {
245 DOWN, /* No slab functionality yet */
246 PARTIAL, /* SLUB: kmem_cache_node available */
Christoph Lameterce8eb6c2013-01-10 19:14:19 +0000247 PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */
Christoph Lameter97d06602012-07-06 15:25:11 -0500248 UP, /* Slab caches usable but not all extras yet */
249 FULL /* Everything is working */
250};
251
252extern enum slab_state slab_state;
253
Christoph Lameter18004c52012-07-06 15:25:12 -0500254/* The slab cache mutex protects the management structures during changes */
255extern struct mutex slab_mutex;
Christoph Lameter9b030cb2012-09-05 00:20:33 +0000256
257/* The list of all slab caches on the system */
Christoph Lameter18004c52012-07-06 15:25:12 -0500258extern struct list_head slab_caches;
259
Christoph Lameter9b030cb2012-09-05 00:20:33 +0000260/* The slab cache that manages slab cache information */
261extern struct kmem_cache *kmem_cache;
262
Vlastimil Babkaaf3b5f82017-02-22 15:41:05 -0800263/* A table of kmalloc cache names and sizes */
264extern const struct kmalloc_info_struct {
Pengfei Licb5d9fb2019-11-30 17:49:21 -0800265 const char *name[NR_KMALLOC_TYPES];
Alexey Dobriyan55de8b92018-04-05 16:20:29 -0700266 unsigned int size;
Vlastimil Babkaaf3b5f82017-02-22 15:41:05 -0800267} kmalloc_info[];
268
Christoph Lameterf97d5f632013-01-10 19:12:17 +0000269#ifndef CONFIG_SLOB
270/* Kmalloc array related functions */
Daniel Sanders34cc6992015-06-24 16:55:57 -0700271void setup_kmalloc_cache_index_table(void);
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800272void create_kmalloc_caches(slab_flags_t);
Christoph Lameter2c59dd62013-01-10 19:14:19 +0000273
274/* Find the kmalloc slab corresponding for a certain size */
275struct kmem_cache *kmalloc_slab(size_t, gfp_t);
Hyeonggon Yooed4cd172022-08-17 19:18:20 +0900276
277void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,
278 int node, size_t orig_size,
279 unsigned long caller);
280void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller);
Christoph Lameterf97d5f632013-01-10 19:12:17 +0000281#endif
282
Long Li44405092020-08-06 23:18:28 -0700283gfp_t kmalloc_fix_flags(gfp_t flags);
Christoph Lameterf97d5f632013-01-10 19:12:17 +0000284
Christoph Lameter9b030cb2012-09-05 00:20:33 +0000285/* Functions provided by the slab allocators */
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800286int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags);
Christoph Lameter97d06602012-07-06 15:25:11 -0500287
Alexey Dobriyan55de8b92018-04-05 16:20:29 -0700288struct kmem_cache *create_kmalloc_cache(const char *name, unsigned int size,
289 slab_flags_t flags, unsigned int useroffset,
290 unsigned int usersize);
Christoph Lameter45530c42012-11-28 16:23:07 +0000291extern void create_boot_cache(struct kmem_cache *, const char *name,
Alexey Dobriyan361d5752018-04-05 16:20:33 -0700292 unsigned int size, slab_flags_t flags,
293 unsigned int useroffset, unsigned int usersize);
Christoph Lameter45530c42012-11-28 16:23:07 +0000294
Joonsoo Kim423c9292014-10-09 15:26:22 -0700295int slab_unmergeable(struct kmem_cache *s);
Alexey Dobriyanf4957d52018-04-05 16:20:37 -0700296struct kmem_cache *find_mergeable(unsigned size, unsigned align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800297 slab_flags_t flags, const char *name, void (*ctor)(void *));
Joonsoo Kim12220de2014-10-09 15:26:24 -0700298#ifndef CONFIG_SLOB
Glauber Costa2633d7a2012-12-18 14:22:34 -0800299struct kmem_cache *
Alexey Dobriyanf4957d52018-04-05 16:20:37 -0700300__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800301 slab_flags_t flags, void (*ctor)(void *));
Joonsoo Kim423c9292014-10-09 15:26:22 -0700302
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -0700303slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -0800304 slab_flags_t flags, const char *name);
Christoph Lametercbb79692012-09-05 00:18:32 +0000305#else
Glauber Costa2633d7a2012-12-18 14:22:34 -0800306static inline struct kmem_cache *
Alexey Dobriyanf4957d52018-04-05 16:20:37 -0700307__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800308 slab_flags_t flags, void (*ctor)(void *))
Christoph Lametercbb79692012-09-05 00:18:32 +0000309{ return NULL; }
Joonsoo Kim423c9292014-10-09 15:26:22 -0700310
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -0700311static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -0800312 slab_flags_t flags, const char *name)
Joonsoo Kim423c9292014-10-09 15:26:22 -0700313{
314 return flags;
315}
Christoph Lametercbb79692012-09-05 00:18:32 +0000316#endif
317
318
Glauber Costad8843922012-10-17 15:36:51 +0400319/* Legal flag mask for kmem_cache_create(), for various configurations */
Nicolas Boichat6d6ea1e2019-03-28 20:43:42 -0700320#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
321 SLAB_CACHE_DMA32 | SLAB_PANIC | \
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -0800322 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS )
Glauber Costad8843922012-10-17 15:36:51 +0400323
324#if defined(CONFIG_DEBUG_SLAB)
325#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
326#elif defined(CONFIG_SLUB_DEBUG)
327#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
Laura Abbottbecfda62016-03-15 14:55:06 -0700328 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
Glauber Costad8843922012-10-17 15:36:51 +0400329#else
330#define SLAB_DEBUG_FLAGS (0)
331#endif
332
333#if defined(CONFIG_SLAB)
334#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
Vladimir Davydov230e9fc2016-01-14 15:18:15 -0800335 SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -0800336 SLAB_ACCOUNT)
Glauber Costad8843922012-10-17 15:36:51 +0400337#elif defined(CONFIG_SLUB)
338#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
Hyeonggon Yooa2859092022-04-06 15:00:03 +0900339 SLAB_TEMPORARY | SLAB_ACCOUNT | SLAB_NO_USER_FLAGS)
Glauber Costad8843922012-10-17 15:36:51 +0400340#else
Rustam Kovhaev34dbc3aa2021-11-19 16:43:37 -0800341#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE)
Glauber Costad8843922012-10-17 15:36:51 +0400342#endif
343
Thomas Garniere70954f2016-12-12 16:41:38 -0800344/* Common flags available with current configuration */
Glauber Costad8843922012-10-17 15:36:51 +0400345#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
346
Thomas Garniere70954f2016-12-12 16:41:38 -0800347/* Common flags permitted for kmem_cache_create */
348#define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \
349 SLAB_RED_ZONE | \
350 SLAB_POISON | \
351 SLAB_STORE_USER | \
352 SLAB_TRACE | \
353 SLAB_CONSISTENCY_CHECKS | \
354 SLAB_MEM_SPREAD | \
355 SLAB_NOLEAKTRACE | \
356 SLAB_RECLAIM_ACCOUNT | \
357 SLAB_TEMPORARY | \
Hyeonggon Yooa2859092022-04-06 15:00:03 +0900358 SLAB_ACCOUNT | \
359 SLAB_NO_USER_FLAGS)
Thomas Garniere70954f2016-12-12 16:41:38 -0800360
Shakeel Buttf9e13c02018-04-05 16:21:57 -0700361bool __kmem_cache_empty(struct kmem_cache *);
Christoph Lameter945cf2b2012-09-04 23:18:33 +0000362int __kmem_cache_shutdown(struct kmem_cache *);
Dmitry Safonov52b4b952016-02-17 13:11:37 -0800363void __kmem_cache_release(struct kmem_cache *);
Tejun Heoc9fc5862017-02-22 15:41:27 -0800364int __kmem_cache_shrink(struct kmem_cache *);
Christoph Lameter41a21282014-05-06 12:50:08 -0700365void slab_kmem_cache_release(struct kmem_cache *);
Christoph Lameter945cf2b2012-09-04 23:18:33 +0000366
Glauber Costab7454ad2012-10-19 18:20:25 +0400367struct seq_file;
368struct file;
Glauber Costab7454ad2012-10-19 18:20:25 +0400369
Glauber Costa0d7561c2012-10-19 18:20:27 +0400370struct slabinfo {
371 unsigned long active_objs;
372 unsigned long num_objs;
373 unsigned long active_slabs;
374 unsigned long num_slabs;
375 unsigned long shared_avail;
376 unsigned int limit;
377 unsigned int batchcount;
378 unsigned int shared;
379 unsigned int objects_per_slab;
380 unsigned int cache_order;
381};
382
383void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
384void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
Glauber Costab7454ad2012-10-19 18:20:25 +0400385ssize_t slabinfo_write(struct file *file, const char __user *buffer,
386 size_t count, loff_t *ppos);
Glauber Costaba6c4962012-12-18 14:22:27 -0800387
Muchun Song1a984c42020-12-14 19:06:24 -0800388static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700389{
390 return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
Roman Gushchind42f3242020-08-06 23:20:39 -0700391 NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700392}
393
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700394#ifdef CONFIG_SLUB_DEBUG
395#ifdef CONFIG_SLUB_DEBUG_ON
396DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
397#else
398DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
399#endif
400extern void print_tracking(struct kmem_cache *s, void *object);
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700401long validate_slab_cache(struct kmem_cache *s);
Marco Elver0d4a0622021-07-14 21:26:34 -0700402static inline bool __slub_debug_enabled(void)
403{
404 return static_branch_unlikely(&slub_debug_enabled);
405}
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700406#else
407static inline void print_tracking(struct kmem_cache *s, void *object)
408{
409}
Marco Elver0d4a0622021-07-14 21:26:34 -0700410static inline bool __slub_debug_enabled(void)
411{
412 return false;
413}
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700414#endif
415
416/*
417 * Returns true if any of the specified slub_debug flags is enabled for the
418 * cache. Use only for flags parsed by setup_slub_debug() as it also enables
419 * the static key.
420 */
421static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
422{
Marco Elver0d4a0622021-07-14 21:26:34 -0700423 if (IS_ENABLED(CONFIG_SLUB_DEBUG))
424 VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
425 if (__slub_debug_enabled())
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700426 return s->flags & flags;
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700427 return false;
428}
429
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700430#ifdef CONFIG_MEMCG_KMEM
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100431/*
432 * slab_objcgs - get the object cgroups vector associated with a slab
433 * @slab: a pointer to the slab struct
434 *
435 * Returns a pointer to the object cgroups vector associated with the slab,
436 * or NULL if no such vector has been associated yet.
437 */
438static inline struct obj_cgroup **slab_objcgs(struct slab *slab)
439{
440 unsigned long memcg_data = READ_ONCE(slab->memcg_data);
441
442 VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS),
443 slab_page(slab));
444 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab));
445
446 return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
447}
448
449int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s,
450 gfp_t gfp, bool new_slab);
Waiman Longfdbcb2a2021-06-28 19:37:19 -0700451void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
452 enum node_stat_item idx, int nr);
Roman Gushchin286e04b2020-08-06 23:20:52 -0700453
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100454static inline void memcg_free_slab_cgroups(struct slab *slab)
Roman Gushchin286e04b2020-08-06 23:20:52 -0700455{
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100456 kfree(slab_objcgs(slab));
457 slab->memcg_data = 0;
Roman Gushchin286e04b2020-08-06 23:20:52 -0700458}
459
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700460static inline size_t obj_full_size(struct kmem_cache *s)
461{
462 /*
463 * For each accounted object there is an extra space which is used
464 * to store obj_cgroup membership. Charge it too.
465 */
466 return s->size + sizeof(struct obj_cgroup *);
467}
468
Roman Gushchinbecaba62020-12-05 22:14:45 -0800469/*
470 * Returns false if the allocation should fail.
471 */
472static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
Muchun Song88f2ef72022-03-22 14:40:56 -0700473 struct list_lru *lru,
Roman Gushchinbecaba62020-12-05 22:14:45 -0800474 struct obj_cgroup **objcgp,
475 size_t objects, gfp_t flags)
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700476{
Roman Gushchin98556092020-08-06 23:21:10 -0700477 struct obj_cgroup *objcg;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700478
Roman Gushchinbecaba62020-12-05 22:14:45 -0800479 if (!memcg_kmem_enabled())
480 return true;
481
482 if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))
483 return true;
484
Roman Gushchin98556092020-08-06 23:21:10 -0700485 objcg = get_obj_cgroup_from_current();
486 if (!objcg)
Roman Gushchinbecaba62020-12-05 22:14:45 -0800487 return true;
Roman Gushchin98556092020-08-06 23:21:10 -0700488
Muchun Song88f2ef72022-03-22 14:40:56 -0700489 if (lru) {
490 int ret;
491 struct mem_cgroup *memcg;
492
493 memcg = get_mem_cgroup_from_objcg(objcg);
494 ret = memcg_list_lru_alloc(memcg, lru, flags);
495 css_put(&memcg->css);
496
497 if (ret)
498 goto out;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700499 }
500
Muchun Song88f2ef72022-03-22 14:40:56 -0700501 if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s)))
502 goto out;
503
Roman Gushchinbecaba62020-12-05 22:14:45 -0800504 *objcgp = objcg;
505 return true;
Muchun Song88f2ef72022-03-22 14:40:56 -0700506out:
507 obj_cgroup_put(objcg);
508 return false;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700509}
510
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700511static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
512 struct obj_cgroup *objcg,
Roman Gushchin10befea2020-08-06 23:21:27 -0700513 gfp_t flags, size_t size,
514 void **p)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700515{
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100516 struct slab *slab;
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700517 unsigned long off;
518 size_t i;
519
Roman Gushchinbecaba62020-12-05 22:14:45 -0800520 if (!memcg_kmem_enabled() || !objcg)
Roman Gushchin10befea2020-08-06 23:21:27 -0700521 return;
522
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700523 for (i = 0; i < size; i++) {
524 if (likely(p[i])) {
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100525 slab = virt_to_slab(p[i]);
Roman Gushchin10befea2020-08-06 23:21:27 -0700526
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100527 if (!slab_objcgs(slab) &&
528 memcg_alloc_slab_cgroups(slab, s, flags,
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800529 false)) {
Roman Gushchin10befea2020-08-06 23:21:27 -0700530 obj_cgroup_uncharge(objcg, obj_full_size(s));
531 continue;
532 }
533
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100534 off = obj_to_index(s, slab, p[i]);
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700535 obj_cgroup_get(objcg);
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100536 slab_objcgs(slab)[off] = objcg;
537 mod_objcg_state(objcg, slab_pgdat(slab),
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700538 cache_vmstat_idx(s), obj_full_size(s));
539 } else {
540 obj_cgroup_uncharge(objcg, obj_full_size(s));
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700541 }
542 }
543 obj_cgroup_put(objcg);
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700544}
545
Muchun Songb77d5b12022-04-29 20:30:44 +0800546static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700547 void **p, int objects)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700548{
Roman Gushchin270c6a72020-12-01 13:58:28 -0800549 struct obj_cgroup **objcgs;
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700550 int i;
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700551
Roman Gushchin10befea2020-08-06 23:21:27 -0700552 if (!memcg_kmem_enabled())
553 return;
554
Muchun Songb77d5b12022-04-29 20:30:44 +0800555 objcgs = slab_objcgs(slab);
556 if (!objcgs)
557 return;
558
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700559 for (i = 0; i < objects; i++) {
Muchun Songb77d5b12022-04-29 20:30:44 +0800560 struct obj_cgroup *objcg;
561 unsigned int off;
Roman Gushchin10befea2020-08-06 23:21:27 -0700562
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100563 off = obj_to_index(s, slab, p[i]);
Roman Gushchin270c6a72020-12-01 13:58:28 -0800564 objcg = objcgs[off];
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700565 if (!objcg)
566 continue;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700567
Roman Gushchin270c6a72020-12-01 13:58:28 -0800568 objcgs[off] = NULL;
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700569 obj_cgroup_uncharge(objcg, obj_full_size(s));
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100570 mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s),
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700571 -obj_full_size(s));
572 obj_cgroup_put(objcg);
573 }
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700574}
575
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700576#else /* CONFIG_MEMCG_KMEM */
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100577static inline struct obj_cgroup **slab_objcgs(struct slab *slab)
578{
579 return NULL;
580}
581
Roman Gushchin98556092020-08-06 23:21:10 -0700582static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
Roman Gushchin4d96ba32019-07-11 20:56:31 -0700583{
584 return NULL;
585}
586
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100587static inline int memcg_alloc_slab_cgroups(struct slab *slab,
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800588 struct kmem_cache *s, gfp_t gfp,
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100589 bool new_slab)
Roman Gushchin286e04b2020-08-06 23:20:52 -0700590{
591 return 0;
592}
593
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100594static inline void memcg_free_slab_cgroups(struct slab *slab)
Roman Gushchin286e04b2020-08-06 23:20:52 -0700595{
596}
597
Roman Gushchinbecaba62020-12-05 22:14:45 -0800598static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
Muchun Song88f2ef72022-03-22 14:40:56 -0700599 struct list_lru *lru,
Roman Gushchinbecaba62020-12-05 22:14:45 -0800600 struct obj_cgroup **objcgp,
601 size_t objects, gfp_t flags)
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700602{
Roman Gushchinbecaba62020-12-05 22:14:45 -0800603 return true;
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700604}
605
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700606static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
607 struct obj_cgroup *objcg,
Roman Gushchin10befea2020-08-06 23:21:27 -0700608 gfp_t flags, size_t size,
609 void **p)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700610{
611}
612
Muchun Songb77d5b12022-04-29 20:30:44 +0800613static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
Bharata B Raod1b2cf62020-10-13 16:53:09 -0700614 void **p, int objects)
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700615{
616}
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700617#endif /* CONFIG_MEMCG_KMEM */
Glauber Costab9ce5ef2012-12-18 14:22:46 -0800618
Vlastimil Babka401fb122021-11-04 11:30:58 +0100619#ifndef CONFIG_SLOB
Kees Cooka64b5372019-07-11 20:53:26 -0700620static inline struct kmem_cache *virt_to_cache(const void *obj)
621{
Matthew Wilcox (Oracle)82c17752021-10-04 14:45:53 +0100622 struct slab *slab;
Kees Cooka64b5372019-07-11 20:53:26 -0700623
Matthew Wilcox (Oracle)82c17752021-10-04 14:45:53 +0100624 slab = virt_to_slab(obj);
625 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n",
Kees Cooka64b5372019-07-11 20:53:26 -0700626 __func__))
627 return NULL;
Matthew Wilcox (Oracle)82c17752021-10-04 14:45:53 +0100628 return slab->slab_cache;
Kees Cooka64b5372019-07-11 20:53:26 -0700629}
630
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100631static __always_inline void account_slab(struct slab *slab, int order,
632 struct kmem_cache *s, gfp_t gfp)
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700633{
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800634 if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100635 memcg_alloc_slab_cgroups(slab, s, gfp, true);
Roman Gushchin2e9bd482021-02-24 12:03:11 -0800636
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100637 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700638 PAGE_SIZE << order);
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700639}
640
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100641static __always_inline void unaccount_slab(struct slab *slab, int order,
642 struct kmem_cache *s)
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700643{
Roman Gushchin10befea2020-08-06 23:21:27 -0700644 if (memcg_kmem_enabled())
Vlastimil Babka4b5f8d9a2021-11-02 22:42:04 +0100645 memcg_free_slab_cgroups(slab);
Roman Gushchin98556092020-08-06 23:21:10 -0700646
Matthew Wilcox (Oracle)b9186532021-10-04 14:45:52 +0100647 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
Roman Gushchinf2fe7b02020-08-06 23:20:59 -0700648 -(PAGE_SIZE << order));
Roman Gushchin6cea1d52019-07-11 20:56:16 -0700649}
650
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700651static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
652{
653 struct kmem_cache *cachep;
654
655 if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700656 !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
657 return s;
658
659 cachep = virt_to_cache(x);
Roman Gushchin10befea2020-08-06 23:21:27 -0700660 if (WARN(cachep && cachep != s,
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700661 "%s: Wrong slab cache. %s but object is from %s\n",
662 __func__, s->name, cachep->name))
663 print_tracking(cachep, x);
664 return cachep;
665}
Hyeonggon Yood6a71642022-08-17 19:18:19 +0900666
667void free_large_kmalloc(struct folio *folio, void *object);
668
Vlastimil Babka401fb122021-11-04 11:30:58 +0100669#endif /* CONFIG_SLOB */
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700670
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700671static inline size_t slab_ksize(const struct kmem_cache *s)
672{
673#ifndef CONFIG_SLUB
674 return s->object_size;
675
676#else /* CONFIG_SLUB */
677# ifdef CONFIG_SLUB_DEBUG
678 /*
679 * Debugging requires use of the padding between object
680 * and whatever may come after it.
681 */
682 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
683 return s->object_size;
684# endif
Alexander Potapenko80a92012016-07-28 15:49:07 -0700685 if (s->flags & SLAB_KASAN)
686 return s->object_size;
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700687 /*
688 * If we have the need to store the freelist pointer
689 * back there or track user information then we can
690 * only use the space before that information.
691 */
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -0800692 if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700693 return s->inuse;
694 /*
695 * Else we can use all the padding etc for the allocation
696 */
697 return s->size;
698#endif
699}
700
701static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
Muchun Song88f2ef72022-03-22 14:40:56 -0700702 struct list_lru *lru,
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700703 struct obj_cgroup **objcgp,
704 size_t size, gfp_t flags)
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700705{
706 flags &= gfp_allowed_mask;
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +0100707
Daniel Vetter95d6c702020-12-14 19:08:34 -0800708 might_alloc(flags);
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700709
Jesper Dangaard Brouerfab99632016-03-15 14:53:38 -0700710 if (should_failslab(s, flags))
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700711 return NULL;
712
Muchun Song88f2ef72022-03-22 14:40:56 -0700713 if (!memcg_slab_pre_alloc_hook(s, lru, objcgp, size, flags))
Roman Gushchinbecaba62020-12-05 22:14:45 -0800714 return NULL;
Vladimir Davydov45264772016-07-26 15:24:21 -0700715
716 return s;
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700717}
718
Roman Gushchin964d4bd2020-08-06 23:20:56 -0700719static inline void slab_post_alloc_hook(struct kmem_cache *s,
Andrey Konovalovda844b72021-04-29 23:00:06 -0700720 struct obj_cgroup *objcg, gfp_t flags,
721 size_t size, void **p, bool init)
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700722{
723 size_t i;
724
725 flags &= gfp_allowed_mask;
Andrey Konovalovda844b72021-04-29 23:00:06 -0700726
727 /*
728 * As memory initialization might be integrated into KASAN,
729 * kasan_slab_alloc and initialization memset must be
730 * kept together to avoid discrepancies in behavior.
731 *
732 * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
733 */
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700734 for (i = 0; i < size; i++) {
Andrey Konovalovda844b72021-04-29 23:00:06 -0700735 p[i] = kasan_slab_alloc(s, p[i], flags, init);
736 if (p[i] && init && !kasan_has_integrated_init())
737 memset(p[i], 0, s->object_size);
Andrey Konovalov53128242019-02-20 22:19:11 -0800738 kmemleak_alloc_recursive(p[i], s->object_size, 1,
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700739 s->flags, flags);
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700740 }
Vladimir Davydov45264772016-07-26 15:24:21 -0700741
Roman Gushchinbecaba62020-12-05 22:14:45 -0800742 memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
Jesper Dangaard Brouer11c7aec2016-03-15 14:53:35 -0700743}
744
Christoph Lameter44c53562014-08-06 16:04:07 -0700745#ifndef CONFIG_SLOB
Christoph Lameterca349562013-01-10 19:14:19 +0000746/*
747 * The slab lists for all objects.
748 */
749struct kmem_cache_node {
750 spinlock_t list_lock;
751
752#ifdef CONFIG_SLAB
753 struct list_head slabs_partial; /* partial list first, better asm code */
754 struct list_head slabs_full;
755 struct list_head slabs_free;
David Rientjesbf00bd32016-12-12 16:41:44 -0800756 unsigned long total_slabs; /* length of all slab lists */
757 unsigned long free_slabs; /* length of free slab list only */
Christoph Lameterca349562013-01-10 19:14:19 +0000758 unsigned long free_objects;
759 unsigned int free_limit;
760 unsigned int colour_next; /* Per-node cache coloring */
761 struct array_cache *shared; /* shared per node */
Joonsoo Kimc8522a32014-08-06 16:04:29 -0700762 struct alien_cache **alien; /* on other nodes */
Christoph Lameterca349562013-01-10 19:14:19 +0000763 unsigned long next_reap; /* updated without locking */
764 int free_touched; /* updated without locking */
765#endif
766
767#ifdef CONFIG_SLUB
768 unsigned long nr_partial;
769 struct list_head partial;
770#ifdef CONFIG_SLUB_DEBUG
771 atomic_long_t nr_slabs;
772 atomic_long_t total_objects;
773 struct list_head full;
774#endif
775#endif
776
777};
Wanpeng Lie25839f2013-07-04 08:33:23 +0800778
Christoph Lameter44c53562014-08-06 16:04:07 -0700779static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
780{
781 return s->node[node];
782}
783
784/*
785 * Iterator over all nodes. The body will be executed for each node that has
786 * a kmem_cache_node structure allocated (which is true for all online nodes)
787 */
788#define for_each_kmem_cache_node(__s, __node, __n) \
Mikulas Patocka91635822014-10-09 15:26:20 -0700789 for (__node = 0; __node < nr_node_ids; __node++) \
790 if ((__n = get_node(__s, __node)))
Christoph Lameter44c53562014-08-06 16:04:07 -0700791
792#endif
793
Yang Shi852d8be2017-11-15 17:32:07 -0800794#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
795void dump_unreclaimable_slab(void);
796#else
797static inline void dump_unreclaimable_slab(void)
798{
799}
800#endif
801
Alexander Potapenko55834c52016-05-20 16:59:11 -0700802void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);
803
Thomas Garnier7c00fce2016-07-26 15:21:56 -0700804#ifdef CONFIG_SLAB_FREELIST_RANDOM
805int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
806 gfp_t gfp);
807void cache_random_seq_destroy(struct kmem_cache *cachep);
808#else
809static inline int cache_random_seq_create(struct kmem_cache *cachep,
810 unsigned int count, gfp_t gfp)
811{
812 return 0;
813}
814static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
815#endif /* CONFIG_SLAB_FREELIST_RANDOM */
816
Alexander Potapenko64713842019-07-11 20:59:19 -0700817static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c)
818{
Kees Cook51cba1e2021-04-01 16:23:43 -0700819 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
820 &init_on_alloc)) {
Alexander Potapenko64713842019-07-11 20:59:19 -0700821 if (c->ctor)
822 return false;
823 if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))
824 return flags & __GFP_ZERO;
825 return true;
826 }
827 return flags & __GFP_ZERO;
828}
829
830static inline bool slab_want_init_on_free(struct kmem_cache *c)
831{
Kees Cook51cba1e2021-04-01 16:23:43 -0700832 if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
833 &init_on_free))
Alexander Potapenko64713842019-07-11 20:59:19 -0700834 return !(c->ctor ||
835 (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)));
836 return false;
837}
838
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -0700839#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
840void debugfs_slab_release(struct kmem_cache *);
841#else
842static inline void debugfs_slab_release(struct kmem_cache *s) { }
843#endif
844
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -0800845#ifdef CONFIG_PRINTK
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800846#define KS_ADDRS_COUNT 16
847struct kmem_obj_info {
848 void *kp_ptr;
Matthew Wilcox (Oracle)72132302021-10-04 14:45:55 +0100849 struct slab *kp_slab;
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800850 void *kp_objp;
851 unsigned long kp_data_offset;
852 struct kmem_cache *kp_slab_cache;
853 void *kp_ret;
854 void *kp_stack[KS_ADDRS_COUNT];
Maninder Singhe548eaa2021-03-16 16:07:11 +0530855 void *kp_free_stack[KS_ADDRS_COUNT];
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800856};
Marco Elver2dfe63e2022-04-14 19:13:40 -0700857void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -0800858#endif
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -0800859
Matthew Wilcox (Oracle)0b3eb0912021-10-04 14:45:56 +0100860#ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR
861void __check_heap_object(const void *ptr, unsigned long n,
862 const struct slab *slab, bool to_user);
863#else
864static inline
865void __check_heap_object(const void *ptr, unsigned long n,
866 const struct slab *slab, bool to_user)
867{
868}
869#endif
870
Andrey Ryabinin5240ab42014-08-06 16:04:14 -0700871#endif /* MM_SLAB_H */