blob: f77d8cd79ef7f7d6855df7ae9bb4659001cee387 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Christoph Lameter81819f02007-05-06 14:49:36 -07002/*
3 * SLUB: A slab allocator that limits cache line use instead of queuing
4 * objects in per cpu and per node lists.
5 *
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -07006 * The allocator synchronizes using per slab locks or atomic operations
Christoph Lameter881db7f2011-06-01 12:25:53 -05007 * and only uses a centralized lock to manage a pool of partial slabs.
Christoph Lameter81819f02007-05-06 14:49:36 -07008 *
Christoph Lametercde53532008-07-04 09:59:22 -07009 * (C) 2007 SGI, Christoph Lameter
Christoph Lameter881db7f2011-06-01 12:25:53 -050010 * (C) 2011 Linux Foundation, Christoph Lameter
Christoph Lameter81819f02007-05-06 14:49:36 -070011 */
12
13#include <linux/mm.h>
Nick Piggin1eb5ac62009-05-05 19:13:44 +100014#include <linux/swap.h> /* struct reclaim_state */
Christoph Lameter81819f02007-05-06 14:49:36 -070015#include <linux/module.h>
16#include <linux/bit_spinlock.h>
17#include <linux/interrupt.h>
Andrew Morton1b3865d2021-06-15 18:23:39 -070018#include <linux/swab.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070019#include <linux/bitops.h>
20#include <linux/slab.h>
Christoph Lameter97d06602012-07-06 15:25:11 -050021#include "slab.h"
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +040022#include <linux/proc_fs.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070023#include <linux/seq_file.h>
Andrey Ryabinina79316c2015-02-13 14:39:38 -080024#include <linux/kasan.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070025#include <linux/cpu.h>
26#include <linux/cpuset.h>
27#include <linux/mempolicy.h>
28#include <linux/ctype.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070029#include <linux/debugobjects.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070030#include <linux/kallsyms.h>
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -080031#include <linux/kfence.h>
Yasunori Gotob9049e22007-10-21 16:41:37 -070032#include <linux/memory.h>
Roman Zippelf8bd2252008-05-01 04:34:31 -070033#include <linux/math64.h>
Akinobu Mita773ff602008-12-23 19:37:01 +090034#include <linux/fault-inject.h>
Pekka Enbergbfa71452011-07-07 22:47:01 +030035#include <linux/stacktrace.h>
Christoph Lameter4de900b2012-01-30 15:53:51 -060036#include <linux/prefetch.h>
Glauber Costa2633d7a2012-12-18 14:22:34 -080037#include <linux/memcontrol.h>
Kees Cook2482ddec2017-09-06 16:19:18 -070038#include <linux/random.h>
Oliver Glitta1f9f78b2021-06-28 19:34:33 -070039#include <kunit/test.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070040
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -070041#include <linux/debugfs.h>
Richard Kennedy4a923792010-10-21 10:29:19 +010042#include <trace/events/kmem.h>
43
Mel Gorman072bb0a2012-07-31 16:43:58 -070044#include "internal.h"
45
Christoph Lameter81819f02007-05-06 14:49:36 -070046/*
47 * Lock order:
Christoph Lameter18004c52012-07-06 15:25:12 -050048 * 1. slab_mutex (Global Mutex)
Christoph Lameter881db7f2011-06-01 12:25:53 -050049 * 2. node->list_lock
50 * 3. slab_lock(page) (Only on some arches and for debugging)
Christoph Lameter81819f02007-05-06 14:49:36 -070051 *
Christoph Lameter18004c52012-07-06 15:25:12 -050052 * slab_mutex
Christoph Lameter881db7f2011-06-01 12:25:53 -050053 *
Christoph Lameter18004c52012-07-06 15:25:12 -050054 * The role of the slab_mutex is to protect the list of all the slabs
Christoph Lameter881db7f2011-06-01 12:25:53 -050055 * and to synchronize major metadata changes to slab cache structures.
56 *
57 * The slab_lock is only used for debugging and on arches that do not
Matthew Wilcoxb7ccc7f2018-06-07 17:08:46 -070058 * have the ability to do a cmpxchg_double. It only protects:
Christoph Lameter881db7f2011-06-01 12:25:53 -050059 * A. page->freelist -> List of object free in a page
Matthew Wilcoxb7ccc7f2018-06-07 17:08:46 -070060 * B. page->inuse -> Number of objects in use
61 * C. page->objects -> Number of objects in page
62 * D. page->frozen -> frozen state
Christoph Lameter881db7f2011-06-01 12:25:53 -050063 *
64 * If a slab is frozen then it is exempt from list management. It is not
Liu Xiang632b2ef2019-05-13 17:16:28 -070065 * on any list except per cpu partial list. The processor that froze the
66 * slab is the one who can perform list operations on the page. Other
67 * processors may put objects onto the freelist but the processor that
68 * froze the slab is the only one that can retrieve the objects from the
69 * page's freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -070070 *
71 * The list_lock protects the partial and full list on each node and
72 * the partial slab counter. If taken then no new slabs may be added or
73 * removed from the lists nor make the number of partial slabs be modified.
74 * (Note that the total number of slabs is an atomic value that may be
75 * modified without taking the list lock).
76 *
77 * The list_lock is a centralized lock and thus we avoid taking it as
78 * much as possible. As long as SLUB does not have to handle partial
79 * slabs, operations can continue without any centralized lock. F.e.
80 * allocating a long series of objects that fill up slabs does not require
81 * the list lock.
Christoph Lameter81819f02007-05-06 14:49:36 -070082 * Interrupts are disabled during allocation and deallocation in order to
83 * make the slab allocator safe to use in the context of an irq. In addition
84 * interrupts are disabled to ensure that the processor does not change
85 * while handling per_cpu slabs, due to kernel preemption.
86 *
87 * SLUB assigns one slab for allocation to each processor.
88 * Allocations only occur from these slabs called cpu slabs.
89 *
Christoph Lameter672bba32007-05-09 02:32:39 -070090 * Slabs with free elements are kept on a partial list and during regular
91 * operations no list for full slabs is used. If an object in a full slab is
Christoph Lameter81819f02007-05-06 14:49:36 -070092 * freed then the slab will show up again on the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -070093 * We track full slabs for debugging purposes though because otherwise we
94 * cannot scan all objects.
Christoph Lameter81819f02007-05-06 14:49:36 -070095 *
96 * Slabs are freed when they become empty. Teardown and setup is
97 * minimal so we rely on the page allocators per cpu caches for
98 * fast frees and allocs.
99 *
Yu Zhaoaed68142019-11-30 17:49:34 -0800100 * page->frozen The slab is frozen and exempt from list processing.
Christoph Lameter4b6f0752007-05-16 22:10:53 -0700101 * This means that the slab is dedicated to a purpose
102 * such as satisfying allocations for a specific
103 * processor. Objects may be freed in the slab while
104 * it is frozen but slab_free will then skip the usual
105 * list operations. It is up to the processor holding
106 * the slab to integrate the slab into the slab lists
107 * when the slab is no longer needed.
108 *
109 * One use of this flag is to mark slabs that are
110 * used for allocations. Then such a slab becomes a cpu
111 * slab. The cpu slab may be equipped with an additional
Christoph Lameterdfb4f092007-10-16 01:26:05 -0700112 * freelist that allows lockless access to
Christoph Lameter894b8782007-05-10 03:15:16 -0700113 * free objects in addition to the regular freelist
114 * that requires the slab lock.
Christoph Lameter81819f02007-05-06 14:49:36 -0700115 *
Yu Zhaoaed68142019-11-30 17:49:34 -0800116 * SLAB_DEBUG_FLAGS Slab requires special handling due to debug
Christoph Lameter81819f02007-05-06 14:49:36 -0700117 * options set. This moves slab handling out of
Christoph Lameter894b8782007-05-10 03:15:16 -0700118 * the fast path and disables lockless freelists.
Christoph Lameter81819f02007-05-06 14:49:36 -0700119 */
120
Vlastimil Babkaca0cab62020-08-06 23:18:51 -0700121#ifdef CONFIG_SLUB_DEBUG
122#ifdef CONFIG_SLUB_DEBUG_ON
123DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
124#else
125DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
126#endif
Stephen Boyd79270292021-06-28 19:34:52 -0700127#endif /* CONFIG_SLUB_DEBUG */
Vlastimil Babkaca0cab62020-08-06 23:18:51 -0700128
Vlastimil Babka59052e82020-08-06 23:18:55 -0700129static inline bool kmem_cache_debug(struct kmem_cache *s)
130{
131 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
Christoph Lameteraf537b02010-07-09 14:07:14 -0500132}
Christoph Lameter5577bd82007-05-16 22:10:56 -0700133
Geert Uytterhoeven117d54d2016-08-04 15:31:55 -0700134void *fixup_red_left(struct kmem_cache *s, void *p)
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700135{
Vlastimil Babka59052e82020-08-06 23:18:55 -0700136 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700137 p += s->red_left_pad;
138
139 return p;
140}
141
Joonsoo Kim345c9052013-06-19 14:05:52 +0900142static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
143{
144#ifdef CONFIG_SLUB_CPU_PARTIAL
145 return !kmem_cache_debug(s);
146#else
147 return false;
148#endif
149}
150
Christoph Lameter81819f02007-05-06 14:49:36 -0700151/*
152 * Issues still to be resolved:
153 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700154 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
155 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700156 * - Variable sizing of the per node arrays
157 */
158
Christoph Lameterb789ef52011-06-01 12:25:49 -0500159/* Enable to log cmpxchg failures */
160#undef SLUB_DEBUG_CMPXCHG
161
Christoph Lameter81819f02007-05-06 14:49:36 -0700162/*
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -0700163 * Minimum number of partial slabs. These will be left on the partial
Christoph Lameter2086d262007-05-06 14:49:46 -0700164 * lists even if they are empty. kmem_cache_shrink may reclaim them.
165 */
Christoph Lameter76be8952007-12-21 14:37:37 -0800166#define MIN_PARTIAL 5
Christoph Lametere95eed52007-05-06 14:49:44 -0700167
Christoph Lameter2086d262007-05-06 14:49:46 -0700168/*
169 * Maximum number of desirable partial slabs.
170 * The existence of more partial slabs makes kmem_cache_shrink
Zhi Yong Wu721ae222013-11-08 20:47:37 +0800171 * sort the partial list by the number of objects in use.
Christoph Lameter2086d262007-05-06 14:49:46 -0700172 */
173#define MAX_PARTIAL 10
174
Laura Abbottbecfda62016-03-15 14:55:06 -0700175#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
Christoph Lameter81819f02007-05-06 14:49:36 -0700176 SLAB_POISON | SLAB_STORE_USER)
Christoph Lameter672bba32007-05-09 02:32:39 -0700177
Christoph Lameter81819f02007-05-06 14:49:36 -0700178/*
Laura Abbott149daaf2016-03-15 14:55:09 -0700179 * These debug flags cannot use CMPXCHG because there might be consistency
180 * issues when checking or reading debug information
181 */
182#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
183 SLAB_TRACE)
184
185
186/*
David Rientjes3de47212009-07-27 18:30:35 -0700187 * Debugging flags that require metadata to be stored in the slab. These get
188 * disabled when slub_debug=O is used and a cache's min order increases with
189 * metadata.
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700190 */
David Rientjes3de47212009-07-27 18:30:35 -0700191#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700192
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400193#define OO_SHIFT 16
194#define OO_MASK ((1 << OO_SHIFT) - 1)
Christoph Lameter50d5c412011-06-01 12:25:45 -0500195#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400196
Christoph Lameter81819f02007-05-06 14:49:36 -0700197/* Internal SLUB flags */
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800198/* Poison object */
Alexey Dobriyan4fd0b462017-11-15 17:32:21 -0800199#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800200/* Use cmpxchg_double */
Alexey Dobriyan4fd0b462017-11-15 17:32:21 -0800201#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
Christoph Lameter81819f02007-05-06 14:49:36 -0700202
Christoph Lameter02cbc872007-05-09 02:32:43 -0700203/*
204 * Tracking user of a slab.
205 */
Ben Greeard6543e32011-07-07 11:36:36 -0700206#define TRACK_ADDRS_COUNT 16
Christoph Lameter02cbc872007-05-09 02:32:43 -0700207struct track {
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300208 unsigned long addr; /* Called from address */
Linus Torvaldsae14c632021-07-17 13:27:00 -0700209#ifdef CONFIG_STACKTRACE
210 unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
Ben Greeard6543e32011-07-07 11:36:36 -0700211#endif
Christoph Lameter02cbc872007-05-09 02:32:43 -0700212 int cpu; /* Was running on cpu */
213 int pid; /* Pid context */
214 unsigned long when; /* When did the operation occur */
215};
216
217enum track_item { TRACK_ALLOC, TRACK_FREE };
218
Christoph Lameterab4d5ed2010-10-05 13:57:26 -0500219#ifdef CONFIG_SYSFS
Christoph Lameter81819f02007-05-06 14:49:36 -0700220static int sysfs_slab_add(struct kmem_cache *);
221static int sysfs_slab_alias(struct kmem_cache *, const char *);
Christoph Lameter81819f02007-05-06 14:49:36 -0700222#else
Christoph Lameter0c710012007-07-17 04:03:24 -0700223static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
224static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
225 { return 0; }
Christoph Lameter81819f02007-05-06 14:49:36 -0700226#endif
227
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -0700228#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
229static void debugfs_slab_add(struct kmem_cache *);
230#else
231static inline void debugfs_slab_add(struct kmem_cache *s) { }
232#endif
233
Christoph Lameter4fdccdf2011-03-22 13:35:00 -0500234static inline void stat(const struct kmem_cache *s, enum stat_item si)
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800235{
236#ifdef CONFIG_SLUB_STATS
Christoph Lameter88da03a2014-04-07 15:39:42 -0700237 /*
238 * The rmw is racy on a preemptible kernel but this is acceptable, so
239 * avoid this_cpu_add()'s irq-disable overhead.
240 */
241 raw_cpu_inc(s->cpu_slab->stat[si]);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800242#endif
243}
244
Vlastimil Babka7e1fa932021-02-24 12:01:12 -0800245/*
246 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
247 * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily
248 * differ during memory hotplug/hotremove operations.
249 * Protected by slab_mutex.
250 */
251static nodemask_t slab_nodes;
252
Christoph Lameter81819f02007-05-06 14:49:36 -0700253/********************************************************************
254 * Core slab cache functions
255 *******************************************************************/
256
Kees Cook2482ddec2017-09-06 16:19:18 -0700257/*
258 * Returns freelist pointer (ptr). With hardening, this is obfuscated
259 * with an XOR of the address where the pointer is held and a per-cache
260 * random number.
261 */
262static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
263 unsigned long ptr_addr)
264{
265#ifdef CONFIG_SLAB_FREELIST_HARDENED
Andrey Konovalovd36a63a2019-02-20 22:19:32 -0800266 /*
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800267 * When CONFIG_KASAN_SW/HW_TAGS is enabled, ptr_addr might be tagged.
Andrey Konovalovd36a63a2019-02-20 22:19:32 -0800268 * Normally, this doesn't cause any issues, as both set_freepointer()
269 * and get_freepointer() are called with a pointer with the same tag.
270 * However, there are some issues with CONFIG_SLUB_DEBUG code. For
271 * example, when __free_slub() iterates over objects in a cache, it
272 * passes untagged pointers to check_object(). check_object() in turns
273 * calls get_freepointer() with an untagged pointer, which causes the
274 * freepointer to be restored incorrectly.
275 */
276 return (void *)((unsigned long)ptr ^ s->random ^
Kees Cook1ad53d92020-04-01 21:04:23 -0700277 swab((unsigned long)kasan_reset_tag((void *)ptr_addr)));
Kees Cook2482ddec2017-09-06 16:19:18 -0700278#else
279 return ptr;
280#endif
281}
282
283/* Returns the freelist pointer recorded at location ptr_addr. */
284static inline void *freelist_dereference(const struct kmem_cache *s,
285 void *ptr_addr)
286{
287 return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
288 (unsigned long)ptr_addr);
289}
290
Christoph Lameter7656c722007-05-09 02:32:40 -0700291static inline void *get_freepointer(struct kmem_cache *s, void *object)
292{
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800293 object = kasan_reset_tag(object);
Kees Cook2482ddec2017-09-06 16:19:18 -0700294 return freelist_dereference(s, object + s->offset);
Christoph Lameter7656c722007-05-09 02:32:40 -0700295}
296
Eric Dumazet0ad95002011-12-16 16:25:34 +0100297static void prefetch_freepointer(const struct kmem_cache *s, void *object)
298{
Vlastimil Babka0882ff92018-08-17 15:44:44 -0700299 prefetch(object + s->offset);
Eric Dumazet0ad95002011-12-16 16:25:34 +0100300}
301
Christoph Lameter1393d9a2011-05-16 15:26:08 -0500302static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
303{
Kees Cook2482ddec2017-09-06 16:19:18 -0700304 unsigned long freepointer_addr;
Christoph Lameter1393d9a2011-05-16 15:26:08 -0500305 void *p;
306
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800307 if (!debug_pagealloc_enabled_static())
Joonsoo Kim922d5662016-03-17 14:17:53 -0700308 return get_freepointer(s, object);
309
Alexander Potapenkof70b0042021-05-22 17:41:56 -0700310 object = kasan_reset_tag(object);
Kees Cook2482ddec2017-09-06 16:19:18 -0700311 freepointer_addr = (unsigned long)object + s->offset;
Christoph Hellwigfe557312020-06-17 09:37:53 +0200312 copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));
Kees Cook2482ddec2017-09-06 16:19:18 -0700313 return freelist_ptr(s, p, freepointer_addr);
Christoph Lameter1393d9a2011-05-16 15:26:08 -0500314}
315
Christoph Lameter7656c722007-05-09 02:32:40 -0700316static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
317{
Kees Cook2482ddec2017-09-06 16:19:18 -0700318 unsigned long freeptr_addr = (unsigned long)object + s->offset;
319
Alexander Popovce6fa912017-09-06 16:19:22 -0700320#ifdef CONFIG_SLAB_FREELIST_HARDENED
321 BUG_ON(object == fp); /* naive detection of double free or corruption */
322#endif
323
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800324 freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
Kees Cook2482ddec2017-09-06 16:19:18 -0700325 *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
Christoph Lameter7656c722007-05-09 02:32:40 -0700326}
327
328/* Loop over all objects in a slab */
Christoph Lameter224a88b2008-04-14 19:11:31 +0300329#define for_each_object(__p, __s, __addr, __objects) \
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700330 for (__p = fixup_red_left(__s, __addr); \
331 __p < (__addr) + (__objects) * (__s)->size; \
332 __p += (__s)->size)
Christoph Lameter7656c722007-05-09 02:32:40 -0700333
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700334static inline unsigned int order_objects(unsigned int order, unsigned int size)
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800335{
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700336 return ((unsigned int)PAGE_SIZE << order) / size;
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800337}
338
Alexey Dobriyan19af27a2018-04-05 16:21:39 -0700339static inline struct kmem_cache_order_objects oo_make(unsigned int order,
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700340 unsigned int size)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300341{
342 struct kmem_cache_order_objects x = {
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700343 (order << OO_SHIFT) + order_objects(order, size)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300344 };
345
346 return x;
347}
348
Alexey Dobriyan19af27a2018-04-05 16:21:39 -0700349static inline unsigned int oo_order(struct kmem_cache_order_objects x)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300350{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400351 return x.x >> OO_SHIFT;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300352}
353
Alexey Dobriyan19af27a2018-04-05 16:21:39 -0700354static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300355{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400356 return x.x & OO_MASK;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300357}
358
Christoph Lameter881db7f2011-06-01 12:25:53 -0500359/*
360 * Per slab locking using the pagelock
361 */
362static __always_inline void slab_lock(struct page *page)
363{
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -0800364 VM_BUG_ON_PAGE(PageTail(page), page);
Christoph Lameter881db7f2011-06-01 12:25:53 -0500365 bit_spin_lock(PG_locked, &page->flags);
366}
367
368static __always_inline void slab_unlock(struct page *page)
369{
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -0800370 VM_BUG_ON_PAGE(PageTail(page), page);
Christoph Lameter881db7f2011-06-01 12:25:53 -0500371 __bit_spin_unlock(PG_locked, &page->flags);
372}
373
Christoph Lameter1d071712011-07-14 12:49:12 -0500374/* Interrupts must be disabled (for the fallback code to work right) */
375static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
Christoph Lameterb789ef52011-06-01 12:25:49 -0500376 void *freelist_old, unsigned long counters_old,
377 void *freelist_new, unsigned long counters_new,
378 const char *n)
379{
Christoph Lameter1d071712011-07-14 12:49:12 -0500380 VM_BUG_ON(!irqs_disabled());
Heiko Carstens25654092012-01-12 17:17:33 -0800381#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
382 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Christoph Lameterb789ef52011-06-01 12:25:49 -0500383 if (s->flags & __CMPXCHG_DOUBLE) {
Jan Beulichcdcd6292012-01-02 17:02:18 +0000384 if (cmpxchg_double(&page->freelist, &page->counters,
Dan Carpenter0aa9a132014-08-06 16:04:48 -0700385 freelist_old, counters_old,
386 freelist_new, counters_new))
Joe Perches6f6528a2015-04-14 15:44:31 -0700387 return true;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500388 } else
389#endif
390 {
Christoph Lameter881db7f2011-06-01 12:25:53 -0500391 slab_lock(page);
Chen Gangd0e0ac92013-07-15 09:05:29 +0800392 if (page->freelist == freelist_old &&
393 page->counters == counters_old) {
Christoph Lameterb789ef52011-06-01 12:25:49 -0500394 page->freelist = freelist_new;
Matthew Wilcox7d27a042018-06-07 17:08:31 -0700395 page->counters = counters_new;
Christoph Lameter881db7f2011-06-01 12:25:53 -0500396 slab_unlock(page);
Joe Perches6f6528a2015-04-14 15:44:31 -0700397 return true;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500398 }
Christoph Lameter881db7f2011-06-01 12:25:53 -0500399 slab_unlock(page);
Christoph Lameterb789ef52011-06-01 12:25:49 -0500400 }
401
402 cpu_relax();
403 stat(s, CMPXCHG_DOUBLE_FAIL);
404
405#ifdef SLUB_DEBUG_CMPXCHG
Fabian Frederickf9f58282014-06-04 16:06:34 -0700406 pr_info("%s %s: cmpxchg double redo ", n, s->name);
Christoph Lameterb789ef52011-06-01 12:25:49 -0500407#endif
408
Joe Perches6f6528a2015-04-14 15:44:31 -0700409 return false;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500410}
411
Christoph Lameter1d071712011-07-14 12:49:12 -0500412static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
413 void *freelist_old, unsigned long counters_old,
414 void *freelist_new, unsigned long counters_new,
415 const char *n)
416{
Heiko Carstens25654092012-01-12 17:17:33 -0800417#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
418 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Christoph Lameter1d071712011-07-14 12:49:12 -0500419 if (s->flags & __CMPXCHG_DOUBLE) {
Jan Beulichcdcd6292012-01-02 17:02:18 +0000420 if (cmpxchg_double(&page->freelist, &page->counters,
Dan Carpenter0aa9a132014-08-06 16:04:48 -0700421 freelist_old, counters_old,
422 freelist_new, counters_new))
Joe Perches6f6528a2015-04-14 15:44:31 -0700423 return true;
Christoph Lameter1d071712011-07-14 12:49:12 -0500424 } else
425#endif
426 {
427 unsigned long flags;
428
429 local_irq_save(flags);
430 slab_lock(page);
Chen Gangd0e0ac92013-07-15 09:05:29 +0800431 if (page->freelist == freelist_old &&
432 page->counters == counters_old) {
Christoph Lameter1d071712011-07-14 12:49:12 -0500433 page->freelist = freelist_new;
Matthew Wilcox7d27a042018-06-07 17:08:31 -0700434 page->counters = counters_new;
Christoph Lameter1d071712011-07-14 12:49:12 -0500435 slab_unlock(page);
436 local_irq_restore(flags);
Joe Perches6f6528a2015-04-14 15:44:31 -0700437 return true;
Christoph Lameter1d071712011-07-14 12:49:12 -0500438 }
439 slab_unlock(page);
440 local_irq_restore(flags);
441 }
442
443 cpu_relax();
444 stat(s, CMPXCHG_DOUBLE_FAIL);
445
446#ifdef SLUB_DEBUG_CMPXCHG
Fabian Frederickf9f58282014-06-04 16:06:34 -0700447 pr_info("%s %s: cmpxchg double redo ", n, s->name);
Christoph Lameter1d071712011-07-14 12:49:12 -0500448#endif
449
Joe Perches6f6528a2015-04-14 15:44:31 -0700450 return false;
Christoph Lameter1d071712011-07-14 12:49:12 -0500451}
452
Christoph Lameter41ecc552007-05-09 02:32:44 -0700453#ifdef CONFIG_SLUB_DEBUG
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800454static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
455static DEFINE_SPINLOCK(object_map_lock);
456
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700457#if IS_ENABLED(CONFIG_KUNIT)
458static bool slab_add_kunit_errors(void)
459{
460 struct kunit_resource *resource;
461
462 if (likely(!current->kunit_test))
463 return false;
464
465 resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
466 if (!resource)
467 return false;
468
469 (*(int *)resource->data)++;
470 kunit_put_resource(resource);
471 return true;
472}
473#else
474static inline bool slab_add_kunit_errors(void) { return false; }
475#endif
476
Christoph Lameter41ecc552007-05-09 02:32:44 -0700477/*
Christoph Lameter5f80b132011-04-15 14:48:13 -0500478 * Determine a map of object in use on a page.
479 *
Christoph Lameter881db7f2011-06-01 12:25:53 -0500480 * Node listlock must be held to guarantee that the page does
Christoph Lameter5f80b132011-04-15 14:48:13 -0500481 * not vanish from under us.
482 */
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800483static unsigned long *get_map(struct kmem_cache *s, struct page *page)
Jules Irenge31364c22020-04-06 20:08:15 -0700484 __acquires(&object_map_lock)
Christoph Lameter5f80b132011-04-15 14:48:13 -0500485{
486 void *p;
487 void *addr = page_address(page);
488
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800489 VM_BUG_ON(!irqs_disabled());
490
491 spin_lock(&object_map_lock);
492
493 bitmap_zero(object_map, page->objects);
494
Christoph Lameter5f80b132011-04-15 14:48:13 -0500495 for (p = page->freelist; p; p = get_freepointer(s, p))
Roman Gushchin4138fdf2020-08-06 23:20:42 -0700496 set_bit(__obj_to_index(s, addr, p), object_map);
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800497
498 return object_map;
499}
500
Jules Irenge81aba9e2020-04-06 20:08:18 -0700501static void put_map(unsigned long *map) __releases(&object_map_lock)
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800502{
503 VM_BUG_ON(map != object_map);
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800504 spin_unlock(&object_map_lock);
Christoph Lameter5f80b132011-04-15 14:48:13 -0500505}
506
Alexey Dobriyan870b1fb2018-04-05 16:21:43 -0700507static inline unsigned int size_from_object(struct kmem_cache *s)
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700508{
509 if (s->flags & SLAB_RED_ZONE)
510 return s->size - s->red_left_pad;
511
512 return s->size;
513}
514
515static inline void *restore_red_left(struct kmem_cache *s, void *p)
516{
517 if (s->flags & SLAB_RED_ZONE)
518 p -= s->red_left_pad;
519
520 return p;
521}
522
Christoph Lameter41ecc552007-05-09 02:32:44 -0700523/*
524 * Debug settings:
525 */
Andrey Ryabinin89d3c872015-11-05 18:51:23 -0800526#if defined(CONFIG_SLUB_DEBUG_ON)
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800527static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700528#else
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800529static slab_flags_t slub_debug;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700530#endif
Christoph Lameter41ecc552007-05-09 02:32:44 -0700531
Vlastimil Babkae17f1df2020-08-06 23:18:35 -0700532static char *slub_debug_string;
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700533static int disable_higher_order_debug;
Christoph Lameter41ecc552007-05-09 02:32:44 -0700534
Christoph Lameter7656c722007-05-09 02:32:40 -0700535/*
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800536 * slub is about to manipulate internal object metadata. This memory lies
537 * outside the range of the allocated object, so accessing it would normally
538 * be reported by kasan as a bounds error. metadata_access_enable() is used
539 * to tell kasan that these accesses are OK.
540 */
541static inline void metadata_access_enable(void)
542{
543 kasan_disable_current();
544}
545
546static inline void metadata_access_disable(void)
547{
548 kasan_enable_current();
549}
550
551/*
Christoph Lameter81819f02007-05-06 14:49:36 -0700552 * Object debugging
553 */
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700554
555/* Verify that a pointer has an address that is valid within a slab page */
556static inline int check_valid_pointer(struct kmem_cache *s,
557 struct page *page, void *object)
558{
559 void *base;
560
561 if (!object)
562 return 1;
563
564 base = page_address(page);
Qian Cai338cfaa2019-02-20 22:19:36 -0800565 object = kasan_reset_tag(object);
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700566 object = restore_red_left(s, object);
567 if (object < base || object >= base + page->objects * s->size ||
568 (object - base) % s->size) {
569 return 0;
570 }
571
572 return 1;
573}
574
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800575static void print_section(char *level, char *text, u8 *addr,
576 unsigned int length)
Christoph Lameter81819f02007-05-06 14:49:36 -0700577{
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800578 metadata_access_enable();
Kuan-Ying Lee340caf12021-08-13 16:54:27 -0700579 print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
580 16, 1, kasan_reset_tag((void *)addr), length, 1);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800581 metadata_access_disable();
Christoph Lameter81819f02007-05-06 14:49:36 -0700582}
583
Waiman Longcbfc35a2020-05-07 18:36:06 -0700584/*
585 * See comment in calculate_sizes().
586 */
587static inline bool freeptr_outside_object(struct kmem_cache *s)
588{
589 return s->offset >= s->inuse;
590}
591
592/*
593 * Return offset of the end of info block which is inuse + free pointer if
594 * not overlapping with object.
595 */
596static inline unsigned int get_info_end(struct kmem_cache *s)
597{
598 if (freeptr_outside_object(s))
599 return s->inuse + sizeof(void *);
600 else
601 return s->inuse;
602}
603
Christoph Lameter81819f02007-05-06 14:49:36 -0700604static struct track *get_track(struct kmem_cache *s, void *object,
605 enum track_item alloc)
606{
607 struct track *p;
608
Waiman Longcbfc35a2020-05-07 18:36:06 -0700609 p = object + get_info_end(s);
Christoph Lameter81819f02007-05-06 14:49:36 -0700610
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800611 return kasan_reset_tag(p + alloc);
Christoph Lameter81819f02007-05-06 14:49:36 -0700612}
613
614static void set_track(struct kmem_cache *s, void *object,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300615 enum track_item alloc, unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -0700616{
Akinobu Mita1a00df42009-03-07 00:36:21 +0900617 struct track *p = get_track(s, object, alloc);
Christoph Lameter81819f02007-05-06 14:49:36 -0700618
Christoph Lameter81819f02007-05-06 14:49:36 -0700619 if (addr) {
Linus Torvaldsae14c632021-07-17 13:27:00 -0700620#ifdef CONFIG_STACKTRACE
621 unsigned int nr_entries;
622
623 metadata_access_enable();
624 nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
625 TRACK_ADDRS_COUNT, 3);
626 metadata_access_disable();
627
628 if (nr_entries < TRACK_ADDRS_COUNT)
629 p->addrs[nr_entries] = 0;
Ben Greeard6543e32011-07-07 11:36:36 -0700630#endif
Christoph Lameter81819f02007-05-06 14:49:36 -0700631 p->addr = addr;
632 p->cpu = smp_processor_id();
Alexey Dobriyan88e4ccf2008-06-23 02:58:37 +0400633 p->pid = current->pid;
Christoph Lameter81819f02007-05-06 14:49:36 -0700634 p->when = jiffies;
Thomas Gleixnerb8ca7ff2019-04-10 12:28:05 +0200635 } else {
Christoph Lameter81819f02007-05-06 14:49:36 -0700636 memset(p, 0, sizeof(struct track));
Thomas Gleixnerb8ca7ff2019-04-10 12:28:05 +0200637 }
Christoph Lameter81819f02007-05-06 14:49:36 -0700638}
639
Christoph Lameter81819f02007-05-06 14:49:36 -0700640static void init_tracking(struct kmem_cache *s, void *object)
641{
Christoph Lameter24922682007-07-17 04:03:18 -0700642 if (!(s->flags & SLAB_STORE_USER))
643 return;
644
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300645 set_track(s, object, TRACK_FREE, 0UL);
646 set_track(s, object, TRACK_ALLOC, 0UL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700647}
648
Chintan Pandya86609d32018-04-05 16:20:15 -0700649static void print_track(const char *s, struct track *t, unsigned long pr_time)
Christoph Lameter81819f02007-05-06 14:49:36 -0700650{
651 if (!t->addr)
652 return;
653
Yafang Shao96b94ab2021-03-19 18:12:45 +0800654 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
Chintan Pandya86609d32018-04-05 16:20:15 -0700655 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
Linus Torvaldsae14c632021-07-17 13:27:00 -0700656#ifdef CONFIG_STACKTRACE
Ben Greeard6543e32011-07-07 11:36:36 -0700657 {
Linus Torvaldsae14c632021-07-17 13:27:00 -0700658 int i;
659 for (i = 0; i < TRACK_ADDRS_COUNT; i++)
660 if (t->addrs[i])
661 pr_err("\t%pS\n", (void *)t->addrs[i]);
662 else
663 break;
Ben Greeard6543e32011-07-07 11:36:36 -0700664 }
665#endif
Christoph Lameter81819f02007-05-06 14:49:36 -0700666}
667
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700668void print_tracking(struct kmem_cache *s, void *object)
Christoph Lameter24922682007-07-17 04:03:18 -0700669{
Chintan Pandya86609d32018-04-05 16:20:15 -0700670 unsigned long pr_time = jiffies;
Christoph Lameter24922682007-07-17 04:03:18 -0700671 if (!(s->flags & SLAB_STORE_USER))
672 return;
673
Chintan Pandya86609d32018-04-05 16:20:15 -0700674 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
675 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
Christoph Lameter24922682007-07-17 04:03:18 -0700676}
677
678static void print_page_info(struct page *page)
679{
Yafang Shao96b94ab2021-03-19 18:12:45 +0800680 pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%#lx(%pGp)\n",
Yafang Shao4a8ef192021-03-19 18:12:44 +0800681 page, page->objects, page->inuse, page->freelist,
682 page->flags, &page->flags);
Christoph Lameter24922682007-07-17 04:03:18 -0700683
684}
685
686static void slab_bug(struct kmem_cache *s, char *fmt, ...)
687{
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700688 struct va_format vaf;
Christoph Lameter24922682007-07-17 04:03:18 -0700689 va_list args;
Christoph Lameter24922682007-07-17 04:03:18 -0700690
691 va_start(args, fmt);
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700692 vaf.fmt = fmt;
693 vaf.va = &args;
Fabian Frederickf9f58282014-06-04 16:06:34 -0700694 pr_err("=============================================================================\n");
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700695 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
Fabian Frederickf9f58282014-06-04 16:06:34 -0700696 pr_err("-----------------------------------------------------------------------------\n\n");
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700697 va_end(args);
Christoph Lameter24922682007-07-17 04:03:18 -0700698}
699
Joe Perches582d1212021-06-28 19:34:49 -0700700__printf(2, 3)
Christoph Lameter24922682007-07-17 04:03:18 -0700701static void slab_fix(struct kmem_cache *s, char *fmt, ...)
702{
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700703 struct va_format vaf;
Christoph Lameter24922682007-07-17 04:03:18 -0700704 va_list args;
Christoph Lameter24922682007-07-17 04:03:18 -0700705
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700706 if (slab_add_kunit_errors())
707 return;
708
Christoph Lameter24922682007-07-17 04:03:18 -0700709 va_start(args, fmt);
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700710 vaf.fmt = fmt;
711 vaf.va = &args;
712 pr_err("FIX %s: %pV\n", s->name, &vaf);
Christoph Lameter24922682007-07-17 04:03:18 -0700713 va_end(args);
Christoph Lameter24922682007-07-17 04:03:18 -0700714}
715
Dongli Zhang52f23472020-06-01 21:45:47 -0700716static bool freelist_corrupted(struct kmem_cache *s, struct page *page,
Eugeniu Roscadc07a722020-09-04 16:35:30 -0700717 void **freelist, void *nextfree)
Dongli Zhang52f23472020-06-01 21:45:47 -0700718{
719 if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
Eugeniu Roscadc07a722020-09-04 16:35:30 -0700720 !check_valid_pointer(s, page, nextfree) && freelist) {
721 object_err(s, page, *freelist, "Freechain corrupt");
722 *freelist = NULL;
Dongli Zhang52f23472020-06-01 21:45:47 -0700723 slab_fix(s, "Isolate corrupted freechain");
724 return true;
725 }
726
727 return false;
728}
729
Christoph Lameter24922682007-07-17 04:03:18 -0700730static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
Christoph Lameter81819f02007-05-06 14:49:36 -0700731{
732 unsigned int off; /* Offset of last byte */
Christoph Lametera973e9d2008-03-01 13:40:44 -0800733 u8 *addr = page_address(page);
Christoph Lameter24922682007-07-17 04:03:18 -0700734
735 print_tracking(s, p);
736
737 print_page_info(page);
738
Yafang Shao96b94ab2021-03-19 18:12:45 +0800739 pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
Fabian Frederickf9f58282014-06-04 16:06:34 -0700740 p, p - addr, get_freepointer(s, p));
Christoph Lameter24922682007-07-17 04:03:18 -0700741
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700742 if (s->flags & SLAB_RED_ZONE)
Kees Cook8669dba2021-06-15 18:23:19 -0700743 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800744 s->red_left_pad);
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700745 else if (p > addr + 16)
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800746 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
Christoph Lameter24922682007-07-17 04:03:18 -0700747
Kees Cook8669dba2021-06-15 18:23:19 -0700748 print_section(KERN_ERR, "Object ", p,
Alexey Dobriyan1b473f22018-04-05 16:21:17 -0700749 min_t(unsigned int, s->object_size, PAGE_SIZE));
Christoph Lameter81819f02007-05-06 14:49:36 -0700750 if (s->flags & SLAB_RED_ZONE)
Kees Cook8669dba2021-06-15 18:23:19 -0700751 print_section(KERN_ERR, "Redzone ", p + s->object_size,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500752 s->inuse - s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -0700753
Waiman Longcbfc35a2020-05-07 18:36:06 -0700754 off = get_info_end(s);
Christoph Lameter81819f02007-05-06 14:49:36 -0700755
Christoph Lameter24922682007-07-17 04:03:18 -0700756 if (s->flags & SLAB_STORE_USER)
Christoph Lameter81819f02007-05-06 14:49:36 -0700757 off += 2 * sizeof(struct track);
Christoph Lameter81819f02007-05-06 14:49:36 -0700758
Alexander Potapenko80a92012016-07-28 15:49:07 -0700759 off += kasan_metadata_size(s);
760
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700761 if (off != size_from_object(s))
Christoph Lameter81819f02007-05-06 14:49:36 -0700762 /* Beginning of the filler is the free pointer */
Kees Cook8669dba2021-06-15 18:23:19 -0700763 print_section(KERN_ERR, "Padding ", p + off,
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800764 size_from_object(s) - off);
Christoph Lameter24922682007-07-17 04:03:18 -0700765
766 dump_stack();
Christoph Lameter81819f02007-05-06 14:49:36 -0700767}
768
Andrey Ryabinin75c66de2015-02-13 14:39:35 -0800769void object_err(struct kmem_cache *s, struct page *page,
Christoph Lameter81819f02007-05-06 14:49:36 -0700770 u8 *object, char *reason)
771{
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700772 if (slab_add_kunit_errors())
773 return;
774
Christoph Lameter3dc50632008-04-23 12:28:01 -0700775 slab_bug(s, "%s", reason);
Christoph Lameter24922682007-07-17 04:03:18 -0700776 print_trailer(s, page, object);
Georgi Djakov65ebdee2021-06-28 19:34:58 -0700777 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter81819f02007-05-06 14:49:36 -0700778}
779
Mathieu Malaterrea38965b2018-06-07 17:05:17 -0700780static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
Chen Gangd0e0ac92013-07-15 09:05:29 +0800781 const char *fmt, ...)
Christoph Lameter81819f02007-05-06 14:49:36 -0700782{
783 va_list args;
784 char buf[100];
785
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700786 if (slab_add_kunit_errors())
787 return;
788
Christoph Lameter24922682007-07-17 04:03:18 -0700789 va_start(args, fmt);
790 vsnprintf(buf, sizeof(buf), fmt, args);
Christoph Lameter81819f02007-05-06 14:49:36 -0700791 va_end(args);
Christoph Lameter3dc50632008-04-23 12:28:01 -0700792 slab_bug(s, "%s", buf);
Christoph Lameter24922682007-07-17 04:03:18 -0700793 print_page_info(page);
Christoph Lameter81819f02007-05-06 14:49:36 -0700794 dump_stack();
Georgi Djakov65ebdee2021-06-28 19:34:58 -0700795 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter81819f02007-05-06 14:49:36 -0700796}
797
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500798static void init_object(struct kmem_cache *s, void *object, u8 val)
Christoph Lameter81819f02007-05-06 14:49:36 -0700799{
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800800 u8 *p = kasan_reset_tag(object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700801
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700802 if (s->flags & SLAB_RED_ZONE)
803 memset(p - s->red_left_pad, val, s->red_left_pad);
804
Christoph Lameter81819f02007-05-06 14:49:36 -0700805 if (s->flags & __OBJECT_POISON) {
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500806 memset(p, POISON_FREE, s->object_size - 1);
807 p[s->object_size - 1] = POISON_END;
Christoph Lameter81819f02007-05-06 14:49:36 -0700808 }
809
810 if (s->flags & SLAB_RED_ZONE)
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500811 memset(p + s->object_size, val, s->inuse - s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -0700812}
813
Christoph Lameter24922682007-07-17 04:03:18 -0700814static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
815 void *from, void *to)
816{
Joe Perches582d1212021-06-28 19:34:49 -0700817 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
Christoph Lameter24922682007-07-17 04:03:18 -0700818 memset(from, data, to - from);
819}
820
821static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
822 u8 *object, char *what,
Pekka Enberg06428782008-01-07 23:20:27 -0800823 u8 *start, unsigned int value, unsigned int bytes)
Christoph Lameter24922682007-07-17 04:03:18 -0700824{
825 u8 *fault;
826 u8 *end;
Miles Chene1b70dd2019-11-30 17:49:31 -0800827 u8 *addr = page_address(page);
Christoph Lameter24922682007-07-17 04:03:18 -0700828
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800829 metadata_access_enable();
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800830 fault = memchr_inv(kasan_reset_tag(start), value, bytes);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800831 metadata_access_disable();
Christoph Lameter24922682007-07-17 04:03:18 -0700832 if (!fault)
833 return 1;
834
835 end = start + bytes;
836 while (end > fault && end[-1] == value)
837 end--;
838
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700839 if (slab_add_kunit_errors())
840 goto skip_bug_print;
841
Christoph Lameter24922682007-07-17 04:03:18 -0700842 slab_bug(s, "%s overwritten", what);
Yafang Shao96b94ab2021-03-19 18:12:45 +0800843 pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
Miles Chene1b70dd2019-11-30 17:49:31 -0800844 fault, end - 1, fault - addr,
845 fault[0], value);
Christoph Lameter24922682007-07-17 04:03:18 -0700846 print_trailer(s, page, object);
Georgi Djakov65ebdee2021-06-28 19:34:58 -0700847 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter24922682007-07-17 04:03:18 -0700848
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700849skip_bug_print:
Christoph Lameter24922682007-07-17 04:03:18 -0700850 restore_bytes(s, what, value, fault, end);
851 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700852}
853
Christoph Lameter81819f02007-05-06 14:49:36 -0700854/*
855 * Object layout:
856 *
857 * object address
858 * Bytes of the object to be managed.
859 * If the freepointer may overlay the object then the free
Waiman Longcbfc35a2020-05-07 18:36:06 -0700860 * pointer is at the middle of the object.
Christoph Lameter672bba32007-05-09 02:32:39 -0700861 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700862 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
863 * 0xa5 (POISON_END)
864 *
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500865 * object + s->object_size
Christoph Lameter81819f02007-05-06 14:49:36 -0700866 * Padding to reach word boundary. This is also used for Redzoning.
Christoph Lameter672bba32007-05-09 02:32:39 -0700867 * Padding is extended by another word if Redzoning is enabled and
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500868 * object_size == inuse.
Christoph Lameter672bba32007-05-09 02:32:39 -0700869 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700870 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
871 * 0xcc (RED_ACTIVE) for objects in use.
872 *
873 * object + s->inuse
Christoph Lameter672bba32007-05-09 02:32:39 -0700874 * Meta data starts here.
875 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700876 * A. Free pointer (if we cannot overwrite object on free)
877 * B. Tracking data for SLAB_STORE_USER
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -0700878 * C. Padding to reach required alignment boundary or at minimum
Christoph Lameter6446faa2008-02-15 23:45:26 -0800879 * one word if debugging is on to be able to detect writes
Christoph Lameter672bba32007-05-09 02:32:39 -0700880 * before the word boundary.
881 *
882 * Padding is done using 0x5a (POISON_INUSE)
Christoph Lameter81819f02007-05-06 14:49:36 -0700883 *
884 * object + s->size
Christoph Lameter672bba32007-05-09 02:32:39 -0700885 * Nothing is used beyond s->size.
Christoph Lameter81819f02007-05-06 14:49:36 -0700886 *
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500887 * If slabcaches are merged then the object_size and inuse boundaries are mostly
Christoph Lameter672bba32007-05-09 02:32:39 -0700888 * ignored. And therefore no slab options that rely on these boundaries
Christoph Lameter81819f02007-05-06 14:49:36 -0700889 * may be used with merged slabcaches.
890 */
891
Christoph Lameter81819f02007-05-06 14:49:36 -0700892static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
893{
Waiman Longcbfc35a2020-05-07 18:36:06 -0700894 unsigned long off = get_info_end(s); /* The end of info */
Christoph Lameter81819f02007-05-06 14:49:36 -0700895
896 if (s->flags & SLAB_STORE_USER)
897 /* We also have user information there */
898 off += 2 * sizeof(struct track);
899
Alexander Potapenko80a92012016-07-28 15:49:07 -0700900 off += kasan_metadata_size(s);
901
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700902 if (size_from_object(s) == off)
Christoph Lameter81819f02007-05-06 14:49:36 -0700903 return 1;
904
Christoph Lameter24922682007-07-17 04:03:18 -0700905 return check_bytes_and_report(s, page, p, "Object padding",
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700906 p + off, POISON_INUSE, size_from_object(s) - off);
Christoph Lameter81819f02007-05-06 14:49:36 -0700907}
908
Christoph Lameter39b26462008-04-14 19:11:30 +0300909/* Check the pad bytes at the end of a slab page */
Christoph Lameter81819f02007-05-06 14:49:36 -0700910static int slab_pad_check(struct kmem_cache *s, struct page *page)
911{
Christoph Lameter24922682007-07-17 04:03:18 -0700912 u8 *start;
913 u8 *fault;
914 u8 *end;
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -0800915 u8 *pad;
Christoph Lameter24922682007-07-17 04:03:18 -0700916 int length;
917 int remainder;
Christoph Lameter81819f02007-05-06 14:49:36 -0700918
919 if (!(s->flags & SLAB_POISON))
920 return 1;
921
Christoph Lametera973e9d2008-03-01 13:40:44 -0800922 start = page_address(page);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -0700923 length = page_size(page);
Christoph Lameter39b26462008-04-14 19:11:30 +0300924 end = start + length;
925 remainder = length % s->size;
Christoph Lameter81819f02007-05-06 14:49:36 -0700926 if (!remainder)
927 return 1;
928
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -0800929 pad = end - remainder;
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800930 metadata_access_enable();
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800931 fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800932 metadata_access_disable();
Christoph Lameter24922682007-07-17 04:03:18 -0700933 if (!fault)
934 return 1;
935 while (end > fault && end[-1] == POISON_INUSE)
936 end--;
937
Miles Chene1b70dd2019-11-30 17:49:31 -0800938 slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu",
939 fault, end - 1, fault - start);
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -0800940 print_section(KERN_ERR, "Padding ", pad, remainder);
Christoph Lameter24922682007-07-17 04:03:18 -0700941
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -0800942 restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
Christoph Lameter24922682007-07-17 04:03:18 -0700943 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700944}
945
946static int check_object(struct kmem_cache *s, struct page *page,
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500947 void *object, u8 val)
Christoph Lameter81819f02007-05-06 14:49:36 -0700948{
949 u8 *p = object;
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500950 u8 *endobject = object + s->object_size;
Christoph Lameter81819f02007-05-06 14:49:36 -0700951
952 if (s->flags & SLAB_RED_ZONE) {
Kees Cook8669dba2021-06-15 18:23:19 -0700953 if (!check_bytes_and_report(s, page, object, "Left Redzone",
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700954 object - s->red_left_pad, val, s->red_left_pad))
955 return 0;
956
Kees Cook8669dba2021-06-15 18:23:19 -0700957 if (!check_bytes_and_report(s, page, object, "Right Redzone",
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500958 endobject, val, s->inuse - s->object_size))
Christoph Lameter81819f02007-05-06 14:49:36 -0700959 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700960 } else {
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500961 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
Ingo Molnar3adbefe2008-02-05 17:57:39 -0800962 check_bytes_and_report(s, page, p, "Alignment padding",
Chen Gangd0e0ac92013-07-15 09:05:29 +0800963 endobject, POISON_INUSE,
964 s->inuse - s->object_size);
Ingo Molnar3adbefe2008-02-05 17:57:39 -0800965 }
Christoph Lameter81819f02007-05-06 14:49:36 -0700966 }
967
968 if (s->flags & SLAB_POISON) {
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500969 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
Christoph Lameter24922682007-07-17 04:03:18 -0700970 (!check_bytes_and_report(s, page, p, "Poison", p,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500971 POISON_FREE, s->object_size - 1) ||
Kees Cook8669dba2021-06-15 18:23:19 -0700972 !check_bytes_and_report(s, page, p, "End Poison",
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500973 p + s->object_size - 1, POISON_END, 1)))
Christoph Lameter81819f02007-05-06 14:49:36 -0700974 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700975 /*
976 * check_pad_bytes cleans up on its own.
977 */
978 check_pad_bytes(s, page, p);
979 }
980
Waiman Longcbfc35a2020-05-07 18:36:06 -0700981 if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
Christoph Lameter81819f02007-05-06 14:49:36 -0700982 /*
983 * Object and freepointer overlap. Cannot check
984 * freepointer while object is allocated.
985 */
986 return 1;
987
988 /* Check free pointer validity */
989 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
990 object_err(s, page, p, "Freepointer corrupt");
991 /*
Nick Andrew9f6c708e2008-12-05 14:08:08 +1100992 * No choice but to zap it and thus lose the remainder
Christoph Lameter81819f02007-05-06 14:49:36 -0700993 * of the free objects in this slab. May cause
Christoph Lameter672bba32007-05-09 02:32:39 -0700994 * another error because the object count is now wrong.
Christoph Lameter81819f02007-05-06 14:49:36 -0700995 */
Christoph Lametera973e9d2008-03-01 13:40:44 -0800996 set_freepointer(s, p, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700997 return 0;
998 }
999 return 1;
1000}
1001
1002static int check_slab(struct kmem_cache *s, struct page *page)
1003{
Christoph Lameter39b26462008-04-14 19:11:30 +03001004 int maxobj;
1005
Christoph Lameter81819f02007-05-06 14:49:36 -07001006 VM_BUG_ON(!irqs_disabled());
1007
1008 if (!PageSlab(page)) {
Christoph Lameter24922682007-07-17 04:03:18 -07001009 slab_err(s, page, "Not a valid slab page");
Christoph Lameter81819f02007-05-06 14:49:36 -07001010 return 0;
1011 }
Christoph Lameter39b26462008-04-14 19:11:30 +03001012
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07001013 maxobj = order_objects(compound_order(page), s->size);
Christoph Lameter39b26462008-04-14 19:11:30 +03001014 if (page->objects > maxobj) {
1015 slab_err(s, page, "objects %u > max %u",
Andrey Ryabininf6edde92014-12-10 15:42:22 -08001016 page->objects, maxobj);
Christoph Lameter39b26462008-04-14 19:11:30 +03001017 return 0;
1018 }
1019 if (page->inuse > page->objects) {
Christoph Lameter24922682007-07-17 04:03:18 -07001020 slab_err(s, page, "inuse %u > max %u",
Andrey Ryabininf6edde92014-12-10 15:42:22 -08001021 page->inuse, page->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07001022 return 0;
1023 }
1024 /* Slab_pad_check fixes things up after itself */
1025 slab_pad_check(s, page);
1026 return 1;
1027}
1028
1029/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001030 * Determine if a certain object on a page is on the freelist. Must hold the
1031 * slab lock to guarantee that the chains are in a consistent state.
Christoph Lameter81819f02007-05-06 14:49:36 -07001032 */
1033static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
1034{
1035 int nr = 0;
Christoph Lameter881db7f2011-06-01 12:25:53 -05001036 void *fp;
Christoph Lameter81819f02007-05-06 14:49:36 -07001037 void *object = NULL;
Andrey Ryabininf6edde92014-12-10 15:42:22 -08001038 int max_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07001039
Christoph Lameter881db7f2011-06-01 12:25:53 -05001040 fp = page->freelist;
Christoph Lameter39b26462008-04-14 19:11:30 +03001041 while (fp && nr <= page->objects) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001042 if (fp == search)
1043 return 1;
1044 if (!check_valid_pointer(s, page, fp)) {
1045 if (object) {
1046 object_err(s, page, object,
1047 "Freechain corrupt");
Christoph Lametera973e9d2008-03-01 13:40:44 -08001048 set_freepointer(s, object, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07001049 } else {
Christoph Lameter24922682007-07-17 04:03:18 -07001050 slab_err(s, page, "Freepointer corrupt");
Christoph Lametera973e9d2008-03-01 13:40:44 -08001051 page->freelist = NULL;
Christoph Lameter39b26462008-04-14 19:11:30 +03001052 page->inuse = page->objects;
Christoph Lameter24922682007-07-17 04:03:18 -07001053 slab_fix(s, "Freelist cleared");
Christoph Lameter81819f02007-05-06 14:49:36 -07001054 return 0;
1055 }
1056 break;
1057 }
1058 object = fp;
1059 fp = get_freepointer(s, object);
1060 nr++;
1061 }
1062
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07001063 max_objects = order_objects(compound_order(page), s->size);
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +04001064 if (max_objects > MAX_OBJS_PER_PAGE)
1065 max_objects = MAX_OBJS_PER_PAGE;
Christoph Lameter224a88b2008-04-14 19:11:31 +03001066
1067 if (page->objects != max_objects) {
Joe Perches756a0252016-03-17 14:19:47 -07001068 slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
1069 page->objects, max_objects);
Christoph Lameter224a88b2008-04-14 19:11:31 +03001070 page->objects = max_objects;
Joe Perches582d1212021-06-28 19:34:49 -07001071 slab_fix(s, "Number of objects adjusted");
Christoph Lameter224a88b2008-04-14 19:11:31 +03001072 }
Christoph Lameter39b26462008-04-14 19:11:30 +03001073 if (page->inuse != page->objects - nr) {
Joe Perches756a0252016-03-17 14:19:47 -07001074 slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
1075 page->inuse, page->objects - nr);
Christoph Lameter39b26462008-04-14 19:11:30 +03001076 page->inuse = page->objects - nr;
Joe Perches582d1212021-06-28 19:34:49 -07001077 slab_fix(s, "Object count adjusted");
Christoph Lameter81819f02007-05-06 14:49:36 -07001078 }
1079 return search == NULL;
1080}
1081
Christoph Lameter0121c6192008-04-29 16:11:12 -07001082static void trace(struct kmem_cache *s, struct page *page, void *object,
1083 int alloc)
Christoph Lameter3ec09742007-05-16 22:11:00 -07001084{
1085 if (s->flags & SLAB_TRACE) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07001086 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
Christoph Lameter3ec09742007-05-16 22:11:00 -07001087 s->name,
1088 alloc ? "alloc" : "free",
1089 object, page->inuse,
1090 page->freelist);
1091
1092 if (!alloc)
Daniel Thompsonaa2efd52017-01-24 15:18:02 -08001093 print_section(KERN_INFO, "Object ", (void *)object,
Chen Gangd0e0ac92013-07-15 09:05:29 +08001094 s->object_size);
Christoph Lameter3ec09742007-05-16 22:11:00 -07001095
1096 dump_stack();
1097 }
1098}
1099
Christoph Lameter643b1132007-05-06 14:49:42 -07001100/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001101 * Tracking of fully allocated slabs for debugging purposes.
Christoph Lameter643b1132007-05-06 14:49:42 -07001102 */
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001103static void add_full(struct kmem_cache *s,
1104 struct kmem_cache_node *n, struct page *page)
Christoph Lameter643b1132007-05-06 14:49:42 -07001105{
Christoph Lameter643b1132007-05-06 14:49:42 -07001106 if (!(s->flags & SLAB_STORE_USER))
1107 return;
1108
David Rientjes255d0882014-02-10 14:25:39 -08001109 lockdep_assert_held(&n->list_lock);
Tobin C. Harding916ac052019-05-13 17:16:12 -07001110 list_add(&page->slab_list, &n->full);
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001111}
Christoph Lameter643b1132007-05-06 14:49:42 -07001112
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001113static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001114{
1115 if (!(s->flags & SLAB_STORE_USER))
1116 return;
1117
David Rientjes255d0882014-02-10 14:25:39 -08001118 lockdep_assert_held(&n->list_lock);
Tobin C. Harding916ac052019-05-13 17:16:12 -07001119 list_del(&page->slab_list);
Christoph Lameter643b1132007-05-06 14:49:42 -07001120}
1121
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001122/* Tracking of the number of slabs for debugging purposes */
1123static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1124{
1125 struct kmem_cache_node *n = get_node(s, node);
1126
1127 return atomic_long_read(&n->nr_slabs);
1128}
1129
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04001130static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1131{
1132 return atomic_long_read(&n->nr_slabs);
1133}
1134
Christoph Lameter205ab992008-04-14 19:11:40 +03001135static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001136{
1137 struct kmem_cache_node *n = get_node(s, node);
1138
1139 /*
1140 * May be called early in order to allocate a slab for the
1141 * kmem_cache_node structure. Solve the chicken-egg
1142 * dilemma by deferring the increment of the count during
1143 * bootstrap (see early_kmem_cache_node_alloc).
1144 */
Joonsoo Kim338b2642013-01-21 17:01:27 +09001145 if (likely(n)) {
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001146 atomic_long_inc(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +03001147 atomic_long_add(objects, &n->total_objects);
1148 }
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001149}
Christoph Lameter205ab992008-04-14 19:11:40 +03001150static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001151{
1152 struct kmem_cache_node *n = get_node(s, node);
1153
1154 atomic_long_dec(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +03001155 atomic_long_sub(objects, &n->total_objects);
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001156}
1157
1158/* Object debug checks for alloc/free paths */
Christoph Lameter3ec09742007-05-16 22:11:00 -07001159static void setup_object_debug(struct kmem_cache *s, struct page *page,
1160 void *object)
1161{
Vlastimil Babka8fc8d662020-08-06 23:18:58 -07001162 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
Christoph Lameter3ec09742007-05-16 22:11:00 -07001163 return;
1164
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001165 init_object(s, object, SLUB_RED_INACTIVE);
Christoph Lameter3ec09742007-05-16 22:11:00 -07001166 init_tracking(s, object);
1167}
1168
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07001169static
1170void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr)
Andrey Konovalova7101222019-02-20 22:19:23 -08001171{
Vlastimil Babka8fc8d662020-08-06 23:18:58 -07001172 if (!kmem_cache_debug_flags(s, SLAB_POISON))
Andrey Konovalova7101222019-02-20 22:19:23 -08001173 return;
1174
1175 metadata_access_enable();
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -08001176 memset(kasan_reset_tag(addr), POISON_INUSE, page_size(page));
Andrey Konovalova7101222019-02-20 22:19:23 -08001177 metadata_access_disable();
1178}
1179
Laura Abbottbecfda62016-03-15 14:55:06 -07001180static inline int alloc_consistency_checks(struct kmem_cache *s,
Qian Cai278d7752019-03-05 15:42:10 -08001181 struct page *page, void *object)
Christoph Lameter81819f02007-05-06 14:49:36 -07001182{
1183 if (!check_slab(s, page))
Laura Abbottbecfda62016-03-15 14:55:06 -07001184 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07001185
Christoph Lameter81819f02007-05-06 14:49:36 -07001186 if (!check_valid_pointer(s, page, object)) {
1187 object_err(s, page, object, "Freelist Pointer check fails");
Laura Abbottbecfda62016-03-15 14:55:06 -07001188 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07001189 }
1190
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001191 if (!check_object(s, page, object, SLUB_RED_INACTIVE))
Laura Abbottbecfda62016-03-15 14:55:06 -07001192 return 0;
1193
1194 return 1;
1195}
1196
1197static noinline int alloc_debug_processing(struct kmem_cache *s,
1198 struct page *page,
1199 void *object, unsigned long addr)
1200{
1201 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
Qian Cai278d7752019-03-05 15:42:10 -08001202 if (!alloc_consistency_checks(s, page, object))
Laura Abbottbecfda62016-03-15 14:55:06 -07001203 goto bad;
1204 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001205
Christoph Lameter3ec09742007-05-16 22:11:00 -07001206 /* Success perform special debug activities for allocs */
1207 if (s->flags & SLAB_STORE_USER)
1208 set_track(s, object, TRACK_ALLOC, addr);
1209 trace(s, page, object, 1);
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001210 init_object(s, object, SLUB_RED_ACTIVE);
Christoph Lameter81819f02007-05-06 14:49:36 -07001211 return 1;
Christoph Lameter3ec09742007-05-16 22:11:00 -07001212
Christoph Lameter81819f02007-05-06 14:49:36 -07001213bad:
1214 if (PageSlab(page)) {
1215 /*
1216 * If this is a slab page then lets do the best we can
1217 * to avoid issues in the future. Marking all objects
Christoph Lameter672bba32007-05-09 02:32:39 -07001218 * as used avoids touching the remaining objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07001219 */
Christoph Lameter24922682007-07-17 04:03:18 -07001220 slab_fix(s, "Marking all objects used");
Christoph Lameter39b26462008-04-14 19:11:30 +03001221 page->inuse = page->objects;
Christoph Lametera973e9d2008-03-01 13:40:44 -08001222 page->freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001223 }
1224 return 0;
1225}
1226
Laura Abbottbecfda62016-03-15 14:55:06 -07001227static inline int free_consistency_checks(struct kmem_cache *s,
1228 struct page *page, void *object, unsigned long addr)
1229{
1230 if (!check_valid_pointer(s, page, object)) {
1231 slab_err(s, page, "Invalid object pointer 0x%p", object);
1232 return 0;
1233 }
1234
1235 if (on_freelist(s, page, object)) {
1236 object_err(s, page, object, "Object already free");
1237 return 0;
1238 }
1239
1240 if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1241 return 0;
1242
1243 if (unlikely(s != page->slab_cache)) {
1244 if (!PageSlab(page)) {
Joe Perches756a0252016-03-17 14:19:47 -07001245 slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
1246 object);
Laura Abbottbecfda62016-03-15 14:55:06 -07001247 } else if (!page->slab_cache) {
1248 pr_err("SLUB <none>: no slab for object 0x%p.\n",
1249 object);
1250 dump_stack();
1251 } else
1252 object_err(s, page, object,
1253 "page slab pointer corrupt.");
1254 return 0;
1255 }
1256 return 1;
1257}
1258
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001259/* Supports checking bulk free of a constructed freelist */
Laura Abbott282acb42016-03-15 14:54:59 -07001260static noinline int free_debug_processing(
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001261 struct kmem_cache *s, struct page *page,
1262 void *head, void *tail, int bulk_cnt,
Laura Abbott282acb42016-03-15 14:54:59 -07001263 unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -07001264{
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001265 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001266 void *object = head;
1267 int cnt = 0;
Kees Cook3f649ab2020-06-03 13:09:38 -07001268 unsigned long flags;
Laura Abbott804aa132016-03-15 14:55:02 -07001269 int ret = 0;
Christoph Lameter5c2e4bb2011-06-01 12:25:54 -05001270
Laura Abbott282acb42016-03-15 14:54:59 -07001271 spin_lock_irqsave(&n->list_lock, flags);
Christoph Lameter881db7f2011-06-01 12:25:53 -05001272 slab_lock(page);
1273
Laura Abbottbecfda62016-03-15 14:55:06 -07001274 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1275 if (!check_slab(s, page))
1276 goto out;
1277 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001278
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001279next_object:
1280 cnt++;
1281
Laura Abbottbecfda62016-03-15 14:55:06 -07001282 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1283 if (!free_consistency_checks(s, page, object, addr))
1284 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07001285 }
Christoph Lameter3ec09742007-05-16 22:11:00 -07001286
Christoph Lameter3ec09742007-05-16 22:11:00 -07001287 if (s->flags & SLAB_STORE_USER)
1288 set_track(s, object, TRACK_FREE, addr);
1289 trace(s, page, object, 0);
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001290 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001291 init_object(s, object, SLUB_RED_INACTIVE);
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001292
1293 /* Reached end of constructed freelist yet? */
1294 if (object != tail) {
1295 object = get_freepointer(s, object);
1296 goto next_object;
1297 }
Laura Abbott804aa132016-03-15 14:55:02 -07001298 ret = 1;
1299
Christoph Lameter5c2e4bb2011-06-01 12:25:54 -05001300out:
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001301 if (cnt != bulk_cnt)
1302 slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
1303 bulk_cnt, cnt);
1304
Christoph Lameter881db7f2011-06-01 12:25:53 -05001305 slab_unlock(page);
Laura Abbott282acb42016-03-15 14:54:59 -07001306 spin_unlock_irqrestore(&n->list_lock, flags);
Laura Abbott804aa132016-03-15 14:55:02 -07001307 if (!ret)
1308 slab_fix(s, "Object at 0x%p not freed", object);
1309 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07001310}
1311
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001312/*
1313 * Parse a block of slub_debug options. Blocks are delimited by ';'
1314 *
1315 * @str: start of block
1316 * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
1317 * @slabs: return start of list of slabs, or NULL when there's no list
1318 * @init: assume this is initial parsing and not per-kmem-create parsing
1319 *
1320 * returns the start of next block if there's any, or NULL
1321 */
1322static char *
1323parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
1324{
1325 bool higher_order_disable = false;
1326
1327 /* Skip any completely empty blocks */
1328 while (*str && *str == ';')
1329 str++;
1330
1331 if (*str == ',') {
1332 /*
1333 * No options but restriction on slabs. This means full
1334 * debugging for slabs matching a pattern.
1335 */
1336 *flags = DEBUG_DEFAULT_FLAGS;
1337 goto check_slabs;
1338 }
1339 *flags = 0;
1340
1341 /* Determine which debug features should be switched on */
1342 for (; *str && *str != ',' && *str != ';'; str++) {
1343 switch (tolower(*str)) {
1344 case '-':
1345 *flags = 0;
1346 break;
1347 case 'f':
1348 *flags |= SLAB_CONSISTENCY_CHECKS;
1349 break;
1350 case 'z':
1351 *flags |= SLAB_RED_ZONE;
1352 break;
1353 case 'p':
1354 *flags |= SLAB_POISON;
1355 break;
1356 case 'u':
1357 *flags |= SLAB_STORE_USER;
1358 break;
1359 case 't':
1360 *flags |= SLAB_TRACE;
1361 break;
1362 case 'a':
1363 *flags |= SLAB_FAILSLAB;
1364 break;
1365 case 'o':
1366 /*
1367 * Avoid enabling debugging on caches if its minimum
1368 * order would increase as a result.
1369 */
1370 higher_order_disable = true;
1371 break;
1372 default:
1373 if (init)
1374 pr_err("slub_debug option '%c' unknown. skipped\n", *str);
1375 }
1376 }
1377check_slabs:
1378 if (*str == ',')
1379 *slabs = ++str;
1380 else
1381 *slabs = NULL;
1382
1383 /* Skip over the slab list */
1384 while (*str && *str != ';')
1385 str++;
1386
1387 /* Skip any completely empty blocks */
1388 while (*str && *str == ';')
1389 str++;
1390
1391 if (init && higher_order_disable)
1392 disable_higher_order_debug = 1;
1393
1394 if (*str)
1395 return str;
1396 else
1397 return NULL;
1398}
1399
Christoph Lameter41ecc552007-05-09 02:32:44 -07001400static int __init setup_slub_debug(char *str)
1401{
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001402 slab_flags_t flags;
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001403 slab_flags_t global_flags;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001404 char *saved_str;
1405 char *slab_list;
1406 bool global_slub_debug_changed = false;
1407 bool slab_list_specified = false;
1408
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001409 global_flags = DEBUG_DEFAULT_FLAGS;
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001410 if (*str++ != '=' || !*str)
1411 /*
1412 * No options specified. Switch on full debugging.
1413 */
1414 goto out;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001415
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001416 saved_str = str;
1417 while (str) {
1418 str = parse_slub_debug_flags(str, &flags, &slab_list, true);
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001419
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001420 if (!slab_list) {
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001421 global_flags = flags;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001422 global_slub_debug_changed = true;
1423 } else {
1424 slab_list_specified = true;
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001425 }
1426 }
1427
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001428 /*
1429 * For backwards compatibility, a single list of flags with list of
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001430 * slabs means debugging is only changed for those slabs, so the global
1431 * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
1432 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001433 * long as there is no option specifying flags without a slab list.
1434 */
1435 if (slab_list_specified) {
1436 if (!global_slub_debug_changed)
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001437 global_flags = slub_debug;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001438 slub_debug_string = saved_str;
1439 }
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001440out:
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001441 slub_debug = global_flags;
Vlastimil Babkaca0cab62020-08-06 23:18:51 -07001442 if (slub_debug != 0 || slub_debug_string)
1443 static_branch_enable(&slub_debug_enabled);
Stephen Boyd02ac47d2021-06-28 19:34:43 -07001444 else
1445 static_branch_disable(&slub_debug_enabled);
Alexander Potapenko64713842019-07-11 20:59:19 -07001446 if ((static_branch_unlikely(&init_on_alloc) ||
1447 static_branch_unlikely(&init_on_free)) &&
1448 (slub_debug & SLAB_POISON))
1449 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
Christoph Lameter41ecc552007-05-09 02:32:44 -07001450 return 1;
1451}
1452
1453__setup("slub_debug", setup_slub_debug);
1454
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001455/*
1456 * kmem_cache_flags - apply debugging options to the cache
1457 * @object_size: the size of an object without meta data
1458 * @flags: flags to set
1459 * @name: name of the cache
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001460 *
1461 * Debug option(s) are applied to @flags. In addition to the debug
1462 * option(s), if a slab name (or multiple) is specified i.e.
1463 * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1464 * then only the select slabs will receive the debug option(s).
1465 */
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -07001466slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -08001467 slab_flags_t flags, const char *name)
Christoph Lameter41ecc552007-05-09 02:32:44 -07001468{
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001469 char *iter;
1470 size_t len;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001471 char *next_block;
1472 slab_flags_t block_flags;
Johannes Bergca220592021-02-24 12:01:04 -08001473 slab_flags_t slub_debug_local = slub_debug;
1474
1475 /*
1476 * If the slab cache is for debugging (e.g. kmemleak) then
1477 * don't store user (stack trace) information by default,
1478 * but let the user enable it via the command line below.
1479 */
1480 if (flags & SLAB_NOLEAKTRACE)
1481 slub_debug_local &= ~SLAB_STORE_USER;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001482
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001483 len = strlen(name);
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001484 next_block = slub_debug_string;
1485 /* Go through all blocks of debug options, see if any matches our slab's name */
1486 while (next_block) {
1487 next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
1488 if (!iter)
1489 continue;
1490 /* Found a block that has a slab list, search it */
1491 while (*iter) {
1492 char *end, *glob;
1493 size_t cmplen;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001494
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001495 end = strchrnul(iter, ',');
1496 if (next_block && next_block < end)
1497 end = next_block - 1;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001498
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001499 glob = strnchr(iter, end - iter, '*');
1500 if (glob)
1501 cmplen = glob - iter;
1502 else
1503 cmplen = max_t(size_t, len, (end - iter));
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001504
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001505 if (!strncmp(name, iter, cmplen)) {
1506 flags |= block_flags;
1507 return flags;
1508 }
1509
1510 if (!*end || *end == ';')
1511 break;
1512 iter = end + 1;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001513 }
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001514 }
Christoph Lameterba0268a2007-09-11 15:24:11 -07001515
Johannes Bergca220592021-02-24 12:01:04 -08001516 return flags | slub_debug_local;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001517}
Jesper Dangaard Brouerb4a64712015-11-20 15:57:41 -08001518#else /* !CONFIG_SLUB_DEBUG */
Christoph Lameter3ec09742007-05-16 22:11:00 -07001519static inline void setup_object_debug(struct kmem_cache *s,
1520 struct page *page, void *object) {}
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07001521static inline
1522void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) {}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001523
Christoph Lameter3ec09742007-05-16 22:11:00 -07001524static inline int alloc_debug_processing(struct kmem_cache *s,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001525 struct page *page, void *object, unsigned long addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001526
Laura Abbott282acb42016-03-15 14:54:59 -07001527static inline int free_debug_processing(
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001528 struct kmem_cache *s, struct page *page,
1529 void *head, void *tail, int bulk_cnt,
Laura Abbott282acb42016-03-15 14:54:59 -07001530 unsigned long addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001531
Christoph Lameter41ecc552007-05-09 02:32:44 -07001532static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1533 { return 1; }
1534static inline int check_object(struct kmem_cache *s, struct page *page,
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001535 void *object, u8 val) { return 1; }
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001536static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1537 struct page *page) {}
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001538static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1539 struct page *page) {}
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -07001540slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -08001541 slab_flags_t flags, const char *name)
Christoph Lameterba0268a2007-09-11 15:24:11 -07001542{
1543 return flags;
1544}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001545#define slub_debug 0
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001546
Ingo Molnarfdaa45e2009-09-15 11:00:26 +02001547#define disable_higher_order_debug 0
1548
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001549static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1550 { return 0; }
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04001551static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1552 { return 0; }
Christoph Lameter205ab992008-04-14 19:11:40 +03001553static inline void inc_slabs_node(struct kmem_cache *s, int node,
1554 int objects) {}
1555static inline void dec_slabs_node(struct kmem_cache *s, int node,
1556 int objects) {}
Christoph Lameter7d550c52010-08-25 14:07:16 -05001557
Dongli Zhang52f23472020-06-01 21:45:47 -07001558static bool freelist_corrupted(struct kmem_cache *s, struct page *page,
Eugeniu Roscadc07a722020-09-04 16:35:30 -07001559 void **freelist, void *nextfree)
Dongli Zhang52f23472020-06-01 21:45:47 -07001560{
1561 return false;
1562}
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001563#endif /* CONFIG_SLUB_DEBUG */
1564
1565/*
1566 * Hooks for other subsystems that check memory allocations. In a typical
1567 * production configuration these hooks all should produce no code at all.
1568 */
Andrey Konovalov0116523c2018-12-28 00:29:37 -08001569static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
Roman Bobnievd56791b2013-10-08 15:58:57 -07001570{
Andrey Konovalov53128242019-02-20 22:19:11 -08001571 ptr = kasan_kmalloc_large(ptr, size, flags);
Andrey Konovalova2f77572019-02-20 22:19:16 -08001572 /* As ptr might get tagged, call kmemleak hook after KASAN. */
Roman Bobnievd56791b2013-10-08 15:58:57 -07001573 kmemleak_alloc(ptr, size, 1, flags);
Andrey Konovalov53128242019-02-20 22:19:11 -08001574 return ptr;
Roman Bobnievd56791b2013-10-08 15:58:57 -07001575}
1576
Dmitry Vyukovee3ce772018-02-06 15:36:27 -08001577static __always_inline void kfree_hook(void *x)
Roman Bobnievd56791b2013-10-08 15:58:57 -07001578{
1579 kmemleak_free(x);
Andrey Konovalov027b37b2021-02-24 12:05:46 -08001580 kasan_kfree_large(x);
Roman Bobnievd56791b2013-10-08 15:58:57 -07001581}
1582
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001583static __always_inline bool slab_free_hook(struct kmem_cache *s,
1584 void *x, bool init)
Roman Bobnievd56791b2013-10-08 15:58:57 -07001585{
1586 kmemleak_free_recursive(x, s->flags);
Christoph Lameter7d550c52010-08-25 14:07:16 -05001587
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001588 /*
1589 * Trouble is that we may no longer disable interrupts in the fast path
1590 * So in order to make the debug calls that expect irqs to be
1591 * disabled we need to disable interrupts temporarily.
1592 */
Levin, Alexander (Sasha Levin)4675ff02017-11-15 17:36:02 -08001593#ifdef CONFIG_LOCKDEP
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001594 {
1595 unsigned long flags;
1596
1597 local_irq_save(flags);
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001598 debug_check_no_locks_freed(x, s->object_size);
1599 local_irq_restore(flags);
1600 }
1601#endif
1602 if (!(s->flags & SLAB_DEBUG_OBJECTS))
1603 debug_check_no_obj_freed(x, s->object_size);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001604
Marco Elvercfbe1632020-08-06 23:19:12 -07001605 /* Use KCSAN to help debug racy use-after-free. */
1606 if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
1607 __kcsan_check_access(x, s->object_size,
1608 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
1609
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001610 /*
1611 * As memory initialization might be integrated into KASAN,
1612 * kasan_slab_free and initialization memset's must be
1613 * kept together to avoid discrepancies in behavior.
1614 *
1615 * The initialization memset's clear the object and the metadata,
1616 * but don't touch the SLAB redzone.
1617 */
1618 if (init) {
1619 int rsize;
1620
1621 if (!kasan_has_integrated_init())
1622 memset(kasan_reset_tag(x), 0, s->object_size);
1623 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
1624 memset((char *)kasan_reset_tag(x) + s->inuse, 0,
1625 s->size - s->inuse - rsize);
1626 }
1627 /* KASAN might put x into memory quarantine, delaying its reuse. */
1628 return kasan_slab_free(s, x, init);
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001629}
Christoph Lameter205ab992008-04-14 19:11:40 +03001630
Andrey Konovalovc3895392018-04-10 16:30:31 -07001631static inline bool slab_free_freelist_hook(struct kmem_cache *s,
1632 void **head, void **tail)
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001633{
Alexander Potapenko64713842019-07-11 20:59:19 -07001634
1635 void *object;
1636 void *next = *head;
1637 void *old_tail = *tail ? *tail : *head;
Alexander Potapenko64713842019-07-11 20:59:19 -07001638
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08001639 if (is_kfence_address(next)) {
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001640 slab_free_hook(s, next, false);
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08001641 return true;
1642 }
1643
Laura Abbottaea4df42019-11-15 17:34:50 -08001644 /* Head and tail of the reconstructed freelist */
1645 *head = NULL;
1646 *tail = NULL;
Laura Abbott1b7e8162019-07-31 15:32:40 -04001647
Laura Abbottaea4df42019-11-15 17:34:50 -08001648 do {
1649 object = next;
1650 next = get_freepointer(s, object);
1651
Andrey Konovalovc3895392018-04-10 16:30:31 -07001652 /* If object's reuse doesn't have to be delayed */
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001653 if (!slab_free_hook(s, object, slab_want_init_on_free(s))) {
Andrey Konovalovc3895392018-04-10 16:30:31 -07001654 /* Move object to the new freelist */
1655 set_freepointer(s, object, *head);
1656 *head = object;
1657 if (!*tail)
1658 *tail = object;
1659 }
1660 } while (object != old_tail);
1661
1662 if (*head == *tail)
1663 *tail = NULL;
1664
1665 return *head != NULL;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001666}
1667
Andrey Konovalov4d176712018-12-28 00:30:23 -08001668static void *setup_object(struct kmem_cache *s, struct page *page,
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001669 void *object)
1670{
1671 setup_object_debug(s, page, object);
Andrey Konovalov4d176712018-12-28 00:30:23 -08001672 object = kasan_init_slab_obj(s, object);
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001673 if (unlikely(s->ctor)) {
1674 kasan_unpoison_object_data(s, object);
1675 s->ctor(object);
1676 kasan_poison_object_data(s, object);
1677 }
Andrey Konovalov4d176712018-12-28 00:30:23 -08001678 return object;
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001679}
1680
Christoph Lameter81819f02007-05-06 14:49:36 -07001681/*
1682 * Slab allocation and freeing
1683 */
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001684static inline struct page *alloc_slab_page(struct kmem_cache *s,
1685 gfp_t flags, int node, struct kmem_cache_order_objects oo)
Christoph Lameter65c33762008-04-14 19:11:40 +03001686{
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001687 struct page *page;
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07001688 unsigned int order = oo_order(oo);
Christoph Lameter65c33762008-04-14 19:11:40 +03001689
Christoph Lameter2154a332010-07-09 14:07:10 -05001690 if (node == NUMA_NO_NODE)
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001691 page = alloc_pages(flags, order);
Christoph Lameter65c33762008-04-14 19:11:40 +03001692 else
Vlastimil Babka96db8002015-09-08 15:03:50 -07001693 page = __alloc_pages_node(node, flags, order);
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001694
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001695 return page;
Christoph Lameter65c33762008-04-14 19:11:40 +03001696}
1697
Thomas Garnier210e7a42016-07-26 15:21:59 -07001698#ifdef CONFIG_SLAB_FREELIST_RANDOM
1699/* Pre-initialize the random sequence cache */
1700static int init_cache_random_seq(struct kmem_cache *s)
1701{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07001702 unsigned int count = oo_objects(s->oo);
Thomas Garnier210e7a42016-07-26 15:21:59 -07001703 int err;
Thomas Garnier210e7a42016-07-26 15:21:59 -07001704
Sean Reesa8100072017-02-08 14:30:59 -08001705 /* Bailout if already initialised */
1706 if (s->random_seq)
1707 return 0;
1708
Thomas Garnier210e7a42016-07-26 15:21:59 -07001709 err = cache_random_seq_create(s, count, GFP_KERNEL);
1710 if (err) {
1711 pr_err("SLUB: Unable to initialize free list for %s\n",
1712 s->name);
1713 return err;
1714 }
1715
1716 /* Transform to an offset on the set of pages */
1717 if (s->random_seq) {
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07001718 unsigned int i;
1719
Thomas Garnier210e7a42016-07-26 15:21:59 -07001720 for (i = 0; i < count; i++)
1721 s->random_seq[i] *= s->size;
1722 }
1723 return 0;
1724}
1725
1726/* Initialize each random sequence freelist per cache */
1727static void __init init_freelist_randomization(void)
1728{
1729 struct kmem_cache *s;
1730
1731 mutex_lock(&slab_mutex);
1732
1733 list_for_each_entry(s, &slab_caches, list)
1734 init_cache_random_seq(s);
1735
1736 mutex_unlock(&slab_mutex);
1737}
1738
1739/* Get the next entry on the pre-computed freelist randomized */
1740static void *next_freelist_entry(struct kmem_cache *s, struct page *page,
1741 unsigned long *pos, void *start,
1742 unsigned long page_limit,
1743 unsigned long freelist_count)
1744{
1745 unsigned int idx;
1746
1747 /*
1748 * If the target page allocation failed, the number of objects on the
1749 * page might be smaller than the usual size defined by the cache.
1750 */
1751 do {
1752 idx = s->random_seq[*pos];
1753 *pos += 1;
1754 if (*pos >= freelist_count)
1755 *pos = 0;
1756 } while (unlikely(idx >= page_limit));
1757
1758 return (char *)start + idx;
1759}
1760
1761/* Shuffle the single linked freelist based on a random pre-computed sequence */
1762static bool shuffle_freelist(struct kmem_cache *s, struct page *page)
1763{
1764 void *start;
1765 void *cur;
1766 void *next;
1767 unsigned long idx, pos, page_limit, freelist_count;
1768
1769 if (page->objects < 2 || !s->random_seq)
1770 return false;
1771
1772 freelist_count = oo_objects(s->oo);
1773 pos = get_random_int() % freelist_count;
1774
1775 page_limit = page->objects * s->size;
1776 start = fixup_red_left(s, page_address(page));
1777
1778 /* First entry is used as the base of the freelist */
1779 cur = next_freelist_entry(s, page, &pos, start, page_limit,
1780 freelist_count);
Andrey Konovalov4d176712018-12-28 00:30:23 -08001781 cur = setup_object(s, page, cur);
Thomas Garnier210e7a42016-07-26 15:21:59 -07001782 page->freelist = cur;
1783
1784 for (idx = 1; idx < page->objects; idx++) {
Thomas Garnier210e7a42016-07-26 15:21:59 -07001785 next = next_freelist_entry(s, page, &pos, start, page_limit,
1786 freelist_count);
Andrey Konovalov4d176712018-12-28 00:30:23 -08001787 next = setup_object(s, page, next);
Thomas Garnier210e7a42016-07-26 15:21:59 -07001788 set_freepointer(s, cur, next);
1789 cur = next;
1790 }
Thomas Garnier210e7a42016-07-26 15:21:59 -07001791 set_freepointer(s, cur, NULL);
1792
1793 return true;
1794}
1795#else
1796static inline int init_cache_random_seq(struct kmem_cache *s)
1797{
1798 return 0;
1799}
1800static inline void init_freelist_randomization(void) { }
1801static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page)
1802{
1803 return false;
1804}
1805#endif /* CONFIG_SLAB_FREELIST_RANDOM */
1806
Christoph Lameter81819f02007-05-06 14:49:36 -07001807static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1808{
Pekka Enberg06428782008-01-07 23:20:27 -08001809 struct page *page;
Christoph Lameter834f3d12008-04-14 19:11:31 +03001810 struct kmem_cache_order_objects oo = s->oo;
Pekka Enbergba522702009-06-24 21:59:51 +03001811 gfp_t alloc_gfp;
Andrey Konovalov4d176712018-12-28 00:30:23 -08001812 void *start, *p, *next;
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07001813 int idx;
Thomas Garnier210e7a42016-07-26 15:21:59 -07001814 bool shuffle;
Christoph Lameter81819f02007-05-06 14:49:36 -07001815
Christoph Lameter7e0528d2011-06-01 12:25:44 -05001816 flags &= gfp_allowed_mask;
1817
Mel Gormand0164ad2015-11-06 16:28:21 -08001818 if (gfpflags_allow_blocking(flags))
Christoph Lameter7e0528d2011-06-01 12:25:44 -05001819 local_irq_enable();
1820
Christoph Lameterb7a49f02008-02-14 14:21:32 -08001821 flags |= s->allocflags;
Mel Gormane12ba742007-10-16 01:25:52 -07001822
Pekka Enbergba522702009-06-24 21:59:51 +03001823 /*
1824 * Let the initial higher-order allocation fail under memory pressure
1825 * so we fall-back to the minimum order allocation.
1826 */
1827 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
Mel Gormand0164ad2015-11-06 16:28:21 -08001828 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
Mel Gorman444eb2a42016-03-17 14:19:23 -07001829 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
Pekka Enbergba522702009-06-24 21:59:51 +03001830
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001831 page = alloc_slab_page(s, alloc_gfp, node, oo);
Christoph Lameter65c33762008-04-14 19:11:40 +03001832 if (unlikely(!page)) {
1833 oo = s->min;
Joonsoo Kim80c3a992014-03-12 17:26:20 +09001834 alloc_gfp = flags;
Christoph Lameter65c33762008-04-14 19:11:40 +03001835 /*
1836 * Allocation may have failed due to fragmentation.
1837 * Try a lower order alloc if possible
1838 */
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001839 page = alloc_slab_page(s, alloc_gfp, node, oo);
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001840 if (unlikely(!page))
1841 goto out;
1842 stat(s, ORDER_FALLBACK);
Christoph Lameter65c33762008-04-14 19:11:40 +03001843 }
Vegard Nossum5a896d92008-04-04 00:54:48 +02001844
Christoph Lameter834f3d12008-04-14 19:11:31 +03001845 page->objects = oo_objects(oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07001846
Roman Gushchin2e9bd482021-02-24 12:03:11 -08001847 account_slab_page(page, oo_order(oo), s, flags);
Roman Gushchin1f3147b2020-12-29 15:15:07 -08001848
Glauber Costa1b4f59e32012-10-22 18:05:36 +04001849 page->slab_cache = s;
Joonsoo Kimc03f94c2012-05-18 00:47:47 +09001850 __SetPageSlab(page);
Michal Hocko2f064f32015-08-21 14:11:51 -07001851 if (page_is_pfmemalloc(page))
Mel Gorman072bb0a2012-07-31 16:43:58 -07001852 SetPageSlabPfmemalloc(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001853
Andrey Konovalova7101222019-02-20 22:19:23 -08001854 kasan_poison_slab(page);
1855
Christoph Lameter81819f02007-05-06 14:49:36 -07001856 start = page_address(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001857
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07001858 setup_page_debug(s, page, start);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001859
Thomas Garnier210e7a42016-07-26 15:21:59 -07001860 shuffle = shuffle_freelist(s, page);
1861
1862 if (!shuffle) {
Andrey Konovalov4d176712018-12-28 00:30:23 -08001863 start = fixup_red_left(s, start);
1864 start = setup_object(s, page, start);
1865 page->freelist = start;
Andrey Konovalov18e50662019-02-20 22:19:28 -08001866 for (idx = 0, p = start; idx < page->objects - 1; idx++) {
1867 next = p + s->size;
1868 next = setup_object(s, page, next);
1869 set_freepointer(s, p, next);
1870 p = next;
1871 }
1872 set_freepointer(s, p, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07001873 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001874
Christoph Lametere6e82ea2011-08-09 16:12:24 -05001875 page->inuse = page->objects;
Christoph Lameter8cb0a502011-06-01 12:25:46 -05001876 page->frozen = 1;
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001877
Christoph Lameter81819f02007-05-06 14:49:36 -07001878out:
Mel Gormand0164ad2015-11-06 16:28:21 -08001879 if (gfpflags_allow_blocking(flags))
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001880 local_irq_disable();
1881 if (!page)
1882 return NULL;
1883
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001884 inc_slabs_node(s, page_to_nid(page), page->objects);
1885
Christoph Lameter81819f02007-05-06 14:49:36 -07001886 return page;
1887}
1888
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001889static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1890{
Long Li44405092020-08-06 23:18:28 -07001891 if (unlikely(flags & GFP_SLAB_BUG_MASK))
1892 flags = kmalloc_fix_flags(flags);
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001893
1894 return allocate_slab(s,
1895 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
1896}
1897
Christoph Lameter81819f02007-05-06 14:49:36 -07001898static void __free_slab(struct kmem_cache *s, struct page *page)
1899{
Christoph Lameter834f3d12008-04-14 19:11:31 +03001900 int order = compound_order(page);
1901 int pages = 1 << order;
Christoph Lameter81819f02007-05-06 14:49:36 -07001902
Vlastimil Babka8fc8d662020-08-06 23:18:58 -07001903 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001904 void *p;
1905
1906 slab_pad_check(s, page);
Christoph Lameter224a88b2008-04-14 19:11:31 +03001907 for_each_object(p, s, page_address(page),
1908 page->objects)
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001909 check_object(s, page, p, SLUB_RED_INACTIVE);
Christoph Lameter81819f02007-05-06 14:49:36 -07001910 }
1911
Mel Gorman072bb0a2012-07-31 16:43:58 -07001912 __ClearPageSlabPfmemalloc(page);
Christoph Lameter49bd5222008-04-14 18:52:18 +03001913 __ClearPageSlab(page);
Vlastimil Babka0c06dd72020-12-14 19:04:29 -08001914 /* In union with page->mapping where page allocator expects NULL */
1915 page->slab_cache = NULL;
Nick Piggin1eb5ac62009-05-05 19:13:44 +10001916 if (current->reclaim_state)
1917 current->reclaim_state->reclaimed_slab += pages;
Roman Gushchin74d555b2020-08-06 23:21:44 -07001918 unaccount_slab_page(page, order, s);
Vladimir Davydov27ee57c2016-03-17 14:17:35 -07001919 __free_pages(page, order);
Christoph Lameter81819f02007-05-06 14:49:36 -07001920}
1921
1922static void rcu_free_slab(struct rcu_head *h)
1923{
Matthew Wilcoxbf68c212018-06-07 17:09:05 -07001924 struct page *page = container_of(h, struct page, rcu_head);
Lai Jiangshanda9a6382011-03-10 15:22:00 +08001925
Glauber Costa1b4f59e32012-10-22 18:05:36 +04001926 __free_slab(page->slab_cache, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001927}
1928
1929static void free_slab(struct kmem_cache *s, struct page *page)
1930{
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -08001931 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
Matthew Wilcoxbf68c212018-06-07 17:09:05 -07001932 call_rcu(&page->rcu_head, rcu_free_slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07001933 } else
1934 __free_slab(s, page);
1935}
1936
1937static void discard_slab(struct kmem_cache *s, struct page *page)
1938{
Christoph Lameter205ab992008-04-14 19:11:40 +03001939 dec_slabs_node(s, page_to_nid(page), page->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07001940 free_slab(s, page);
1941}
1942
1943/*
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001944 * Management of partially allocated slabs.
Christoph Lameter81819f02007-05-06 14:49:36 -07001945 */
Steven Rostedt1e4dd942014-02-10 14:25:46 -08001946static inline void
1947__add_partial(struct kmem_cache_node *n, struct page *page, int tail)
Christoph Lameter81819f02007-05-06 14:49:36 -07001948{
Christoph Lametere95eed52007-05-06 14:49:44 -07001949 n->nr_partial++;
Shaohua Li136333d2011-08-24 08:57:52 +08001950 if (tail == DEACTIVATE_TO_TAIL)
Tobin C. Harding916ac052019-05-13 17:16:12 -07001951 list_add_tail(&page->slab_list, &n->partial);
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001952 else
Tobin C. Harding916ac052019-05-13 17:16:12 -07001953 list_add(&page->slab_list, &n->partial);
Christoph Lameter81819f02007-05-06 14:49:36 -07001954}
1955
Steven Rostedt1e4dd942014-02-10 14:25:46 -08001956static inline void add_partial(struct kmem_cache_node *n,
1957 struct page *page, int tail)
1958{
1959 lockdep_assert_held(&n->list_lock);
1960 __add_partial(n, page, tail);
1961}
1962
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001963static inline void remove_partial(struct kmem_cache_node *n,
Christoph Lameter62e346a2010-09-28 08:10:28 -05001964 struct page *page)
1965{
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001966 lockdep_assert_held(&n->list_lock);
Tobin C. Harding916ac052019-05-13 17:16:12 -07001967 list_del(&page->slab_list);
Dmitry Safonov52b4b952016-02-17 13:11:37 -08001968 n->nr_partial--;
Christoph Lameter62e346a2010-09-28 08:10:28 -05001969}
1970
Christoph Lameter81819f02007-05-06 14:49:36 -07001971/*
Christoph Lameter7ced3712012-05-09 10:09:53 -05001972 * Remove slab from the partial list, freeze it and
1973 * return the pointer to the freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07001974 *
Christoph Lameter497b66f2011-08-09 16:12:26 -05001975 * Returns a list of objects or NULL if it fails.
Christoph Lameter81819f02007-05-06 14:49:36 -07001976 */
Christoph Lameter497b66f2011-08-09 16:12:26 -05001977static inline void *acquire_slab(struct kmem_cache *s,
Christoph Lameteracd19fd2011-08-09 16:12:25 -05001978 struct kmem_cache_node *n, struct page *page,
Joonsoo Kim633b0762013-01-21 17:01:25 +09001979 int mode, int *objects)
Christoph Lameter81819f02007-05-06 14:49:36 -07001980{
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001981 void *freelist;
1982 unsigned long counters;
1983 struct page new;
1984
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001985 lockdep_assert_held(&n->list_lock);
1986
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001987 /*
1988 * Zap the freelist and set the frozen bit.
1989 * The old freelist is the list of objects for the
1990 * per cpu allocation list.
1991 */
Christoph Lameter7ced3712012-05-09 10:09:53 -05001992 freelist = page->freelist;
1993 counters = page->counters;
1994 new.counters = counters;
Joonsoo Kim633b0762013-01-21 17:01:25 +09001995 *objects = new.objects - new.inuse;
Pekka Enberg23910c52012-06-04 10:14:58 +03001996 if (mode) {
Christoph Lameter7ced3712012-05-09 10:09:53 -05001997 new.inuse = page->objects;
Pekka Enberg23910c52012-06-04 10:14:58 +03001998 new.freelist = NULL;
1999 } else {
2000 new.freelist = freelist;
2001 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002002
Dave Hansena0132ac2014-01-29 14:05:50 -08002003 VM_BUG_ON(new.frozen);
Christoph Lameter7ced3712012-05-09 10:09:53 -05002004 new.frozen = 1;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002005
Christoph Lameter7ced3712012-05-09 10:09:53 -05002006 if (!__cmpxchg_double_slab(s, page,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002007 freelist, counters,
Joonsoo Kim02d76332012-05-17 00:13:02 +09002008 new.freelist, new.counters,
Christoph Lameter7ced3712012-05-09 10:09:53 -05002009 "acquire_slab"))
Christoph Lameter7ced3712012-05-09 10:09:53 -05002010 return NULL;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002011
2012 remove_partial(n, page);
Christoph Lameter7ced3712012-05-09 10:09:53 -05002013 WARN_ON(!freelist);
Christoph Lameter49e22582011-08-09 16:12:27 -05002014 return freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07002015}
2016
Joonsoo Kim633b0762013-01-21 17:01:25 +09002017static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002018static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
Christoph Lameter49e22582011-08-09 16:12:27 -05002019
Christoph Lameter81819f02007-05-06 14:49:36 -07002020/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002021 * Try to allocate a partial slab from a specific node.
Christoph Lameter81819f02007-05-06 14:49:36 -07002022 */
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002023static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
2024 struct kmem_cache_cpu *c, gfp_t flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07002025{
Christoph Lameter49e22582011-08-09 16:12:27 -05002026 struct page *page, *page2;
2027 void *object = NULL;
Alexey Dobriyane5d99982018-04-05 16:21:10 -07002028 unsigned int available = 0;
Joonsoo Kim633b0762013-01-21 17:01:25 +09002029 int objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07002030
2031 /*
2032 * Racy check. If we mistakenly see no partial slabs then we
2033 * just allocate an empty slab. If we mistakenly try to get a
Chen Tao70b6d252020-10-15 20:10:01 -07002034 * partial slab and there is none available then get_partial()
Christoph Lameter672bba32007-05-09 02:32:39 -07002035 * will return NULL.
Christoph Lameter81819f02007-05-06 14:49:36 -07002036 */
2037 if (!n || !n->nr_partial)
2038 return NULL;
2039
2040 spin_lock(&n->list_lock);
Tobin C. Harding916ac052019-05-13 17:16:12 -07002041 list_for_each_entry_safe(page, page2, &n->partial, slab_list) {
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002042 void *t;
Christoph Lameter49e22582011-08-09 16:12:27 -05002043
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002044 if (!pfmemalloc_match(page, flags))
2045 continue;
2046
Joonsoo Kim633b0762013-01-21 17:01:25 +09002047 t = acquire_slab(s, n, page, object == NULL, &objects);
Christoph Lameter49e22582011-08-09 16:12:27 -05002048 if (!t)
Linus Torvalds9b1ea292021-03-10 10:18:04 -08002049 break;
Christoph Lameter49e22582011-08-09 16:12:27 -05002050
Joonsoo Kim633b0762013-01-21 17:01:25 +09002051 available += objects;
Alex,Shi12d79632011-09-07 10:26:36 +08002052 if (!object) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002053 c->page = page;
Christoph Lameter49e22582011-08-09 16:12:27 -05002054 stat(s, ALLOC_FROM_PARTIAL);
Christoph Lameter49e22582011-08-09 16:12:27 -05002055 object = t;
Christoph Lameter49e22582011-08-09 16:12:27 -05002056 } else {
Joonsoo Kim633b0762013-01-21 17:01:25 +09002057 put_cpu_partial(s, page, 0);
Alex Shi8028dce2012-02-03 23:34:56 +08002058 stat(s, CPU_PARTIAL_NODE);
Christoph Lameter49e22582011-08-09 16:12:27 -05002059 }
Joonsoo Kim345c9052013-06-19 14:05:52 +09002060 if (!kmem_cache_has_cpu_partial(s)
Wei Yange6d0e1d2017-07-06 15:36:34 -07002061 || available > slub_cpu_partial(s) / 2)
Christoph Lameter49e22582011-08-09 16:12:27 -05002062 break;
2063
Christoph Lameter497b66f2011-08-09 16:12:26 -05002064 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002065 spin_unlock(&n->list_lock);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002066 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07002067}
2068
2069/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002070 * Get a page from somewhere. Search in increasing NUMA distances.
Christoph Lameter81819f02007-05-06 14:49:36 -07002071 */
Joonsoo Kimde3ec032012-01-27 00:12:23 -08002072static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
Christoph Lameteracd19fd2011-08-09 16:12:25 -05002073 struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002074{
2075#ifdef CONFIG_NUMA
2076 struct zonelist *zonelist;
Mel Gormandd1a2392008-04-28 02:12:17 -07002077 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07002078 struct zone *zone;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002079 enum zone_type highest_zoneidx = gfp_zone(flags);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002080 void *object;
Mel Gormancc9a6c82012-03-21 16:34:11 -07002081 unsigned int cpuset_mems_cookie;
Christoph Lameter81819f02007-05-06 14:49:36 -07002082
2083 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002084 * The defrag ratio allows a configuration of the tradeoffs between
2085 * inter node defragmentation and node local allocations. A lower
2086 * defrag_ratio increases the tendency to do local allocations
2087 * instead of attempting to obtain partial slabs from other nodes.
Christoph Lameter81819f02007-05-06 14:49:36 -07002088 *
Christoph Lameter672bba32007-05-09 02:32:39 -07002089 * If the defrag_ratio is set to 0 then kmalloc() always
2090 * returns node local objects. If the ratio is higher then kmalloc()
2091 * may return off node objects because partial slabs are obtained
2092 * from other nodes and filled up.
Christoph Lameter81819f02007-05-06 14:49:36 -07002093 *
Li Peng43efd3e2016-05-19 17:10:43 -07002094 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
2095 * (which makes defrag_ratio = 1000) then every (well almost)
2096 * allocation will first attempt to defrag slab caches on other nodes.
2097 * This means scanning over all nodes to look for partial slabs which
2098 * may be expensive if we do it every time we are trying to find a slab
Christoph Lameter672bba32007-05-09 02:32:39 -07002099 * with available objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07002100 */
Christoph Lameter98246012008-01-07 23:20:26 -08002101 if (!s->remote_node_defrag_ratio ||
2102 get_cycles() % 1024 > s->remote_node_defrag_ratio)
Christoph Lameter81819f02007-05-06 14:49:36 -07002103 return NULL;
2104
Mel Gormancc9a6c82012-03-21 16:34:11 -07002105 do {
Mel Gormand26914d2014-04-03 14:47:24 -07002106 cpuset_mems_cookie = read_mems_allowed_begin();
David Rientjes2a389612014-04-07 15:37:29 -07002107 zonelist = node_zonelist(mempolicy_slab_node(), flags);
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002108 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
Mel Gormancc9a6c82012-03-21 16:34:11 -07002109 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07002110
Mel Gormancc9a6c82012-03-21 16:34:11 -07002111 n = get_node(s, zone_to_nid(zone));
Christoph Lameter81819f02007-05-06 14:49:36 -07002112
Vladimir Davydovdee2f8a2014-12-12 16:58:28 -08002113 if (n && cpuset_zone_allowed(zone, flags) &&
Mel Gormancc9a6c82012-03-21 16:34:11 -07002114 n->nr_partial > s->min_partial) {
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002115 object = get_partial_node(s, n, c, flags);
Mel Gormancc9a6c82012-03-21 16:34:11 -07002116 if (object) {
2117 /*
Mel Gormand26914d2014-04-03 14:47:24 -07002118 * Don't check read_mems_allowed_retry()
2119 * here - if mems_allowed was updated in
2120 * parallel, that was a harmless race
2121 * between allocation and the cpuset
2122 * update
Mel Gormancc9a6c82012-03-21 16:34:11 -07002123 */
Mel Gormancc9a6c82012-03-21 16:34:11 -07002124 return object;
2125 }
Miao Xiec0ff7452010-05-24 14:32:08 -07002126 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002127 }
Mel Gormand26914d2014-04-03 14:47:24 -07002128 } while (read_mems_allowed_retry(cpuset_mems_cookie));
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07002129#endif /* CONFIG_NUMA */
Christoph Lameter81819f02007-05-06 14:49:36 -07002130 return NULL;
2131}
2132
2133/*
2134 * Get a partial page, lock it and return it.
2135 */
Christoph Lameter497b66f2011-08-09 16:12:26 -05002136static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
Christoph Lameteracd19fd2011-08-09 16:12:25 -05002137 struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002138{
Christoph Lameter497b66f2011-08-09 16:12:26 -05002139 void *object;
Joonsoo Kima561ce02014-10-09 15:26:15 -07002140 int searchnode = node;
2141
2142 if (node == NUMA_NO_NODE)
2143 searchnode = numa_mem_id();
Christoph Lameter81819f02007-05-06 14:49:36 -07002144
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002145 object = get_partial_node(s, get_node(s, searchnode), c, flags);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002146 if (object || node != NUMA_NO_NODE)
2147 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07002148
Christoph Lameteracd19fd2011-08-09 16:12:25 -05002149 return get_any_partial(s, flags, c);
Christoph Lameter81819f02007-05-06 14:49:36 -07002150}
2151
Thomas Gleixner923717c2019-10-15 21:18:12 +02002152#ifdef CONFIG_PREEMPTION
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002153/*
Ethon Paul0d645ed2020-06-04 16:49:34 -07002154 * Calculate the next globally unique transaction for disambiguation
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002155 * during cmpxchg. The transactions start with the cpu number and are then
2156 * incremented by CONFIG_NR_CPUS.
2157 */
2158#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
2159#else
2160/*
2161 * No preemption supported therefore also no need to check for
2162 * different cpus.
2163 */
2164#define TID_STEP 1
2165#endif
2166
2167static inline unsigned long next_tid(unsigned long tid)
2168{
2169 return tid + TID_STEP;
2170}
2171
Qian Cai9d5f0be2019-09-23 15:33:52 -07002172#ifdef SLUB_DEBUG_CMPXCHG
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002173static inline unsigned int tid_to_cpu(unsigned long tid)
2174{
2175 return tid % TID_STEP;
2176}
2177
2178static inline unsigned long tid_to_event(unsigned long tid)
2179{
2180 return tid / TID_STEP;
2181}
Qian Cai9d5f0be2019-09-23 15:33:52 -07002182#endif
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002183
2184static inline unsigned int init_tid(int cpu)
2185{
2186 return cpu;
2187}
2188
2189static inline void note_cmpxchg_failure(const char *n,
2190 const struct kmem_cache *s, unsigned long tid)
2191{
2192#ifdef SLUB_DEBUG_CMPXCHG
2193 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
2194
Fabian Frederickf9f58282014-06-04 16:06:34 -07002195 pr_info("%s %s: cmpxchg redo ", n, s->name);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002196
Thomas Gleixner923717c2019-10-15 21:18:12 +02002197#ifdef CONFIG_PREEMPTION
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002198 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
Fabian Frederickf9f58282014-06-04 16:06:34 -07002199 pr_warn("due to cpu change %d -> %d\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002200 tid_to_cpu(tid), tid_to_cpu(actual_tid));
2201 else
2202#endif
2203 if (tid_to_event(tid) != tid_to_event(actual_tid))
Fabian Frederickf9f58282014-06-04 16:06:34 -07002204 pr_warn("due to cpu running other code. Event %ld->%ld\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002205 tid_to_event(tid), tid_to_event(actual_tid));
2206 else
Fabian Frederickf9f58282014-06-04 16:06:34 -07002207 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002208 actual_tid, tid, next_tid(tid));
2209#endif
Christoph Lameter4fdccdf2011-03-22 13:35:00 -05002210 stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002211}
2212
Fengguang Wu788e1aa2012-09-28 16:34:05 +08002213static void init_kmem_cache_cpus(struct kmem_cache *s)
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002214{
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002215 int cpu;
2216
2217 for_each_possible_cpu(cpu)
2218 per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002219}
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002220
2221/*
2222 * Remove the cpu slab
2223 */
Chen Gangd0e0ac92013-07-15 09:05:29 +08002224static void deactivate_slab(struct kmem_cache *s, struct page *page,
Wei Yangd4ff6d32017-07-06 15:36:25 -07002225 void *freelist, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002226{
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002227 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002228 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002229 int lock = 0, free_delta = 0;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002230 enum slab_modes l = M_NONE, m = M_NONE;
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002231 void *nextfree, *freelist_iter, *freelist_tail;
Shaohua Li136333d2011-08-24 08:57:52 +08002232 int tail = DEACTIVATE_TO_HEAD;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002233 struct page new;
2234 struct page old;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08002235
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002236 if (page->freelist) {
Christoph Lameter84e554e62009-12-18 16:26:23 -06002237 stat(s, DEACTIVATE_REMOTE_FREES);
Shaohua Li136333d2011-08-24 08:57:52 +08002238 tail = DEACTIVATE_TO_TAIL;
Christoph Lameter894b8782007-05-10 03:15:16 -07002239 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002240
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002241 /*
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002242 * Stage one: Count the objects on cpu's freelist as free_delta and
2243 * remember the last object in freelist_tail for later splicing.
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002244 */
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002245 freelist_tail = NULL;
2246 freelist_iter = freelist;
2247 while (freelist_iter) {
2248 nextfree = get_freepointer(s, freelist_iter);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002249
Dongli Zhang52f23472020-06-01 21:45:47 -07002250 /*
2251 * If 'nextfree' is invalid, it is possible that the object at
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002252 * 'freelist_iter' is already corrupted. So isolate all objects
2253 * starting at 'freelist_iter' by skipping them.
Dongli Zhang52f23472020-06-01 21:45:47 -07002254 */
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002255 if (freelist_corrupted(s, page, &freelist_iter, nextfree))
Dongli Zhang52f23472020-06-01 21:45:47 -07002256 break;
2257
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002258 freelist_tail = freelist_iter;
2259 free_delta++;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002260
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002261 freelist_iter = nextfree;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002262 }
2263
2264 /*
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002265 * Stage two: Unfreeze the page while splicing the per-cpu
2266 * freelist to the head of page's freelist.
2267 *
2268 * Ensure that the page is unfrozen while the list presence
2269 * reflects the actual number of objects during unfreeze.
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002270 *
2271 * We setup the list membership and then perform a cmpxchg
2272 * with the count. If there is a mismatch then the page
2273 * is not unfrozen but the page is on the wrong list.
2274 *
2275 * Then we restart the process which may have to remove
2276 * the page from the list that we just put it on again
2277 * because the number of objects in the slab may have
2278 * changed.
2279 */
2280redo:
2281
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002282 old.freelist = READ_ONCE(page->freelist);
2283 old.counters = READ_ONCE(page->counters);
Dave Hansena0132ac2014-01-29 14:05:50 -08002284 VM_BUG_ON(!old.frozen);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002285
2286 /* Determine target state of the slab */
2287 new.counters = old.counters;
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002288 if (freelist_tail) {
2289 new.inuse -= free_delta;
2290 set_freepointer(s, freelist_tail, old.freelist);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002291 new.freelist = freelist;
2292 } else
2293 new.freelist = old.freelist;
2294
2295 new.frozen = 0;
2296
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07002297 if (!new.inuse && n->nr_partial >= s->min_partial)
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002298 m = M_FREE;
2299 else if (new.freelist) {
2300 m = M_PARTIAL;
2301 if (!lock) {
2302 lock = 1;
2303 /*
Wei Yang8bb4e7a2019-03-05 15:46:22 -08002304 * Taking the spinlock removes the possibility
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002305 * that acquire_slab() will see a slab page that
2306 * is frozen
2307 */
2308 spin_lock(&n->list_lock);
2309 }
2310 } else {
2311 m = M_FULL;
Vlastimil Babka965c4842020-12-14 19:04:36 -08002312 if (kmem_cache_debug_flags(s, SLAB_STORE_USER) && !lock) {
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002313 lock = 1;
2314 /*
2315 * This also ensures that the scanning of full
2316 * slabs from diagnostic functions will not see
2317 * any frozen slabs.
2318 */
2319 spin_lock(&n->list_lock);
2320 }
2321 }
2322
2323 if (l != m) {
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002324 if (l == M_PARTIAL)
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002325 remove_partial(n, page);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002326 else if (l == M_FULL)
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002327 remove_full(s, n, page);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002328
Wei Yang88349a22018-12-28 00:33:13 -08002329 if (m == M_PARTIAL)
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002330 add_partial(n, page, tail);
Wei Yang88349a22018-12-28 00:33:13 -08002331 else if (m == M_FULL)
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002332 add_full(s, n, page);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002333 }
2334
2335 l = m;
Christoph Lameter1d071712011-07-14 12:49:12 -05002336 if (!__cmpxchg_double_slab(s, page,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002337 old.freelist, old.counters,
2338 new.freelist, new.counters,
2339 "unfreezing slab"))
2340 goto redo;
2341
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002342 if (lock)
2343 spin_unlock(&n->list_lock);
2344
Wei Yang88349a22018-12-28 00:33:13 -08002345 if (m == M_PARTIAL)
2346 stat(s, tail);
2347 else if (m == M_FULL)
2348 stat(s, DEACTIVATE_FULL);
2349 else if (m == M_FREE) {
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002350 stat(s, DEACTIVATE_EMPTY);
2351 discard_slab(s, page);
2352 stat(s, FREE_SLAB);
2353 }
Wei Yangd4ff6d32017-07-06 15:36:25 -07002354
2355 c->page = NULL;
2356 c->freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07002357}
2358
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002359/*
2360 * Unfreeze all the cpu partial slabs.
2361 *
Christoph Lameter59a09912012-11-28 16:23:00 +00002362 * This function must be called with interrupts disabled
2363 * for the cpu using c (or some other guarantee must be there
2364 * to guarantee no concurrent accesses).
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002365 */
Christoph Lameter59a09912012-11-28 16:23:00 +00002366static void unfreeze_partials(struct kmem_cache *s,
2367 struct kmem_cache_cpu *c)
Christoph Lameter49e22582011-08-09 16:12:27 -05002368{
Joonsoo Kim345c9052013-06-19 14:05:52 +09002369#ifdef CONFIG_SLUB_CPU_PARTIAL
Joonsoo Kim43d77862012-06-09 02:23:16 +09002370 struct kmem_cache_node *n = NULL, *n2 = NULL;
Shaohua Li9ada1932011-11-14 13:34:13 +08002371 struct page *page, *discard_page = NULL;
Christoph Lameter49e22582011-08-09 16:12:27 -05002372
chenqiwu4c7ba222020-04-01 21:04:16 -07002373 while ((page = slub_percpu_partial(c))) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002374 struct page new;
2375 struct page old;
2376
chenqiwu4c7ba222020-04-01 21:04:16 -07002377 slub_set_percpu_partial(c, page);
Joonsoo Kim43d77862012-06-09 02:23:16 +09002378
2379 n2 = get_node(s, page_to_nid(page));
2380 if (n != n2) {
2381 if (n)
2382 spin_unlock(&n->list_lock);
2383
2384 n = n2;
2385 spin_lock(&n->list_lock);
2386 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002387
2388 do {
2389
2390 old.freelist = page->freelist;
2391 old.counters = page->counters;
Dave Hansena0132ac2014-01-29 14:05:50 -08002392 VM_BUG_ON(!old.frozen);
Christoph Lameter49e22582011-08-09 16:12:27 -05002393
2394 new.counters = old.counters;
2395 new.freelist = old.freelist;
2396
2397 new.frozen = 0;
2398
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002399 } while (!__cmpxchg_double_slab(s, page,
Christoph Lameter49e22582011-08-09 16:12:27 -05002400 old.freelist, old.counters,
2401 new.freelist, new.counters,
2402 "unfreezing slab"));
2403
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07002404 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
Shaohua Li9ada1932011-11-14 13:34:13 +08002405 page->next = discard_page;
2406 discard_page = page;
Joonsoo Kim43d77862012-06-09 02:23:16 +09002407 } else {
2408 add_partial(n, page, DEACTIVATE_TO_TAIL);
2409 stat(s, FREE_ADD_PARTIAL);
Christoph Lameter49e22582011-08-09 16:12:27 -05002410 }
2411 }
2412
2413 if (n)
2414 spin_unlock(&n->list_lock);
Shaohua Li9ada1932011-11-14 13:34:13 +08002415
2416 while (discard_page) {
2417 page = discard_page;
2418 discard_page = discard_page->next;
2419
2420 stat(s, DEACTIVATE_EMPTY);
2421 discard_slab(s, page);
2422 stat(s, FREE_SLAB);
2423 }
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07002424#endif /* CONFIG_SLUB_CPU_PARTIAL */
Christoph Lameter49e22582011-08-09 16:12:27 -05002425}
2426
2427/*
Wei Yang9234bae2019-03-05 15:43:10 -08002428 * Put a page that was just frozen (in __slab_free|get_partial_node) into a
2429 * partial page slot if available.
Christoph Lameter49e22582011-08-09 16:12:27 -05002430 *
2431 * If we did not find a slot then simply move all the partials to the
2432 * per node partial list.
2433 */
Joonsoo Kim633b0762013-01-21 17:01:25 +09002434static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
Christoph Lameter49e22582011-08-09 16:12:27 -05002435{
Joonsoo Kim345c9052013-06-19 14:05:52 +09002436#ifdef CONFIG_SLUB_CPU_PARTIAL
Christoph Lameter49e22582011-08-09 16:12:27 -05002437 struct page *oldpage;
2438 int pages;
2439 int pobjects;
2440
Vladimir Davydovd6e0b7f2015-02-12 14:59:47 -08002441 preempt_disable();
Christoph Lameter49e22582011-08-09 16:12:27 -05002442 do {
2443 pages = 0;
2444 pobjects = 0;
2445 oldpage = this_cpu_read(s->cpu_slab->partial);
2446
2447 if (oldpage) {
2448 pobjects = oldpage->pobjects;
2449 pages = oldpage->pages;
chenqiwubbd4e302020-04-01 21:04:19 -07002450 if (drain && pobjects > slub_cpu_partial(s)) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002451 unsigned long flags;
2452 /*
2453 * partial array is full. Move the existing
2454 * set to the per node partial list.
2455 */
2456 local_irq_save(flags);
Christoph Lameter59a09912012-11-28 16:23:00 +00002457 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
Christoph Lameter49e22582011-08-09 16:12:27 -05002458 local_irq_restore(flags);
Joonsoo Kime24fc412012-06-23 03:22:38 +09002459 oldpage = NULL;
Christoph Lameter49e22582011-08-09 16:12:27 -05002460 pobjects = 0;
2461 pages = 0;
Alex Shi8028dce2012-02-03 23:34:56 +08002462 stat(s, CPU_PARTIAL_DRAIN);
Christoph Lameter49e22582011-08-09 16:12:27 -05002463 }
2464 }
2465
2466 pages++;
2467 pobjects += page->objects - page->inuse;
2468
2469 page->pages = pages;
2470 page->pobjects = pobjects;
2471 page->next = oldpage;
2472
Chen Gangd0e0ac92013-07-15 09:05:29 +08002473 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
2474 != oldpage);
chenqiwubbd4e302020-04-01 21:04:19 -07002475 if (unlikely(!slub_cpu_partial(s))) {
Vladimir Davydovd6e0b7f2015-02-12 14:59:47 -08002476 unsigned long flags;
2477
2478 local_irq_save(flags);
2479 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
2480 local_irq_restore(flags);
2481 }
2482 preempt_enable();
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07002483#endif /* CONFIG_SLUB_CPU_PARTIAL */
Christoph Lameter49e22582011-08-09 16:12:27 -05002484}
2485
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002486static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002487{
Christoph Lameter84e554e62009-12-18 16:26:23 -06002488 stat(s, CPUSLAB_FLUSH);
Wei Yangd4ff6d32017-07-06 15:36:25 -07002489 deactivate_slab(s, c->page, c->freelist, c);
Christoph Lameterc17dda42012-05-09 10:09:57 -05002490
2491 c->tid = next_tid(c->tid);
Christoph Lameter81819f02007-05-06 14:49:36 -07002492}
2493
2494/*
2495 * Flush cpu slab.
Christoph Lameter6446faa2008-02-15 23:45:26 -08002496 *
Christoph Lameter81819f02007-05-06 14:49:36 -07002497 * Called from IPI handler with interrupts disabled.
2498 */
Christoph Lameter0c710012007-07-17 04:03:24 -07002499static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
Christoph Lameter81819f02007-05-06 14:49:36 -07002500{
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06002501 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
Christoph Lameter81819f02007-05-06 14:49:36 -07002502
Wei Yang1265ef22018-12-28 00:33:06 -08002503 if (c->page)
2504 flush_slab(s, c);
Christoph Lameter49e22582011-08-09 16:12:27 -05002505
Wei Yang1265ef22018-12-28 00:33:06 -08002506 unfreeze_partials(s, c);
Christoph Lameter81819f02007-05-06 14:49:36 -07002507}
2508
2509static void flush_cpu_slab(void *d)
2510{
2511 struct kmem_cache *s = d;
Christoph Lameter81819f02007-05-06 14:49:36 -07002512
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002513 __flush_cpu_slab(s, smp_processor_id());
Christoph Lameter81819f02007-05-06 14:49:36 -07002514}
2515
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002516static bool has_cpu_slab(int cpu, void *info)
2517{
2518 struct kmem_cache *s = info;
2519 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2520
Wei Yanga93cf072017-07-06 15:36:31 -07002521 return c->page || slub_percpu_partial(c);
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002522}
2523
Christoph Lameter81819f02007-05-06 14:49:36 -07002524static void flush_all(struct kmem_cache *s)
2525{
Sebastian Andrzej Siewiorcb923152020-01-17 10:01:37 +01002526 on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07002527}
2528
2529/*
Sebastian Andrzej Siewiora96a87b2016-08-18 14:57:19 +02002530 * Use the cpu notifier to insure that the cpu slabs are flushed when
2531 * necessary.
2532 */
2533static int slub_cpu_dead(unsigned int cpu)
2534{
2535 struct kmem_cache *s;
2536 unsigned long flags;
2537
2538 mutex_lock(&slab_mutex);
2539 list_for_each_entry(s, &slab_caches, list) {
2540 local_irq_save(flags);
2541 __flush_cpu_slab(s, cpu);
2542 local_irq_restore(flags);
2543 }
2544 mutex_unlock(&slab_mutex);
2545 return 0;
2546}
2547
2548/*
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002549 * Check if the objects in a per cpu structure fit numa
2550 * locality expectations.
2551 */
Christoph Lameter57d437d2012-05-09 10:09:59 -05002552static inline int node_match(struct page *page, int node)
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002553{
2554#ifdef CONFIG_NUMA
Wei Yang6159d0f2018-12-28 00:33:09 -08002555 if (node != NUMA_NO_NODE && page_to_nid(page) != node)
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002556 return 0;
2557#endif
2558 return 1;
2559}
2560
David Rientjes9a02d692014-06-04 16:06:36 -07002561#ifdef CONFIG_SLUB_DEBUG
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002562static int count_free(struct page *page)
2563{
2564 return page->objects - page->inuse;
2565}
2566
David Rientjes9a02d692014-06-04 16:06:36 -07002567static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2568{
2569 return atomic_long_read(&n->total_objects);
2570}
2571#endif /* CONFIG_SLUB_DEBUG */
2572
2573#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002574static unsigned long count_partial(struct kmem_cache_node *n,
2575 int (*get_count)(struct page *))
2576{
2577 unsigned long flags;
2578 unsigned long x = 0;
2579 struct page *page;
2580
2581 spin_lock_irqsave(&n->list_lock, flags);
Tobin C. Harding916ac052019-05-13 17:16:12 -07002582 list_for_each_entry(page, &n->partial, slab_list)
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002583 x += get_count(page);
2584 spin_unlock_irqrestore(&n->list_lock, flags);
2585 return x;
2586}
David Rientjes9a02d692014-06-04 16:06:36 -07002587#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04002588
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002589static noinline void
2590slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2591{
David Rientjes9a02d692014-06-04 16:06:36 -07002592#ifdef CONFIG_SLUB_DEBUG
2593 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
2594 DEFAULT_RATELIMIT_BURST);
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002595 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07002596 struct kmem_cache_node *n;
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002597
David Rientjes9a02d692014-06-04 16:06:36 -07002598 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
2599 return;
2600
Vlastimil Babka5b3810e2016-03-15 14:56:33 -07002601 pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
2602 nid, gfpflags, &gfpflags);
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07002603 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
Fabian Frederickf9f58282014-06-04 16:06:34 -07002604 s->name, s->object_size, s->size, oo_order(s->oo),
2605 oo_order(s->min));
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002606
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05002607 if (oo_order(s->min) > get_order(s->object_size))
Fabian Frederickf9f58282014-06-04 16:06:34 -07002608 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
2609 s->name);
David Rientjesfa5ec8a2009-07-07 00:14:14 -07002610
Christoph Lameterfa45dc22014-08-06 16:04:09 -07002611 for_each_kmem_cache_node(s, node, n) {
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002612 unsigned long nr_slabs;
2613 unsigned long nr_objs;
2614 unsigned long nr_free;
2615
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04002616 nr_free = count_partial(n, count_free);
2617 nr_slabs = node_nr_slabs(n);
2618 nr_objs = node_nr_objs(n);
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002619
Fabian Frederickf9f58282014-06-04 16:06:34 -07002620 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002621 node, nr_slabs, nr_objs, nr_free);
2622 }
David Rientjes9a02d692014-06-04 16:06:36 -07002623#endif
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002624}
2625
Christoph Lameter497b66f2011-08-09 16:12:26 -05002626static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
2627 int node, struct kmem_cache_cpu **pc)
2628{
Christoph Lameter6faa6832012-05-09 10:09:51 -05002629 void *freelist;
Christoph Lameter188fd062012-05-09 10:09:55 -05002630 struct kmem_cache_cpu *c = *pc;
2631 struct page *page;
Christoph Lameter497b66f2011-08-09 16:12:26 -05002632
Matthew Wilcox128227e2018-06-07 17:05:13 -07002633 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
2634
Christoph Lameter188fd062012-05-09 10:09:55 -05002635 freelist = get_partial(s, flags, node, c);
2636
2637 if (freelist)
2638 return freelist;
2639
2640 page = new_slab(s, flags, node);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002641 if (page) {
Christoph Lameter7c8e0182014-06-04 16:07:56 -07002642 c = raw_cpu_ptr(s->cpu_slab);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002643 if (c->page)
2644 flush_slab(s, c);
2645
2646 /*
2647 * No other reference to the page yet so we can
2648 * muck around with it freely without cmpxchg
2649 */
Christoph Lameter6faa6832012-05-09 10:09:51 -05002650 freelist = page->freelist;
Christoph Lameter497b66f2011-08-09 16:12:26 -05002651 page->freelist = NULL;
2652
2653 stat(s, ALLOC_SLAB);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002654 c->page = page;
2655 *pc = c;
Peng Wangedde82b2019-03-05 15:42:00 -08002656 }
Christoph Lameter497b66f2011-08-09 16:12:26 -05002657
Christoph Lameter6faa6832012-05-09 10:09:51 -05002658 return freelist;
Christoph Lameter497b66f2011-08-09 16:12:26 -05002659}
2660
Mel Gorman072bb0a2012-07-31 16:43:58 -07002661static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
2662{
2663 if (unlikely(PageSlabPfmemalloc(page)))
2664 return gfp_pfmemalloc_allowed(gfpflags);
2665
2666 return true;
2667}
2668
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002669/*
Chen Gangd0e0ac92013-07-15 09:05:29 +08002670 * Check the page->freelist of a page and either transfer the freelist to the
2671 * per cpu freelist or deactivate the page.
Christoph Lameter213eeb92011-11-11 14:07:14 -06002672 *
2673 * The page is still frozen if the return value is not NULL.
2674 *
2675 * If this function returns NULL then the page has been unfrozen.
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002676 *
2677 * This function must be called with interrupt disabled.
Christoph Lameter213eeb92011-11-11 14:07:14 -06002678 */
2679static inline void *get_freelist(struct kmem_cache *s, struct page *page)
2680{
2681 struct page new;
2682 unsigned long counters;
2683 void *freelist;
2684
2685 do {
2686 freelist = page->freelist;
2687 counters = page->counters;
Christoph Lameter6faa6832012-05-09 10:09:51 -05002688
Christoph Lameter213eeb92011-11-11 14:07:14 -06002689 new.counters = counters;
Dave Hansena0132ac2014-01-29 14:05:50 -08002690 VM_BUG_ON(!new.frozen);
Christoph Lameter213eeb92011-11-11 14:07:14 -06002691
2692 new.inuse = page->objects;
2693 new.frozen = freelist != NULL;
2694
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002695 } while (!__cmpxchg_double_slab(s, page,
Christoph Lameter213eeb92011-11-11 14:07:14 -06002696 freelist, counters,
2697 NULL, new.counters,
2698 "get_freelist"));
2699
2700 return freelist;
2701}
2702
2703/*
Christoph Lameter894b8782007-05-10 03:15:16 -07002704 * Slow path. The lockless freelist is empty or we need to perform
2705 * debugging duties.
Christoph Lameter81819f02007-05-06 14:49:36 -07002706 *
Christoph Lameter894b8782007-05-10 03:15:16 -07002707 * Processing is still very fast if new objects have been freed to the
2708 * regular freelist. In that case we simply take over the regular freelist
2709 * as the lockless freelist and zap the regular freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07002710 *
Christoph Lameter894b8782007-05-10 03:15:16 -07002711 * If that is not working then we fall back to the partial lists. We take the
2712 * first element of the freelist as the object to allocate now and move the
2713 * rest of the freelist to the lockless freelist.
2714 *
2715 * And if we were unable to get a new slab from the partial slab lists then
Christoph Lameter6446faa2008-02-15 23:45:26 -08002716 * we need to allocate a new slab. This is the slowest path since it involves
2717 * a call to the page allocator and the setup of a new slab.
Christoph Lametera380a3c2015-11-20 15:57:35 -08002718 *
2719 * Version of __slab_alloc to use when we know that interrupts are
2720 * already disabled (which is the case for bulk allocation).
Christoph Lameter81819f02007-05-06 14:49:36 -07002721 */
Christoph Lametera380a3c2015-11-20 15:57:35 -08002722static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03002723 unsigned long addr, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002724{
Christoph Lameter6faa6832012-05-09 10:09:51 -05002725 void *freelist;
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002726 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07002727
Abel Wu9f986d92020-10-13 16:48:43 -07002728 stat(s, ALLOC_SLOWPATH);
2729
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002730 page = c->page;
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002731 if (!page) {
2732 /*
2733 * if the node is not online or has no normal memory, just
2734 * ignore the node constraint
2735 */
2736 if (unlikely(node != NUMA_NO_NODE &&
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08002737 !node_isset(node, slab_nodes)))
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002738 node = NUMA_NO_NODE;
Christoph Lameter81819f02007-05-06 14:49:36 -07002739 goto new_slab;
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002740 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002741redo:
Christoph Lameter6faa6832012-05-09 10:09:51 -05002742
Christoph Lameter57d437d2012-05-09 10:09:59 -05002743 if (unlikely(!node_match(page, node))) {
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002744 /*
2745 * same as above but node_match() being false already
2746 * implies node != NUMA_NO_NODE
2747 */
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08002748 if (!node_isset(node, slab_nodes)) {
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002749 node = NUMA_NO_NODE;
2750 goto redo;
2751 } else {
Joonsoo Kima561ce02014-10-09 15:26:15 -07002752 stat(s, ALLOC_NODE_MISMATCH);
Wei Yangd4ff6d32017-07-06 15:36:25 -07002753 deactivate_slab(s, page, c->freelist, c);
Joonsoo Kima561ce02014-10-09 15:26:15 -07002754 goto new_slab;
2755 }
Christoph Lameterfc59c052011-06-01 12:25:56 -05002756 }
Christoph Lameter6446faa2008-02-15 23:45:26 -08002757
Mel Gorman072bb0a2012-07-31 16:43:58 -07002758 /*
2759 * By rights, we should be searching for a slab page that was
2760 * PFMEMALLOC but right now, we are losing the pfmemalloc
2761 * information when the page leaves the per-cpu allocator
2762 */
2763 if (unlikely(!pfmemalloc_match(page, gfpflags))) {
Wei Yangd4ff6d32017-07-06 15:36:25 -07002764 deactivate_slab(s, page, c->freelist, c);
Mel Gorman072bb0a2012-07-31 16:43:58 -07002765 goto new_slab;
2766 }
2767
Eric Dumazet73736e02011-12-13 04:57:06 +01002768 /* must check again c->freelist in case of cpu migration or IRQ */
Christoph Lameter6faa6832012-05-09 10:09:51 -05002769 freelist = c->freelist;
2770 if (freelist)
Eric Dumazet73736e02011-12-13 04:57:06 +01002771 goto load_freelist;
2772
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002773 freelist = get_freelist(s, page);
Christoph Lameter6446faa2008-02-15 23:45:26 -08002774
Christoph Lameter6faa6832012-05-09 10:09:51 -05002775 if (!freelist) {
Christoph Lameter03e404a2011-06-01 12:25:58 -05002776 c->page = NULL;
2777 stat(s, DEACTIVATE_BYPASS);
Christoph Lameterfc59c052011-06-01 12:25:56 -05002778 goto new_slab;
Christoph Lameter03e404a2011-06-01 12:25:58 -05002779 }
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002780
Christoph Lameter81819f02007-05-06 14:49:36 -07002781 stat(s, ALLOC_REFILL);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08002782
Christoph Lameter894b8782007-05-10 03:15:16 -07002783load_freelist:
Christoph Lameter507effe2012-05-09 10:09:52 -05002784 /*
2785 * freelist is pointing to the list of objects to be used.
2786 * page is pointing to the page from which the objects are obtained.
2787 * That page must be frozen for per cpu allocations to work.
2788 */
Dave Hansena0132ac2014-01-29 14:05:50 -08002789 VM_BUG_ON(!c->page->frozen);
Christoph Lameter6faa6832012-05-09 10:09:51 -05002790 c->freelist = get_freepointer(s, freelist);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002791 c->tid = next_tid(c->tid);
Christoph Lameter6faa6832012-05-09 10:09:51 -05002792 return freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07002793
Christoph Lameter81819f02007-05-06 14:49:36 -07002794new_slab:
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002795
Wei Yanga93cf072017-07-06 15:36:31 -07002796 if (slub_percpu_partial(c)) {
2797 page = c->page = slub_percpu_partial(c);
2798 slub_set_percpu_partial(c, page);
Christoph Lameter49e22582011-08-09 16:12:27 -05002799 stat(s, CPU_PARTIAL_ALLOC);
Christoph Lameter49e22582011-08-09 16:12:27 -05002800 goto redo;
Christoph Lameter81819f02007-05-06 14:49:36 -07002801 }
2802
Christoph Lameter188fd062012-05-09 10:09:55 -05002803 freelist = new_slab_objects(s, gfpflags, node, &c);
Christoph Lameterb811c202007-10-16 23:25:51 -07002804
Christoph Lameterf46974362012-05-09 10:09:54 -05002805 if (unlikely(!freelist)) {
David Rientjes9a02d692014-06-04 16:06:36 -07002806 slab_out_of_memory(s, gfpflags, node);
Christoph Lameterf46974362012-05-09 10:09:54 -05002807 return NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07002808 }
Christoph Lameter894b8782007-05-10 03:15:16 -07002809
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002810 page = c->page;
Christoph Lameter5091b742012-07-31 16:44:00 -07002811 if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
Christoph Lameter81819f02007-05-06 14:49:36 -07002812 goto load_freelist;
Christoph Lameter894b8782007-05-10 03:15:16 -07002813
Christoph Lameter497b66f2011-08-09 16:12:26 -05002814 /* Only entered in the debug case */
Chen Gangd0e0ac92013-07-15 09:05:29 +08002815 if (kmem_cache_debug(s) &&
2816 !alloc_debug_processing(s, page, freelist, addr))
Christoph Lameter497b66f2011-08-09 16:12:26 -05002817 goto new_slab; /* Slab failed checks. Next slab needed */
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002818
Wei Yangd4ff6d32017-07-06 15:36:25 -07002819 deactivate_slab(s, page, get_freepointer(s, freelist), c);
Christoph Lameter6faa6832012-05-09 10:09:51 -05002820 return freelist;
Christoph Lameter894b8782007-05-10 03:15:16 -07002821}
2822
2823/*
Christoph Lametera380a3c2015-11-20 15:57:35 -08002824 * Another one that disabled interrupt and compensates for possible
2825 * cpu changes by refetching the per cpu area pointer.
2826 */
2827static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2828 unsigned long addr, struct kmem_cache_cpu *c)
2829{
2830 void *p;
2831 unsigned long flags;
2832
2833 local_irq_save(flags);
Thomas Gleixner923717c2019-10-15 21:18:12 +02002834#ifdef CONFIG_PREEMPTION
Christoph Lametera380a3c2015-11-20 15:57:35 -08002835 /*
2836 * We may have been preempted and rescheduled on a different
2837 * cpu before disabling interrupts. Need to reload cpu area
2838 * pointer.
2839 */
2840 c = this_cpu_ptr(s->cpu_slab);
2841#endif
2842
2843 p = ___slab_alloc(s, gfpflags, node, addr, c);
2844 local_irq_restore(flags);
2845 return p;
2846}
2847
2848/*
Alexander Potapenko0f181f92019-10-14 14:11:57 -07002849 * If the object has been wiped upon free, make sure it's fully initialized by
2850 * zeroing out freelist pointer.
2851 */
2852static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
2853 void *obj)
2854{
2855 if (unlikely(slab_want_init_on_free(s)) && obj)
Andrey Konovalovce5716c62021-01-23 21:01:38 -08002856 memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
2857 0, sizeof(void *));
Alexander Potapenko0f181f92019-10-14 14:11:57 -07002858}
2859
2860/*
Christoph Lameter894b8782007-05-10 03:15:16 -07002861 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
2862 * have the fastpath folded into their functions. So no function call
2863 * overhead for requests that can be satisfied on the fastpath.
2864 *
2865 * The fastpath works by first checking if the lockless freelist can be used.
2866 * If not then __slab_alloc is called for slow processing.
2867 *
2868 * Otherwise we can simply pick the next object from the lockless free list.
2869 */
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002870static __always_inline void *slab_alloc_node(struct kmem_cache *s,
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002871 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
Christoph Lameter894b8782007-05-10 03:15:16 -07002872{
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08002873 void *object;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002874 struct kmem_cache_cpu *c;
Christoph Lameter57d437d2012-05-09 10:09:59 -05002875 struct page *page;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002876 unsigned long tid;
Roman Gushchin964d4bd2020-08-06 23:20:56 -07002877 struct obj_cgroup *objcg = NULL;
Andrey Konovalovda844b72021-04-29 23:00:06 -07002878 bool init = false;
Christoph Lameter1f842602008-01-07 23:20:30 -08002879
Roman Gushchin964d4bd2020-08-06 23:20:56 -07002880 s = slab_pre_alloc_hook(s, &objcg, 1, gfpflags);
Vladimir Davydov8135be52014-12-12 16:56:38 -08002881 if (!s)
Akinobu Mita773ff602008-12-23 19:37:01 +09002882 return NULL;
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002883
2884 object = kfence_alloc(s, orig_size, gfpflags);
2885 if (unlikely(object))
2886 goto out;
2887
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002888redo:
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002889 /*
2890 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
2891 * enabled. We may switch back and forth between cpus while
2892 * reading from one cpu area. That does not matter as long
2893 * as we end up on the original cpu again when doing the cmpxchg.
Christoph Lameter7cccd80b2013-01-23 21:45:48 +00002894 *
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002895 * We should guarantee that tid and kmem_cache are retrieved on
Thomas Gleixner923717c2019-10-15 21:18:12 +02002896 * the same cpu. It could be different if CONFIG_PREEMPTION so we need
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002897 * to check if it is matched or not.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002898 */
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002899 do {
2900 tid = this_cpu_read(s->cpu_slab->tid);
2901 c = raw_cpu_ptr(s->cpu_slab);
Thomas Gleixner923717c2019-10-15 21:18:12 +02002902 } while (IS_ENABLED(CONFIG_PREEMPTION) &&
Mark Rutland859b7a02015-03-25 15:55:23 -07002903 unlikely(tid != READ_ONCE(c->tid)));
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002904
2905 /*
2906 * Irqless object alloc/free algorithm used here depends on sequence
2907 * of fetching cpu_slab's data. tid should be fetched before anything
2908 * on c to guarantee that object and page associated with previous tid
2909 * won't be used with current tid. If we fetch tid first, object and
2910 * page could be one associated with next tid and our alloc/free
2911 * request will be failed. In this case, we will retry. So, no problem.
2912 */
2913 barrier();
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002914
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002915 /*
2916 * The transaction ids are globally unique per cpu and per operation on
2917 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
2918 * occurs on the right processor and that there was no operation on the
2919 * linked list in between.
2920 */
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002921
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06002922 object = c->freelist;
Christoph Lameter57d437d2012-05-09 10:09:59 -05002923 page = c->page;
Laurent Dufour22e46632020-11-13 22:51:53 -08002924 if (unlikely(!object || !page || !node_match(page, node))) {
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002925 object = __slab_alloc(s, gfpflags, node, addr, c);
Dave Hansen8eae1492014-06-04 16:06:37 -07002926 } else {
Eric Dumazet0ad95002011-12-16 16:25:34 +01002927 void *next_object = get_freepointer_safe(s, object);
2928
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002929 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002930 * The cmpxchg will only match if there was no additional
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002931 * operation and if we are on the right processor.
2932 *
Chen Gangd0e0ac92013-07-15 09:05:29 +08002933 * The cmpxchg does the following atomically (without lock
2934 * semantics!)
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002935 * 1. Relocate first pointer to the current per cpu area.
2936 * 2. Verify that tid and freelist have not been changed
2937 * 3. If they were not changed replace tid and freelist
2938 *
Chen Gangd0e0ac92013-07-15 09:05:29 +08002939 * Since this is without lock semantics the protection is only
2940 * against code executing on this cpu *not* from access by
2941 * other cpus.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002942 */
Christoph Lameter933393f2011-12-22 11:58:51 -06002943 if (unlikely(!this_cpu_cmpxchg_double(
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002944 s->cpu_slab->freelist, s->cpu_slab->tid,
2945 object, tid,
Eric Dumazet0ad95002011-12-16 16:25:34 +01002946 next_object, next_tid(tid)))) {
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002947
2948 note_cmpxchg_failure("slab_alloc", s, tid);
2949 goto redo;
2950 }
Eric Dumazet0ad95002011-12-16 16:25:34 +01002951 prefetch_freepointer(s, next_object);
Christoph Lameter84e554e62009-12-18 16:26:23 -06002952 stat(s, ALLOC_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07002953 }
Alexander Potapenko0f181f92019-10-14 14:11:57 -07002954
Andrey Konovalovce5716c62021-01-23 21:01:38 -08002955 maybe_wipe_obj_freeptr(s, object);
Andrey Konovalovda844b72021-04-29 23:00:06 -07002956 init = slab_want_init_on_alloc(gfpflags, s);
Christoph Lameterd07dbea2007-07-17 04:03:23 -07002957
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002958out:
Andrey Konovalovda844b72021-04-29 23:00:06 -07002959 slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init);
Vegard Nossum5a896d92008-04-04 00:54:48 +02002960
Christoph Lameter894b8782007-05-10 03:15:16 -07002961 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07002962}
2963
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002964static __always_inline void *slab_alloc(struct kmem_cache *s,
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002965 gfp_t gfpflags, unsigned long addr, size_t orig_size)
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002966{
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002967 return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr, orig_size);
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002968}
2969
Christoph Lameter81819f02007-05-06 14:49:36 -07002970void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
2971{
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002972 void *ret = slab_alloc(s, gfpflags, _RET_IP_, s->object_size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002973
Chen Gangd0e0ac92013-07-15 09:05:29 +08002974 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
2975 s->size, gfpflags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002976
2977 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07002978}
2979EXPORT_SYMBOL(kmem_cache_alloc);
2980
Li Zefan0f24f122009-12-11 15:45:30 +08002981#ifdef CONFIG_TRACING
Richard Kennedy4a923792010-10-21 10:29:19 +01002982void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002983{
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002984 void *ret = slab_alloc(s, gfpflags, _RET_IP_, size);
Richard Kennedy4a923792010-10-21 10:29:19 +01002985 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
Andrey Konovalov0116523c2018-12-28 00:29:37 -08002986 ret = kasan_kmalloc(s, ret, size, gfpflags);
Richard Kennedy4a923792010-10-21 10:29:19 +01002987 return ret;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002988}
Richard Kennedy4a923792010-10-21 10:29:19 +01002989EXPORT_SYMBOL(kmem_cache_alloc_trace);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002990#endif
2991
Christoph Lameter81819f02007-05-06 14:49:36 -07002992#ifdef CONFIG_NUMA
2993void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
2994{
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08002995 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, s->object_size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002996
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02002997 trace_kmem_cache_alloc_node(_RET_IP_, ret,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05002998 s->object_size, s->size, gfpflags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002999
3000 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003001}
3002EXPORT_SYMBOL(kmem_cache_alloc_node);
Christoph Lameter81819f02007-05-06 14:49:36 -07003003
Li Zefan0f24f122009-12-11 15:45:30 +08003004#ifdef CONFIG_TRACING
Richard Kennedy4a923792010-10-21 10:29:19 +01003005void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003006 gfp_t gfpflags,
Richard Kennedy4a923792010-10-21 10:29:19 +01003007 int node, size_t size)
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003008{
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003009 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, size);
Richard Kennedy4a923792010-10-21 10:29:19 +01003010
3011 trace_kmalloc_node(_RET_IP_, ret,
3012 size, s->size, gfpflags, node);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08003013
Andrey Konovalov0116523c2018-12-28 00:29:37 -08003014 ret = kasan_kmalloc(s, ret, size, gfpflags);
Richard Kennedy4a923792010-10-21 10:29:19 +01003015 return ret;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003016}
Richard Kennedy4a923792010-10-21 10:29:19 +01003017EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003018#endif
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07003019#endif /* CONFIG_NUMA */
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003020
Christoph Lameter81819f02007-05-06 14:49:36 -07003021/*
Kim Phillips94e4d712015-02-10 14:09:37 -08003022 * Slow path handling. This may still be called frequently since objects
Christoph Lameter894b8782007-05-10 03:15:16 -07003023 * have a longer lifetime than the cpu slabs in most processing loads.
Christoph Lameter81819f02007-05-06 14:49:36 -07003024 *
Christoph Lameter894b8782007-05-10 03:15:16 -07003025 * So we still attempt to reduce cache line usage. Just take the slab
3026 * lock and free the item. If there is no additional partial page
3027 * handling required then we can return immediately.
Christoph Lameter81819f02007-05-06 14:49:36 -07003028 */
Christoph Lameter894b8782007-05-10 03:15:16 -07003029static void __slab_free(struct kmem_cache *s, struct page *page,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003030 void *head, void *tail, int cnt,
3031 unsigned long addr)
3032
Christoph Lameter81819f02007-05-06 14:49:36 -07003033{
3034 void *prior;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003035 int was_frozen;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003036 struct page new;
3037 unsigned long counters;
3038 struct kmem_cache_node *n = NULL;
Kees Cook3f649ab2020-06-03 13:09:38 -07003039 unsigned long flags;
Christoph Lameter81819f02007-05-06 14:49:36 -07003040
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003041 stat(s, FREE_SLOWPATH);
Christoph Lameter81819f02007-05-06 14:49:36 -07003042
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003043 if (kfence_free(head))
3044 return;
3045
Christoph Lameter19c7ff92012-05-30 12:54:46 -05003046 if (kmem_cache_debug(s) &&
Laura Abbott282acb42016-03-15 14:54:59 -07003047 !free_debug_processing(s, page, head, tail, cnt, addr))
Christoph Lameter80f08c12011-06-01 12:25:55 -05003048 return;
Christoph Lameter6446faa2008-02-15 23:45:26 -08003049
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003050 do {
Joonsoo Kim837d6782012-08-16 00:02:40 +09003051 if (unlikely(n)) {
3052 spin_unlock_irqrestore(&n->list_lock, flags);
3053 n = NULL;
3054 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003055 prior = page->freelist;
3056 counters = page->counters;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003057 set_freepointer(s, tail, prior);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003058 new.counters = counters;
3059 was_frozen = new.frozen;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003060 new.inuse -= cnt;
Joonsoo Kim837d6782012-08-16 00:02:40 +09003061 if ((!new.inuse || !prior) && !was_frozen) {
Christoph Lameter49e22582011-08-09 16:12:27 -05003062
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003063 if (kmem_cache_has_cpu_partial(s) && !prior) {
Christoph Lameter49e22582011-08-09 16:12:27 -05003064
3065 /*
Chen Gangd0e0ac92013-07-15 09:05:29 +08003066 * Slab was on no list before and will be
3067 * partially empty
3068 * We can defer the list move and instead
3069 * freeze it.
Christoph Lameter49e22582011-08-09 16:12:27 -05003070 */
3071 new.frozen = 1;
3072
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003073 } else { /* Needs to be taken off a list */
Christoph Lameter49e22582011-08-09 16:12:27 -05003074
LQYMGTb455def2014-12-10 15:42:13 -08003075 n = get_node(s, page_to_nid(page));
Christoph Lameter49e22582011-08-09 16:12:27 -05003076 /*
3077 * Speculatively acquire the list_lock.
3078 * If the cmpxchg does not succeed then we may
3079 * drop the list_lock without any processing.
3080 *
3081 * Otherwise the list_lock will synchronize with
3082 * other processors updating the list of slabs.
3083 */
3084 spin_lock_irqsave(&n->list_lock, flags);
3085
3086 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003087 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003088
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003089 } while (!cmpxchg_double_slab(s, page,
3090 prior, counters,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003091 head, new.counters,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003092 "__slab_free"));
Christoph Lameter81819f02007-05-06 14:49:36 -07003093
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003094 if (likely(!n)) {
Christoph Lameter49e22582011-08-09 16:12:27 -05003095
Abel Wuc270cf32020-10-13 16:48:40 -07003096 if (likely(was_frozen)) {
3097 /*
3098 * The list lock was not taken therefore no list
3099 * activity can be necessary.
3100 */
3101 stat(s, FREE_FROZEN);
3102 } else if (new.frozen) {
3103 /*
3104 * If we just froze the page then put it onto the
3105 * per cpu partial list.
3106 */
Christoph Lameter49e22582011-08-09 16:12:27 -05003107 put_cpu_partial(s, page, 1);
Alex Shi8028dce2012-02-03 23:34:56 +08003108 stat(s, CPU_PARTIAL_FREE);
3109 }
Abel Wuc270cf32020-10-13 16:48:40 -07003110
LQYMGTb455def2014-12-10 15:42:13 -08003111 return;
3112 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003113
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07003114 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
Joonsoo Kim837d6782012-08-16 00:02:40 +09003115 goto slab_empty;
Christoph Lameter81819f02007-05-06 14:49:36 -07003116
Joonsoo Kim837d6782012-08-16 00:02:40 +09003117 /*
3118 * Objects left in the slab. If it was not on the partial list before
3119 * then add it.
3120 */
Joonsoo Kim345c9052013-06-19 14:05:52 +09003121 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
Liu Xianga4d3f892019-05-13 17:16:22 -07003122 remove_full(s, n, page);
Joonsoo Kim837d6782012-08-16 00:02:40 +09003123 add_partial(n, page, DEACTIVATE_TO_TAIL);
3124 stat(s, FREE_ADD_PARTIAL);
Christoph Lameter81819f02007-05-06 14:49:36 -07003125 }
Christoph Lameter80f08c12011-06-01 12:25:55 -05003126 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07003127 return;
3128
3129slab_empty:
Christoph Lametera973e9d2008-03-01 13:40:44 -08003130 if (prior) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003131 /*
Christoph Lameter6fbabb22011-08-08 11:16:56 -05003132 * Slab on the partial list.
Christoph Lameter81819f02007-05-06 14:49:36 -07003133 */
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05003134 remove_partial(n, page);
Christoph Lameter84e554e62009-12-18 16:26:23 -06003135 stat(s, FREE_REMOVE_PARTIAL);
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003136 } else {
Christoph Lameter6fbabb22011-08-08 11:16:56 -05003137 /* Slab must be on the full list */
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003138 remove_full(s, n, page);
3139 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003140
Christoph Lameter80f08c12011-06-01 12:25:55 -05003141 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter84e554e62009-12-18 16:26:23 -06003142 stat(s, FREE_SLAB);
Christoph Lameter81819f02007-05-06 14:49:36 -07003143 discard_slab(s, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07003144}
3145
Christoph Lameter894b8782007-05-10 03:15:16 -07003146/*
3147 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
3148 * can perform fastpath freeing without additional function calls.
3149 *
3150 * The fastpath is only possible if we are freeing to the current cpu slab
3151 * of this processor. This typically the case if we have just allocated
3152 * the item before.
3153 *
3154 * If fastpath is not possible then fall back to __slab_free where we deal
3155 * with all sorts of special processing.
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003156 *
3157 * Bulk free of a freelist with several objects (all pointing to the
3158 * same page) possible by specifying head and tail ptr, plus objects
3159 * count (cnt). Bulk free indicated by tail pointer being set.
Christoph Lameter894b8782007-05-10 03:15:16 -07003160 */
Alexander Potapenko80a92012016-07-28 15:49:07 -07003161static __always_inline void do_slab_free(struct kmem_cache *s,
3162 struct page *page, void *head, void *tail,
3163 int cnt, unsigned long addr)
Christoph Lameter894b8782007-05-10 03:15:16 -07003164{
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003165 void *tail_obj = tail ? : head;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003166 struct kmem_cache_cpu *c;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003167 unsigned long tid;
Roman Gushchin964d4bd2020-08-06 23:20:56 -07003168
Bharata B Raod1b2cf62020-10-13 16:53:09 -07003169 memcg_slab_free_hook(s, &head, 1);
Christoph Lametera24c5a02011-03-15 12:45:21 -05003170redo:
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003171 /*
3172 * Determine the currently cpus per cpu slab.
3173 * The cpu may change afterward. However that does not matter since
3174 * data is retrieved via this pointer. If we are on the same cpu
Jesper Dangaard Brouer2ae44002015-09-04 15:45:31 -07003175 * during the cmpxchg then the free will succeed.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003176 */
Joonsoo Kim9aabf812015-02-10 14:09:32 -08003177 do {
3178 tid = this_cpu_read(s->cpu_slab->tid);
3179 c = raw_cpu_ptr(s->cpu_slab);
Thomas Gleixner923717c2019-10-15 21:18:12 +02003180 } while (IS_ENABLED(CONFIG_PREEMPTION) &&
Mark Rutland859b7a02015-03-25 15:55:23 -07003181 unlikely(tid != READ_ONCE(c->tid)));
Christoph Lameterc016b0b2010-08-20 12:37:16 -05003182
Joonsoo Kim9aabf812015-02-10 14:09:32 -08003183 /* Same with comment on barrier() in slab_alloc_node() */
3184 barrier();
Christoph Lameterc016b0b2010-08-20 12:37:16 -05003185
Christoph Lameter442b06b2011-05-17 16:29:31 -05003186 if (likely(page == c->page)) {
Linus Torvalds50761902020-03-17 11:04:09 -07003187 void **freelist = READ_ONCE(c->freelist);
3188
3189 set_freepointer(s, tail_obj, freelist);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003190
Christoph Lameter933393f2011-12-22 11:58:51 -06003191 if (unlikely(!this_cpu_cmpxchg_double(
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003192 s->cpu_slab->freelist, s->cpu_slab->tid,
Linus Torvalds50761902020-03-17 11:04:09 -07003193 freelist, tid,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003194 head, next_tid(tid)))) {
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003195
3196 note_cmpxchg_failure("slab_free", s, tid);
3197 goto redo;
3198 }
Christoph Lameter84e554e62009-12-18 16:26:23 -06003199 stat(s, FREE_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07003200 } else
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003201 __slab_free(s, page, head, tail_obj, cnt, addr);
Christoph Lameter894b8782007-05-10 03:15:16 -07003202
Christoph Lameter894b8782007-05-10 03:15:16 -07003203}
3204
Alexander Potapenko80a92012016-07-28 15:49:07 -07003205static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
3206 void *head, void *tail, int cnt,
3207 unsigned long addr)
3208{
Alexander Potapenko80a92012016-07-28 15:49:07 -07003209 /*
Andrey Konovalovc3895392018-04-10 16:30:31 -07003210 * With KASAN enabled slab_free_freelist_hook modifies the freelist
3211 * to remove objects, whose reuse must be delayed.
Alexander Potapenko80a92012016-07-28 15:49:07 -07003212 */
Andrey Konovalovc3895392018-04-10 16:30:31 -07003213 if (slab_free_freelist_hook(s, &head, &tail))
3214 do_slab_free(s, page, head, tail, cnt, addr);
Alexander Potapenko80a92012016-07-28 15:49:07 -07003215}
3216
Andrey Konovalov2bd926b2018-12-28 00:29:53 -08003217#ifdef CONFIG_KASAN_GENERIC
Alexander Potapenko80a92012016-07-28 15:49:07 -07003218void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
3219{
3220 do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
3221}
3222#endif
3223
Christoph Lameter81819f02007-05-06 14:49:36 -07003224void kmem_cache_free(struct kmem_cache *s, void *x)
3225{
Glauber Costab9ce5ef2012-12-18 14:22:46 -08003226 s = cache_from_obj(s, x);
3227 if (!s)
Christoph Lameter79576102012-09-04 23:06:14 +00003228 return;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003229 slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
Jacob Wen3544de8e2021-02-24 12:00:55 -08003230 trace_kmem_cache_free(_RET_IP_, x, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07003231}
3232EXPORT_SYMBOL(kmem_cache_free);
3233
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003234struct detached_freelist {
3235 struct page *page;
3236 void *tail;
3237 void *freelist;
3238 int cnt;
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003239 struct kmem_cache *s;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003240};
3241
Shakeel Butt1ed7ce52021-08-13 16:54:31 -07003242static inline void free_nonslab_page(struct page *page, void *object)
Shakeel Buttf227f0f2021-07-29 14:53:50 -07003243{
3244 unsigned int order = compound_order(page);
3245
3246 VM_BUG_ON_PAGE(!PageCompound(page), page);
Shakeel Butt1ed7ce52021-08-13 16:54:31 -07003247 kfree_hook(object);
Shakeel Buttf227f0f2021-07-29 14:53:50 -07003248 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, -(PAGE_SIZE << order));
3249 __free_pages(page, order);
3250}
3251
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003252/*
3253 * This function progressively scans the array with free objects (with
3254 * a limited look ahead) and extract objects belonging to the same
3255 * page. It builds a detached freelist directly within the given
3256 * page/objects. This can happen without any need for
3257 * synchronization, because the objects are owned by running process.
3258 * The freelist is build up as a single linked list in the objects.
3259 * The idea is, that this detached freelist can then be bulk
3260 * transferred to the real freelist(s), but only requiring a single
3261 * synchronization primitive. Look ahead in the array is limited due
3262 * to performance reasons.
3263 */
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003264static inline
3265int build_detached_freelist(struct kmem_cache *s, size_t size,
3266 void **p, struct detached_freelist *df)
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003267{
3268 size_t first_skipped_index = 0;
3269 int lookahead = 3;
3270 void *object;
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003271 struct page *page;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003272
3273 /* Always re-init detached_freelist */
3274 df->page = NULL;
3275
3276 do {
3277 object = p[--size];
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003278 /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003279 } while (!object && size);
3280
3281 if (!object)
3282 return 0;
3283
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003284 page = virt_to_head_page(object);
3285 if (!s) {
3286 /* Handle kalloc'ed objects */
3287 if (unlikely(!PageSlab(page))) {
Shakeel Butt1ed7ce52021-08-13 16:54:31 -07003288 free_nonslab_page(page, object);
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003289 p[size] = NULL; /* mark object processed */
3290 return size;
3291 }
3292 /* Derive kmem_cache from object */
3293 df->s = page->slab_cache;
3294 } else {
3295 df->s = cache_from_obj(s, object); /* Support for memcg */
3296 }
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003297
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003298 if (is_kfence_address(object)) {
Andrey Konovalovd57a9642021-04-29 23:00:09 -07003299 slab_free_hook(df->s, object, false);
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003300 __kfence_free(object);
3301 p[size] = NULL; /* mark object processed */
3302 return size;
3303 }
3304
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003305 /* Start new detached freelist */
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003306 df->page = page;
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003307 set_freepointer(df->s, object, NULL);
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003308 df->tail = object;
3309 df->freelist = object;
3310 p[size] = NULL; /* mark object processed */
3311 df->cnt = 1;
3312
3313 while (size) {
3314 object = p[--size];
3315 if (!object)
3316 continue; /* Skip processed objects */
3317
3318 /* df->page is always set at this point */
3319 if (df->page == virt_to_head_page(object)) {
3320 /* Opportunity build freelist */
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003321 set_freepointer(df->s, object, df->freelist);
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003322 df->freelist = object;
3323 df->cnt++;
3324 p[size] = NULL; /* mark object processed */
3325
3326 continue;
3327 }
3328
3329 /* Limit look ahead search */
3330 if (!--lookahead)
3331 break;
3332
3333 if (!first_skipped_index)
3334 first_skipped_index = size + 1;
3335 }
3336
3337 return first_skipped_index;
3338}
3339
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003340/* Note that interrupts must be enabled when calling this function. */
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003341void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
Christoph Lameter484748f2015-09-04 15:45:34 -07003342{
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003343 if (WARN_ON(!size))
3344 return;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07003345
Bharata B Raod1b2cf62020-10-13 16:53:09 -07003346 memcg_slab_free_hook(s, p, size);
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003347 do {
3348 struct detached_freelist df;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07003349
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003350 size = build_detached_freelist(s, size, p, &df);
Arnd Bergmann84582c82016-12-12 16:41:35 -08003351 if (!df.page)
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003352 continue;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07003353
Zhiyuan Dai457c82c2021-02-24 12:01:26 -08003354 slab_free(df.s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_);
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003355 } while (likely(size));
Christoph Lameter484748f2015-09-04 15:45:34 -07003356}
3357EXPORT_SYMBOL(kmem_cache_free_bulk);
3358
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003359/* Note that interrupts must be enabled when calling this function. */
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08003360int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3361 void **p)
Christoph Lameter484748f2015-09-04 15:45:34 -07003362{
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003363 struct kmem_cache_cpu *c;
3364 int i;
Roman Gushchin964d4bd2020-08-06 23:20:56 -07003365 struct obj_cgroup *objcg = NULL;
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003366
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08003367 /* memcg and kmem_cache debug support */
Roman Gushchin964d4bd2020-08-06 23:20:56 -07003368 s = slab_pre_alloc_hook(s, &objcg, size, flags);
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08003369 if (unlikely(!s))
3370 return false;
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003371 /*
3372 * Drain objects in the per cpu slab, while disabling local
3373 * IRQs, which protects against PREEMPT and interrupts
3374 * handlers invoking normal fastpath.
3375 */
3376 local_irq_disable();
3377 c = this_cpu_ptr(s->cpu_slab);
3378
3379 for (i = 0; i < size; i++) {
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003380 void *object = kfence_alloc(s, s->object_size, flags);
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003381
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003382 if (unlikely(object)) {
3383 p[i] = object;
3384 continue;
3385 }
3386
3387 object = c->freelist;
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003388 if (unlikely(!object)) {
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003389 /*
Jann Hornfd4d9c72020-03-17 01:28:45 +01003390 * We may have removed an object from c->freelist using
3391 * the fastpath in the previous iteration; in that case,
3392 * c->tid has not been bumped yet.
3393 * Since ___slab_alloc() may reenable interrupts while
3394 * allocating memory, we should bump c->tid now.
3395 */
3396 c->tid = next_tid(c->tid);
3397
3398 /*
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003399 * Invoking slow path likely have side-effect
3400 * of re-populating per CPU c->freelist
3401 */
Christoph Lameter87098372015-11-20 15:57:38 -08003402 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003403 _RET_IP_, c);
Christoph Lameter87098372015-11-20 15:57:38 -08003404 if (unlikely(!p[i]))
3405 goto error;
3406
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003407 c = this_cpu_ptr(s->cpu_slab);
Alexander Potapenko0f181f92019-10-14 14:11:57 -07003408 maybe_wipe_obj_freeptr(s, p[i]);
3409
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003410 continue; /* goto for-loop */
3411 }
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003412 c->freelist = get_freepointer(s, object);
3413 p[i] = object;
Alexander Potapenko0f181f92019-10-14 14:11:57 -07003414 maybe_wipe_obj_freeptr(s, p[i]);
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003415 }
3416 c->tid = next_tid(c->tid);
3417 local_irq_enable();
3418
Andrey Konovalovda844b72021-04-29 23:00:06 -07003419 /*
3420 * memcg and kmem_cache debug support and memory initialization.
3421 * Done outside of the IRQ disabled fastpath loop.
3422 */
3423 slab_post_alloc_hook(s, objcg, flags, size, p,
3424 slab_want_init_on_alloc(flags, s));
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08003425 return i;
Christoph Lameter87098372015-11-20 15:57:38 -08003426error:
Christoph Lameter87098372015-11-20 15:57:38 -08003427 local_irq_enable();
Andrey Konovalovda844b72021-04-29 23:00:06 -07003428 slab_post_alloc_hook(s, objcg, flags, i, p, false);
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08003429 __kmem_cache_free_bulk(s, i, p);
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08003430 return 0;
Christoph Lameter484748f2015-09-04 15:45:34 -07003431}
3432EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3433
3434
Christoph Lameter81819f02007-05-06 14:49:36 -07003435/*
Christoph Lameter672bba32007-05-09 02:32:39 -07003436 * Object placement in a slab is made very easy because we always start at
3437 * offset 0. If we tune the size of the object to the alignment then we can
3438 * get the required alignment by putting one properly sized object after
3439 * another.
Christoph Lameter81819f02007-05-06 14:49:36 -07003440 *
3441 * Notice that the allocation order determines the sizes of the per cpu
3442 * caches. Each processor has always one slab available for allocations.
3443 * Increasing the allocation order reduces the number of times that slabs
Christoph Lameter672bba32007-05-09 02:32:39 -07003444 * must be moved on and off the partial lists and is therefore a factor in
Christoph Lameter81819f02007-05-06 14:49:36 -07003445 * locking overhead.
Christoph Lameter81819f02007-05-06 14:49:36 -07003446 */
3447
3448/*
Ingo Molnarf0953a12021-05-06 18:06:47 -07003449 * Minimum / Maximum order of slab pages. This influences locking overhead
Christoph Lameter81819f02007-05-06 14:49:36 -07003450 * and slab fragmentation. A higher order reduces the number of partial slabs
3451 * and increases the number of allocations possible without having to
3452 * take the list_lock.
3453 */
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003454static unsigned int slub_min_order;
3455static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
3456static unsigned int slub_min_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07003457
3458/*
Christoph Lameter81819f02007-05-06 14:49:36 -07003459 * Calculate the order of allocation given an slab object size.
3460 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003461 * The order of allocation has significant impact on performance and other
3462 * system components. Generally order 0 allocations should be preferred since
3463 * order 0 does not cause fragmentation in the page allocator. Larger objects
3464 * be problematic to put into order 0 slabs because there may be too much
Christoph Lameterc124f5b2008-04-14 19:13:29 +03003465 * unused space left. We go to a higher order if more than 1/16th of the slab
Christoph Lameter672bba32007-05-09 02:32:39 -07003466 * would be wasted.
Christoph Lameter81819f02007-05-06 14:49:36 -07003467 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003468 * In order to reach satisfactory performance we must ensure that a minimum
3469 * number of objects is in one slab. Otherwise we may generate too much
3470 * activity on the partial lists which requires taking the list_lock. This is
3471 * less a concern for large slabs though which are rarely used.
Christoph Lameter81819f02007-05-06 14:49:36 -07003472 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003473 * slub_max_order specifies the order where we begin to stop considering the
3474 * number of objects in a slab as critical. If we reach slub_max_order then
3475 * we try to keep the page order as low as possible. So we accept more waste
3476 * of space in favor of a small page order.
3477 *
3478 * Higher order allocations also allow the placement of more objects in a
3479 * slab and thereby reduce object handling overhead. If the user has
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -07003480 * requested a higher minimum order then we start with that one instead of
Christoph Lameter672bba32007-05-09 02:32:39 -07003481 * the smallest order which will fit the object.
Christoph Lameter81819f02007-05-06 14:49:36 -07003482 */
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003483static inline unsigned int slab_order(unsigned int size,
3484 unsigned int min_objects, unsigned int max_order,
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003485 unsigned int fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07003486{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003487 unsigned int min_order = slub_min_order;
3488 unsigned int order;
Christoph Lameter81819f02007-05-06 14:49:36 -07003489
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003490 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +04003491 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
Christoph Lameter39b26462008-04-14 19:11:30 +03003492
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003493 for (order = max(min_order, (unsigned int)get_order(min_objects * size));
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003494 order <= max_order; order++) {
3495
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003496 unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
3497 unsigned int rem;
Christoph Lameter81819f02007-05-06 14:49:36 -07003498
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003499 rem = slab_size % size;
Christoph Lameter81819f02007-05-06 14:49:36 -07003500
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003501 if (rem <= slab_size / fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07003502 break;
Christoph Lameter81819f02007-05-06 14:49:36 -07003503 }
Christoph Lameter672bba32007-05-09 02:32:39 -07003504
Christoph Lameter81819f02007-05-06 14:49:36 -07003505 return order;
3506}
3507
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003508static inline int calculate_order(unsigned int size)
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003509{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003510 unsigned int order;
3511 unsigned int min_objects;
3512 unsigned int max_objects;
Vlastimil Babka32862222021-02-09 13:42:32 -08003513 unsigned int nr_cpus;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003514
3515 /*
3516 * Attempt to find best configuration for a slab. This
3517 * works by first attempting to generate a layout with
3518 * the best configuration and backing off gradually.
3519 *
Wei Yang422ff4d2015-11-05 18:45:46 -08003520 * First we increase the acceptable waste in a slab. Then
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003521 * we reduce the minimum objects required in a slab.
3522 */
3523 min_objects = slub_min_objects;
Vlastimil Babka32862222021-02-09 13:42:32 -08003524 if (!min_objects) {
3525 /*
3526 * Some architectures will only update present cpus when
3527 * onlining them, so don't trust the number if it's just 1. But
3528 * we also don't want to use nr_cpu_ids always, as on some other
3529 * architectures, there can be many possible cpus, but never
3530 * onlined. Here we compromise between trying to avoid too high
3531 * order on systems that appear larger than they are, and too
3532 * low order on systems that appear smaller than they are.
3533 */
3534 nr_cpus = num_present_cpus();
3535 if (nr_cpus <= 1)
3536 nr_cpus = nr_cpu_ids;
3537 min_objects = 4 * (fls(nr_cpus) + 1);
3538 }
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003539 max_objects = order_objects(slub_max_order, size);
Zhang Yanmine8120ff2009-02-12 18:00:17 +02003540 min_objects = min(min_objects, max_objects);
3541
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003542 while (min_objects > 1) {
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003543 unsigned int fraction;
3544
Christoph Lameterc124f5b2008-04-14 19:13:29 +03003545 fraction = 16;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003546 while (fraction >= 4) {
3547 order = slab_order(size, min_objects,
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003548 slub_max_order, fraction);
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003549 if (order <= slub_max_order)
3550 return order;
3551 fraction /= 2;
3552 }
Amerigo Wang5086c389c2009-08-19 21:44:13 +03003553 min_objects--;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003554 }
3555
3556 /*
3557 * We were unable to place multiple objects in a slab. Now
3558 * lets see if we can place a single object there.
3559 */
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003560 order = slab_order(size, 1, slub_max_order, 1);
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003561 if (order <= slub_max_order)
3562 return order;
3563
3564 /*
3565 * Doh this slab cannot be placed using slub_max_order.
3566 */
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003567 order = slab_order(size, 1, MAX_ORDER, 1);
David Rientjes818cf592009-04-23 09:58:22 +03003568 if (order < MAX_ORDER)
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003569 return order;
3570 return -ENOSYS;
3571}
3572
Pekka Enberg5595cff2008-08-05 09:28:47 +03003573static void
Joonsoo Kim40534972012-05-11 00:50:47 +09003574init_kmem_cache_node(struct kmem_cache_node *n)
Christoph Lameter81819f02007-05-06 14:49:36 -07003575{
3576 n->nr_partial = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07003577 spin_lock_init(&n->list_lock);
3578 INIT_LIST_HEAD(&n->partial);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003579#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter0f389ec2008-04-14 18:53:02 +03003580 atomic_long_set(&n->nr_slabs, 0);
Salman Qazi02b71b72008-09-11 12:25:41 -07003581 atomic_long_set(&n->total_objects, 0);
Christoph Lameter643b1132007-05-06 14:49:42 -07003582 INIT_LIST_HEAD(&n->full);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003583#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003584}
3585
Christoph Lameter55136592010-08-20 12:37:13 -05003586static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003587{
Christoph Lameter6c182dc2010-08-20 12:37:14 -05003588 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
Christoph Lameter95a05b42013-01-10 19:14:19 +00003589 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003590
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003591 /*
Chris Metcalfd4d84fe2011-06-02 10:19:41 -04003592 * Must align to double word boundary for the double cmpxchg
3593 * instructions to work; see __pcpu_double_call_return_bool().
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003594 */
Chris Metcalfd4d84fe2011-06-02 10:19:41 -04003595 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
3596 2 * sizeof(void *));
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003597
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003598 if (!s->cpu_slab)
3599 return 0;
3600
3601 init_kmem_cache_cpus(s);
3602
3603 return 1;
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003604}
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003605
Christoph Lameter51df1142010-08-20 12:37:15 -05003606static struct kmem_cache *kmem_cache_node;
3607
Christoph Lameter81819f02007-05-06 14:49:36 -07003608/*
3609 * No kmalloc_node yet so do it by hand. We know that this is the first
3610 * slab on the node for this slabcache. There are no concurrent accesses
3611 * possible.
3612 *
Zhi Yong Wu721ae222013-11-08 20:47:37 +08003613 * Note that this function only works on the kmem_cache_node
3614 * when allocating for the kmem_cache_node. This is used for bootstrapping
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003615 * memory on a fresh node that has no slab structures yet.
Christoph Lameter81819f02007-05-06 14:49:36 -07003616 */
Christoph Lameter55136592010-08-20 12:37:13 -05003617static void early_kmem_cache_node_alloc(int node)
Christoph Lameter81819f02007-05-06 14:49:36 -07003618{
3619 struct page *page;
3620 struct kmem_cache_node *n;
3621
Christoph Lameter51df1142010-08-20 12:37:15 -05003622 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
Christoph Lameter81819f02007-05-06 14:49:36 -07003623
Christoph Lameter51df1142010-08-20 12:37:15 -05003624 page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
Christoph Lameter81819f02007-05-06 14:49:36 -07003625
3626 BUG_ON(!page);
Christoph Lametera2f92ee2007-08-22 14:01:57 -07003627 if (page_to_nid(page) != node) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07003628 pr_err("SLUB: Unable to allocate memory from node %d\n", node);
3629 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
Christoph Lametera2f92ee2007-08-22 14:01:57 -07003630 }
3631
Christoph Lameter81819f02007-05-06 14:49:36 -07003632 n = page->freelist;
3633 BUG_ON(!n);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003634#ifdef CONFIG_SLUB_DEBUG
Christoph Lameterf7cb1932010-09-29 07:15:01 -05003635 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
Christoph Lameter51df1142010-08-20 12:37:15 -05003636 init_tracking(kmem_cache_node, n);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003637#endif
Andrey Konovalovda844b72021-04-29 23:00:06 -07003638 n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
Andrey Konovalov12b22382018-12-28 00:29:41 -08003639 page->freelist = get_freepointer(kmem_cache_node, n);
3640 page->inuse = 1;
3641 page->frozen = 0;
3642 kmem_cache_node->node[node] = n;
Joonsoo Kim40534972012-05-11 00:50:47 +09003643 init_kmem_cache_node(n);
Christoph Lameter51df1142010-08-20 12:37:15 -05003644 inc_slabs_node(kmem_cache_node, node, page->objects);
Christoph Lameter6446faa2008-02-15 23:45:26 -08003645
Dave Hansen67b6c902014-01-24 07:20:23 -08003646 /*
Steven Rostedt1e4dd942014-02-10 14:25:46 -08003647 * No locks need to be taken here as it has just been
3648 * initialized and there is no concurrent access.
Dave Hansen67b6c902014-01-24 07:20:23 -08003649 */
Steven Rostedt1e4dd942014-02-10 14:25:46 -08003650 __add_partial(n, page, DEACTIVATE_TO_HEAD);
Christoph Lameter81819f02007-05-06 14:49:36 -07003651}
3652
3653static void free_kmem_cache_nodes(struct kmem_cache *s)
3654{
3655 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003656 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07003657
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003658 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003659 s->node[node] = NULL;
Alexander Potapenkoea37df52017-09-06 16:19:15 -07003660 kmem_cache_free(kmem_cache_node, n);
Christoph Lameter81819f02007-05-06 14:49:36 -07003661 }
3662}
3663
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003664void __kmem_cache_release(struct kmem_cache *s)
3665{
Thomas Garnier210e7a42016-07-26 15:21:59 -07003666 cache_random_seq_destroy(s);
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003667 free_percpu(s->cpu_slab);
3668 free_kmem_cache_nodes(s);
3669}
3670
Christoph Lameter55136592010-08-20 12:37:13 -05003671static int init_kmem_cache_nodes(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07003672{
3673 int node;
Christoph Lameter81819f02007-05-06 14:49:36 -07003674
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08003675 for_each_node_mask(node, slab_nodes) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003676 struct kmem_cache_node *n;
3677
Alexander Duyck73367bd2010-05-21 14:41:35 -07003678 if (slab_state == DOWN) {
Christoph Lameter55136592010-08-20 12:37:13 -05003679 early_kmem_cache_node_alloc(node);
Alexander Duyck73367bd2010-05-21 14:41:35 -07003680 continue;
Christoph Lameter81819f02007-05-06 14:49:36 -07003681 }
Christoph Lameter51df1142010-08-20 12:37:15 -05003682 n = kmem_cache_alloc_node(kmem_cache_node,
Christoph Lameter55136592010-08-20 12:37:13 -05003683 GFP_KERNEL, node);
Alexander Duyck73367bd2010-05-21 14:41:35 -07003684
3685 if (!n) {
3686 free_kmem_cache_nodes(s);
3687 return 0;
3688 }
3689
Joonsoo Kim40534972012-05-11 00:50:47 +09003690 init_kmem_cache_node(n);
Alexander Potapenkoea37df52017-09-06 16:19:15 -07003691 s->node[node] = n;
Christoph Lameter81819f02007-05-06 14:49:36 -07003692 }
3693 return 1;
3694}
Christoph Lameter81819f02007-05-06 14:49:36 -07003695
David Rientjesc0bdb232009-02-25 09:16:35 +02003696static void set_min_partial(struct kmem_cache *s, unsigned long min)
David Rientjes3b89d7d2009-02-22 17:40:07 -08003697{
3698 if (min < MIN_PARTIAL)
3699 min = MIN_PARTIAL;
3700 else if (min > MAX_PARTIAL)
3701 min = MAX_PARTIAL;
3702 s->min_partial = min;
3703}
3704
Wei Yange6d0e1d2017-07-06 15:36:34 -07003705static void set_cpu_partial(struct kmem_cache *s)
3706{
3707#ifdef CONFIG_SLUB_CPU_PARTIAL
3708 /*
3709 * cpu_partial determined the maximum number of objects kept in the
3710 * per cpu partial lists of a processor.
3711 *
3712 * Per cpu partial lists mainly contain slabs that just have one
3713 * object freed. If they are used for allocation then they can be
3714 * filled up again with minimal effort. The slab will never hit the
3715 * per node partial lists and therefore no locking will be required.
3716 *
3717 * This setting also determines
3718 *
3719 * A) The number of objects from per cpu partial slabs dumped to the
3720 * per node list when we reach the limit.
3721 * B) The number of objects in cpu partial slabs to extract from the
3722 * per node list when we run out of per cpu objects. We only fetch
3723 * 50% to keep some capacity around for frees.
3724 */
3725 if (!kmem_cache_has_cpu_partial(s))
chenqiwubbd4e302020-04-01 21:04:19 -07003726 slub_set_cpu_partial(s, 0);
Wei Yange6d0e1d2017-07-06 15:36:34 -07003727 else if (s->size >= PAGE_SIZE)
chenqiwubbd4e302020-04-01 21:04:19 -07003728 slub_set_cpu_partial(s, 2);
Wei Yange6d0e1d2017-07-06 15:36:34 -07003729 else if (s->size >= 1024)
chenqiwubbd4e302020-04-01 21:04:19 -07003730 slub_set_cpu_partial(s, 6);
Wei Yange6d0e1d2017-07-06 15:36:34 -07003731 else if (s->size >= 256)
chenqiwubbd4e302020-04-01 21:04:19 -07003732 slub_set_cpu_partial(s, 13);
Wei Yange6d0e1d2017-07-06 15:36:34 -07003733 else
chenqiwubbd4e302020-04-01 21:04:19 -07003734 slub_set_cpu_partial(s, 30);
Wei Yange6d0e1d2017-07-06 15:36:34 -07003735#endif
3736}
3737
Christoph Lameter81819f02007-05-06 14:49:36 -07003738/*
3739 * calculate_sizes() determines the order and the distribution of data within
3740 * a slab object.
3741 */
Christoph Lameter06b285d2008-04-14 19:11:41 +03003742static int calculate_sizes(struct kmem_cache *s, int forced_order)
Christoph Lameter81819f02007-05-06 14:49:36 -07003743{
Alexey Dobriyand50112e2017-11-15 17:32:18 -08003744 slab_flags_t flags = s->flags;
Alexey Dobriyanbe4a7982018-04-05 16:21:28 -07003745 unsigned int size = s->object_size;
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003746 unsigned int order;
Christoph Lameter81819f02007-05-06 14:49:36 -07003747
3748 /*
Christoph Lameterd8b42bf2008-02-15 23:45:25 -08003749 * Round up object size to the next word boundary. We can only
3750 * place the free pointer at word boundaries and this determines
3751 * the possible location of the free pointer.
3752 */
3753 size = ALIGN(size, sizeof(void *));
3754
3755#ifdef CONFIG_SLUB_DEBUG
3756 /*
Christoph Lameter81819f02007-05-06 14:49:36 -07003757 * Determine if we can poison the object itself. If the user of
3758 * the slab may touch the object after free or before allocation
3759 * then we should never poison the object itself.
3760 */
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -08003761 if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
Christoph Lameterc59def92007-05-16 22:10:50 -07003762 !s->ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07003763 s->flags |= __OBJECT_POISON;
3764 else
3765 s->flags &= ~__OBJECT_POISON;
3766
Christoph Lameter81819f02007-05-06 14:49:36 -07003767
3768 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07003769 * If we are Redzoning then check if there is some space between the
Christoph Lameter81819f02007-05-06 14:49:36 -07003770 * end of the object and the free pointer. If not then add an
Christoph Lameter672bba32007-05-09 02:32:39 -07003771 * additional word to have some bytes to store Redzone information.
Christoph Lameter81819f02007-05-06 14:49:36 -07003772 */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05003773 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
Christoph Lameter81819f02007-05-06 14:49:36 -07003774 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07003775#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003776
3777 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07003778 * With that we have determined the number of bytes in actual use
Kees Cooke41a49fa2021-06-15 18:23:26 -07003779 * by the object and redzoning.
Christoph Lameter81819f02007-05-06 14:49:36 -07003780 */
3781 s->inuse = size;
3782
Kees Cook74c1d3e2021-06-15 18:23:22 -07003783 if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
3784 ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
3785 s->ctor) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003786 /*
3787 * Relocate free pointer after the object if it is not
3788 * permitted to overwrite the first word of the object on
3789 * kmem_cache_free.
3790 *
3791 * This is the case if we do RCU, have a constructor or
Kees Cook74c1d3e2021-06-15 18:23:22 -07003792 * destructor, are poisoning the objects, or are
3793 * redzoning an object smaller than sizeof(void *).
Waiman Longcbfc35a2020-05-07 18:36:06 -07003794 *
3795 * The assumption that s->offset >= s->inuse means free
3796 * pointer is outside of the object is used in the
3797 * freeptr_outside_object() function. If that is no
3798 * longer true, the function needs to be modified.
Christoph Lameter81819f02007-05-06 14:49:36 -07003799 */
3800 s->offset = size;
3801 size += sizeof(void *);
Kees Cooke41a49fa2021-06-15 18:23:26 -07003802 } else {
Kees Cook3202fa62020-04-01 21:04:27 -07003803 /*
3804 * Store freelist pointer near middle of object to keep
3805 * it away from the edges of the object to avoid small
3806 * sized over/underflows from neighboring allocations.
3807 */
Kees Cooke41a49fa2021-06-15 18:23:26 -07003808 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
Christoph Lameter81819f02007-05-06 14:49:36 -07003809 }
3810
Christoph Lameterc12b3c62007-05-23 13:57:31 -07003811#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07003812 if (flags & SLAB_STORE_USER)
3813 /*
3814 * Need to store information about allocs and frees after
3815 * the object.
3816 */
3817 size += 2 * sizeof(struct track);
Alexander Potapenko80a92012016-07-28 15:49:07 -07003818#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003819
Alexander Potapenko80a92012016-07-28 15:49:07 -07003820 kasan_cache_create(s, &size, &s->flags);
3821#ifdef CONFIG_SLUB_DEBUG
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -07003822 if (flags & SLAB_RED_ZONE) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003823 /*
3824 * Add some empty padding so that we can catch
3825 * overwrites from earlier objects rather than let
3826 * tracking information or the free pointer be
Frederik Schwarzer0211a9c82008-12-29 22:14:56 +01003827 * corrupted if a user writes before the start
Christoph Lameter81819f02007-05-06 14:49:36 -07003828 * of the object.
3829 */
3830 size += sizeof(void *);
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -07003831
3832 s->red_left_pad = sizeof(void *);
3833 s->red_left_pad = ALIGN(s->red_left_pad, s->align);
3834 size += s->red_left_pad;
3835 }
Christoph Lameter41ecc552007-05-09 02:32:44 -07003836#endif
Christoph Lameter672bba32007-05-09 02:32:39 -07003837
Christoph Lameter81819f02007-05-06 14:49:36 -07003838 /*
Christoph Lameter81819f02007-05-06 14:49:36 -07003839 * SLUB stores one object immediately after another beginning from
3840 * offset 0. In order to align the objects we have to simply size
3841 * each object to conform to the alignment.
3842 */
Christoph Lameter45906852012-11-28 16:23:16 +00003843 size = ALIGN(size, s->align);
Christoph Lameter81819f02007-05-06 14:49:36 -07003844 s->size = size;
Roman Gushchin4138fdf2020-08-06 23:20:42 -07003845 s->reciprocal_size = reciprocal_value(size);
Christoph Lameter06b285d2008-04-14 19:11:41 +03003846 if (forced_order >= 0)
3847 order = forced_order;
3848 else
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003849 order = calculate_order(size);
Christoph Lameter81819f02007-05-06 14:49:36 -07003850
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003851 if ((int)order < 0)
Christoph Lameter81819f02007-05-06 14:49:36 -07003852 return 0;
3853
Christoph Lameterb7a49f02008-02-14 14:21:32 -08003854 s->allocflags = 0;
Christoph Lameter834f3d12008-04-14 19:11:31 +03003855 if (order)
Christoph Lameterb7a49f02008-02-14 14:21:32 -08003856 s->allocflags |= __GFP_COMP;
3857
3858 if (s->flags & SLAB_CACHE_DMA)
Christoph Lameter2c59dd62013-01-10 19:14:19 +00003859 s->allocflags |= GFP_DMA;
Christoph Lameterb7a49f02008-02-14 14:21:32 -08003860
Nicolas Boichat6d6ea1e2019-03-28 20:43:42 -07003861 if (s->flags & SLAB_CACHE_DMA32)
3862 s->allocflags |= GFP_DMA32;
3863
Christoph Lameterb7a49f02008-02-14 14:21:32 -08003864 if (s->flags & SLAB_RECLAIM_ACCOUNT)
3865 s->allocflags |= __GFP_RECLAIMABLE;
3866
Christoph Lameter81819f02007-05-06 14:49:36 -07003867 /*
3868 * Determine the number of objects per slab
3869 */
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003870 s->oo = oo_make(order, size);
3871 s->min = oo_make(get_order(size), size);
Christoph Lameter205ab992008-04-14 19:11:40 +03003872 if (oo_objects(s->oo) > oo_objects(s->max))
3873 s->max = s->oo;
Christoph Lameter81819f02007-05-06 14:49:36 -07003874
Christoph Lameter834f3d12008-04-14 19:11:31 +03003875 return !!oo_objects(s->oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07003876}
3877
Alexey Dobriyand50112e2017-11-15 17:32:18 -08003878static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07003879{
Nikolay Borisov37540002021-02-24 12:00:58 -08003880 s->flags = kmem_cache_flags(s->size, flags, s->name);
Kees Cook2482ddec2017-09-06 16:19:18 -07003881#ifdef CONFIG_SLAB_FREELIST_HARDENED
3882 s->random = get_random_long();
3883#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003884
Christoph Lameter06b285d2008-04-14 19:11:41 +03003885 if (!calculate_sizes(s, -1))
Christoph Lameter81819f02007-05-06 14:49:36 -07003886 goto error;
David Rientjes3de47212009-07-27 18:30:35 -07003887 if (disable_higher_order_debug) {
3888 /*
3889 * Disable debugging flags that store metadata if the min slab
3890 * order increased.
3891 */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05003892 if (get_order(s->size) > get_order(s->object_size)) {
David Rientjes3de47212009-07-27 18:30:35 -07003893 s->flags &= ~DEBUG_METADATA_FLAGS;
3894 s->offset = 0;
3895 if (!calculate_sizes(s, -1))
3896 goto error;
3897 }
3898 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003899
Heiko Carstens25654092012-01-12 17:17:33 -08003900#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
3901 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Laura Abbott149daaf2016-03-15 14:55:09 -07003902 if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
Christoph Lameterb789ef52011-06-01 12:25:49 -05003903 /* Enable fast mode */
3904 s->flags |= __CMPXCHG_DOUBLE;
3905#endif
3906
David Rientjes3b89d7d2009-02-22 17:40:07 -08003907 /*
3908 * The larger the object size is, the more pages we want on the partial
3909 * list to avoid pounding the page allocator excessively.
3910 */
Christoph Lameter49e22582011-08-09 16:12:27 -05003911 set_min_partial(s, ilog2(s->size) / 2);
3912
Wei Yange6d0e1d2017-07-06 15:36:34 -07003913 set_cpu_partial(s);
Christoph Lameter49e22582011-08-09 16:12:27 -05003914
Christoph Lameter81819f02007-05-06 14:49:36 -07003915#ifdef CONFIG_NUMA
Christoph Lametere2cb96b2008-08-19 08:51:22 -05003916 s->remote_node_defrag_ratio = 1000;
Christoph Lameter81819f02007-05-06 14:49:36 -07003917#endif
Thomas Garnier210e7a42016-07-26 15:21:59 -07003918
3919 /* Initialize the pre-computed randomized freelist if slab is up */
3920 if (slab_state >= UP) {
3921 if (init_cache_random_seq(s))
3922 goto error;
3923 }
3924
Christoph Lameter55136592010-08-20 12:37:13 -05003925 if (!init_kmem_cache_nodes(s))
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003926 goto error;
Christoph Lameter81819f02007-05-06 14:49:36 -07003927
Christoph Lameter55136592010-08-20 12:37:13 -05003928 if (alloc_kmem_cache_cpus(s))
Christoph Lameter278b1bb2012-09-05 00:20:34 +00003929 return 0;
Christoph Lameterff120592009-12-18 16:26:22 -06003930
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003931 free_kmem_cache_nodes(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07003932error:
Christoph Lameter278b1bb2012-09-05 00:20:34 +00003933 return -EINVAL;
Christoph Lameter81819f02007-05-06 14:49:36 -07003934}
Christoph Lameter81819f02007-05-06 14:49:36 -07003935
Christoph Lameter33b12c32008-04-25 12:22:43 -07003936static void list_slab_objects(struct kmem_cache *s, struct page *page,
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07003937 const char *text)
Christoph Lameter81819f02007-05-06 14:49:36 -07003938{
Christoph Lameter33b12c32008-04-25 12:22:43 -07003939#ifdef CONFIG_SLUB_DEBUG
3940 void *addr = page_address(page);
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07003941 unsigned long *map;
Christoph Lameter33b12c32008-04-25 12:22:43 -07003942 void *p;
Christopher Lameteraa456c72020-06-01 21:45:53 -07003943
Christoph Lameter945cf2b2012-09-04 23:18:33 +00003944 slab_err(s, page, text, s->name);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003945 slab_lock(page);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003946
Yu Zhao90e9f6a2020-01-30 22:11:57 -08003947 map = get_map(s, page);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003948 for_each_object(p, s, addr, page->objects) {
3949
Roman Gushchin4138fdf2020-08-06 23:20:42 -07003950 if (!test_bit(__obj_to_index(s, addr, p), map)) {
Yafang Shao96b94ab2021-03-19 18:12:45 +08003951 pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003952 print_tracking(s, p);
3953 }
3954 }
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07003955 put_map(map);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003956 slab_unlock(page);
3957#endif
3958}
3959
Christoph Lameter81819f02007-05-06 14:49:36 -07003960/*
Christoph Lameter599870b2008-04-23 12:36:52 -07003961 * Attempt to free all partial slabs on a node.
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003962 * This is called from __kmem_cache_shutdown(). We must take list_lock
3963 * because sysfs file might still access partial list after the shutdowning.
Christoph Lameter81819f02007-05-06 14:49:36 -07003964 */
Christoph Lameter599870b2008-04-23 12:36:52 -07003965static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
Christoph Lameter81819f02007-05-06 14:49:36 -07003966{
Chris Wilson60398922016-08-10 16:27:58 -07003967 LIST_HEAD(discard);
Christoph Lameter81819f02007-05-06 14:49:36 -07003968 struct page *page, *h;
3969
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003970 BUG_ON(irqs_disabled());
3971 spin_lock_irq(&n->list_lock);
Tobin C. Harding916ac052019-05-13 17:16:12 -07003972 list_for_each_entry_safe(page, h, &n->partial, slab_list) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003973 if (!page->inuse) {
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003974 remove_partial(n, page);
Tobin C. Harding916ac052019-05-13 17:16:12 -07003975 list_add(&page->slab_list, &discard);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003976 } else {
3977 list_slab_objects(s, page,
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07003978 "Objects remaining in %s on __kmem_cache_shutdown()");
Christoph Lameter599870b2008-04-23 12:36:52 -07003979 }
Christoph Lameter33b12c32008-04-25 12:22:43 -07003980 }
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003981 spin_unlock_irq(&n->list_lock);
Chris Wilson60398922016-08-10 16:27:58 -07003982
Tobin C. Harding916ac052019-05-13 17:16:12 -07003983 list_for_each_entry_safe(page, h, &discard, slab_list)
Chris Wilson60398922016-08-10 16:27:58 -07003984 discard_slab(s, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07003985}
3986
Shakeel Buttf9e13c02018-04-05 16:21:57 -07003987bool __kmem_cache_empty(struct kmem_cache *s)
3988{
3989 int node;
3990 struct kmem_cache_node *n;
3991
3992 for_each_kmem_cache_node(s, node, n)
3993 if (n->nr_partial || slabs_node(s, node))
3994 return false;
3995 return true;
3996}
3997
Christoph Lameter81819f02007-05-06 14:49:36 -07003998/*
Christoph Lameter672bba32007-05-09 02:32:39 -07003999 * Release all resources used by a slab cache.
Christoph Lameter81819f02007-05-06 14:49:36 -07004000 */
Dmitry Safonov52b4b952016-02-17 13:11:37 -08004001int __kmem_cache_shutdown(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07004002{
4003 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004004 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07004005
4006 flush_all(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07004007 /* Attempt to free all objects */
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004008 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter599870b2008-04-23 12:36:52 -07004009 free_partial(s, n);
4010 if (n->nr_partial || slabs_node(s, node))
Christoph Lameter81819f02007-05-06 14:49:36 -07004011 return 1;
4012 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004013 return 0;
4014}
4015
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -08004016#ifdef CONFIG_PRINTK
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004017void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
4018{
4019 void *base;
4020 int __maybe_unused i;
4021 unsigned int objnr;
4022 void *objp;
4023 void *objp0;
4024 struct kmem_cache *s = page->slab_cache;
4025 struct track __maybe_unused *trackp;
4026
4027 kpp->kp_ptr = object;
4028 kpp->kp_page = page;
4029 kpp->kp_slab_cache = s;
4030 base = page_address(page);
4031 objp0 = kasan_reset_tag(object);
4032#ifdef CONFIG_SLUB_DEBUG
4033 objp = restore_red_left(s, objp0);
4034#else
4035 objp = objp0;
4036#endif
4037 objnr = obj_to_index(s, page, objp);
4038 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
4039 objp = base + s->size * objnr;
4040 kpp->kp_objp = objp;
4041 if (WARN_ON_ONCE(objp < base || objp >= base + page->objects * s->size || (objp - base) % s->size) ||
4042 !(s->flags & SLAB_STORE_USER))
4043 return;
4044#ifdef CONFIG_SLUB_DEBUG
Maninder Singh0cbc1242021-03-16 16:07:10 +05304045 objp = fixup_red_left(s, objp);
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004046 trackp = get_track(s, objp, TRACK_ALLOC);
4047 kpp->kp_ret = (void *)trackp->addr;
Linus Torvaldsae14c632021-07-17 13:27:00 -07004048#ifdef CONFIG_STACKTRACE
4049 for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
4050 kpp->kp_stack[i] = (void *)trackp->addrs[i];
4051 if (!kpp->kp_stack[i])
4052 break;
4053 }
Maninder Singhe548eaa2021-03-16 16:07:11 +05304054
Linus Torvaldsae14c632021-07-17 13:27:00 -07004055 trackp = get_track(s, objp, TRACK_FREE);
4056 for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
4057 kpp->kp_free_stack[i] = (void *)trackp->addrs[i];
4058 if (!kpp->kp_free_stack[i])
4059 break;
Maninder Singhe548eaa2021-03-16 16:07:11 +05304060 }
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004061#endif
4062#endif
4063}
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -08004064#endif
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004065
Christoph Lameter81819f02007-05-06 14:49:36 -07004066/********************************************************************
4067 * Kmalloc subsystem
4068 *******************************************************************/
4069
Christoph Lameter81819f02007-05-06 14:49:36 -07004070static int __init setup_slub_min_order(char *str)
4071{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004072 get_option(&str, (int *)&slub_min_order);
Christoph Lameter81819f02007-05-06 14:49:36 -07004073
4074 return 1;
4075}
4076
4077__setup("slub_min_order=", setup_slub_min_order);
4078
4079static int __init setup_slub_max_order(char *str)
4080{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004081 get_option(&str, (int *)&slub_max_order);
4082 slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004083
4084 return 1;
4085}
4086
4087__setup("slub_max_order=", setup_slub_max_order);
4088
4089static int __init setup_slub_min_objects(char *str)
4090{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004091 get_option(&str, (int *)&slub_min_objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07004092
4093 return 1;
4094}
4095
4096__setup("slub_min_objects=", setup_slub_min_objects);
4097
Christoph Lameter81819f02007-05-06 14:49:36 -07004098void *__kmalloc(size_t size, gfp_t flags)
4099{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004100 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004101 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004102
Christoph Lameter95a05b42013-01-10 19:14:19 +00004103 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02004104 return kmalloc_large(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004105
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004106 s = kmalloc_slab(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004107
4108 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004109 return s;
4110
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004111 ret = slab_alloc(s, flags, _RET_IP_, size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004112
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004113 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004114
Andrey Konovalov0116523c2018-12-28 00:29:37 -08004115 ret = kasan_kmalloc(s, ret, size, flags);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08004116
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004117 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004118}
4119EXPORT_SYMBOL(__kmalloc);
4120
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004121#ifdef CONFIG_NUMA
Christoph Lameterf619cfe2008-03-01 13:56:40 -08004122static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
4123{
Vegard Nossumb1eeab62008-11-25 16:55:53 +01004124 struct page *page;
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01004125 void *ptr = NULL;
Vlastimil Babka6a486c02019-10-06 17:58:42 -07004126 unsigned int order = get_order(size);
Christoph Lameterf619cfe2008-03-01 13:56:40 -08004127
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -08004128 flags |= __GFP_COMP;
Vlastimil Babka6a486c02019-10-06 17:58:42 -07004129 page = alloc_pages_node(node, flags, order);
4130 if (page) {
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01004131 ptr = page_address(page);
Muchun Song96403bf2021-02-24 12:04:26 -08004132 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
4133 PAGE_SIZE << order);
Vlastimil Babka6a486c02019-10-06 17:58:42 -07004134 }
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01004135
Andrey Konovalov0116523c2018-12-28 00:29:37 -08004136 return kmalloc_large_node_hook(ptr, size, flags);
Christoph Lameterf619cfe2008-03-01 13:56:40 -08004137}
4138
Christoph Lameter81819f02007-05-06 14:49:36 -07004139void *__kmalloc_node(size_t size, gfp_t flags, int node)
4140{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004141 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004142 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004143
Christoph Lameter95a05b42013-01-10 19:14:19 +00004144 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004145 ret = kmalloc_large_node(size, flags, node);
4146
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004147 trace_kmalloc_node(_RET_IP_, ret,
4148 size, PAGE_SIZE << get_order(size),
4149 flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004150
4151 return ret;
4152 }
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004153
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004154 s = kmalloc_slab(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004155
4156 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004157 return s;
4158
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004159 ret = slab_alloc_node(s, flags, node, _RET_IP_, size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004160
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004161 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004162
Andrey Konovalov0116523c2018-12-28 00:29:37 -08004163 ret = kasan_kmalloc(s, ret, size, flags);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08004164
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004165 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004166}
4167EXPORT_SYMBOL(__kmalloc_node);
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07004168#endif /* CONFIG_NUMA */
Christoph Lameter81819f02007-05-06 14:49:36 -07004169
Kees Cooked18adc2016-06-23 15:24:05 -07004170#ifdef CONFIG_HARDENED_USERCOPY
4171/*
Kees Cookafcc90f82018-01-10 15:17:01 -08004172 * Rejects incorrectly sized objects and objects that are to be copied
4173 * to/from userspace but do not fall entirely within the containing slab
4174 * cache's usercopy region.
Kees Cooked18adc2016-06-23 15:24:05 -07004175 *
4176 * Returns NULL if check passes, otherwise const char * to name of cache
4177 * to indicate an error.
4178 */
Kees Cookf4e6e282018-01-10 14:48:22 -08004179void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
4180 bool to_user)
Kees Cooked18adc2016-06-23 15:24:05 -07004181{
4182 struct kmem_cache *s;
Alexey Dobriyan44065b22018-04-05 16:21:20 -07004183 unsigned int offset;
Kees Cooked18adc2016-06-23 15:24:05 -07004184 size_t object_size;
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004185 bool is_kfence = is_kfence_address(ptr);
Kees Cooked18adc2016-06-23 15:24:05 -07004186
Andrey Konovalov96fedce2019-01-08 15:23:15 -08004187 ptr = kasan_reset_tag(ptr);
4188
Kees Cooked18adc2016-06-23 15:24:05 -07004189 /* Find object and usable object size. */
4190 s = page->slab_cache;
Kees Cooked18adc2016-06-23 15:24:05 -07004191
4192 /* Reject impossible pointers. */
4193 if (ptr < page_address(page))
Kees Cookf4e6e282018-01-10 14:48:22 -08004194 usercopy_abort("SLUB object not in SLUB page?!", NULL,
4195 to_user, 0, n);
Kees Cooked18adc2016-06-23 15:24:05 -07004196
4197 /* Find offset within object. */
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004198 if (is_kfence)
4199 offset = ptr - kfence_object_start(ptr);
4200 else
4201 offset = (ptr - page_address(page)) % s->size;
Kees Cooked18adc2016-06-23 15:24:05 -07004202
4203 /* Adjust for redzone and reject if within the redzone. */
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004204 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
Kees Cooked18adc2016-06-23 15:24:05 -07004205 if (offset < s->red_left_pad)
Kees Cookf4e6e282018-01-10 14:48:22 -08004206 usercopy_abort("SLUB object in left red zone",
4207 s->name, to_user, offset, n);
Kees Cooked18adc2016-06-23 15:24:05 -07004208 offset -= s->red_left_pad;
4209 }
4210
Kees Cookafcc90f82018-01-10 15:17:01 -08004211 /* Allow address range falling entirely within usercopy region. */
4212 if (offset >= s->useroffset &&
4213 offset - s->useroffset <= s->usersize &&
4214 n <= s->useroffset - offset + s->usersize)
Kees Cookf4e6e282018-01-10 14:48:22 -08004215 return;
Kees Cooked18adc2016-06-23 15:24:05 -07004216
Kees Cookafcc90f82018-01-10 15:17:01 -08004217 /*
4218 * If the copy is still within the allocated object, produce
4219 * a warning instead of rejecting the copy. This is intended
4220 * to be a temporary method to find any missing usercopy
4221 * whitelists.
4222 */
4223 object_size = slab_ksize(s);
Kees Cook2d891fb2017-11-30 13:04:32 -08004224 if (usercopy_fallback &&
4225 offset <= object_size && n <= object_size - offset) {
Kees Cookafcc90f82018-01-10 15:17:01 -08004226 usercopy_warn("SLUB object", s->name, to_user, offset, n);
4227 return;
4228 }
4229
Kees Cookf4e6e282018-01-10 14:48:22 -08004230 usercopy_abort("SLUB object", s->name, to_user, offset, n);
Kees Cooked18adc2016-06-23 15:24:05 -07004231}
4232#endif /* CONFIG_HARDENED_USERCOPY */
4233
Marco Elver10d1f8c2019-07-11 20:54:14 -07004234size_t __ksize(const void *object)
Christoph Lameter81819f02007-05-06 14:49:36 -07004235{
Christoph Lameter272c1d22007-06-08 13:46:49 -07004236 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07004237
Christoph Lameteref8b4522007-10-16 01:24:46 -07004238 if (unlikely(object == ZERO_SIZE_PTR))
Christoph Lameter272c1d22007-06-08 13:46:49 -07004239 return 0;
4240
Vegard Nossum294a80a2007-12-04 23:45:30 -08004241 page = virt_to_head_page(object);
Vegard Nossum294a80a2007-12-04 23:45:30 -08004242
Pekka Enberg76994412008-05-22 19:22:25 +03004243 if (unlikely(!PageSlab(page))) {
4244 WARN_ON(!PageCompound(page));
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07004245 return page_size(page);
Pekka Enberg76994412008-05-22 19:22:25 +03004246 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004247
Glauber Costa1b4f59e32012-10-22 18:05:36 +04004248 return slab_ksize(page->slab_cache);
Christoph Lameter81819f02007-05-06 14:49:36 -07004249}
Marco Elver10d1f8c2019-07-11 20:54:14 -07004250EXPORT_SYMBOL(__ksize);
Christoph Lameter81819f02007-05-06 14:49:36 -07004251
4252void kfree(const void *x)
4253{
Christoph Lameter81819f02007-05-06 14:49:36 -07004254 struct page *page;
Christoph Lameter5bb983b2008-02-07 17:47:41 -08004255 void *object = (void *)x;
Christoph Lameter81819f02007-05-06 14:49:36 -07004256
Pekka Enberg2121db72009-03-25 11:05:57 +02004257 trace_kfree(_RET_IP_, x);
4258
Satyam Sharma2408c552007-10-16 01:24:44 -07004259 if (unlikely(ZERO_OR_NULL_PTR(x)))
Christoph Lameter81819f02007-05-06 14:49:36 -07004260 return;
4261
Christoph Lameterb49af682007-05-06 14:49:41 -07004262 page = virt_to_head_page(x);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004263 if (unlikely(!PageSlab(page))) {
Shakeel Butt1ed7ce52021-08-13 16:54:31 -07004264 free_nonslab_page(page, object);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004265 return;
4266 }
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08004267 slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
Christoph Lameter81819f02007-05-06 14:49:36 -07004268}
4269EXPORT_SYMBOL(kfree);
4270
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004271#define SHRINK_PROMOTE_MAX 32
4272
Christoph Lameter2086d262007-05-06 14:49:46 -07004273/*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004274 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
4275 * up most to the head of the partial lists. New allocations will then
4276 * fill those up and thus they can be removed from the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -07004277 *
4278 * The slabs with the least items are placed last. This results in them
4279 * being allocated from last increasing the chance that the last objects
4280 * are freed in them.
Christoph Lameter2086d262007-05-06 14:49:46 -07004281 */
Tejun Heoc9fc5862017-02-22 15:41:27 -08004282int __kmem_cache_shrink(struct kmem_cache *s)
Christoph Lameter2086d262007-05-06 14:49:46 -07004283{
4284 int node;
4285 int i;
4286 struct kmem_cache_node *n;
4287 struct page *page;
4288 struct page *t;
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004289 struct list_head discard;
4290 struct list_head promote[SHRINK_PROMOTE_MAX];
Christoph Lameter2086d262007-05-06 14:49:46 -07004291 unsigned long flags;
Vladimir Davydovce3712d2015-02-12 14:59:44 -08004292 int ret = 0;
Christoph Lameter2086d262007-05-06 14:49:46 -07004293
Christoph Lameter2086d262007-05-06 14:49:46 -07004294 flush_all(s);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004295 for_each_kmem_cache_node(s, node, n) {
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004296 INIT_LIST_HEAD(&discard);
4297 for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
4298 INIT_LIST_HEAD(promote + i);
Christoph Lameter2086d262007-05-06 14:49:46 -07004299
4300 spin_lock_irqsave(&n->list_lock, flags);
4301
4302 /*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004303 * Build lists of slabs to discard or promote.
Christoph Lameter2086d262007-05-06 14:49:46 -07004304 *
Christoph Lameter672bba32007-05-09 02:32:39 -07004305 * Note that concurrent frees may occur while we hold the
4306 * list_lock. page->inuse here is the upper limit.
Christoph Lameter2086d262007-05-06 14:49:46 -07004307 */
Tobin C. Harding916ac052019-05-13 17:16:12 -07004308 list_for_each_entry_safe(page, t, &n->partial, slab_list) {
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004309 int free = page->objects - page->inuse;
4310
4311 /* Do not reread page->inuse */
4312 barrier();
4313
4314 /* We do not keep full slabs on the list */
4315 BUG_ON(free <= 0);
4316
4317 if (free == page->objects) {
Tobin C. Harding916ac052019-05-13 17:16:12 -07004318 list_move(&page->slab_list, &discard);
Christoph Lameter69cb8e62011-08-09 16:12:22 -05004319 n->nr_partial--;
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004320 } else if (free <= SHRINK_PROMOTE_MAX)
Tobin C. Harding916ac052019-05-13 17:16:12 -07004321 list_move(&page->slab_list, promote + free - 1);
Christoph Lameter2086d262007-05-06 14:49:46 -07004322 }
4323
Christoph Lameter2086d262007-05-06 14:49:46 -07004324 /*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004325 * Promote the slabs filled up most to the head of the
4326 * partial list.
Christoph Lameter2086d262007-05-06 14:49:46 -07004327 */
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004328 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
4329 list_splice(promote + i, &n->partial);
Christoph Lameter2086d262007-05-06 14:49:46 -07004330
Christoph Lameter2086d262007-05-06 14:49:46 -07004331 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter69cb8e62011-08-09 16:12:22 -05004332
4333 /* Release empty slabs */
Tobin C. Harding916ac052019-05-13 17:16:12 -07004334 list_for_each_entry_safe(page, t, &discard, slab_list)
Christoph Lameter69cb8e62011-08-09 16:12:22 -05004335 discard_slab(s, page);
Vladimir Davydovce3712d2015-02-12 14:59:44 -08004336
4337 if (slabs_node(s, node))
4338 ret = 1;
Christoph Lameter2086d262007-05-06 14:49:46 -07004339 }
4340
Vladimir Davydovce3712d2015-02-12 14:59:44 -08004341 return ret;
Christoph Lameter2086d262007-05-06 14:49:46 -07004342}
Christoph Lameter2086d262007-05-06 14:49:46 -07004343
Yasunori Gotob9049e22007-10-21 16:41:37 -07004344static int slab_mem_going_offline_callback(void *arg)
4345{
4346 struct kmem_cache *s;
4347
Christoph Lameter18004c52012-07-06 15:25:12 -05004348 mutex_lock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004349 list_for_each_entry(s, &slab_caches, list)
Tejun Heoc9fc5862017-02-22 15:41:27 -08004350 __kmem_cache_shrink(s);
Christoph Lameter18004c52012-07-06 15:25:12 -05004351 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004352
4353 return 0;
4354}
4355
4356static void slab_mem_offline_callback(void *arg)
4357{
Yasunori Gotob9049e22007-10-21 16:41:37 -07004358 struct memory_notify *marg = arg;
4359 int offline_node;
4360
Lai Jiangshanb9d5ab22012-12-11 16:01:05 -08004361 offline_node = marg->status_change_nid_normal;
Yasunori Gotob9049e22007-10-21 16:41:37 -07004362
4363 /*
4364 * If the node still has available memory. we need kmem_cache_node
4365 * for it yet.
4366 */
4367 if (offline_node < 0)
4368 return;
4369
Christoph Lameter18004c52012-07-06 15:25:12 -05004370 mutex_lock(&slab_mutex);
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004371 node_clear(offline_node, slab_nodes);
Vlastimil Babka666716f2021-02-24 12:01:08 -08004372 /*
4373 * We no longer free kmem_cache_node structures here, as it would be
4374 * racy with all get_node() users, and infeasible to protect them with
4375 * slab_mutex.
4376 */
Christoph Lameter18004c52012-07-06 15:25:12 -05004377 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004378}
4379
4380static int slab_mem_going_online_callback(void *arg)
4381{
4382 struct kmem_cache_node *n;
4383 struct kmem_cache *s;
4384 struct memory_notify *marg = arg;
Lai Jiangshanb9d5ab22012-12-11 16:01:05 -08004385 int nid = marg->status_change_nid_normal;
Yasunori Gotob9049e22007-10-21 16:41:37 -07004386 int ret = 0;
4387
4388 /*
4389 * If the node's memory is already available, then kmem_cache_node is
4390 * already created. Nothing to do.
4391 */
4392 if (nid < 0)
4393 return 0;
4394
4395 /*
Christoph Lameter0121c6192008-04-29 16:11:12 -07004396 * We are bringing a node online. No memory is available yet. We must
Yasunori Gotob9049e22007-10-21 16:41:37 -07004397 * allocate a kmem_cache_node structure in order to bring the node
4398 * online.
4399 */
Christoph Lameter18004c52012-07-06 15:25:12 -05004400 mutex_lock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004401 list_for_each_entry(s, &slab_caches, list) {
4402 /*
Vlastimil Babka666716f2021-02-24 12:01:08 -08004403 * The structure may already exist if the node was previously
4404 * onlined and offlined.
4405 */
4406 if (get_node(s, nid))
4407 continue;
4408 /*
Yasunori Gotob9049e22007-10-21 16:41:37 -07004409 * XXX: kmem_cache_alloc_node will fallback to other nodes
4410 * since memory is not yet available from the node that
4411 * is brought up.
4412 */
Christoph Lameter8de66a02010-08-25 14:51:14 -05004413 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004414 if (!n) {
4415 ret = -ENOMEM;
4416 goto out;
4417 }
Joonsoo Kim40534972012-05-11 00:50:47 +09004418 init_kmem_cache_node(n);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004419 s->node[nid] = n;
4420 }
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004421 /*
4422 * Any cache created after this point will also have kmem_cache_node
4423 * initialized for the new node.
4424 */
4425 node_set(nid, slab_nodes);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004426out:
Christoph Lameter18004c52012-07-06 15:25:12 -05004427 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004428 return ret;
4429}
4430
4431static int slab_memory_callback(struct notifier_block *self,
4432 unsigned long action, void *arg)
4433{
4434 int ret = 0;
4435
4436 switch (action) {
4437 case MEM_GOING_ONLINE:
4438 ret = slab_mem_going_online_callback(arg);
4439 break;
4440 case MEM_GOING_OFFLINE:
4441 ret = slab_mem_going_offline_callback(arg);
4442 break;
4443 case MEM_OFFLINE:
4444 case MEM_CANCEL_ONLINE:
4445 slab_mem_offline_callback(arg);
4446 break;
4447 case MEM_ONLINE:
4448 case MEM_CANCEL_OFFLINE:
4449 break;
4450 }
KAMEZAWA Hiroyukidc19f9d2008-12-01 13:13:48 -08004451 if (ret)
4452 ret = notifier_from_errno(ret);
4453 else
4454 ret = NOTIFY_OK;
Yasunori Gotob9049e22007-10-21 16:41:37 -07004455 return ret;
4456}
4457
Andrew Morton3ac38fa2013-04-29 15:08:06 -07004458static struct notifier_block slab_memory_callback_nb = {
4459 .notifier_call = slab_memory_callback,
4460 .priority = SLAB_CALLBACK_PRI,
4461};
Yasunori Gotob9049e22007-10-21 16:41:37 -07004462
Christoph Lameter81819f02007-05-06 14:49:36 -07004463/********************************************************************
4464 * Basic setup of slabs
4465 *******************************************************************/
4466
Christoph Lameter51df1142010-08-20 12:37:15 -05004467/*
4468 * Used for early kmem_cache structures that were allocated using
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004469 * the page allocator. Allocate them properly then fix up the pointers
4470 * that may be pointing to the wrong kmem_cache structure.
Christoph Lameter51df1142010-08-20 12:37:15 -05004471 */
4472
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004473static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
Christoph Lameter51df1142010-08-20 12:37:15 -05004474{
4475 int node;
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004476 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004477 struct kmem_cache_node *n;
Christoph Lameter51df1142010-08-20 12:37:15 -05004478
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004479 memcpy(s, static_cache, kmem_cache->object_size);
Christoph Lameter51df1142010-08-20 12:37:15 -05004480
Glauber Costa7d557b32013-02-22 20:20:00 +04004481 /*
4482 * This runs very early, and only the boot processor is supposed to be
4483 * up. Even if it weren't true, IRQs are not up so we couldn't fire
4484 * IPIs around.
4485 */
4486 __flush_cpu_slab(s, smp_processor_id());
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004487 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter51df1142010-08-20 12:37:15 -05004488 struct page *p;
4489
Tobin C. Harding916ac052019-05-13 17:16:12 -07004490 list_for_each_entry(p, &n->partial, slab_list)
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004491 p->slab_cache = s;
Christoph Lameter51df1142010-08-20 12:37:15 -05004492
Li Zefan607bf322011-04-12 15:22:26 +08004493#ifdef CONFIG_SLUB_DEBUG
Tobin C. Harding916ac052019-05-13 17:16:12 -07004494 list_for_each_entry(p, &n->full, slab_list)
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004495 p->slab_cache = s;
Christoph Lameter51df1142010-08-20 12:37:15 -05004496#endif
Christoph Lameter51df1142010-08-20 12:37:15 -05004497 }
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004498 list_add(&s->list, &slab_caches);
4499 return s;
Christoph Lameter51df1142010-08-20 12:37:15 -05004500}
4501
Christoph Lameter81819f02007-05-06 14:49:36 -07004502void __init kmem_cache_init(void)
4503{
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004504 static __initdata struct kmem_cache boot_kmem_cache,
4505 boot_kmem_cache_node;
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004506 int node;
Christoph Lameter51df1142010-08-20 12:37:15 -05004507
Stanislaw Gruszkafc8d8622012-01-10 15:07:32 -08004508 if (debug_guardpage_minorder())
4509 slub_max_order = 0;
4510
Stephen Boyd79270292021-06-28 19:34:52 -07004511 /* Print slub debugging pointers without hashing */
4512 if (__slub_debug_enabled())
4513 no_hash_pointers_enable(NULL);
4514
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004515 kmem_cache_node = &boot_kmem_cache_node;
4516 kmem_cache = &boot_kmem_cache;
Christoph Lameter51df1142010-08-20 12:37:15 -05004517
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004518 /*
4519 * Initialize the nodemask for which we will allocate per node
4520 * structures. Here we don't need taking slab_mutex yet.
4521 */
4522 for_each_node_state(node, N_NORMAL_MEMORY)
4523 node_set(node, slab_nodes);
4524
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004525 create_boot_cache(kmem_cache_node, "kmem_cache_node",
David Windsor8eb82842017-06-10 22:50:28 -04004526 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004527
Andrew Morton3ac38fa2013-04-29 15:08:06 -07004528 register_hotmemory_notifier(&slab_memory_callback_nb);
Christoph Lameter81819f02007-05-06 14:49:36 -07004529
4530 /* Able to allocate the per node structures */
4531 slab_state = PARTIAL;
4532
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004533 create_boot_cache(kmem_cache, "kmem_cache",
4534 offsetof(struct kmem_cache, node) +
4535 nr_node_ids * sizeof(struct kmem_cache_node *),
David Windsor8eb82842017-06-10 22:50:28 -04004536 SLAB_HWCACHE_ALIGN, 0, 0);
Christoph Lameter8a13a4c2012-09-04 23:18:33 +00004537
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004538 kmem_cache = bootstrap(&boot_kmem_cache);
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004539 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
Christoph Lameter51df1142010-08-20 12:37:15 -05004540
4541 /* Now we can use the kmem_cache to allocate kmalloc slabs */
Daniel Sanders34cc6992015-06-24 16:55:57 -07004542 setup_kmalloc_cache_index_table();
Christoph Lameterf97d5f632013-01-10 19:12:17 +00004543 create_kmalloc_caches(0);
Christoph Lameter81819f02007-05-06 14:49:36 -07004544
Thomas Garnier210e7a42016-07-26 15:21:59 -07004545 /* Setup random freelists for each cache */
4546 init_freelist_randomization();
4547
Sebastian Andrzej Siewiora96a87b2016-08-18 14:57:19 +02004548 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
4549 slub_cpu_dead);
Christoph Lameter81819f02007-05-06 14:49:36 -07004550
Alexey Dobriyanb9726c22019-03-05 15:48:26 -08004551 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
Christoph Lameterf97d5f632013-01-10 19:12:17 +00004552 cache_line_size(),
Christoph Lameter81819f02007-05-06 14:49:36 -07004553 slub_min_order, slub_max_order, slub_min_objects,
4554 nr_cpu_ids, nr_node_ids);
4555}
4556
Pekka Enberg7e85ee02009-06-12 14:03:06 +03004557void __init kmem_cache_init_late(void)
4558{
Pekka Enberg7e85ee02009-06-12 14:03:06 +03004559}
4560
Glauber Costa2633d7a2012-12-18 14:22:34 -08004561struct kmem_cache *
Alexey Dobriyanf4957d52018-04-05 16:20:37 -07004562__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -08004563 slab_flags_t flags, void (*ctor)(void *))
Christoph Lameter81819f02007-05-06 14:49:36 -07004564{
Roman Gushchin10befea2020-08-06 23:21:27 -07004565 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004566
Vladimir Davydova44cb9442014-04-07 15:39:23 -07004567 s = find_mergeable(size, align, flags, name, ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07004568 if (s) {
4569 s->refcount++;
Vladimir Davydov84d0ddd2014-04-07 15:39:29 -07004570
Christoph Lameter81819f02007-05-06 14:49:36 -07004571 /*
4572 * Adjust the object sizes so that we clear
4573 * the complete object on kzalloc.
4574 */
Alexey Dobriyan1b473f22018-04-05 16:21:17 -07004575 s->object_size = max(s->object_size, size);
Alexey Dobriyan52ee6d72018-04-05 16:21:06 -07004576 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
Christoph Lameter6446faa2008-02-15 23:45:26 -08004577
David Rientjes7b8f3b62008-12-17 22:09:46 -08004578 if (sysfs_slab_alias(s, name)) {
David Rientjes7b8f3b62008-12-17 22:09:46 -08004579 s->refcount--;
Christoph Lametercbb79692012-09-05 00:18:32 +00004580 s = NULL;
David Rientjes7b8f3b62008-12-17 22:09:46 -08004581 }
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07004582 }
Christoph Lameter6446faa2008-02-15 23:45:26 -08004583
Christoph Lametercbb79692012-09-05 00:18:32 +00004584 return s;
4585}
Pekka Enberg84c1cf62010-09-14 23:21:12 +03004586
Alexey Dobriyand50112e2017-11-15 17:32:18 -08004587int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
Christoph Lametercbb79692012-09-05 00:18:32 +00004588{
Pekka Enbergaac3a162012-09-05 12:07:44 +03004589 int err;
Christoph Lameter20cea962012-07-06 15:25:13 -05004590
Pekka Enbergaac3a162012-09-05 12:07:44 +03004591 err = kmem_cache_open(s, flags);
4592 if (err)
4593 return err;
Christoph Lameter20cea962012-07-06 15:25:13 -05004594
Christoph Lameter45530c42012-11-28 16:23:07 +00004595 /* Mutex is not taken during early boot */
4596 if (slab_state <= UP)
4597 return 0;
4598
Pekka Enbergaac3a162012-09-05 12:07:44 +03004599 err = sysfs_slab_add(s);
Pekka Enbergaac3a162012-09-05 12:07:44 +03004600 if (err)
Dmitry Safonov52b4b952016-02-17 13:11:37 -08004601 __kmem_cache_release(s);
Pekka Enbergaac3a162012-09-05 12:07:44 +03004602
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07004603 if (s->flags & SLAB_STORE_USER)
4604 debugfs_slab_add(s);
4605
Pekka Enbergaac3a162012-09-05 12:07:44 +03004606 return err;
Christoph Lameter81819f02007-05-06 14:49:36 -07004607}
Christoph Lameter81819f02007-05-06 14:49:36 -07004608
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004609void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07004610{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004611 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004612 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004613
Christoph Lameter95a05b42013-01-10 19:14:19 +00004614 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02004615 return kmalloc_large(size, gfpflags);
4616
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004617 s = kmalloc_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07004618
Satyam Sharma2408c552007-10-16 01:24:44 -07004619 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004620 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004621
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004622 ret = slab_alloc(s, gfpflags, caller, size);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004623
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004624 /* Honor the call site pointer we received. */
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004625 trace_kmalloc(caller, ret, size, s->size, gfpflags);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004626
4627 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004628}
Daniel Vetterfd7cb572020-03-23 15:49:00 +01004629EXPORT_SYMBOL(__kmalloc_track_caller);
Christoph Lameter81819f02007-05-06 14:49:36 -07004630
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004631#ifdef CONFIG_NUMA
Christoph Lameter81819f02007-05-06 14:49:36 -07004632void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004633 int node, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07004634{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004635 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004636 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004637
Christoph Lameter95a05b42013-01-10 19:14:19 +00004638 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
Xiaotian Fengd3e14aa2010-04-08 17:26:44 +08004639 ret = kmalloc_large_node(size, gfpflags, node);
4640
4641 trace_kmalloc_node(caller, ret,
4642 size, PAGE_SIZE << get_order(size),
4643 gfpflags, node);
4644
4645 return ret;
4646 }
Pekka Enbergeada35e2008-02-11 22:47:46 +02004647
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004648 s = kmalloc_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07004649
Satyam Sharma2408c552007-10-16 01:24:44 -07004650 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004651 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004652
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004653 ret = slab_alloc_node(s, gfpflags, node, caller, size);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004654
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004655 /* Honor the call site pointer we received. */
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004656 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004657
4658 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004659}
Daniel Vetterfd7cb572020-03-23 15:49:00 +01004660EXPORT_SYMBOL(__kmalloc_node_track_caller);
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004661#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004662
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004663#ifdef CONFIG_SYSFS
Christoph Lameter205ab992008-04-14 19:11:40 +03004664static int count_inuse(struct page *page)
4665{
4666 return page->inuse;
4667}
4668
4669static int count_total(struct page *page)
4670{
4671 return page->objects;
4672}
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004673#endif
Christoph Lameter205ab992008-04-14 19:11:40 +03004674
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004675#ifdef CONFIG_SLUB_DEBUG
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004676static void validate_slab(struct kmem_cache *s, struct page *page)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004677{
4678 void *p;
Christoph Lametera973e9d2008-03-01 13:40:44 -08004679 void *addr = page_address(page);
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004680 unsigned long *map;
4681
4682 slab_lock(page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004683
Yu Zhaodd98afd2019-11-30 17:49:37 -08004684 if (!check_slab(s, page) || !on_freelist(s, page, NULL))
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004685 goto unlock;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004686
4687 /* Now we know that a valid freelist exists */
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004688 map = get_map(s, page);
Christoph Lameter5f80b132011-04-15 14:48:13 -05004689 for_each_object(p, s, addr, page->objects) {
Roman Gushchin4138fdf2020-08-06 23:20:42 -07004690 u8 val = test_bit(__obj_to_index(s, addr, p), map) ?
Yu Zhaodd98afd2019-11-30 17:49:37 -08004691 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004692
Yu Zhaodd98afd2019-11-30 17:49:37 -08004693 if (!check_object(s, page, p, val))
4694 break;
4695 }
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004696 put_map(map);
4697unlock:
Christoph Lameter881db7f2011-06-01 12:25:53 -05004698 slab_unlock(page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004699}
4700
Christoph Lameter434e2452007-07-17 04:03:30 -07004701static int validate_slab_node(struct kmem_cache *s,
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004702 struct kmem_cache_node *n)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004703{
4704 unsigned long count = 0;
4705 struct page *page;
4706 unsigned long flags;
4707
4708 spin_lock_irqsave(&n->list_lock, flags);
4709
Tobin C. Harding916ac052019-05-13 17:16:12 -07004710 list_for_each_entry(page, &n->partial, slab_list) {
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004711 validate_slab(s, page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004712 count++;
4713 }
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07004714 if (count != n->nr_partial) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07004715 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
4716 s->name, count, n->nr_partial);
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07004717 slab_add_kunit_errors();
4718 }
Christoph Lameter53e15af2007-05-06 14:49:43 -07004719
4720 if (!(s->flags & SLAB_STORE_USER))
4721 goto out;
4722
Tobin C. Harding916ac052019-05-13 17:16:12 -07004723 list_for_each_entry(page, &n->full, slab_list) {
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004724 validate_slab(s, page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004725 count++;
4726 }
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07004727 if (count != atomic_long_read(&n->nr_slabs)) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07004728 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
4729 s->name, count, atomic_long_read(&n->nr_slabs));
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07004730 slab_add_kunit_errors();
4731 }
Christoph Lameter53e15af2007-05-06 14:49:43 -07004732
4733out:
4734 spin_unlock_irqrestore(&n->list_lock, flags);
4735 return count;
4736}
4737
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07004738long validate_slab_cache(struct kmem_cache *s)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004739{
4740 int node;
4741 unsigned long count = 0;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004742 struct kmem_cache_node *n;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004743
4744 flush_all(s);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004745 for_each_kmem_cache_node(s, node, n)
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004746 count += validate_slab_node(s, n);
4747
Christoph Lameter53e15af2007-05-06 14:49:43 -07004748 return count;
4749}
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07004750EXPORT_SYMBOL(validate_slab_cache);
4751
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07004752#ifdef CONFIG_DEBUG_FS
Christoph Lameter88a420e2007-05-06 14:49:45 -07004753/*
Christoph Lameter672bba32007-05-09 02:32:39 -07004754 * Generate lists of code addresses where slabcache objects are allocated
Christoph Lameter88a420e2007-05-06 14:49:45 -07004755 * and freed.
4756 */
4757
4758struct location {
4759 unsigned long count;
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004760 unsigned long addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004761 long long sum_time;
4762 long min_time;
4763 long max_time;
4764 long min_pid;
4765 long max_pid;
Rusty Russell174596a2009-01-01 10:12:29 +10304766 DECLARE_BITMAP(cpus, NR_CPUS);
Christoph Lameter45edfa52007-05-09 02:32:45 -07004767 nodemask_t nodes;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004768};
4769
4770struct loc_track {
4771 unsigned long max;
4772 unsigned long count;
4773 struct location *loc;
4774};
4775
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07004776static struct dentry *slab_debugfs_root;
4777
Christoph Lameter88a420e2007-05-06 14:49:45 -07004778static void free_loc_track(struct loc_track *t)
4779{
4780 if (t->max)
4781 free_pages((unsigned long)t->loc,
4782 get_order(sizeof(struct location) * t->max));
4783}
4784
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004785static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004786{
4787 struct location *l;
4788 int order;
4789
Christoph Lameter88a420e2007-05-06 14:49:45 -07004790 order = get_order(sizeof(struct location) * max);
4791
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004792 l = (void *)__get_free_pages(flags, order);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004793 if (!l)
4794 return 0;
4795
4796 if (t->count) {
4797 memcpy(l, t->loc, sizeof(struct location) * t->count);
4798 free_loc_track(t);
4799 }
4800 t->max = max;
4801 t->loc = l;
4802 return 1;
4803}
4804
4805static int add_location(struct loc_track *t, struct kmem_cache *s,
Christoph Lameter45edfa52007-05-09 02:32:45 -07004806 const struct track *track)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004807{
4808 long start, end, pos;
4809 struct location *l;
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004810 unsigned long caddr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004811 unsigned long age = jiffies - track->when;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004812
4813 start = -1;
4814 end = t->count;
4815
4816 for ( ; ; ) {
4817 pos = start + (end - start + 1) / 2;
4818
4819 /*
4820 * There is nothing at "end". If we end up there
4821 * we need to add something to before end.
4822 */
4823 if (pos == end)
4824 break;
4825
4826 caddr = t->loc[pos].addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004827 if (track->addr == caddr) {
4828
4829 l = &t->loc[pos];
4830 l->count++;
4831 if (track->when) {
4832 l->sum_time += age;
4833 if (age < l->min_time)
4834 l->min_time = age;
4835 if (age > l->max_time)
4836 l->max_time = age;
4837
4838 if (track->pid < l->min_pid)
4839 l->min_pid = track->pid;
4840 if (track->pid > l->max_pid)
4841 l->max_pid = track->pid;
4842
Rusty Russell174596a2009-01-01 10:12:29 +10304843 cpumask_set_cpu(track->cpu,
4844 to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07004845 }
4846 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004847 return 1;
4848 }
4849
Christoph Lameter45edfa52007-05-09 02:32:45 -07004850 if (track->addr < caddr)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004851 end = pos;
4852 else
4853 start = pos;
4854 }
4855
4856 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07004857 * Not found. Insert new tracking element.
Christoph Lameter88a420e2007-05-06 14:49:45 -07004858 */
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004859 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
Christoph Lameter88a420e2007-05-06 14:49:45 -07004860 return 0;
4861
4862 l = t->loc + pos;
4863 if (pos < t->count)
4864 memmove(l + 1, l,
4865 (t->count - pos) * sizeof(struct location));
4866 t->count++;
4867 l->count = 1;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004868 l->addr = track->addr;
4869 l->sum_time = age;
4870 l->min_time = age;
4871 l->max_time = age;
4872 l->min_pid = track->pid;
4873 l->max_pid = track->pid;
Rusty Russell174596a2009-01-01 10:12:29 +10304874 cpumask_clear(to_cpumask(l->cpus));
4875 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07004876 nodes_clear(l->nodes);
4877 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004878 return 1;
4879}
4880
4881static void process_slab(struct loc_track *t, struct kmem_cache *s,
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004882 struct page *page, enum track_item alloc)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004883{
Christoph Lametera973e9d2008-03-01 13:40:44 -08004884 void *addr = page_address(page);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004885 void *p;
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004886 unsigned long *map;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004887
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004888 map = get_map(s, page);
Christoph Lameter224a88b2008-04-14 19:11:31 +03004889 for_each_object(p, s, addr, page->objects)
Roman Gushchin4138fdf2020-08-06 23:20:42 -07004890 if (!test_bit(__obj_to_index(s, addr, p), map))
Christoph Lameter45edfa52007-05-09 02:32:45 -07004891 add_location(t, s, get_track(s, p, alloc));
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004892 put_map(map);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004893}
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07004894#endif /* CONFIG_DEBUG_FS */
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07004895#endif /* CONFIG_SLUB_DEBUG */
Christoph Lameter88a420e2007-05-06 14:49:45 -07004896
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004897#ifdef CONFIG_SYSFS
Christoph Lameter81819f02007-05-06 14:49:36 -07004898enum slab_stat_type {
Christoph Lameter205ab992008-04-14 19:11:40 +03004899 SL_ALL, /* All slabs */
4900 SL_PARTIAL, /* Only partially allocated slabs */
4901 SL_CPU, /* Only slabs used for cpu caches */
4902 SL_OBJECTS, /* Determine allocated objects not slabs */
4903 SL_TOTAL /* Determine object capacity not slabs */
Christoph Lameter81819f02007-05-06 14:49:36 -07004904};
4905
Christoph Lameter205ab992008-04-14 19:11:40 +03004906#define SO_ALL (1 << SL_ALL)
Christoph Lameter81819f02007-05-06 14:49:36 -07004907#define SO_PARTIAL (1 << SL_PARTIAL)
4908#define SO_CPU (1 << SL_CPU)
4909#define SO_OBJECTS (1 << SL_OBJECTS)
Christoph Lameter205ab992008-04-14 19:11:40 +03004910#define SO_TOTAL (1 << SL_TOTAL)
Christoph Lameter81819f02007-05-06 14:49:36 -07004911
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03004912static ssize_t show_slab_objects(struct kmem_cache *s,
Joe Perchesbf16d192020-12-14 19:14:57 -08004913 char *buf, unsigned long flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07004914{
4915 unsigned long total = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07004916 int node;
4917 int x;
4918 unsigned long *nodes;
Joe Perchesbf16d192020-12-14 19:14:57 -08004919 int len = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07004920
Kees Cook6396bb22018-06-12 14:03:40 -07004921 nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03004922 if (!nodes)
4923 return -ENOMEM;
Christoph Lameter81819f02007-05-06 14:49:36 -07004924
Christoph Lameter205ab992008-04-14 19:11:40 +03004925 if (flags & SO_CPU) {
4926 int cpu;
Christoph Lameter81819f02007-05-06 14:49:36 -07004927
Christoph Lameter205ab992008-04-14 19:11:40 +03004928 for_each_possible_cpu(cpu) {
Chen Gangd0e0ac92013-07-15 09:05:29 +08004929 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
4930 cpu);
Christoph Lameterec3ab082012-05-09 10:09:56 -05004931 int node;
Christoph Lameter49e22582011-08-09 16:12:27 -05004932 struct page *page;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07004933
Jason Low4db0c3c2015-04-15 16:14:08 -07004934 page = READ_ONCE(c->page);
Christoph Lameterec3ab082012-05-09 10:09:56 -05004935 if (!page)
4936 continue;
Christoph Lameter205ab992008-04-14 19:11:40 +03004937
Christoph Lameterec3ab082012-05-09 10:09:56 -05004938 node = page_to_nid(page);
4939 if (flags & SO_TOTAL)
4940 x = page->objects;
4941 else if (flags & SO_OBJECTS)
4942 x = page->inuse;
4943 else
4944 x = 1;
Christoph Lameter49e22582011-08-09 16:12:27 -05004945
Christoph Lameterec3ab082012-05-09 10:09:56 -05004946 total += x;
4947 nodes[node] += x;
4948
Wei Yanga93cf072017-07-06 15:36:31 -07004949 page = slub_percpu_partial_read_once(c);
Christoph Lameter49e22582011-08-09 16:12:27 -05004950 if (page) {
Li Zefan8afb1472013-09-10 11:43:37 +08004951 node = page_to_nid(page);
4952 if (flags & SO_TOTAL)
4953 WARN_ON_ONCE(1);
4954 else if (flags & SO_OBJECTS)
4955 WARN_ON_ONCE(1);
4956 else
4957 x = page->pages;
Eric Dumazetbc6697d2011-11-22 16:02:02 +01004958 total += x;
4959 nodes[node] += x;
Christoph Lameter49e22582011-08-09 16:12:27 -05004960 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004961 }
4962 }
4963
Qian Caie4f8e512019-10-14 14:11:51 -07004964 /*
4965 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
4966 * already held which will conflict with an existing lock order:
4967 *
4968 * mem_hotplug_lock->slab_mutex->kernfs_mutex
4969 *
4970 * We don't really need mem_hotplug_lock (to hold off
4971 * slab_mem_going_offline_callback) here because slab's memory hot
4972 * unplug code doesn't destroy the kmem_cache->node[] data.
4973 */
4974
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004975#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter205ab992008-04-14 19:11:40 +03004976 if (flags & SO_ALL) {
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004977 struct kmem_cache_node *n;
4978
4979 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter81819f02007-05-06 14:49:36 -07004980
Chen Gangd0e0ac92013-07-15 09:05:29 +08004981 if (flags & SO_TOTAL)
4982 x = atomic_long_read(&n->total_objects);
4983 else if (flags & SO_OBJECTS)
4984 x = atomic_long_read(&n->total_objects) -
4985 count_partial(n, count_free);
Christoph Lameter205ab992008-04-14 19:11:40 +03004986 else
4987 x = atomic_long_read(&n->nr_slabs);
4988 total += x;
4989 nodes[node] += x;
4990 }
4991
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004992 } else
4993#endif
4994 if (flags & SO_PARTIAL) {
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004995 struct kmem_cache_node *n;
Christoph Lameter205ab992008-04-14 19:11:40 +03004996
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004997 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter205ab992008-04-14 19:11:40 +03004998 if (flags & SO_TOTAL)
4999 x = count_partial(n, count_total);
5000 else if (flags & SO_OBJECTS)
5001 x = count_partial(n, count_inuse);
Christoph Lameter81819f02007-05-06 14:49:36 -07005002 else
5003 x = n->nr_partial;
5004 total += x;
5005 nodes[node] += x;
5006 }
Christoph Lameter81819f02007-05-06 14:49:36 -07005007 }
Joe Perchesbf16d192020-12-14 19:14:57 -08005008
5009 len += sysfs_emit_at(buf, len, "%lu", total);
Christoph Lameter81819f02007-05-06 14:49:36 -07005010#ifdef CONFIG_NUMA
Joe Perchesbf16d192020-12-14 19:14:57 -08005011 for (node = 0; node < nr_node_ids; node++) {
Christoph Lameter81819f02007-05-06 14:49:36 -07005012 if (nodes[node])
Joe Perchesbf16d192020-12-14 19:14:57 -08005013 len += sysfs_emit_at(buf, len, " N%d=%lu",
5014 node, nodes[node]);
5015 }
Christoph Lameter81819f02007-05-06 14:49:36 -07005016#endif
Joe Perchesbf16d192020-12-14 19:14:57 -08005017 len += sysfs_emit_at(buf, len, "\n");
Christoph Lameter81819f02007-05-06 14:49:36 -07005018 kfree(nodes);
Joe Perchesbf16d192020-12-14 19:14:57 -08005019
5020 return len;
Christoph Lameter81819f02007-05-06 14:49:36 -07005021}
5022
Christoph Lameter81819f02007-05-06 14:49:36 -07005023#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
Phil Carmody497888c2011-07-14 15:07:13 +03005024#define to_slab(n) container_of(n, struct kmem_cache, kobj)
Christoph Lameter81819f02007-05-06 14:49:36 -07005025
5026struct slab_attribute {
5027 struct attribute attr;
5028 ssize_t (*show)(struct kmem_cache *s, char *buf);
5029 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
5030};
5031
5032#define SLAB_ATTR_RO(_name) \
Vasiliy Kulikovab067e992011-09-27 21:54:53 +04005033 static struct slab_attribute _name##_attr = \
5034 __ATTR(_name, 0400, _name##_show, NULL)
Christoph Lameter81819f02007-05-06 14:49:36 -07005035
5036#define SLAB_ATTR(_name) \
5037 static struct slab_attribute _name##_attr = \
Vasiliy Kulikovab067e992011-09-27 21:54:53 +04005038 __ATTR(_name, 0600, _name##_show, _name##_store)
Christoph Lameter81819f02007-05-06 14:49:36 -07005039
Christoph Lameter81819f02007-05-06 14:49:36 -07005040static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
5041{
Joe Perchesbf16d192020-12-14 19:14:57 -08005042 return sysfs_emit(buf, "%u\n", s->size);
Christoph Lameter81819f02007-05-06 14:49:36 -07005043}
5044SLAB_ATTR_RO(slab_size);
5045
5046static ssize_t align_show(struct kmem_cache *s, char *buf)
5047{
Joe Perchesbf16d192020-12-14 19:14:57 -08005048 return sysfs_emit(buf, "%u\n", s->align);
Christoph Lameter81819f02007-05-06 14:49:36 -07005049}
5050SLAB_ATTR_RO(align);
5051
5052static ssize_t object_size_show(struct kmem_cache *s, char *buf)
5053{
Joe Perchesbf16d192020-12-14 19:14:57 -08005054 return sysfs_emit(buf, "%u\n", s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -07005055}
5056SLAB_ATTR_RO(object_size);
5057
5058static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
5059{
Joe Perchesbf16d192020-12-14 19:14:57 -08005060 return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07005061}
5062SLAB_ATTR_RO(objs_per_slab);
5063
5064static ssize_t order_show(struct kmem_cache *s, char *buf)
5065{
Joe Perchesbf16d192020-12-14 19:14:57 -08005066 return sysfs_emit(buf, "%u\n", oo_order(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07005067}
Vlastimil Babka32a6f402020-08-06 23:18:41 -07005068SLAB_ATTR_RO(order);
Christoph Lameter81819f02007-05-06 14:49:36 -07005069
David Rientjes73d342b2009-02-22 17:40:09 -08005070static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
5071{
Joe Perchesbf16d192020-12-14 19:14:57 -08005072 return sysfs_emit(buf, "%lu\n", s->min_partial);
David Rientjes73d342b2009-02-22 17:40:09 -08005073}
5074
5075static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
5076 size_t length)
5077{
5078 unsigned long min;
5079 int err;
5080
Jingoo Han3dbb95f2013-09-11 14:20:25 -07005081 err = kstrtoul(buf, 10, &min);
David Rientjes73d342b2009-02-22 17:40:09 -08005082 if (err)
5083 return err;
5084
David Rientjesc0bdb232009-02-25 09:16:35 +02005085 set_min_partial(s, min);
David Rientjes73d342b2009-02-22 17:40:09 -08005086 return length;
5087}
5088SLAB_ATTR(min_partial);
5089
Christoph Lameter49e22582011-08-09 16:12:27 -05005090static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
5091{
Joe Perchesbf16d192020-12-14 19:14:57 -08005092 return sysfs_emit(buf, "%u\n", slub_cpu_partial(s));
Christoph Lameter49e22582011-08-09 16:12:27 -05005093}
5094
5095static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
5096 size_t length)
5097{
Alexey Dobriyane5d99982018-04-05 16:21:10 -07005098 unsigned int objects;
Christoph Lameter49e22582011-08-09 16:12:27 -05005099 int err;
5100
Alexey Dobriyane5d99982018-04-05 16:21:10 -07005101 err = kstrtouint(buf, 10, &objects);
Christoph Lameter49e22582011-08-09 16:12:27 -05005102 if (err)
5103 return err;
Joonsoo Kim345c9052013-06-19 14:05:52 +09005104 if (objects && !kmem_cache_has_cpu_partial(s))
David Rientjes74ee4ef2012-01-09 13:19:45 -08005105 return -EINVAL;
Christoph Lameter49e22582011-08-09 16:12:27 -05005106
Wei Yange6d0e1d2017-07-06 15:36:34 -07005107 slub_set_cpu_partial(s, objects);
Christoph Lameter49e22582011-08-09 16:12:27 -05005108 flush_all(s);
5109 return length;
5110}
5111SLAB_ATTR(cpu_partial);
5112
Christoph Lameter81819f02007-05-06 14:49:36 -07005113static ssize_t ctor_show(struct kmem_cache *s, char *buf)
5114{
Joe Perches62c70bc2011-01-13 15:45:52 -08005115 if (!s->ctor)
5116 return 0;
Joe Perchesbf16d192020-12-14 19:14:57 -08005117 return sysfs_emit(buf, "%pS\n", s->ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07005118}
5119SLAB_ATTR_RO(ctor);
5120
Christoph Lameter81819f02007-05-06 14:49:36 -07005121static ssize_t aliases_show(struct kmem_cache *s, char *buf)
5122{
Joe Perchesbf16d192020-12-14 19:14:57 -08005123 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07005124}
5125SLAB_ATTR_RO(aliases);
5126
Christoph Lameter81819f02007-05-06 14:49:36 -07005127static ssize_t partial_show(struct kmem_cache *s, char *buf)
5128{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08005129 return show_slab_objects(s, buf, SO_PARTIAL);
Christoph Lameter81819f02007-05-06 14:49:36 -07005130}
5131SLAB_ATTR_RO(partial);
5132
5133static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
5134{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08005135 return show_slab_objects(s, buf, SO_CPU);
Christoph Lameter81819f02007-05-06 14:49:36 -07005136}
5137SLAB_ATTR_RO(cpu_slabs);
5138
5139static ssize_t objects_show(struct kmem_cache *s, char *buf)
5140{
Christoph Lameter205ab992008-04-14 19:11:40 +03005141 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
Christoph Lameter81819f02007-05-06 14:49:36 -07005142}
5143SLAB_ATTR_RO(objects);
5144
Christoph Lameter205ab992008-04-14 19:11:40 +03005145static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
5146{
5147 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
5148}
5149SLAB_ATTR_RO(objects_partial);
5150
Christoph Lameter49e22582011-08-09 16:12:27 -05005151static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
5152{
5153 int objects = 0;
5154 int pages = 0;
5155 int cpu;
Joe Perchesbf16d192020-12-14 19:14:57 -08005156 int len = 0;
Christoph Lameter49e22582011-08-09 16:12:27 -05005157
5158 for_each_online_cpu(cpu) {
Wei Yanga93cf072017-07-06 15:36:31 -07005159 struct page *page;
5160
5161 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
Christoph Lameter49e22582011-08-09 16:12:27 -05005162
5163 if (page) {
5164 pages += page->pages;
5165 objects += page->pobjects;
5166 }
5167 }
5168
Joe Perchesbf16d192020-12-14 19:14:57 -08005169 len += sysfs_emit_at(buf, len, "%d(%d)", objects, pages);
Christoph Lameter49e22582011-08-09 16:12:27 -05005170
5171#ifdef CONFIG_SMP
5172 for_each_online_cpu(cpu) {
Wei Yanga93cf072017-07-06 15:36:31 -07005173 struct page *page;
5174
5175 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
Joe Perchesbf16d192020-12-14 19:14:57 -08005176 if (page)
5177 len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
5178 cpu, page->pobjects, page->pages);
Christoph Lameter49e22582011-08-09 16:12:27 -05005179 }
5180#endif
Joe Perchesbf16d192020-12-14 19:14:57 -08005181 len += sysfs_emit_at(buf, len, "\n");
5182
5183 return len;
Christoph Lameter49e22582011-08-09 16:12:27 -05005184}
5185SLAB_ATTR_RO(slabs_cpu_partial);
5186
Christoph Lameter81819f02007-05-06 14:49:36 -07005187static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
5188{
Joe Perchesbf16d192020-12-14 19:14:57 -08005189 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
Christoph Lameter81819f02007-05-06 14:49:36 -07005190}
Vlastimil Babka8f58119a2020-08-06 23:18:48 -07005191SLAB_ATTR_RO(reclaim_account);
Christoph Lameter81819f02007-05-06 14:49:36 -07005192
5193static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
5194{
Joe Perchesbf16d192020-12-14 19:14:57 -08005195 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
Christoph Lameter81819f02007-05-06 14:49:36 -07005196}
5197SLAB_ATTR_RO(hwcache_align);
5198
5199#ifdef CONFIG_ZONE_DMA
5200static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
5201{
Joe Perchesbf16d192020-12-14 19:14:57 -08005202 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
Christoph Lameter81819f02007-05-06 14:49:36 -07005203}
5204SLAB_ATTR_RO(cache_dma);
5205#endif
5206
David Windsor8eb82842017-06-10 22:50:28 -04005207static ssize_t usersize_show(struct kmem_cache *s, char *buf)
5208{
Joe Perchesbf16d192020-12-14 19:14:57 -08005209 return sysfs_emit(buf, "%u\n", s->usersize);
David Windsor8eb82842017-06-10 22:50:28 -04005210}
5211SLAB_ATTR_RO(usersize);
5212
Christoph Lameter81819f02007-05-06 14:49:36 -07005213static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
5214{
Joe Perchesbf16d192020-12-14 19:14:57 -08005215 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
Christoph Lameter81819f02007-05-06 14:49:36 -07005216}
5217SLAB_ATTR_RO(destroy_by_rcu);
5218
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005219#ifdef CONFIG_SLUB_DEBUG
Christoph Lametera5a84752010-10-05 13:57:27 -05005220static ssize_t slabs_show(struct kmem_cache *s, char *buf)
5221{
5222 return show_slab_objects(s, buf, SO_ALL);
5223}
5224SLAB_ATTR_RO(slabs);
5225
5226static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
5227{
5228 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
5229}
5230SLAB_ATTR_RO(total_objects);
5231
5232static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
5233{
Joe Perchesbf16d192020-12-14 19:14:57 -08005234 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
Christoph Lametera5a84752010-10-05 13:57:27 -05005235}
Vlastimil Babka060807f2020-08-06 23:18:45 -07005236SLAB_ATTR_RO(sanity_checks);
Christoph Lametera5a84752010-10-05 13:57:27 -05005237
5238static ssize_t trace_show(struct kmem_cache *s, char *buf)
5239{
Joe Perchesbf16d192020-12-14 19:14:57 -08005240 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
Christoph Lametera5a84752010-10-05 13:57:27 -05005241}
Vlastimil Babka060807f2020-08-06 23:18:45 -07005242SLAB_ATTR_RO(trace);
Christoph Lametera5a84752010-10-05 13:57:27 -05005243
Christoph Lameter81819f02007-05-06 14:49:36 -07005244static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
5245{
Joe Perchesbf16d192020-12-14 19:14:57 -08005246 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
Christoph Lameter81819f02007-05-06 14:49:36 -07005247}
5248
Vlastimil Babkaad38b5b2020-08-06 23:18:38 -07005249SLAB_ATTR_RO(red_zone);
Christoph Lameter81819f02007-05-06 14:49:36 -07005250
5251static ssize_t poison_show(struct kmem_cache *s, char *buf)
5252{
Joe Perchesbf16d192020-12-14 19:14:57 -08005253 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
Christoph Lameter81819f02007-05-06 14:49:36 -07005254}
5255
Vlastimil Babkaad38b5b2020-08-06 23:18:38 -07005256SLAB_ATTR_RO(poison);
Christoph Lameter81819f02007-05-06 14:49:36 -07005257
5258static ssize_t store_user_show(struct kmem_cache *s, char *buf)
5259{
Joe Perchesbf16d192020-12-14 19:14:57 -08005260 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
Christoph Lameter81819f02007-05-06 14:49:36 -07005261}
5262
Vlastimil Babkaad38b5b2020-08-06 23:18:38 -07005263SLAB_ATTR_RO(store_user);
Christoph Lameter81819f02007-05-06 14:49:36 -07005264
Christoph Lameter53e15af2007-05-06 14:49:43 -07005265static ssize_t validate_show(struct kmem_cache *s, char *buf)
5266{
5267 return 0;
5268}
5269
5270static ssize_t validate_store(struct kmem_cache *s,
5271 const char *buf, size_t length)
5272{
Christoph Lameter434e2452007-07-17 04:03:30 -07005273 int ret = -EINVAL;
5274
5275 if (buf[0] == '1') {
5276 ret = validate_slab_cache(s);
5277 if (ret >= 0)
5278 ret = length;
5279 }
5280 return ret;
Christoph Lameter53e15af2007-05-06 14:49:43 -07005281}
5282SLAB_ATTR(validate);
Christoph Lametera5a84752010-10-05 13:57:27 -05005283
Christoph Lametera5a84752010-10-05 13:57:27 -05005284#endif /* CONFIG_SLUB_DEBUG */
5285
5286#ifdef CONFIG_FAILSLAB
5287static ssize_t failslab_show(struct kmem_cache *s, char *buf)
5288{
Joe Perchesbf16d192020-12-14 19:14:57 -08005289 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
Christoph Lametera5a84752010-10-05 13:57:27 -05005290}
Vlastimil Babka060807f2020-08-06 23:18:45 -07005291SLAB_ATTR_RO(failslab);
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005292#endif
Christoph Lameter53e15af2007-05-06 14:49:43 -07005293
Christoph Lameter2086d262007-05-06 14:49:46 -07005294static ssize_t shrink_show(struct kmem_cache *s, char *buf)
5295{
5296 return 0;
5297}
5298
5299static ssize_t shrink_store(struct kmem_cache *s,
5300 const char *buf, size_t length)
5301{
Vladimir Davydov832f37f2015-02-12 14:59:41 -08005302 if (buf[0] == '1')
Roman Gushchin10befea2020-08-06 23:21:27 -07005303 kmem_cache_shrink(s);
Vladimir Davydov832f37f2015-02-12 14:59:41 -08005304 else
Christoph Lameter2086d262007-05-06 14:49:46 -07005305 return -EINVAL;
5306 return length;
5307}
5308SLAB_ATTR(shrink);
5309
Christoph Lameter81819f02007-05-06 14:49:36 -07005310#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08005311static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
Christoph Lameter81819f02007-05-06 14:49:36 -07005312{
Joe Perchesbf16d192020-12-14 19:14:57 -08005313 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
Christoph Lameter81819f02007-05-06 14:49:36 -07005314}
5315
Christoph Lameter98246012008-01-07 23:20:26 -08005316static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
Christoph Lameter81819f02007-05-06 14:49:36 -07005317 const char *buf, size_t length)
5318{
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005319 unsigned int ratio;
Christoph Lameter0121c6192008-04-29 16:11:12 -07005320 int err;
Christoph Lameter81819f02007-05-06 14:49:36 -07005321
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005322 err = kstrtouint(buf, 10, &ratio);
Christoph Lameter0121c6192008-04-29 16:11:12 -07005323 if (err)
5324 return err;
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005325 if (ratio > 100)
5326 return -ERANGE;
Christoph Lameter0121c6192008-04-29 16:11:12 -07005327
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005328 s->remote_node_defrag_ratio = ratio * 10;
Christoph Lameter0121c6192008-04-29 16:11:12 -07005329
Christoph Lameter81819f02007-05-06 14:49:36 -07005330 return length;
5331}
Christoph Lameter98246012008-01-07 23:20:26 -08005332SLAB_ATTR(remote_node_defrag_ratio);
Christoph Lameter81819f02007-05-06 14:49:36 -07005333#endif
5334
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005335#ifdef CONFIG_SLUB_STATS
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005336static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
5337{
5338 unsigned long sum = 0;
5339 int cpu;
Joe Perchesbf16d192020-12-14 19:14:57 -08005340 int len = 0;
Kees Cook6da2ec52018-06-12 13:55:00 -07005341 int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005342
5343 if (!data)
5344 return -ENOMEM;
5345
5346 for_each_online_cpu(cpu) {
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06005347 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005348
5349 data[cpu] = x;
5350 sum += x;
5351 }
5352
Joe Perchesbf16d192020-12-14 19:14:57 -08005353 len += sysfs_emit_at(buf, len, "%lu", sum);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005354
Christoph Lameter50ef37b2008-04-14 18:52:05 +03005355#ifdef CONFIG_SMP
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005356 for_each_online_cpu(cpu) {
Joe Perchesbf16d192020-12-14 19:14:57 -08005357 if (data[cpu])
5358 len += sysfs_emit_at(buf, len, " C%d=%u",
5359 cpu, data[cpu]);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005360 }
Christoph Lameter50ef37b2008-04-14 18:52:05 +03005361#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005362 kfree(data);
Joe Perchesbf16d192020-12-14 19:14:57 -08005363 len += sysfs_emit_at(buf, len, "\n");
5364
5365 return len;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005366}
5367
David Rientjes78eb00c2009-10-15 02:20:22 -07005368static void clear_stat(struct kmem_cache *s, enum stat_item si)
5369{
5370 int cpu;
5371
5372 for_each_online_cpu(cpu)
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06005373 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
David Rientjes78eb00c2009-10-15 02:20:22 -07005374}
5375
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005376#define STAT_ATTR(si, text) \
5377static ssize_t text##_show(struct kmem_cache *s, char *buf) \
5378{ \
5379 return show_stat(s, buf, si); \
5380} \
David Rientjes78eb00c2009-10-15 02:20:22 -07005381static ssize_t text##_store(struct kmem_cache *s, \
5382 const char *buf, size_t length) \
5383{ \
5384 if (buf[0] != '0') \
5385 return -EINVAL; \
5386 clear_stat(s, si); \
5387 return length; \
5388} \
5389SLAB_ATTR(text); \
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005390
5391STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5392STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5393STAT_ATTR(FREE_FASTPATH, free_fastpath);
5394STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5395STAT_ATTR(FREE_FROZEN, free_frozen);
5396STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5397STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5398STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5399STAT_ATTR(ALLOC_SLAB, alloc_slab);
5400STAT_ATTR(ALLOC_REFILL, alloc_refill);
Christoph Lametere36a2652011-06-01 12:25:57 -05005401STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005402STAT_ATTR(FREE_SLAB, free_slab);
5403STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5404STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5405STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5406STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5407STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5408STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
Christoph Lameter03e404a2011-06-01 12:25:58 -05005409STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
Christoph Lameter65c33762008-04-14 19:11:40 +03005410STAT_ATTR(ORDER_FALLBACK, order_fallback);
Christoph Lameterb789ef52011-06-01 12:25:49 -05005411STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5412STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
Christoph Lameter49e22582011-08-09 16:12:27 -05005413STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5414STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
Alex Shi8028dce2012-02-03 23:34:56 +08005415STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5416STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07005417#endif /* CONFIG_SLUB_STATS */
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005418
Pekka Enberg06428782008-01-07 23:20:27 -08005419static struct attribute *slab_attrs[] = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005420 &slab_size_attr.attr,
5421 &object_size_attr.attr,
5422 &objs_per_slab_attr.attr,
5423 &order_attr.attr,
David Rientjes73d342b2009-02-22 17:40:09 -08005424 &min_partial_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005425 &cpu_partial_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005426 &objects_attr.attr,
Christoph Lameter205ab992008-04-14 19:11:40 +03005427 &objects_partial_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005428 &partial_attr.attr,
5429 &cpu_slabs_attr.attr,
5430 &ctor_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005431 &aliases_attr.attr,
5432 &align_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005433 &hwcache_align_attr.attr,
5434 &reclaim_account_attr.attr,
5435 &destroy_by_rcu_attr.attr,
Christoph Lametera5a84752010-10-05 13:57:27 -05005436 &shrink_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005437 &slabs_cpu_partial_attr.attr,
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005438#ifdef CONFIG_SLUB_DEBUG
Christoph Lametera5a84752010-10-05 13:57:27 -05005439 &total_objects_attr.attr,
5440 &slabs_attr.attr,
5441 &sanity_checks_attr.attr,
5442 &trace_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005443 &red_zone_attr.attr,
5444 &poison_attr.attr,
5445 &store_user_attr.attr,
Christoph Lameter53e15af2007-05-06 14:49:43 -07005446 &validate_attr.attr,
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005447#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07005448#ifdef CONFIG_ZONE_DMA
5449 &cache_dma_attr.attr,
5450#endif
5451#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08005452 &remote_node_defrag_ratio_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005453#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005454#ifdef CONFIG_SLUB_STATS
5455 &alloc_fastpath_attr.attr,
5456 &alloc_slowpath_attr.attr,
5457 &free_fastpath_attr.attr,
5458 &free_slowpath_attr.attr,
5459 &free_frozen_attr.attr,
5460 &free_add_partial_attr.attr,
5461 &free_remove_partial_attr.attr,
5462 &alloc_from_partial_attr.attr,
5463 &alloc_slab_attr.attr,
5464 &alloc_refill_attr.attr,
Christoph Lametere36a2652011-06-01 12:25:57 -05005465 &alloc_node_mismatch_attr.attr,
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005466 &free_slab_attr.attr,
5467 &cpuslab_flush_attr.attr,
5468 &deactivate_full_attr.attr,
5469 &deactivate_empty_attr.attr,
5470 &deactivate_to_head_attr.attr,
5471 &deactivate_to_tail_attr.attr,
5472 &deactivate_remote_frees_attr.attr,
Christoph Lameter03e404a2011-06-01 12:25:58 -05005473 &deactivate_bypass_attr.attr,
Christoph Lameter65c33762008-04-14 19:11:40 +03005474 &order_fallback_attr.attr,
Christoph Lameterb789ef52011-06-01 12:25:49 -05005475 &cmpxchg_double_fail_attr.attr,
5476 &cmpxchg_double_cpu_fail_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005477 &cpu_partial_alloc_attr.attr,
5478 &cpu_partial_free_attr.attr,
Alex Shi8028dce2012-02-03 23:34:56 +08005479 &cpu_partial_node_attr.attr,
5480 &cpu_partial_drain_attr.attr,
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005481#endif
Dmitry Monakhov4c13dd32010-02-26 09:36:12 +03005482#ifdef CONFIG_FAILSLAB
5483 &failslab_attr.attr,
5484#endif
David Windsor8eb82842017-06-10 22:50:28 -04005485 &usersize_attr.attr,
Dmitry Monakhov4c13dd32010-02-26 09:36:12 +03005486
Christoph Lameter81819f02007-05-06 14:49:36 -07005487 NULL
5488};
5489
Arvind Yadav1fdaaa22017-09-06 16:21:56 -07005490static const struct attribute_group slab_attr_group = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005491 .attrs = slab_attrs,
5492};
5493
5494static ssize_t slab_attr_show(struct kobject *kobj,
5495 struct attribute *attr,
5496 char *buf)
5497{
5498 struct slab_attribute *attribute;
5499 struct kmem_cache *s;
5500 int err;
5501
5502 attribute = to_slab_attr(attr);
5503 s = to_slab(kobj);
5504
5505 if (!attribute->show)
5506 return -EIO;
5507
5508 err = attribute->show(s, buf);
5509
5510 return err;
5511}
5512
5513static ssize_t slab_attr_store(struct kobject *kobj,
5514 struct attribute *attr,
5515 const char *buf, size_t len)
5516{
5517 struct slab_attribute *attribute;
5518 struct kmem_cache *s;
5519 int err;
5520
5521 attribute = to_slab_attr(attr);
5522 s = to_slab(kobj);
5523
5524 if (!attribute->store)
5525 return -EIO;
5526
5527 err = attribute->store(s, buf, len);
Christoph Lameter81819f02007-05-06 14:49:36 -07005528 return err;
5529}
5530
Christoph Lameter41a21282014-05-06 12:50:08 -07005531static void kmem_cache_release(struct kobject *k)
5532{
5533 slab_kmem_cache_release(to_slab(k));
5534}
5535
Emese Revfy52cf25d2010-01-19 02:58:23 +01005536static const struct sysfs_ops slab_sysfs_ops = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005537 .show = slab_attr_show,
5538 .store = slab_attr_store,
5539};
5540
5541static struct kobj_type slab_ktype = {
5542 .sysfs_ops = &slab_sysfs_ops,
Christoph Lameter41a21282014-05-06 12:50:08 -07005543 .release = kmem_cache_release,
Christoph Lameter81819f02007-05-06 14:49:36 -07005544};
5545
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005546static struct kset *slab_kset;
Christoph Lameter81819f02007-05-06 14:49:36 -07005547
Vladimir Davydov9a417072014-04-07 15:39:31 -07005548static inline struct kset *cache_kset(struct kmem_cache *s)
5549{
Vladimir Davydov9a417072014-04-07 15:39:31 -07005550 return slab_kset;
5551}
5552
Christoph Lameter81819f02007-05-06 14:49:36 -07005553#define ID_STR_LENGTH 64
5554
5555/* Create a unique string id for a slab cache:
Christoph Lameter6446faa2008-02-15 23:45:26 -08005556 *
5557 * Format :[flags-]size
Christoph Lameter81819f02007-05-06 14:49:36 -07005558 */
5559static char *create_unique_id(struct kmem_cache *s)
5560{
5561 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5562 char *p = name;
5563
5564 BUG_ON(!name);
5565
5566 *p++ = ':';
5567 /*
5568 * First flags affecting slabcache operations. We will only
5569 * get here for aliasable slabs so we do not need to support
5570 * too many flags. The flags here must cover all flags that
5571 * are matched during merging to guarantee that the id is
5572 * unique.
5573 */
5574 if (s->flags & SLAB_CACHE_DMA)
5575 *p++ = 'd';
Nicolas Boichat6d6ea1e2019-03-28 20:43:42 -07005576 if (s->flags & SLAB_CACHE_DMA32)
5577 *p++ = 'D';
Christoph Lameter81819f02007-05-06 14:49:36 -07005578 if (s->flags & SLAB_RECLAIM_ACCOUNT)
5579 *p++ = 'a';
Laura Abbottbecfda62016-03-15 14:55:06 -07005580 if (s->flags & SLAB_CONSISTENCY_CHECKS)
Christoph Lameter81819f02007-05-06 14:49:36 -07005581 *p++ = 'F';
Vladimir Davydov230e9fc2016-01-14 15:18:15 -08005582 if (s->flags & SLAB_ACCOUNT)
5583 *p++ = 'A';
Christoph Lameter81819f02007-05-06 14:49:36 -07005584 if (p != name + 1)
5585 *p++ = '-';
Alexey Dobriyan44065b22018-04-05 16:21:20 -07005586 p += sprintf(p, "%07u", s->size);
Glauber Costa2633d7a2012-12-18 14:22:34 -08005587
Christoph Lameter81819f02007-05-06 14:49:36 -07005588 BUG_ON(p > name + ID_STR_LENGTH - 1);
5589 return name;
5590}
5591
5592static int sysfs_slab_add(struct kmem_cache *s)
5593{
5594 int err;
5595 const char *name;
Tejun Heo1663f262017-02-22 15:41:39 -08005596 struct kset *kset = cache_kset(s);
Christoph Lameter45530c42012-11-28 16:23:07 +00005597 int unmergeable = slab_unmergeable(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07005598
Tejun Heo1663f262017-02-22 15:41:39 -08005599 if (!kset) {
5600 kobject_init(&s->kobj, &slab_ktype);
5601 return 0;
5602 }
5603
Miles Chen11066382017-11-15 17:32:25 -08005604 if (!unmergeable && disable_higher_order_debug &&
5605 (slub_debug & DEBUG_METADATA_FLAGS))
5606 unmergeable = 1;
5607
Christoph Lameter81819f02007-05-06 14:49:36 -07005608 if (unmergeable) {
5609 /*
5610 * Slabcache can never be merged so we can use the name proper.
5611 * This is typically the case for debug situations. In that
5612 * case we can catch duplicate names easily.
5613 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005614 sysfs_remove_link(&slab_kset->kobj, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005615 name = s->name;
5616 } else {
5617 /*
5618 * Create a unique name for the slab as a target
5619 * for the symlinks.
5620 */
5621 name = create_unique_id(s);
5622 }
5623
Tejun Heo1663f262017-02-22 15:41:39 -08005624 s->kobj.kset = kset;
Tetsuo Handa26e4f202014-01-04 16:32:31 +09005625 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
Wang Hai757fed12021-01-28 19:32:50 +08005626 if (err)
Konstantin Khlebnikov80da0262015-09-04 15:45:51 -07005627 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07005628
5629 err = sysfs_create_group(&s->kobj, &slab_attr_group);
Dave Jones54b6a732014-04-07 15:39:32 -07005630 if (err)
5631 goto out_del_kobj;
Vladimir Davydov9a417072014-04-07 15:39:31 -07005632
Christoph Lameter81819f02007-05-06 14:49:36 -07005633 if (!unmergeable) {
5634 /* Setup first alias */
5635 sysfs_slab_alias(s, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005636 }
Dave Jones54b6a732014-04-07 15:39:32 -07005637out:
5638 if (!unmergeable)
5639 kfree(name);
5640 return err;
5641out_del_kobj:
5642 kobject_del(&s->kobj);
Dave Jones54b6a732014-04-07 15:39:32 -07005643 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07005644}
5645
Mikulas Patockad50d82f2018-06-27 23:26:09 -07005646void sysfs_slab_unlink(struct kmem_cache *s)
5647{
5648 if (slab_state >= FULL)
5649 kobject_del(&s->kobj);
5650}
5651
Tejun Heobf5eb3d2017-02-22 15:41:11 -08005652void sysfs_slab_release(struct kmem_cache *s)
5653{
5654 if (slab_state >= FULL)
5655 kobject_put(&s->kobj);
Christoph Lameter81819f02007-05-06 14:49:36 -07005656}
5657
5658/*
5659 * Need to buffer aliases during bootup until sysfs becomes
Nick Andrew9f6c708e2008-12-05 14:08:08 +11005660 * available lest we lose that information.
Christoph Lameter81819f02007-05-06 14:49:36 -07005661 */
5662struct saved_alias {
5663 struct kmem_cache *s;
5664 const char *name;
5665 struct saved_alias *next;
5666};
5667
Adrian Bunk5af328a2007-07-17 04:03:27 -07005668static struct saved_alias *alias_list;
Christoph Lameter81819f02007-05-06 14:49:36 -07005669
5670static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
5671{
5672 struct saved_alias *al;
5673
Christoph Lameter97d06602012-07-06 15:25:11 -05005674 if (slab_state == FULL) {
Christoph Lameter81819f02007-05-06 14:49:36 -07005675 /*
5676 * If we have a leftover link then remove it.
5677 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005678 sysfs_remove_link(&slab_kset->kobj, name);
5679 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005680 }
5681
5682 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
5683 if (!al)
5684 return -ENOMEM;
5685
5686 al->s = s;
5687 al->name = name;
5688 al->next = alias_list;
5689 alias_list = al;
5690 return 0;
5691}
5692
5693static int __init slab_sysfs_init(void)
5694{
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07005695 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07005696 int err;
5697
Christoph Lameter18004c52012-07-06 15:25:12 -05005698 mutex_lock(&slab_mutex);
Christoph Lameter2bce6482010-07-19 11:39:11 -05005699
Christoph Lameterd7660ce2020-06-01 21:45:50 -07005700 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005701 if (!slab_kset) {
Christoph Lameter18004c52012-07-06 15:25:12 -05005702 mutex_unlock(&slab_mutex);
Fabian Frederickf9f58282014-06-04 16:06:34 -07005703 pr_err("Cannot register slab subsystem.\n");
Christoph Lameter81819f02007-05-06 14:49:36 -07005704 return -ENOSYS;
5705 }
5706
Christoph Lameter97d06602012-07-06 15:25:11 -05005707 slab_state = FULL;
Christoph Lameter26a7bd02007-05-09 02:32:39 -07005708
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07005709 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter26a7bd02007-05-09 02:32:39 -07005710 err = sysfs_slab_add(s);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07005711 if (err)
Fabian Frederickf9f58282014-06-04 16:06:34 -07005712 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
5713 s->name);
Christoph Lameter26a7bd02007-05-09 02:32:39 -07005714 }
Christoph Lameter81819f02007-05-06 14:49:36 -07005715
5716 while (alias_list) {
5717 struct saved_alias *al = alias_list;
5718
5719 alias_list = alias_list->next;
5720 err = sysfs_slab_alias(al->s, al->name);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07005721 if (err)
Fabian Frederickf9f58282014-06-04 16:06:34 -07005722 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
5723 al->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005724 kfree(al);
5725 }
5726
Christoph Lameter18004c52012-07-06 15:25:12 -05005727 mutex_unlock(&slab_mutex);
Christoph Lameter81819f02007-05-06 14:49:36 -07005728 return 0;
5729}
5730
5731__initcall(slab_sysfs_init);
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005732#endif /* CONFIG_SYSFS */
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005733
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07005734#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
5735static int slab_debugfs_show(struct seq_file *seq, void *v)
5736{
5737
5738 struct location *l;
5739 unsigned int idx = *(unsigned int *)v;
5740 struct loc_track *t = seq->private;
5741
5742 if (idx < t->count) {
5743 l = &t->loc[idx];
5744
5745 seq_printf(seq, "%7ld ", l->count);
5746
5747 if (l->addr)
5748 seq_printf(seq, "%pS", (void *)l->addr);
5749 else
5750 seq_puts(seq, "<not-available>");
5751
5752 if (l->sum_time != l->min_time) {
5753 seq_printf(seq, " age=%ld/%llu/%ld",
5754 l->min_time, div_u64(l->sum_time, l->count),
5755 l->max_time);
5756 } else
5757 seq_printf(seq, " age=%ld", l->min_time);
5758
5759 if (l->min_pid != l->max_pid)
5760 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
5761 else
5762 seq_printf(seq, " pid=%ld",
5763 l->min_pid);
5764
5765 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
5766 seq_printf(seq, " cpus=%*pbl",
5767 cpumask_pr_args(to_cpumask(l->cpus)));
5768
5769 if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
5770 seq_printf(seq, " nodes=%*pbl",
5771 nodemask_pr_args(&l->nodes));
5772
5773 seq_puts(seq, "\n");
5774 }
5775
5776 if (!idx && !t->count)
5777 seq_puts(seq, "No data\n");
5778
5779 return 0;
5780}
5781
5782static void slab_debugfs_stop(struct seq_file *seq, void *v)
5783{
5784}
5785
5786static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
5787{
5788 struct loc_track *t = seq->private;
5789
5790 v = ppos;
5791 ++*ppos;
5792 if (*ppos <= t->count)
5793 return v;
5794
5795 return NULL;
5796}
5797
5798static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
5799{
5800 return ppos;
5801}
5802
5803static const struct seq_operations slab_debugfs_sops = {
5804 .start = slab_debugfs_start,
5805 .next = slab_debugfs_next,
5806 .stop = slab_debugfs_stop,
5807 .show = slab_debugfs_show,
5808};
5809
5810static int slab_debug_trace_open(struct inode *inode, struct file *filep)
5811{
5812
5813 struct kmem_cache_node *n;
5814 enum track_item alloc;
5815 int node;
5816 struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
5817 sizeof(struct loc_track));
5818 struct kmem_cache *s = file_inode(filep)->i_private;
5819
5820 if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
5821 alloc = TRACK_ALLOC;
5822 else
5823 alloc = TRACK_FREE;
5824
5825 if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL))
5826 return -ENOMEM;
5827
5828 /* Push back cpu slabs */
5829 flush_all(s);
5830
5831 for_each_kmem_cache_node(s, node, n) {
5832 unsigned long flags;
5833 struct page *page;
5834
5835 if (!atomic_long_read(&n->nr_slabs))
5836 continue;
5837
5838 spin_lock_irqsave(&n->list_lock, flags);
5839 list_for_each_entry(page, &n->partial, slab_list)
5840 process_slab(t, s, page, alloc);
5841 list_for_each_entry(page, &n->full, slab_list)
5842 process_slab(t, s, page, alloc);
5843 spin_unlock_irqrestore(&n->list_lock, flags);
5844 }
5845
5846 return 0;
5847}
5848
5849static int slab_debug_trace_release(struct inode *inode, struct file *file)
5850{
5851 struct seq_file *seq = file->private_data;
5852 struct loc_track *t = seq->private;
5853
5854 free_loc_track(t);
5855 return seq_release_private(inode, file);
5856}
5857
5858static const struct file_operations slab_debugfs_fops = {
5859 .open = slab_debug_trace_open,
5860 .read = seq_read,
5861 .llseek = seq_lseek,
5862 .release = slab_debug_trace_release,
5863};
5864
5865static void debugfs_slab_add(struct kmem_cache *s)
5866{
5867 struct dentry *slab_cache_dir;
5868
5869 if (unlikely(!slab_debugfs_root))
5870 return;
5871
5872 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
5873
5874 debugfs_create_file("alloc_traces", 0400,
5875 slab_cache_dir, s, &slab_debugfs_fops);
5876
5877 debugfs_create_file("free_traces", 0400,
5878 slab_cache_dir, s, &slab_debugfs_fops);
5879}
5880
5881void debugfs_slab_release(struct kmem_cache *s)
5882{
5883 debugfs_remove_recursive(debugfs_lookup(s->name, slab_debugfs_root));
5884}
5885
5886static int __init slab_debugfs_init(void)
5887{
5888 struct kmem_cache *s;
5889
5890 slab_debugfs_root = debugfs_create_dir("slab", NULL);
5891
5892 list_for_each_entry(s, &slab_caches, list)
5893 if (s->flags & SLAB_STORE_USER)
5894 debugfs_slab_add(s);
5895
5896 return 0;
5897
5898}
5899__initcall(slab_debugfs_init);
5900#endif
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005901/*
5902 * The /proc/slabinfo ABI
5903 */
Yang Shi5b365772017-11-15 17:32:03 -08005904#ifdef CONFIG_SLUB_DEBUG
Glauber Costa0d7561c2012-10-19 18:20:27 +04005905void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005906{
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005907 unsigned long nr_slabs = 0;
Christoph Lameter205ab992008-04-14 19:11:40 +03005908 unsigned long nr_objs = 0;
5909 unsigned long nr_free = 0;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005910 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005911 struct kmem_cache_node *n;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005912
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005913 for_each_kmem_cache_node(s, node, n) {
Wanpeng Lic17fd132013-07-04 08:33:26 +08005914 nr_slabs += node_nr_slabs(n);
5915 nr_objs += node_nr_objs(n);
Christoph Lameter205ab992008-04-14 19:11:40 +03005916 nr_free += count_partial(n, count_free);
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005917 }
5918
Glauber Costa0d7561c2012-10-19 18:20:27 +04005919 sinfo->active_objs = nr_objs - nr_free;
5920 sinfo->num_objs = nr_objs;
5921 sinfo->active_slabs = nr_slabs;
5922 sinfo->num_slabs = nr_slabs;
5923 sinfo->objects_per_slab = oo_objects(s->oo);
5924 sinfo->cache_order = oo_order(s->oo);
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005925}
5926
Glauber Costa0d7561c2012-10-19 18:20:27 +04005927void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005928{
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005929}
5930
Glauber Costab7454ad2012-10-19 18:20:25 +04005931ssize_t slabinfo_write(struct file *file, const char __user *buffer,
5932 size_t count, loff_t *ppos)
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005933{
Glauber Costab7454ad2012-10-19 18:20:25 +04005934 return -EIO;
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005935}
Yang Shi5b365772017-11-15 17:32:03 -08005936#endif /* CONFIG_SLUB_DEBUG */