blob: 862dbd9af4f521fe820a9b5bdb69e7a726ba3d5d [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Christoph Lameter81819f02007-05-06 14:49:36 -07002/*
3 * SLUB: A slab allocator that limits cache line use instead of queuing
4 * objects in per cpu and per node lists.
5 *
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -07006 * The allocator synchronizes using per slab locks or atomic operations
Christoph Lameter881db7f2011-06-01 12:25:53 -05007 * and only uses a centralized lock to manage a pool of partial slabs.
Christoph Lameter81819f02007-05-06 14:49:36 -07008 *
Christoph Lametercde53532008-07-04 09:59:22 -07009 * (C) 2007 SGI, Christoph Lameter
Christoph Lameter881db7f2011-06-01 12:25:53 -050010 * (C) 2011 Linux Foundation, Christoph Lameter
Christoph Lameter81819f02007-05-06 14:49:36 -070011 */
12
13#include <linux/mm.h>
Nick Piggin1eb5ac62009-05-05 19:13:44 +100014#include <linux/swap.h> /* struct reclaim_state */
Christoph Lameter81819f02007-05-06 14:49:36 -070015#include <linux/module.h>
16#include <linux/bit_spinlock.h>
17#include <linux/interrupt.h>
Andrew Morton1b3865d2021-06-15 18:23:39 -070018#include <linux/swab.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070019#include <linux/bitops.h>
20#include <linux/slab.h>
Christoph Lameter97d06602012-07-06 15:25:11 -050021#include "slab.h"
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +040022#include <linux/proc_fs.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070023#include <linux/seq_file.h>
Andrey Ryabinina79316c2015-02-13 14:39:38 -080024#include <linux/kasan.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070025#include <linux/cpu.h>
26#include <linux/cpuset.h>
27#include <linux/mempolicy.h>
28#include <linux/ctype.h>
Oliver Glitta5cf909c2021-07-07 18:07:47 -070029#include <linux/stackdepot.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070030#include <linux/debugobjects.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070031#include <linux/kallsyms.h>
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -080032#include <linux/kfence.h>
Yasunori Gotob9049e22007-10-21 16:41:37 -070033#include <linux/memory.h>
Roman Zippelf8bd2252008-05-01 04:34:31 -070034#include <linux/math64.h>
Akinobu Mita773ff602008-12-23 19:37:01 +090035#include <linux/fault-inject.h>
Pekka Enbergbfa71452011-07-07 22:47:01 +030036#include <linux/stacktrace.h>
Christoph Lameter4de900b2012-01-30 15:53:51 -060037#include <linux/prefetch.h>
Glauber Costa2633d7a2012-12-18 14:22:34 -080038#include <linux/memcontrol.h>
Kees Cook2482ddec2017-09-06 16:19:18 -070039#include <linux/random.h>
Oliver Glitta1f9f78b2021-06-28 19:34:33 -070040#include <kunit/test.h>
Oliver Glitta553c0362021-05-21 14:11:26 +020041#include <linux/sort.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070042
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -070043#include <linux/debugfs.h>
Richard Kennedy4a923792010-10-21 10:29:19 +010044#include <trace/events/kmem.h>
45
Mel Gorman072bb0a2012-07-31 16:43:58 -070046#include "internal.h"
47
Christoph Lameter81819f02007-05-06 14:49:36 -070048/*
49 * Lock order:
Christoph Lameter18004c52012-07-06 15:25:12 -050050 * 1. slab_mutex (Global Mutex)
Vlastimil Babkabd0e7492021-05-22 01:59:38 +020051 * 2. node->list_lock (Spinlock)
52 * 3. kmem_cache->cpu_slab->lock (Local lock)
Vlastimil Babkac2092c12021-11-15 16:55:15 +010053 * 4. slab_lock(slab) (Only on some arches or for debugging)
Vlastimil Babkabd0e7492021-05-22 01:59:38 +020054 * 5. object_map_lock (Only for debugging)
Christoph Lameter81819f02007-05-06 14:49:36 -070055 *
Christoph Lameter18004c52012-07-06 15:25:12 -050056 * slab_mutex
Christoph Lameter881db7f2011-06-01 12:25:53 -050057 *
Christoph Lameter18004c52012-07-06 15:25:12 -050058 * The role of the slab_mutex is to protect the list of all the slabs
Christoph Lameter881db7f2011-06-01 12:25:53 -050059 * and to synchronize major metadata changes to slab cache structures.
Vlastimil Babkabd0e7492021-05-22 01:59:38 +020060 * Also synchronizes memory hotplug callbacks.
61 *
62 * slab_lock
63 *
64 * The slab_lock is a wrapper around the page lock, thus it is a bit
65 * spinlock.
Christoph Lameter881db7f2011-06-01 12:25:53 -050066 *
67 * The slab_lock is only used for debugging and on arches that do not
Matthew Wilcoxb7ccc7f2018-06-07 17:08:46 -070068 * have the ability to do a cmpxchg_double. It only protects:
Vlastimil Babkac2092c12021-11-15 16:55:15 +010069 * A. slab->freelist -> List of free objects in a slab
70 * B. slab->inuse -> Number of objects in use
71 * C. slab->objects -> Number of objects in slab
72 * D. slab->frozen -> frozen state
Christoph Lameter881db7f2011-06-01 12:25:53 -050073 *
Vlastimil Babkabd0e7492021-05-22 01:59:38 +020074 * Frozen slabs
75 *
Christoph Lameter881db7f2011-06-01 12:25:53 -050076 * If a slab is frozen then it is exempt from list management. It is not
Liu Xiang632b2ef2019-05-13 17:16:28 -070077 * on any list except per cpu partial list. The processor that froze the
Vlastimil Babkac2092c12021-11-15 16:55:15 +010078 * slab is the one who can perform list operations on the slab. Other
Liu Xiang632b2ef2019-05-13 17:16:28 -070079 * processors may put objects onto the freelist but the processor that
80 * froze the slab is the only one that can retrieve the objects from the
Vlastimil Babkac2092c12021-11-15 16:55:15 +010081 * slab's freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -070082 *
Vlastimil Babkabd0e7492021-05-22 01:59:38 +020083 * list_lock
84 *
Christoph Lameter81819f02007-05-06 14:49:36 -070085 * The list_lock protects the partial and full list on each node and
86 * the partial slab counter. If taken then no new slabs may be added or
87 * removed from the lists nor make the number of partial slabs be modified.
88 * (Note that the total number of slabs is an atomic value that may be
89 * modified without taking the list lock).
90 *
91 * The list_lock is a centralized lock and thus we avoid taking it as
92 * much as possible. As long as SLUB does not have to handle partial
93 * slabs, operations can continue without any centralized lock. F.e.
94 * allocating a long series of objects that fill up slabs does not require
95 * the list lock.
Vlastimil Babkabd0e7492021-05-22 01:59:38 +020096 *
97 * cpu_slab->lock local lock
98 *
99 * This locks protect slowpath manipulation of all kmem_cache_cpu fields
100 * except the stat counters. This is a percpu structure manipulated only by
101 * the local cpu, so the lock protects against being preempted or interrupted
102 * by an irq. Fast path operations rely on lockless operations instead.
103 * On PREEMPT_RT, the local lock does not actually disable irqs (and thus
104 * prevent the lockless operations), so fastpath operations also need to take
105 * the lock and are no longer lockless.
106 *
107 * lockless fastpaths
108 *
109 * The fast path allocation (slab_alloc_node()) and freeing (do_slab_free())
110 * are fully lockless when satisfied from the percpu slab (and when
111 * cmpxchg_double is possible to use, otherwise slab_lock is taken).
112 * They also don't disable preemption or migration or irqs. They rely on
113 * the transaction id (tid) field to detect being preempted or moved to
114 * another cpu.
115 *
116 * irq, preemption, migration considerations
117 *
118 * Interrupts are disabled as part of list_lock or local_lock operations, or
119 * around the slab_lock operation, in order to make the slab allocator safe
120 * to use in the context of an irq.
121 *
122 * In addition, preemption (or migration on PREEMPT_RT) is disabled in the
123 * allocation slowpath, bulk allocation, and put_cpu_partial(), so that the
124 * local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer
125 * doesn't have to be revalidated in each section protected by the local lock.
Christoph Lameter81819f02007-05-06 14:49:36 -0700126 *
127 * SLUB assigns one slab for allocation to each processor.
128 * Allocations only occur from these slabs called cpu slabs.
129 *
Christoph Lameter672bba32007-05-09 02:32:39 -0700130 * Slabs with free elements are kept on a partial list and during regular
131 * operations no list for full slabs is used. If an object in a full slab is
Christoph Lameter81819f02007-05-06 14:49:36 -0700132 * freed then the slab will show up again on the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -0700133 * We track full slabs for debugging purposes though because otherwise we
134 * cannot scan all objects.
Christoph Lameter81819f02007-05-06 14:49:36 -0700135 *
136 * Slabs are freed when they become empty. Teardown and setup is
137 * minimal so we rely on the page allocators per cpu caches for
138 * fast frees and allocs.
139 *
Vlastimil Babkac2092c12021-11-15 16:55:15 +0100140 * slab->frozen The slab is frozen and exempt from list processing.
Christoph Lameter4b6f0752007-05-16 22:10:53 -0700141 * This means that the slab is dedicated to a purpose
142 * such as satisfying allocations for a specific
143 * processor. Objects may be freed in the slab while
144 * it is frozen but slab_free will then skip the usual
145 * list operations. It is up to the processor holding
146 * the slab to integrate the slab into the slab lists
147 * when the slab is no longer needed.
148 *
149 * One use of this flag is to mark slabs that are
150 * used for allocations. Then such a slab becomes a cpu
151 * slab. The cpu slab may be equipped with an additional
Christoph Lameterdfb4f092007-10-16 01:26:05 -0700152 * freelist that allows lockless access to
Christoph Lameter894b8782007-05-10 03:15:16 -0700153 * free objects in addition to the regular freelist
154 * that requires the slab lock.
Christoph Lameter81819f02007-05-06 14:49:36 -0700155 *
Yu Zhaoaed68142019-11-30 17:49:34 -0800156 * SLAB_DEBUG_FLAGS Slab requires special handling due to debug
Christoph Lameter81819f02007-05-06 14:49:36 -0700157 * options set. This moves slab handling out of
Christoph Lameter894b8782007-05-10 03:15:16 -0700158 * the fast path and disables lockless freelists.
Christoph Lameter81819f02007-05-06 14:49:36 -0700159 */
160
Vlastimil Babka25c00c52021-05-21 14:03:23 +0200161/*
162 * We could simply use migrate_disable()/enable() but as long as it's a
163 * function call even on !PREEMPT_RT, use inline preempt_disable() there.
164 */
165#ifndef CONFIG_PREEMPT_RT
166#define slub_get_cpu_ptr(var) get_cpu_ptr(var)
167#define slub_put_cpu_ptr(var) put_cpu_ptr(var)
168#else
169#define slub_get_cpu_ptr(var) \
170({ \
171 migrate_disable(); \
172 this_cpu_ptr(var); \
173})
174#define slub_put_cpu_ptr(var) \
175do { \
176 (void)(var); \
177 migrate_enable(); \
178} while (0)
179#endif
180
Vlastimil Babkaca0cab62020-08-06 23:18:51 -0700181#ifdef CONFIG_SLUB_DEBUG
182#ifdef CONFIG_SLUB_DEBUG_ON
183DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
184#else
185DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
186#endif
Stephen Boyd79270292021-06-28 19:34:52 -0700187#endif /* CONFIG_SLUB_DEBUG */
Vlastimil Babkaca0cab62020-08-06 23:18:51 -0700188
Vlastimil Babka59052e82020-08-06 23:18:55 -0700189static inline bool kmem_cache_debug(struct kmem_cache *s)
190{
191 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
Christoph Lameteraf537b02010-07-09 14:07:14 -0500192}
Christoph Lameter5577bd82007-05-16 22:10:56 -0700193
Geert Uytterhoeven117d54d2016-08-04 15:31:55 -0700194void *fixup_red_left(struct kmem_cache *s, void *p)
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700195{
Vlastimil Babka59052e82020-08-06 23:18:55 -0700196 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700197 p += s->red_left_pad;
198
199 return p;
200}
201
Joonsoo Kim345c9052013-06-19 14:05:52 +0900202static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
203{
204#ifdef CONFIG_SLUB_CPU_PARTIAL
205 return !kmem_cache_debug(s);
206#else
207 return false;
208#endif
209}
210
Christoph Lameter81819f02007-05-06 14:49:36 -0700211/*
212 * Issues still to be resolved:
213 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700214 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
215 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700216 * - Variable sizing of the per node arrays
217 */
218
Christoph Lameterb789ef52011-06-01 12:25:49 -0500219/* Enable to log cmpxchg failures */
220#undef SLUB_DEBUG_CMPXCHG
221
Christoph Lameter81819f02007-05-06 14:49:36 -0700222/*
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -0700223 * Minimum number of partial slabs. These will be left on the partial
Christoph Lameter2086d262007-05-06 14:49:46 -0700224 * lists even if they are empty. kmem_cache_shrink may reclaim them.
225 */
Christoph Lameter76be8952007-12-21 14:37:37 -0800226#define MIN_PARTIAL 5
Christoph Lametere95eed52007-05-06 14:49:44 -0700227
Christoph Lameter2086d262007-05-06 14:49:46 -0700228/*
229 * Maximum number of desirable partial slabs.
230 * The existence of more partial slabs makes kmem_cache_shrink
Zhi Yong Wu721ae222013-11-08 20:47:37 +0800231 * sort the partial list by the number of objects in use.
Christoph Lameter2086d262007-05-06 14:49:46 -0700232 */
233#define MAX_PARTIAL 10
234
Laura Abbottbecfda62016-03-15 14:55:06 -0700235#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
Christoph Lameter81819f02007-05-06 14:49:36 -0700236 SLAB_POISON | SLAB_STORE_USER)
Christoph Lameter672bba32007-05-09 02:32:39 -0700237
Christoph Lameter81819f02007-05-06 14:49:36 -0700238/*
Laura Abbott149daaf2016-03-15 14:55:09 -0700239 * These debug flags cannot use CMPXCHG because there might be consistency
240 * issues when checking or reading debug information
241 */
242#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
243 SLAB_TRACE)
244
245
246/*
David Rientjes3de47212009-07-27 18:30:35 -0700247 * Debugging flags that require metadata to be stored in the slab. These get
248 * disabled when slub_debug=O is used and a cache's min order increases with
249 * metadata.
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700250 */
David Rientjes3de47212009-07-27 18:30:35 -0700251#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700252
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400253#define OO_SHIFT 16
254#define OO_MASK ((1 << OO_SHIFT) - 1)
Vlastimil Babkac2092c12021-11-15 16:55:15 +0100255#define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400256
Christoph Lameter81819f02007-05-06 14:49:36 -0700257/* Internal SLUB flags */
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800258/* Poison object */
Alexey Dobriyan4fd0b462017-11-15 17:32:21 -0800259#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800260/* Use cmpxchg_double */
Alexey Dobriyan4fd0b462017-11-15 17:32:21 -0800261#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
Christoph Lameter81819f02007-05-06 14:49:36 -0700262
Christoph Lameter02cbc872007-05-09 02:32:43 -0700263/*
264 * Tracking user of a slab.
265 */
Ben Greeard6543e32011-07-07 11:36:36 -0700266#define TRACK_ADDRS_COUNT 16
Christoph Lameter02cbc872007-05-09 02:32:43 -0700267struct track {
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300268 unsigned long addr; /* Called from address */
Oliver Glitta5cf909c2021-07-07 18:07:47 -0700269#ifdef CONFIG_STACKDEPOT
270 depot_stack_handle_t handle;
Ben Greeard6543e32011-07-07 11:36:36 -0700271#endif
Christoph Lameter02cbc872007-05-09 02:32:43 -0700272 int cpu; /* Was running on cpu */
273 int pid; /* Pid context */
274 unsigned long when; /* When did the operation occur */
275};
276
277enum track_item { TRACK_ALLOC, TRACK_FREE };
278
Christoph Lameterab4d5ed2010-10-05 13:57:26 -0500279#ifdef CONFIG_SYSFS
Christoph Lameter81819f02007-05-06 14:49:36 -0700280static int sysfs_slab_add(struct kmem_cache *);
281static int sysfs_slab_alias(struct kmem_cache *, const char *);
Christoph Lameter81819f02007-05-06 14:49:36 -0700282#else
Christoph Lameter0c710012007-07-17 04:03:24 -0700283static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
284static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
285 { return 0; }
Christoph Lameter81819f02007-05-06 14:49:36 -0700286#endif
287
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -0700288#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
289static void debugfs_slab_add(struct kmem_cache *);
290#else
291static inline void debugfs_slab_add(struct kmem_cache *s) { }
292#endif
293
Christoph Lameter4fdccdf2011-03-22 13:35:00 -0500294static inline void stat(const struct kmem_cache *s, enum stat_item si)
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800295{
296#ifdef CONFIG_SLUB_STATS
Christoph Lameter88da03a2014-04-07 15:39:42 -0700297 /*
298 * The rmw is racy on a preemptible kernel but this is acceptable, so
299 * avoid this_cpu_add()'s irq-disable overhead.
300 */
301 raw_cpu_inc(s->cpu_slab->stat[si]);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800302#endif
303}
304
Vlastimil Babka7e1fa932021-02-24 12:01:12 -0800305/*
306 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
307 * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily
308 * differ during memory hotplug/hotremove operations.
309 * Protected by slab_mutex.
310 */
311static nodemask_t slab_nodes;
312
Christoph Lameter81819f02007-05-06 14:49:36 -0700313/********************************************************************
314 * Core slab cache functions
315 *******************************************************************/
316
Kees Cook2482ddec2017-09-06 16:19:18 -0700317/*
318 * Returns freelist pointer (ptr). With hardening, this is obfuscated
319 * with an XOR of the address where the pointer is held and a per-cache
320 * random number.
321 */
322static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
323 unsigned long ptr_addr)
324{
325#ifdef CONFIG_SLAB_FREELIST_HARDENED
Andrey Konovalovd36a63a2019-02-20 22:19:32 -0800326 /*
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800327 * When CONFIG_KASAN_SW/HW_TAGS is enabled, ptr_addr might be tagged.
Andrey Konovalovd36a63a2019-02-20 22:19:32 -0800328 * Normally, this doesn't cause any issues, as both set_freepointer()
329 * and get_freepointer() are called with a pointer with the same tag.
330 * However, there are some issues with CONFIG_SLUB_DEBUG code. For
331 * example, when __free_slub() iterates over objects in a cache, it
332 * passes untagged pointers to check_object(). check_object() in turns
333 * calls get_freepointer() with an untagged pointer, which causes the
334 * freepointer to be restored incorrectly.
335 */
336 return (void *)((unsigned long)ptr ^ s->random ^
Kees Cook1ad53d92020-04-01 21:04:23 -0700337 swab((unsigned long)kasan_reset_tag((void *)ptr_addr)));
Kees Cook2482ddec2017-09-06 16:19:18 -0700338#else
339 return ptr;
340#endif
341}
342
343/* Returns the freelist pointer recorded at location ptr_addr. */
344static inline void *freelist_dereference(const struct kmem_cache *s,
345 void *ptr_addr)
346{
347 return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
348 (unsigned long)ptr_addr);
349}
350
Christoph Lameter7656c722007-05-09 02:32:40 -0700351static inline void *get_freepointer(struct kmem_cache *s, void *object)
352{
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800353 object = kasan_reset_tag(object);
Kees Cook2482ddec2017-09-06 16:19:18 -0700354 return freelist_dereference(s, object + s->offset);
Christoph Lameter7656c722007-05-09 02:32:40 -0700355}
356
Eric Dumazet0ad95002011-12-16 16:25:34 +0100357static void prefetch_freepointer(const struct kmem_cache *s, void *object)
358{
Hyeonggon Yoo04b4b002021-11-05 13:35:24 -0700359 prefetchw(object + s->offset);
Eric Dumazet0ad95002011-12-16 16:25:34 +0100360}
361
Christoph Lameter1393d9a2011-05-16 15:26:08 -0500362static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
363{
Kees Cook2482ddec2017-09-06 16:19:18 -0700364 unsigned long freepointer_addr;
Christoph Lameter1393d9a2011-05-16 15:26:08 -0500365 void *p;
366
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800367 if (!debug_pagealloc_enabled_static())
Joonsoo Kim922d5662016-03-17 14:17:53 -0700368 return get_freepointer(s, object);
369
Alexander Potapenkof70b0042021-05-22 17:41:56 -0700370 object = kasan_reset_tag(object);
Kees Cook2482ddec2017-09-06 16:19:18 -0700371 freepointer_addr = (unsigned long)object + s->offset;
Christoph Hellwigfe557312020-06-17 09:37:53 +0200372 copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));
Kees Cook2482ddec2017-09-06 16:19:18 -0700373 return freelist_ptr(s, p, freepointer_addr);
Christoph Lameter1393d9a2011-05-16 15:26:08 -0500374}
375
Christoph Lameter7656c722007-05-09 02:32:40 -0700376static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
377{
Kees Cook2482ddec2017-09-06 16:19:18 -0700378 unsigned long freeptr_addr = (unsigned long)object + s->offset;
379
Alexander Popovce6fa912017-09-06 16:19:22 -0700380#ifdef CONFIG_SLAB_FREELIST_HARDENED
381 BUG_ON(object == fp); /* naive detection of double free or corruption */
382#endif
383
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800384 freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
Kees Cook2482ddec2017-09-06 16:19:18 -0700385 *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
Christoph Lameter7656c722007-05-09 02:32:40 -0700386}
387
388/* Loop over all objects in a slab */
Christoph Lameter224a88b2008-04-14 19:11:31 +0300389#define for_each_object(__p, __s, __addr, __objects) \
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700390 for (__p = fixup_red_left(__s, __addr); \
391 __p < (__addr) + (__objects) * (__s)->size; \
392 __p += (__s)->size)
Christoph Lameter7656c722007-05-09 02:32:40 -0700393
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700394static inline unsigned int order_objects(unsigned int order, unsigned int size)
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800395{
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700396 return ((unsigned int)PAGE_SIZE << order) / size;
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800397}
398
Alexey Dobriyan19af27a2018-04-05 16:21:39 -0700399static inline struct kmem_cache_order_objects oo_make(unsigned int order,
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700400 unsigned int size)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300401{
402 struct kmem_cache_order_objects x = {
Matthew Wilcox9736d2a2018-06-07 17:09:10 -0700403 (order << OO_SHIFT) + order_objects(order, size)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300404 };
405
406 return x;
407}
408
Alexey Dobriyan19af27a2018-04-05 16:21:39 -0700409static inline unsigned int oo_order(struct kmem_cache_order_objects x)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300410{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400411 return x.x >> OO_SHIFT;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300412}
413
Alexey Dobriyan19af27a2018-04-05 16:21:39 -0700414static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300415{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400416 return x.x & OO_MASK;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300417}
418
Vlastimil Babkab47291e2021-11-05 13:35:17 -0700419#ifdef CONFIG_SLUB_CPU_PARTIAL
420static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
421{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100422 unsigned int nr_slabs;
Vlastimil Babkab47291e2021-11-05 13:35:17 -0700423
424 s->cpu_partial = nr_objects;
425
426 /*
427 * We take the number of objects but actually limit the number of
Vlastimil Babkac2092c12021-11-15 16:55:15 +0100428 * slabs on the per cpu partial list, in order to limit excessive
429 * growth of the list. For simplicity we assume that the slabs will
Vlastimil Babkab47291e2021-11-05 13:35:17 -0700430 * be half-full.
431 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100432 nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
433 s->cpu_partial_slabs = nr_slabs;
Vlastimil Babkab47291e2021-11-05 13:35:17 -0700434}
435#else
436static inline void
437slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
438{
439}
440#endif /* CONFIG_SLUB_CPU_PARTIAL */
441
Christoph Lameter881db7f2011-06-01 12:25:53 -0500442/*
443 * Per slab locking using the pagelock
444 */
Vlastimil Babka03938952021-10-26 13:39:14 +0200445static __always_inline void __slab_lock(struct slab *slab)
Christoph Lameter881db7f2011-06-01 12:25:53 -0500446{
Vlastimil Babka03938952021-10-26 13:39:14 +0200447 struct page *page = slab_page(slab);
448
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -0800449 VM_BUG_ON_PAGE(PageTail(page), page);
Christoph Lameter881db7f2011-06-01 12:25:53 -0500450 bit_spin_lock(PG_locked, &page->flags);
451}
452
Vlastimil Babka03938952021-10-26 13:39:14 +0200453static __always_inline void __slab_unlock(struct slab *slab)
Christoph Lameter881db7f2011-06-01 12:25:53 -0500454{
Vlastimil Babka03938952021-10-26 13:39:14 +0200455 struct page *page = slab_page(slab);
456
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -0800457 VM_BUG_ON_PAGE(PageTail(page), page);
Christoph Lameter881db7f2011-06-01 12:25:53 -0500458 __bit_spin_unlock(PG_locked, &page->flags);
459}
460
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100461static __always_inline void slab_lock(struct slab *slab, unsigned long *flags)
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +0200462{
463 if (IS_ENABLED(CONFIG_PREEMPT_RT))
464 local_irq_save(*flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100465 __slab_lock(slab);
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +0200466}
467
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100468static __always_inline void slab_unlock(struct slab *slab, unsigned long *flags)
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +0200469{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100470 __slab_unlock(slab);
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +0200471 if (IS_ENABLED(CONFIG_PREEMPT_RT))
472 local_irq_restore(*flags);
473}
474
475/*
476 * Interrupts must be disabled (for the fallback code to work right), typically
477 * by an _irqsave() lock variant. Except on PREEMPT_RT where locks are different
478 * so we disable interrupts as part of slab_[un]lock().
479 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100480static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,
Christoph Lameterb789ef52011-06-01 12:25:49 -0500481 void *freelist_old, unsigned long counters_old,
482 void *freelist_new, unsigned long counters_new,
483 const char *n)
484{
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +0200485 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
486 lockdep_assert_irqs_disabled();
Heiko Carstens25654092012-01-12 17:17:33 -0800487#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
488 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Christoph Lameterb789ef52011-06-01 12:25:49 -0500489 if (s->flags & __CMPXCHG_DOUBLE) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100490 if (cmpxchg_double(&slab->freelist, &slab->counters,
Dan Carpenter0aa9a132014-08-06 16:04:48 -0700491 freelist_old, counters_old,
492 freelist_new, counters_new))
Joe Perches6f6528a2015-04-14 15:44:31 -0700493 return true;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500494 } else
495#endif
496 {
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +0200497 /* init to 0 to prevent spurious warnings */
498 unsigned long flags = 0;
499
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100500 slab_lock(slab, &flags);
501 if (slab->freelist == freelist_old &&
502 slab->counters == counters_old) {
503 slab->freelist = freelist_new;
504 slab->counters = counters_new;
505 slab_unlock(slab, &flags);
Joe Perches6f6528a2015-04-14 15:44:31 -0700506 return true;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500507 }
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100508 slab_unlock(slab, &flags);
Christoph Lameterb789ef52011-06-01 12:25:49 -0500509 }
510
511 cpu_relax();
512 stat(s, CMPXCHG_DOUBLE_FAIL);
513
514#ifdef SLUB_DEBUG_CMPXCHG
Fabian Frederickf9f58282014-06-04 16:06:34 -0700515 pr_info("%s %s: cmpxchg double redo ", n, s->name);
Christoph Lameterb789ef52011-06-01 12:25:49 -0500516#endif
517
Joe Perches6f6528a2015-04-14 15:44:31 -0700518 return false;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500519}
520
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100521static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,
Christoph Lameter1d071712011-07-14 12:49:12 -0500522 void *freelist_old, unsigned long counters_old,
523 void *freelist_new, unsigned long counters_new,
524 const char *n)
525{
Heiko Carstens25654092012-01-12 17:17:33 -0800526#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
527 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Christoph Lameter1d071712011-07-14 12:49:12 -0500528 if (s->flags & __CMPXCHG_DOUBLE) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100529 if (cmpxchg_double(&slab->freelist, &slab->counters,
Dan Carpenter0aa9a132014-08-06 16:04:48 -0700530 freelist_old, counters_old,
531 freelist_new, counters_new))
Joe Perches6f6528a2015-04-14 15:44:31 -0700532 return true;
Christoph Lameter1d071712011-07-14 12:49:12 -0500533 } else
534#endif
535 {
536 unsigned long flags;
537
538 local_irq_save(flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100539 __slab_lock(slab);
540 if (slab->freelist == freelist_old &&
541 slab->counters == counters_old) {
542 slab->freelist = freelist_new;
543 slab->counters = counters_new;
544 __slab_unlock(slab);
Christoph Lameter1d071712011-07-14 12:49:12 -0500545 local_irq_restore(flags);
Joe Perches6f6528a2015-04-14 15:44:31 -0700546 return true;
Christoph Lameter1d071712011-07-14 12:49:12 -0500547 }
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100548 __slab_unlock(slab);
Christoph Lameter1d071712011-07-14 12:49:12 -0500549 local_irq_restore(flags);
550 }
551
552 cpu_relax();
553 stat(s, CMPXCHG_DOUBLE_FAIL);
554
555#ifdef SLUB_DEBUG_CMPXCHG
Fabian Frederickf9f58282014-06-04 16:06:34 -0700556 pr_info("%s %s: cmpxchg double redo ", n, s->name);
Christoph Lameter1d071712011-07-14 12:49:12 -0500557#endif
558
Joe Perches6f6528a2015-04-14 15:44:31 -0700559 return false;
Christoph Lameter1d071712011-07-14 12:49:12 -0500560}
561
Christoph Lameter41ecc552007-05-09 02:32:44 -0700562#ifdef CONFIG_SLUB_DEBUG
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800563static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
Sebastian Andrzej Siewior94ef0302020-07-16 18:47:50 +0200564static DEFINE_RAW_SPINLOCK(object_map_lock);
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800565
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +0200566static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100567 struct slab *slab)
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +0200568{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100569 void *addr = slab_address(slab);
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +0200570 void *p;
571
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100572 bitmap_zero(obj_map, slab->objects);
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +0200573
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100574 for (p = slab->freelist; p; p = get_freepointer(s, p))
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +0200575 set_bit(__obj_to_index(s, addr, p), obj_map);
576}
577
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700578#if IS_ENABLED(CONFIG_KUNIT)
579static bool slab_add_kunit_errors(void)
580{
581 struct kunit_resource *resource;
582
583 if (likely(!current->kunit_test))
584 return false;
585
586 resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
587 if (!resource)
588 return false;
589
590 (*(int *)resource->data)++;
591 kunit_put_resource(resource);
592 return true;
593}
594#else
595static inline bool slab_add_kunit_errors(void) { return false; }
596#endif
597
Christoph Lameter41ecc552007-05-09 02:32:44 -0700598/*
Vlastimil Babkac2092c12021-11-15 16:55:15 +0100599 * Determine a map of objects in use in a slab.
Christoph Lameter5f80b132011-04-15 14:48:13 -0500600 *
Vlastimil Babkac2092c12021-11-15 16:55:15 +0100601 * Node listlock must be held to guarantee that the slab does
Christoph Lameter5f80b132011-04-15 14:48:13 -0500602 * not vanish from under us.
603 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100604static unsigned long *get_map(struct kmem_cache *s, struct slab *slab)
Jules Irenge31364c22020-04-06 20:08:15 -0700605 __acquires(&object_map_lock)
Christoph Lameter5f80b132011-04-15 14:48:13 -0500606{
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800607 VM_BUG_ON(!irqs_disabled());
608
Sebastian Andrzej Siewior94ef0302020-07-16 18:47:50 +0200609 raw_spin_lock(&object_map_lock);
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800610
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100611 __fill_map(object_map, s, slab);
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800612
613 return object_map;
614}
615
Jules Irenge81aba9e2020-04-06 20:08:18 -0700616static void put_map(unsigned long *map) __releases(&object_map_lock)
Yu Zhao90e9f6a2020-01-30 22:11:57 -0800617{
618 VM_BUG_ON(map != object_map);
Sebastian Andrzej Siewior94ef0302020-07-16 18:47:50 +0200619 raw_spin_unlock(&object_map_lock);
Christoph Lameter5f80b132011-04-15 14:48:13 -0500620}
621
Alexey Dobriyan870b1fb2018-04-05 16:21:43 -0700622static inline unsigned int size_from_object(struct kmem_cache *s)
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700623{
624 if (s->flags & SLAB_RED_ZONE)
625 return s->size - s->red_left_pad;
626
627 return s->size;
628}
629
630static inline void *restore_red_left(struct kmem_cache *s, void *p)
631{
632 if (s->flags & SLAB_RED_ZONE)
633 p -= s->red_left_pad;
634
635 return p;
636}
637
Christoph Lameter41ecc552007-05-09 02:32:44 -0700638/*
639 * Debug settings:
640 */
Andrey Ryabinin89d3c872015-11-05 18:51:23 -0800641#if defined(CONFIG_SLUB_DEBUG_ON)
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800642static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700643#else
Alexey Dobriyand50112e2017-11-15 17:32:18 -0800644static slab_flags_t slub_debug;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700645#endif
Christoph Lameter41ecc552007-05-09 02:32:44 -0700646
Vlastimil Babkae17f1df2020-08-06 23:18:35 -0700647static char *slub_debug_string;
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700648static int disable_higher_order_debug;
Christoph Lameter41ecc552007-05-09 02:32:44 -0700649
Christoph Lameter7656c722007-05-09 02:32:40 -0700650/*
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800651 * slub is about to manipulate internal object metadata. This memory lies
652 * outside the range of the allocated object, so accessing it would normally
653 * be reported by kasan as a bounds error. metadata_access_enable() is used
654 * to tell kasan that these accesses are OK.
655 */
656static inline void metadata_access_enable(void)
657{
658 kasan_disable_current();
659}
660
661static inline void metadata_access_disable(void)
662{
663 kasan_enable_current();
664}
665
666/*
Christoph Lameter81819f02007-05-06 14:49:36 -0700667 * Object debugging
668 */
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700669
670/* Verify that a pointer has an address that is valid within a slab page */
671static inline int check_valid_pointer(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100672 struct slab *slab, void *object)
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700673{
674 void *base;
675
676 if (!object)
677 return 1;
678
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100679 base = slab_address(slab);
Qian Cai338cfaa2019-02-20 22:19:36 -0800680 object = kasan_reset_tag(object);
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700681 object = restore_red_left(s, object);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100682 if (object < base || object >= base + slab->objects * s->size ||
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700683 (object - base) % s->size) {
684 return 0;
685 }
686
687 return 1;
688}
689
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800690static void print_section(char *level, char *text, u8 *addr,
691 unsigned int length)
Christoph Lameter81819f02007-05-06 14:49:36 -0700692{
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800693 metadata_access_enable();
Kuan-Ying Lee340caf12021-08-13 16:54:27 -0700694 print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
695 16, 1, kasan_reset_tag((void *)addr), length, 1);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800696 metadata_access_disable();
Christoph Lameter81819f02007-05-06 14:49:36 -0700697}
698
Waiman Longcbfc35a2020-05-07 18:36:06 -0700699/*
700 * See comment in calculate_sizes().
701 */
702static inline bool freeptr_outside_object(struct kmem_cache *s)
703{
704 return s->offset >= s->inuse;
705}
706
707/*
708 * Return offset of the end of info block which is inuse + free pointer if
709 * not overlapping with object.
710 */
711static inline unsigned int get_info_end(struct kmem_cache *s)
712{
713 if (freeptr_outside_object(s))
714 return s->inuse + sizeof(void *);
715 else
716 return s->inuse;
717}
718
Christoph Lameter81819f02007-05-06 14:49:36 -0700719static struct track *get_track(struct kmem_cache *s, void *object,
720 enum track_item alloc)
721{
722 struct track *p;
723
Waiman Longcbfc35a2020-05-07 18:36:06 -0700724 p = object + get_info_end(s);
Christoph Lameter81819f02007-05-06 14:49:36 -0700725
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800726 return kasan_reset_tag(p + alloc);
Christoph Lameter81819f02007-05-06 14:49:36 -0700727}
728
Oliver Glitta5cf909c2021-07-07 18:07:47 -0700729#ifdef CONFIG_STACKDEPOT
Sebastian Andrzej Siewiorc4cf6782022-06-07 17:20:10 +0200730static noinline depot_stack_handle_t set_track_prepare(void)
731{
732 depot_stack_handle_t handle;
Oliver Glitta5cf909c2021-07-07 18:07:47 -0700733 unsigned long entries[TRACK_ADDRS_COUNT];
Vlastimil Babka0cd1a022022-02-04 17:44:40 +0100734 unsigned int nr_entries;
Linus Torvaldsae14c632021-07-17 13:27:00 -0700735
Oliver Glitta5cf909c2021-07-07 18:07:47 -0700736 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
Sebastian Andrzej Siewiorc4cf6782022-06-07 17:20:10 +0200737 handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);
738
739 return handle;
740}
741#else
742static inline depot_stack_handle_t set_track_prepare(void)
743{
744 return 0;
745}
Ben Greeard6543e32011-07-07 11:36:36 -0700746#endif
Oliver Glitta5cf909c2021-07-07 18:07:47 -0700747
Sebastian Andrzej Siewiorc4cf6782022-06-07 17:20:10 +0200748static void set_track_update(struct kmem_cache *s, void *object,
749 enum track_item alloc, unsigned long addr,
750 depot_stack_handle_t handle)
751{
752 struct track *p = get_track(s, object, alloc);
753
754#ifdef CONFIG_STACKDEPOT
755 p->handle = handle;
756#endif
Vlastimil Babka0cd1a022022-02-04 17:44:40 +0100757 p->addr = addr;
758 p->cpu = smp_processor_id();
759 p->pid = current->pid;
760 p->when = jiffies;
Christoph Lameter81819f02007-05-06 14:49:36 -0700761}
762
Sebastian Andrzej Siewiorc4cf6782022-06-07 17:20:10 +0200763static __always_inline void set_track(struct kmem_cache *s, void *object,
764 enum track_item alloc, unsigned long addr)
765{
766 depot_stack_handle_t handle = set_track_prepare();
767
768 set_track_update(s, object, alloc, addr, handle);
769}
770
Christoph Lameter81819f02007-05-06 14:49:36 -0700771static void init_tracking(struct kmem_cache *s, void *object)
772{
Vlastimil Babka0cd1a022022-02-04 17:44:40 +0100773 struct track *p;
774
Christoph Lameter24922682007-07-17 04:03:18 -0700775 if (!(s->flags & SLAB_STORE_USER))
776 return;
777
Vlastimil Babka0cd1a022022-02-04 17:44:40 +0100778 p = get_track(s, object, TRACK_ALLOC);
779 memset(p, 0, 2*sizeof(struct track));
Christoph Lameter81819f02007-05-06 14:49:36 -0700780}
781
Chintan Pandya86609d32018-04-05 16:20:15 -0700782static void print_track(const char *s, struct track *t, unsigned long pr_time)
Christoph Lameter81819f02007-05-06 14:49:36 -0700783{
Oliver Glitta5cf909c2021-07-07 18:07:47 -0700784 depot_stack_handle_t handle __maybe_unused;
785
Christoph Lameter81819f02007-05-06 14:49:36 -0700786 if (!t->addr)
787 return;
788
Yafang Shao96b94ab2021-03-19 18:12:45 +0800789 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
Chintan Pandya86609d32018-04-05 16:20:15 -0700790 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
Oliver Glitta5cf909c2021-07-07 18:07:47 -0700791#ifdef CONFIG_STACKDEPOT
792 handle = READ_ONCE(t->handle);
793 if (handle)
794 stack_depot_print(handle);
795 else
796 pr_err("object allocation/free stack trace missing\n");
Ben Greeard6543e32011-07-07 11:36:36 -0700797#endif
Christoph Lameter81819f02007-05-06 14:49:36 -0700798}
799
Vlastimil Babkae42f1742020-08-06 23:19:05 -0700800void print_tracking(struct kmem_cache *s, void *object)
Christoph Lameter24922682007-07-17 04:03:18 -0700801{
Chintan Pandya86609d32018-04-05 16:20:15 -0700802 unsigned long pr_time = jiffies;
Christoph Lameter24922682007-07-17 04:03:18 -0700803 if (!(s->flags & SLAB_STORE_USER))
804 return;
805
Chintan Pandya86609d32018-04-05 16:20:15 -0700806 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
807 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
Christoph Lameter24922682007-07-17 04:03:18 -0700808}
809
Matthew Wilcox (Oracle)fb012e22021-10-04 14:46:34 +0100810static void print_slab_info(const struct slab *slab)
Christoph Lameter24922682007-07-17 04:03:18 -0700811{
Matthew Wilcox (Oracle)fb012e22021-10-04 14:46:34 +0100812 struct folio *folio = (struct folio *)slab_folio(slab);
Christoph Lameter24922682007-07-17 04:03:18 -0700813
Matthew Wilcox (Oracle)fb012e22021-10-04 14:46:34 +0100814 pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",
815 slab, slab->objects, slab->inuse, slab->freelist,
816 folio_flags(folio, 0));
Christoph Lameter24922682007-07-17 04:03:18 -0700817}
818
819static void slab_bug(struct kmem_cache *s, char *fmt, ...)
820{
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700821 struct va_format vaf;
Christoph Lameter24922682007-07-17 04:03:18 -0700822 va_list args;
Christoph Lameter24922682007-07-17 04:03:18 -0700823
824 va_start(args, fmt);
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700825 vaf.fmt = fmt;
826 vaf.va = &args;
Fabian Frederickf9f58282014-06-04 16:06:34 -0700827 pr_err("=============================================================================\n");
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700828 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
Fabian Frederickf9f58282014-06-04 16:06:34 -0700829 pr_err("-----------------------------------------------------------------------------\n\n");
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700830 va_end(args);
Christoph Lameter24922682007-07-17 04:03:18 -0700831}
832
Joe Perches582d1212021-06-28 19:34:49 -0700833__printf(2, 3)
Christoph Lameter24922682007-07-17 04:03:18 -0700834static void slab_fix(struct kmem_cache *s, char *fmt, ...)
835{
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700836 struct va_format vaf;
Christoph Lameter24922682007-07-17 04:03:18 -0700837 va_list args;
Christoph Lameter24922682007-07-17 04:03:18 -0700838
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700839 if (slab_add_kunit_errors())
840 return;
841
Christoph Lameter24922682007-07-17 04:03:18 -0700842 va_start(args, fmt);
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700843 vaf.fmt = fmt;
844 vaf.va = &args;
845 pr_err("FIX %s: %pV\n", s->name, &vaf);
Christoph Lameter24922682007-07-17 04:03:18 -0700846 va_end(args);
Christoph Lameter24922682007-07-17 04:03:18 -0700847}
848
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100849static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
Christoph Lameter81819f02007-05-06 14:49:36 -0700850{
851 unsigned int off; /* Offset of last byte */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100852 u8 *addr = slab_address(slab);
Christoph Lameter24922682007-07-17 04:03:18 -0700853
854 print_tracking(s, p);
855
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100856 print_slab_info(slab);
Christoph Lameter24922682007-07-17 04:03:18 -0700857
Yafang Shao96b94ab2021-03-19 18:12:45 +0800858 pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
Fabian Frederickf9f58282014-06-04 16:06:34 -0700859 p, p - addr, get_freepointer(s, p));
Christoph Lameter24922682007-07-17 04:03:18 -0700860
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700861 if (s->flags & SLAB_RED_ZONE)
Kees Cook8669dba2021-06-15 18:23:19 -0700862 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800863 s->red_left_pad);
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700864 else if (p > addr + 16)
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800865 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
Christoph Lameter24922682007-07-17 04:03:18 -0700866
Kees Cook8669dba2021-06-15 18:23:19 -0700867 print_section(KERN_ERR, "Object ", p,
Alexey Dobriyan1b473f22018-04-05 16:21:17 -0700868 min_t(unsigned int, s->object_size, PAGE_SIZE));
Christoph Lameter81819f02007-05-06 14:49:36 -0700869 if (s->flags & SLAB_RED_ZONE)
Kees Cook8669dba2021-06-15 18:23:19 -0700870 print_section(KERN_ERR, "Redzone ", p + s->object_size,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500871 s->inuse - s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -0700872
Waiman Longcbfc35a2020-05-07 18:36:06 -0700873 off = get_info_end(s);
Christoph Lameter81819f02007-05-06 14:49:36 -0700874
Christoph Lameter24922682007-07-17 04:03:18 -0700875 if (s->flags & SLAB_STORE_USER)
Christoph Lameter81819f02007-05-06 14:49:36 -0700876 off += 2 * sizeof(struct track);
Christoph Lameter81819f02007-05-06 14:49:36 -0700877
Alexander Potapenko80a92012016-07-28 15:49:07 -0700878 off += kasan_metadata_size(s);
879
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700880 if (off != size_from_object(s))
Christoph Lameter81819f02007-05-06 14:49:36 -0700881 /* Beginning of the filler is the free pointer */
Kees Cook8669dba2021-06-15 18:23:19 -0700882 print_section(KERN_ERR, "Padding ", p + off,
Daniel Thompsonaa2efd52017-01-24 15:18:02 -0800883 size_from_object(s) - off);
Christoph Lameter24922682007-07-17 04:03:18 -0700884
885 dump_stack();
Christoph Lameter81819f02007-05-06 14:49:36 -0700886}
887
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100888static void object_err(struct kmem_cache *s, struct slab *slab,
Christoph Lameter81819f02007-05-06 14:49:36 -0700889 u8 *object, char *reason)
890{
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700891 if (slab_add_kunit_errors())
892 return;
893
Christoph Lameter3dc50632008-04-23 12:28:01 -0700894 slab_bug(s, "%s", reason);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100895 print_trailer(s, slab, object);
Georgi Djakov65ebdee2021-06-28 19:34:58 -0700896 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter81819f02007-05-06 14:49:36 -0700897}
898
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100899static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
Vlastimil Babkaae16d052021-10-26 18:22:44 +0200900 void **freelist, void *nextfree)
901{
902 if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100903 !check_valid_pointer(s, slab, nextfree) && freelist) {
904 object_err(s, slab, *freelist, "Freechain corrupt");
Vlastimil Babkaae16d052021-10-26 18:22:44 +0200905 *freelist = NULL;
906 slab_fix(s, "Isolate corrupted freechain");
907 return true;
908 }
909
910 return false;
911}
912
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100913static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
Chen Gangd0e0ac92013-07-15 09:05:29 +0800914 const char *fmt, ...)
Christoph Lameter81819f02007-05-06 14:49:36 -0700915{
916 va_list args;
917 char buf[100];
918
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700919 if (slab_add_kunit_errors())
920 return;
921
Christoph Lameter24922682007-07-17 04:03:18 -0700922 va_start(args, fmt);
923 vsnprintf(buf, sizeof(buf), fmt, args);
Christoph Lameter81819f02007-05-06 14:49:36 -0700924 va_end(args);
Christoph Lameter3dc50632008-04-23 12:28:01 -0700925 slab_bug(s, "%s", buf);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100926 print_slab_info(slab);
Christoph Lameter81819f02007-05-06 14:49:36 -0700927 dump_stack();
Georgi Djakov65ebdee2021-06-28 19:34:58 -0700928 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter81819f02007-05-06 14:49:36 -0700929}
930
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500931static void init_object(struct kmem_cache *s, void *object, u8 val)
Christoph Lameter81819f02007-05-06 14:49:36 -0700932{
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800933 u8 *p = kasan_reset_tag(object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700934
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -0700935 if (s->flags & SLAB_RED_ZONE)
936 memset(p - s->red_left_pad, val, s->red_left_pad);
937
Christoph Lameter81819f02007-05-06 14:49:36 -0700938 if (s->flags & __OBJECT_POISON) {
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500939 memset(p, POISON_FREE, s->object_size - 1);
940 p[s->object_size - 1] = POISON_END;
Christoph Lameter81819f02007-05-06 14:49:36 -0700941 }
942
943 if (s->flags & SLAB_RED_ZONE)
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500944 memset(p + s->object_size, val, s->inuse - s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -0700945}
946
Christoph Lameter24922682007-07-17 04:03:18 -0700947static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
948 void *from, void *to)
949{
Joe Perches582d1212021-06-28 19:34:49 -0700950 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
Christoph Lameter24922682007-07-17 04:03:18 -0700951 memset(from, data, to - from);
952}
953
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100954static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
Christoph Lameter24922682007-07-17 04:03:18 -0700955 u8 *object, char *what,
Pekka Enberg06428782008-01-07 23:20:27 -0800956 u8 *start, unsigned int value, unsigned int bytes)
Christoph Lameter24922682007-07-17 04:03:18 -0700957{
958 u8 *fault;
959 u8 *end;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100960 u8 *addr = slab_address(slab);
Christoph Lameter24922682007-07-17 04:03:18 -0700961
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800962 metadata_access_enable();
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -0800963 fault = memchr_inv(kasan_reset_tag(start), value, bytes);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800964 metadata_access_disable();
Christoph Lameter24922682007-07-17 04:03:18 -0700965 if (!fault)
966 return 1;
967
968 end = start + bytes;
969 while (end > fault && end[-1] == value)
970 end--;
971
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700972 if (slab_add_kunit_errors())
973 goto skip_bug_print;
974
Christoph Lameter24922682007-07-17 04:03:18 -0700975 slab_bug(s, "%s overwritten", what);
Yafang Shao96b94ab2021-03-19 18:12:45 +0800976 pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
Miles Chene1b70dd2019-11-30 17:49:31 -0800977 fault, end - 1, fault - addr,
978 fault[0], value);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +0100979 print_trailer(s, slab, object);
Georgi Djakov65ebdee2021-06-28 19:34:58 -0700980 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter24922682007-07-17 04:03:18 -0700981
Oliver Glitta1f9f78b2021-06-28 19:34:33 -0700982skip_bug_print:
Christoph Lameter24922682007-07-17 04:03:18 -0700983 restore_bytes(s, what, value, fault, end);
984 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700985}
986
Christoph Lameter81819f02007-05-06 14:49:36 -0700987/*
988 * Object layout:
989 *
990 * object address
991 * Bytes of the object to be managed.
992 * If the freepointer may overlay the object then the free
Waiman Longcbfc35a2020-05-07 18:36:06 -0700993 * pointer is at the middle of the object.
Christoph Lameter672bba32007-05-09 02:32:39 -0700994 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700995 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
996 * 0xa5 (POISON_END)
997 *
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500998 * object + s->object_size
Christoph Lameter81819f02007-05-06 14:49:36 -0700999 * Padding to reach word boundary. This is also used for Redzoning.
Christoph Lameter672bba32007-05-09 02:32:39 -07001000 * Padding is extended by another word if Redzoning is enabled and
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05001001 * object_size == inuse.
Christoph Lameter672bba32007-05-09 02:32:39 -07001002 *
Christoph Lameter81819f02007-05-06 14:49:36 -07001003 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
1004 * 0xcc (RED_ACTIVE) for objects in use.
1005 *
1006 * object + s->inuse
Christoph Lameter672bba32007-05-09 02:32:39 -07001007 * Meta data starts here.
1008 *
Christoph Lameter81819f02007-05-06 14:49:36 -07001009 * A. Free pointer (if we cannot overwrite object on free)
1010 * B. Tracking data for SLAB_STORE_USER
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -07001011 * C. Padding to reach required alignment boundary or at minimum
Christoph Lameter6446faa2008-02-15 23:45:26 -08001012 * one word if debugging is on to be able to detect writes
Christoph Lameter672bba32007-05-09 02:32:39 -07001013 * before the word boundary.
1014 *
1015 * Padding is done using 0x5a (POISON_INUSE)
Christoph Lameter81819f02007-05-06 14:49:36 -07001016 *
1017 * object + s->size
Christoph Lameter672bba32007-05-09 02:32:39 -07001018 * Nothing is used beyond s->size.
Christoph Lameter81819f02007-05-06 14:49:36 -07001019 *
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05001020 * If slabcaches are merged then the object_size and inuse boundaries are mostly
Christoph Lameter672bba32007-05-09 02:32:39 -07001021 * ignored. And therefore no slab options that rely on these boundaries
Christoph Lameter81819f02007-05-06 14:49:36 -07001022 * may be used with merged slabcaches.
1023 */
1024
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001025static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
Christoph Lameter81819f02007-05-06 14:49:36 -07001026{
Waiman Longcbfc35a2020-05-07 18:36:06 -07001027 unsigned long off = get_info_end(s); /* The end of info */
Christoph Lameter81819f02007-05-06 14:49:36 -07001028
1029 if (s->flags & SLAB_STORE_USER)
1030 /* We also have user information there */
1031 off += 2 * sizeof(struct track);
1032
Alexander Potapenko80a92012016-07-28 15:49:07 -07001033 off += kasan_metadata_size(s);
1034
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -07001035 if (size_from_object(s) == off)
Christoph Lameter81819f02007-05-06 14:49:36 -07001036 return 1;
1037
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001038 return check_bytes_and_report(s, slab, p, "Object padding",
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -07001039 p + off, POISON_INUSE, size_from_object(s) - off);
Christoph Lameter81819f02007-05-06 14:49:36 -07001040}
1041
Christoph Lameter39b26462008-04-14 19:11:30 +03001042/* Check the pad bytes at the end of a slab page */
Miaohe Lina204e6d2022-04-19 20:03:52 +08001043static void slab_pad_check(struct kmem_cache *s, struct slab *slab)
Christoph Lameter81819f02007-05-06 14:49:36 -07001044{
Christoph Lameter24922682007-07-17 04:03:18 -07001045 u8 *start;
1046 u8 *fault;
1047 u8 *end;
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -08001048 u8 *pad;
Christoph Lameter24922682007-07-17 04:03:18 -07001049 int length;
1050 int remainder;
Christoph Lameter81819f02007-05-06 14:49:36 -07001051
1052 if (!(s->flags & SLAB_POISON))
Miaohe Lina204e6d2022-04-19 20:03:52 +08001053 return;
Christoph Lameter81819f02007-05-06 14:49:36 -07001054
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001055 start = slab_address(slab);
1056 length = slab_size(slab);
Christoph Lameter39b26462008-04-14 19:11:30 +03001057 end = start + length;
1058 remainder = length % s->size;
Christoph Lameter81819f02007-05-06 14:49:36 -07001059 if (!remainder)
Miaohe Lina204e6d2022-04-19 20:03:52 +08001060 return;
Christoph Lameter81819f02007-05-06 14:49:36 -07001061
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -08001062 pad = end - remainder;
Andrey Ryabinina79316c2015-02-13 14:39:38 -08001063 metadata_access_enable();
Andrey Konovalovaa1ef4d2020-12-22 12:02:17 -08001064 fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
Andrey Ryabinina79316c2015-02-13 14:39:38 -08001065 metadata_access_disable();
Christoph Lameter24922682007-07-17 04:03:18 -07001066 if (!fault)
Miaohe Lina204e6d2022-04-19 20:03:52 +08001067 return;
Christoph Lameter24922682007-07-17 04:03:18 -07001068 while (end > fault && end[-1] == POISON_INUSE)
1069 end--;
1070
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001071 slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu",
Miles Chene1b70dd2019-11-30 17:49:31 -08001072 fault, end - 1, fault - start);
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -08001073 print_section(KERN_ERR, "Padding ", pad, remainder);
Christoph Lameter24922682007-07-17 04:03:18 -07001074
Balasubramani Vivekanandan5d682682018-01-31 16:15:43 -08001075 restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
Christoph Lameter81819f02007-05-06 14:49:36 -07001076}
1077
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001078static int check_object(struct kmem_cache *s, struct slab *slab,
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001079 void *object, u8 val)
Christoph Lameter81819f02007-05-06 14:49:36 -07001080{
1081 u8 *p = object;
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05001082 u8 *endobject = object + s->object_size;
Christoph Lameter81819f02007-05-06 14:49:36 -07001083
1084 if (s->flags & SLAB_RED_ZONE) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001085 if (!check_bytes_and_report(s, slab, object, "Left Redzone",
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -07001086 object - s->red_left_pad, val, s->red_left_pad))
1087 return 0;
1088
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001089 if (!check_bytes_and_report(s, slab, object, "Right Redzone",
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05001090 endobject, val, s->inuse - s->object_size))
Christoph Lameter81819f02007-05-06 14:49:36 -07001091 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07001092 } else {
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05001093 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001094 check_bytes_and_report(s, slab, p, "Alignment padding",
Chen Gangd0e0ac92013-07-15 09:05:29 +08001095 endobject, POISON_INUSE,
1096 s->inuse - s->object_size);
Ingo Molnar3adbefe2008-02-05 17:57:39 -08001097 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001098 }
1099
1100 if (s->flags & SLAB_POISON) {
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001101 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001102 (!check_bytes_and_report(s, slab, p, "Poison", p,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05001103 POISON_FREE, s->object_size - 1) ||
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001104 !check_bytes_and_report(s, slab, p, "End Poison",
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05001105 p + s->object_size - 1, POISON_END, 1)))
Christoph Lameter81819f02007-05-06 14:49:36 -07001106 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07001107 /*
1108 * check_pad_bytes cleans up on its own.
1109 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001110 check_pad_bytes(s, slab, p);
Christoph Lameter81819f02007-05-06 14:49:36 -07001111 }
1112
Waiman Longcbfc35a2020-05-07 18:36:06 -07001113 if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
Christoph Lameter81819f02007-05-06 14:49:36 -07001114 /*
1115 * Object and freepointer overlap. Cannot check
1116 * freepointer while object is allocated.
1117 */
1118 return 1;
1119
1120 /* Check free pointer validity */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001121 if (!check_valid_pointer(s, slab, get_freepointer(s, p))) {
1122 object_err(s, slab, p, "Freepointer corrupt");
Christoph Lameter81819f02007-05-06 14:49:36 -07001123 /*
Nick Andrew9f6c708e2008-12-05 14:08:08 +11001124 * No choice but to zap it and thus lose the remainder
Christoph Lameter81819f02007-05-06 14:49:36 -07001125 * of the free objects in this slab. May cause
Christoph Lameter672bba32007-05-09 02:32:39 -07001126 * another error because the object count is now wrong.
Christoph Lameter81819f02007-05-06 14:49:36 -07001127 */
Christoph Lametera973e9d2008-03-01 13:40:44 -08001128 set_freepointer(s, p, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07001129 return 0;
1130 }
1131 return 1;
1132}
1133
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001134static int check_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter81819f02007-05-06 14:49:36 -07001135{
Christoph Lameter39b26462008-04-14 19:11:30 +03001136 int maxobj;
1137
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001138 if (!folio_test_slab(slab_folio(slab))) {
1139 slab_err(s, slab, "Not a valid slab page");
Christoph Lameter81819f02007-05-06 14:49:36 -07001140 return 0;
1141 }
Christoph Lameter39b26462008-04-14 19:11:30 +03001142
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001143 maxobj = order_objects(slab_order(slab), s->size);
1144 if (slab->objects > maxobj) {
1145 slab_err(s, slab, "objects %u > max %u",
1146 slab->objects, maxobj);
Christoph Lameter39b26462008-04-14 19:11:30 +03001147 return 0;
1148 }
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001149 if (slab->inuse > slab->objects) {
1150 slab_err(s, slab, "inuse %u > max %u",
1151 slab->inuse, slab->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07001152 return 0;
1153 }
1154 /* Slab_pad_check fixes things up after itself */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001155 slab_pad_check(s, slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07001156 return 1;
1157}
1158
1159/*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01001160 * Determine if a certain object in a slab is on the freelist. Must hold the
Christoph Lameter672bba32007-05-09 02:32:39 -07001161 * slab lock to guarantee that the chains are in a consistent state.
Christoph Lameter81819f02007-05-06 14:49:36 -07001162 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001163static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
Christoph Lameter81819f02007-05-06 14:49:36 -07001164{
1165 int nr = 0;
Christoph Lameter881db7f2011-06-01 12:25:53 -05001166 void *fp;
Christoph Lameter81819f02007-05-06 14:49:36 -07001167 void *object = NULL;
Andrey Ryabininf6edde92014-12-10 15:42:22 -08001168 int max_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07001169
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001170 fp = slab->freelist;
1171 while (fp && nr <= slab->objects) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001172 if (fp == search)
1173 return 1;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001174 if (!check_valid_pointer(s, slab, fp)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001175 if (object) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001176 object_err(s, slab, object,
Christoph Lameter81819f02007-05-06 14:49:36 -07001177 "Freechain corrupt");
Christoph Lametera973e9d2008-03-01 13:40:44 -08001178 set_freepointer(s, object, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07001179 } else {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001180 slab_err(s, slab, "Freepointer corrupt");
1181 slab->freelist = NULL;
1182 slab->inuse = slab->objects;
Christoph Lameter24922682007-07-17 04:03:18 -07001183 slab_fix(s, "Freelist cleared");
Christoph Lameter81819f02007-05-06 14:49:36 -07001184 return 0;
1185 }
1186 break;
1187 }
1188 object = fp;
1189 fp = get_freepointer(s, object);
1190 nr++;
1191 }
1192
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001193 max_objects = order_objects(slab_order(slab), s->size);
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +04001194 if (max_objects > MAX_OBJS_PER_PAGE)
1195 max_objects = MAX_OBJS_PER_PAGE;
Christoph Lameter224a88b2008-04-14 19:11:31 +03001196
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001197 if (slab->objects != max_objects) {
1198 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
1199 slab->objects, max_objects);
1200 slab->objects = max_objects;
Joe Perches582d1212021-06-28 19:34:49 -07001201 slab_fix(s, "Number of objects adjusted");
Christoph Lameter224a88b2008-04-14 19:11:31 +03001202 }
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001203 if (slab->inuse != slab->objects - nr) {
1204 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
1205 slab->inuse, slab->objects - nr);
1206 slab->inuse = slab->objects - nr;
Joe Perches582d1212021-06-28 19:34:49 -07001207 slab_fix(s, "Object count adjusted");
Christoph Lameter81819f02007-05-06 14:49:36 -07001208 }
1209 return search == NULL;
1210}
1211
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001212static void trace(struct kmem_cache *s, struct slab *slab, void *object,
Christoph Lameter0121c6192008-04-29 16:11:12 -07001213 int alloc)
Christoph Lameter3ec09742007-05-16 22:11:00 -07001214{
1215 if (s->flags & SLAB_TRACE) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07001216 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
Christoph Lameter3ec09742007-05-16 22:11:00 -07001217 s->name,
1218 alloc ? "alloc" : "free",
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001219 object, slab->inuse,
1220 slab->freelist);
Christoph Lameter3ec09742007-05-16 22:11:00 -07001221
1222 if (!alloc)
Daniel Thompsonaa2efd52017-01-24 15:18:02 -08001223 print_section(KERN_INFO, "Object ", (void *)object,
Chen Gangd0e0ac92013-07-15 09:05:29 +08001224 s->object_size);
Christoph Lameter3ec09742007-05-16 22:11:00 -07001225
1226 dump_stack();
1227 }
1228}
1229
Christoph Lameter643b1132007-05-06 14:49:42 -07001230/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001231 * Tracking of fully allocated slabs for debugging purposes.
Christoph Lameter643b1132007-05-06 14:49:42 -07001232 */
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001233static void add_full(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001234 struct kmem_cache_node *n, struct slab *slab)
Christoph Lameter643b1132007-05-06 14:49:42 -07001235{
Christoph Lameter643b1132007-05-06 14:49:42 -07001236 if (!(s->flags & SLAB_STORE_USER))
1237 return;
1238
David Rientjes255d0882014-02-10 14:25:39 -08001239 lockdep_assert_held(&n->list_lock);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001240 list_add(&slab->slab_list, &n->full);
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001241}
Christoph Lameter643b1132007-05-06 14:49:42 -07001242
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001243static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001244{
1245 if (!(s->flags & SLAB_STORE_USER))
1246 return;
1247
David Rientjes255d0882014-02-10 14:25:39 -08001248 lockdep_assert_held(&n->list_lock);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001249 list_del(&slab->slab_list);
Christoph Lameter643b1132007-05-06 14:49:42 -07001250}
1251
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001252/* Tracking of the number of slabs for debugging purposes */
1253static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1254{
1255 struct kmem_cache_node *n = get_node(s, node);
1256
1257 return atomic_long_read(&n->nr_slabs);
1258}
1259
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04001260static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1261{
1262 return atomic_long_read(&n->nr_slabs);
1263}
1264
Christoph Lameter205ab992008-04-14 19:11:40 +03001265static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001266{
1267 struct kmem_cache_node *n = get_node(s, node);
1268
1269 /*
1270 * May be called early in order to allocate a slab for the
1271 * kmem_cache_node structure. Solve the chicken-egg
1272 * dilemma by deferring the increment of the count during
1273 * bootstrap (see early_kmem_cache_node_alloc).
1274 */
Joonsoo Kim338b2642013-01-21 17:01:27 +09001275 if (likely(n)) {
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001276 atomic_long_inc(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +03001277 atomic_long_add(objects, &n->total_objects);
1278 }
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001279}
Christoph Lameter205ab992008-04-14 19:11:40 +03001280static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001281{
1282 struct kmem_cache_node *n = get_node(s, node);
1283
1284 atomic_long_dec(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +03001285 atomic_long_sub(objects, &n->total_objects);
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001286}
1287
1288/* Object debug checks for alloc/free paths */
JaeSang Yooc0f81a92022-04-11 16:25:34 +09001289static void setup_object_debug(struct kmem_cache *s, void *object)
Christoph Lameter3ec09742007-05-16 22:11:00 -07001290{
Vlastimil Babka8fc8d662020-08-06 23:18:58 -07001291 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
Christoph Lameter3ec09742007-05-16 22:11:00 -07001292 return;
1293
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001294 init_object(s, object, SLUB_RED_INACTIVE);
Christoph Lameter3ec09742007-05-16 22:11:00 -07001295 init_tracking(s, object);
1296}
1297
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07001298static
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001299void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
Andrey Konovalova7101222019-02-20 22:19:23 -08001300{
Vlastimil Babka8fc8d662020-08-06 23:18:58 -07001301 if (!kmem_cache_debug_flags(s, SLAB_POISON))
Andrey Konovalova7101222019-02-20 22:19:23 -08001302 return;
1303
1304 metadata_access_enable();
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001305 memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
Andrey Konovalova7101222019-02-20 22:19:23 -08001306 metadata_access_disable();
1307}
1308
Laura Abbottbecfda62016-03-15 14:55:06 -07001309static inline int alloc_consistency_checks(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001310 struct slab *slab, void *object)
Christoph Lameter81819f02007-05-06 14:49:36 -07001311{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001312 if (!check_slab(s, slab))
Laura Abbottbecfda62016-03-15 14:55:06 -07001313 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07001314
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001315 if (!check_valid_pointer(s, slab, object)) {
1316 object_err(s, slab, object, "Freelist Pointer check fails");
Laura Abbottbecfda62016-03-15 14:55:06 -07001317 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07001318 }
1319
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001320 if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
Laura Abbottbecfda62016-03-15 14:55:06 -07001321 return 0;
1322
1323 return 1;
1324}
1325
1326static noinline int alloc_debug_processing(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001327 struct slab *slab,
Laura Abbottbecfda62016-03-15 14:55:06 -07001328 void *object, unsigned long addr)
1329{
1330 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001331 if (!alloc_consistency_checks(s, slab, object))
Laura Abbottbecfda62016-03-15 14:55:06 -07001332 goto bad;
1333 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001334
Christoph Lameter3ec09742007-05-16 22:11:00 -07001335 /* Success perform special debug activities for allocs */
1336 if (s->flags & SLAB_STORE_USER)
1337 set_track(s, object, TRACK_ALLOC, addr);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001338 trace(s, slab, object, 1);
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001339 init_object(s, object, SLUB_RED_ACTIVE);
Christoph Lameter81819f02007-05-06 14:49:36 -07001340 return 1;
Christoph Lameter3ec09742007-05-16 22:11:00 -07001341
Christoph Lameter81819f02007-05-06 14:49:36 -07001342bad:
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001343 if (folio_test_slab(slab_folio(slab))) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001344 /*
1345 * If this is a slab page then lets do the best we can
1346 * to avoid issues in the future. Marking all objects
Christoph Lameter672bba32007-05-09 02:32:39 -07001347 * as used avoids touching the remaining objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07001348 */
Christoph Lameter24922682007-07-17 04:03:18 -07001349 slab_fix(s, "Marking all objects used");
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001350 slab->inuse = slab->objects;
1351 slab->freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001352 }
1353 return 0;
1354}
1355
Laura Abbottbecfda62016-03-15 14:55:06 -07001356static inline int free_consistency_checks(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001357 struct slab *slab, void *object, unsigned long addr)
Laura Abbottbecfda62016-03-15 14:55:06 -07001358{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001359 if (!check_valid_pointer(s, slab, object)) {
1360 slab_err(s, slab, "Invalid object pointer 0x%p", object);
Laura Abbottbecfda62016-03-15 14:55:06 -07001361 return 0;
1362 }
1363
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001364 if (on_freelist(s, slab, object)) {
1365 object_err(s, slab, object, "Object already free");
Laura Abbottbecfda62016-03-15 14:55:06 -07001366 return 0;
1367 }
1368
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001369 if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
Laura Abbottbecfda62016-03-15 14:55:06 -07001370 return 0;
1371
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001372 if (unlikely(s != slab->slab_cache)) {
1373 if (!folio_test_slab(slab_folio(slab))) {
1374 slab_err(s, slab, "Attempt to free object(0x%p) outside of slab",
Joe Perches756a0252016-03-17 14:19:47 -07001375 object);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001376 } else if (!slab->slab_cache) {
Laura Abbottbecfda62016-03-15 14:55:06 -07001377 pr_err("SLUB <none>: no slab for object 0x%p.\n",
1378 object);
1379 dump_stack();
1380 } else
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001381 object_err(s, slab, object,
Laura Abbottbecfda62016-03-15 14:55:06 -07001382 "page slab pointer corrupt.");
1383 return 0;
1384 }
1385 return 1;
1386}
1387
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001388/* Supports checking bulk free of a constructed freelist */
Laura Abbott282acb42016-03-15 14:54:59 -07001389static noinline int free_debug_processing(
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001390 struct kmem_cache *s, struct slab *slab,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001391 void *head, void *tail, int bulk_cnt,
Laura Abbott282acb42016-03-15 14:54:59 -07001392 unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -07001393{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001394 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001395 void *object = head;
1396 int cnt = 0;
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +02001397 unsigned long flags, flags2;
Laura Abbott804aa132016-03-15 14:55:02 -07001398 int ret = 0;
Sebastian Andrzej Siewiorc4cf6782022-06-07 17:20:10 +02001399 depot_stack_handle_t handle = 0;
1400
1401 if (s->flags & SLAB_STORE_USER)
1402 handle = set_track_prepare();
Christoph Lameter5c2e4bb2011-06-01 12:25:54 -05001403
Laura Abbott282acb42016-03-15 14:54:59 -07001404 spin_lock_irqsave(&n->list_lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001405 slab_lock(slab, &flags2);
Christoph Lameter881db7f2011-06-01 12:25:53 -05001406
Laura Abbottbecfda62016-03-15 14:55:06 -07001407 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001408 if (!check_slab(s, slab))
Laura Abbottbecfda62016-03-15 14:55:06 -07001409 goto out;
1410 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001411
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001412next_object:
1413 cnt++;
1414
Laura Abbottbecfda62016-03-15 14:55:06 -07001415 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001416 if (!free_consistency_checks(s, slab, object, addr))
Laura Abbottbecfda62016-03-15 14:55:06 -07001417 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07001418 }
Christoph Lameter3ec09742007-05-16 22:11:00 -07001419
Christoph Lameter3ec09742007-05-16 22:11:00 -07001420 if (s->flags & SLAB_STORE_USER)
Sebastian Andrzej Siewiorc4cf6782022-06-07 17:20:10 +02001421 set_track_update(s, object, TRACK_FREE, addr, handle);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001422 trace(s, slab, object, 0);
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001423 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001424 init_object(s, object, SLUB_RED_INACTIVE);
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001425
1426 /* Reached end of constructed freelist yet? */
1427 if (object != tail) {
1428 object = get_freepointer(s, object);
1429 goto next_object;
1430 }
Laura Abbott804aa132016-03-15 14:55:02 -07001431 ret = 1;
1432
Christoph Lameter5c2e4bb2011-06-01 12:25:54 -05001433out:
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001434 if (cnt != bulk_cnt)
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001435 slab_err(s, slab, "Bulk freelist count(%d) invalid(%d)\n",
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001436 bulk_cnt, cnt);
1437
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001438 slab_unlock(slab, &flags2);
Laura Abbott282acb42016-03-15 14:54:59 -07001439 spin_unlock_irqrestore(&n->list_lock, flags);
Laura Abbott804aa132016-03-15 14:55:02 -07001440 if (!ret)
1441 slab_fix(s, "Object at 0x%p not freed", object);
1442 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07001443}
1444
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001445/*
1446 * Parse a block of slub_debug options. Blocks are delimited by ';'
1447 *
1448 * @str: start of block
1449 * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
1450 * @slabs: return start of list of slabs, or NULL when there's no list
1451 * @init: assume this is initial parsing and not per-kmem-create parsing
1452 *
1453 * returns the start of next block if there's any, or NULL
1454 */
1455static char *
1456parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
1457{
1458 bool higher_order_disable = false;
1459
1460 /* Skip any completely empty blocks */
1461 while (*str && *str == ';')
1462 str++;
1463
1464 if (*str == ',') {
1465 /*
1466 * No options but restriction on slabs. This means full
1467 * debugging for slabs matching a pattern.
1468 */
1469 *flags = DEBUG_DEFAULT_FLAGS;
1470 goto check_slabs;
1471 }
1472 *flags = 0;
1473
1474 /* Determine which debug features should be switched on */
1475 for (; *str && *str != ',' && *str != ';'; str++) {
1476 switch (tolower(*str)) {
1477 case '-':
1478 *flags = 0;
1479 break;
1480 case 'f':
1481 *flags |= SLAB_CONSISTENCY_CHECKS;
1482 break;
1483 case 'z':
1484 *flags |= SLAB_RED_ZONE;
1485 break;
1486 case 'p':
1487 *flags |= SLAB_POISON;
1488 break;
1489 case 'u':
1490 *flags |= SLAB_STORE_USER;
1491 break;
1492 case 't':
1493 *flags |= SLAB_TRACE;
1494 break;
1495 case 'a':
1496 *flags |= SLAB_FAILSLAB;
1497 break;
1498 case 'o':
1499 /*
1500 * Avoid enabling debugging on caches if its minimum
1501 * order would increase as a result.
1502 */
1503 higher_order_disable = true;
1504 break;
1505 default:
1506 if (init)
1507 pr_err("slub_debug option '%c' unknown. skipped\n", *str);
1508 }
1509 }
1510check_slabs:
1511 if (*str == ',')
1512 *slabs = ++str;
1513 else
1514 *slabs = NULL;
1515
1516 /* Skip over the slab list */
1517 while (*str && *str != ';')
1518 str++;
1519
1520 /* Skip any completely empty blocks */
1521 while (*str && *str == ';')
1522 str++;
1523
1524 if (init && higher_order_disable)
1525 disable_higher_order_debug = 1;
1526
1527 if (*str)
1528 return str;
1529 else
1530 return NULL;
1531}
1532
Christoph Lameter41ecc552007-05-09 02:32:44 -07001533static int __init setup_slub_debug(char *str)
1534{
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001535 slab_flags_t flags;
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001536 slab_flags_t global_flags;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001537 char *saved_str;
1538 char *slab_list;
1539 bool global_slub_debug_changed = false;
1540 bool slab_list_specified = false;
1541
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001542 global_flags = DEBUG_DEFAULT_FLAGS;
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001543 if (*str++ != '=' || !*str)
1544 /*
1545 * No options specified. Switch on full debugging.
1546 */
1547 goto out;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001548
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001549 saved_str = str;
1550 while (str) {
1551 str = parse_slub_debug_flags(str, &flags, &slab_list, true);
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001552
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001553 if (!slab_list) {
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001554 global_flags = flags;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001555 global_slub_debug_changed = true;
1556 } else {
1557 slab_list_specified = true;
Oliver Glitta5cf909c2021-07-07 18:07:47 -07001558 if (flags & SLAB_STORE_USER)
1559 stack_depot_want_early_init();
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001560 }
1561 }
1562
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001563 /*
1564 * For backwards compatibility, a single list of flags with list of
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001565 * slabs means debugging is only changed for those slabs, so the global
1566 * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
1567 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001568 * long as there is no option specifying flags without a slab list.
1569 */
1570 if (slab_list_specified) {
1571 if (!global_slub_debug_changed)
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001572 global_flags = slub_debug;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001573 slub_debug_string = saved_str;
1574 }
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001575out:
Vlastimil Babkaa7f1d482021-08-13 16:54:34 -07001576 slub_debug = global_flags;
Oliver Glitta5cf909c2021-07-07 18:07:47 -07001577 if (slub_debug & SLAB_STORE_USER)
1578 stack_depot_want_early_init();
Vlastimil Babkaca0cab62020-08-06 23:18:51 -07001579 if (slub_debug != 0 || slub_debug_string)
1580 static_branch_enable(&slub_debug_enabled);
Stephen Boyd02ac47d2021-06-28 19:34:43 -07001581 else
1582 static_branch_disable(&slub_debug_enabled);
Alexander Potapenko64713842019-07-11 20:59:19 -07001583 if ((static_branch_unlikely(&init_on_alloc) ||
1584 static_branch_unlikely(&init_on_free)) &&
1585 (slub_debug & SLAB_POISON))
1586 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
Christoph Lameter41ecc552007-05-09 02:32:44 -07001587 return 1;
1588}
1589
1590__setup("slub_debug", setup_slub_debug);
1591
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001592/*
1593 * kmem_cache_flags - apply debugging options to the cache
1594 * @object_size: the size of an object without meta data
1595 * @flags: flags to set
1596 * @name: name of the cache
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001597 *
1598 * Debug option(s) are applied to @flags. In addition to the debug
1599 * option(s), if a slab name (or multiple) is specified i.e.
1600 * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1601 * then only the select slabs will receive the debug option(s).
1602 */
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -07001603slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -08001604 slab_flags_t flags, const char *name)
Christoph Lameter41ecc552007-05-09 02:32:44 -07001605{
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001606 char *iter;
1607 size_t len;
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001608 char *next_block;
1609 slab_flags_t block_flags;
Johannes Bergca220592021-02-24 12:01:04 -08001610 slab_flags_t slub_debug_local = slub_debug;
1611
Hyeonggon Yooa2859092022-04-06 15:00:03 +09001612 if (flags & SLAB_NO_USER_FLAGS)
1613 return flags;
1614
Johannes Bergca220592021-02-24 12:01:04 -08001615 /*
1616 * If the slab cache is for debugging (e.g. kmemleak) then
1617 * don't store user (stack trace) information by default,
1618 * but let the user enable it via the command line below.
1619 */
1620 if (flags & SLAB_NOLEAKTRACE)
1621 slub_debug_local &= ~SLAB_STORE_USER;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001622
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001623 len = strlen(name);
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001624 next_block = slub_debug_string;
1625 /* Go through all blocks of debug options, see if any matches our slab's name */
1626 while (next_block) {
1627 next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
1628 if (!iter)
1629 continue;
1630 /* Found a block that has a slab list, search it */
1631 while (*iter) {
1632 char *end, *glob;
1633 size_t cmplen;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001634
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001635 end = strchrnul(iter, ',');
1636 if (next_block && next_block < end)
1637 end = next_block - 1;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001638
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001639 glob = strnchr(iter, end - iter, '*');
1640 if (glob)
1641 cmplen = glob - iter;
1642 else
1643 cmplen = max_t(size_t, len, (end - iter));
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001644
Vlastimil Babkae17f1df2020-08-06 23:18:35 -07001645 if (!strncmp(name, iter, cmplen)) {
1646 flags |= block_flags;
1647 return flags;
1648 }
1649
1650 if (!*end || *end == ';')
1651 break;
1652 iter = end + 1;
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001653 }
Aaron Tomlinc5fd3ca2018-10-26 15:03:15 -07001654 }
Christoph Lameterba0268a2007-09-11 15:24:11 -07001655
Johannes Bergca220592021-02-24 12:01:04 -08001656 return flags | slub_debug_local;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001657}
Jesper Dangaard Brouerb4a64712015-11-20 15:57:41 -08001658#else /* !CONFIG_SLUB_DEBUG */
JaeSang Yooc0f81a92022-04-11 16:25:34 +09001659static inline void setup_object_debug(struct kmem_cache *s, void *object) {}
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07001660static inline
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001661void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001662
Christoph Lameter3ec09742007-05-16 22:11:00 -07001663static inline int alloc_debug_processing(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001664 struct slab *slab, void *object, unsigned long addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001665
Laura Abbott282acb42016-03-15 14:54:59 -07001666static inline int free_debug_processing(
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001667 struct kmem_cache *s, struct slab *slab,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001668 void *head, void *tail, int bulk_cnt,
Laura Abbott282acb42016-03-15 14:54:59 -07001669 unsigned long addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001670
Miaohe Lina204e6d2022-04-19 20:03:52 +08001671static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001672static inline int check_object(struct kmem_cache *s, struct slab *slab,
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001673 void *object, u8 val) { return 1; }
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001674static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001675 struct slab *slab) {}
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001676static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001677 struct slab *slab) {}
Alexey Dobriyan0293d1f2018-04-05 16:21:24 -07001678slab_flags_t kmem_cache_flags(unsigned int object_size,
Nikolay Borisov37540002021-02-24 12:00:58 -08001679 slab_flags_t flags, const char *name)
Christoph Lameterba0268a2007-09-11 15:24:11 -07001680{
1681 return flags;
1682}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001683#define slub_debug 0
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001684
Ingo Molnarfdaa45e2009-09-15 11:00:26 +02001685#define disable_higher_order_debug 0
1686
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001687static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1688 { return 0; }
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04001689static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1690 { return 0; }
Christoph Lameter205ab992008-04-14 19:11:40 +03001691static inline void inc_slabs_node(struct kmem_cache *s, int node,
1692 int objects) {}
1693static inline void dec_slabs_node(struct kmem_cache *s, int node,
1694 int objects) {}
Christoph Lameter7d550c52010-08-25 14:07:16 -05001695
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001696static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
Eugeniu Roscadc07a722020-09-04 16:35:30 -07001697 void **freelist, void *nextfree)
Dongli Zhang52f23472020-06-01 21:45:47 -07001698{
1699 return false;
1700}
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001701#endif /* CONFIG_SLUB_DEBUG */
1702
1703/*
1704 * Hooks for other subsystems that check memory allocations. In a typical
1705 * production configuration these hooks all should produce no code at all.
1706 */
Andrey Konovalov0116523c2018-12-28 00:29:37 -08001707static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
Roman Bobnievd56791b2013-10-08 15:58:57 -07001708{
Andrey Konovalov53128242019-02-20 22:19:11 -08001709 ptr = kasan_kmalloc_large(ptr, size, flags);
Andrey Konovalova2f77572019-02-20 22:19:16 -08001710 /* As ptr might get tagged, call kmemleak hook after KASAN. */
Roman Bobnievd56791b2013-10-08 15:58:57 -07001711 kmemleak_alloc(ptr, size, 1, flags);
Andrey Konovalov53128242019-02-20 22:19:11 -08001712 return ptr;
Roman Bobnievd56791b2013-10-08 15:58:57 -07001713}
1714
Dmitry Vyukovee3ce772018-02-06 15:36:27 -08001715static __always_inline void kfree_hook(void *x)
Roman Bobnievd56791b2013-10-08 15:58:57 -07001716{
1717 kmemleak_free(x);
Andrey Konovalov027b37b2021-02-24 12:05:46 -08001718 kasan_kfree_large(x);
Roman Bobnievd56791b2013-10-08 15:58:57 -07001719}
1720
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001721static __always_inline bool slab_free_hook(struct kmem_cache *s,
1722 void *x, bool init)
Roman Bobnievd56791b2013-10-08 15:58:57 -07001723{
1724 kmemleak_free_recursive(x, s->flags);
Christoph Lameter7d550c52010-08-25 14:07:16 -05001725
Vlastimil Babka84048032021-05-21 01:25:06 +02001726 debug_check_no_locks_freed(x, s->object_size);
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001727
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001728 if (!(s->flags & SLAB_DEBUG_OBJECTS))
1729 debug_check_no_obj_freed(x, s->object_size);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001730
Marco Elvercfbe1632020-08-06 23:19:12 -07001731 /* Use KCSAN to help debug racy use-after-free. */
1732 if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
1733 __kcsan_check_access(x, s->object_size,
1734 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
1735
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001736 /*
1737 * As memory initialization might be integrated into KASAN,
1738 * kasan_slab_free and initialization memset's must be
1739 * kept together to avoid discrepancies in behavior.
1740 *
1741 * The initialization memset's clear the object and the metadata,
1742 * but don't touch the SLAB redzone.
1743 */
1744 if (init) {
1745 int rsize;
1746
1747 if (!kasan_has_integrated_init())
1748 memset(kasan_reset_tag(x), 0, s->object_size);
1749 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
1750 memset((char *)kasan_reset_tag(x) + s->inuse, 0,
1751 s->size - s->inuse - rsize);
1752 }
1753 /* KASAN might put x into memory quarantine, delaying its reuse. */
1754 return kasan_slab_free(s, x, init);
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001755}
Christoph Lameter205ab992008-04-14 19:11:40 +03001756
Andrey Konovalovc3895392018-04-10 16:30:31 -07001757static inline bool slab_free_freelist_hook(struct kmem_cache *s,
Miaohe Lin899447f2021-10-18 15:15:55 -07001758 void **head, void **tail,
1759 int *cnt)
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001760{
Alexander Potapenko64713842019-07-11 20:59:19 -07001761
1762 void *object;
1763 void *next = *head;
1764 void *old_tail = *tail ? *tail : *head;
Alexander Potapenko64713842019-07-11 20:59:19 -07001765
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08001766 if (is_kfence_address(next)) {
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001767 slab_free_hook(s, next, false);
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08001768 return true;
1769 }
1770
Laura Abbottaea4df42019-11-15 17:34:50 -08001771 /* Head and tail of the reconstructed freelist */
1772 *head = NULL;
1773 *tail = NULL;
Laura Abbott1b7e8162019-07-31 15:32:40 -04001774
Laura Abbottaea4df42019-11-15 17:34:50 -08001775 do {
1776 object = next;
1777 next = get_freepointer(s, object);
1778
Andrey Konovalovc3895392018-04-10 16:30:31 -07001779 /* If object's reuse doesn't have to be delayed */
Andrey Konovalovd57a9642021-04-29 23:00:09 -07001780 if (!slab_free_hook(s, object, slab_want_init_on_free(s))) {
Andrey Konovalovc3895392018-04-10 16:30:31 -07001781 /* Move object to the new freelist */
1782 set_freepointer(s, object, *head);
1783 *head = object;
1784 if (!*tail)
1785 *tail = object;
Miaohe Lin899447f2021-10-18 15:15:55 -07001786 } else {
1787 /*
1788 * Adjust the reconstructed freelist depth
1789 * accordingly if object's reuse is delayed.
1790 */
1791 --(*cnt);
Andrey Konovalovc3895392018-04-10 16:30:31 -07001792 }
1793 } while (object != old_tail);
1794
1795 if (*head == *tail)
1796 *tail = NULL;
1797
1798 return *head != NULL;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001799}
1800
JaeSang Yooc0f81a92022-04-11 16:25:34 +09001801static void *setup_object(struct kmem_cache *s, void *object)
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001802{
JaeSang Yooc0f81a92022-04-11 16:25:34 +09001803 setup_object_debug(s, object);
Andrey Konovalov4d176712018-12-28 00:30:23 -08001804 object = kasan_init_slab_obj(s, object);
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001805 if (unlikely(s->ctor)) {
1806 kasan_unpoison_object_data(s, object);
1807 s->ctor(object);
1808 kasan_poison_object_data(s, object);
1809 }
Andrey Konovalov4d176712018-12-28 00:30:23 -08001810 return object;
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001811}
1812
Christoph Lameter81819f02007-05-06 14:49:36 -07001813/*
1814 * Slab allocation and freeing
1815 */
Xiongwei Songa485e1d2022-03-10 22:07:01 +08001816static inline struct slab *alloc_slab_page(gfp_t flags, int node,
1817 struct kmem_cache_order_objects oo)
Christoph Lameter65c33762008-04-14 19:11:40 +03001818{
Vlastimil Babka45387b82021-10-26 16:35:02 +02001819 struct folio *folio;
1820 struct slab *slab;
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07001821 unsigned int order = oo_order(oo);
Christoph Lameter65c33762008-04-14 19:11:40 +03001822
Christoph Lameter2154a332010-07-09 14:07:10 -05001823 if (node == NUMA_NO_NODE)
Vlastimil Babka45387b82021-10-26 16:35:02 +02001824 folio = (struct folio *)alloc_pages(flags, order);
Christoph Lameter65c33762008-04-14 19:11:40 +03001825 else
Vlastimil Babka45387b82021-10-26 16:35:02 +02001826 folio = (struct folio *)__alloc_pages_node(node, flags, order);
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001827
Vlastimil Babka45387b82021-10-26 16:35:02 +02001828 if (!folio)
1829 return NULL;
1830
1831 slab = folio_slab(folio);
1832 __folio_set_slab(folio);
1833 if (page_is_pfmemalloc(folio_page(folio, 0)))
1834 slab_set_pfmemalloc(slab);
1835
1836 return slab;
Christoph Lameter65c33762008-04-14 19:11:40 +03001837}
1838
Thomas Garnier210e7a42016-07-26 15:21:59 -07001839#ifdef CONFIG_SLAB_FREELIST_RANDOM
1840/* Pre-initialize the random sequence cache */
1841static int init_cache_random_seq(struct kmem_cache *s)
1842{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07001843 unsigned int count = oo_objects(s->oo);
Thomas Garnier210e7a42016-07-26 15:21:59 -07001844 int err;
Thomas Garnier210e7a42016-07-26 15:21:59 -07001845
Sean Reesa8100072017-02-08 14:30:59 -08001846 /* Bailout if already initialised */
1847 if (s->random_seq)
1848 return 0;
1849
Thomas Garnier210e7a42016-07-26 15:21:59 -07001850 err = cache_random_seq_create(s, count, GFP_KERNEL);
1851 if (err) {
1852 pr_err("SLUB: Unable to initialize free list for %s\n",
1853 s->name);
1854 return err;
1855 }
1856
1857 /* Transform to an offset on the set of pages */
1858 if (s->random_seq) {
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07001859 unsigned int i;
1860
Thomas Garnier210e7a42016-07-26 15:21:59 -07001861 for (i = 0; i < count; i++)
1862 s->random_seq[i] *= s->size;
1863 }
1864 return 0;
1865}
1866
1867/* Initialize each random sequence freelist per cache */
1868static void __init init_freelist_randomization(void)
1869{
1870 struct kmem_cache *s;
1871
1872 mutex_lock(&slab_mutex);
1873
1874 list_for_each_entry(s, &slab_caches, list)
1875 init_cache_random_seq(s);
1876
1877 mutex_unlock(&slab_mutex);
1878}
1879
1880/* Get the next entry on the pre-computed freelist randomized */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001881static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab,
Thomas Garnier210e7a42016-07-26 15:21:59 -07001882 unsigned long *pos, void *start,
1883 unsigned long page_limit,
1884 unsigned long freelist_count)
1885{
1886 unsigned int idx;
1887
1888 /*
1889 * If the target page allocation failed, the number of objects on the
1890 * page might be smaller than the usual size defined by the cache.
1891 */
1892 do {
1893 idx = s->random_seq[*pos];
1894 *pos += 1;
1895 if (*pos >= freelist_count)
1896 *pos = 0;
1897 } while (unlikely(idx >= page_limit));
1898
1899 return (char *)start + idx;
1900}
1901
1902/* Shuffle the single linked freelist based on a random pre-computed sequence */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001903static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
Thomas Garnier210e7a42016-07-26 15:21:59 -07001904{
1905 void *start;
1906 void *cur;
1907 void *next;
1908 unsigned long idx, pos, page_limit, freelist_count;
1909
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001910 if (slab->objects < 2 || !s->random_seq)
Thomas Garnier210e7a42016-07-26 15:21:59 -07001911 return false;
1912
1913 freelist_count = oo_objects(s->oo);
1914 pos = get_random_int() % freelist_count;
1915
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001916 page_limit = slab->objects * s->size;
1917 start = fixup_red_left(s, slab_address(slab));
Thomas Garnier210e7a42016-07-26 15:21:59 -07001918
1919 /* First entry is used as the base of the freelist */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001920 cur = next_freelist_entry(s, slab, &pos, start, page_limit,
Thomas Garnier210e7a42016-07-26 15:21:59 -07001921 freelist_count);
JaeSang Yooc0f81a92022-04-11 16:25:34 +09001922 cur = setup_object(s, cur);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001923 slab->freelist = cur;
Thomas Garnier210e7a42016-07-26 15:21:59 -07001924
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001925 for (idx = 1; idx < slab->objects; idx++) {
1926 next = next_freelist_entry(s, slab, &pos, start, page_limit,
Thomas Garnier210e7a42016-07-26 15:21:59 -07001927 freelist_count);
JaeSang Yooc0f81a92022-04-11 16:25:34 +09001928 next = setup_object(s, next);
Thomas Garnier210e7a42016-07-26 15:21:59 -07001929 set_freepointer(s, cur, next);
1930 cur = next;
1931 }
Thomas Garnier210e7a42016-07-26 15:21:59 -07001932 set_freepointer(s, cur, NULL);
1933
1934 return true;
1935}
1936#else
1937static inline int init_cache_random_seq(struct kmem_cache *s)
1938{
1939 return 0;
1940}
1941static inline void init_freelist_randomization(void) { }
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001942static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
Thomas Garnier210e7a42016-07-26 15:21:59 -07001943{
1944 return false;
1945}
1946#endif /* CONFIG_SLAB_FREELIST_RANDOM */
1947
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001948static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
Christoph Lameter81819f02007-05-06 14:49:36 -07001949{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001950 struct slab *slab;
Christoph Lameter834f3d12008-04-14 19:11:31 +03001951 struct kmem_cache_order_objects oo = s->oo;
Pekka Enbergba522702009-06-24 21:59:51 +03001952 gfp_t alloc_gfp;
Andrey Konovalov4d176712018-12-28 00:30:23 -08001953 void *start, *p, *next;
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07001954 int idx;
Thomas Garnier210e7a42016-07-26 15:21:59 -07001955 bool shuffle;
Christoph Lameter81819f02007-05-06 14:49:36 -07001956
Christoph Lameter7e0528d2011-06-01 12:25:44 -05001957 flags &= gfp_allowed_mask;
1958
Christoph Lameterb7a49f02008-02-14 14:21:32 -08001959 flags |= s->allocflags;
Mel Gormane12ba742007-10-16 01:25:52 -07001960
Pekka Enbergba522702009-06-24 21:59:51 +03001961 /*
1962 * Let the initial higher-order allocation fail under memory pressure
1963 * so we fall-back to the minimum order allocation.
1964 */
1965 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
Mel Gormand0164ad2015-11-06 16:28:21 -08001966 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
Jiyoup Kim27c08f72022-04-10 00:05:37 +09001967 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
Pekka Enbergba522702009-06-24 21:59:51 +03001968
Xiongwei Songa485e1d2022-03-10 22:07:01 +08001969 slab = alloc_slab_page(alloc_gfp, node, oo);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001970 if (unlikely(!slab)) {
Christoph Lameter65c33762008-04-14 19:11:40 +03001971 oo = s->min;
Joonsoo Kim80c3a992014-03-12 17:26:20 +09001972 alloc_gfp = flags;
Christoph Lameter65c33762008-04-14 19:11:40 +03001973 /*
1974 * Allocation may have failed due to fragmentation.
1975 * Try a lower order alloc if possible
1976 */
Xiongwei Songa485e1d2022-03-10 22:07:01 +08001977 slab = alloc_slab_page(alloc_gfp, node, oo);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001978 if (unlikely(!slab))
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001979 goto out;
1980 stat(s, ORDER_FALLBACK);
Christoph Lameter65c33762008-04-14 19:11:40 +03001981 }
Vegard Nossum5a896d92008-04-04 00:54:48 +02001982
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001983 slab->objects = oo_objects(oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07001984
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001985 account_slab(slab, oo_order(oo), s, flags);
Roman Gushchin1f3147b2020-12-29 15:15:07 -08001986
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001987 slab->slab_cache = s;
Christoph Lameter81819f02007-05-06 14:49:36 -07001988
Matthew Wilcox (Oracle)6e48a962021-10-04 14:46:46 +01001989 kasan_poison_slab(slab);
Andrey Konovalova7101222019-02-20 22:19:23 -08001990
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001991 start = slab_address(slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07001992
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001993 setup_slab_debug(s, slab, start);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001994
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01001995 shuffle = shuffle_freelist(s, slab);
Thomas Garnier210e7a42016-07-26 15:21:59 -07001996
1997 if (!shuffle) {
Andrey Konovalov4d176712018-12-28 00:30:23 -08001998 start = fixup_red_left(s, start);
JaeSang Yooc0f81a92022-04-11 16:25:34 +09001999 start = setup_object(s, start);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002000 slab->freelist = start;
2001 for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
Andrey Konovalov18e50662019-02-20 22:19:28 -08002002 next = p + s->size;
JaeSang Yooc0f81a92022-04-11 16:25:34 +09002003 next = setup_object(s, next);
Andrey Konovalov18e50662019-02-20 22:19:28 -08002004 set_freepointer(s, p, next);
2005 p = next;
2006 }
2007 set_freepointer(s, p, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07002008 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002009
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002010 slab->inuse = slab->objects;
2011 slab->frozen = 1;
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07002012
Christoph Lameter81819f02007-05-06 14:49:36 -07002013out:
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002014 if (!slab)
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07002015 return NULL;
2016
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002017 inc_slabs_node(s, slab_nid(slab), slab->objects);
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07002018
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002019 return slab;
Christoph Lameter81819f02007-05-06 14:49:36 -07002020}
2021
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002022static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07002023{
Long Li44405092020-08-06 23:18:28 -07002024 if (unlikely(flags & GFP_SLAB_BUG_MASK))
2025 flags = kmalloc_fix_flags(flags);
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07002026
Vlastimil Babka53a0de02021-05-11 13:01:34 +02002027 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
2028
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07002029 return allocate_slab(s,
2030 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
2031}
2032
Vlastimil Babka4020b4a2021-10-29 12:18:24 +02002033static void __free_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter81819f02007-05-06 14:49:36 -07002034{
Vlastimil Babka4020b4a2021-10-29 12:18:24 +02002035 struct folio *folio = slab_folio(slab);
2036 int order = folio_order(folio);
Christoph Lameter834f3d12008-04-14 19:11:31 +03002037 int pages = 1 << order;
Christoph Lameter81819f02007-05-06 14:49:36 -07002038
Vlastimil Babka8fc8d662020-08-06 23:18:58 -07002039 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002040 void *p;
2041
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002042 slab_pad_check(s, slab);
Vlastimil Babka4020b4a2021-10-29 12:18:24 +02002043 for_each_object(p, s, slab_address(slab), slab->objects)
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002044 check_object(s, slab, p, SLUB_RED_INACTIVE);
Christoph Lameter81819f02007-05-06 14:49:36 -07002045 }
2046
Vlastimil Babka4020b4a2021-10-29 12:18:24 +02002047 __slab_clear_pfmemalloc(slab);
2048 __folio_clear_slab(folio);
2049 folio->mapping = NULL;
Nick Piggin1eb5ac62009-05-05 19:13:44 +10002050 if (current->reclaim_state)
2051 current->reclaim_state->reclaimed_slab += pages;
Vlastimil Babka4020b4a2021-10-29 12:18:24 +02002052 unaccount_slab(slab, order, s);
2053 __free_pages(folio_page(folio, 0), order);
Christoph Lameter81819f02007-05-06 14:49:36 -07002054}
2055
2056static void rcu_free_slab(struct rcu_head *h)
2057{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002058 struct slab *slab = container_of(h, struct slab, rcu_head);
Lai Jiangshanda9a6382011-03-10 15:22:00 +08002059
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002060 __free_slab(slab->slab_cache, slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07002061}
2062
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002063static void free_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter81819f02007-05-06 14:49:36 -07002064{
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -08002065 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002066 call_rcu(&slab->rcu_head, rcu_free_slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07002067 } else
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002068 __free_slab(s, slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07002069}
2070
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002071static void discard_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter81819f02007-05-06 14:49:36 -07002072{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002073 dec_slabs_node(s, slab_nid(slab), slab->objects);
2074 free_slab(s, slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07002075}
2076
2077/*
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05002078 * Management of partially allocated slabs.
Christoph Lameter81819f02007-05-06 14:49:36 -07002079 */
Steven Rostedt1e4dd942014-02-10 14:25:46 -08002080static inline void
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002081__add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
Christoph Lameter81819f02007-05-06 14:49:36 -07002082{
Christoph Lametere95eed52007-05-06 14:49:44 -07002083 n->nr_partial++;
Shaohua Li136333d2011-08-24 08:57:52 +08002084 if (tail == DEACTIVATE_TO_TAIL)
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002085 list_add_tail(&slab->slab_list, &n->partial);
Christoph Lameter7c2e1322008-01-07 23:20:27 -08002086 else
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002087 list_add(&slab->slab_list, &n->partial);
Christoph Lameter81819f02007-05-06 14:49:36 -07002088}
2089
Steven Rostedt1e4dd942014-02-10 14:25:46 -08002090static inline void add_partial(struct kmem_cache_node *n,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002091 struct slab *slab, int tail)
Steven Rostedt1e4dd942014-02-10 14:25:46 -08002092{
2093 lockdep_assert_held(&n->list_lock);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002094 __add_partial(n, slab, tail);
Steven Rostedt1e4dd942014-02-10 14:25:46 -08002095}
2096
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05002097static inline void remove_partial(struct kmem_cache_node *n,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002098 struct slab *slab)
Christoph Lameter62e346a2010-09-28 08:10:28 -05002099{
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002100 lockdep_assert_held(&n->list_lock);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002101 list_del(&slab->slab_list);
Dmitry Safonov52b4b952016-02-17 13:11:37 -08002102 n->nr_partial--;
Christoph Lameter62e346a2010-09-28 08:10:28 -05002103}
2104
Christoph Lameter81819f02007-05-06 14:49:36 -07002105/*
Christoph Lameter7ced3712012-05-09 10:09:53 -05002106 * Remove slab from the partial list, freeze it and
2107 * return the pointer to the freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07002108 *
Christoph Lameter497b66f2011-08-09 16:12:26 -05002109 * Returns a list of objects or NULL if it fails.
Christoph Lameter81819f02007-05-06 14:49:36 -07002110 */
Christoph Lameter497b66f2011-08-09 16:12:26 -05002111static inline void *acquire_slab(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002112 struct kmem_cache_node *n, struct slab *slab,
Vlastimil Babkab47291e2021-11-05 13:35:17 -07002113 int mode)
Christoph Lameter81819f02007-05-06 14:49:36 -07002114{
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002115 void *freelist;
2116 unsigned long counters;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002117 struct slab new;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002118
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002119 lockdep_assert_held(&n->list_lock);
2120
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002121 /*
2122 * Zap the freelist and set the frozen bit.
2123 * The old freelist is the list of objects for the
2124 * per cpu allocation list.
2125 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002126 freelist = slab->freelist;
2127 counters = slab->counters;
Christoph Lameter7ced3712012-05-09 10:09:53 -05002128 new.counters = counters;
Pekka Enberg23910c52012-06-04 10:14:58 +03002129 if (mode) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002130 new.inuse = slab->objects;
Pekka Enberg23910c52012-06-04 10:14:58 +03002131 new.freelist = NULL;
2132 } else {
2133 new.freelist = freelist;
2134 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002135
Dave Hansena0132ac2014-01-29 14:05:50 -08002136 VM_BUG_ON(new.frozen);
Christoph Lameter7ced3712012-05-09 10:09:53 -05002137 new.frozen = 1;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002138
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002139 if (!__cmpxchg_double_slab(s, slab,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002140 freelist, counters,
Joonsoo Kim02d76332012-05-17 00:13:02 +09002141 new.freelist, new.counters,
Christoph Lameter7ced3712012-05-09 10:09:53 -05002142 "acquire_slab"))
Christoph Lameter7ced3712012-05-09 10:09:53 -05002143 return NULL;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002144
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002145 remove_partial(n, slab);
Christoph Lameter7ced3712012-05-09 10:09:53 -05002146 WARN_ON(!freelist);
Christoph Lameter49e22582011-08-09 16:12:27 -05002147 return freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07002148}
2149
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002150#ifdef CONFIG_SLUB_CPU_PARTIAL
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002151static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002152#else
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002153static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002154 int drain) { }
2155#endif
Matthew Wilcox (Oracle)01b34d12021-10-04 14:46:40 +01002156static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
Christoph Lameter49e22582011-08-09 16:12:27 -05002157
Christoph Lameter81819f02007-05-06 14:49:36 -07002158/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002159 * Try to allocate a partial slab from a specific node.
Christoph Lameter81819f02007-05-06 14:49:36 -07002160 */
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002161static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002162 struct slab **ret_slab, gfp_t gfpflags)
Christoph Lameter81819f02007-05-06 14:49:36 -07002163{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002164 struct slab *slab, *slab2;
Christoph Lameter49e22582011-08-09 16:12:27 -05002165 void *object = NULL;
Vlastimil Babka4b1f4492021-05-11 17:45:26 +02002166 unsigned long flags;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002167 unsigned int partial_slabs = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07002168
2169 /*
2170 * Racy check. If we mistakenly see no partial slabs then we
2171 * just allocate an empty slab. If we mistakenly try to get a
Chen Tao70b6d252020-10-15 20:10:01 -07002172 * partial slab and there is none available then get_partial()
Christoph Lameter672bba32007-05-09 02:32:39 -07002173 * will return NULL.
Christoph Lameter81819f02007-05-06 14:49:36 -07002174 */
2175 if (!n || !n->nr_partial)
2176 return NULL;
2177
Vlastimil Babka4b1f4492021-05-11 17:45:26 +02002178 spin_lock_irqsave(&n->list_lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002179 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002180 void *t;
Christoph Lameter49e22582011-08-09 16:12:27 -05002181
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002182 if (!pfmemalloc_match(slab, gfpflags))
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07002183 continue;
2184
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002185 t = acquire_slab(s, n, slab, object == NULL);
Christoph Lameter49e22582011-08-09 16:12:27 -05002186 if (!t)
Linus Torvalds9b1ea292021-03-10 10:18:04 -08002187 break;
Christoph Lameter49e22582011-08-09 16:12:27 -05002188
Alex,Shi12d79632011-09-07 10:26:36 +08002189 if (!object) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002190 *ret_slab = slab;
Christoph Lameter49e22582011-08-09 16:12:27 -05002191 stat(s, ALLOC_FROM_PARTIAL);
Christoph Lameter49e22582011-08-09 16:12:27 -05002192 object = t;
Christoph Lameter49e22582011-08-09 16:12:27 -05002193 } else {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002194 put_cpu_partial(s, slab, 0);
Alex Shi8028dce2012-02-03 23:34:56 +08002195 stat(s, CPU_PARTIAL_NODE);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002196 partial_slabs++;
Christoph Lameter49e22582011-08-09 16:12:27 -05002197 }
Vlastimil Babkab47291e2021-11-05 13:35:17 -07002198#ifdef CONFIG_SLUB_CPU_PARTIAL
Joonsoo Kim345c9052013-06-19 14:05:52 +09002199 if (!kmem_cache_has_cpu_partial(s)
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002200 || partial_slabs > s->cpu_partial_slabs / 2)
Christoph Lameter49e22582011-08-09 16:12:27 -05002201 break;
Vlastimil Babkab47291e2021-11-05 13:35:17 -07002202#else
2203 break;
2204#endif
Christoph Lameter49e22582011-08-09 16:12:27 -05002205
Christoph Lameter497b66f2011-08-09 16:12:26 -05002206 }
Vlastimil Babka4b1f4492021-05-11 17:45:26 +02002207 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002208 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07002209}
2210
2211/*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002212 * Get a slab from somewhere. Search in increasing NUMA distances.
Christoph Lameter81819f02007-05-06 14:49:36 -07002213 */
Joonsoo Kimde3ec032012-01-27 00:12:23 -08002214static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002215 struct slab **ret_slab)
Christoph Lameter81819f02007-05-06 14:49:36 -07002216{
2217#ifdef CONFIG_NUMA
2218 struct zonelist *zonelist;
Mel Gormandd1a2392008-04-28 02:12:17 -07002219 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07002220 struct zone *zone;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002221 enum zone_type highest_zoneidx = gfp_zone(flags);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002222 void *object;
Mel Gormancc9a6c82012-03-21 16:34:11 -07002223 unsigned int cpuset_mems_cookie;
Christoph Lameter81819f02007-05-06 14:49:36 -07002224
2225 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002226 * The defrag ratio allows a configuration of the tradeoffs between
2227 * inter node defragmentation and node local allocations. A lower
2228 * defrag_ratio increases the tendency to do local allocations
2229 * instead of attempting to obtain partial slabs from other nodes.
Christoph Lameter81819f02007-05-06 14:49:36 -07002230 *
Christoph Lameter672bba32007-05-09 02:32:39 -07002231 * If the defrag_ratio is set to 0 then kmalloc() always
2232 * returns node local objects. If the ratio is higher then kmalloc()
2233 * may return off node objects because partial slabs are obtained
2234 * from other nodes and filled up.
Christoph Lameter81819f02007-05-06 14:49:36 -07002235 *
Li Peng43efd3e2016-05-19 17:10:43 -07002236 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
2237 * (which makes defrag_ratio = 1000) then every (well almost)
2238 * allocation will first attempt to defrag slab caches on other nodes.
2239 * This means scanning over all nodes to look for partial slabs which
2240 * may be expensive if we do it every time we are trying to find a slab
Christoph Lameter672bba32007-05-09 02:32:39 -07002241 * with available objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07002242 */
Christoph Lameter98246012008-01-07 23:20:26 -08002243 if (!s->remote_node_defrag_ratio ||
2244 get_cycles() % 1024 > s->remote_node_defrag_ratio)
Christoph Lameter81819f02007-05-06 14:49:36 -07002245 return NULL;
2246
Mel Gormancc9a6c82012-03-21 16:34:11 -07002247 do {
Mel Gormand26914d2014-04-03 14:47:24 -07002248 cpuset_mems_cookie = read_mems_allowed_begin();
David Rientjes2a389612014-04-07 15:37:29 -07002249 zonelist = node_zonelist(mempolicy_slab_node(), flags);
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002250 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
Mel Gormancc9a6c82012-03-21 16:34:11 -07002251 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07002252
Mel Gormancc9a6c82012-03-21 16:34:11 -07002253 n = get_node(s, zone_to_nid(zone));
Christoph Lameter81819f02007-05-06 14:49:36 -07002254
Vladimir Davydovdee2f8a2014-12-12 16:58:28 -08002255 if (n && cpuset_zone_allowed(zone, flags) &&
Mel Gormancc9a6c82012-03-21 16:34:11 -07002256 n->nr_partial > s->min_partial) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002257 object = get_partial_node(s, n, ret_slab, flags);
Mel Gormancc9a6c82012-03-21 16:34:11 -07002258 if (object) {
2259 /*
Mel Gormand26914d2014-04-03 14:47:24 -07002260 * Don't check read_mems_allowed_retry()
2261 * here - if mems_allowed was updated in
2262 * parallel, that was a harmless race
2263 * between allocation and the cpuset
2264 * update
Mel Gormancc9a6c82012-03-21 16:34:11 -07002265 */
Mel Gormancc9a6c82012-03-21 16:34:11 -07002266 return object;
2267 }
Miao Xiec0ff7452010-05-24 14:32:08 -07002268 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002269 }
Mel Gormand26914d2014-04-03 14:47:24 -07002270 } while (read_mems_allowed_retry(cpuset_mems_cookie));
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07002271#endif /* CONFIG_NUMA */
Christoph Lameter81819f02007-05-06 14:49:36 -07002272 return NULL;
2273}
2274
2275/*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002276 * Get a partial slab, lock it and return it.
Christoph Lameter81819f02007-05-06 14:49:36 -07002277 */
Christoph Lameter497b66f2011-08-09 16:12:26 -05002278static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002279 struct slab **ret_slab)
Christoph Lameter81819f02007-05-06 14:49:36 -07002280{
Christoph Lameter497b66f2011-08-09 16:12:26 -05002281 void *object;
Joonsoo Kima561ce02014-10-09 15:26:15 -07002282 int searchnode = node;
2283
2284 if (node == NUMA_NO_NODE)
2285 searchnode = numa_mem_id();
Christoph Lameter81819f02007-05-06 14:49:36 -07002286
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002287 object = get_partial_node(s, get_node(s, searchnode), ret_slab, flags);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002288 if (object || node != NUMA_NO_NODE)
2289 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07002290
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002291 return get_any_partial(s, flags, ret_slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07002292}
2293
Thomas Gleixner923717c2019-10-15 21:18:12 +02002294#ifdef CONFIG_PREEMPTION
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002295/*
Ethon Paul0d645ed2020-06-04 16:49:34 -07002296 * Calculate the next globally unique transaction for disambiguation
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002297 * during cmpxchg. The transactions start with the cpu number and are then
2298 * incremented by CONFIG_NR_CPUS.
2299 */
2300#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
2301#else
2302/*
2303 * No preemption supported therefore also no need to check for
2304 * different cpus.
2305 */
2306#define TID_STEP 1
2307#endif
2308
2309static inline unsigned long next_tid(unsigned long tid)
2310{
2311 return tid + TID_STEP;
2312}
2313
Qian Cai9d5f0be2019-09-23 15:33:52 -07002314#ifdef SLUB_DEBUG_CMPXCHG
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002315static inline unsigned int tid_to_cpu(unsigned long tid)
2316{
2317 return tid % TID_STEP;
2318}
2319
2320static inline unsigned long tid_to_event(unsigned long tid)
2321{
2322 return tid / TID_STEP;
2323}
Qian Cai9d5f0be2019-09-23 15:33:52 -07002324#endif
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002325
2326static inline unsigned int init_tid(int cpu)
2327{
2328 return cpu;
2329}
2330
2331static inline void note_cmpxchg_failure(const char *n,
2332 const struct kmem_cache *s, unsigned long tid)
2333{
2334#ifdef SLUB_DEBUG_CMPXCHG
2335 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
2336
Fabian Frederickf9f58282014-06-04 16:06:34 -07002337 pr_info("%s %s: cmpxchg redo ", n, s->name);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002338
Thomas Gleixner923717c2019-10-15 21:18:12 +02002339#ifdef CONFIG_PREEMPTION
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002340 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
Fabian Frederickf9f58282014-06-04 16:06:34 -07002341 pr_warn("due to cpu change %d -> %d\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002342 tid_to_cpu(tid), tid_to_cpu(actual_tid));
2343 else
2344#endif
2345 if (tid_to_event(tid) != tid_to_event(actual_tid))
Fabian Frederickf9f58282014-06-04 16:06:34 -07002346 pr_warn("due to cpu running other code. Event %ld->%ld\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002347 tid_to_event(tid), tid_to_event(actual_tid));
2348 else
Fabian Frederickf9f58282014-06-04 16:06:34 -07002349 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002350 actual_tid, tid, next_tid(tid));
2351#endif
Christoph Lameter4fdccdf2011-03-22 13:35:00 -05002352 stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002353}
2354
Fengguang Wu788e1aa2012-09-28 16:34:05 +08002355static void init_kmem_cache_cpus(struct kmem_cache *s)
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002356{
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002357 int cpu;
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002358 struct kmem_cache_cpu *c;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002359
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002360 for_each_possible_cpu(cpu) {
2361 c = per_cpu_ptr(s->cpu_slab, cpu);
2362 local_lock_init(&c->lock);
2363 c->tid = init_tid(cpu);
2364 }
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002365}
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002366
2367/*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002368 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
Vlastimil Babkaa019d202021-05-12 13:53:34 +02002369 * unfreezes the slabs and puts it on the proper list.
2370 * Assumes the slab has been already safely taken away from kmem_cache_cpu
2371 * by the caller.
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002372 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002373static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
Vlastimil Babkaa019d202021-05-12 13:53:34 +02002374 void *freelist)
Christoph Lameter81819f02007-05-06 14:49:36 -07002375{
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002376 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE, M_FULL_NOLIST };
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002377 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002378 int free_delta = 0;
2379 enum slab_modes mode = M_NONE;
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002380 void *nextfree, *freelist_iter, *freelist_tail;
Shaohua Li136333d2011-08-24 08:57:52 +08002381 int tail = DEACTIVATE_TO_HEAD;
Vlastimil Babka3406e912021-05-12 13:59:58 +02002382 unsigned long flags = 0;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002383 struct slab new;
2384 struct slab old;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08002385
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002386 if (slab->freelist) {
Christoph Lameter84e554e62009-12-18 16:26:23 -06002387 stat(s, DEACTIVATE_REMOTE_FREES);
Shaohua Li136333d2011-08-24 08:57:52 +08002388 tail = DEACTIVATE_TO_TAIL;
Christoph Lameter894b8782007-05-10 03:15:16 -07002389 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002390
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002391 /*
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002392 * Stage one: Count the objects on cpu's freelist as free_delta and
2393 * remember the last object in freelist_tail for later splicing.
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002394 */
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002395 freelist_tail = NULL;
2396 freelist_iter = freelist;
2397 while (freelist_iter) {
2398 nextfree = get_freepointer(s, freelist_iter);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002399
Dongli Zhang52f23472020-06-01 21:45:47 -07002400 /*
2401 * If 'nextfree' is invalid, it is possible that the object at
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002402 * 'freelist_iter' is already corrupted. So isolate all objects
2403 * starting at 'freelist_iter' by skipping them.
Dongli Zhang52f23472020-06-01 21:45:47 -07002404 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002405 if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
Dongli Zhang52f23472020-06-01 21:45:47 -07002406 break;
2407
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002408 freelist_tail = freelist_iter;
2409 free_delta++;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002410
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002411 freelist_iter = nextfree;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002412 }
2413
2414 /*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002415 * Stage two: Unfreeze the slab while splicing the per-cpu
2416 * freelist to the head of slab's freelist.
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002417 *
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002418 * Ensure that the slab is unfrozen while the list presence
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002419 * reflects the actual number of objects during unfreeze.
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002420 *
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002421 * We first perform cmpxchg holding lock and insert to list
2422 * when it succeed. If there is mismatch then the slab is not
2423 * unfrozen and number of objects in the slab may have changed.
2424 * Then release lock and retry cmpxchg again.
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002425 */
2426redo:
2427
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002428 old.freelist = READ_ONCE(slab->freelist);
2429 old.counters = READ_ONCE(slab->counters);
Dave Hansena0132ac2014-01-29 14:05:50 -08002430 VM_BUG_ON(!old.frozen);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002431
2432 /* Determine target state of the slab */
2433 new.counters = old.counters;
Vlastimil Babkad930ff02021-02-24 12:01:19 -08002434 if (freelist_tail) {
2435 new.inuse -= free_delta;
2436 set_freepointer(s, freelist_tail, old.freelist);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002437 new.freelist = freelist;
2438 } else
2439 new.freelist = old.freelist;
2440
2441 new.frozen = 0;
2442
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002443 if (!new.inuse && n->nr_partial >= s->min_partial) {
2444 mode = M_FREE;
2445 } else if (new.freelist) {
2446 mode = M_PARTIAL;
2447 /*
2448 * Taking the spinlock removes the possibility that
2449 * acquire_slab() will see a slab that is frozen
2450 */
2451 spin_lock_irqsave(&n->list_lock, flags);
2452 } else if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) {
2453 mode = M_FULL;
2454 /*
2455 * This also ensures that the scanning of full
2456 * slabs from diagnostic functions will not see
2457 * any frozen slabs.
2458 */
2459 spin_lock_irqsave(&n->list_lock, flags);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002460 } else {
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002461 mode = M_FULL_NOLIST;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002462 }
2463
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002464
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002465 if (!cmpxchg_double_slab(s, slab,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002466 old.freelist, old.counters,
2467 new.freelist, new.counters,
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002468 "unfreezing slab")) {
2469 if (mode == M_PARTIAL || mode == M_FULL)
2470 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002471 goto redo;
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002472 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002473
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002474
2475 if (mode == M_PARTIAL) {
2476 add_partial(n, slab, tail);
Vlastimil Babka3406e912021-05-12 13:59:58 +02002477 spin_unlock_irqrestore(&n->list_lock, flags);
Wei Yang88349a22018-12-28 00:33:13 -08002478 stat(s, tail);
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002479 } else if (mode == M_FREE) {
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002480 stat(s, DEACTIVATE_EMPTY);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002481 discard_slab(s, slab);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002482 stat(s, FREE_SLAB);
Hyeonggon Yoo6d3a16d2022-03-07 07:40:56 +00002483 } else if (mode == M_FULL) {
2484 add_full(s, n, slab);
2485 spin_unlock_irqrestore(&n->list_lock, flags);
2486 stat(s, DEACTIVATE_FULL);
2487 } else if (mode == M_FULL_NOLIST) {
2488 stat(s, DEACTIVATE_FULL);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002489 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002490}
2491
Joonsoo Kim345c9052013-06-19 14:05:52 +09002492#ifdef CONFIG_SLUB_CPU_PARTIAL
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002493static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab)
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002494{
Joonsoo Kim43d77862012-06-09 02:23:16 +09002495 struct kmem_cache_node *n = NULL, *n2 = NULL;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002496 struct slab *slab, *slab_to_discard = NULL;
Vlastimil Babka7cf9f3b2021-05-21 01:16:54 +02002497 unsigned long flags = 0;
Christoph Lameter49e22582011-08-09 16:12:27 -05002498
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002499 while (partial_slab) {
2500 struct slab new;
2501 struct slab old;
Christoph Lameter49e22582011-08-09 16:12:27 -05002502
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002503 slab = partial_slab;
2504 partial_slab = slab->next;
Joonsoo Kim43d77862012-06-09 02:23:16 +09002505
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002506 n2 = get_node(s, slab_nid(slab));
Joonsoo Kim43d77862012-06-09 02:23:16 +09002507 if (n != n2) {
2508 if (n)
Vlastimil Babka7cf9f3b2021-05-21 01:16:54 +02002509 spin_unlock_irqrestore(&n->list_lock, flags);
Joonsoo Kim43d77862012-06-09 02:23:16 +09002510
2511 n = n2;
Vlastimil Babka7cf9f3b2021-05-21 01:16:54 +02002512 spin_lock_irqsave(&n->list_lock, flags);
Joonsoo Kim43d77862012-06-09 02:23:16 +09002513 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002514
2515 do {
2516
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002517 old.freelist = slab->freelist;
2518 old.counters = slab->counters;
Dave Hansena0132ac2014-01-29 14:05:50 -08002519 VM_BUG_ON(!old.frozen);
Christoph Lameter49e22582011-08-09 16:12:27 -05002520
2521 new.counters = old.counters;
2522 new.freelist = old.freelist;
2523
2524 new.frozen = 0;
2525
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002526 } while (!__cmpxchg_double_slab(s, slab,
Christoph Lameter49e22582011-08-09 16:12:27 -05002527 old.freelist, old.counters,
2528 new.freelist, new.counters,
2529 "unfreezing slab"));
2530
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07002531 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002532 slab->next = slab_to_discard;
2533 slab_to_discard = slab;
Joonsoo Kim43d77862012-06-09 02:23:16 +09002534 } else {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002535 add_partial(n, slab, DEACTIVATE_TO_TAIL);
Joonsoo Kim43d77862012-06-09 02:23:16 +09002536 stat(s, FREE_ADD_PARTIAL);
Christoph Lameter49e22582011-08-09 16:12:27 -05002537 }
2538 }
2539
2540 if (n)
Vlastimil Babka7cf9f3b2021-05-21 01:16:54 +02002541 spin_unlock_irqrestore(&n->list_lock, flags);
Vlastimil Babka8de06a62021-05-20 14:01:57 +02002542
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002543 while (slab_to_discard) {
2544 slab = slab_to_discard;
2545 slab_to_discard = slab_to_discard->next;
Shaohua Li9ada1932011-11-14 13:34:13 +08002546
2547 stat(s, DEACTIVATE_EMPTY);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002548 discard_slab(s, slab);
Shaohua Li9ada1932011-11-14 13:34:13 +08002549 stat(s, FREE_SLAB);
2550 }
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002551}
2552
2553/*
2554 * Unfreeze all the cpu partial slabs.
2555 */
2556static void unfreeze_partials(struct kmem_cache *s)
2557{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002558 struct slab *partial_slab;
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002559 unsigned long flags;
2560
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002561 local_lock_irqsave(&s->cpu_slab->lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002562 partial_slab = this_cpu_read(s->cpu_slab->partial);
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002563 this_cpu_write(s->cpu_slab->partial, NULL);
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002564 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002565
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002566 if (partial_slab)
2567 __unfreeze_partials(s, partial_slab);
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002568}
2569
2570static void unfreeze_partials_cpu(struct kmem_cache *s,
2571 struct kmem_cache_cpu *c)
2572{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002573 struct slab *partial_slab;
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002574
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002575 partial_slab = slub_percpu_partial(c);
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002576 c->partial = NULL;
2577
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002578 if (partial_slab)
2579 __unfreeze_partials(s, partial_slab);
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002580}
2581
Christoph Lameter49e22582011-08-09 16:12:27 -05002582/*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002583 * Put a slab that was just frozen (in __slab_free|get_partial_node) into a
2584 * partial slab slot if available.
Christoph Lameter49e22582011-08-09 16:12:27 -05002585 *
2586 * If we did not find a slot then simply move all the partials to the
2587 * per node partial list.
2588 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002589static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
Christoph Lameter49e22582011-08-09 16:12:27 -05002590{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002591 struct slab *oldslab;
2592 struct slab *slab_to_unfreeze = NULL;
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002593 unsigned long flags;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002594 int slabs = 0;
Christoph Lameter49e22582011-08-09 16:12:27 -05002595
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002596 local_lock_irqsave(&s->cpu_slab->lock, flags);
Christoph Lameter49e22582011-08-09 16:12:27 -05002597
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002598 oldslab = this_cpu_read(s->cpu_slab->partial);
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002599
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002600 if (oldslab) {
2601 if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002602 /*
2603 * Partial array is full. Move the existing set to the
2604 * per node partial list. Postpone the actual unfreezing
2605 * outside of the critical section.
2606 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002607 slab_to_unfreeze = oldslab;
2608 oldslab = NULL;
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002609 } else {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002610 slabs = oldslab->slabs;
Christoph Lameter49e22582011-08-09 16:12:27 -05002611 }
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002612 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002613
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002614 slabs++;
Christoph Lameter49e22582011-08-09 16:12:27 -05002615
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002616 slab->slabs = slabs;
2617 slab->next = oldslab;
Christoph Lameter49e22582011-08-09 16:12:27 -05002618
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002619 this_cpu_write(s->cpu_slab->partial, slab);
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002620
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002621 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002622
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002623 if (slab_to_unfreeze) {
2624 __unfreeze_partials(s, slab_to_unfreeze);
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002625 stat(s, CPU_PARTIAL_DRAIN);
2626 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002627}
2628
Vlastimil Babkae0a043a2021-07-28 12:26:27 +02002629#else /* CONFIG_SLUB_CPU_PARTIAL */
2630
2631static inline void unfreeze_partials(struct kmem_cache *s) { }
2632static inline void unfreeze_partials_cpu(struct kmem_cache *s,
2633 struct kmem_cache_cpu *c) { }
2634
2635#endif /* CONFIG_SLUB_CPU_PARTIAL */
2636
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002637static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002638{
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002639 unsigned long flags;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002640 struct slab *slab;
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002641 void *freelist;
2642
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002643 local_lock_irqsave(&s->cpu_slab->lock, flags);
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002644
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002645 slab = c->slab;
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002646 freelist = c->freelist;
Christoph Lameterc17dda42012-05-09 10:09:57 -05002647
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002648 c->slab = NULL;
Vlastimil Babkaa019d202021-05-12 13:53:34 +02002649 c->freelist = NULL;
Christoph Lameterc17dda42012-05-09 10:09:57 -05002650 c->tid = next_tid(c->tid);
Vlastimil Babkaa019d202021-05-12 13:53:34 +02002651
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002652 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkaa019d202021-05-12 13:53:34 +02002653
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002654 if (slab) {
2655 deactivate_slab(s, slab, freelist);
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002656 stat(s, CPUSLAB_FLUSH);
2657 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002658}
2659
Christoph Lameter0c710012007-07-17 04:03:24 -07002660static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
Christoph Lameter81819f02007-05-06 14:49:36 -07002661{
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06002662 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
Vlastimil Babka08beb542021-06-03 19:17:42 +02002663 void *freelist = c->freelist;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002664 struct slab *slab = c->slab;
Christoph Lameter81819f02007-05-06 14:49:36 -07002665
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002666 c->slab = NULL;
Vlastimil Babka08beb542021-06-03 19:17:42 +02002667 c->freelist = NULL;
2668 c->tid = next_tid(c->tid);
2669
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002670 if (slab) {
2671 deactivate_slab(s, slab, freelist);
Vlastimil Babka08beb542021-06-03 19:17:42 +02002672 stat(s, CPUSLAB_FLUSH);
2673 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002674
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002675 unfreeze_partials_cpu(s, c);
Christoph Lameter81819f02007-05-06 14:49:36 -07002676}
2677
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002678struct slub_flush_work {
2679 struct work_struct work;
2680 struct kmem_cache *s;
2681 bool skip;
2682};
2683
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002684/*
2685 * Flush cpu slab.
2686 *
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002687 * Called from CPU work handler with migration disabled.
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002688 */
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002689static void flush_cpu_slab(struct work_struct *w)
Christoph Lameter81819f02007-05-06 14:49:36 -07002690{
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002691 struct kmem_cache *s;
2692 struct kmem_cache_cpu *c;
2693 struct slub_flush_work *sfw;
2694
2695 sfw = container_of(w, struct slub_flush_work, work);
2696
2697 s = sfw->s;
2698 c = this_cpu_ptr(s->cpu_slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07002699
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002700 if (c->slab)
Vlastimil Babkafc1455f2021-05-20 16:39:51 +02002701 flush_slab(s, c);
2702
2703 unfreeze_partials(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07002704}
2705
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002706static bool has_cpu_slab(int cpu, struct kmem_cache *s)
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002707{
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002708 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2709
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002710 return c->slab || slub_percpu_partial(c);
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002711}
2712
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002713static DEFINE_MUTEX(flush_lock);
2714static DEFINE_PER_CPU(struct slub_flush_work, slub_flush);
2715
2716static void flush_all_cpus_locked(struct kmem_cache *s)
2717{
2718 struct slub_flush_work *sfw;
2719 unsigned int cpu;
2720
2721 lockdep_assert_cpus_held();
2722 mutex_lock(&flush_lock);
2723
2724 for_each_online_cpu(cpu) {
2725 sfw = &per_cpu(slub_flush, cpu);
2726 if (!has_cpu_slab(cpu, s)) {
2727 sfw->skip = true;
2728 continue;
2729 }
2730 INIT_WORK(&sfw->work, flush_cpu_slab);
2731 sfw->skip = false;
2732 sfw->s = s;
2733 schedule_work_on(cpu, &sfw->work);
2734 }
2735
2736 for_each_online_cpu(cpu) {
2737 sfw = &per_cpu(slub_flush, cpu);
2738 if (sfw->skip)
2739 continue;
2740 flush_work(&sfw->work);
2741 }
2742
2743 mutex_unlock(&flush_lock);
2744}
2745
Christoph Lameter81819f02007-05-06 14:49:36 -07002746static void flush_all(struct kmem_cache *s)
2747{
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01002748 cpus_read_lock();
2749 flush_all_cpus_locked(s);
2750 cpus_read_unlock();
Christoph Lameter81819f02007-05-06 14:49:36 -07002751}
2752
2753/*
Sebastian Andrzej Siewiora96a87b2016-08-18 14:57:19 +02002754 * Use the cpu notifier to insure that the cpu slabs are flushed when
2755 * necessary.
2756 */
2757static int slub_cpu_dead(unsigned int cpu)
2758{
2759 struct kmem_cache *s;
Sebastian Andrzej Siewiora96a87b2016-08-18 14:57:19 +02002760
2761 mutex_lock(&slab_mutex);
Vlastimil Babka0e7ac732021-05-21 01:48:56 +02002762 list_for_each_entry(s, &slab_caches, list)
Sebastian Andrzej Siewiora96a87b2016-08-18 14:57:19 +02002763 __flush_cpu_slab(s, cpu);
Sebastian Andrzej Siewiora96a87b2016-08-18 14:57:19 +02002764 mutex_unlock(&slab_mutex);
2765 return 0;
2766}
2767
2768/*
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002769 * Check if the objects in a per cpu structure fit numa
2770 * locality expectations.
2771 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002772static inline int node_match(struct slab *slab, int node)
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002773{
2774#ifdef CONFIG_NUMA
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002775 if (node != NUMA_NO_NODE && slab_nid(slab) != node)
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002776 return 0;
2777#endif
2778 return 1;
2779}
2780
David Rientjes9a02d692014-06-04 16:06:36 -07002781#ifdef CONFIG_SLUB_DEBUG
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002782static int count_free(struct slab *slab)
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002783{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002784 return slab->objects - slab->inuse;
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002785}
2786
David Rientjes9a02d692014-06-04 16:06:36 -07002787static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2788{
2789 return atomic_long_read(&n->total_objects);
2790}
2791#endif /* CONFIG_SLUB_DEBUG */
2792
2793#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002794static unsigned long count_partial(struct kmem_cache_node *n,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002795 int (*get_count)(struct slab *))
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002796{
2797 unsigned long flags;
2798 unsigned long x = 0;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002799 struct slab *slab;
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002800
2801 spin_lock_irqsave(&n->list_lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002802 list_for_each_entry(slab, &n->partial, slab_list)
2803 x += get_count(slab);
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002804 spin_unlock_irqrestore(&n->list_lock, flags);
2805 return x;
2806}
David Rientjes9a02d692014-06-04 16:06:36 -07002807#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04002808
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002809static noinline void
2810slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2811{
David Rientjes9a02d692014-06-04 16:06:36 -07002812#ifdef CONFIG_SLUB_DEBUG
2813 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
2814 DEFAULT_RATELIMIT_BURST);
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002815 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07002816 struct kmem_cache_node *n;
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002817
David Rientjes9a02d692014-06-04 16:06:36 -07002818 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
2819 return;
2820
Vlastimil Babka5b3810e2016-03-15 14:56:33 -07002821 pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
2822 nid, gfpflags, &gfpflags);
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07002823 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
Fabian Frederickf9f58282014-06-04 16:06:34 -07002824 s->name, s->object_size, s->size, oo_order(s->oo),
2825 oo_order(s->min));
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002826
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05002827 if (oo_order(s->min) > get_order(s->object_size))
Fabian Frederickf9f58282014-06-04 16:06:34 -07002828 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
2829 s->name);
David Rientjesfa5ec8a2009-07-07 00:14:14 -07002830
Christoph Lameterfa45dc22014-08-06 16:04:09 -07002831 for_each_kmem_cache_node(s, node, n) {
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002832 unsigned long nr_slabs;
2833 unsigned long nr_objs;
2834 unsigned long nr_free;
2835
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04002836 nr_free = count_partial(n, count_free);
2837 nr_slabs = node_nr_slabs(n);
2838 nr_objs = node_nr_objs(n);
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002839
Fabian Frederickf9f58282014-06-04 16:06:34 -07002840 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002841 node, nr_slabs, nr_objs, nr_free);
2842 }
David Rientjes9a02d692014-06-04 16:06:36 -07002843#endif
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002844}
2845
Matthew Wilcox (Oracle)01b34d12021-10-04 14:46:40 +01002846static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
Mel Gorman072bb0a2012-07-31 16:43:58 -07002847{
Matthew Wilcox (Oracle)01b34d12021-10-04 14:46:40 +01002848 if (unlikely(slab_test_pfmemalloc(slab)))
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002849 return gfp_pfmemalloc_allowed(gfpflags);
2850
2851 return true;
2852}
2853
2854/*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002855 * Check the slab->freelist and either transfer the freelist to the
2856 * per cpu freelist or deactivate the slab.
Christoph Lameter213eeb92011-11-11 14:07:14 -06002857 *
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002858 * The slab is still frozen if the return value is not NULL.
Christoph Lameter213eeb92011-11-11 14:07:14 -06002859 *
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002860 * If this function returns NULL then the slab has been unfrozen.
Christoph Lameter213eeb92011-11-11 14:07:14 -06002861 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002862static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
Christoph Lameter213eeb92011-11-11 14:07:14 -06002863{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002864 struct slab new;
Christoph Lameter213eeb92011-11-11 14:07:14 -06002865 unsigned long counters;
2866 void *freelist;
2867
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002868 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
2869
Christoph Lameter213eeb92011-11-11 14:07:14 -06002870 do {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002871 freelist = slab->freelist;
2872 counters = slab->counters;
Christoph Lameter6faa6832012-05-09 10:09:51 -05002873
Christoph Lameter213eeb92011-11-11 14:07:14 -06002874 new.counters = counters;
Dave Hansena0132ac2014-01-29 14:05:50 -08002875 VM_BUG_ON(!new.frozen);
Christoph Lameter213eeb92011-11-11 14:07:14 -06002876
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002877 new.inuse = slab->objects;
Christoph Lameter213eeb92011-11-11 14:07:14 -06002878 new.frozen = freelist != NULL;
2879
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002880 } while (!__cmpxchg_double_slab(s, slab,
Christoph Lameter213eeb92011-11-11 14:07:14 -06002881 freelist, counters,
2882 NULL, new.counters,
2883 "get_freelist"));
2884
2885 return freelist;
2886}
2887
2888/*
Christoph Lameter894b8782007-05-10 03:15:16 -07002889 * Slow path. The lockless freelist is empty or we need to perform
2890 * debugging duties.
Christoph Lameter81819f02007-05-06 14:49:36 -07002891 *
Christoph Lameter894b8782007-05-10 03:15:16 -07002892 * Processing is still very fast if new objects have been freed to the
2893 * regular freelist. In that case we simply take over the regular freelist
2894 * as the lockless freelist and zap the regular freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07002895 *
Christoph Lameter894b8782007-05-10 03:15:16 -07002896 * If that is not working then we fall back to the partial lists. We take the
2897 * first element of the freelist as the object to allocate now and move the
2898 * rest of the freelist to the lockless freelist.
2899 *
2900 * And if we were unable to get a new slab from the partial slab lists then
Christoph Lameter6446faa2008-02-15 23:45:26 -08002901 * we need to allocate a new slab. This is the slowest path since it involves
2902 * a call to the page allocator and the setup of a new slab.
Christoph Lametera380a3c2015-11-20 15:57:35 -08002903 *
Vlastimil Babkae5000592021-05-07 19:32:31 +02002904 * Version of __slab_alloc to use when we know that preemption is
Christoph Lametera380a3c2015-11-20 15:57:35 -08002905 * already disabled (which is the case for bulk allocation).
Christoph Lameter81819f02007-05-06 14:49:36 -07002906 */
Christoph Lametera380a3c2015-11-20 15:57:35 -08002907static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03002908 unsigned long addr, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002909{
Christoph Lameter6faa6832012-05-09 10:09:51 -05002910 void *freelist;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002911 struct slab *slab;
Vlastimil Babkae5000592021-05-07 19:32:31 +02002912 unsigned long flags;
Christoph Lameter81819f02007-05-06 14:49:36 -07002913
Abel Wu9f986d92020-10-13 16:48:43 -07002914 stat(s, ALLOC_SLOWPATH);
2915
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002916reread_slab:
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002917
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002918 slab = READ_ONCE(c->slab);
2919 if (!slab) {
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002920 /*
2921 * if the node is not online or has no normal memory, just
2922 * ignore the node constraint
2923 */
2924 if (unlikely(node != NUMA_NO_NODE &&
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08002925 !node_isset(node, slab_nodes)))
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002926 node = NUMA_NO_NODE;
Christoph Lameter81819f02007-05-06 14:49:36 -07002927 goto new_slab;
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002928 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002929redo:
Christoph Lameter6faa6832012-05-09 10:09:51 -05002930
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002931 if (unlikely(!node_match(slab, node))) {
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002932 /*
2933 * same as above but node_match() being false already
2934 * implies node != NUMA_NO_NODE
2935 */
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08002936 if (!node_isset(node, slab_nodes)) {
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002937 node = NUMA_NO_NODE;
Vlastimil Babka0715e6c2020-03-21 18:22:37 -07002938 } else {
Joonsoo Kima561ce02014-10-09 15:26:15 -07002939 stat(s, ALLOC_NODE_MISMATCH);
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002940 goto deactivate_slab;
Joonsoo Kima561ce02014-10-09 15:26:15 -07002941 }
Christoph Lameterfc59c052011-06-01 12:25:56 -05002942 }
Christoph Lameter6446faa2008-02-15 23:45:26 -08002943
Mel Gorman072bb0a2012-07-31 16:43:58 -07002944 /*
2945 * By rights, we should be searching for a slab page that was
2946 * PFMEMALLOC but right now, we are losing the pfmemalloc
2947 * information when the page leaves the per-cpu allocator
2948 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002949 if (unlikely(!pfmemalloc_match(slab, gfpflags)))
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002950 goto deactivate_slab;
Mel Gorman072bb0a2012-07-31 16:43:58 -07002951
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002952 /* must check again c->slab in case we got preempted and it changed */
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002953 local_lock_irqsave(&s->cpu_slab->lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002954 if (unlikely(slab != c->slab)) {
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002955 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002956 goto reread_slab;
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002957 }
Christoph Lameter6faa6832012-05-09 10:09:51 -05002958 freelist = c->freelist;
2959 if (freelist)
Eric Dumazet73736e02011-12-13 04:57:06 +01002960 goto load_freelist;
2961
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002962 freelist = get_freelist(s, slab);
Christoph Lameter6446faa2008-02-15 23:45:26 -08002963
Christoph Lameter6faa6832012-05-09 10:09:51 -05002964 if (!freelist) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002965 c->slab = NULL;
Jann Horneeaa3452022-06-08 20:22:05 +02002966 c->tid = next_tid(c->tid);
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002967 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Christoph Lameter03e404a2011-06-01 12:25:58 -05002968 stat(s, DEACTIVATE_BYPASS);
Christoph Lameterfc59c052011-06-01 12:25:56 -05002969 goto new_slab;
Christoph Lameter03e404a2011-06-01 12:25:58 -05002970 }
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002971
Christoph Lameter81819f02007-05-06 14:49:36 -07002972 stat(s, ALLOC_REFILL);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08002973
Christoph Lameter894b8782007-05-10 03:15:16 -07002974load_freelist:
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002975
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002976 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002977
Christoph Lameter507effe2012-05-09 10:09:52 -05002978 /*
2979 * freelist is pointing to the list of objects to be used.
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002980 * slab is pointing to the slab from which the objects are obtained.
2981 * That slab must be frozen for per cpu allocations to work.
Christoph Lameter507effe2012-05-09 10:09:52 -05002982 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002983 VM_BUG_ON(!c->slab->frozen);
Christoph Lameter6faa6832012-05-09 10:09:51 -05002984 c->freelist = get_freepointer(s, freelist);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002985 c->tid = next_tid(c->tid);
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002986 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Christoph Lameter6faa6832012-05-09 10:09:51 -05002987 return freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07002988
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002989deactivate_slab:
2990
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002991 local_lock_irqsave(&s->cpu_slab->lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002992 if (slab != c->slab) {
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02002993 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkac2092c12021-11-15 16:55:15 +01002994 goto reread_slab;
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02002995 }
Vlastimil Babkaa019d202021-05-12 13:53:34 +02002996 freelist = c->freelist;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01002997 c->slab = NULL;
Vlastimil Babkaa019d202021-05-12 13:53:34 +02002998 c->freelist = NULL;
Jann Horneeaa3452022-06-08 20:22:05 +02002999 c->tid = next_tid(c->tid);
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003000 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003001 deactivate_slab(s, slab, freelist);
Vlastimil Babka0b303fb2021-05-08 02:28:02 +02003002
Christoph Lameter81819f02007-05-06 14:49:36 -07003003new_slab:
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003004
Wei Yanga93cf072017-07-06 15:36:31 -07003005 if (slub_percpu_partial(c)) {
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003006 local_lock_irqsave(&s->cpu_slab->lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003007 if (unlikely(c->slab)) {
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003008 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003009 goto reread_slab;
Vlastimil Babkafa417ab2021-05-10 13:56:17 +02003010 }
Vlastimil Babka4b1f4492021-05-11 17:45:26 +02003011 if (unlikely(!slub_percpu_partial(c))) {
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003012 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003013 /* we were preempted and partial list got empty */
3014 goto new_objects;
Vlastimil Babka4b1f4492021-05-11 17:45:26 +02003015 }
Vlastimil Babkafa417ab2021-05-10 13:56:17 +02003016
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003017 slab = c->slab = slub_percpu_partial(c);
3018 slub_set_percpu_partial(c, slab);
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003019 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Christoph Lameter49e22582011-08-09 16:12:27 -05003020 stat(s, CPU_PARTIAL_ALLOC);
Christoph Lameter49e22582011-08-09 16:12:27 -05003021 goto redo;
Christoph Lameter81819f02007-05-06 14:49:36 -07003022 }
3023
Vlastimil Babkafa417ab2021-05-10 13:56:17 +02003024new_objects:
3025
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003026 freelist = get_partial(s, gfpflags, node, &slab);
Vlastimil Babka3f2b77e2021-05-11 16:37:51 +02003027 if (freelist)
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003028 goto check_new_slab;
Vlastimil Babka2a904902021-05-11 12:45:48 +02003029
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003030 slub_put_cpu_ptr(s->cpu_slab);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003031 slab = new_slab(s, gfpflags, node);
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003032 c = slub_get_cpu_ptr(s->cpu_slab);
Christoph Lameterb811c202007-10-16 23:25:51 -07003033
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003034 if (unlikely(!slab)) {
David Rientjes9a02d692014-06-04 16:06:36 -07003035 slab_out_of_memory(s, gfpflags, node);
Christoph Lameterf46974362012-05-09 10:09:54 -05003036 return NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07003037 }
Christoph Lameter894b8782007-05-10 03:15:16 -07003038
Vlastimil Babka53a0de02021-05-11 13:01:34 +02003039 /*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003040 * No other reference to the slab yet so we can
Vlastimil Babka53a0de02021-05-11 13:01:34 +02003041 * muck around with it freely without cmpxchg
3042 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003043 freelist = slab->freelist;
3044 slab->freelist = NULL;
Vlastimil Babka53a0de02021-05-11 13:01:34 +02003045
3046 stat(s, ALLOC_SLAB);
Vlastimil Babka53a0de02021-05-11 13:01:34 +02003047
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003048check_new_slab:
Christoph Lameter894b8782007-05-10 03:15:16 -07003049
Vlastimil Babka1572df72021-05-11 18:25:09 +02003050 if (kmem_cache_debug(s)) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003051 if (!alloc_debug_processing(s, slab, freelist, addr)) {
Vlastimil Babka1572df72021-05-11 18:25:09 +02003052 /* Slab failed checks. Next slab needed */
3053 goto new_slab;
Vlastimil Babkafa417ab2021-05-10 13:56:17 +02003054 } else {
Vlastimil Babka1572df72021-05-11 18:25:09 +02003055 /*
3056 * For debug case, we don't load freelist so that all
3057 * allocations go through alloc_debug_processing()
3058 */
3059 goto return_single;
Vlastimil Babkafa417ab2021-05-10 13:56:17 +02003060 }
Vlastimil Babka1572df72021-05-11 18:25:09 +02003061 }
3062
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003063 if (unlikely(!pfmemalloc_match(slab, gfpflags)))
Vlastimil Babka1572df72021-05-11 18:25:09 +02003064 /*
3065 * For !pfmemalloc_match() case we don't load freelist so that
3066 * we don't make further mismatched allocations easier.
3067 */
3068 goto return_single;
3069
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003070retry_load_slab:
Vlastimil Babkacfdf8362021-05-12 14:04:43 +02003071
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003072 local_lock_irqsave(&s->cpu_slab->lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003073 if (unlikely(c->slab)) {
Vlastimil Babkacfdf8362021-05-12 14:04:43 +02003074 void *flush_freelist = c->freelist;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003075 struct slab *flush_slab = c->slab;
Vlastimil Babkacfdf8362021-05-12 14:04:43 +02003076
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003077 c->slab = NULL;
Vlastimil Babkacfdf8362021-05-12 14:04:43 +02003078 c->freelist = NULL;
3079 c->tid = next_tid(c->tid);
3080
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003081 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
Vlastimil Babkacfdf8362021-05-12 14:04:43 +02003082
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003083 deactivate_slab(s, flush_slab, flush_freelist);
Vlastimil Babkacfdf8362021-05-12 14:04:43 +02003084
3085 stat(s, CPUSLAB_FLUSH);
3086
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003087 goto retry_load_slab;
Vlastimil Babkacfdf8362021-05-12 14:04:43 +02003088 }
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003089 c->slab = slab;
Vlastimil Babka3f2b77e2021-05-11 16:37:51 +02003090
Vlastimil Babka1572df72021-05-11 18:25:09 +02003091 goto load_freelist;
3092
3093return_single:
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003094
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003095 deactivate_slab(s, slab, get_freepointer(s, freelist));
Christoph Lameter6faa6832012-05-09 10:09:51 -05003096 return freelist;
Christoph Lameter894b8782007-05-10 03:15:16 -07003097}
3098
3099/*
Vlastimil Babkae5000592021-05-07 19:32:31 +02003100 * A wrapper for ___slab_alloc() for contexts where preemption is not yet
3101 * disabled. Compensates for possible cpu changes by refetching the per cpu area
3102 * pointer.
Christoph Lametera380a3c2015-11-20 15:57:35 -08003103 */
3104static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3105 unsigned long addr, struct kmem_cache_cpu *c)
3106{
3107 void *p;
Christoph Lametera380a3c2015-11-20 15:57:35 -08003108
Vlastimil Babkae5000592021-05-07 19:32:31 +02003109#ifdef CONFIG_PREEMPT_COUNT
Christoph Lametera380a3c2015-11-20 15:57:35 -08003110 /*
3111 * We may have been preempted and rescheduled on a different
Vlastimil Babkae5000592021-05-07 19:32:31 +02003112 * cpu before disabling preemption. Need to reload cpu area
Christoph Lametera380a3c2015-11-20 15:57:35 -08003113 * pointer.
3114 */
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003115 c = slub_get_cpu_ptr(s->cpu_slab);
Christoph Lametera380a3c2015-11-20 15:57:35 -08003116#endif
3117
3118 p = ___slab_alloc(s, gfpflags, node, addr, c);
Vlastimil Babkae5000592021-05-07 19:32:31 +02003119#ifdef CONFIG_PREEMPT_COUNT
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003120 slub_put_cpu_ptr(s->cpu_slab);
Vlastimil Babkae5000592021-05-07 19:32:31 +02003121#endif
Christoph Lametera380a3c2015-11-20 15:57:35 -08003122 return p;
3123}
3124
3125/*
Alexander Potapenko0f181f92019-10-14 14:11:57 -07003126 * If the object has been wiped upon free, make sure it's fully initialized by
3127 * zeroing out freelist pointer.
3128 */
3129static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
3130 void *obj)
3131{
3132 if (unlikely(slab_want_init_on_free(s)) && obj)
Andrey Konovalovce5716c62021-01-23 21:01:38 -08003133 memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
3134 0, sizeof(void *));
Alexander Potapenko0f181f92019-10-14 14:11:57 -07003135}
3136
3137/*
Christoph Lameter894b8782007-05-10 03:15:16 -07003138 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
3139 * have the fastpath folded into their functions. So no function call
3140 * overhead for requests that can be satisfied on the fastpath.
3141 *
3142 * The fastpath works by first checking if the lockless freelist can be used.
3143 * If not then __slab_alloc is called for slow processing.
3144 *
3145 * Otherwise we can simply pick the next object from the lockless free list.
3146 */
Muchun Song88f2ef72022-03-22 14:40:56 -07003147static __always_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003148 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
Christoph Lameter894b8782007-05-10 03:15:16 -07003149{
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08003150 void *object;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003151 struct kmem_cache_cpu *c;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003152 struct slab *slab;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003153 unsigned long tid;
Roman Gushchin964d4bd2020-08-06 23:20:56 -07003154 struct obj_cgroup *objcg = NULL;
Andrey Konovalovda844b72021-04-29 23:00:06 -07003155 bool init = false;
Christoph Lameter1f842602008-01-07 23:20:30 -08003156
Muchun Song88f2ef72022-03-22 14:40:56 -07003157 s = slab_pre_alloc_hook(s, lru, &objcg, 1, gfpflags);
Vladimir Davydov8135be52014-12-12 16:56:38 -08003158 if (!s)
Akinobu Mita773ff602008-12-23 19:37:01 +09003159 return NULL;
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003160
3161 object = kfence_alloc(s, orig_size, gfpflags);
3162 if (unlikely(object))
3163 goto out;
3164
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003165redo:
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003166 /*
3167 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
3168 * enabled. We may switch back and forth between cpus while
3169 * reading from one cpu area. That does not matter as long
3170 * as we end up on the original cpu again when doing the cmpxchg.
Christoph Lameter7cccd80b2013-01-23 21:45:48 +00003171 *
Vlastimil Babka9b4bc852021-05-18 02:01:39 +02003172 * We must guarantee that tid and kmem_cache_cpu are retrieved on the
3173 * same cpu. We read first the kmem_cache_cpu pointer and use it to read
3174 * the tid. If we are preempted and switched to another cpu between the
3175 * two reads, it's OK as the two are still associated with the same cpu
3176 * and cmpxchg later will validate the cpu.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003177 */
Vlastimil Babka9b4bc852021-05-18 02:01:39 +02003178 c = raw_cpu_ptr(s->cpu_slab);
3179 tid = READ_ONCE(c->tid);
Joonsoo Kim9aabf812015-02-10 14:09:32 -08003180
3181 /*
3182 * Irqless object alloc/free algorithm used here depends on sequence
3183 * of fetching cpu_slab's data. tid should be fetched before anything
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003184 * on c to guarantee that object and slab associated with previous tid
Joonsoo Kim9aabf812015-02-10 14:09:32 -08003185 * won't be used with current tid. If we fetch tid first, object and
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003186 * slab could be one associated with next tid and our alloc/free
Joonsoo Kim9aabf812015-02-10 14:09:32 -08003187 * request will be failed. In this case, we will retry. So, no problem.
3188 */
3189 barrier();
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003190
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003191 /*
3192 * The transaction ids are globally unique per cpu and per operation on
3193 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
3194 * occurs on the right processor and that there was no operation on the
3195 * linked list in between.
3196 */
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003197
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003198 object = c->freelist;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003199 slab = c->slab;
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003200 /*
3201 * We cannot use the lockless fastpath on PREEMPT_RT because if a
3202 * slowpath has taken the local_lock_irqsave(), it is not protected
3203 * against a fast path operation in an irq handler. So we need to take
3204 * the slow path which uses local_lock. It is still relatively fast if
3205 * there is a suitable cpu freelist.
3206 */
3207 if (IS_ENABLED(CONFIG_PREEMPT_RT) ||
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003208 unlikely(!object || !slab || !node_match(slab, node))) {
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003209 object = __slab_alloc(s, gfpflags, node, addr, c);
Dave Hansen8eae1492014-06-04 16:06:37 -07003210 } else {
Eric Dumazet0ad95002011-12-16 16:25:34 +01003211 void *next_object = get_freepointer_safe(s, object);
3212
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003213 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003214 * The cmpxchg will only match if there was no additional
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003215 * operation and if we are on the right processor.
3216 *
Chen Gangd0e0ac92013-07-15 09:05:29 +08003217 * The cmpxchg does the following atomically (without lock
3218 * semantics!)
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003219 * 1. Relocate first pointer to the current per cpu area.
3220 * 2. Verify that tid and freelist have not been changed
3221 * 3. If they were not changed replace tid and freelist
3222 *
Chen Gangd0e0ac92013-07-15 09:05:29 +08003223 * Since this is without lock semantics the protection is only
3224 * against code executing on this cpu *not* from access by
3225 * other cpus.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003226 */
Christoph Lameter933393f2011-12-22 11:58:51 -06003227 if (unlikely(!this_cpu_cmpxchg_double(
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003228 s->cpu_slab->freelist, s->cpu_slab->tid,
3229 object, tid,
Eric Dumazet0ad95002011-12-16 16:25:34 +01003230 next_object, next_tid(tid)))) {
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003231
3232 note_cmpxchg_failure("slab_alloc", s, tid);
3233 goto redo;
3234 }
Eric Dumazet0ad95002011-12-16 16:25:34 +01003235 prefetch_freepointer(s, next_object);
Christoph Lameter84e554e62009-12-18 16:26:23 -06003236 stat(s, ALLOC_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07003237 }
Alexander Potapenko0f181f92019-10-14 14:11:57 -07003238
Andrey Konovalovce5716c62021-01-23 21:01:38 -08003239 maybe_wipe_obj_freeptr(s, object);
Andrey Konovalovda844b72021-04-29 23:00:06 -07003240 init = slab_want_init_on_alloc(gfpflags, s);
Christoph Lameterd07dbea2007-07-17 04:03:23 -07003241
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003242out:
Andrey Konovalovda844b72021-04-29 23:00:06 -07003243 slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init);
Vegard Nossum5a896d92008-04-04 00:54:48 +02003244
Christoph Lameter894b8782007-05-10 03:15:16 -07003245 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07003246}
3247
Muchun Song88f2ef72022-03-22 14:40:56 -07003248static __always_inline void *slab_alloc(struct kmem_cache *s, struct list_lru *lru,
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003249 gfp_t gfpflags, unsigned long addr, size_t orig_size)
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03003250{
Muchun Song88f2ef72022-03-22 14:40:56 -07003251 return slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, addr, orig_size);
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03003252}
3253
Muchun Song88f2ef72022-03-22 14:40:56 -07003254static __always_inline
3255void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
3256 gfp_t gfpflags)
Christoph Lameter81819f02007-05-06 14:49:36 -07003257{
Muchun Song88f2ef72022-03-22 14:40:56 -07003258 void *ret = slab_alloc(s, lru, gfpflags, _RET_IP_, s->object_size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003259
Vasily Averinb347aa72022-06-03 06:21:49 +03003260 trace_kmem_cache_alloc(_RET_IP_, ret, s, s->object_size,
Chen Gangd0e0ac92013-07-15 09:05:29 +08003261 s->size, gfpflags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003262
3263 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003264}
Muchun Song88f2ef72022-03-22 14:40:56 -07003265
3266void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
3267{
3268 return __kmem_cache_alloc_lru(s, NULL, gfpflags);
3269}
Christoph Lameter81819f02007-05-06 14:49:36 -07003270EXPORT_SYMBOL(kmem_cache_alloc);
3271
Muchun Song88f2ef72022-03-22 14:40:56 -07003272void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
3273 gfp_t gfpflags)
3274{
3275 return __kmem_cache_alloc_lru(s, lru, gfpflags);
3276}
3277EXPORT_SYMBOL(kmem_cache_alloc_lru);
3278
Li Zefan0f24f122009-12-11 15:45:30 +08003279#ifdef CONFIG_TRACING
Richard Kennedy4a923792010-10-21 10:29:19 +01003280void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003281{
Muchun Song88f2ef72022-03-22 14:40:56 -07003282 void *ret = slab_alloc(s, NULL, gfpflags, _RET_IP_, size);
Vasily Averinb347aa72022-06-03 06:21:49 +03003283 trace_kmalloc(_RET_IP_, ret, s, size, s->size, gfpflags);
Andrey Konovalov0116523c2018-12-28 00:29:37 -08003284 ret = kasan_kmalloc(s, ret, size, gfpflags);
Richard Kennedy4a923792010-10-21 10:29:19 +01003285 return ret;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003286}
Richard Kennedy4a923792010-10-21 10:29:19 +01003287EXPORT_SYMBOL(kmem_cache_alloc_trace);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003288#endif
3289
Christoph Lameter81819f02007-05-06 14:49:36 -07003290#ifdef CONFIG_NUMA
3291void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
3292{
Muchun Song88f2ef72022-03-22 14:40:56 -07003293 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003294
Vasily Averinb347aa72022-06-03 06:21:49 +03003295 trace_kmem_cache_alloc_node(_RET_IP_, ret, s,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05003296 s->object_size, s->size, gfpflags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003297
3298 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003299}
3300EXPORT_SYMBOL(kmem_cache_alloc_node);
Christoph Lameter81819f02007-05-06 14:49:36 -07003301
Li Zefan0f24f122009-12-11 15:45:30 +08003302#ifdef CONFIG_TRACING
Richard Kennedy4a923792010-10-21 10:29:19 +01003303void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003304 gfp_t gfpflags,
Richard Kennedy4a923792010-10-21 10:29:19 +01003305 int node, size_t size)
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003306{
Muchun Song88f2ef72022-03-22 14:40:56 -07003307 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
Richard Kennedy4a923792010-10-21 10:29:19 +01003308
Vasily Averinb347aa72022-06-03 06:21:49 +03003309 trace_kmalloc_node(_RET_IP_, ret, s,
Richard Kennedy4a923792010-10-21 10:29:19 +01003310 size, s->size, gfpflags, node);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08003311
Andrey Konovalov0116523c2018-12-28 00:29:37 -08003312 ret = kasan_kmalloc(s, ret, size, gfpflags);
Richard Kennedy4a923792010-10-21 10:29:19 +01003313 return ret;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003314}
Richard Kennedy4a923792010-10-21 10:29:19 +01003315EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003316#endif
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07003317#endif /* CONFIG_NUMA */
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003318
Christoph Lameter81819f02007-05-06 14:49:36 -07003319/*
Kim Phillips94e4d712015-02-10 14:09:37 -08003320 * Slow path handling. This may still be called frequently since objects
Christoph Lameter894b8782007-05-10 03:15:16 -07003321 * have a longer lifetime than the cpu slabs in most processing loads.
Christoph Lameter81819f02007-05-06 14:49:36 -07003322 *
Christoph Lameter894b8782007-05-10 03:15:16 -07003323 * So we still attempt to reduce cache line usage. Just take the slab
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003324 * lock and free the item. If there is no additional partial slab
Christoph Lameter894b8782007-05-10 03:15:16 -07003325 * handling required then we can return immediately.
Christoph Lameter81819f02007-05-06 14:49:36 -07003326 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003327static void __slab_free(struct kmem_cache *s, struct slab *slab,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003328 void *head, void *tail, int cnt,
3329 unsigned long addr)
3330
Christoph Lameter81819f02007-05-06 14:49:36 -07003331{
3332 void *prior;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003333 int was_frozen;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003334 struct slab new;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003335 unsigned long counters;
3336 struct kmem_cache_node *n = NULL;
Kees Cook3f649ab2020-06-03 13:09:38 -07003337 unsigned long flags;
Christoph Lameter81819f02007-05-06 14:49:36 -07003338
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003339 stat(s, FREE_SLOWPATH);
Christoph Lameter81819f02007-05-06 14:49:36 -07003340
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003341 if (kfence_free(head))
3342 return;
3343
Christoph Lameter19c7ff92012-05-30 12:54:46 -05003344 if (kmem_cache_debug(s) &&
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003345 !free_debug_processing(s, slab, head, tail, cnt, addr))
Christoph Lameter80f08c12011-06-01 12:25:55 -05003346 return;
Christoph Lameter6446faa2008-02-15 23:45:26 -08003347
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003348 do {
Joonsoo Kim837d6782012-08-16 00:02:40 +09003349 if (unlikely(n)) {
3350 spin_unlock_irqrestore(&n->list_lock, flags);
3351 n = NULL;
3352 }
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003353 prior = slab->freelist;
3354 counters = slab->counters;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003355 set_freepointer(s, tail, prior);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003356 new.counters = counters;
3357 was_frozen = new.frozen;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003358 new.inuse -= cnt;
Joonsoo Kim837d6782012-08-16 00:02:40 +09003359 if ((!new.inuse || !prior) && !was_frozen) {
Christoph Lameter49e22582011-08-09 16:12:27 -05003360
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003361 if (kmem_cache_has_cpu_partial(s) && !prior) {
Christoph Lameter49e22582011-08-09 16:12:27 -05003362
3363 /*
Chen Gangd0e0ac92013-07-15 09:05:29 +08003364 * Slab was on no list before and will be
3365 * partially empty
3366 * We can defer the list move and instead
3367 * freeze it.
Christoph Lameter49e22582011-08-09 16:12:27 -05003368 */
3369 new.frozen = 1;
3370
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003371 } else { /* Needs to be taken off a list */
Christoph Lameter49e22582011-08-09 16:12:27 -05003372
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003373 n = get_node(s, slab_nid(slab));
Christoph Lameter49e22582011-08-09 16:12:27 -05003374 /*
3375 * Speculatively acquire the list_lock.
3376 * If the cmpxchg does not succeed then we may
3377 * drop the list_lock without any processing.
3378 *
3379 * Otherwise the list_lock will synchronize with
3380 * other processors updating the list of slabs.
3381 */
3382 spin_lock_irqsave(&n->list_lock, flags);
3383
3384 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003385 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003386
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003387 } while (!cmpxchg_double_slab(s, slab,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003388 prior, counters,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003389 head, new.counters,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003390 "__slab_free"));
Christoph Lameter81819f02007-05-06 14:49:36 -07003391
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003392 if (likely(!n)) {
Christoph Lameter49e22582011-08-09 16:12:27 -05003393
Abel Wuc270cf32020-10-13 16:48:40 -07003394 if (likely(was_frozen)) {
3395 /*
3396 * The list lock was not taken therefore no list
3397 * activity can be necessary.
3398 */
3399 stat(s, FREE_FROZEN);
3400 } else if (new.frozen) {
3401 /*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003402 * If we just froze the slab then put it onto the
Abel Wuc270cf32020-10-13 16:48:40 -07003403 * per cpu partial list.
3404 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003405 put_cpu_partial(s, slab, 1);
Alex Shi8028dce2012-02-03 23:34:56 +08003406 stat(s, CPU_PARTIAL_FREE);
3407 }
Abel Wuc270cf32020-10-13 16:48:40 -07003408
LQYMGTb455def2014-12-10 15:42:13 -08003409 return;
3410 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003411
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07003412 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
Joonsoo Kim837d6782012-08-16 00:02:40 +09003413 goto slab_empty;
Christoph Lameter81819f02007-05-06 14:49:36 -07003414
Joonsoo Kim837d6782012-08-16 00:02:40 +09003415 /*
3416 * Objects left in the slab. If it was not on the partial list before
3417 * then add it.
3418 */
Joonsoo Kim345c9052013-06-19 14:05:52 +09003419 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003420 remove_full(s, n, slab);
3421 add_partial(n, slab, DEACTIVATE_TO_TAIL);
Joonsoo Kim837d6782012-08-16 00:02:40 +09003422 stat(s, FREE_ADD_PARTIAL);
Christoph Lameter81819f02007-05-06 14:49:36 -07003423 }
Christoph Lameter80f08c12011-06-01 12:25:55 -05003424 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07003425 return;
3426
3427slab_empty:
Christoph Lametera973e9d2008-03-01 13:40:44 -08003428 if (prior) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003429 /*
Christoph Lameter6fbabb22011-08-08 11:16:56 -05003430 * Slab on the partial list.
Christoph Lameter81819f02007-05-06 14:49:36 -07003431 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003432 remove_partial(n, slab);
Christoph Lameter84e554e62009-12-18 16:26:23 -06003433 stat(s, FREE_REMOVE_PARTIAL);
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003434 } else {
Christoph Lameter6fbabb22011-08-08 11:16:56 -05003435 /* Slab must be on the full list */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003436 remove_full(s, n, slab);
Peter Zijlstrac65c1872014-01-10 13:23:49 +01003437 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05003438
Christoph Lameter80f08c12011-06-01 12:25:55 -05003439 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter84e554e62009-12-18 16:26:23 -06003440 stat(s, FREE_SLAB);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003441 discard_slab(s, slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07003442}
3443
Christoph Lameter894b8782007-05-10 03:15:16 -07003444/*
3445 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
3446 * can perform fastpath freeing without additional function calls.
3447 *
3448 * The fastpath is only possible if we are freeing to the current cpu slab
3449 * of this processor. This typically the case if we have just allocated
3450 * the item before.
3451 *
3452 * If fastpath is not possible then fall back to __slab_free where we deal
3453 * with all sorts of special processing.
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003454 *
3455 * Bulk free of a freelist with several objects (all pointing to the
Vlastimil Babkac2092c12021-11-15 16:55:15 +01003456 * same slab) possible by specifying head and tail ptr, plus objects
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003457 * count (cnt). Bulk free indicated by tail pointer being set.
Christoph Lameter894b8782007-05-10 03:15:16 -07003458 */
Alexander Potapenko80a92012016-07-28 15:49:07 -07003459static __always_inline void do_slab_free(struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003460 struct slab *slab, void *head, void *tail,
Alexander Potapenko80a92012016-07-28 15:49:07 -07003461 int cnt, unsigned long addr)
Christoph Lameter894b8782007-05-10 03:15:16 -07003462{
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003463 void *tail_obj = tail ? : head;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003464 struct kmem_cache_cpu *c;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003465 unsigned long tid;
Roman Gushchin964d4bd2020-08-06 23:20:56 -07003466
Christoph Lametera24c5a02011-03-15 12:45:21 -05003467redo:
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003468 /*
3469 * Determine the currently cpus per cpu slab.
3470 * The cpu may change afterward. However that does not matter since
3471 * data is retrieved via this pointer. If we are on the same cpu
Jesper Dangaard Brouer2ae44002015-09-04 15:45:31 -07003472 * during the cmpxchg then the free will succeed.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003473 */
Vlastimil Babka9b4bc852021-05-18 02:01:39 +02003474 c = raw_cpu_ptr(s->cpu_slab);
3475 tid = READ_ONCE(c->tid);
Christoph Lameterc016b0b2010-08-20 12:37:16 -05003476
Joonsoo Kim9aabf812015-02-10 14:09:32 -08003477 /* Same with comment on barrier() in slab_alloc_node() */
3478 barrier();
Christoph Lameterc016b0b2010-08-20 12:37:16 -05003479
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003480 if (likely(slab == c->slab)) {
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003481#ifndef CONFIG_PREEMPT_RT
Linus Torvalds50761902020-03-17 11:04:09 -07003482 void **freelist = READ_ONCE(c->freelist);
3483
3484 set_freepointer(s, tail_obj, freelist);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003485
Christoph Lameter933393f2011-12-22 11:58:51 -06003486 if (unlikely(!this_cpu_cmpxchg_double(
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003487 s->cpu_slab->freelist, s->cpu_slab->tid,
Linus Torvalds50761902020-03-17 11:04:09 -07003488 freelist, tid,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003489 head, next_tid(tid)))) {
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003490
3491 note_cmpxchg_failure("slab_free", s, tid);
3492 goto redo;
3493 }
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003494#else /* CONFIG_PREEMPT_RT */
3495 /*
3496 * We cannot use the lockless fastpath on PREEMPT_RT because if
3497 * a slowpath has taken the local_lock_irqsave(), it is not
3498 * protected against a fast path operation in an irq handler. So
3499 * we need to take the local_lock. We shouldn't simply defer to
3500 * __slab_free() as that wouldn't use the cpu freelist at all.
3501 */
3502 void **freelist;
3503
3504 local_lock(&s->cpu_slab->lock);
3505 c = this_cpu_ptr(s->cpu_slab);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003506 if (unlikely(slab != c->slab)) {
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003507 local_unlock(&s->cpu_slab->lock);
3508 goto redo;
3509 }
3510 tid = c->tid;
3511 freelist = c->freelist;
3512
3513 set_freepointer(s, tail_obj, freelist);
3514 c->freelist = head;
3515 c->tid = next_tid(tid);
3516
3517 local_unlock(&s->cpu_slab->lock);
3518#endif
Christoph Lameter84e554e62009-12-18 16:26:23 -06003519 stat(s, FREE_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07003520 } else
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003521 __slab_free(s, slab, head, tail_obj, cnt, addr);
Christoph Lameter894b8782007-05-10 03:15:16 -07003522
Christoph Lameter894b8782007-05-10 03:15:16 -07003523}
3524
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003525static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,
Muchun Songb77d5b12022-04-29 20:30:44 +08003526 void *head, void *tail, void **p, int cnt,
Alexander Potapenko80a92012016-07-28 15:49:07 -07003527 unsigned long addr)
3528{
Muchun Songb77d5b12022-04-29 20:30:44 +08003529 memcg_slab_free_hook(s, slab, p, cnt);
Alexander Potapenko80a92012016-07-28 15:49:07 -07003530 /*
Andrey Konovalovc3895392018-04-10 16:30:31 -07003531 * With KASAN enabled slab_free_freelist_hook modifies the freelist
3532 * to remove objects, whose reuse must be delayed.
Alexander Potapenko80a92012016-07-28 15:49:07 -07003533 */
Miaohe Lin899447f2021-10-18 15:15:55 -07003534 if (slab_free_freelist_hook(s, &head, &tail, &cnt))
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003535 do_slab_free(s, slab, head, tail, cnt, addr);
Alexander Potapenko80a92012016-07-28 15:49:07 -07003536}
3537
Andrey Konovalov2bd926b2018-12-28 00:29:53 -08003538#ifdef CONFIG_KASAN_GENERIC
Alexander Potapenko80a92012016-07-28 15:49:07 -07003539void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
3540{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003541 do_slab_free(cache, virt_to_slab(x), x, NULL, 1, addr);
Alexander Potapenko80a92012016-07-28 15:49:07 -07003542}
3543#endif
3544
Christoph Lameter81819f02007-05-06 14:49:36 -07003545void kmem_cache_free(struct kmem_cache *s, void *x)
3546{
Glauber Costab9ce5ef2012-12-18 14:22:46 -08003547 s = cache_from_obj(s, x);
3548 if (!s)
Christoph Lameter79576102012-09-04 23:06:14 +00003549 return;
Jacob Wen3544de8e2021-02-24 12:00:55 -08003550 trace_kmem_cache_free(_RET_IP_, x, s->name);
Muchun Songb77d5b12022-04-29 20:30:44 +08003551 slab_free(s, virt_to_slab(x), x, NULL, &x, 1, _RET_IP_);
Christoph Lameter81819f02007-05-06 14:49:36 -07003552}
3553EXPORT_SYMBOL(kmem_cache_free);
3554
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003555struct detached_freelist {
Matthew Wilcox (Oracle)cc465c3b2021-10-04 14:45:58 +01003556 struct slab *slab;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003557 void *tail;
3558 void *freelist;
3559 int cnt;
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003560 struct kmem_cache *s;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003561};
3562
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01003563static inline void free_large_kmalloc(struct folio *folio, void *object)
Shakeel Buttf227f0f2021-07-29 14:53:50 -07003564{
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01003565 unsigned int order = folio_order(folio);
Shakeel Buttf227f0f2021-07-29 14:53:50 -07003566
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01003567 if (WARN_ON_ONCE(order == 0))
Kefeng Wangd0fe47c2021-11-05 13:35:14 -07003568 pr_warn_once("object pointer: 0x%p\n", object);
3569
Shakeel Butt1ed7ce52021-08-13 16:54:31 -07003570 kfree_hook(object);
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01003571 mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
3572 -(PAGE_SIZE << order));
3573 __free_pages(folio_page(folio, 0), order);
Shakeel Buttf227f0f2021-07-29 14:53:50 -07003574}
3575
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003576/*
3577 * This function progressively scans the array with free objects (with
3578 * a limited look ahead) and extract objects belonging to the same
Matthew Wilcox (Oracle)cc465c3b2021-10-04 14:45:58 +01003579 * slab. It builds a detached freelist directly within the given
3580 * slab/objects. This can happen without any need for
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003581 * synchronization, because the objects are owned by running process.
3582 * The freelist is build up as a single linked list in the objects.
3583 * The idea is, that this detached freelist can then be bulk
3584 * transferred to the real freelist(s), but only requiring a single
3585 * synchronization primitive. Look ahead in the array is limited due
3586 * to performance reasons.
3587 */
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003588static inline
3589int build_detached_freelist(struct kmem_cache *s, size_t size,
3590 void **p, struct detached_freelist *df)
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003591{
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003592 int lookahead = 3;
3593 void *object;
Matthew Wilcox (Oracle)cc465c3b2021-10-04 14:45:58 +01003594 struct folio *folio;
Muchun Songb77d5b12022-04-29 20:30:44 +08003595 size_t same;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003596
Muchun Songb77d5b12022-04-29 20:30:44 +08003597 object = p[--size];
Matthew Wilcox (Oracle)cc465c3b2021-10-04 14:45:58 +01003598 folio = virt_to_folio(object);
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003599 if (!s) {
3600 /* Handle kalloc'ed objects */
Matthew Wilcox (Oracle)cc465c3b2021-10-04 14:45:58 +01003601 if (unlikely(!folio_test_slab(folio))) {
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01003602 free_large_kmalloc(folio, object);
Muchun Songb77d5b12022-04-29 20:30:44 +08003603 df->slab = NULL;
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003604 return size;
3605 }
3606 /* Derive kmem_cache from object */
Muchun Songb77d5b12022-04-29 20:30:44 +08003607 df->slab = folio_slab(folio);
3608 df->s = df->slab->slab_cache;
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003609 } else {
Muchun Songb77d5b12022-04-29 20:30:44 +08003610 df->slab = folio_slab(folio);
Jesper Dangaard Brouerca257192016-03-15 14:54:00 -07003611 df->s = cache_from_obj(s, object); /* Support for memcg */
3612 }
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003613
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003614 /* Start new detached freelist */
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003615 df->tail = object;
3616 df->freelist = object;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003617 df->cnt = 1;
3618
Muchun Songb77d5b12022-04-29 20:30:44 +08003619 if (is_kfence_address(object))
3620 return size;
3621
3622 set_freepointer(df->s, object, NULL);
3623
3624 same = size;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003625 while (size) {
3626 object = p[--size];
Matthew Wilcox (Oracle)cc465c3b2021-10-04 14:45:58 +01003627 /* df->slab is always set at this point */
3628 if (df->slab == virt_to_slab(object)) {
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003629 /* Opportunity build freelist */
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003630 set_freepointer(df->s, object, df->freelist);
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003631 df->freelist = object;
3632 df->cnt++;
Muchun Songb77d5b12022-04-29 20:30:44 +08003633 same--;
3634 if (size != same)
3635 swap(p[size], p[same]);
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003636 continue;
3637 }
3638
3639 /* Limit look ahead search */
3640 if (!--lookahead)
3641 break;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003642 }
3643
Muchun Songb77d5b12022-04-29 20:30:44 +08003644 return same;
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003645}
3646
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003647/* Note that interrupts must be enabled when calling this function. */
Jesper Dangaard Brouer376bf122016-03-15 14:53:32 -07003648void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
Christoph Lameter484748f2015-09-04 15:45:34 -07003649{
Hyeonggon Yoo2055e672022-06-15 00:26:34 +09003650 if (!size)
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003651 return;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07003652
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003653 do {
3654 struct detached_freelist df;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07003655
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003656 size = build_detached_freelist(s, size, p, &df);
Matthew Wilcox (Oracle)cc465c3b2021-10-04 14:45:58 +01003657 if (!df.slab)
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003658 continue;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07003659
Muchun Songb77d5b12022-04-29 20:30:44 +08003660 slab_free(df.s, df.slab, df.freelist, df.tail, &p[size], df.cnt,
3661 _RET_IP_);
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08003662 } while (likely(size));
Christoph Lameter484748f2015-09-04 15:45:34 -07003663}
3664EXPORT_SYMBOL(kmem_cache_free_bulk);
3665
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003666/* Note that interrupts must be enabled when calling this function. */
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08003667int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3668 void **p)
Christoph Lameter484748f2015-09-04 15:45:34 -07003669{
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003670 struct kmem_cache_cpu *c;
3671 int i;
Roman Gushchin964d4bd2020-08-06 23:20:56 -07003672 struct obj_cgroup *objcg = NULL;
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003673
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08003674 /* memcg and kmem_cache debug support */
Muchun Song88f2ef72022-03-22 14:40:56 -07003675 s = slab_pre_alloc_hook(s, NULL, &objcg, size, flags);
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08003676 if (unlikely(!s))
3677 return false;
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003678 /*
3679 * Drain objects in the per cpu slab, while disabling local
3680 * IRQs, which protects against PREEMPT and interrupts
3681 * handlers invoking normal fastpath.
3682 */
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003683 c = slub_get_cpu_ptr(s->cpu_slab);
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003684 local_lock_irq(&s->cpu_slab->lock);
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003685
3686 for (i = 0; i < size; i++) {
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003687 void *object = kfence_alloc(s, s->object_size, flags);
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003688
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08003689 if (unlikely(object)) {
3690 p[i] = object;
3691 continue;
3692 }
3693
3694 object = c->freelist;
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003695 if (unlikely(!object)) {
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003696 /*
Jann Hornfd4d9c72020-03-17 01:28:45 +01003697 * We may have removed an object from c->freelist using
3698 * the fastpath in the previous iteration; in that case,
3699 * c->tid has not been bumped yet.
3700 * Since ___slab_alloc() may reenable interrupts while
3701 * allocating memory, we should bump c->tid now.
3702 */
3703 c->tid = next_tid(c->tid);
3704
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003705 local_unlock_irq(&s->cpu_slab->lock);
Vlastimil Babkae5000592021-05-07 19:32:31 +02003706
Jann Hornfd4d9c72020-03-17 01:28:45 +01003707 /*
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003708 * Invoking slow path likely have side-effect
3709 * of re-populating per CPU c->freelist
3710 */
Christoph Lameter87098372015-11-20 15:57:38 -08003711 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003712 _RET_IP_, c);
Christoph Lameter87098372015-11-20 15:57:38 -08003713 if (unlikely(!p[i]))
3714 goto error;
3715
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003716 c = this_cpu_ptr(s->cpu_slab);
Alexander Potapenko0f181f92019-10-14 14:11:57 -07003717 maybe_wipe_obj_freeptr(s, p[i]);
3718
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003719 local_lock_irq(&s->cpu_slab->lock);
Vlastimil Babkae5000592021-05-07 19:32:31 +02003720
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07003721 continue; /* goto for-loop */
3722 }
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003723 c->freelist = get_freepointer(s, object);
3724 p[i] = object;
Alexander Potapenko0f181f92019-10-14 14:11:57 -07003725 maybe_wipe_obj_freeptr(s, p[i]);
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003726 }
3727 c->tid = next_tid(c->tid);
Vlastimil Babkabd0e7492021-05-22 01:59:38 +02003728 local_unlock_irq(&s->cpu_slab->lock);
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003729 slub_put_cpu_ptr(s->cpu_slab);
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07003730
Andrey Konovalovda844b72021-04-29 23:00:06 -07003731 /*
3732 * memcg and kmem_cache debug support and memory initialization.
3733 * Done outside of the IRQ disabled fastpath loop.
3734 */
3735 slab_post_alloc_hook(s, objcg, flags, size, p,
3736 slab_want_init_on_alloc(flags, s));
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08003737 return i;
Christoph Lameter87098372015-11-20 15:57:38 -08003738error:
Vlastimil Babka25c00c52021-05-21 14:03:23 +02003739 slub_put_cpu_ptr(s->cpu_slab);
Andrey Konovalovda844b72021-04-29 23:00:06 -07003740 slab_post_alloc_hook(s, objcg, flags, i, p, false);
Hyeonggon Yoo2055e672022-06-15 00:26:34 +09003741 kmem_cache_free_bulk(s, i, p);
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08003742 return 0;
Christoph Lameter484748f2015-09-04 15:45:34 -07003743}
3744EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3745
3746
Christoph Lameter81819f02007-05-06 14:49:36 -07003747/*
Christoph Lameter672bba32007-05-09 02:32:39 -07003748 * Object placement in a slab is made very easy because we always start at
3749 * offset 0. If we tune the size of the object to the alignment then we can
3750 * get the required alignment by putting one properly sized object after
3751 * another.
Christoph Lameter81819f02007-05-06 14:49:36 -07003752 *
3753 * Notice that the allocation order determines the sizes of the per cpu
3754 * caches. Each processor has always one slab available for allocations.
3755 * Increasing the allocation order reduces the number of times that slabs
Christoph Lameter672bba32007-05-09 02:32:39 -07003756 * must be moved on and off the partial lists and is therefore a factor in
Christoph Lameter81819f02007-05-06 14:49:36 -07003757 * locking overhead.
Christoph Lameter81819f02007-05-06 14:49:36 -07003758 */
3759
3760/*
Ingo Molnarf0953a12021-05-06 18:06:47 -07003761 * Minimum / Maximum order of slab pages. This influences locking overhead
Christoph Lameter81819f02007-05-06 14:49:36 -07003762 * and slab fragmentation. A higher order reduces the number of partial slabs
3763 * and increases the number of allocations possible without having to
3764 * take the list_lock.
3765 */
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003766static unsigned int slub_min_order;
3767static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
3768static unsigned int slub_min_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07003769
3770/*
Christoph Lameter81819f02007-05-06 14:49:36 -07003771 * Calculate the order of allocation given an slab object size.
3772 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003773 * The order of allocation has significant impact on performance and other
3774 * system components. Generally order 0 allocations should be preferred since
3775 * order 0 does not cause fragmentation in the page allocator. Larger objects
3776 * be problematic to put into order 0 slabs because there may be too much
Christoph Lameterc124f5b2008-04-14 19:13:29 +03003777 * unused space left. We go to a higher order if more than 1/16th of the slab
Christoph Lameter672bba32007-05-09 02:32:39 -07003778 * would be wasted.
Christoph Lameter81819f02007-05-06 14:49:36 -07003779 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003780 * In order to reach satisfactory performance we must ensure that a minimum
3781 * number of objects is in one slab. Otherwise we may generate too much
3782 * activity on the partial lists which requires taking the list_lock. This is
3783 * less a concern for large slabs though which are rarely used.
Christoph Lameter81819f02007-05-06 14:49:36 -07003784 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003785 * slub_max_order specifies the order where we begin to stop considering the
3786 * number of objects in a slab as critical. If we reach slub_max_order then
3787 * we try to keep the page order as low as possible. So we accept more waste
3788 * of space in favor of a small page order.
3789 *
3790 * Higher order allocations also allow the placement of more objects in a
3791 * slab and thereby reduce object handling overhead. If the user has
Bhaskar Chowdhurydc842072021-04-29 22:54:51 -07003792 * requested a higher minimum order then we start with that one instead of
Christoph Lameter672bba32007-05-09 02:32:39 -07003793 * the smallest order which will fit the object.
Christoph Lameter81819f02007-05-06 14:49:36 -07003794 */
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +01003795static inline unsigned int calc_slab_order(unsigned int size,
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003796 unsigned int min_objects, unsigned int max_order,
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003797 unsigned int fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07003798{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003799 unsigned int min_order = slub_min_order;
3800 unsigned int order;
Christoph Lameter81819f02007-05-06 14:49:36 -07003801
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003802 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +04003803 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
Christoph Lameter39b26462008-04-14 19:11:30 +03003804
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003805 for (order = max(min_order, (unsigned int)get_order(min_objects * size));
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003806 order <= max_order; order++) {
3807
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003808 unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
3809 unsigned int rem;
Christoph Lameter81819f02007-05-06 14:49:36 -07003810
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003811 rem = slab_size % size;
Christoph Lameter81819f02007-05-06 14:49:36 -07003812
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003813 if (rem <= slab_size / fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07003814 break;
Christoph Lameter81819f02007-05-06 14:49:36 -07003815 }
Christoph Lameter672bba32007-05-09 02:32:39 -07003816
Christoph Lameter81819f02007-05-06 14:49:36 -07003817 return order;
3818}
3819
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003820static inline int calculate_order(unsigned int size)
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003821{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003822 unsigned int order;
3823 unsigned int min_objects;
3824 unsigned int max_objects;
Vlastimil Babka32862222021-02-09 13:42:32 -08003825 unsigned int nr_cpus;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003826
3827 /*
3828 * Attempt to find best configuration for a slab. This
3829 * works by first attempting to generate a layout with
3830 * the best configuration and backing off gradually.
3831 *
Wei Yang422ff4d2015-11-05 18:45:46 -08003832 * First we increase the acceptable waste in a slab. Then
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003833 * we reduce the minimum objects required in a slab.
3834 */
3835 min_objects = slub_min_objects;
Vlastimil Babka32862222021-02-09 13:42:32 -08003836 if (!min_objects) {
3837 /*
3838 * Some architectures will only update present cpus when
3839 * onlining them, so don't trust the number if it's just 1. But
3840 * we also don't want to use nr_cpu_ids always, as on some other
3841 * architectures, there can be many possible cpus, but never
3842 * onlined. Here we compromise between trying to avoid too high
3843 * order on systems that appear larger than they are, and too
3844 * low order on systems that appear smaller than they are.
3845 */
3846 nr_cpus = num_present_cpus();
3847 if (nr_cpus <= 1)
3848 nr_cpus = nr_cpu_ids;
3849 min_objects = 4 * (fls(nr_cpus) + 1);
3850 }
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003851 max_objects = order_objects(slub_max_order, size);
Zhang Yanmine8120ff2009-02-12 18:00:17 +02003852 min_objects = min(min_objects, max_objects);
3853
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003854 while (min_objects > 1) {
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07003855 unsigned int fraction;
3856
Christoph Lameterc124f5b2008-04-14 19:13:29 +03003857 fraction = 16;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003858 while (fraction >= 4) {
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +01003859 order = calc_slab_order(size, min_objects,
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07003860 slub_max_order, fraction);
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003861 if (order <= slub_max_order)
3862 return order;
3863 fraction /= 2;
3864 }
Amerigo Wang5086c389c2009-08-19 21:44:13 +03003865 min_objects--;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003866 }
3867
3868 /*
3869 * We were unable to place multiple objects in a slab. Now
3870 * lets see if we can place a single object there.
3871 */
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +01003872 order = calc_slab_order(size, 1, slub_max_order, 1);
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003873 if (order <= slub_max_order)
3874 return order;
3875
3876 /*
3877 * Doh this slab cannot be placed using slub_max_order.
3878 */
Matthew Wilcox (Oracle)d1220192021-10-04 14:45:51 +01003879 order = calc_slab_order(size, 1, MAX_ORDER, 1);
David Rientjes818cf592009-04-23 09:58:22 +03003880 if (order < MAX_ORDER)
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003881 return order;
3882 return -ENOSYS;
3883}
3884
Pekka Enberg5595cff2008-08-05 09:28:47 +03003885static void
Joonsoo Kim40534972012-05-11 00:50:47 +09003886init_kmem_cache_node(struct kmem_cache_node *n)
Christoph Lameter81819f02007-05-06 14:49:36 -07003887{
3888 n->nr_partial = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07003889 spin_lock_init(&n->list_lock);
3890 INIT_LIST_HEAD(&n->partial);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003891#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter0f389ec2008-04-14 18:53:02 +03003892 atomic_long_set(&n->nr_slabs, 0);
Salman Qazi02b71b72008-09-11 12:25:41 -07003893 atomic_long_set(&n->total_objects, 0);
Christoph Lameter643b1132007-05-06 14:49:42 -07003894 INIT_LIST_HEAD(&n->full);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003895#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003896}
3897
Christoph Lameter55136592010-08-20 12:37:13 -05003898static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003899{
Christoph Lameter6c182dc2010-08-20 12:37:14 -05003900 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
Christoph Lameter95a05b42013-01-10 19:14:19 +00003901 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003902
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003903 /*
Chris Metcalfd4d84fe2011-06-02 10:19:41 -04003904 * Must align to double word boundary for the double cmpxchg
3905 * instructions to work; see __pcpu_double_call_return_bool().
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003906 */
Chris Metcalfd4d84fe2011-06-02 10:19:41 -04003907 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
3908 2 * sizeof(void *));
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003909
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003910 if (!s->cpu_slab)
3911 return 0;
3912
3913 init_kmem_cache_cpus(s);
3914
3915 return 1;
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003916}
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003917
Christoph Lameter51df1142010-08-20 12:37:15 -05003918static struct kmem_cache *kmem_cache_node;
3919
Christoph Lameter81819f02007-05-06 14:49:36 -07003920/*
3921 * No kmalloc_node yet so do it by hand. We know that this is the first
3922 * slab on the node for this slabcache. There are no concurrent accesses
3923 * possible.
3924 *
Zhi Yong Wu721ae222013-11-08 20:47:37 +08003925 * Note that this function only works on the kmem_cache_node
3926 * when allocating for the kmem_cache_node. This is used for bootstrapping
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003927 * memory on a fresh node that has no slab structures yet.
Christoph Lameter81819f02007-05-06 14:49:36 -07003928 */
Christoph Lameter55136592010-08-20 12:37:13 -05003929static void early_kmem_cache_node_alloc(int node)
Christoph Lameter81819f02007-05-06 14:49:36 -07003930{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003931 struct slab *slab;
Christoph Lameter81819f02007-05-06 14:49:36 -07003932 struct kmem_cache_node *n;
3933
Christoph Lameter51df1142010-08-20 12:37:15 -05003934 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
Christoph Lameter81819f02007-05-06 14:49:36 -07003935
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003936 slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
Christoph Lameter81819f02007-05-06 14:49:36 -07003937
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003938 BUG_ON(!slab);
3939 if (slab_nid(slab) != node) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07003940 pr_err("SLUB: Unable to allocate memory from node %d\n", node);
3941 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
Christoph Lametera2f92ee2007-08-22 14:01:57 -07003942 }
3943
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003944 n = slab->freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07003945 BUG_ON(!n);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003946#ifdef CONFIG_SLUB_DEBUG
Christoph Lameterf7cb1932010-09-29 07:15:01 -05003947 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
Christoph Lameter51df1142010-08-20 12:37:15 -05003948 init_tracking(kmem_cache_node, n);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003949#endif
Andrey Konovalovda844b72021-04-29 23:00:06 -07003950 n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003951 slab->freelist = get_freepointer(kmem_cache_node, n);
3952 slab->inuse = 1;
3953 slab->frozen = 0;
Andrey Konovalov12b22382018-12-28 00:29:41 -08003954 kmem_cache_node->node[node] = n;
Joonsoo Kim40534972012-05-11 00:50:47 +09003955 init_kmem_cache_node(n);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003956 inc_slabs_node(kmem_cache_node, node, slab->objects);
Christoph Lameter6446faa2008-02-15 23:45:26 -08003957
Dave Hansen67b6c902014-01-24 07:20:23 -08003958 /*
Steven Rostedt1e4dd942014-02-10 14:25:46 -08003959 * No locks need to be taken here as it has just been
3960 * initialized and there is no concurrent access.
Dave Hansen67b6c902014-01-24 07:20:23 -08003961 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01003962 __add_partial(n, slab, DEACTIVATE_TO_HEAD);
Christoph Lameter81819f02007-05-06 14:49:36 -07003963}
3964
3965static void free_kmem_cache_nodes(struct kmem_cache *s)
3966{
3967 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003968 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07003969
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003970 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003971 s->node[node] = NULL;
Alexander Potapenkoea37df52017-09-06 16:19:15 -07003972 kmem_cache_free(kmem_cache_node, n);
Christoph Lameter81819f02007-05-06 14:49:36 -07003973 }
3974}
3975
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003976void __kmem_cache_release(struct kmem_cache *s)
3977{
Thomas Garnier210e7a42016-07-26 15:21:59 -07003978 cache_random_seq_destroy(s);
Dmitry Safonov52b4b952016-02-17 13:11:37 -08003979 free_percpu(s->cpu_slab);
3980 free_kmem_cache_nodes(s);
3981}
3982
Christoph Lameter55136592010-08-20 12:37:13 -05003983static int init_kmem_cache_nodes(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07003984{
3985 int node;
Christoph Lameter81819f02007-05-06 14:49:36 -07003986
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08003987 for_each_node_mask(node, slab_nodes) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003988 struct kmem_cache_node *n;
3989
Alexander Duyck73367bd2010-05-21 14:41:35 -07003990 if (slab_state == DOWN) {
Christoph Lameter55136592010-08-20 12:37:13 -05003991 early_kmem_cache_node_alloc(node);
Alexander Duyck73367bd2010-05-21 14:41:35 -07003992 continue;
Christoph Lameter81819f02007-05-06 14:49:36 -07003993 }
Christoph Lameter51df1142010-08-20 12:37:15 -05003994 n = kmem_cache_alloc_node(kmem_cache_node,
Christoph Lameter55136592010-08-20 12:37:13 -05003995 GFP_KERNEL, node);
Alexander Duyck73367bd2010-05-21 14:41:35 -07003996
3997 if (!n) {
3998 free_kmem_cache_nodes(s);
3999 return 0;
4000 }
4001
Joonsoo Kim40534972012-05-11 00:50:47 +09004002 init_kmem_cache_node(n);
Alexander Potapenkoea37df52017-09-06 16:19:15 -07004003 s->node[node] = n;
Christoph Lameter81819f02007-05-06 14:49:36 -07004004 }
4005 return 1;
4006}
Christoph Lameter81819f02007-05-06 14:49:36 -07004007
Wei Yange6d0e1d2017-07-06 15:36:34 -07004008static void set_cpu_partial(struct kmem_cache *s)
4009{
4010#ifdef CONFIG_SLUB_CPU_PARTIAL
Vlastimil Babkab47291e2021-11-05 13:35:17 -07004011 unsigned int nr_objects;
4012
Wei Yange6d0e1d2017-07-06 15:36:34 -07004013 /*
4014 * cpu_partial determined the maximum number of objects kept in the
4015 * per cpu partial lists of a processor.
4016 *
4017 * Per cpu partial lists mainly contain slabs that just have one
4018 * object freed. If they are used for allocation then they can be
4019 * filled up again with minimal effort. The slab will never hit the
4020 * per node partial lists and therefore no locking will be required.
4021 *
Vlastimil Babkab47291e2021-11-05 13:35:17 -07004022 * For backwards compatibility reasons, this is determined as number
4023 * of objects, even though we now limit maximum number of pages, see
4024 * slub_set_cpu_partial()
Wei Yange6d0e1d2017-07-06 15:36:34 -07004025 */
4026 if (!kmem_cache_has_cpu_partial(s))
Vlastimil Babkab47291e2021-11-05 13:35:17 -07004027 nr_objects = 0;
Wei Yange6d0e1d2017-07-06 15:36:34 -07004028 else if (s->size >= PAGE_SIZE)
Vlastimil Babkab47291e2021-11-05 13:35:17 -07004029 nr_objects = 6;
Wei Yange6d0e1d2017-07-06 15:36:34 -07004030 else if (s->size >= 1024)
Vlastimil Babka23e98ad2021-11-05 13:35:20 -07004031 nr_objects = 24;
Wei Yange6d0e1d2017-07-06 15:36:34 -07004032 else if (s->size >= 256)
Vlastimil Babka23e98ad2021-11-05 13:35:20 -07004033 nr_objects = 52;
Wei Yange6d0e1d2017-07-06 15:36:34 -07004034 else
Vlastimil Babka23e98ad2021-11-05 13:35:20 -07004035 nr_objects = 120;
Vlastimil Babkab47291e2021-11-05 13:35:17 -07004036
4037 slub_set_cpu_partial(s, nr_objects);
Wei Yange6d0e1d2017-07-06 15:36:34 -07004038#endif
4039}
4040
Christoph Lameter81819f02007-05-06 14:49:36 -07004041/*
4042 * calculate_sizes() determines the order and the distribution of data within
4043 * a slab object.
4044 */
Miaohe Linae44d812022-03-09 17:20:36 +08004045static int calculate_sizes(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07004046{
Alexey Dobriyand50112e2017-11-15 17:32:18 -08004047 slab_flags_t flags = s->flags;
Alexey Dobriyanbe4a7982018-04-05 16:21:28 -07004048 unsigned int size = s->object_size;
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004049 unsigned int order;
Christoph Lameter81819f02007-05-06 14:49:36 -07004050
4051 /*
Christoph Lameterd8b42bf2008-02-15 23:45:25 -08004052 * Round up object size to the next word boundary. We can only
4053 * place the free pointer at word boundaries and this determines
4054 * the possible location of the free pointer.
4055 */
4056 size = ALIGN(size, sizeof(void *));
4057
4058#ifdef CONFIG_SLUB_DEBUG
4059 /*
Christoph Lameter81819f02007-05-06 14:49:36 -07004060 * Determine if we can poison the object itself. If the user of
4061 * the slab may touch the object after free or before allocation
4062 * then we should never poison the object itself.
4063 */
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -08004064 if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
Christoph Lameterc59def92007-05-16 22:10:50 -07004065 !s->ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07004066 s->flags |= __OBJECT_POISON;
4067 else
4068 s->flags &= ~__OBJECT_POISON;
4069
Christoph Lameter81819f02007-05-06 14:49:36 -07004070
4071 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07004072 * If we are Redzoning then check if there is some space between the
Christoph Lameter81819f02007-05-06 14:49:36 -07004073 * end of the object and the free pointer. If not then add an
Christoph Lameter672bba32007-05-09 02:32:39 -07004074 * additional word to have some bytes to store Redzone information.
Christoph Lameter81819f02007-05-06 14:49:36 -07004075 */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05004076 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
Christoph Lameter81819f02007-05-06 14:49:36 -07004077 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07004078#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004079
4080 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07004081 * With that we have determined the number of bytes in actual use
Kees Cooke41a49fa2021-06-15 18:23:26 -07004082 * by the object and redzoning.
Christoph Lameter81819f02007-05-06 14:49:36 -07004083 */
4084 s->inuse = size;
4085
Kees Cook74c1d3e2021-06-15 18:23:22 -07004086 if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
4087 ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
4088 s->ctor) {
Christoph Lameter81819f02007-05-06 14:49:36 -07004089 /*
4090 * Relocate free pointer after the object if it is not
4091 * permitted to overwrite the first word of the object on
4092 * kmem_cache_free.
4093 *
4094 * This is the case if we do RCU, have a constructor or
Kees Cook74c1d3e2021-06-15 18:23:22 -07004095 * destructor, are poisoning the objects, or are
4096 * redzoning an object smaller than sizeof(void *).
Waiman Longcbfc35a2020-05-07 18:36:06 -07004097 *
4098 * The assumption that s->offset >= s->inuse means free
4099 * pointer is outside of the object is used in the
4100 * freeptr_outside_object() function. If that is no
4101 * longer true, the function needs to be modified.
Christoph Lameter81819f02007-05-06 14:49:36 -07004102 */
4103 s->offset = size;
4104 size += sizeof(void *);
Kees Cooke41a49fa2021-06-15 18:23:26 -07004105 } else {
Kees Cook3202fa62020-04-01 21:04:27 -07004106 /*
4107 * Store freelist pointer near middle of object to keep
4108 * it away from the edges of the object to avoid small
4109 * sized over/underflows from neighboring allocations.
4110 */
Kees Cooke41a49fa2021-06-15 18:23:26 -07004111 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
Christoph Lameter81819f02007-05-06 14:49:36 -07004112 }
4113
Christoph Lameterc12b3c62007-05-23 13:57:31 -07004114#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07004115 if (flags & SLAB_STORE_USER)
4116 /*
4117 * Need to store information about allocs and frees after
4118 * the object.
4119 */
4120 size += 2 * sizeof(struct track);
Alexander Potapenko80a92012016-07-28 15:49:07 -07004121#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004122
Alexander Potapenko80a92012016-07-28 15:49:07 -07004123 kasan_cache_create(s, &size, &s->flags);
4124#ifdef CONFIG_SLUB_DEBUG
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -07004125 if (flags & SLAB_RED_ZONE) {
Christoph Lameter81819f02007-05-06 14:49:36 -07004126 /*
4127 * Add some empty padding so that we can catch
4128 * overwrites from earlier objects rather than let
4129 * tracking information or the free pointer be
Frederik Schwarzer0211a9c82008-12-29 22:14:56 +01004130 * corrupted if a user writes before the start
Christoph Lameter81819f02007-05-06 14:49:36 -07004131 * of the object.
4132 */
4133 size += sizeof(void *);
Joonsoo Kimd86bd1b2016-03-15 14:55:12 -07004134
4135 s->red_left_pad = sizeof(void *);
4136 s->red_left_pad = ALIGN(s->red_left_pad, s->align);
4137 size += s->red_left_pad;
4138 }
Christoph Lameter41ecc552007-05-09 02:32:44 -07004139#endif
Christoph Lameter672bba32007-05-09 02:32:39 -07004140
Christoph Lameter81819f02007-05-06 14:49:36 -07004141 /*
Christoph Lameter81819f02007-05-06 14:49:36 -07004142 * SLUB stores one object immediately after another beginning from
4143 * offset 0. In order to align the objects we have to simply size
4144 * each object to conform to the alignment.
4145 */
Christoph Lameter45906852012-11-28 16:23:16 +00004146 size = ALIGN(size, s->align);
Christoph Lameter81819f02007-05-06 14:49:36 -07004147 s->size = size;
Roman Gushchin4138fdf2020-08-06 23:20:42 -07004148 s->reciprocal_size = reciprocal_value(size);
Miaohe Linae44d812022-03-09 17:20:36 +08004149 order = calculate_order(size);
Christoph Lameter81819f02007-05-06 14:49:36 -07004150
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004151 if ((int)order < 0)
Christoph Lameter81819f02007-05-06 14:49:36 -07004152 return 0;
4153
Christoph Lameterb7a49f02008-02-14 14:21:32 -08004154 s->allocflags = 0;
Christoph Lameter834f3d12008-04-14 19:11:31 +03004155 if (order)
Christoph Lameterb7a49f02008-02-14 14:21:32 -08004156 s->allocflags |= __GFP_COMP;
4157
4158 if (s->flags & SLAB_CACHE_DMA)
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004159 s->allocflags |= GFP_DMA;
Christoph Lameterb7a49f02008-02-14 14:21:32 -08004160
Nicolas Boichat6d6ea1e2019-03-28 20:43:42 -07004161 if (s->flags & SLAB_CACHE_DMA32)
4162 s->allocflags |= GFP_DMA32;
4163
Christoph Lameterb7a49f02008-02-14 14:21:32 -08004164 if (s->flags & SLAB_RECLAIM_ACCOUNT)
4165 s->allocflags |= __GFP_RECLAIMABLE;
4166
Christoph Lameter81819f02007-05-06 14:49:36 -07004167 /*
4168 * Determine the number of objects per slab
4169 */
Matthew Wilcox9736d2a2018-06-07 17:09:10 -07004170 s->oo = oo_make(order, size);
4171 s->min = oo_make(get_order(size), size);
Christoph Lameter81819f02007-05-06 14:49:36 -07004172
Christoph Lameter834f3d12008-04-14 19:11:31 +03004173 return !!oo_objects(s->oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07004174}
4175
Alexey Dobriyand50112e2017-11-15 17:32:18 -08004176static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07004177{
Nikolay Borisov37540002021-02-24 12:00:58 -08004178 s->flags = kmem_cache_flags(s->size, flags, s->name);
Kees Cook2482ddec2017-09-06 16:19:18 -07004179#ifdef CONFIG_SLAB_FREELIST_HARDENED
4180 s->random = get_random_long();
4181#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004182
Miaohe Linae44d812022-03-09 17:20:36 +08004183 if (!calculate_sizes(s))
Christoph Lameter81819f02007-05-06 14:49:36 -07004184 goto error;
David Rientjes3de47212009-07-27 18:30:35 -07004185 if (disable_higher_order_debug) {
4186 /*
4187 * Disable debugging flags that store metadata if the min slab
4188 * order increased.
4189 */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05004190 if (get_order(s->size) > get_order(s->object_size)) {
David Rientjes3de47212009-07-27 18:30:35 -07004191 s->flags &= ~DEBUG_METADATA_FLAGS;
4192 s->offset = 0;
Miaohe Linae44d812022-03-09 17:20:36 +08004193 if (!calculate_sizes(s))
David Rientjes3de47212009-07-27 18:30:35 -07004194 goto error;
4195 }
4196 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004197
Heiko Carstens25654092012-01-12 17:17:33 -08004198#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
4199 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Laura Abbott149daaf2016-03-15 14:55:09 -07004200 if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
Christoph Lameterb789ef52011-06-01 12:25:49 -05004201 /* Enable fast mode */
4202 s->flags |= __CMPXCHG_DOUBLE;
4203#endif
4204
David Rientjes3b89d7d2009-02-22 17:40:07 -08004205 /*
Vlastimil Babkac2092c12021-11-15 16:55:15 +01004206 * The larger the object size is, the more slabs we want on the partial
David Rientjes3b89d7d2009-02-22 17:40:07 -08004207 * list to avoid pounding the page allocator excessively.
4208 */
Hyeonggon Yoo5182f3c2022-03-07 07:40:55 +00004209 s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
4210 s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
Christoph Lameter49e22582011-08-09 16:12:27 -05004211
Wei Yange6d0e1d2017-07-06 15:36:34 -07004212 set_cpu_partial(s);
Christoph Lameter49e22582011-08-09 16:12:27 -05004213
Christoph Lameter81819f02007-05-06 14:49:36 -07004214#ifdef CONFIG_NUMA
Christoph Lametere2cb96b2008-08-19 08:51:22 -05004215 s->remote_node_defrag_ratio = 1000;
Christoph Lameter81819f02007-05-06 14:49:36 -07004216#endif
Thomas Garnier210e7a42016-07-26 15:21:59 -07004217
4218 /* Initialize the pre-computed randomized freelist if slab is up */
4219 if (slab_state >= UP) {
4220 if (init_cache_random_seq(s))
4221 goto error;
4222 }
4223
Christoph Lameter55136592010-08-20 12:37:13 -05004224 if (!init_kmem_cache_nodes(s))
Christoph Lameterdfb4f092007-10-16 01:26:05 -07004225 goto error;
Christoph Lameter81819f02007-05-06 14:49:36 -07004226
Christoph Lameter55136592010-08-20 12:37:13 -05004227 if (alloc_kmem_cache_cpus(s))
Christoph Lameter278b1bb2012-09-05 00:20:34 +00004228 return 0;
Christoph Lameterff120592009-12-18 16:26:22 -06004229
Christoph Lameter81819f02007-05-06 14:49:36 -07004230error:
Miaohe Lin9037c572021-10-18 15:15:59 -07004231 __kmem_cache_release(s);
Christoph Lameter278b1bb2012-09-05 00:20:34 +00004232 return -EINVAL;
Christoph Lameter81819f02007-05-06 14:49:36 -07004233}
Christoph Lameter81819f02007-05-06 14:49:36 -07004234
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004235static void list_slab_objects(struct kmem_cache *s, struct slab *slab,
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07004236 const char *text)
Christoph Lameter81819f02007-05-06 14:49:36 -07004237{
Christoph Lameter33b12c32008-04-25 12:22:43 -07004238#ifdef CONFIG_SLUB_DEBUG
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004239 void *addr = slab_address(slab);
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +02004240 unsigned long flags;
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07004241 unsigned long *map;
Christoph Lameter33b12c32008-04-25 12:22:43 -07004242 void *p;
Christopher Lameteraa456c72020-06-01 21:45:53 -07004243
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004244 slab_err(s, slab, text, s->name);
4245 slab_lock(slab, &flags);
Christoph Lameter33b12c32008-04-25 12:22:43 -07004246
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004247 map = get_map(s, slab);
4248 for_each_object(p, s, addr, slab->objects) {
Christoph Lameter33b12c32008-04-25 12:22:43 -07004249
Roman Gushchin4138fdf2020-08-06 23:20:42 -07004250 if (!test_bit(__obj_to_index(s, addr, p), map)) {
Yafang Shao96b94ab2021-03-19 18:12:45 +08004251 pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
Christoph Lameter33b12c32008-04-25 12:22:43 -07004252 print_tracking(s, p);
4253 }
4254 }
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07004255 put_map(map);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004256 slab_unlock(slab, &flags);
Christoph Lameter33b12c32008-04-25 12:22:43 -07004257#endif
4258}
4259
Christoph Lameter81819f02007-05-06 14:49:36 -07004260/*
Christoph Lameter599870b2008-04-23 12:36:52 -07004261 * Attempt to free all partial slabs on a node.
Dmitry Safonov52b4b952016-02-17 13:11:37 -08004262 * This is called from __kmem_cache_shutdown(). We must take list_lock
4263 * because sysfs file might still access partial list after the shutdowning.
Christoph Lameter81819f02007-05-06 14:49:36 -07004264 */
Christoph Lameter599870b2008-04-23 12:36:52 -07004265static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
Christoph Lameter81819f02007-05-06 14:49:36 -07004266{
Chris Wilson60398922016-08-10 16:27:58 -07004267 LIST_HEAD(discard);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004268 struct slab *slab, *h;
Christoph Lameter81819f02007-05-06 14:49:36 -07004269
Dmitry Safonov52b4b952016-02-17 13:11:37 -08004270 BUG_ON(irqs_disabled());
4271 spin_lock_irq(&n->list_lock);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004272 list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
4273 if (!slab->inuse) {
4274 remove_partial(n, slab);
4275 list_add(&slab->slab_list, &discard);
Christoph Lameter33b12c32008-04-25 12:22:43 -07004276 } else {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004277 list_slab_objects(s, slab,
Sebastian Andrzej Siewior55860d92020-06-25 20:29:55 -07004278 "Objects remaining in %s on __kmem_cache_shutdown()");
Christoph Lameter599870b2008-04-23 12:36:52 -07004279 }
Christoph Lameter33b12c32008-04-25 12:22:43 -07004280 }
Dmitry Safonov52b4b952016-02-17 13:11:37 -08004281 spin_unlock_irq(&n->list_lock);
Chris Wilson60398922016-08-10 16:27:58 -07004282
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004283 list_for_each_entry_safe(slab, h, &discard, slab_list)
4284 discard_slab(s, slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07004285}
4286
Shakeel Buttf9e13c02018-04-05 16:21:57 -07004287bool __kmem_cache_empty(struct kmem_cache *s)
4288{
4289 int node;
4290 struct kmem_cache_node *n;
4291
4292 for_each_kmem_cache_node(s, node, n)
4293 if (n->nr_partial || slabs_node(s, node))
4294 return false;
4295 return true;
4296}
4297
Christoph Lameter81819f02007-05-06 14:49:36 -07004298/*
Christoph Lameter672bba32007-05-09 02:32:39 -07004299 * Release all resources used by a slab cache.
Christoph Lameter81819f02007-05-06 14:49:36 -07004300 */
Dmitry Safonov52b4b952016-02-17 13:11:37 -08004301int __kmem_cache_shutdown(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07004302{
4303 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004304 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07004305
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01004306 flush_all_cpus_locked(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07004307 /* Attempt to free all objects */
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004308 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter599870b2008-04-23 12:36:52 -07004309 free_partial(s, n);
4310 if (n->nr_partial || slabs_node(s, node))
Christoph Lameter81819f02007-05-06 14:49:36 -07004311 return 1;
4312 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004313 return 0;
4314}
4315
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -08004316#ifdef CONFIG_PRINTK
Marco Elver2dfe63e2022-04-14 19:13:40 -07004317void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004318{
4319 void *base;
4320 int __maybe_unused i;
4321 unsigned int objnr;
4322 void *objp;
4323 void *objp0;
Matthew Wilcox (Oracle)72132302021-10-04 14:45:55 +01004324 struct kmem_cache *s = slab->slab_cache;
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004325 struct track __maybe_unused *trackp;
4326
4327 kpp->kp_ptr = object;
Matthew Wilcox (Oracle)72132302021-10-04 14:45:55 +01004328 kpp->kp_slab = slab;
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004329 kpp->kp_slab_cache = s;
Matthew Wilcox (Oracle)72132302021-10-04 14:45:55 +01004330 base = slab_address(slab);
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004331 objp0 = kasan_reset_tag(object);
4332#ifdef CONFIG_SLUB_DEBUG
4333 objp = restore_red_left(s, objp0);
4334#else
4335 objp = objp0;
4336#endif
Vlastimil Babka40f3bf02021-11-02 15:42:04 +01004337 objnr = obj_to_index(s, slab, objp);
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004338 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
4339 objp = base + s->size * objnr;
4340 kpp->kp_objp = objp;
Matthew Wilcox (Oracle)72132302021-10-04 14:45:55 +01004341 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
4342 || (objp - base) % s->size) ||
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004343 !(s->flags & SLAB_STORE_USER))
4344 return;
4345#ifdef CONFIG_SLUB_DEBUG
Maninder Singh0cbc1242021-03-16 16:07:10 +05304346 objp = fixup_red_left(s, objp);
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004347 trackp = get_track(s, objp, TRACK_ALLOC);
4348 kpp->kp_ret = (void *)trackp->addr;
Oliver Glitta5cf909c2021-07-07 18:07:47 -07004349#ifdef CONFIG_STACKDEPOT
4350 {
4351 depot_stack_handle_t handle;
4352 unsigned long *entries;
4353 unsigned int nr_entries;
Maninder Singhe548eaa2021-03-16 16:07:11 +05304354
Oliver Glitta5cf909c2021-07-07 18:07:47 -07004355 handle = READ_ONCE(trackp->handle);
4356 if (handle) {
4357 nr_entries = stack_depot_fetch(handle, &entries);
4358 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
4359 kpp->kp_stack[i] = (void *)entries[i];
4360 }
4361
4362 trackp = get_track(s, objp, TRACK_FREE);
4363 handle = READ_ONCE(trackp->handle);
4364 if (handle) {
4365 nr_entries = stack_depot_fetch(handle, &entries);
4366 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
4367 kpp->kp_free_stack[i] = (void *)entries[i];
4368 }
Maninder Singhe548eaa2021-03-16 16:07:11 +05304369 }
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004370#endif
4371#endif
4372}
Paul E. McKenney5bb1bb32021-01-07 13:46:11 -08004373#endif
Paul E. McKenney8e7f37f2020-12-07 17:41:02 -08004374
Christoph Lameter81819f02007-05-06 14:49:36 -07004375/********************************************************************
4376 * Kmalloc subsystem
4377 *******************************************************************/
4378
Christoph Lameter81819f02007-05-06 14:49:36 -07004379static int __init setup_slub_min_order(char *str)
4380{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004381 get_option(&str, (int *)&slub_min_order);
Christoph Lameter81819f02007-05-06 14:49:36 -07004382
4383 return 1;
4384}
4385
4386__setup("slub_min_order=", setup_slub_min_order);
4387
4388static int __init setup_slub_max_order(char *str)
4389{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004390 get_option(&str, (int *)&slub_max_order);
4391 slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004392
4393 return 1;
4394}
4395
4396__setup("slub_max_order=", setup_slub_max_order);
4397
4398static int __init setup_slub_min_objects(char *str)
4399{
Alexey Dobriyan19af27a2018-04-05 16:21:39 -07004400 get_option(&str, (int *)&slub_min_objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07004401
4402 return 1;
4403}
4404
4405__setup("slub_min_objects=", setup_slub_min_objects);
4406
Christoph Lameter81819f02007-05-06 14:49:36 -07004407void *__kmalloc(size_t size, gfp_t flags)
4408{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004409 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004410 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004411
Christoph Lameter95a05b42013-01-10 19:14:19 +00004412 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02004413 return kmalloc_large(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004414
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004415 s = kmalloc_slab(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004416
4417 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004418 return s;
4419
Muchun Song88f2ef72022-03-22 14:40:56 -07004420 ret = slab_alloc(s, NULL, flags, _RET_IP_, size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004421
Vasily Averinb347aa72022-06-03 06:21:49 +03004422 trace_kmalloc(_RET_IP_, ret, s, size, s->size, flags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004423
Andrey Konovalov0116523c2018-12-28 00:29:37 -08004424 ret = kasan_kmalloc(s, ret, size, flags);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08004425
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004426 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004427}
4428EXPORT_SYMBOL(__kmalloc);
4429
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004430#ifdef CONFIG_NUMA
Christoph Lameterf619cfe2008-03-01 13:56:40 -08004431static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
4432{
Vegard Nossumb1eeab62008-11-25 16:55:53 +01004433 struct page *page;
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01004434 void *ptr = NULL;
Vlastimil Babka6a486c02019-10-06 17:58:42 -07004435 unsigned int order = get_order(size);
Christoph Lameterf619cfe2008-03-01 13:56:40 -08004436
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -08004437 flags |= __GFP_COMP;
Vlastimil Babka6a486c02019-10-06 17:58:42 -07004438 page = alloc_pages_node(node, flags, order);
4439 if (page) {
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01004440 ptr = page_address(page);
Muchun Song96403bf2021-02-24 12:04:26 -08004441 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
4442 PAGE_SIZE << order);
Vlastimil Babka6a486c02019-10-06 17:58:42 -07004443 }
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01004444
Andrey Konovalov0116523c2018-12-28 00:29:37 -08004445 return kmalloc_large_node_hook(ptr, size, flags);
Christoph Lameterf619cfe2008-03-01 13:56:40 -08004446}
4447
Christoph Lameter81819f02007-05-06 14:49:36 -07004448void *__kmalloc_node(size_t size, gfp_t flags, int node)
4449{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004450 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004451 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004452
Christoph Lameter95a05b42013-01-10 19:14:19 +00004453 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004454 ret = kmalloc_large_node(size, flags, node);
4455
Vasily Averinb347aa72022-06-03 06:21:49 +03004456 trace_kmalloc_node(_RET_IP_, ret, NULL,
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004457 size, PAGE_SIZE << get_order(size),
4458 flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004459
4460 return ret;
4461 }
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004462
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004463 s = kmalloc_slab(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004464
4465 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004466 return s;
4467
Muchun Song88f2ef72022-03-22 14:40:56 -07004468 ret = slab_alloc_node(s, NULL, flags, node, _RET_IP_, size);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004469
Vasily Averinb347aa72022-06-03 06:21:49 +03004470 trace_kmalloc_node(_RET_IP_, ret, s, size, s->size, flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004471
Andrey Konovalov0116523c2018-12-28 00:29:37 -08004472 ret = kasan_kmalloc(s, ret, size, flags);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08004473
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03004474 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004475}
4476EXPORT_SYMBOL(__kmalloc_node);
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07004477#endif /* CONFIG_NUMA */
Christoph Lameter81819f02007-05-06 14:49:36 -07004478
Kees Cooked18adc2016-06-23 15:24:05 -07004479#ifdef CONFIG_HARDENED_USERCOPY
4480/*
Kees Cookafcc90f82018-01-10 15:17:01 -08004481 * Rejects incorrectly sized objects and objects that are to be copied
4482 * to/from userspace but do not fall entirely within the containing slab
4483 * cache's usercopy region.
Kees Cooked18adc2016-06-23 15:24:05 -07004484 *
4485 * Returns NULL if check passes, otherwise const char * to name of cache
4486 * to indicate an error.
4487 */
Matthew Wilcox (Oracle)0b3eb0912021-10-04 14:45:56 +01004488void __check_heap_object(const void *ptr, unsigned long n,
4489 const struct slab *slab, bool to_user)
Kees Cooked18adc2016-06-23 15:24:05 -07004490{
4491 struct kmem_cache *s;
Alexey Dobriyan44065b22018-04-05 16:21:20 -07004492 unsigned int offset;
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004493 bool is_kfence = is_kfence_address(ptr);
Kees Cooked18adc2016-06-23 15:24:05 -07004494
Andrey Konovalov96fedce2019-01-08 15:23:15 -08004495 ptr = kasan_reset_tag(ptr);
4496
Kees Cooked18adc2016-06-23 15:24:05 -07004497 /* Find object and usable object size. */
Matthew Wilcox (Oracle)0b3eb0912021-10-04 14:45:56 +01004498 s = slab->slab_cache;
Kees Cooked18adc2016-06-23 15:24:05 -07004499
4500 /* Reject impossible pointers. */
Matthew Wilcox (Oracle)0b3eb0912021-10-04 14:45:56 +01004501 if (ptr < slab_address(slab))
Kees Cookf4e6e282018-01-10 14:48:22 -08004502 usercopy_abort("SLUB object not in SLUB page?!", NULL,
4503 to_user, 0, n);
Kees Cooked18adc2016-06-23 15:24:05 -07004504
4505 /* Find offset within object. */
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004506 if (is_kfence)
4507 offset = ptr - kfence_object_start(ptr);
4508 else
Matthew Wilcox (Oracle)0b3eb0912021-10-04 14:45:56 +01004509 offset = (ptr - slab_address(slab)) % s->size;
Kees Cooked18adc2016-06-23 15:24:05 -07004510
4511 /* Adjust for redzone and reject if within the redzone. */
Alexander Potapenkob89fb5e2021-02-25 17:19:16 -08004512 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
Kees Cooked18adc2016-06-23 15:24:05 -07004513 if (offset < s->red_left_pad)
Kees Cookf4e6e282018-01-10 14:48:22 -08004514 usercopy_abort("SLUB object in left red zone",
4515 s->name, to_user, offset, n);
Kees Cooked18adc2016-06-23 15:24:05 -07004516 offset -= s->red_left_pad;
4517 }
4518
Kees Cookafcc90f82018-01-10 15:17:01 -08004519 /* Allow address range falling entirely within usercopy region. */
4520 if (offset >= s->useroffset &&
4521 offset - s->useroffset <= s->usersize &&
4522 n <= s->useroffset - offset + s->usersize)
Kees Cookf4e6e282018-01-10 14:48:22 -08004523 return;
Kees Cooked18adc2016-06-23 15:24:05 -07004524
Kees Cookf4e6e282018-01-10 14:48:22 -08004525 usercopy_abort("SLUB object", s->name, to_user, offset, n);
Kees Cooked18adc2016-06-23 15:24:05 -07004526}
4527#endif /* CONFIG_HARDENED_USERCOPY */
4528
Marco Elver10d1f8c2019-07-11 20:54:14 -07004529size_t __ksize(const void *object)
Christoph Lameter81819f02007-05-06 14:49:36 -07004530{
Matthew Wilcox (Oracle)0c248112021-10-04 14:45:54 +01004531 struct folio *folio;
Christoph Lameter81819f02007-05-06 14:49:36 -07004532
Christoph Lameteref8b4522007-10-16 01:24:46 -07004533 if (unlikely(object == ZERO_SIZE_PTR))
Christoph Lameter272c1d22007-06-08 13:46:49 -07004534 return 0;
4535
Matthew Wilcox (Oracle)0c248112021-10-04 14:45:54 +01004536 folio = virt_to_folio(object);
Vegard Nossum294a80a2007-12-04 23:45:30 -08004537
Matthew Wilcox (Oracle)0c248112021-10-04 14:45:54 +01004538 if (unlikely(!folio_test_slab(folio)))
4539 return folio_size(folio);
Christoph Lameter81819f02007-05-06 14:49:36 -07004540
Matthew Wilcox (Oracle)0c248112021-10-04 14:45:54 +01004541 return slab_ksize(folio_slab(folio)->slab_cache);
Christoph Lameter81819f02007-05-06 14:49:36 -07004542}
Marco Elver10d1f8c2019-07-11 20:54:14 -07004543EXPORT_SYMBOL(__ksize);
Christoph Lameter81819f02007-05-06 14:49:36 -07004544
4545void kfree(const void *x)
4546{
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01004547 struct folio *folio;
4548 struct slab *slab;
Christoph Lameter5bb983b2008-02-07 17:47:41 -08004549 void *object = (void *)x;
Christoph Lameter81819f02007-05-06 14:49:36 -07004550
Pekka Enberg2121db72009-03-25 11:05:57 +02004551 trace_kfree(_RET_IP_, x);
4552
Satyam Sharma2408c552007-10-16 01:24:44 -07004553 if (unlikely(ZERO_OR_NULL_PTR(x)))
Christoph Lameter81819f02007-05-06 14:49:36 -07004554 return;
4555
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01004556 folio = virt_to_folio(x);
4557 if (unlikely(!folio_test_slab(folio))) {
4558 free_large_kmalloc(folio, object);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004559 return;
4560 }
Matthew Wilcox (Oracle)d835eef2021-10-04 14:45:59 +01004561 slab = folio_slab(folio);
Muchun Songb77d5b12022-04-29 20:30:44 +08004562 slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_);
Christoph Lameter81819f02007-05-06 14:49:36 -07004563}
4564EXPORT_SYMBOL(kfree);
4565
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004566#define SHRINK_PROMOTE_MAX 32
4567
Christoph Lameter2086d262007-05-06 14:49:46 -07004568/*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004569 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
4570 * up most to the head of the partial lists. New allocations will then
4571 * fill those up and thus they can be removed from the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -07004572 *
4573 * The slabs with the least items are placed last. This results in them
4574 * being allocated from last increasing the chance that the last objects
4575 * are freed in them.
Christoph Lameter2086d262007-05-06 14:49:46 -07004576 */
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01004577static int __kmem_cache_do_shrink(struct kmem_cache *s)
Christoph Lameter2086d262007-05-06 14:49:46 -07004578{
4579 int node;
4580 int i;
4581 struct kmem_cache_node *n;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004582 struct slab *slab;
4583 struct slab *t;
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004584 struct list_head discard;
4585 struct list_head promote[SHRINK_PROMOTE_MAX];
Christoph Lameter2086d262007-05-06 14:49:46 -07004586 unsigned long flags;
Vladimir Davydovce3712d2015-02-12 14:59:44 -08004587 int ret = 0;
Christoph Lameter2086d262007-05-06 14:49:46 -07004588
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004589 for_each_kmem_cache_node(s, node, n) {
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004590 INIT_LIST_HEAD(&discard);
4591 for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
4592 INIT_LIST_HEAD(promote + i);
Christoph Lameter2086d262007-05-06 14:49:46 -07004593
4594 spin_lock_irqsave(&n->list_lock, flags);
4595
4596 /*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004597 * Build lists of slabs to discard or promote.
Christoph Lameter2086d262007-05-06 14:49:46 -07004598 *
Christoph Lameter672bba32007-05-09 02:32:39 -07004599 * Note that concurrent frees may occur while we hold the
Vlastimil Babkac2092c12021-11-15 16:55:15 +01004600 * list_lock. slab->inuse here is the upper limit.
Christoph Lameter2086d262007-05-06 14:49:46 -07004601 */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004602 list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
4603 int free = slab->objects - slab->inuse;
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004604
Vlastimil Babkac2092c12021-11-15 16:55:15 +01004605 /* Do not reread slab->inuse */
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004606 barrier();
4607
4608 /* We do not keep full slabs on the list */
4609 BUG_ON(free <= 0);
4610
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004611 if (free == slab->objects) {
4612 list_move(&slab->slab_list, &discard);
Christoph Lameter69cb8e62011-08-09 16:12:22 -05004613 n->nr_partial--;
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004614 } else if (free <= SHRINK_PROMOTE_MAX)
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004615 list_move(&slab->slab_list, promote + free - 1);
Christoph Lameter2086d262007-05-06 14:49:46 -07004616 }
4617
Christoph Lameter2086d262007-05-06 14:49:46 -07004618 /*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004619 * Promote the slabs filled up most to the head of the
4620 * partial list.
Christoph Lameter2086d262007-05-06 14:49:46 -07004621 */
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004622 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
4623 list_splice(promote + i, &n->partial);
Christoph Lameter2086d262007-05-06 14:49:46 -07004624
Christoph Lameter2086d262007-05-06 14:49:46 -07004625 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter69cb8e62011-08-09 16:12:22 -05004626
4627 /* Release empty slabs */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004628 list_for_each_entry_safe(slab, t, &discard, slab_list)
4629 discard_slab(s, slab);
Vladimir Davydovce3712d2015-02-12 14:59:44 -08004630
4631 if (slabs_node(s, node))
4632 ret = 1;
Christoph Lameter2086d262007-05-06 14:49:46 -07004633 }
4634
Vladimir Davydovce3712d2015-02-12 14:59:44 -08004635 return ret;
Christoph Lameter2086d262007-05-06 14:49:46 -07004636}
Christoph Lameter2086d262007-05-06 14:49:46 -07004637
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01004638int __kmem_cache_shrink(struct kmem_cache *s)
4639{
4640 flush_all(s);
4641 return __kmem_cache_do_shrink(s);
4642}
4643
Yasunori Gotob9049e22007-10-21 16:41:37 -07004644static int slab_mem_going_offline_callback(void *arg)
4645{
4646 struct kmem_cache *s;
4647
Christoph Lameter18004c52012-07-06 15:25:12 -05004648 mutex_lock(&slab_mutex);
Sebastian Andrzej Siewior5a836bf2021-02-26 17:11:55 +01004649 list_for_each_entry(s, &slab_caches, list) {
4650 flush_all_cpus_locked(s);
4651 __kmem_cache_do_shrink(s);
4652 }
Christoph Lameter18004c52012-07-06 15:25:12 -05004653 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004654
4655 return 0;
4656}
4657
4658static void slab_mem_offline_callback(void *arg)
4659{
Yasunori Gotob9049e22007-10-21 16:41:37 -07004660 struct memory_notify *marg = arg;
4661 int offline_node;
4662
Lai Jiangshanb9d5ab22012-12-11 16:01:05 -08004663 offline_node = marg->status_change_nid_normal;
Yasunori Gotob9049e22007-10-21 16:41:37 -07004664
4665 /*
4666 * If the node still has available memory. we need kmem_cache_node
4667 * for it yet.
4668 */
4669 if (offline_node < 0)
4670 return;
4671
Christoph Lameter18004c52012-07-06 15:25:12 -05004672 mutex_lock(&slab_mutex);
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004673 node_clear(offline_node, slab_nodes);
Vlastimil Babka666716f2021-02-24 12:01:08 -08004674 /*
4675 * We no longer free kmem_cache_node structures here, as it would be
4676 * racy with all get_node() users, and infeasible to protect them with
4677 * slab_mutex.
4678 */
Christoph Lameter18004c52012-07-06 15:25:12 -05004679 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004680}
4681
4682static int slab_mem_going_online_callback(void *arg)
4683{
4684 struct kmem_cache_node *n;
4685 struct kmem_cache *s;
4686 struct memory_notify *marg = arg;
Lai Jiangshanb9d5ab22012-12-11 16:01:05 -08004687 int nid = marg->status_change_nid_normal;
Yasunori Gotob9049e22007-10-21 16:41:37 -07004688 int ret = 0;
4689
4690 /*
4691 * If the node's memory is already available, then kmem_cache_node is
4692 * already created. Nothing to do.
4693 */
4694 if (nid < 0)
4695 return 0;
4696
4697 /*
Christoph Lameter0121c6192008-04-29 16:11:12 -07004698 * We are bringing a node online. No memory is available yet. We must
Yasunori Gotob9049e22007-10-21 16:41:37 -07004699 * allocate a kmem_cache_node structure in order to bring the node
4700 * online.
4701 */
Christoph Lameter18004c52012-07-06 15:25:12 -05004702 mutex_lock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004703 list_for_each_entry(s, &slab_caches, list) {
4704 /*
Vlastimil Babka666716f2021-02-24 12:01:08 -08004705 * The structure may already exist if the node was previously
4706 * onlined and offlined.
4707 */
4708 if (get_node(s, nid))
4709 continue;
4710 /*
Yasunori Gotob9049e22007-10-21 16:41:37 -07004711 * XXX: kmem_cache_alloc_node will fallback to other nodes
4712 * since memory is not yet available from the node that
4713 * is brought up.
4714 */
Christoph Lameter8de66a02010-08-25 14:51:14 -05004715 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004716 if (!n) {
4717 ret = -ENOMEM;
4718 goto out;
4719 }
Joonsoo Kim40534972012-05-11 00:50:47 +09004720 init_kmem_cache_node(n);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004721 s->node[nid] = n;
4722 }
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004723 /*
4724 * Any cache created after this point will also have kmem_cache_node
4725 * initialized for the new node.
4726 */
4727 node_set(nid, slab_nodes);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004728out:
Christoph Lameter18004c52012-07-06 15:25:12 -05004729 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004730 return ret;
4731}
4732
4733static int slab_memory_callback(struct notifier_block *self,
4734 unsigned long action, void *arg)
4735{
4736 int ret = 0;
4737
4738 switch (action) {
4739 case MEM_GOING_ONLINE:
4740 ret = slab_mem_going_online_callback(arg);
4741 break;
4742 case MEM_GOING_OFFLINE:
4743 ret = slab_mem_going_offline_callback(arg);
4744 break;
4745 case MEM_OFFLINE:
4746 case MEM_CANCEL_ONLINE:
4747 slab_mem_offline_callback(arg);
4748 break;
4749 case MEM_ONLINE:
4750 case MEM_CANCEL_OFFLINE:
4751 break;
4752 }
KAMEZAWA Hiroyukidc19f9d2008-12-01 13:13:48 -08004753 if (ret)
4754 ret = notifier_from_errno(ret);
4755 else
4756 ret = NOTIFY_OK;
Yasunori Gotob9049e22007-10-21 16:41:37 -07004757 return ret;
4758}
4759
Andrew Morton3ac38fa2013-04-29 15:08:06 -07004760static struct notifier_block slab_memory_callback_nb = {
4761 .notifier_call = slab_memory_callback,
4762 .priority = SLAB_CALLBACK_PRI,
4763};
Yasunori Gotob9049e22007-10-21 16:41:37 -07004764
Christoph Lameter81819f02007-05-06 14:49:36 -07004765/********************************************************************
4766 * Basic setup of slabs
4767 *******************************************************************/
4768
Christoph Lameter51df1142010-08-20 12:37:15 -05004769/*
4770 * Used for early kmem_cache structures that were allocated using
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004771 * the page allocator. Allocate them properly then fix up the pointers
4772 * that may be pointing to the wrong kmem_cache structure.
Christoph Lameter51df1142010-08-20 12:37:15 -05004773 */
4774
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004775static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
Christoph Lameter51df1142010-08-20 12:37:15 -05004776{
4777 int node;
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004778 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004779 struct kmem_cache_node *n;
Christoph Lameter51df1142010-08-20 12:37:15 -05004780
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004781 memcpy(s, static_cache, kmem_cache->object_size);
Christoph Lameter51df1142010-08-20 12:37:15 -05004782
Glauber Costa7d557b32013-02-22 20:20:00 +04004783 /*
4784 * This runs very early, and only the boot processor is supposed to be
4785 * up. Even if it weren't true, IRQs are not up so we couldn't fire
4786 * IPIs around.
4787 */
4788 __flush_cpu_slab(s, smp_processor_id());
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004789 for_each_kmem_cache_node(s, node, n) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004790 struct slab *p;
Christoph Lameter51df1142010-08-20 12:37:15 -05004791
Tobin C. Harding916ac052019-05-13 17:16:12 -07004792 list_for_each_entry(p, &n->partial, slab_list)
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004793 p->slab_cache = s;
Christoph Lameter51df1142010-08-20 12:37:15 -05004794
Li Zefan607bf322011-04-12 15:22:26 +08004795#ifdef CONFIG_SLUB_DEBUG
Tobin C. Harding916ac052019-05-13 17:16:12 -07004796 list_for_each_entry(p, &n->full, slab_list)
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004797 p->slab_cache = s;
Christoph Lameter51df1142010-08-20 12:37:15 -05004798#endif
Christoph Lameter51df1142010-08-20 12:37:15 -05004799 }
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004800 list_add(&s->list, &slab_caches);
4801 return s;
Christoph Lameter51df1142010-08-20 12:37:15 -05004802}
4803
Christoph Lameter81819f02007-05-06 14:49:36 -07004804void __init kmem_cache_init(void)
4805{
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004806 static __initdata struct kmem_cache boot_kmem_cache,
4807 boot_kmem_cache_node;
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004808 int node;
Christoph Lameter51df1142010-08-20 12:37:15 -05004809
Stanislaw Gruszkafc8d8622012-01-10 15:07:32 -08004810 if (debug_guardpage_minorder())
4811 slub_max_order = 0;
4812
Stephen Boyd79270292021-06-28 19:34:52 -07004813 /* Print slub debugging pointers without hashing */
4814 if (__slub_debug_enabled())
4815 no_hash_pointers_enable(NULL);
4816
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004817 kmem_cache_node = &boot_kmem_cache_node;
4818 kmem_cache = &boot_kmem_cache;
Christoph Lameter51df1142010-08-20 12:37:15 -05004819
Vlastimil Babka7e1fa932021-02-24 12:01:12 -08004820 /*
4821 * Initialize the nodemask for which we will allocate per node
4822 * structures. Here we don't need taking slab_mutex yet.
4823 */
4824 for_each_node_state(node, N_NORMAL_MEMORY)
4825 node_set(node, slab_nodes);
4826
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004827 create_boot_cache(kmem_cache_node, "kmem_cache_node",
David Windsor8eb82842017-06-10 22:50:28 -04004828 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
Yasunori Gotob9049e22007-10-21 16:41:37 -07004829
Andrew Morton3ac38fa2013-04-29 15:08:06 -07004830 register_hotmemory_notifier(&slab_memory_callback_nb);
Christoph Lameter81819f02007-05-06 14:49:36 -07004831
4832 /* Able to allocate the per node structures */
4833 slab_state = PARTIAL;
4834
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004835 create_boot_cache(kmem_cache, "kmem_cache",
4836 offsetof(struct kmem_cache, node) +
4837 nr_node_ids * sizeof(struct kmem_cache_node *),
David Windsor8eb82842017-06-10 22:50:28 -04004838 SLAB_HWCACHE_ALIGN, 0, 0);
Christoph Lameter8a13a4c2012-09-04 23:18:33 +00004839
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004840 kmem_cache = bootstrap(&boot_kmem_cache);
Christoph Lameterdffb4d62012-11-28 16:23:07 +00004841 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
Christoph Lameter51df1142010-08-20 12:37:15 -05004842
4843 /* Now we can use the kmem_cache to allocate kmalloc slabs */
Daniel Sanders34cc6992015-06-24 16:55:57 -07004844 setup_kmalloc_cache_index_table();
Christoph Lameterf97d5f632013-01-10 19:12:17 +00004845 create_kmalloc_caches(0);
Christoph Lameter81819f02007-05-06 14:49:36 -07004846
Thomas Garnier210e7a42016-07-26 15:21:59 -07004847 /* Setup random freelists for each cache */
4848 init_freelist_randomization();
4849
Sebastian Andrzej Siewiora96a87b2016-08-18 14:57:19 +02004850 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
4851 slub_cpu_dead);
Christoph Lameter81819f02007-05-06 14:49:36 -07004852
Alexey Dobriyanb9726c22019-03-05 15:48:26 -08004853 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
Christoph Lameterf97d5f632013-01-10 19:12:17 +00004854 cache_line_size(),
Christoph Lameter81819f02007-05-06 14:49:36 -07004855 slub_min_order, slub_max_order, slub_min_objects,
4856 nr_cpu_ids, nr_node_ids);
4857}
4858
Pekka Enberg7e85ee02009-06-12 14:03:06 +03004859void __init kmem_cache_init_late(void)
4860{
Pekka Enberg7e85ee02009-06-12 14:03:06 +03004861}
4862
Glauber Costa2633d7a2012-12-18 14:22:34 -08004863struct kmem_cache *
Alexey Dobriyanf4957d52018-04-05 16:20:37 -07004864__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
Alexey Dobriyand50112e2017-11-15 17:32:18 -08004865 slab_flags_t flags, void (*ctor)(void *))
Christoph Lameter81819f02007-05-06 14:49:36 -07004866{
Roman Gushchin10befea2020-08-06 23:21:27 -07004867 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004868
Vladimir Davydova44cb9442014-04-07 15:39:23 -07004869 s = find_mergeable(size, align, flags, name, ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07004870 if (s) {
Xiongwei Songefb93522022-05-31 08:55:50 +08004871 if (sysfs_slab_alias(s, name))
4872 return NULL;
4873
Christoph Lameter81819f02007-05-06 14:49:36 -07004874 s->refcount++;
Vladimir Davydov84d0ddd2014-04-07 15:39:29 -07004875
Christoph Lameter81819f02007-05-06 14:49:36 -07004876 /*
4877 * Adjust the object sizes so that we clear
4878 * the complete object on kzalloc.
4879 */
Alexey Dobriyan1b473f22018-04-05 16:21:17 -07004880 s->object_size = max(s->object_size, size);
Alexey Dobriyan52ee6d72018-04-05 16:21:06 -07004881 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07004882 }
Christoph Lameter6446faa2008-02-15 23:45:26 -08004883
Christoph Lametercbb79692012-09-05 00:18:32 +00004884 return s;
4885}
Pekka Enberg84c1cf62010-09-14 23:21:12 +03004886
Alexey Dobriyand50112e2017-11-15 17:32:18 -08004887int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
Christoph Lametercbb79692012-09-05 00:18:32 +00004888{
Pekka Enbergaac3a162012-09-05 12:07:44 +03004889 int err;
Christoph Lameter20cea962012-07-06 15:25:13 -05004890
Pekka Enbergaac3a162012-09-05 12:07:44 +03004891 err = kmem_cache_open(s, flags);
4892 if (err)
4893 return err;
Christoph Lameter20cea962012-07-06 15:25:13 -05004894
Christoph Lameter45530c42012-11-28 16:23:07 +00004895 /* Mutex is not taken during early boot */
4896 if (slab_state <= UP)
4897 return 0;
4898
Pekka Enbergaac3a162012-09-05 12:07:44 +03004899 err = sysfs_slab_add(s);
Miaohe Lin67823a52021-10-18 15:16:02 -07004900 if (err) {
Dmitry Safonov52b4b952016-02-17 13:11:37 -08004901 __kmem_cache_release(s);
Miaohe Lin67823a52021-10-18 15:16:02 -07004902 return err;
4903 }
Pekka Enbergaac3a162012-09-05 12:07:44 +03004904
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07004905 if (s->flags & SLAB_STORE_USER)
4906 debugfs_slab_add(s);
4907
Miaohe Lin67823a52021-10-18 15:16:02 -07004908 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07004909}
Christoph Lameter81819f02007-05-06 14:49:36 -07004910
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004911void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07004912{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004913 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004914 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004915
Christoph Lameter95a05b42013-01-10 19:14:19 +00004916 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02004917 return kmalloc_large(size, gfpflags);
4918
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004919 s = kmalloc_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07004920
Satyam Sharma2408c552007-10-16 01:24:44 -07004921 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004922 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004923
Muchun Song88f2ef72022-03-22 14:40:56 -07004924 ret = slab_alloc(s, NULL, gfpflags, caller, size);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004925
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004926 /* Honor the call site pointer we received. */
Vasily Averinb347aa72022-06-03 06:21:49 +03004927 trace_kmalloc(caller, ret, s, size, s->size, gfpflags);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004928
4929 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004930}
Daniel Vetterfd7cb572020-03-23 15:49:00 +01004931EXPORT_SYMBOL(__kmalloc_track_caller);
Christoph Lameter81819f02007-05-06 14:49:36 -07004932
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004933#ifdef CONFIG_NUMA
Christoph Lameter81819f02007-05-06 14:49:36 -07004934void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004935 int node, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07004936{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004937 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004938 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004939
Christoph Lameter95a05b42013-01-10 19:14:19 +00004940 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
Xiaotian Fengd3e14aa2010-04-08 17:26:44 +08004941 ret = kmalloc_large_node(size, gfpflags, node);
4942
Vasily Averinb347aa72022-06-03 06:21:49 +03004943 trace_kmalloc_node(caller, ret, NULL,
Xiaotian Fengd3e14aa2010-04-08 17:26:44 +08004944 size, PAGE_SIZE << get_order(size),
4945 gfpflags, node);
4946
4947 return ret;
4948 }
Pekka Enbergeada35e2008-02-11 22:47:46 +02004949
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004950 s = kmalloc_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07004951
Satyam Sharma2408c552007-10-16 01:24:44 -07004952 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004953 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004954
Muchun Song88f2ef72022-03-22 14:40:56 -07004955 ret = slab_alloc_node(s, NULL, gfpflags, node, caller, size);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004956
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004957 /* Honor the call site pointer we received. */
Vasily Averinb347aa72022-06-03 06:21:49 +03004958 trace_kmalloc_node(caller, ret, s, size, s->size, gfpflags, node);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004959
4960 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004961}
Daniel Vetterfd7cb572020-03-23 15:49:00 +01004962EXPORT_SYMBOL(__kmalloc_node_track_caller);
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004963#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004964
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004965#ifdef CONFIG_SYSFS
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004966static int count_inuse(struct slab *slab)
Christoph Lameter205ab992008-04-14 19:11:40 +03004967{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004968 return slab->inuse;
Christoph Lameter205ab992008-04-14 19:11:40 +03004969}
4970
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004971static int count_total(struct slab *slab)
Christoph Lameter205ab992008-04-14 19:11:40 +03004972{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004973 return slab->objects;
Christoph Lameter205ab992008-04-14 19:11:40 +03004974}
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004975#endif
Christoph Lameter205ab992008-04-14 19:11:40 +03004976
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004977#ifdef CONFIG_SLUB_DEBUG
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004978static void validate_slab(struct kmem_cache *s, struct slab *slab,
Vlastimil Babka0a19e7d2021-05-23 01:37:07 +02004979 unsigned long *obj_map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004980{
4981 void *p;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004982 void *addr = slab_address(slab);
Vlastimil Babkaa2b4ae82021-06-04 12:55:55 +02004983 unsigned long flags;
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004984
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004985 slab_lock(slab, &flags);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004986
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004987 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004988 goto unlock;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004989
4990 /* Now we know that a valid freelist exists */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004991 __fill_map(obj_map, s, slab);
4992 for_each_object(p, s, addr, slab->objects) {
Vlastimil Babka0a19e7d2021-05-23 01:37:07 +02004993 u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
Yu Zhaodd98afd2019-11-30 17:49:37 -08004994 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004995
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01004996 if (!check_object(s, slab, p, val))
Yu Zhaodd98afd2019-11-30 17:49:37 -08004997 break;
4998 }
Yu Zhao90e9f6a2020-01-30 22:11:57 -08004999unlock:
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005000 slab_unlock(slab, &flags);
Christoph Lameter53e15af2007-05-06 14:49:43 -07005001}
5002
Christoph Lameter434e2452007-07-17 04:03:30 -07005003static int validate_slab_node(struct kmem_cache *s,
Vlastimil Babka0a19e7d2021-05-23 01:37:07 +02005004 struct kmem_cache_node *n, unsigned long *obj_map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07005005{
5006 unsigned long count = 0;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005007 struct slab *slab;
Christoph Lameter53e15af2007-05-06 14:49:43 -07005008 unsigned long flags;
5009
5010 spin_lock_irqsave(&n->list_lock, flags);
5011
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005012 list_for_each_entry(slab, &n->partial, slab_list) {
5013 validate_slab(s, slab, obj_map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07005014 count++;
5015 }
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07005016 if (count != n->nr_partial) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07005017 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
5018 s->name, count, n->nr_partial);
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07005019 slab_add_kunit_errors();
5020 }
Christoph Lameter53e15af2007-05-06 14:49:43 -07005021
5022 if (!(s->flags & SLAB_STORE_USER))
5023 goto out;
5024
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005025 list_for_each_entry(slab, &n->full, slab_list) {
5026 validate_slab(s, slab, obj_map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07005027 count++;
5028 }
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07005029 if (count != atomic_long_read(&n->nr_slabs)) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07005030 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
5031 s->name, count, atomic_long_read(&n->nr_slabs));
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07005032 slab_add_kunit_errors();
5033 }
Christoph Lameter53e15af2007-05-06 14:49:43 -07005034
5035out:
5036 spin_unlock_irqrestore(&n->list_lock, flags);
5037 return count;
5038}
5039
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07005040long validate_slab_cache(struct kmem_cache *s)
Christoph Lameter53e15af2007-05-06 14:49:43 -07005041{
5042 int node;
5043 unsigned long count = 0;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005044 struct kmem_cache_node *n;
Vlastimil Babka0a19e7d2021-05-23 01:37:07 +02005045 unsigned long *obj_map;
5046
5047 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
5048 if (!obj_map)
5049 return -ENOMEM;
Christoph Lameter53e15af2007-05-06 14:49:43 -07005050
5051 flush_all(s);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005052 for_each_kmem_cache_node(s, node, n)
Vlastimil Babka0a19e7d2021-05-23 01:37:07 +02005053 count += validate_slab_node(s, n, obj_map);
5054
5055 bitmap_free(obj_map);
Yu Zhao90e9f6a2020-01-30 22:11:57 -08005056
Christoph Lameter53e15af2007-05-06 14:49:43 -07005057 return count;
5058}
Oliver Glitta1f9f78b2021-06-28 19:34:33 -07005059EXPORT_SYMBOL(validate_slab_cache);
5060
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07005061#ifdef CONFIG_DEBUG_FS
Christoph Lameter88a420e2007-05-06 14:49:45 -07005062/*
Christoph Lameter672bba32007-05-09 02:32:39 -07005063 * Generate lists of code addresses where slabcache objects are allocated
Christoph Lameter88a420e2007-05-06 14:49:45 -07005064 * and freed.
5065 */
5066
5067struct location {
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02005068 depot_stack_handle_t handle;
Christoph Lameter88a420e2007-05-06 14:49:45 -07005069 unsigned long count;
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03005070 unsigned long addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07005071 long long sum_time;
5072 long min_time;
5073 long max_time;
5074 long min_pid;
5075 long max_pid;
Rusty Russell174596a2009-01-01 10:12:29 +10305076 DECLARE_BITMAP(cpus, NR_CPUS);
Christoph Lameter45edfa52007-05-09 02:32:45 -07005077 nodemask_t nodes;
Christoph Lameter88a420e2007-05-06 14:49:45 -07005078};
5079
5080struct loc_track {
5081 unsigned long max;
5082 unsigned long count;
5083 struct location *loc;
Gerald Schaefer005a79e2021-12-10 14:47:02 -08005084 loff_t idx;
Christoph Lameter88a420e2007-05-06 14:49:45 -07005085};
5086
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07005087static struct dentry *slab_debugfs_root;
5088
Christoph Lameter88a420e2007-05-06 14:49:45 -07005089static void free_loc_track(struct loc_track *t)
5090{
5091 if (t->max)
5092 free_pages((unsigned long)t->loc,
5093 get_order(sizeof(struct location) * t->max));
5094}
5095
Christoph Lameter68dff6a2007-07-17 04:03:20 -07005096static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
Christoph Lameter88a420e2007-05-06 14:49:45 -07005097{
5098 struct location *l;
5099 int order;
5100
Christoph Lameter88a420e2007-05-06 14:49:45 -07005101 order = get_order(sizeof(struct location) * max);
5102
Christoph Lameter68dff6a2007-07-17 04:03:20 -07005103 l = (void *)__get_free_pages(flags, order);
Christoph Lameter88a420e2007-05-06 14:49:45 -07005104 if (!l)
5105 return 0;
5106
5107 if (t->count) {
5108 memcpy(l, t->loc, sizeof(struct location) * t->count);
5109 free_loc_track(t);
5110 }
5111 t->max = max;
5112 t->loc = l;
5113 return 1;
5114}
5115
5116static int add_location(struct loc_track *t, struct kmem_cache *s,
Christoph Lameter45edfa52007-05-09 02:32:45 -07005117 const struct track *track)
Christoph Lameter88a420e2007-05-06 14:49:45 -07005118{
5119 long start, end, pos;
5120 struct location *l;
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02005121 unsigned long caddr, chandle;
Christoph Lameter45edfa52007-05-09 02:32:45 -07005122 unsigned long age = jiffies - track->when;
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02005123 depot_stack_handle_t handle = 0;
Christoph Lameter88a420e2007-05-06 14:49:45 -07005124
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02005125#ifdef CONFIG_STACKDEPOT
5126 handle = READ_ONCE(track->handle);
5127#endif
Christoph Lameter88a420e2007-05-06 14:49:45 -07005128 start = -1;
5129 end = t->count;
5130
5131 for ( ; ; ) {
5132 pos = start + (end - start + 1) / 2;
5133
5134 /*
5135 * There is nothing at "end". If we end up there
5136 * we need to add something to before end.
5137 */
5138 if (pos == end)
5139 break;
5140
5141 caddr = t->loc[pos].addr;
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02005142 chandle = t->loc[pos].handle;
5143 if ((track->addr == caddr) && (handle == chandle)) {
Christoph Lameter45edfa52007-05-09 02:32:45 -07005144
5145 l = &t->loc[pos];
5146 l->count++;
5147 if (track->when) {
5148 l->sum_time += age;
5149 if (age < l->min_time)
5150 l->min_time = age;
5151 if (age > l->max_time)
5152 l->max_time = age;
5153
5154 if (track->pid < l->min_pid)
5155 l->min_pid = track->pid;
5156 if (track->pid > l->max_pid)
5157 l->max_pid = track->pid;
5158
Rusty Russell174596a2009-01-01 10:12:29 +10305159 cpumask_set_cpu(track->cpu,
5160 to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07005161 }
5162 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07005163 return 1;
5164 }
5165
Christoph Lameter45edfa52007-05-09 02:32:45 -07005166 if (track->addr < caddr)
Christoph Lameter88a420e2007-05-06 14:49:45 -07005167 end = pos;
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02005168 else if (track->addr == caddr && handle < chandle)
5169 end = pos;
Christoph Lameter88a420e2007-05-06 14:49:45 -07005170 else
5171 start = pos;
5172 }
5173
5174 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07005175 * Not found. Insert new tracking element.
Christoph Lameter88a420e2007-05-06 14:49:45 -07005176 */
Christoph Lameter68dff6a2007-07-17 04:03:20 -07005177 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
Christoph Lameter88a420e2007-05-06 14:49:45 -07005178 return 0;
5179
5180 l = t->loc + pos;
5181 if (pos < t->count)
5182 memmove(l + 1, l,
5183 (t->count - pos) * sizeof(struct location));
5184 t->count++;
5185 l->count = 1;
Christoph Lameter45edfa52007-05-09 02:32:45 -07005186 l->addr = track->addr;
5187 l->sum_time = age;
5188 l->min_time = age;
5189 l->max_time = age;
5190 l->min_pid = track->pid;
5191 l->max_pid = track->pid;
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02005192 l->handle = handle;
Rusty Russell174596a2009-01-01 10:12:29 +10305193 cpumask_clear(to_cpumask(l->cpus));
5194 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07005195 nodes_clear(l->nodes);
5196 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07005197 return 1;
5198}
5199
5200static void process_slab(struct loc_track *t, struct kmem_cache *s,
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005201 struct slab *slab, enum track_item alloc,
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02005202 unsigned long *obj_map)
Christoph Lameter88a420e2007-05-06 14:49:45 -07005203{
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005204 void *addr = slab_address(slab);
Christoph Lameter88a420e2007-05-06 14:49:45 -07005205 void *p;
5206
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005207 __fill_map(obj_map, s, slab);
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02005208
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005209 for_each_object(p, s, addr, slab->objects)
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02005210 if (!test_bit(__obj_to_index(s, addr, p), obj_map))
Christoph Lameter45edfa52007-05-09 02:32:45 -07005211 add_location(t, s, get_track(s, p, alloc));
Christoph Lameter88a420e2007-05-06 14:49:45 -07005212}
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07005213#endif /* CONFIG_DEBUG_FS */
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07005214#endif /* CONFIG_SLUB_DEBUG */
Christoph Lameter88a420e2007-05-06 14:49:45 -07005215
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005216#ifdef CONFIG_SYSFS
Christoph Lameter81819f02007-05-06 14:49:36 -07005217enum slab_stat_type {
Christoph Lameter205ab992008-04-14 19:11:40 +03005218 SL_ALL, /* All slabs */
5219 SL_PARTIAL, /* Only partially allocated slabs */
5220 SL_CPU, /* Only slabs used for cpu caches */
5221 SL_OBJECTS, /* Determine allocated objects not slabs */
5222 SL_TOTAL /* Determine object capacity not slabs */
Christoph Lameter81819f02007-05-06 14:49:36 -07005223};
5224
Christoph Lameter205ab992008-04-14 19:11:40 +03005225#define SO_ALL (1 << SL_ALL)
Christoph Lameter81819f02007-05-06 14:49:36 -07005226#define SO_PARTIAL (1 << SL_PARTIAL)
5227#define SO_CPU (1 << SL_CPU)
5228#define SO_OBJECTS (1 << SL_OBJECTS)
Christoph Lameter205ab992008-04-14 19:11:40 +03005229#define SO_TOTAL (1 << SL_TOTAL)
Christoph Lameter81819f02007-05-06 14:49:36 -07005230
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03005231static ssize_t show_slab_objects(struct kmem_cache *s,
Joe Perchesbf16d192020-12-14 19:14:57 -08005232 char *buf, unsigned long flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07005233{
5234 unsigned long total = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07005235 int node;
5236 int x;
5237 unsigned long *nodes;
Joe Perchesbf16d192020-12-14 19:14:57 -08005238 int len = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07005239
Kees Cook6396bb22018-06-12 14:03:40 -07005240 nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03005241 if (!nodes)
5242 return -ENOMEM;
Christoph Lameter81819f02007-05-06 14:49:36 -07005243
Christoph Lameter205ab992008-04-14 19:11:40 +03005244 if (flags & SO_CPU) {
5245 int cpu;
Christoph Lameter81819f02007-05-06 14:49:36 -07005246
Christoph Lameter205ab992008-04-14 19:11:40 +03005247 for_each_possible_cpu(cpu) {
Chen Gangd0e0ac92013-07-15 09:05:29 +08005248 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
5249 cpu);
Christoph Lameterec3ab082012-05-09 10:09:56 -05005250 int node;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005251 struct slab *slab;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07005252
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005253 slab = READ_ONCE(c->slab);
5254 if (!slab)
Christoph Lameterec3ab082012-05-09 10:09:56 -05005255 continue;
Christoph Lameter205ab992008-04-14 19:11:40 +03005256
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005257 node = slab_nid(slab);
Christoph Lameterec3ab082012-05-09 10:09:56 -05005258 if (flags & SO_TOTAL)
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005259 x = slab->objects;
Christoph Lameterec3ab082012-05-09 10:09:56 -05005260 else if (flags & SO_OBJECTS)
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005261 x = slab->inuse;
Christoph Lameterec3ab082012-05-09 10:09:56 -05005262 else
5263 x = 1;
Christoph Lameter49e22582011-08-09 16:12:27 -05005264
Christoph Lameterec3ab082012-05-09 10:09:56 -05005265 total += x;
5266 nodes[node] += x;
5267
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +01005268#ifdef CONFIG_SLUB_CPU_PARTIAL
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005269 slab = slub_percpu_partial_read_once(c);
5270 if (slab) {
5271 node = slab_nid(slab);
Li Zefan8afb1472013-09-10 11:43:37 +08005272 if (flags & SO_TOTAL)
5273 WARN_ON_ONCE(1);
5274 else if (flags & SO_OBJECTS)
5275 WARN_ON_ONCE(1);
5276 else
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005277 x = slab->slabs;
Eric Dumazetbc6697d2011-11-22 16:02:02 +01005278 total += x;
5279 nodes[node] += x;
Christoph Lameter49e22582011-08-09 16:12:27 -05005280 }
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +01005281#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07005282 }
5283 }
5284
Qian Caie4f8e512019-10-14 14:11:51 -07005285 /*
5286 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
5287 * already held which will conflict with an existing lock order:
5288 *
5289 * mem_hotplug_lock->slab_mutex->kernfs_mutex
5290 *
5291 * We don't really need mem_hotplug_lock (to hold off
5292 * slab_mem_going_offline_callback) here because slab's memory hot
5293 * unplug code doesn't destroy the kmem_cache->node[] data.
5294 */
5295
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005296#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter205ab992008-04-14 19:11:40 +03005297 if (flags & SO_ALL) {
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005298 struct kmem_cache_node *n;
5299
5300 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter81819f02007-05-06 14:49:36 -07005301
Chen Gangd0e0ac92013-07-15 09:05:29 +08005302 if (flags & SO_TOTAL)
5303 x = atomic_long_read(&n->total_objects);
5304 else if (flags & SO_OBJECTS)
5305 x = atomic_long_read(&n->total_objects) -
5306 count_partial(n, count_free);
Christoph Lameter205ab992008-04-14 19:11:40 +03005307 else
5308 x = atomic_long_read(&n->nr_slabs);
5309 total += x;
5310 nodes[node] += x;
5311 }
5312
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005313 } else
5314#endif
5315 if (flags & SO_PARTIAL) {
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005316 struct kmem_cache_node *n;
Christoph Lameter205ab992008-04-14 19:11:40 +03005317
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005318 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter205ab992008-04-14 19:11:40 +03005319 if (flags & SO_TOTAL)
5320 x = count_partial(n, count_total);
5321 else if (flags & SO_OBJECTS)
5322 x = count_partial(n, count_inuse);
Christoph Lameter81819f02007-05-06 14:49:36 -07005323 else
5324 x = n->nr_partial;
5325 total += x;
5326 nodes[node] += x;
5327 }
Christoph Lameter81819f02007-05-06 14:49:36 -07005328 }
Joe Perchesbf16d192020-12-14 19:14:57 -08005329
5330 len += sysfs_emit_at(buf, len, "%lu", total);
Christoph Lameter81819f02007-05-06 14:49:36 -07005331#ifdef CONFIG_NUMA
Joe Perchesbf16d192020-12-14 19:14:57 -08005332 for (node = 0; node < nr_node_ids; node++) {
Christoph Lameter81819f02007-05-06 14:49:36 -07005333 if (nodes[node])
Joe Perchesbf16d192020-12-14 19:14:57 -08005334 len += sysfs_emit_at(buf, len, " N%d=%lu",
5335 node, nodes[node]);
5336 }
Christoph Lameter81819f02007-05-06 14:49:36 -07005337#endif
Joe Perchesbf16d192020-12-14 19:14:57 -08005338 len += sysfs_emit_at(buf, len, "\n");
Christoph Lameter81819f02007-05-06 14:49:36 -07005339 kfree(nodes);
Joe Perchesbf16d192020-12-14 19:14:57 -08005340
5341 return len;
Christoph Lameter81819f02007-05-06 14:49:36 -07005342}
5343
Christoph Lameter81819f02007-05-06 14:49:36 -07005344#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
Phil Carmody497888c2011-07-14 15:07:13 +03005345#define to_slab(n) container_of(n, struct kmem_cache, kobj)
Christoph Lameter81819f02007-05-06 14:49:36 -07005346
5347struct slab_attribute {
5348 struct attribute attr;
5349 ssize_t (*show)(struct kmem_cache *s, char *buf);
5350 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
5351};
5352
5353#define SLAB_ATTR_RO(_name) \
Lianjie Zhangd1d28bd2022-03-06 15:38:18 +08005354 static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400)
Christoph Lameter81819f02007-05-06 14:49:36 -07005355
5356#define SLAB_ATTR(_name) \
Lianjie Zhangd1d28bd2022-03-06 15:38:18 +08005357 static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600)
Christoph Lameter81819f02007-05-06 14:49:36 -07005358
Christoph Lameter81819f02007-05-06 14:49:36 -07005359static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
5360{
Joe Perchesbf16d192020-12-14 19:14:57 -08005361 return sysfs_emit(buf, "%u\n", s->size);
Christoph Lameter81819f02007-05-06 14:49:36 -07005362}
5363SLAB_ATTR_RO(slab_size);
5364
5365static ssize_t align_show(struct kmem_cache *s, char *buf)
5366{
Joe Perchesbf16d192020-12-14 19:14:57 -08005367 return sysfs_emit(buf, "%u\n", s->align);
Christoph Lameter81819f02007-05-06 14:49:36 -07005368}
5369SLAB_ATTR_RO(align);
5370
5371static ssize_t object_size_show(struct kmem_cache *s, char *buf)
5372{
Joe Perchesbf16d192020-12-14 19:14:57 -08005373 return sysfs_emit(buf, "%u\n", s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -07005374}
5375SLAB_ATTR_RO(object_size);
5376
5377static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
5378{
Joe Perchesbf16d192020-12-14 19:14:57 -08005379 return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07005380}
5381SLAB_ATTR_RO(objs_per_slab);
5382
5383static ssize_t order_show(struct kmem_cache *s, char *buf)
5384{
Joe Perchesbf16d192020-12-14 19:14:57 -08005385 return sysfs_emit(buf, "%u\n", oo_order(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07005386}
Vlastimil Babka32a6f402020-08-06 23:18:41 -07005387SLAB_ATTR_RO(order);
Christoph Lameter81819f02007-05-06 14:49:36 -07005388
David Rientjes73d342b2009-02-22 17:40:09 -08005389static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
5390{
Joe Perchesbf16d192020-12-14 19:14:57 -08005391 return sysfs_emit(buf, "%lu\n", s->min_partial);
David Rientjes73d342b2009-02-22 17:40:09 -08005392}
5393
5394static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
5395 size_t length)
5396{
5397 unsigned long min;
5398 int err;
5399
Jingoo Han3dbb95f2013-09-11 14:20:25 -07005400 err = kstrtoul(buf, 10, &min);
David Rientjes73d342b2009-02-22 17:40:09 -08005401 if (err)
5402 return err;
5403
Hyeonggon Yoo5182f3c2022-03-07 07:40:55 +00005404 s->min_partial = min;
David Rientjes73d342b2009-02-22 17:40:09 -08005405 return length;
5406}
5407SLAB_ATTR(min_partial);
5408
Christoph Lameter49e22582011-08-09 16:12:27 -05005409static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
5410{
Vlastimil Babkab47291e2021-11-05 13:35:17 -07005411 unsigned int nr_partial = 0;
5412#ifdef CONFIG_SLUB_CPU_PARTIAL
5413 nr_partial = s->cpu_partial;
5414#endif
5415
5416 return sysfs_emit(buf, "%u\n", nr_partial);
Christoph Lameter49e22582011-08-09 16:12:27 -05005417}
5418
5419static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
5420 size_t length)
5421{
Alexey Dobriyane5d99982018-04-05 16:21:10 -07005422 unsigned int objects;
Christoph Lameter49e22582011-08-09 16:12:27 -05005423 int err;
5424
Alexey Dobriyane5d99982018-04-05 16:21:10 -07005425 err = kstrtouint(buf, 10, &objects);
Christoph Lameter49e22582011-08-09 16:12:27 -05005426 if (err)
5427 return err;
Joonsoo Kim345c9052013-06-19 14:05:52 +09005428 if (objects && !kmem_cache_has_cpu_partial(s))
David Rientjes74ee4ef2012-01-09 13:19:45 -08005429 return -EINVAL;
Christoph Lameter49e22582011-08-09 16:12:27 -05005430
Wei Yange6d0e1d2017-07-06 15:36:34 -07005431 slub_set_cpu_partial(s, objects);
Christoph Lameter49e22582011-08-09 16:12:27 -05005432 flush_all(s);
5433 return length;
5434}
5435SLAB_ATTR(cpu_partial);
5436
Christoph Lameter81819f02007-05-06 14:49:36 -07005437static ssize_t ctor_show(struct kmem_cache *s, char *buf)
5438{
Joe Perches62c70bc2011-01-13 15:45:52 -08005439 if (!s->ctor)
5440 return 0;
Joe Perchesbf16d192020-12-14 19:14:57 -08005441 return sysfs_emit(buf, "%pS\n", s->ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07005442}
5443SLAB_ATTR_RO(ctor);
5444
Christoph Lameter81819f02007-05-06 14:49:36 -07005445static ssize_t aliases_show(struct kmem_cache *s, char *buf)
5446{
Joe Perchesbf16d192020-12-14 19:14:57 -08005447 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07005448}
5449SLAB_ATTR_RO(aliases);
5450
Christoph Lameter81819f02007-05-06 14:49:36 -07005451static ssize_t partial_show(struct kmem_cache *s, char *buf)
5452{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08005453 return show_slab_objects(s, buf, SO_PARTIAL);
Christoph Lameter81819f02007-05-06 14:49:36 -07005454}
5455SLAB_ATTR_RO(partial);
5456
5457static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
5458{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08005459 return show_slab_objects(s, buf, SO_CPU);
Christoph Lameter81819f02007-05-06 14:49:36 -07005460}
5461SLAB_ATTR_RO(cpu_slabs);
5462
5463static ssize_t objects_show(struct kmem_cache *s, char *buf)
5464{
Christoph Lameter205ab992008-04-14 19:11:40 +03005465 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
Christoph Lameter81819f02007-05-06 14:49:36 -07005466}
5467SLAB_ATTR_RO(objects);
5468
Christoph Lameter205ab992008-04-14 19:11:40 +03005469static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
5470{
5471 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
5472}
5473SLAB_ATTR_RO(objects_partial);
5474
Christoph Lameter49e22582011-08-09 16:12:27 -05005475static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
5476{
5477 int objects = 0;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005478 int slabs = 0;
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +01005479 int cpu __maybe_unused;
Joe Perchesbf16d192020-12-14 19:14:57 -08005480 int len = 0;
Christoph Lameter49e22582011-08-09 16:12:27 -05005481
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +01005482#ifdef CONFIG_SLUB_CPU_PARTIAL
Christoph Lameter49e22582011-08-09 16:12:27 -05005483 for_each_online_cpu(cpu) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005484 struct slab *slab;
Wei Yanga93cf072017-07-06 15:36:31 -07005485
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005486 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
Christoph Lameter49e22582011-08-09 16:12:27 -05005487
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005488 if (slab)
5489 slabs += slab->slabs;
Christoph Lameter49e22582011-08-09 16:12:27 -05005490 }
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +01005491#endif
Christoph Lameter49e22582011-08-09 16:12:27 -05005492
Vlastimil Babkac2092c12021-11-15 16:55:15 +01005493 /* Approximate half-full slabs, see slub_set_cpu_partial() */
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005494 objects = (slabs * oo_objects(s->oo)) / 2;
5495 len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
Christoph Lameter49e22582011-08-09 16:12:27 -05005496
Vlastimil Babka9c01e9a2021-11-10 14:12:45 +01005497#if defined(CONFIG_SLUB_CPU_PARTIAL) && defined(CONFIG_SMP)
Christoph Lameter49e22582011-08-09 16:12:27 -05005498 for_each_online_cpu(cpu) {
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005499 struct slab *slab;
Wei Yanga93cf072017-07-06 15:36:31 -07005500
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005501 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
5502 if (slab) {
5503 slabs = READ_ONCE(slab->slabs);
5504 objects = (slabs * oo_objects(s->oo)) / 2;
Joe Perchesbf16d192020-12-14 19:14:57 -08005505 len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01005506 cpu, objects, slabs);
Vlastimil Babkab47291e2021-11-05 13:35:17 -07005507 }
Christoph Lameter49e22582011-08-09 16:12:27 -05005508 }
5509#endif
Joe Perchesbf16d192020-12-14 19:14:57 -08005510 len += sysfs_emit_at(buf, len, "\n");
5511
5512 return len;
Christoph Lameter49e22582011-08-09 16:12:27 -05005513}
5514SLAB_ATTR_RO(slabs_cpu_partial);
5515
Christoph Lameter81819f02007-05-06 14:49:36 -07005516static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
5517{
Joe Perchesbf16d192020-12-14 19:14:57 -08005518 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
Christoph Lameter81819f02007-05-06 14:49:36 -07005519}
Vlastimil Babka8f58119a2020-08-06 23:18:48 -07005520SLAB_ATTR_RO(reclaim_account);
Christoph Lameter81819f02007-05-06 14:49:36 -07005521
5522static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
5523{
Joe Perchesbf16d192020-12-14 19:14:57 -08005524 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
Christoph Lameter81819f02007-05-06 14:49:36 -07005525}
5526SLAB_ATTR_RO(hwcache_align);
5527
5528#ifdef CONFIG_ZONE_DMA
5529static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
5530{
Joe Perchesbf16d192020-12-14 19:14:57 -08005531 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
Christoph Lameter81819f02007-05-06 14:49:36 -07005532}
5533SLAB_ATTR_RO(cache_dma);
5534#endif
5535
David Windsor8eb82842017-06-10 22:50:28 -04005536static ssize_t usersize_show(struct kmem_cache *s, char *buf)
5537{
Joe Perchesbf16d192020-12-14 19:14:57 -08005538 return sysfs_emit(buf, "%u\n", s->usersize);
David Windsor8eb82842017-06-10 22:50:28 -04005539}
5540SLAB_ATTR_RO(usersize);
5541
Christoph Lameter81819f02007-05-06 14:49:36 -07005542static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
5543{
Joe Perchesbf16d192020-12-14 19:14:57 -08005544 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
Christoph Lameter81819f02007-05-06 14:49:36 -07005545}
5546SLAB_ATTR_RO(destroy_by_rcu);
5547
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005548#ifdef CONFIG_SLUB_DEBUG
Christoph Lametera5a84752010-10-05 13:57:27 -05005549static ssize_t slabs_show(struct kmem_cache *s, char *buf)
5550{
5551 return show_slab_objects(s, buf, SO_ALL);
5552}
5553SLAB_ATTR_RO(slabs);
5554
5555static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
5556{
5557 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
5558}
5559SLAB_ATTR_RO(total_objects);
5560
5561static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
5562{
Joe Perchesbf16d192020-12-14 19:14:57 -08005563 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
Christoph Lametera5a84752010-10-05 13:57:27 -05005564}
Vlastimil Babka060807f2020-08-06 23:18:45 -07005565SLAB_ATTR_RO(sanity_checks);
Christoph Lametera5a84752010-10-05 13:57:27 -05005566
5567static ssize_t trace_show(struct kmem_cache *s, char *buf)
5568{
Joe Perchesbf16d192020-12-14 19:14:57 -08005569 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
Christoph Lametera5a84752010-10-05 13:57:27 -05005570}
Vlastimil Babka060807f2020-08-06 23:18:45 -07005571SLAB_ATTR_RO(trace);
Christoph Lametera5a84752010-10-05 13:57:27 -05005572
Christoph Lameter81819f02007-05-06 14:49:36 -07005573static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
5574{
Joe Perchesbf16d192020-12-14 19:14:57 -08005575 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
Christoph Lameter81819f02007-05-06 14:49:36 -07005576}
5577
Vlastimil Babkaad38b5b2020-08-06 23:18:38 -07005578SLAB_ATTR_RO(red_zone);
Christoph Lameter81819f02007-05-06 14:49:36 -07005579
5580static ssize_t poison_show(struct kmem_cache *s, char *buf)
5581{
Joe Perchesbf16d192020-12-14 19:14:57 -08005582 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
Christoph Lameter81819f02007-05-06 14:49:36 -07005583}
5584
Vlastimil Babkaad38b5b2020-08-06 23:18:38 -07005585SLAB_ATTR_RO(poison);
Christoph Lameter81819f02007-05-06 14:49:36 -07005586
5587static ssize_t store_user_show(struct kmem_cache *s, char *buf)
5588{
Joe Perchesbf16d192020-12-14 19:14:57 -08005589 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
Christoph Lameter81819f02007-05-06 14:49:36 -07005590}
5591
Vlastimil Babkaad38b5b2020-08-06 23:18:38 -07005592SLAB_ATTR_RO(store_user);
Christoph Lameter81819f02007-05-06 14:49:36 -07005593
Christoph Lameter53e15af2007-05-06 14:49:43 -07005594static ssize_t validate_show(struct kmem_cache *s, char *buf)
5595{
5596 return 0;
5597}
5598
5599static ssize_t validate_store(struct kmem_cache *s,
5600 const char *buf, size_t length)
5601{
Christoph Lameter434e2452007-07-17 04:03:30 -07005602 int ret = -EINVAL;
5603
5604 if (buf[0] == '1') {
5605 ret = validate_slab_cache(s);
5606 if (ret >= 0)
5607 ret = length;
5608 }
5609 return ret;
Christoph Lameter53e15af2007-05-06 14:49:43 -07005610}
5611SLAB_ATTR(validate);
Christoph Lametera5a84752010-10-05 13:57:27 -05005612
Christoph Lametera5a84752010-10-05 13:57:27 -05005613#endif /* CONFIG_SLUB_DEBUG */
5614
5615#ifdef CONFIG_FAILSLAB
5616static ssize_t failslab_show(struct kmem_cache *s, char *buf)
5617{
Joe Perchesbf16d192020-12-14 19:14:57 -08005618 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
Christoph Lametera5a84752010-10-05 13:57:27 -05005619}
Vlastimil Babka060807f2020-08-06 23:18:45 -07005620SLAB_ATTR_RO(failslab);
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005621#endif
Christoph Lameter53e15af2007-05-06 14:49:43 -07005622
Christoph Lameter2086d262007-05-06 14:49:46 -07005623static ssize_t shrink_show(struct kmem_cache *s, char *buf)
5624{
5625 return 0;
5626}
5627
5628static ssize_t shrink_store(struct kmem_cache *s,
5629 const char *buf, size_t length)
5630{
Vladimir Davydov832f37f2015-02-12 14:59:41 -08005631 if (buf[0] == '1')
Roman Gushchin10befea2020-08-06 23:21:27 -07005632 kmem_cache_shrink(s);
Vladimir Davydov832f37f2015-02-12 14:59:41 -08005633 else
Christoph Lameter2086d262007-05-06 14:49:46 -07005634 return -EINVAL;
5635 return length;
5636}
5637SLAB_ATTR(shrink);
5638
Christoph Lameter81819f02007-05-06 14:49:36 -07005639#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08005640static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
Christoph Lameter81819f02007-05-06 14:49:36 -07005641{
Joe Perchesbf16d192020-12-14 19:14:57 -08005642 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
Christoph Lameter81819f02007-05-06 14:49:36 -07005643}
5644
Christoph Lameter98246012008-01-07 23:20:26 -08005645static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
Christoph Lameter81819f02007-05-06 14:49:36 -07005646 const char *buf, size_t length)
5647{
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005648 unsigned int ratio;
Christoph Lameter0121c6192008-04-29 16:11:12 -07005649 int err;
Christoph Lameter81819f02007-05-06 14:49:36 -07005650
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005651 err = kstrtouint(buf, 10, &ratio);
Christoph Lameter0121c6192008-04-29 16:11:12 -07005652 if (err)
5653 return err;
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005654 if (ratio > 100)
5655 return -ERANGE;
Christoph Lameter0121c6192008-04-29 16:11:12 -07005656
Alexey Dobriyaneb7235e2018-04-05 16:20:48 -07005657 s->remote_node_defrag_ratio = ratio * 10;
Christoph Lameter0121c6192008-04-29 16:11:12 -07005658
Christoph Lameter81819f02007-05-06 14:49:36 -07005659 return length;
5660}
Christoph Lameter98246012008-01-07 23:20:26 -08005661SLAB_ATTR(remote_node_defrag_ratio);
Christoph Lameter81819f02007-05-06 14:49:36 -07005662#endif
5663
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005664#ifdef CONFIG_SLUB_STATS
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005665static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
5666{
5667 unsigned long sum = 0;
5668 int cpu;
Joe Perchesbf16d192020-12-14 19:14:57 -08005669 int len = 0;
Kees Cook6da2ec52018-06-12 13:55:00 -07005670 int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005671
5672 if (!data)
5673 return -ENOMEM;
5674
5675 for_each_online_cpu(cpu) {
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06005676 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005677
5678 data[cpu] = x;
5679 sum += x;
5680 }
5681
Joe Perchesbf16d192020-12-14 19:14:57 -08005682 len += sysfs_emit_at(buf, len, "%lu", sum);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005683
Christoph Lameter50ef37b2008-04-14 18:52:05 +03005684#ifdef CONFIG_SMP
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005685 for_each_online_cpu(cpu) {
Joe Perchesbf16d192020-12-14 19:14:57 -08005686 if (data[cpu])
5687 len += sysfs_emit_at(buf, len, " C%d=%u",
5688 cpu, data[cpu]);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005689 }
Christoph Lameter50ef37b2008-04-14 18:52:05 +03005690#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005691 kfree(data);
Joe Perchesbf16d192020-12-14 19:14:57 -08005692 len += sysfs_emit_at(buf, len, "\n");
5693
5694 return len;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005695}
5696
David Rientjes78eb00c2009-10-15 02:20:22 -07005697static void clear_stat(struct kmem_cache *s, enum stat_item si)
5698{
5699 int cpu;
5700
5701 for_each_online_cpu(cpu)
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06005702 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
David Rientjes78eb00c2009-10-15 02:20:22 -07005703}
5704
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005705#define STAT_ATTR(si, text) \
5706static ssize_t text##_show(struct kmem_cache *s, char *buf) \
5707{ \
5708 return show_stat(s, buf, si); \
5709} \
David Rientjes78eb00c2009-10-15 02:20:22 -07005710static ssize_t text##_store(struct kmem_cache *s, \
5711 const char *buf, size_t length) \
5712{ \
5713 if (buf[0] != '0') \
5714 return -EINVAL; \
5715 clear_stat(s, si); \
5716 return length; \
5717} \
5718SLAB_ATTR(text); \
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005719
5720STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5721STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5722STAT_ATTR(FREE_FASTPATH, free_fastpath);
5723STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5724STAT_ATTR(FREE_FROZEN, free_frozen);
5725STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5726STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5727STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5728STAT_ATTR(ALLOC_SLAB, alloc_slab);
5729STAT_ATTR(ALLOC_REFILL, alloc_refill);
Christoph Lametere36a2652011-06-01 12:25:57 -05005730STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005731STAT_ATTR(FREE_SLAB, free_slab);
5732STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5733STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5734STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5735STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5736STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5737STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
Christoph Lameter03e404a2011-06-01 12:25:58 -05005738STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
Christoph Lameter65c33762008-04-14 19:11:40 +03005739STAT_ATTR(ORDER_FALLBACK, order_fallback);
Christoph Lameterb789ef52011-06-01 12:25:49 -05005740STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5741STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
Christoph Lameter49e22582011-08-09 16:12:27 -05005742STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5743STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
Alex Shi8028dce2012-02-03 23:34:56 +08005744STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5745STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
Tobin C. Harding6dfd1b62019-05-13 17:16:09 -07005746#endif /* CONFIG_SLUB_STATS */
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005747
Pekka Enberg06428782008-01-07 23:20:27 -08005748static struct attribute *slab_attrs[] = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005749 &slab_size_attr.attr,
5750 &object_size_attr.attr,
5751 &objs_per_slab_attr.attr,
5752 &order_attr.attr,
David Rientjes73d342b2009-02-22 17:40:09 -08005753 &min_partial_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005754 &cpu_partial_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005755 &objects_attr.attr,
Christoph Lameter205ab992008-04-14 19:11:40 +03005756 &objects_partial_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005757 &partial_attr.attr,
5758 &cpu_slabs_attr.attr,
5759 &ctor_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005760 &aliases_attr.attr,
5761 &align_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005762 &hwcache_align_attr.attr,
5763 &reclaim_account_attr.attr,
5764 &destroy_by_rcu_attr.attr,
Christoph Lametera5a84752010-10-05 13:57:27 -05005765 &shrink_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005766 &slabs_cpu_partial_attr.attr,
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005767#ifdef CONFIG_SLUB_DEBUG
Christoph Lametera5a84752010-10-05 13:57:27 -05005768 &total_objects_attr.attr,
5769 &slabs_attr.attr,
5770 &sanity_checks_attr.attr,
5771 &trace_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005772 &red_zone_attr.attr,
5773 &poison_attr.attr,
5774 &store_user_attr.attr,
Christoph Lameter53e15af2007-05-06 14:49:43 -07005775 &validate_attr.attr,
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005776#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07005777#ifdef CONFIG_ZONE_DMA
5778 &cache_dma_attr.attr,
5779#endif
5780#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08005781 &remote_node_defrag_ratio_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005782#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005783#ifdef CONFIG_SLUB_STATS
5784 &alloc_fastpath_attr.attr,
5785 &alloc_slowpath_attr.attr,
5786 &free_fastpath_attr.attr,
5787 &free_slowpath_attr.attr,
5788 &free_frozen_attr.attr,
5789 &free_add_partial_attr.attr,
5790 &free_remove_partial_attr.attr,
5791 &alloc_from_partial_attr.attr,
5792 &alloc_slab_attr.attr,
5793 &alloc_refill_attr.attr,
Christoph Lametere36a2652011-06-01 12:25:57 -05005794 &alloc_node_mismatch_attr.attr,
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005795 &free_slab_attr.attr,
5796 &cpuslab_flush_attr.attr,
5797 &deactivate_full_attr.attr,
5798 &deactivate_empty_attr.attr,
5799 &deactivate_to_head_attr.attr,
5800 &deactivate_to_tail_attr.attr,
5801 &deactivate_remote_frees_attr.attr,
Christoph Lameter03e404a2011-06-01 12:25:58 -05005802 &deactivate_bypass_attr.attr,
Christoph Lameter65c33762008-04-14 19:11:40 +03005803 &order_fallback_attr.attr,
Christoph Lameterb789ef52011-06-01 12:25:49 -05005804 &cmpxchg_double_fail_attr.attr,
5805 &cmpxchg_double_cpu_fail_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005806 &cpu_partial_alloc_attr.attr,
5807 &cpu_partial_free_attr.attr,
Alex Shi8028dce2012-02-03 23:34:56 +08005808 &cpu_partial_node_attr.attr,
5809 &cpu_partial_drain_attr.attr,
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005810#endif
Dmitry Monakhov4c13dd32010-02-26 09:36:12 +03005811#ifdef CONFIG_FAILSLAB
5812 &failslab_attr.attr,
5813#endif
David Windsor8eb82842017-06-10 22:50:28 -04005814 &usersize_attr.attr,
Dmitry Monakhov4c13dd32010-02-26 09:36:12 +03005815
Christoph Lameter81819f02007-05-06 14:49:36 -07005816 NULL
5817};
5818
Arvind Yadav1fdaaa22017-09-06 16:21:56 -07005819static const struct attribute_group slab_attr_group = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005820 .attrs = slab_attrs,
5821};
5822
5823static ssize_t slab_attr_show(struct kobject *kobj,
5824 struct attribute *attr,
5825 char *buf)
5826{
5827 struct slab_attribute *attribute;
5828 struct kmem_cache *s;
5829 int err;
5830
5831 attribute = to_slab_attr(attr);
5832 s = to_slab(kobj);
5833
5834 if (!attribute->show)
5835 return -EIO;
5836
5837 err = attribute->show(s, buf);
5838
5839 return err;
5840}
5841
5842static ssize_t slab_attr_store(struct kobject *kobj,
5843 struct attribute *attr,
5844 const char *buf, size_t len)
5845{
5846 struct slab_attribute *attribute;
5847 struct kmem_cache *s;
5848 int err;
5849
5850 attribute = to_slab_attr(attr);
5851 s = to_slab(kobj);
5852
5853 if (!attribute->store)
5854 return -EIO;
5855
5856 err = attribute->store(s, buf, len);
Christoph Lameter81819f02007-05-06 14:49:36 -07005857 return err;
5858}
5859
Christoph Lameter41a21282014-05-06 12:50:08 -07005860static void kmem_cache_release(struct kobject *k)
5861{
5862 slab_kmem_cache_release(to_slab(k));
5863}
5864
Emese Revfy52cf25d2010-01-19 02:58:23 +01005865static const struct sysfs_ops slab_sysfs_ops = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005866 .show = slab_attr_show,
5867 .store = slab_attr_store,
5868};
5869
5870static struct kobj_type slab_ktype = {
5871 .sysfs_ops = &slab_sysfs_ops,
Christoph Lameter41a21282014-05-06 12:50:08 -07005872 .release = kmem_cache_release,
Christoph Lameter81819f02007-05-06 14:49:36 -07005873};
5874
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005875static struct kset *slab_kset;
Christoph Lameter81819f02007-05-06 14:49:36 -07005876
Vladimir Davydov9a417072014-04-07 15:39:31 -07005877static inline struct kset *cache_kset(struct kmem_cache *s)
5878{
Vladimir Davydov9a417072014-04-07 15:39:31 -07005879 return slab_kset;
5880}
5881
Christoph Lameter81819f02007-05-06 14:49:36 -07005882#define ID_STR_LENGTH 64
5883
5884/* Create a unique string id for a slab cache:
Christoph Lameter6446faa2008-02-15 23:45:26 -08005885 *
5886 * Format :[flags-]size
Christoph Lameter81819f02007-05-06 14:49:36 -07005887 */
5888static char *create_unique_id(struct kmem_cache *s)
5889{
5890 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5891 char *p = name;
5892
5893 BUG_ON(!name);
5894
5895 *p++ = ':';
5896 /*
5897 * First flags affecting slabcache operations. We will only
5898 * get here for aliasable slabs so we do not need to support
5899 * too many flags. The flags here must cover all flags that
5900 * are matched during merging to guarantee that the id is
5901 * unique.
5902 */
5903 if (s->flags & SLAB_CACHE_DMA)
5904 *p++ = 'd';
Nicolas Boichat6d6ea1e2019-03-28 20:43:42 -07005905 if (s->flags & SLAB_CACHE_DMA32)
5906 *p++ = 'D';
Christoph Lameter81819f02007-05-06 14:49:36 -07005907 if (s->flags & SLAB_RECLAIM_ACCOUNT)
5908 *p++ = 'a';
Laura Abbottbecfda62016-03-15 14:55:06 -07005909 if (s->flags & SLAB_CONSISTENCY_CHECKS)
Christoph Lameter81819f02007-05-06 14:49:36 -07005910 *p++ = 'F';
Vladimir Davydov230e9fc2016-01-14 15:18:15 -08005911 if (s->flags & SLAB_ACCOUNT)
5912 *p++ = 'A';
Christoph Lameter81819f02007-05-06 14:49:36 -07005913 if (p != name + 1)
5914 *p++ = '-';
Alexey Dobriyan44065b22018-04-05 16:21:20 -07005915 p += sprintf(p, "%07u", s->size);
Glauber Costa2633d7a2012-12-18 14:22:34 -08005916
Christoph Lameter81819f02007-05-06 14:49:36 -07005917 BUG_ON(p > name + ID_STR_LENGTH - 1);
5918 return name;
5919}
5920
5921static int sysfs_slab_add(struct kmem_cache *s)
5922{
5923 int err;
5924 const char *name;
Tejun Heo1663f262017-02-22 15:41:39 -08005925 struct kset *kset = cache_kset(s);
Christoph Lameter45530c42012-11-28 16:23:07 +00005926 int unmergeable = slab_unmergeable(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07005927
Tejun Heo1663f262017-02-22 15:41:39 -08005928 if (!kset) {
5929 kobject_init(&s->kobj, &slab_ktype);
5930 return 0;
5931 }
5932
Miles Chen11066382017-11-15 17:32:25 -08005933 if (!unmergeable && disable_higher_order_debug &&
5934 (slub_debug & DEBUG_METADATA_FLAGS))
5935 unmergeable = 1;
5936
Christoph Lameter81819f02007-05-06 14:49:36 -07005937 if (unmergeable) {
5938 /*
5939 * Slabcache can never be merged so we can use the name proper.
5940 * This is typically the case for debug situations. In that
5941 * case we can catch duplicate names easily.
5942 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005943 sysfs_remove_link(&slab_kset->kobj, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005944 name = s->name;
5945 } else {
5946 /*
5947 * Create a unique name for the slab as a target
5948 * for the symlinks.
5949 */
5950 name = create_unique_id(s);
5951 }
5952
Tejun Heo1663f262017-02-22 15:41:39 -08005953 s->kobj.kset = kset;
Tetsuo Handa26e4f202014-01-04 16:32:31 +09005954 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
Wang Hai757fed12021-01-28 19:32:50 +08005955 if (err)
Konstantin Khlebnikov80da0262015-09-04 15:45:51 -07005956 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07005957
5958 err = sysfs_create_group(&s->kobj, &slab_attr_group);
Dave Jones54b6a732014-04-07 15:39:32 -07005959 if (err)
5960 goto out_del_kobj;
Vladimir Davydov9a417072014-04-07 15:39:31 -07005961
Christoph Lameter81819f02007-05-06 14:49:36 -07005962 if (!unmergeable) {
5963 /* Setup first alias */
5964 sysfs_slab_alias(s, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005965 }
Dave Jones54b6a732014-04-07 15:39:32 -07005966out:
5967 if (!unmergeable)
5968 kfree(name);
5969 return err;
5970out_del_kobj:
5971 kobject_del(&s->kobj);
Dave Jones54b6a732014-04-07 15:39:32 -07005972 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07005973}
5974
Mikulas Patockad50d82f2018-06-27 23:26:09 -07005975void sysfs_slab_unlink(struct kmem_cache *s)
5976{
5977 if (slab_state >= FULL)
5978 kobject_del(&s->kobj);
5979}
5980
Tejun Heobf5eb3d2017-02-22 15:41:11 -08005981void sysfs_slab_release(struct kmem_cache *s)
5982{
5983 if (slab_state >= FULL)
5984 kobject_put(&s->kobj);
Christoph Lameter81819f02007-05-06 14:49:36 -07005985}
5986
5987/*
5988 * Need to buffer aliases during bootup until sysfs becomes
Nick Andrew9f6c708e2008-12-05 14:08:08 +11005989 * available lest we lose that information.
Christoph Lameter81819f02007-05-06 14:49:36 -07005990 */
5991struct saved_alias {
5992 struct kmem_cache *s;
5993 const char *name;
5994 struct saved_alias *next;
5995};
5996
Adrian Bunk5af328a2007-07-17 04:03:27 -07005997static struct saved_alias *alias_list;
Christoph Lameter81819f02007-05-06 14:49:36 -07005998
5999static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
6000{
6001 struct saved_alias *al;
6002
Christoph Lameter97d06602012-07-06 15:25:11 -05006003 if (slab_state == FULL) {
Christoph Lameter81819f02007-05-06 14:49:36 -07006004 /*
6005 * If we have a leftover link then remove it.
6006 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06006007 sysfs_remove_link(&slab_kset->kobj, name);
6008 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
Christoph Lameter81819f02007-05-06 14:49:36 -07006009 }
6010
6011 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
6012 if (!al)
6013 return -ENOMEM;
6014
6015 al->s = s;
6016 al->name = name;
6017 al->next = alias_list;
6018 alias_list = al;
6019 return 0;
6020}
6021
6022static int __init slab_sysfs_init(void)
6023{
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07006024 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07006025 int err;
6026
Christoph Lameter18004c52012-07-06 15:25:12 -05006027 mutex_lock(&slab_mutex);
Christoph Lameter2bce6482010-07-19 11:39:11 -05006028
Christoph Lameterd7660ce2020-06-01 21:45:50 -07006029 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06006030 if (!slab_kset) {
Christoph Lameter18004c52012-07-06 15:25:12 -05006031 mutex_unlock(&slab_mutex);
Fabian Frederickf9f58282014-06-04 16:06:34 -07006032 pr_err("Cannot register slab subsystem.\n");
Christoph Lameter81819f02007-05-06 14:49:36 -07006033 return -ENOSYS;
6034 }
6035
Christoph Lameter97d06602012-07-06 15:25:11 -05006036 slab_state = FULL;
Christoph Lameter26a7bd02007-05-09 02:32:39 -07006037
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07006038 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter26a7bd02007-05-09 02:32:39 -07006039 err = sysfs_slab_add(s);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07006040 if (err)
Fabian Frederickf9f58282014-06-04 16:06:34 -07006041 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
6042 s->name);
Christoph Lameter26a7bd02007-05-09 02:32:39 -07006043 }
Christoph Lameter81819f02007-05-06 14:49:36 -07006044
6045 while (alias_list) {
6046 struct saved_alias *al = alias_list;
6047
6048 alias_list = alias_list->next;
6049 err = sysfs_slab_alias(al->s, al->name);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07006050 if (err)
Fabian Frederickf9f58282014-06-04 16:06:34 -07006051 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
6052 al->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07006053 kfree(al);
6054 }
6055
Christoph Lameter18004c52012-07-06 15:25:12 -05006056 mutex_unlock(&slab_mutex);
Christoph Lameter81819f02007-05-06 14:49:36 -07006057 return 0;
6058}
6059
6060__initcall(slab_sysfs_init);
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05006061#endif /* CONFIG_SYSFS */
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006062
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006063#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
6064static int slab_debugfs_show(struct seq_file *seq, void *v)
6065{
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006066 struct loc_track *t = seq->private;
Gerald Schaefer005a79e2021-12-10 14:47:02 -08006067 struct location *l;
6068 unsigned long idx;
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006069
Gerald Schaefer005a79e2021-12-10 14:47:02 -08006070 idx = (unsigned long) t->idx;
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006071 if (idx < t->count) {
6072 l = &t->loc[idx];
6073
6074 seq_printf(seq, "%7ld ", l->count);
6075
6076 if (l->addr)
6077 seq_printf(seq, "%pS", (void *)l->addr);
6078 else
6079 seq_puts(seq, "<not-available>");
6080
6081 if (l->sum_time != l->min_time) {
6082 seq_printf(seq, " age=%ld/%llu/%ld",
6083 l->min_time, div_u64(l->sum_time, l->count),
6084 l->max_time);
6085 } else
6086 seq_printf(seq, " age=%ld", l->min_time);
6087
6088 if (l->min_pid != l->max_pid)
6089 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
6090 else
6091 seq_printf(seq, " pid=%ld",
6092 l->min_pid);
6093
6094 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
6095 seq_printf(seq, " cpus=%*pbl",
6096 cpumask_pr_args(to_cpumask(l->cpus)));
6097
6098 if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
6099 seq_printf(seq, " nodes=%*pbl",
6100 nodemask_pr_args(&l->nodes));
6101
Oliver Glitta8ea9fb92021-05-21 14:11:25 +02006102#ifdef CONFIG_STACKDEPOT
6103 {
6104 depot_stack_handle_t handle;
6105 unsigned long *entries;
6106 unsigned int nr_entries, j;
6107
6108 handle = READ_ONCE(l->handle);
6109 if (handle) {
6110 nr_entries = stack_depot_fetch(handle, &entries);
6111 seq_puts(seq, "\n");
6112 for (j = 0; j < nr_entries; j++)
6113 seq_printf(seq, " %pS\n", (void *)entries[j]);
6114 }
6115 }
6116#endif
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006117 seq_puts(seq, "\n");
6118 }
6119
6120 if (!idx && !t->count)
6121 seq_puts(seq, "No data\n");
6122
6123 return 0;
6124}
6125
6126static void slab_debugfs_stop(struct seq_file *seq, void *v)
6127{
6128}
6129
6130static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
6131{
6132 struct loc_track *t = seq->private;
6133
Gerald Schaefer005a79e2021-12-10 14:47:02 -08006134 t->idx = ++(*ppos);
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006135 if (*ppos <= t->count)
Gerald Schaefer005a79e2021-12-10 14:47:02 -08006136 return ppos;
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006137
6138 return NULL;
6139}
6140
Oliver Glitta553c0362021-05-21 14:11:26 +02006141static int cmp_loc_by_count(const void *a, const void *b, const void *data)
6142{
6143 struct location *loc1 = (struct location *)a;
6144 struct location *loc2 = (struct location *)b;
6145
6146 if (loc1->count > loc2->count)
6147 return -1;
6148 else
6149 return 1;
6150}
6151
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006152static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
6153{
Gerald Schaefer005a79e2021-12-10 14:47:02 -08006154 struct loc_track *t = seq->private;
6155
6156 t->idx = *ppos;
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006157 return ppos;
6158}
6159
6160static const struct seq_operations slab_debugfs_sops = {
6161 .start = slab_debugfs_start,
6162 .next = slab_debugfs_next,
6163 .stop = slab_debugfs_stop,
6164 .show = slab_debugfs_show,
6165};
6166
6167static int slab_debug_trace_open(struct inode *inode, struct file *filep)
6168{
6169
6170 struct kmem_cache_node *n;
6171 enum track_item alloc;
6172 int node;
6173 struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
6174 sizeof(struct loc_track));
6175 struct kmem_cache *s = file_inode(filep)->i_private;
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02006176 unsigned long *obj_map;
6177
Miaohe Lin2127d222021-10-18 15:15:52 -07006178 if (!t)
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02006179 return -ENOMEM;
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006180
Miaohe Lin2127d222021-10-18 15:15:52 -07006181 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
6182 if (!obj_map) {
6183 seq_release_private(inode, filep);
6184 return -ENOMEM;
6185 }
6186
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006187 if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
6188 alloc = TRACK_ALLOC;
6189 else
6190 alloc = TRACK_FREE;
6191
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02006192 if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
6193 bitmap_free(obj_map);
Miaohe Lin2127d222021-10-18 15:15:52 -07006194 seq_release_private(inode, filep);
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006195 return -ENOMEM;
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02006196 }
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006197
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006198 for_each_kmem_cache_node(s, node, n) {
6199 unsigned long flags;
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01006200 struct slab *slab;
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006201
6202 if (!atomic_long_read(&n->nr_slabs))
6203 continue;
6204
6205 spin_lock_irqsave(&n->list_lock, flags);
Vlastimil Babkabb192ed2021-11-03 15:39:59 +01006206 list_for_each_entry(slab, &n->partial, slab_list)
6207 process_slab(t, s, slab, alloc, obj_map);
6208 list_for_each_entry(slab, &n->full, slab_list)
6209 process_slab(t, s, slab, alloc, obj_map);
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006210 spin_unlock_irqrestore(&n->list_lock, flags);
6211 }
6212
Oliver Glitta553c0362021-05-21 14:11:26 +02006213 /* Sort locations by count */
6214 sort_r(t->loc, t->count, sizeof(struct location),
6215 cmp_loc_by_count, NULL, NULL);
6216
Vlastimil Babkab3fd64e2021-05-23 01:28:37 +02006217 bitmap_free(obj_map);
Faiyaz Mohammed64dd6842021-06-28 19:34:55 -07006218 return 0;
6219}
6220
6221static int slab_debug_trace_release(struct inode *inode, struct file *file)
6222{
6223 struct seq_file *seq = file->private_data;
6224 struct loc_track *t = seq->private;
6225
6226 free_loc_track(t);
6227 return seq_release_private(inode, file);
6228}
6229
6230static const struct file_operations slab_debugfs_fops = {
6231 .open = slab_debug_trace_open,
6232 .read = seq_read,
6233 .llseek = seq_lseek,
6234 .release = slab_debug_trace_release,
6235};
6236
6237static void debugfs_slab_add(struct kmem_cache *s)
6238{
6239 struct dentry *slab_cache_dir;
6240
6241 if (unlikely(!slab_debugfs_root))
6242 return;
6243
6244 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
6245
6246 debugfs_create_file("alloc_traces", 0400,
6247 slab_cache_dir, s, &slab_debugfs_fops);
6248
6249 debugfs_create_file("free_traces", 0400,
6250 slab_cache_dir, s, &slab_debugfs_fops);
6251}
6252
6253void debugfs_slab_release(struct kmem_cache *s)
6254{
6255 debugfs_remove_recursive(debugfs_lookup(s->name, slab_debugfs_root));
6256}
6257
6258static int __init slab_debugfs_init(void)
6259{
6260 struct kmem_cache *s;
6261
6262 slab_debugfs_root = debugfs_create_dir("slab", NULL);
6263
6264 list_for_each_entry(s, &slab_caches, list)
6265 if (s->flags & SLAB_STORE_USER)
6266 debugfs_slab_add(s);
6267
6268 return 0;
6269
6270}
6271__initcall(slab_debugfs_init);
6272#endif
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006273/*
6274 * The /proc/slabinfo ABI
6275 */
Yang Shi5b365772017-11-15 17:32:03 -08006276#ifdef CONFIG_SLUB_DEBUG
Glauber Costa0d7561c2012-10-19 18:20:27 +04006277void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006278{
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006279 unsigned long nr_slabs = 0;
Christoph Lameter205ab992008-04-14 19:11:40 +03006280 unsigned long nr_objs = 0;
6281 unsigned long nr_free = 0;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006282 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07006283 struct kmem_cache_node *n;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006284
Christoph Lameterfa45dc22014-08-06 16:04:09 -07006285 for_each_kmem_cache_node(s, node, n) {
Wanpeng Lic17fd132013-07-04 08:33:26 +08006286 nr_slabs += node_nr_slabs(n);
6287 nr_objs += node_nr_objs(n);
Christoph Lameter205ab992008-04-14 19:11:40 +03006288 nr_free += count_partial(n, count_free);
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006289 }
6290
Glauber Costa0d7561c2012-10-19 18:20:27 +04006291 sinfo->active_objs = nr_objs - nr_free;
6292 sinfo->num_objs = nr_objs;
6293 sinfo->active_slabs = nr_slabs;
6294 sinfo->num_slabs = nr_slabs;
6295 sinfo->objects_per_slab = oo_objects(s->oo);
6296 sinfo->cache_order = oo_order(s->oo);
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01006297}
6298
Glauber Costa0d7561c2012-10-19 18:20:27 +04006299void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04006300{
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04006301}
6302
Glauber Costab7454ad2012-10-19 18:20:25 +04006303ssize_t slabinfo_write(struct file *file, const char __user *buffer,
6304 size_t count, loff_t *ppos)
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04006305{
Glauber Costab7454ad2012-10-19 18:20:25 +04006306 return -EIO;
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04006307}
Yang Shi5b365772017-11-15 17:32:03 -08006308#endif /* CONFIG_SLUB_DEBUG */