Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 2 | #ifndef MM_SLAB_H |
| 3 | #define MM_SLAB_H |
| 4 | /* |
| 5 | * Internal slab definitions |
| 6 | */ |
| 7 | |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 8 | /* Reuses the bits in struct page */ |
| 9 | struct slab { |
| 10 | unsigned long __page_flags; |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 11 | |
| 12 | #if defined(CONFIG_SLAB) |
| 13 | |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 14 | union { |
| 15 | struct list_head slab_list; |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 16 | struct rcu_head rcu_head; |
| 17 | }; |
| 18 | struct kmem_cache *slab_cache; |
| 19 | void *freelist; /* array of free object indexes */ |
| 20 | void *s_mem; /* first object */ |
| 21 | unsigned int active; |
| 22 | |
| 23 | #elif defined(CONFIG_SLUB) |
| 24 | |
| 25 | union { |
| 26 | struct list_head slab_list; |
| 27 | struct rcu_head rcu_head; |
Vlastimil Babka | 9c01e9a | 2021-11-10 14:12:45 +0100 | [diff] [blame] | 28 | #ifdef CONFIG_SLUB_CPU_PARTIAL |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 29 | struct { |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 30 | struct slab *next; |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 31 | int slabs; /* Nr of slabs left */ |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 32 | }; |
Vlastimil Babka | 9c01e9a | 2021-11-10 14:12:45 +0100 | [diff] [blame] | 33 | #endif |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 34 | }; |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 35 | struct kmem_cache *slab_cache; |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 36 | /* Double-word boundary */ |
| 37 | void *freelist; /* first free object */ |
| 38 | union { |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 39 | unsigned long counters; |
| 40 | struct { |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 41 | unsigned inuse:16; |
| 42 | unsigned objects:15; |
| 43 | unsigned frozen:1; |
| 44 | }; |
| 45 | }; |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 46 | unsigned int __unused; |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 47 | |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 48 | #elif defined(CONFIG_SLOB) |
| 49 | |
| 50 | struct list_head slab_list; |
| 51 | void *__unused_1; |
| 52 | void *freelist; /* first free block */ |
Hyeonggon Yoo | b01af5c | 2021-12-12 06:52:41 +0000 | [diff] [blame] | 53 | long units; |
| 54 | unsigned int __unused_2; |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 55 | |
| 56 | #else |
| 57 | #error "Unexpected slab allocator configured" |
| 58 | #endif |
| 59 | |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 60 | atomic_t __page_refcount; |
| 61 | #ifdef CONFIG_MEMCG |
| 62 | unsigned long memcg_data; |
| 63 | #endif |
| 64 | }; |
| 65 | |
| 66 | #define SLAB_MATCH(pg, sl) \ |
| 67 | static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) |
| 68 | SLAB_MATCH(flags, __page_flags); |
| 69 | SLAB_MATCH(compound_head, slab_list); /* Ensure bit 0 is clear */ |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 70 | #ifndef CONFIG_SLOB |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 71 | SLAB_MATCH(rcu_head, rcu_head); |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 72 | #endif |
Matthew Wilcox (Oracle) | d122019 | 2021-10-04 14:45:51 +0100 | [diff] [blame] | 73 | SLAB_MATCH(_refcount, __page_refcount); |
| 74 | #ifdef CONFIG_MEMCG |
| 75 | SLAB_MATCH(memcg_data, memcg_data); |
| 76 | #endif |
| 77 | #undef SLAB_MATCH |
| 78 | static_assert(sizeof(struct slab) <= sizeof(struct page)); |
| 79 | |
| 80 | /** |
| 81 | * folio_slab - Converts from folio to slab. |
| 82 | * @folio: The folio. |
| 83 | * |
| 84 | * Currently struct slab is a different representation of a folio where |
| 85 | * folio_test_slab() is true. |
| 86 | * |
| 87 | * Return: The slab which contains this folio. |
| 88 | */ |
| 89 | #define folio_slab(folio) (_Generic((folio), \ |
| 90 | const struct folio *: (const struct slab *)(folio), \ |
| 91 | struct folio *: (struct slab *)(folio))) |
| 92 | |
| 93 | /** |
| 94 | * slab_folio - The folio allocated for a slab |
| 95 | * @slab: The slab. |
| 96 | * |
| 97 | * Slabs are allocated as folios that contain the individual objects and are |
| 98 | * using some fields in the first struct page of the folio - those fields are |
| 99 | * now accessed by struct slab. It is occasionally necessary to convert back to |
| 100 | * a folio in order to communicate with the rest of the mm. Please use this |
| 101 | * helper function instead of casting yourself, as the implementation may change |
| 102 | * in the future. |
| 103 | */ |
| 104 | #define slab_folio(s) (_Generic((s), \ |
| 105 | const struct slab *: (const struct folio *)s, \ |
| 106 | struct slab *: (struct folio *)s)) |
| 107 | |
| 108 | /** |
| 109 | * page_slab - Converts from first struct page to slab. |
| 110 | * @p: The first (either head of compound or single) page of slab. |
| 111 | * |
| 112 | * A temporary wrapper to convert struct page to struct slab in situations where |
| 113 | * we know the page is the compound head, or single order-0 page. |
| 114 | * |
| 115 | * Long-term ideally everything would work with struct slab directly or go |
| 116 | * through folio to struct slab. |
| 117 | * |
| 118 | * Return: The slab which contains this page |
| 119 | */ |
| 120 | #define page_slab(p) (_Generic((p), \ |
| 121 | const struct page *: (const struct slab *)(p), \ |
| 122 | struct page *: (struct slab *)(p))) |
| 123 | |
| 124 | /** |
| 125 | * slab_page - The first struct page allocated for a slab |
| 126 | * @slab: The slab. |
| 127 | * |
| 128 | * A convenience wrapper for converting slab to the first struct page of the |
| 129 | * underlying folio, to communicate with code not yet converted to folio or |
| 130 | * struct slab. |
| 131 | */ |
| 132 | #define slab_page(s) folio_page(slab_folio(s), 0) |
| 133 | |
| 134 | /* |
| 135 | * If network-based swap is enabled, sl*b must keep track of whether pages |
| 136 | * were allocated from pfmemalloc reserves. |
| 137 | */ |
| 138 | static inline bool slab_test_pfmemalloc(const struct slab *slab) |
| 139 | { |
| 140 | return folio_test_active((struct folio *)slab_folio(slab)); |
| 141 | } |
| 142 | |
| 143 | static inline void slab_set_pfmemalloc(struct slab *slab) |
| 144 | { |
| 145 | folio_set_active(slab_folio(slab)); |
| 146 | } |
| 147 | |
| 148 | static inline void slab_clear_pfmemalloc(struct slab *slab) |
| 149 | { |
| 150 | folio_clear_active(slab_folio(slab)); |
| 151 | } |
| 152 | |
| 153 | static inline void __slab_clear_pfmemalloc(struct slab *slab) |
| 154 | { |
| 155 | __folio_clear_active(slab_folio(slab)); |
| 156 | } |
| 157 | |
| 158 | static inline void *slab_address(const struct slab *slab) |
| 159 | { |
| 160 | return folio_address(slab_folio(slab)); |
| 161 | } |
| 162 | |
| 163 | static inline int slab_nid(const struct slab *slab) |
| 164 | { |
| 165 | return folio_nid(slab_folio(slab)); |
| 166 | } |
| 167 | |
| 168 | static inline pg_data_t *slab_pgdat(const struct slab *slab) |
| 169 | { |
| 170 | return folio_pgdat(slab_folio(slab)); |
| 171 | } |
| 172 | |
| 173 | static inline struct slab *virt_to_slab(const void *addr) |
| 174 | { |
| 175 | struct folio *folio = virt_to_folio(addr); |
| 176 | |
| 177 | if (!folio_test_slab(folio)) |
| 178 | return NULL; |
| 179 | |
| 180 | return folio_slab(folio); |
| 181 | } |
| 182 | |
| 183 | static inline int slab_order(const struct slab *slab) |
| 184 | { |
| 185 | return folio_order((struct folio *)slab_folio(slab)); |
| 186 | } |
| 187 | |
| 188 | static inline size_t slab_size(const struct slab *slab) |
| 189 | { |
| 190 | return PAGE_SIZE << slab_order(slab); |
| 191 | } |
| 192 | |
Joonsoo Kim | 07f361b | 2014-10-09 15:26:00 -0700 | [diff] [blame] | 193 | #ifdef CONFIG_SLOB |
| 194 | /* |
| 195 | * Common fields provided in kmem_cache by all slab allocators |
| 196 | * This struct is either used directly by the allocator (SLOB) |
| 197 | * or the allocator must include definitions for all fields |
| 198 | * provided in kmem_cache_common in their definition of kmem_cache. |
| 199 | * |
| 200 | * Once we can do anonymous structs (C11 standard) we could put a |
| 201 | * anonymous struct definition in these allocators so that the |
| 202 | * separate allocations in the kmem_cache structure of SLAB and |
| 203 | * SLUB is no longer needed. |
| 204 | */ |
| 205 | struct kmem_cache { |
| 206 | unsigned int object_size;/* The original size of the object */ |
| 207 | unsigned int size; /* The aligned/padded/added on size */ |
| 208 | unsigned int align; /* Alignment as calculated */ |
Alexey Dobriyan | d50112e | 2017-11-15 17:32:18 -0800 | [diff] [blame] | 209 | slab_flags_t flags; /* Active flags on the slab */ |
Alexey Dobriyan | 7bbdb81 | 2018-04-05 16:21:31 -0700 | [diff] [blame] | 210 | unsigned int useroffset;/* Usercopy region offset */ |
| 211 | unsigned int usersize; /* Usercopy region size */ |
Joonsoo Kim | 07f361b | 2014-10-09 15:26:00 -0700 | [diff] [blame] | 212 | const char *name; /* Slab name for sysfs */ |
| 213 | int refcount; /* Use counter */ |
| 214 | void (*ctor)(void *); /* Called on object slot creation */ |
| 215 | struct list_head list; /* List of all slab caches on the system */ |
| 216 | }; |
| 217 | |
| 218 | #endif /* CONFIG_SLOB */ |
| 219 | |
| 220 | #ifdef CONFIG_SLAB |
| 221 | #include <linux/slab_def.h> |
| 222 | #endif |
| 223 | |
| 224 | #ifdef CONFIG_SLUB |
| 225 | #include <linux/slub_def.h> |
| 226 | #endif |
| 227 | |
| 228 | #include <linux/memcontrol.h> |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 229 | #include <linux/fault-inject.h> |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 230 | #include <linux/kasan.h> |
| 231 | #include <linux/kmemleak.h> |
Thomas Garnier | 7c00fce | 2016-07-26 15:21:56 -0700 | [diff] [blame] | 232 | #include <linux/random.h> |
Peter Zijlstra | d92a8cf | 2017-03-03 10:13:38 +0100 | [diff] [blame] | 233 | #include <linux/sched/mm.h> |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 234 | #include <linux/list_lru.h> |
Joonsoo Kim | 07f361b | 2014-10-09 15:26:00 -0700 | [diff] [blame] | 235 | |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 236 | /* |
| 237 | * State of the slab allocator. |
| 238 | * |
| 239 | * This is used to describe the states of the allocator during bootup. |
| 240 | * Allocators use this to gradually bootstrap themselves. Most allocators |
| 241 | * have the problem that the structures used for managing slab caches are |
| 242 | * allocated from slab caches themselves. |
| 243 | */ |
| 244 | enum slab_state { |
| 245 | DOWN, /* No slab functionality yet */ |
| 246 | PARTIAL, /* SLUB: kmem_cache_node available */ |
Christoph Lameter | ce8eb6c | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 247 | PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */ |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 248 | UP, /* Slab caches usable but not all extras yet */ |
| 249 | FULL /* Everything is working */ |
| 250 | }; |
| 251 | |
| 252 | extern enum slab_state slab_state; |
| 253 | |
Christoph Lameter | 18004c5 | 2012-07-06 15:25:12 -0500 | [diff] [blame] | 254 | /* The slab cache mutex protects the management structures during changes */ |
| 255 | extern struct mutex slab_mutex; |
Christoph Lameter | 9b030cb | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 256 | |
| 257 | /* The list of all slab caches on the system */ |
Christoph Lameter | 18004c5 | 2012-07-06 15:25:12 -0500 | [diff] [blame] | 258 | extern struct list_head slab_caches; |
| 259 | |
Christoph Lameter | 9b030cb | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 260 | /* The slab cache that manages slab cache information */ |
| 261 | extern struct kmem_cache *kmem_cache; |
| 262 | |
Vlastimil Babka | af3b5f8 | 2017-02-22 15:41:05 -0800 | [diff] [blame] | 263 | /* A table of kmalloc cache names and sizes */ |
| 264 | extern const struct kmalloc_info_struct { |
Pengfei Li | cb5d9fb | 2019-11-30 17:49:21 -0800 | [diff] [blame] | 265 | const char *name[NR_KMALLOC_TYPES]; |
Alexey Dobriyan | 55de8b9 | 2018-04-05 16:20:29 -0700 | [diff] [blame] | 266 | unsigned int size; |
Vlastimil Babka | af3b5f8 | 2017-02-22 15:41:05 -0800 | [diff] [blame] | 267 | } kmalloc_info[]; |
| 268 | |
Christoph Lameter | f97d5f63 | 2013-01-10 19:12:17 +0000 | [diff] [blame] | 269 | #ifndef CONFIG_SLOB |
| 270 | /* Kmalloc array related functions */ |
Daniel Sanders | 34cc699 | 2015-06-24 16:55:57 -0700 | [diff] [blame] | 271 | void setup_kmalloc_cache_index_table(void); |
Alexey Dobriyan | d50112e | 2017-11-15 17:32:18 -0800 | [diff] [blame] | 272 | void create_kmalloc_caches(slab_flags_t); |
Christoph Lameter | 2c59dd6 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 273 | |
| 274 | /* Find the kmalloc slab corresponding for a certain size */ |
| 275 | struct kmem_cache *kmalloc_slab(size_t, gfp_t); |
Christoph Lameter | f97d5f63 | 2013-01-10 19:12:17 +0000 | [diff] [blame] | 276 | #endif |
| 277 | |
Long Li | 4440509 | 2020-08-06 23:18:28 -0700 | [diff] [blame] | 278 | gfp_t kmalloc_fix_flags(gfp_t flags); |
Christoph Lameter | f97d5f63 | 2013-01-10 19:12:17 +0000 | [diff] [blame] | 279 | |
Christoph Lameter | 9b030cb | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 280 | /* Functions provided by the slab allocators */ |
Alexey Dobriyan | d50112e | 2017-11-15 17:32:18 -0800 | [diff] [blame] | 281 | int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags); |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 282 | |
Alexey Dobriyan | 55de8b9 | 2018-04-05 16:20:29 -0700 | [diff] [blame] | 283 | struct kmem_cache *create_kmalloc_cache(const char *name, unsigned int size, |
| 284 | slab_flags_t flags, unsigned int useroffset, |
| 285 | unsigned int usersize); |
Christoph Lameter | 45530c4 | 2012-11-28 16:23:07 +0000 | [diff] [blame] | 286 | extern void create_boot_cache(struct kmem_cache *, const char *name, |
Alexey Dobriyan | 361d575 | 2018-04-05 16:20:33 -0700 | [diff] [blame] | 287 | unsigned int size, slab_flags_t flags, |
| 288 | unsigned int useroffset, unsigned int usersize); |
Christoph Lameter | 45530c4 | 2012-11-28 16:23:07 +0000 | [diff] [blame] | 289 | |
Joonsoo Kim | 423c929 | 2014-10-09 15:26:22 -0700 | [diff] [blame] | 290 | int slab_unmergeable(struct kmem_cache *s); |
Alexey Dobriyan | f4957d5 | 2018-04-05 16:20:37 -0700 | [diff] [blame] | 291 | struct kmem_cache *find_mergeable(unsigned size, unsigned align, |
Alexey Dobriyan | d50112e | 2017-11-15 17:32:18 -0800 | [diff] [blame] | 292 | slab_flags_t flags, const char *name, void (*ctor)(void *)); |
Joonsoo Kim | 12220de | 2014-10-09 15:26:24 -0700 | [diff] [blame] | 293 | #ifndef CONFIG_SLOB |
Glauber Costa | 2633d7a | 2012-12-18 14:22:34 -0800 | [diff] [blame] | 294 | struct kmem_cache * |
Alexey Dobriyan | f4957d5 | 2018-04-05 16:20:37 -0700 | [diff] [blame] | 295 | __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, |
Alexey Dobriyan | d50112e | 2017-11-15 17:32:18 -0800 | [diff] [blame] | 296 | slab_flags_t flags, void (*ctor)(void *)); |
Joonsoo Kim | 423c929 | 2014-10-09 15:26:22 -0700 | [diff] [blame] | 297 | |
Alexey Dobriyan | 0293d1f | 2018-04-05 16:21:24 -0700 | [diff] [blame] | 298 | slab_flags_t kmem_cache_flags(unsigned int object_size, |
Nikolay Borisov | 3754000 | 2021-02-24 12:00:58 -0800 | [diff] [blame] | 299 | slab_flags_t flags, const char *name); |
Christoph Lameter | cbb7969 | 2012-09-05 00:18:32 +0000 | [diff] [blame] | 300 | #else |
Glauber Costa | 2633d7a | 2012-12-18 14:22:34 -0800 | [diff] [blame] | 301 | static inline struct kmem_cache * |
Alexey Dobriyan | f4957d5 | 2018-04-05 16:20:37 -0700 | [diff] [blame] | 302 | __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, |
Alexey Dobriyan | d50112e | 2017-11-15 17:32:18 -0800 | [diff] [blame] | 303 | slab_flags_t flags, void (*ctor)(void *)) |
Christoph Lameter | cbb7969 | 2012-09-05 00:18:32 +0000 | [diff] [blame] | 304 | { return NULL; } |
Joonsoo Kim | 423c929 | 2014-10-09 15:26:22 -0700 | [diff] [blame] | 305 | |
Alexey Dobriyan | 0293d1f | 2018-04-05 16:21:24 -0700 | [diff] [blame] | 306 | static inline slab_flags_t kmem_cache_flags(unsigned int object_size, |
Nikolay Borisov | 3754000 | 2021-02-24 12:00:58 -0800 | [diff] [blame] | 307 | slab_flags_t flags, const char *name) |
Joonsoo Kim | 423c929 | 2014-10-09 15:26:22 -0700 | [diff] [blame] | 308 | { |
| 309 | return flags; |
| 310 | } |
Christoph Lameter | cbb7969 | 2012-09-05 00:18:32 +0000 | [diff] [blame] | 311 | #endif |
| 312 | |
| 313 | |
Glauber Costa | d884392 | 2012-10-17 15:36:51 +0400 | [diff] [blame] | 314 | /* Legal flag mask for kmem_cache_create(), for various configurations */ |
Nicolas Boichat | 6d6ea1e | 2019-03-28 20:43:42 -0700 | [diff] [blame] | 315 | #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \ |
| 316 | SLAB_CACHE_DMA32 | SLAB_PANIC | \ |
Paul E. McKenney | 5f0d5a3 | 2017-01-18 02:53:44 -0800 | [diff] [blame] | 317 | SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS ) |
Glauber Costa | d884392 | 2012-10-17 15:36:51 +0400 | [diff] [blame] | 318 | |
| 319 | #if defined(CONFIG_DEBUG_SLAB) |
| 320 | #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) |
| 321 | #elif defined(CONFIG_SLUB_DEBUG) |
| 322 | #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ |
Laura Abbott | becfda6 | 2016-03-15 14:55:06 -0700 | [diff] [blame] | 323 | SLAB_TRACE | SLAB_CONSISTENCY_CHECKS) |
Glauber Costa | d884392 | 2012-10-17 15:36:51 +0400 | [diff] [blame] | 324 | #else |
| 325 | #define SLAB_DEBUG_FLAGS (0) |
| 326 | #endif |
| 327 | |
| 328 | #if defined(CONFIG_SLAB) |
| 329 | #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \ |
Vladimir Davydov | 230e9fc | 2016-01-14 15:18:15 -0800 | [diff] [blame] | 330 | SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \ |
Levin, Alexander (Sasha Levin) | 75f296d | 2017-11-15 17:35:54 -0800 | [diff] [blame] | 331 | SLAB_ACCOUNT) |
Glauber Costa | d884392 | 2012-10-17 15:36:51 +0400 | [diff] [blame] | 332 | #elif defined(CONFIG_SLUB) |
| 333 | #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ |
Hyeonggon Yoo | a285909 | 2022-04-06 15:00:03 +0900 | [diff] [blame] | 334 | SLAB_TEMPORARY | SLAB_ACCOUNT | SLAB_NO_USER_FLAGS) |
Glauber Costa | d884392 | 2012-10-17 15:36:51 +0400 | [diff] [blame] | 335 | #else |
Rustam Kovhaev | 34dbc3aa | 2021-11-19 16:43:37 -0800 | [diff] [blame] | 336 | #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE) |
Glauber Costa | d884392 | 2012-10-17 15:36:51 +0400 | [diff] [blame] | 337 | #endif |
| 338 | |
Thomas Garnier | e70954f | 2016-12-12 16:41:38 -0800 | [diff] [blame] | 339 | /* Common flags available with current configuration */ |
Glauber Costa | d884392 | 2012-10-17 15:36:51 +0400 | [diff] [blame] | 340 | #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS) |
| 341 | |
Thomas Garnier | e70954f | 2016-12-12 16:41:38 -0800 | [diff] [blame] | 342 | /* Common flags permitted for kmem_cache_create */ |
| 343 | #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \ |
| 344 | SLAB_RED_ZONE | \ |
| 345 | SLAB_POISON | \ |
| 346 | SLAB_STORE_USER | \ |
| 347 | SLAB_TRACE | \ |
| 348 | SLAB_CONSISTENCY_CHECKS | \ |
| 349 | SLAB_MEM_SPREAD | \ |
| 350 | SLAB_NOLEAKTRACE | \ |
| 351 | SLAB_RECLAIM_ACCOUNT | \ |
| 352 | SLAB_TEMPORARY | \ |
Hyeonggon Yoo | a285909 | 2022-04-06 15:00:03 +0900 | [diff] [blame] | 353 | SLAB_ACCOUNT | \ |
| 354 | SLAB_NO_USER_FLAGS) |
Thomas Garnier | e70954f | 2016-12-12 16:41:38 -0800 | [diff] [blame] | 355 | |
Shakeel Butt | f9e13c0 | 2018-04-05 16:21:57 -0700 | [diff] [blame] | 356 | bool __kmem_cache_empty(struct kmem_cache *); |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 357 | int __kmem_cache_shutdown(struct kmem_cache *); |
Dmitry Safonov | 52b4b95 | 2016-02-17 13:11:37 -0800 | [diff] [blame] | 358 | void __kmem_cache_release(struct kmem_cache *); |
Tejun Heo | c9fc586 | 2017-02-22 15:41:27 -0800 | [diff] [blame] | 359 | int __kmem_cache_shrink(struct kmem_cache *); |
Christoph Lameter | 41a2128 | 2014-05-06 12:50:08 -0700 | [diff] [blame] | 360 | void slab_kmem_cache_release(struct kmem_cache *); |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 361 | |
Glauber Costa | b7454ad | 2012-10-19 18:20:25 +0400 | [diff] [blame] | 362 | struct seq_file; |
| 363 | struct file; |
Glauber Costa | b7454ad | 2012-10-19 18:20:25 +0400 | [diff] [blame] | 364 | |
Glauber Costa | 0d7561c | 2012-10-19 18:20:27 +0400 | [diff] [blame] | 365 | struct slabinfo { |
| 366 | unsigned long active_objs; |
| 367 | unsigned long num_objs; |
| 368 | unsigned long active_slabs; |
| 369 | unsigned long num_slabs; |
| 370 | unsigned long shared_avail; |
| 371 | unsigned int limit; |
| 372 | unsigned int batchcount; |
| 373 | unsigned int shared; |
| 374 | unsigned int objects_per_slab; |
| 375 | unsigned int cache_order; |
| 376 | }; |
| 377 | |
| 378 | void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); |
| 379 | void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s); |
Glauber Costa | b7454ad | 2012-10-19 18:20:25 +0400 | [diff] [blame] | 380 | ssize_t slabinfo_write(struct file *file, const char __user *buffer, |
| 381 | size_t count, loff_t *ppos); |
Glauber Costa | ba6c496 | 2012-12-18 14:22:27 -0800 | [diff] [blame] | 382 | |
Christoph Lameter | 484748f | 2015-09-04 15:45:34 -0700 | [diff] [blame] | 383 | /* |
| 384 | * Generic implementation of bulk operations |
| 385 | * These are useful for situations in which the allocator cannot |
Jesper Dangaard Brouer | 9f706d6 | 2016-03-15 14:54:03 -0700 | [diff] [blame] | 386 | * perform optimizations. In that case segments of the object listed |
Christoph Lameter | 484748f | 2015-09-04 15:45:34 -0700 | [diff] [blame] | 387 | * may be allocated or freed using these operations. |
| 388 | */ |
| 389 | void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **); |
Jesper Dangaard Brouer | 865762a | 2015-11-20 15:57:58 -0800 | [diff] [blame] | 390 | int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **); |
Christoph Lameter | 484748f | 2015-09-04 15:45:34 -0700 | [diff] [blame] | 391 | |
Muchun Song | 1a984c4 | 2020-12-14 19:06:24 -0800 | [diff] [blame] | 392 | static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) |
Roman Gushchin | 6cea1d5 | 2019-07-11 20:56:16 -0700 | [diff] [blame] | 393 | { |
| 394 | return (s->flags & SLAB_RECLAIM_ACCOUNT) ? |
Roman Gushchin | d42f324 | 2020-08-06 23:20:39 -0700 | [diff] [blame] | 395 | NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B; |
Roman Gushchin | 6cea1d5 | 2019-07-11 20:56:16 -0700 | [diff] [blame] | 396 | } |
| 397 | |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 398 | #ifdef CONFIG_SLUB_DEBUG |
| 399 | #ifdef CONFIG_SLUB_DEBUG_ON |
| 400 | DECLARE_STATIC_KEY_TRUE(slub_debug_enabled); |
| 401 | #else |
| 402 | DECLARE_STATIC_KEY_FALSE(slub_debug_enabled); |
| 403 | #endif |
| 404 | extern void print_tracking(struct kmem_cache *s, void *object); |
Oliver Glitta | 1f9f78b | 2021-06-28 19:34:33 -0700 | [diff] [blame] | 405 | long validate_slab_cache(struct kmem_cache *s); |
Marco Elver | 0d4a062 | 2021-07-14 21:26:34 -0700 | [diff] [blame] | 406 | static inline bool __slub_debug_enabled(void) |
| 407 | { |
| 408 | return static_branch_unlikely(&slub_debug_enabled); |
| 409 | } |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 410 | #else |
| 411 | static inline void print_tracking(struct kmem_cache *s, void *object) |
| 412 | { |
| 413 | } |
Marco Elver | 0d4a062 | 2021-07-14 21:26:34 -0700 | [diff] [blame] | 414 | static inline bool __slub_debug_enabled(void) |
| 415 | { |
| 416 | return false; |
| 417 | } |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 418 | #endif |
| 419 | |
| 420 | /* |
| 421 | * Returns true if any of the specified slub_debug flags is enabled for the |
| 422 | * cache. Use only for flags parsed by setup_slub_debug() as it also enables |
| 423 | * the static key. |
| 424 | */ |
| 425 | static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags) |
| 426 | { |
Marco Elver | 0d4a062 | 2021-07-14 21:26:34 -0700 | [diff] [blame] | 427 | if (IS_ENABLED(CONFIG_SLUB_DEBUG)) |
| 428 | VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS)); |
| 429 | if (__slub_debug_enabled()) |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 430 | return s->flags & flags; |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 431 | return false; |
| 432 | } |
| 433 | |
Kirill Tkhai | 84c07d1 | 2018-08-17 15:47:25 -0700 | [diff] [blame] | 434 | #ifdef CONFIG_MEMCG_KMEM |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 435 | /* |
| 436 | * slab_objcgs - get the object cgroups vector associated with a slab |
| 437 | * @slab: a pointer to the slab struct |
| 438 | * |
| 439 | * Returns a pointer to the object cgroups vector associated with the slab, |
| 440 | * or NULL if no such vector has been associated yet. |
| 441 | */ |
| 442 | static inline struct obj_cgroup **slab_objcgs(struct slab *slab) |
| 443 | { |
| 444 | unsigned long memcg_data = READ_ONCE(slab->memcg_data); |
| 445 | |
| 446 | VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), |
| 447 | slab_page(slab)); |
| 448 | VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab)); |
| 449 | |
| 450 | return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); |
| 451 | } |
| 452 | |
| 453 | int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, |
| 454 | gfp_t gfp, bool new_slab); |
Waiman Long | fdbcb2a | 2021-06-28 19:37:19 -0700 | [diff] [blame] | 455 | void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, |
| 456 | enum node_stat_item idx, int nr); |
Roman Gushchin | 286e04b | 2020-08-06 23:20:52 -0700 | [diff] [blame] | 457 | |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 458 | static inline void memcg_free_slab_cgroups(struct slab *slab) |
Roman Gushchin | 286e04b | 2020-08-06 23:20:52 -0700 | [diff] [blame] | 459 | { |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 460 | kfree(slab_objcgs(slab)); |
| 461 | slab->memcg_data = 0; |
Roman Gushchin | 286e04b | 2020-08-06 23:20:52 -0700 | [diff] [blame] | 462 | } |
| 463 | |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 464 | static inline size_t obj_full_size(struct kmem_cache *s) |
| 465 | { |
| 466 | /* |
| 467 | * For each accounted object there is an extra space which is used |
| 468 | * to store obj_cgroup membership. Charge it too. |
| 469 | */ |
| 470 | return s->size + sizeof(struct obj_cgroup *); |
| 471 | } |
| 472 | |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 473 | /* |
| 474 | * Returns false if the allocation should fail. |
| 475 | */ |
| 476 | static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 477 | struct list_lru *lru, |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 478 | struct obj_cgroup **objcgp, |
| 479 | size_t objects, gfp_t flags) |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 480 | { |
Roman Gushchin | 9855609 | 2020-08-06 23:21:10 -0700 | [diff] [blame] | 481 | struct obj_cgroup *objcg; |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 482 | |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 483 | if (!memcg_kmem_enabled()) |
| 484 | return true; |
| 485 | |
| 486 | if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)) |
| 487 | return true; |
| 488 | |
Roman Gushchin | 9855609 | 2020-08-06 23:21:10 -0700 | [diff] [blame] | 489 | objcg = get_obj_cgroup_from_current(); |
| 490 | if (!objcg) |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 491 | return true; |
Roman Gushchin | 9855609 | 2020-08-06 23:21:10 -0700 | [diff] [blame] | 492 | |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 493 | if (lru) { |
| 494 | int ret; |
| 495 | struct mem_cgroup *memcg; |
| 496 | |
| 497 | memcg = get_mem_cgroup_from_objcg(objcg); |
| 498 | ret = memcg_list_lru_alloc(memcg, lru, flags); |
| 499 | css_put(&memcg->css); |
| 500 | |
| 501 | if (ret) |
| 502 | goto out; |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 503 | } |
| 504 | |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 505 | if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) |
| 506 | goto out; |
| 507 | |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 508 | *objcgp = objcg; |
| 509 | return true; |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 510 | out: |
| 511 | obj_cgroup_put(objcg); |
| 512 | return false; |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 513 | } |
| 514 | |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 515 | static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, |
| 516 | struct obj_cgroup *objcg, |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 517 | gfp_t flags, size_t size, |
| 518 | void **p) |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 519 | { |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 520 | struct slab *slab; |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 521 | unsigned long off; |
| 522 | size_t i; |
| 523 | |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 524 | if (!memcg_kmem_enabled() || !objcg) |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 525 | return; |
| 526 | |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 527 | for (i = 0; i < size; i++) { |
| 528 | if (likely(p[i])) { |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 529 | slab = virt_to_slab(p[i]); |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 530 | |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 531 | if (!slab_objcgs(slab) && |
| 532 | memcg_alloc_slab_cgroups(slab, s, flags, |
Roman Gushchin | 2e9bd48 | 2021-02-24 12:03:11 -0800 | [diff] [blame] | 533 | false)) { |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 534 | obj_cgroup_uncharge(objcg, obj_full_size(s)); |
| 535 | continue; |
| 536 | } |
| 537 | |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 538 | off = obj_to_index(s, slab, p[i]); |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 539 | obj_cgroup_get(objcg); |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 540 | slab_objcgs(slab)[off] = objcg; |
| 541 | mod_objcg_state(objcg, slab_pgdat(slab), |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 542 | cache_vmstat_idx(s), obj_full_size(s)); |
| 543 | } else { |
| 544 | obj_cgroup_uncharge(objcg, obj_full_size(s)); |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 545 | } |
| 546 | } |
| 547 | obj_cgroup_put(objcg); |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 548 | } |
| 549 | |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 550 | static inline void memcg_slab_free_hook(struct kmem_cache *s_orig, |
| 551 | void **p, int objects) |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 552 | { |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 553 | struct kmem_cache *s; |
Roman Gushchin | 270c6a7 | 2020-12-01 13:58:28 -0800 | [diff] [blame] | 554 | struct obj_cgroup **objcgs; |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 555 | struct obj_cgroup *objcg; |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 556 | struct slab *slab; |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 557 | unsigned int off; |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 558 | int i; |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 559 | |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 560 | if (!memcg_kmem_enabled()) |
| 561 | return; |
| 562 | |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 563 | for (i = 0; i < objects; i++) { |
| 564 | if (unlikely(!p[i])) |
| 565 | continue; |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 566 | |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 567 | slab = virt_to_slab(p[i]); |
| 568 | /* we could be given a kmalloc_large() object, skip those */ |
| 569 | if (!slab) |
| 570 | continue; |
| 571 | |
| 572 | objcgs = slab_objcgs(slab); |
Roman Gushchin | 270c6a7 | 2020-12-01 13:58:28 -0800 | [diff] [blame] | 573 | if (!objcgs) |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 574 | continue; |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 575 | |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 576 | if (!s_orig) |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 577 | s = slab->slab_cache; |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 578 | else |
| 579 | s = s_orig; |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 580 | |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 581 | off = obj_to_index(s, slab, p[i]); |
Roman Gushchin | 270c6a7 | 2020-12-01 13:58:28 -0800 | [diff] [blame] | 582 | objcg = objcgs[off]; |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 583 | if (!objcg) |
| 584 | continue; |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 585 | |
Roman Gushchin | 270c6a7 | 2020-12-01 13:58:28 -0800 | [diff] [blame] | 586 | objcgs[off] = NULL; |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 587 | obj_cgroup_uncharge(objcg, obj_full_size(s)); |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 588 | mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 589 | -obj_full_size(s)); |
| 590 | obj_cgroup_put(objcg); |
| 591 | } |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 592 | } |
| 593 | |
Kirill Tkhai | 84c07d1 | 2018-08-17 15:47:25 -0700 | [diff] [blame] | 594 | #else /* CONFIG_MEMCG_KMEM */ |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 595 | static inline struct obj_cgroup **slab_objcgs(struct slab *slab) |
| 596 | { |
| 597 | return NULL; |
| 598 | } |
| 599 | |
Roman Gushchin | 9855609 | 2020-08-06 23:21:10 -0700 | [diff] [blame] | 600 | static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) |
Roman Gushchin | 4d96ba3 | 2019-07-11 20:56:31 -0700 | [diff] [blame] | 601 | { |
| 602 | return NULL; |
| 603 | } |
| 604 | |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 605 | static inline int memcg_alloc_slab_cgroups(struct slab *slab, |
Roman Gushchin | 2e9bd48 | 2021-02-24 12:03:11 -0800 | [diff] [blame] | 606 | struct kmem_cache *s, gfp_t gfp, |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 607 | bool new_slab) |
Roman Gushchin | 286e04b | 2020-08-06 23:20:52 -0700 | [diff] [blame] | 608 | { |
| 609 | return 0; |
| 610 | } |
| 611 | |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 612 | static inline void memcg_free_slab_cgroups(struct slab *slab) |
Roman Gushchin | 286e04b | 2020-08-06 23:20:52 -0700 | [diff] [blame] | 613 | { |
| 614 | } |
| 615 | |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 616 | static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 617 | struct list_lru *lru, |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 618 | struct obj_cgroup **objcgp, |
| 619 | size_t objects, gfp_t flags) |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 620 | { |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 621 | return true; |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 622 | } |
| 623 | |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 624 | static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, |
| 625 | struct obj_cgroup *objcg, |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 626 | gfp_t flags, size_t size, |
| 627 | void **p) |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 628 | { |
| 629 | } |
| 630 | |
Bharata B Rao | d1b2cf6 | 2020-10-13 16:53:09 -0700 | [diff] [blame] | 631 | static inline void memcg_slab_free_hook(struct kmem_cache *s, |
| 632 | void **p, int objects) |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 633 | { |
| 634 | } |
Kirill Tkhai | 84c07d1 | 2018-08-17 15:47:25 -0700 | [diff] [blame] | 635 | #endif /* CONFIG_MEMCG_KMEM */ |
Glauber Costa | b9ce5ef | 2012-12-18 14:22:46 -0800 | [diff] [blame] | 636 | |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 637 | #ifndef CONFIG_SLOB |
Kees Cook | a64b537 | 2019-07-11 20:53:26 -0700 | [diff] [blame] | 638 | static inline struct kmem_cache *virt_to_cache(const void *obj) |
| 639 | { |
Matthew Wilcox (Oracle) | 82c1775 | 2021-10-04 14:45:53 +0100 | [diff] [blame] | 640 | struct slab *slab; |
Kees Cook | a64b537 | 2019-07-11 20:53:26 -0700 | [diff] [blame] | 641 | |
Matthew Wilcox (Oracle) | 82c1775 | 2021-10-04 14:45:53 +0100 | [diff] [blame] | 642 | slab = virt_to_slab(obj); |
| 643 | if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", |
Kees Cook | a64b537 | 2019-07-11 20:53:26 -0700 | [diff] [blame] | 644 | __func__)) |
| 645 | return NULL; |
Matthew Wilcox (Oracle) | 82c1775 | 2021-10-04 14:45:53 +0100 | [diff] [blame] | 646 | return slab->slab_cache; |
Kees Cook | a64b537 | 2019-07-11 20:53:26 -0700 | [diff] [blame] | 647 | } |
| 648 | |
Matthew Wilcox (Oracle) | b918653 | 2021-10-04 14:45:52 +0100 | [diff] [blame] | 649 | static __always_inline void account_slab(struct slab *slab, int order, |
| 650 | struct kmem_cache *s, gfp_t gfp) |
Roman Gushchin | 6cea1d5 | 2019-07-11 20:56:16 -0700 | [diff] [blame] | 651 | { |
Roman Gushchin | 2e9bd48 | 2021-02-24 12:03:11 -0800 | [diff] [blame] | 652 | if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT)) |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 653 | memcg_alloc_slab_cgroups(slab, s, gfp, true); |
Roman Gushchin | 2e9bd48 | 2021-02-24 12:03:11 -0800 | [diff] [blame] | 654 | |
Matthew Wilcox (Oracle) | b918653 | 2021-10-04 14:45:52 +0100 | [diff] [blame] | 655 | mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 656 | PAGE_SIZE << order); |
Roman Gushchin | 6cea1d5 | 2019-07-11 20:56:16 -0700 | [diff] [blame] | 657 | } |
| 658 | |
Matthew Wilcox (Oracle) | b918653 | 2021-10-04 14:45:52 +0100 | [diff] [blame] | 659 | static __always_inline void unaccount_slab(struct slab *slab, int order, |
| 660 | struct kmem_cache *s) |
Roman Gushchin | 6cea1d5 | 2019-07-11 20:56:16 -0700 | [diff] [blame] | 661 | { |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 662 | if (memcg_kmem_enabled()) |
Vlastimil Babka | 4b5f8d9a | 2021-11-02 22:42:04 +0100 | [diff] [blame] | 663 | memcg_free_slab_cgroups(slab); |
Roman Gushchin | 9855609 | 2020-08-06 23:21:10 -0700 | [diff] [blame] | 664 | |
Matthew Wilcox (Oracle) | b918653 | 2021-10-04 14:45:52 +0100 | [diff] [blame] | 665 | mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), |
Roman Gushchin | f2fe7b0 | 2020-08-06 23:20:59 -0700 | [diff] [blame] | 666 | -(PAGE_SIZE << order)); |
Roman Gushchin | 6cea1d5 | 2019-07-11 20:56:16 -0700 | [diff] [blame] | 667 | } |
| 668 | |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 669 | static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) |
| 670 | { |
| 671 | struct kmem_cache *cachep; |
| 672 | |
| 673 | if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) && |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 674 | !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) |
| 675 | return s; |
| 676 | |
| 677 | cachep = virt_to_cache(x); |
Roman Gushchin | 10befea | 2020-08-06 23:21:27 -0700 | [diff] [blame] | 678 | if (WARN(cachep && cachep != s, |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 679 | "%s: Wrong slab cache. %s but object is from %s\n", |
| 680 | __func__, s->name, cachep->name)) |
| 681 | print_tracking(cachep, x); |
| 682 | return cachep; |
| 683 | } |
Vlastimil Babka | 401fb12 | 2021-11-04 11:30:58 +0100 | [diff] [blame] | 684 | #endif /* CONFIG_SLOB */ |
Vlastimil Babka | e42f174 | 2020-08-06 23:19:05 -0700 | [diff] [blame] | 685 | |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 686 | static inline size_t slab_ksize(const struct kmem_cache *s) |
| 687 | { |
| 688 | #ifndef CONFIG_SLUB |
| 689 | return s->object_size; |
| 690 | |
| 691 | #else /* CONFIG_SLUB */ |
| 692 | # ifdef CONFIG_SLUB_DEBUG |
| 693 | /* |
| 694 | * Debugging requires use of the padding between object |
| 695 | * and whatever may come after it. |
| 696 | */ |
| 697 | if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) |
| 698 | return s->object_size; |
| 699 | # endif |
Alexander Potapenko | 80a9201 | 2016-07-28 15:49:07 -0700 | [diff] [blame] | 700 | if (s->flags & SLAB_KASAN) |
| 701 | return s->object_size; |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 702 | /* |
| 703 | * If we have the need to store the freelist pointer |
| 704 | * back there or track user information then we can |
| 705 | * only use the space before that information. |
| 706 | */ |
Paul E. McKenney | 5f0d5a3 | 2017-01-18 02:53:44 -0800 | [diff] [blame] | 707 | if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER)) |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 708 | return s->inuse; |
| 709 | /* |
| 710 | * Else we can use all the padding etc for the allocation |
| 711 | */ |
| 712 | return s->size; |
| 713 | #endif |
| 714 | } |
| 715 | |
| 716 | static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 717 | struct list_lru *lru, |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 718 | struct obj_cgroup **objcgp, |
| 719 | size_t size, gfp_t flags) |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 720 | { |
| 721 | flags &= gfp_allowed_mask; |
Peter Zijlstra | d92a8cf | 2017-03-03 10:13:38 +0100 | [diff] [blame] | 722 | |
Daniel Vetter | 95d6c70 | 2020-12-14 19:08:34 -0800 | [diff] [blame] | 723 | might_alloc(flags); |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 724 | |
Jesper Dangaard Brouer | fab9963 | 2016-03-15 14:53:38 -0700 | [diff] [blame] | 725 | if (should_failslab(s, flags)) |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 726 | return NULL; |
| 727 | |
Muchun Song | 88f2ef7 | 2022-03-22 14:40:56 -0700 | [diff] [blame] | 728 | if (!memcg_slab_pre_alloc_hook(s, lru, objcgp, size, flags)) |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 729 | return NULL; |
Vladimir Davydov | 4526477 | 2016-07-26 15:24:21 -0700 | [diff] [blame] | 730 | |
| 731 | return s; |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 732 | } |
| 733 | |
Roman Gushchin | 964d4bd | 2020-08-06 23:20:56 -0700 | [diff] [blame] | 734 | static inline void slab_post_alloc_hook(struct kmem_cache *s, |
Andrey Konovalov | da844b7 | 2021-04-29 23:00:06 -0700 | [diff] [blame] | 735 | struct obj_cgroup *objcg, gfp_t flags, |
| 736 | size_t size, void **p, bool init) |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 737 | { |
| 738 | size_t i; |
| 739 | |
| 740 | flags &= gfp_allowed_mask; |
Andrey Konovalov | da844b7 | 2021-04-29 23:00:06 -0700 | [diff] [blame] | 741 | |
| 742 | /* |
| 743 | * As memory initialization might be integrated into KASAN, |
| 744 | * kasan_slab_alloc and initialization memset must be |
| 745 | * kept together to avoid discrepancies in behavior. |
| 746 | * |
| 747 | * As p[i] might get tagged, memset and kmemleak hook come after KASAN. |
| 748 | */ |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 749 | for (i = 0; i < size; i++) { |
Andrey Konovalov | da844b7 | 2021-04-29 23:00:06 -0700 | [diff] [blame] | 750 | p[i] = kasan_slab_alloc(s, p[i], flags, init); |
| 751 | if (p[i] && init && !kasan_has_integrated_init()) |
| 752 | memset(p[i], 0, s->object_size); |
Andrey Konovalov | 5312824 | 2019-02-20 22:19:11 -0800 | [diff] [blame] | 753 | kmemleak_alloc_recursive(p[i], s->object_size, 1, |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 754 | s->flags, flags); |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 755 | } |
Vladimir Davydov | 4526477 | 2016-07-26 15:24:21 -0700 | [diff] [blame] | 756 | |
Roman Gushchin | becaba6 | 2020-12-05 22:14:45 -0800 | [diff] [blame] | 757 | memcg_slab_post_alloc_hook(s, objcg, flags, size, p); |
Jesper Dangaard Brouer | 11c7aec | 2016-03-15 14:53:35 -0700 | [diff] [blame] | 758 | } |
| 759 | |
Christoph Lameter | 44c5356 | 2014-08-06 16:04:07 -0700 | [diff] [blame] | 760 | #ifndef CONFIG_SLOB |
Christoph Lameter | ca34956 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 761 | /* |
| 762 | * The slab lists for all objects. |
| 763 | */ |
| 764 | struct kmem_cache_node { |
| 765 | spinlock_t list_lock; |
| 766 | |
| 767 | #ifdef CONFIG_SLAB |
| 768 | struct list_head slabs_partial; /* partial list first, better asm code */ |
| 769 | struct list_head slabs_full; |
| 770 | struct list_head slabs_free; |
David Rientjes | bf00bd3 | 2016-12-12 16:41:44 -0800 | [diff] [blame] | 771 | unsigned long total_slabs; /* length of all slab lists */ |
| 772 | unsigned long free_slabs; /* length of free slab list only */ |
Christoph Lameter | ca34956 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 773 | unsigned long free_objects; |
| 774 | unsigned int free_limit; |
| 775 | unsigned int colour_next; /* Per-node cache coloring */ |
| 776 | struct array_cache *shared; /* shared per node */ |
Joonsoo Kim | c8522a3 | 2014-08-06 16:04:29 -0700 | [diff] [blame] | 777 | struct alien_cache **alien; /* on other nodes */ |
Christoph Lameter | ca34956 | 2013-01-10 19:14:19 +0000 | [diff] [blame] | 778 | unsigned long next_reap; /* updated without locking */ |
| 779 | int free_touched; /* updated without locking */ |
| 780 | #endif |
| 781 | |
| 782 | #ifdef CONFIG_SLUB |
| 783 | unsigned long nr_partial; |
| 784 | struct list_head partial; |
| 785 | #ifdef CONFIG_SLUB_DEBUG |
| 786 | atomic_long_t nr_slabs; |
| 787 | atomic_long_t total_objects; |
| 788 | struct list_head full; |
| 789 | #endif |
| 790 | #endif |
| 791 | |
| 792 | }; |
Wanpeng Li | e25839f | 2013-07-04 08:33:23 +0800 | [diff] [blame] | 793 | |
Christoph Lameter | 44c5356 | 2014-08-06 16:04:07 -0700 | [diff] [blame] | 794 | static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) |
| 795 | { |
| 796 | return s->node[node]; |
| 797 | } |
| 798 | |
| 799 | /* |
| 800 | * Iterator over all nodes. The body will be executed for each node that has |
| 801 | * a kmem_cache_node structure allocated (which is true for all online nodes) |
| 802 | */ |
| 803 | #define for_each_kmem_cache_node(__s, __node, __n) \ |
Mikulas Patocka | 9163582 | 2014-10-09 15:26:20 -0700 | [diff] [blame] | 804 | for (__node = 0; __node < nr_node_ids; __node++) \ |
| 805 | if ((__n = get_node(__s, __node))) |
Christoph Lameter | 44c5356 | 2014-08-06 16:04:07 -0700 | [diff] [blame] | 806 | |
| 807 | #endif |
| 808 | |
Yang Shi | 852d8be | 2017-11-15 17:32:07 -0800 | [diff] [blame] | 809 | #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG) |
| 810 | void dump_unreclaimable_slab(void); |
| 811 | #else |
| 812 | static inline void dump_unreclaimable_slab(void) |
| 813 | { |
| 814 | } |
| 815 | #endif |
| 816 | |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 817 | void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr); |
| 818 | |
Thomas Garnier | 7c00fce | 2016-07-26 15:21:56 -0700 | [diff] [blame] | 819 | #ifdef CONFIG_SLAB_FREELIST_RANDOM |
| 820 | int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, |
| 821 | gfp_t gfp); |
| 822 | void cache_random_seq_destroy(struct kmem_cache *cachep); |
| 823 | #else |
| 824 | static inline int cache_random_seq_create(struct kmem_cache *cachep, |
| 825 | unsigned int count, gfp_t gfp) |
| 826 | { |
| 827 | return 0; |
| 828 | } |
| 829 | static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { } |
| 830 | #endif /* CONFIG_SLAB_FREELIST_RANDOM */ |
| 831 | |
Alexander Potapenko | 6471384 | 2019-07-11 20:59:19 -0700 | [diff] [blame] | 832 | static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c) |
| 833 | { |
Kees Cook | 51cba1e | 2021-04-01 16:23:43 -0700 | [diff] [blame] | 834 | if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, |
| 835 | &init_on_alloc)) { |
Alexander Potapenko | 6471384 | 2019-07-11 20:59:19 -0700 | [diff] [blame] | 836 | if (c->ctor) |
| 837 | return false; |
| 838 | if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) |
| 839 | return flags & __GFP_ZERO; |
| 840 | return true; |
| 841 | } |
| 842 | return flags & __GFP_ZERO; |
| 843 | } |
| 844 | |
| 845 | static inline bool slab_want_init_on_free(struct kmem_cache *c) |
| 846 | { |
Kees Cook | 51cba1e | 2021-04-01 16:23:43 -0700 | [diff] [blame] | 847 | if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, |
| 848 | &init_on_free)) |
Alexander Potapenko | 6471384 | 2019-07-11 20:59:19 -0700 | [diff] [blame] | 849 | return !(c->ctor || |
| 850 | (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); |
| 851 | return false; |
| 852 | } |
| 853 | |
Faiyaz Mohammed | 64dd684 | 2021-06-28 19:34:55 -0700 | [diff] [blame] | 854 | #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) |
| 855 | void debugfs_slab_release(struct kmem_cache *); |
| 856 | #else |
| 857 | static inline void debugfs_slab_release(struct kmem_cache *s) { } |
| 858 | #endif |
| 859 | |
Paul E. McKenney | 5bb1bb3 | 2021-01-07 13:46:11 -0800 | [diff] [blame] | 860 | #ifdef CONFIG_PRINTK |
Paul E. McKenney | 8e7f37f | 2020-12-07 17:41:02 -0800 | [diff] [blame] | 861 | #define KS_ADDRS_COUNT 16 |
| 862 | struct kmem_obj_info { |
| 863 | void *kp_ptr; |
Matthew Wilcox (Oracle) | 7213230 | 2021-10-04 14:45:55 +0100 | [diff] [blame] | 864 | struct slab *kp_slab; |
Paul E. McKenney | 8e7f37f | 2020-12-07 17:41:02 -0800 | [diff] [blame] | 865 | void *kp_objp; |
| 866 | unsigned long kp_data_offset; |
| 867 | struct kmem_cache *kp_slab_cache; |
| 868 | void *kp_ret; |
| 869 | void *kp_stack[KS_ADDRS_COUNT]; |
Maninder Singh | e548eaa | 2021-03-16 16:07:11 +0530 | [diff] [blame] | 870 | void *kp_free_stack[KS_ADDRS_COUNT]; |
Paul E. McKenney | 8e7f37f | 2020-12-07 17:41:02 -0800 | [diff] [blame] | 871 | }; |
Marco Elver | 2dfe63e | 2022-04-14 19:13:40 -0700 | [diff] [blame] | 872 | void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab); |
Paul E. McKenney | 5bb1bb3 | 2021-01-07 13:46:11 -0800 | [diff] [blame] | 873 | #endif |
Paul E. McKenney | 8e7f37f | 2020-12-07 17:41:02 -0800 | [diff] [blame] | 874 | |
Matthew Wilcox (Oracle) | 0b3eb091 | 2021-10-04 14:45:56 +0100 | [diff] [blame] | 875 | #ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR |
| 876 | void __check_heap_object(const void *ptr, unsigned long n, |
| 877 | const struct slab *slab, bool to_user); |
| 878 | #else |
| 879 | static inline |
| 880 | void __check_heap_object(const void *ptr, unsigned long n, |
| 881 | const struct slab *slab, bool to_user) |
| 882 | { |
| 883 | } |
| 884 | #endif |
| 885 | |
Andrey Ryabinin | 5240ab4 | 2014-08-06 16:04:14 -0700 | [diff] [blame] | 886 | #endif /* MM_SLAB_H */ |