blob: 13ab3b771409c34c3e68d4133e264f10f256eaae [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Tim Chen67afa382017-02-22 15:45:39 -08002/*
3 * Manage cache of swap slots to be used for and returned from
4 * swap.
5 *
6 * Copyright(c) 2016 Intel Corporation.
7 *
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
9 *
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
13 * we need a new slot.
14 *
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
Ingo Molnarf0953a12021-05-06 18:06:47 -070019 * allows the slots to coalesce and reduce fragmentation.
Tim Chen67afa382017-02-22 15:45:39 -080020 *
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
24 *
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
28 */
29
30#include <linux/swap_slots.h>
31#include <linux/cpu.h>
32#include <linux/cpumask.h>
Jakub Kicinski8581fd42021-12-02 12:34:00 -080033#include <linux/slab.h>
Tim Chen67afa382017-02-22 15:45:39 -080034#include <linux/vmalloc.h>
35#include <linux/mutex.h>
Huang Ying54f180d2017-05-08 15:57:40 -070036#include <linux/mm.h>
Tim Chen67afa382017-02-22 15:45:39 -080037
Tim Chen67afa382017-02-22 15:45:39 -080038static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
39static bool swap_slot_cache_active;
Huang Yingba81f832017-02-22 15:45:46 -080040bool swap_slot_cache_enabled;
Tim Chen67afa382017-02-22 15:45:39 -080041static bool swap_slot_cache_initialized;
Colin Ian King31f21da2018-08-17 15:46:54 -070042static DEFINE_MUTEX(swap_slots_cache_mutex);
Tim Chen67afa382017-02-22 15:45:39 -080043/* Serialize swap slots cache enable/disable operations */
Colin Ian King31f21da2018-08-17 15:46:54 -070044static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
Tim Chen67afa382017-02-22 15:45:39 -080045
46static void __drain_swap_slots_cache(unsigned int type);
Tim Chen67afa382017-02-22 15:45:39 -080047
Zhen Leie0f3ebb2020-08-06 23:20:11 -070048#define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
Tim Chen67afa382017-02-22 15:45:39 -080049#define SLOTS_CACHE 0x1
50#define SLOTS_CACHE_RET 0x2
51
52static void deactivate_swap_slots_cache(void)
53{
54 mutex_lock(&swap_slots_cache_mutex);
55 swap_slot_cache_active = false;
56 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
57 mutex_unlock(&swap_slots_cache_mutex);
58}
59
60static void reactivate_swap_slots_cache(void)
61{
62 mutex_lock(&swap_slots_cache_mutex);
63 swap_slot_cache_active = true;
64 mutex_unlock(&swap_slots_cache_mutex);
65}
66
67/* Must not be called with cpu hot plug lock */
68void disable_swap_slots_cache_lock(void)
69{
70 mutex_lock(&swap_slots_cache_enable_mutex);
71 swap_slot_cache_enabled = false;
72 if (swap_slot_cache_initialized) {
73 /* serialize with cpu hotplug operations */
Sebastian Andrzej Siewior7625ecc2021-08-03 16:16:03 +020074 cpus_read_lock();
Tim Chen67afa382017-02-22 15:45:39 -080075 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
Sebastian Andrzej Siewior7625ecc2021-08-03 16:16:03 +020076 cpus_read_unlock();
Tim Chen67afa382017-02-22 15:45:39 -080077 }
78}
79
80static void __reenable_swap_slots_cache(void)
81{
82 swap_slot_cache_enabled = has_usable_swap();
83}
84
85void reenable_swap_slots_cache_unlock(void)
86{
87 __reenable_swap_slots_cache();
88 mutex_unlock(&swap_slots_cache_enable_mutex);
89}
90
91static bool check_cache_active(void)
92{
93 long pages;
94
Zhen Leie0f3ebb2020-08-06 23:20:11 -070095 if (!swap_slot_cache_enabled)
Tim Chen67afa382017-02-22 15:45:39 -080096 return false;
97
98 pages = get_nr_swap_pages();
99 if (!swap_slot_cache_active) {
100 if (pages > num_online_cpus() *
101 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
102 reactivate_swap_slots_cache();
103 goto out;
104 }
105
106 /* if global pool of slot caches too low, deactivate cache */
107 if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
108 deactivate_swap_slots_cache();
109out:
110 return swap_slot_cache_active;
111}
112
113static int alloc_swap_slot_cache(unsigned int cpu)
114{
115 struct swap_slots_cache *cache;
116 swp_entry_t *slots, *slots_ret;
117
118 /*
119 * Do allocation outside swap_slots_cache_mutex
Matthew Wilcox (Oracle)e2e3fdc2022-05-12 20:23:02 -0700120 * as kvzalloc could trigger reclaim and folio_alloc_swap,
Tim Chen67afa382017-02-22 15:45:39 -0800121 * which can lock swap_slots_cache_mutex.
122 */
Kees Cook778e1cd2018-06-12 14:04:48 -0700123 slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
Huang Ying54f180d2017-05-08 15:57:40 -0700124 GFP_KERNEL);
Tim Chen67afa382017-02-22 15:45:39 -0800125 if (!slots)
126 return -ENOMEM;
127
Kees Cook778e1cd2018-06-12 14:04:48 -0700128 slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
Huang Ying54f180d2017-05-08 15:57:40 -0700129 GFP_KERNEL);
Tim Chen67afa382017-02-22 15:45:39 -0800130 if (!slots_ret) {
Huang Ying54f180d2017-05-08 15:57:40 -0700131 kvfree(slots);
Tim Chen67afa382017-02-22 15:45:39 -0800132 return -ENOMEM;
133 }
134
135 mutex_lock(&swap_slots_cache_mutex);
136 cache = &per_cpu(swp_slots, cpu);
Zhen Leif90eae22020-08-06 23:20:05 -0700137 if (cache->slots || cache->slots_ret) {
Tim Chen67afa382017-02-22 15:45:39 -0800138 /* cache already allocated */
Zhen Leif90eae22020-08-06 23:20:05 -0700139 mutex_unlock(&swap_slots_cache_mutex);
140
141 kvfree(slots);
142 kvfree(slots_ret);
143
144 return 0;
145 }
146
Tim Chen67afa382017-02-22 15:45:39 -0800147 if (!cache->lock_initialized) {
148 mutex_init(&cache->alloc_lock);
149 spin_lock_init(&cache->free_lock);
150 cache->lock_initialized = true;
151 }
152 cache->nr = 0;
153 cache->cur = 0;
154 cache->n_ret = 0;
Tim Chena2e16732017-11-15 17:34:18 -0800155 /*
156 * We initialized alloc_lock and free_lock earlier. We use
157 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
158 * the corresponding lock and use the cache. Memory barrier below
159 * ensures the assumption.
160 */
161 mb();
Tim Chen67afa382017-02-22 15:45:39 -0800162 cache->slots = slots;
Tim Chen67afa382017-02-22 15:45:39 -0800163 cache->slots_ret = slots_ret;
Tim Chen67afa382017-02-22 15:45:39 -0800164 mutex_unlock(&swap_slots_cache_mutex);
Tim Chen67afa382017-02-22 15:45:39 -0800165 return 0;
166}
167
168static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
169 bool free_slots)
170{
171 struct swap_slots_cache *cache;
172 swp_entry_t *slots = NULL;
173
174 cache = &per_cpu(swp_slots, cpu);
175 if ((type & SLOTS_CACHE) && cache->slots) {
176 mutex_lock(&cache->alloc_lock);
177 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
178 cache->cur = 0;
179 cache->nr = 0;
180 if (free_slots && cache->slots) {
Huang Ying54f180d2017-05-08 15:57:40 -0700181 kvfree(cache->slots);
Tim Chen67afa382017-02-22 15:45:39 -0800182 cache->slots = NULL;
183 }
184 mutex_unlock(&cache->alloc_lock);
185 }
186 if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
187 spin_lock_irq(&cache->free_lock);
188 swapcache_free_entries(cache->slots_ret, cache->n_ret);
189 cache->n_ret = 0;
190 if (free_slots && cache->slots_ret) {
191 slots = cache->slots_ret;
192 cache->slots_ret = NULL;
193 }
194 spin_unlock_irq(&cache->free_lock);
Yang Li191a7222021-02-24 12:02:55 -0800195 kvfree(slots);
Tim Chen67afa382017-02-22 15:45:39 -0800196 }
197}
198
199static void __drain_swap_slots_cache(unsigned int type)
200{
201 unsigned int cpu;
202
203 /*
204 * This function is called during
205 * 1) swapoff, when we have to make sure no
206 * left over slots are in cache when we remove
207 * a swap device;
208 * 2) disabling of swap slot cache, when we run low
209 * on swap slots when allocating memory and need
210 * to return swap slots to global pool.
211 *
212 * We cannot acquire cpu hot plug lock here as
213 * this function can be invoked in the cpu
214 * hot plug path:
215 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
Matthew Wilcox (Oracle)e2e3fdc2022-05-12 20:23:02 -0700216 * -> memory allocation -> direct reclaim -> folio_alloc_swap
Tim Chen67afa382017-02-22 15:45:39 -0800217 * -> drain_swap_slots_cache
218 *
219 * Hence the loop over current online cpu below could miss cpu that
220 * is being brought online but not yet marked as online.
221 * That is okay as we do not schedule and run anything on a
222 * cpu before it has been marked online. Hence, we will not
223 * fill any swap slots in slots cache of such cpu.
224 * There are no slots on such cpu that need to be drained.
225 */
226 for_each_online_cpu(cpu)
227 drain_slots_cache_cpu(cpu, type, false);
228}
229
230static int free_slot_cache(unsigned int cpu)
231{
232 mutex_lock(&swap_slots_cache_mutex);
233 drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
234 mutex_unlock(&swap_slots_cache_mutex);
235 return 0;
236}
237
Miaohe Linf3bc52c2020-10-13 16:52:18 -0700238void enable_swap_slots_cache(void)
Tim Chen67afa382017-02-22 15:45:39 -0800239{
Tim Chen67afa382017-02-22 15:45:39 -0800240 mutex_lock(&swap_slots_cache_enable_mutex);
Zhen Leid69a9572020-08-06 23:20:08 -0700241 if (!swap_slot_cache_initialized) {
242 int ret;
243
244 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
245 alloc_swap_slot_cache, free_slot_cache);
246 if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
247 "without swap slots cache.\n", __func__))
248 goto out_unlock;
249
250 swap_slot_cache_initialized = true;
Tim Chen67afa382017-02-22 15:45:39 -0800251 }
252
Tim Chen67afa382017-02-22 15:45:39 -0800253 __reenable_swap_slots_cache();
254out_unlock:
255 mutex_unlock(&swap_slots_cache_enable_mutex);
Tim Chen67afa382017-02-22 15:45:39 -0800256}
257
258/* called with swap slot cache's alloc lock held */
259static int refill_swap_slots_cache(struct swap_slots_cache *cache)
260{
Miaohe Linf19c2562022-05-19 14:08:51 -0700261 if (!use_swap_slot_cache)
Tim Chen67afa382017-02-22 15:45:39 -0800262 return 0;
263
264 cache->cur = 0;
265 if (swap_slot_cache_active)
Huang Ying5d5e8f12018-08-21 21:52:20 -0700266 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
Ryan Roberts9faaa0f2024-04-08 19:39:43 +0100267 cache->slots, 0);
Tim Chen67afa382017-02-22 15:45:39 -0800268
269 return cache->nr;
270}
271
Miaohe Linbc4a68a2022-05-19 14:08:50 -0700272void free_swap_slot(swp_entry_t entry)
Tim Chen67afa382017-02-22 15:45:39 -0800273{
274 struct swap_slots_cache *cache;
275
Chengming Zhou0827a1f2024-02-04 03:06:00 +0000276 /* Large folio swap slot is not covered. */
277 zswap_invalidate(entry);
278
Sebastian Andrzej Siewiorf07e0f842017-07-10 15:49:29 -0700279 cache = raw_cpu_ptr(&swp_slots);
Tim Chena2e16732017-11-15 17:34:18 -0800280 if (likely(use_swap_slot_cache && cache->slots_ret)) {
Tim Chen67afa382017-02-22 15:45:39 -0800281 spin_lock_irq(&cache->free_lock);
282 /* Swap slots cache may be deactivated before acquiring lock */
Sebastian Andrzej Siewiorf07e0f842017-07-10 15:49:29 -0700283 if (!use_swap_slot_cache || !cache->slots_ret) {
Tim Chen67afa382017-02-22 15:45:39 -0800284 spin_unlock_irq(&cache->free_lock);
285 goto direct_free;
286 }
287 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
288 /*
289 * Return slots to global pool.
290 * The current swap_map value is SWAP_HAS_CACHE.
291 * Set it to 0 to indicate it is available for
292 * allocation in global pool
293 */
294 swapcache_free_entries(cache->slots_ret, cache->n_ret);
295 cache->n_ret = 0;
296 }
297 cache->slots_ret[cache->n_ret++] = entry;
298 spin_unlock_irq(&cache->free_lock);
299 } else {
300direct_free:
301 swapcache_free_entries(&entry, 1);
302 }
Tim Chen67afa382017-02-22 15:45:39 -0800303}
304
Matthew Wilcox (Oracle)e2e3fdc2022-05-12 20:23:02 -0700305swp_entry_t folio_alloc_swap(struct folio *folio)
Tim Chen67afa382017-02-22 15:45:39 -0800306{
Wei Yang2406b762020-04-01 21:06:16 -0700307 swp_entry_t entry;
Tim Chen67afa382017-02-22 15:45:39 -0800308 struct swap_slots_cache *cache;
309
Huang Ying38d8b4e2017-07-06 15:37:18 -0700310 entry.val = 0;
311
Matthew Wilcox (Oracle)e2e3fdc2022-05-12 20:23:02 -0700312 if (folio_test_large(folio)) {
Barry Songf238b8c2024-03-23 00:41:36 +1300313 if (IS_ENABLED(CONFIG_THP_SWAP))
Ryan Roberts9faaa0f2024-04-08 19:39:43 +0100314 get_swap_pages(1, &entry, folio_order(folio));
Tejun Heobb98f2c2018-06-07 17:05:31 -0700315 goto out;
Huang Ying38d8b4e2017-07-06 15:37:18 -0700316 }
317
Tim Chen67afa382017-02-22 15:45:39 -0800318 /*
319 * Preemption is allowed here, because we may sleep
320 * in refill_swap_slots_cache(). But it is safe, because
321 * accesses to the per-CPU data structure are protected by the
322 * mutex cache->alloc_lock.
323 *
324 * The alloc path here does not touch cache->slots_ret
325 * so cache->free_lock is not taken.
326 */
327 cache = raw_cpu_ptr(&swp_slots);
328
Tim Chena2e16732017-11-15 17:34:18 -0800329 if (likely(check_cache_active() && cache->slots)) {
Tim Chen67afa382017-02-22 15:45:39 -0800330 mutex_lock(&cache->alloc_lock);
331 if (cache->slots) {
332repeat:
333 if (cache->nr) {
Wei Yang2406b762020-04-01 21:06:16 -0700334 entry = cache->slots[cache->cur];
335 cache->slots[cache->cur++].val = 0;
Tim Chen67afa382017-02-22 15:45:39 -0800336 cache->nr--;
Wei Yang2406b762020-04-01 21:06:16 -0700337 } else if (refill_swap_slots_cache(cache)) {
338 goto repeat;
Tim Chen67afa382017-02-22 15:45:39 -0800339 }
340 }
341 mutex_unlock(&cache->alloc_lock);
342 if (entry.val)
Tejun Heobb98f2c2018-06-07 17:05:31 -0700343 goto out;
Tim Chen67afa382017-02-22 15:45:39 -0800344 }
345
Ryan Roberts9faaa0f2024-04-08 19:39:43 +0100346 get_swap_pages(1, &entry, 0);
Tejun Heobb98f2c2018-06-07 17:05:31 -0700347out:
Matthew Wilcox (Oracle)e2e3fdc2022-05-12 20:23:02 -0700348 if (mem_cgroup_try_charge_swap(folio, entry)) {
Matthew Wilcox (Oracle)4081f742022-09-02 20:46:09 +0100349 put_swap_folio(folio, entry);
Tejun Heobb98f2c2018-06-07 17:05:31 -0700350 entry.val = 0;
351 }
Tim Chen67afa382017-02-22 15:45:39 -0800352 return entry;
353}