Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1 | /* |
| 2 | * zsmalloc memory allocator |
| 3 | * |
| 4 | * Copyright (C) 2011 Nitin Gupta |
Minchan Kim | 31fc00b | 2014-01-30 15:45:55 -0800 | [diff] [blame] | 5 | * Copyright (C) 2012, 2013 Minchan Kim |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 6 | * |
| 7 | * This code is released using a dual license strategy: BSD/GPL |
| 8 | * You can choose the license that better fits your requirements. |
| 9 | * |
| 10 | * Released under the terms of 3-clause BSD License |
| 11 | * Released under the terms of GNU General Public License Version 2.0 |
| 12 | */ |
| 13 | |
Nitin Gupta | 2db51da | 2012-06-09 17:41:14 -0700 | [diff] [blame] | 14 | /* |
Nitin Gupta | 2db51da | 2012-06-09 17:41:14 -0700 | [diff] [blame] | 15 | * Following is how we use various fields and flags of underlying |
| 16 | * struct page(s) to form a zspage. |
| 17 | * |
| 18 | * Usage of struct page fields: |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 19 | * page->private: points to zspage |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 20 | * page->index: links together all component pages of a zspage |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 21 | * For the huge page, this is always 0, so we use this field |
| 22 | * to store handle. |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 23 | * page->page_type: first object offset in a subpage of zspage |
Nitin Gupta | 2db51da | 2012-06-09 17:41:14 -0700 | [diff] [blame] | 24 | * |
| 25 | * Usage of struct page flags: |
| 26 | * PG_private: identifies the first component page |
Xishi Qiu | 399d8ee | 2017-02-22 15:45:01 -0800 | [diff] [blame] | 27 | * PG_owner_priv_1: identifies the huge component page |
Nitin Gupta | 2db51da | 2012-06-09 17:41:14 -0700 | [diff] [blame] | 28 | * |
| 29 | */ |
| 30 | |
Dan Streetman | 4abaac9 | 2016-05-26 15:16:27 -0700 | [diff] [blame] | 31 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 32 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 33 | /* |
| 34 | * lock ordering: |
| 35 | * page_lock |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 36 | * pool->lock |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 37 | * zspage->lock |
| 38 | */ |
| 39 | |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 40 | #include <linux/module.h> |
| 41 | #include <linux/kernel.h> |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 42 | #include <linux/sched.h> |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 43 | #include <linux/bitops.h> |
| 44 | #include <linux/errno.h> |
| 45 | #include <linux/highmem.h> |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 46 | #include <linux/string.h> |
| 47 | #include <linux/slab.h> |
Mike Rapoport | ca5999f | 2020-06-08 21:32:38 -0700 | [diff] [blame] | 48 | #include <linux/pgtable.h> |
Mike Rapoport | 65fddcf | 2020-06-08 21:32:42 -0700 | [diff] [blame] | 49 | #include <asm/tlbflush.h> |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 50 | #include <linux/cpumask.h> |
| 51 | #include <linux/cpu.h> |
Seth Jennings | 0cbb613 | 2012-02-13 08:47:49 -0600 | [diff] [blame] | 52 | #include <linux/vmalloc.h> |
Sergey Senozhatsky | 759b26b | 2015-11-06 16:29:29 -0800 | [diff] [blame] | 53 | #include <linux/preempt.h> |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 54 | #include <linux/spinlock.h> |
Aliaksei Karaliou | 93144ca | 2018-01-31 16:18:40 -0800 | [diff] [blame] | 55 | #include <linux/shrinker.h> |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 56 | #include <linux/types.h> |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 57 | #include <linux/debugfs.h> |
Minchan Kim | bcf1647 | 2014-01-30 15:45:50 -0800 | [diff] [blame] | 58 | #include <linux/zsmalloc.h> |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 59 | #include <linux/zpool.h> |
Minchan Kim | dd4123f | 2016-07-26 15:26:50 -0700 | [diff] [blame] | 60 | #include <linux/migrate.h> |
Henry Burns | 701d678 | 2019-08-24 17:55:06 -0700 | [diff] [blame] | 61 | #include <linux/wait.h> |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 62 | #include <linux/pagemap.h> |
Sergey Senozhatsky | cdc346b | 2018-01-04 16:18:02 -0800 | [diff] [blame] | 63 | #include <linux/fs.h> |
Mike Galbraith | a372659 | 2022-01-21 22:14:17 -0800 | [diff] [blame] | 64 | #include <linux/local_lock.h> |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 65 | |
| 66 | #define ZSPAGE_MAGIC 0x58 |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 67 | |
| 68 | /* |
Shijie Luo | cb152a1 | 2021-05-06 18:05:51 -0700 | [diff] [blame] | 69 | * This must be power of 2 and greater than or equal to sizeof(link_free). |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 70 | * These two conditions ensure that any 'struct link_free' itself doesn't |
| 71 | * span more than 1 page which avoids complex case of mapping 2 pages simply |
| 72 | * to restore link_free pointer values. |
| 73 | */ |
| 74 | #define ZS_ALIGN 8 |
| 75 | |
| 76 | /* |
| 77 | * A single 'zspage' is composed of up to 2^N discontiguous 0-order (single) |
| 78 | * pages. ZS_MAX_ZSPAGE_ORDER defines upper limit on N. |
| 79 | */ |
| 80 | #define ZS_MAX_ZSPAGE_ORDER 2 |
| 81 | #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER) |
| 82 | |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 83 | #define ZS_HANDLE_SIZE (sizeof(unsigned long)) |
| 84 | |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 85 | /* |
| 86 | * Object location (<PFN>, <obj_idx>) is encoded as |
Randy Dunlap | b956b5a | 2020-08-11 18:33:31 -0700 | [diff] [blame] | 87 | * a single (unsigned long) handle value. |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 88 | * |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 89 | * Note that object index <obj_idx> starts from 0. |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 90 | * |
| 91 | * This is made more complicated by various memory models and PAE. |
| 92 | */ |
| 93 | |
Kirill A. Shutemov | 02390b8 | 2018-02-14 14:16:49 +0300 | [diff] [blame] | 94 | #ifndef MAX_POSSIBLE_PHYSMEM_BITS |
| 95 | #ifdef MAX_PHYSMEM_BITS |
| 96 | #define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS |
| 97 | #else |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 98 | /* |
| 99 | * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just |
| 100 | * be PAGE_SHIFT |
| 101 | */ |
Kirill A. Shutemov | 02390b8 | 2018-02-14 14:16:49 +0300 | [diff] [blame] | 102 | #define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 103 | #endif |
| 104 | #endif |
Kirill A. Shutemov | 02390b8 | 2018-02-14 14:16:49 +0300 | [diff] [blame] | 105 | |
| 106 | #define _PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 107 | |
| 108 | /* |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 109 | * Head in allocated object should have OBJ_ALLOCATED_TAG |
| 110 | * to identify the object was allocated or not. |
| 111 | * It's okay to add the status bit in the least bit because |
| 112 | * header keeps handle which is 4byte-aligned address so we |
| 113 | * have room for two bit at least. |
| 114 | */ |
| 115 | #define OBJ_ALLOCATED_TAG 1 |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 116 | |
| 117 | #ifdef CONFIG_ZPOOL |
| 118 | /* |
| 119 | * The second least-significant bit in the object's header identifies if the |
| 120 | * value stored at the header is a deferred handle from the last reclaim |
| 121 | * attempt. |
| 122 | * |
| 123 | * As noted above, this is valid because we have room for two bits. |
| 124 | */ |
| 125 | #define OBJ_DEFERRED_HANDLE_TAG 2 |
| 126 | #define OBJ_TAG_BITS 2 |
| 127 | #define OBJ_TAG_MASK (OBJ_ALLOCATED_TAG | OBJ_DEFERRED_HANDLE_TAG) |
| 128 | #else |
| 129 | #define OBJ_TAG_BITS 1 |
| 130 | #define OBJ_TAG_MASK OBJ_ALLOCATED_TAG |
| 131 | #endif /* CONFIG_ZPOOL */ |
| 132 | |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 133 | #define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS) |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 134 | #define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1) |
| 135 | |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 136 | #define HUGE_BITS 1 |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 137 | #define FULLNESS_BITS 2 |
| 138 | #define CLASS_BITS 8 |
| 139 | #define ISOLATED_BITS 3 |
| 140 | #define MAGIC_VAL_BITS 8 |
| 141 | |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 142 | #define MAX(a, b) ((a) >= (b) ? (a) : (b)) |
| 143 | /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */ |
| 144 | #define ZS_MIN_ALLOC_SIZE \ |
| 145 | MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS)) |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 146 | /* each chunk includes extra space to keep handle */ |
Minchan Kim | 7b60a68 | 2015-04-15 16:15:39 -0700 | [diff] [blame] | 147 | #define ZS_MAX_ALLOC_SIZE PAGE_SIZE |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 148 | |
| 149 | /* |
Weijie Yang | 7eb5251 | 2014-06-04 16:11:08 -0700 | [diff] [blame] | 150 | * On systems with 4K page size, this gives 255 size classes! There is a |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 151 | * trader-off here: |
| 152 | * - Large number of size classes is potentially wasteful as free page are |
| 153 | * spread across these classes |
| 154 | * - Small number of size classes causes large internal fragmentation |
| 155 | * - Probably its better to use specific size classes (empirically |
| 156 | * determined). NOTE: all those class sizes must be set as multiple of |
| 157 | * ZS_ALIGN to make sure link_free itself never has to span 2 pages. |
| 158 | * |
| 159 | * ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN |
| 160 | * (reason above) |
| 161 | */ |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 162 | #define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_BITS) |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 163 | #define ZS_SIZE_CLASSES (DIV_ROUND_UP(ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE, \ |
| 164 | ZS_SIZE_CLASS_DELTA) + 1) |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 165 | |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 166 | enum fullness_group { |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 167 | ZS_EMPTY, |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 168 | ZS_ALMOST_EMPTY, |
| 169 | ZS_ALMOST_FULL, |
| 170 | ZS_FULL, |
| 171 | NR_ZS_FULLNESS, |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 172 | }; |
| 173 | |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 174 | enum class_stat_type { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 175 | CLASS_EMPTY, |
| 176 | CLASS_ALMOST_EMPTY, |
| 177 | CLASS_ALMOST_FULL, |
| 178 | CLASS_FULL, |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 179 | OBJ_ALLOCATED, |
| 180 | OBJ_USED, |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 181 | NR_ZS_STAT_TYPE, |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 182 | }; |
| 183 | |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 184 | struct zs_size_stat { |
| 185 | unsigned long objs[NR_ZS_STAT_TYPE]; |
| 186 | }; |
| 187 | |
Sergey Senozhatsky | 5724459 | 2015-09-08 15:04:27 -0700 | [diff] [blame] | 188 | #ifdef CONFIG_ZSMALLOC_STAT |
| 189 | static struct dentry *zs_stat_root; |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 190 | #endif |
| 191 | |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 192 | /* |
| 193 | * We assign a page to ZS_ALMOST_EMPTY fullness group when: |
| 194 | * n <= N / f, where |
| 195 | * n = number of allocated objects |
| 196 | * N = total number of objects zspage can store |
Wang Sheng-Hui | 6dd9737 | 2014-10-09 15:29:59 -0700 | [diff] [blame] | 197 | * f = fullness_threshold_frac |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 198 | * |
| 199 | * Similarly, we assign zspage to: |
| 200 | * ZS_ALMOST_FULL when n > N / f |
| 201 | * ZS_EMPTY when n == 0 |
| 202 | * ZS_FULL when n == N |
| 203 | * |
| 204 | * (see: fix_fullness_group()) |
| 205 | */ |
| 206 | static const int fullness_threshold_frac = 4; |
Sergey Senozhatsky | 010b495 | 2018-04-05 16:24:43 -0700 | [diff] [blame] | 207 | static size_t huge_class_size; |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 208 | |
| 209 | struct size_class { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 210 | struct list_head fullness_list[NR_ZS_FULLNESS]; |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 211 | /* |
| 212 | * Size of objects stored in this class. Must be multiple |
| 213 | * of ZS_ALIGN. |
| 214 | */ |
| 215 | int size; |
Minchan Kim | 1fc6e27 | 2016-07-26 15:23:11 -0700 | [diff] [blame] | 216 | int objs_per_zspage; |
Weijie Yang | 7dfa461 | 2016-01-14 15:22:40 -0800 | [diff] [blame] | 217 | /* Number of PAGE_SIZE sized pages to combine to form a 'zspage' */ |
| 218 | int pages_per_zspage; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 219 | |
| 220 | unsigned int index; |
| 221 | struct zs_size_stat stats; |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 222 | }; |
| 223 | |
| 224 | /* |
| 225 | * Placed within free objects to form a singly linked list. |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 226 | * For every zspage, zspage->freeobj gives head of this list. |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 227 | * |
| 228 | * This must be power of 2 and less than or equal to ZS_ALIGN |
| 229 | */ |
| 230 | struct link_free { |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 231 | union { |
| 232 | /* |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 233 | * Free object index; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 234 | * It's valid for non-allocated object |
| 235 | */ |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 236 | unsigned long next; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 237 | /* |
| 238 | * Handle of allocated object. |
| 239 | */ |
| 240 | unsigned long handle; |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 241 | #ifdef CONFIG_ZPOOL |
| 242 | /* |
| 243 | * Deferred handle of a reclaimed object. |
| 244 | */ |
| 245 | unsigned long deferred_handle; |
| 246 | #endif |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 247 | }; |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 248 | }; |
| 249 | |
| 250 | struct zs_pool { |
Sergey SENOZHATSKY | 6f3526d | 2015-11-06 16:29:21 -0800 | [diff] [blame] | 251 | const char *name; |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 252 | |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 253 | struct size_class *size_class[ZS_SIZE_CLASSES]; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 254 | struct kmem_cache *handle_cachep; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 255 | struct kmem_cache *zspage_cachep; |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 256 | |
Minchan Kim | 13de893 | 2014-10-09 15:29:48 -0700 | [diff] [blame] | 257 | atomic_long_t pages_allocated; |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 258 | |
Sergey Senozhatsky | 7d3f393 | 2015-09-08 15:04:35 -0700 | [diff] [blame] | 259 | struct zs_pool_stats stats; |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 260 | |
| 261 | /* Compact classes */ |
| 262 | struct shrinker shrinker; |
Aliaksei Karaliou | 93144ca | 2018-01-31 16:18:40 -0800 | [diff] [blame] | 263 | |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 264 | #ifdef CONFIG_ZPOOL |
| 265 | /* List tracking the zspages in LRU order by most recently added object */ |
| 266 | struct list_head lru; |
Nhat Pham | bd0fded | 2022-11-28 11:16:14 -0800 | [diff] [blame] | 267 | struct zpool *zpool; |
| 268 | const struct zpool_ops *zpool_ops; |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 269 | #endif |
| 270 | |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 271 | #ifdef CONFIG_ZSMALLOC_STAT |
| 272 | struct dentry *stat_dentry; |
| 273 | #endif |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 274 | #ifdef CONFIG_COMPACTION |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 275 | struct work_struct free_work; |
| 276 | #endif |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 277 | spinlock_t lock; |
Seth Jennings | 0959c63 | 2012-08-08 15:12:17 +0900 | [diff] [blame] | 278 | }; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 279 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 280 | struct zspage { |
| 281 | struct { |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 282 | unsigned int huge:HUGE_BITS; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 283 | unsigned int fullness:FULLNESS_BITS; |
Minchan Kim | 85d492f | 2017-04-13 14:56:40 -0700 | [diff] [blame] | 284 | unsigned int class:CLASS_BITS + 1; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 285 | unsigned int isolated:ISOLATED_BITS; |
| 286 | unsigned int magic:MAGIC_VAL_BITS; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 287 | }; |
| 288 | unsigned int inuse; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 289 | unsigned int freeobj; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 290 | struct page *first_page; |
| 291 | struct list_head list; /* fullness list */ |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 292 | |
| 293 | #ifdef CONFIG_ZPOOL |
| 294 | /* links the zspage to the lru list in the pool */ |
| 295 | struct list_head lru; |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 296 | bool under_reclaim; |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 297 | #endif |
| 298 | |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 299 | struct zs_pool *pool; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 300 | rwlock_t lock; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 301 | }; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 302 | |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 303 | struct mapping_area { |
Mike Galbraith | a372659 | 2022-01-21 22:14:17 -0800 | [diff] [blame] | 304 | local_lock_t lock; |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 305 | char *vm_buf; /* copy buffer for objects that span pages */ |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 306 | char *vm_addr; /* address of kmap_atomic()'ed pages */ |
| 307 | enum zs_mapmode vm_mm; /* mapping mode */ |
| 308 | }; |
| 309 | |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 310 | /* huge object: pages_per_zspage == 1 && maxobj_per_zspage == 1 */ |
| 311 | static void SetZsHugePage(struct zspage *zspage) |
| 312 | { |
| 313 | zspage->huge = 1; |
| 314 | } |
| 315 | |
| 316 | static bool ZsHugePage(struct zspage *zspage) |
| 317 | { |
| 318 | return zspage->huge; |
| 319 | } |
| 320 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 321 | static void migrate_lock_init(struct zspage *zspage); |
| 322 | static void migrate_read_lock(struct zspage *zspage); |
| 323 | static void migrate_read_unlock(struct zspage *zspage); |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 324 | |
| 325 | #ifdef CONFIG_COMPACTION |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 326 | static void migrate_write_lock(struct zspage *zspage); |
| 327 | static void migrate_write_lock_nested(struct zspage *zspage); |
| 328 | static void migrate_write_unlock(struct zspage *zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 329 | static void kick_deferred_free(struct zs_pool *pool); |
| 330 | static void init_deferred_free(struct zs_pool *pool); |
| 331 | static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage); |
| 332 | #else |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 333 | static void migrate_write_lock(struct zspage *zspage) {} |
| 334 | static void migrate_write_lock_nested(struct zspage *zspage) {} |
| 335 | static void migrate_write_unlock(struct zspage *zspage) {} |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 336 | static void kick_deferred_free(struct zs_pool *pool) {} |
| 337 | static void init_deferred_free(struct zs_pool *pool) {} |
| 338 | static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {} |
| 339 | #endif |
| 340 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 341 | static int create_cache(struct zs_pool *pool) |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 342 | { |
| 343 | pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE, |
| 344 | 0, 0, NULL); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 345 | if (!pool->handle_cachep) |
| 346 | return 1; |
| 347 | |
| 348 | pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage), |
| 349 | 0, 0, NULL); |
| 350 | if (!pool->zspage_cachep) { |
| 351 | kmem_cache_destroy(pool->handle_cachep); |
| 352 | pool->handle_cachep = NULL; |
| 353 | return 1; |
| 354 | } |
| 355 | |
| 356 | return 0; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 357 | } |
| 358 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 359 | static void destroy_cache(struct zs_pool *pool) |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 360 | { |
Sergey Senozhatsky | cd10add | 2015-09-08 15:04:55 -0700 | [diff] [blame] | 361 | kmem_cache_destroy(pool->handle_cachep); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 362 | kmem_cache_destroy(pool->zspage_cachep); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 363 | } |
| 364 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 365 | static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp) |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 366 | { |
| 367 | return (unsigned long)kmem_cache_alloc(pool->handle_cachep, |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 368 | gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE)); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 369 | } |
| 370 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 371 | static void cache_free_handle(struct zs_pool *pool, unsigned long handle) |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 372 | { |
| 373 | kmem_cache_free(pool->handle_cachep, (void *)handle); |
| 374 | } |
| 375 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 376 | static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags) |
| 377 | { |
Miaohe Lin | f023130 | 2021-02-25 17:18:27 -0800 | [diff] [blame] | 378 | return kmem_cache_zalloc(pool->zspage_cachep, |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 379 | flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE)); |
Xishi Qiu | 399d8ee | 2017-02-22 15:45:01 -0800 | [diff] [blame] | 380 | } |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 381 | |
| 382 | static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage) |
| 383 | { |
| 384 | kmem_cache_free(pool->zspage_cachep, zspage); |
| 385 | } |
| 386 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 387 | /* pool->lock(which owns the handle) synchronizes races */ |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 388 | static void record_obj(unsigned long handle, unsigned long obj) |
| 389 | { |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 390 | *(unsigned long *)handle = obj; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 391 | } |
| 392 | |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 393 | /* zpool driver */ |
| 394 | |
| 395 | #ifdef CONFIG_ZPOOL |
| 396 | |
Sergey SENOZHATSKY | 6f3526d | 2015-11-06 16:29:21 -0800 | [diff] [blame] | 397 | static void *zs_zpool_create(const char *name, gfp_t gfp, |
Krzysztof Kozlowski | 7867277 | 2015-09-08 15:05:03 -0700 | [diff] [blame] | 398 | const struct zpool_ops *zpool_ops, |
Dan Streetman | 479305f | 2015-06-25 15:00:40 -0700 | [diff] [blame] | 399 | struct zpool *zpool) |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 400 | { |
Sergey Senozhatsky | d0d8da2 | 2016-05-20 16:59:48 -0700 | [diff] [blame] | 401 | /* |
| 402 | * Ignore global gfp flags: zs_malloc() may be invoked from |
| 403 | * different contexts and its caller must provide a valid |
| 404 | * gfp mask. |
| 405 | */ |
Nhat Pham | bd0fded | 2022-11-28 11:16:14 -0800 | [diff] [blame] | 406 | struct zs_pool *pool = zs_create_pool(name); |
| 407 | |
| 408 | if (pool) { |
| 409 | pool->zpool = zpool; |
| 410 | pool->zpool_ops = zpool_ops; |
| 411 | } |
| 412 | |
| 413 | return pool; |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 414 | } |
| 415 | |
| 416 | static void zs_zpool_destroy(void *pool) |
| 417 | { |
| 418 | zs_destroy_pool(pool); |
| 419 | } |
| 420 | |
| 421 | static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp, |
| 422 | unsigned long *handle) |
| 423 | { |
Sergey Senozhatsky | d0d8da2 | 2016-05-20 16:59:48 -0700 | [diff] [blame] | 424 | *handle = zs_malloc(pool, size, gfp); |
Hui Zhu | c7e6f17 | 2022-07-14 16:07:57 +0800 | [diff] [blame] | 425 | |
Deming Wang | 65917b53 | 2022-11-03 22:38:18 -0400 | [diff] [blame] | 426 | if (IS_ERR_VALUE(*handle)) |
Hui Zhu | c7e6f17 | 2022-07-14 16:07:57 +0800 | [diff] [blame] | 427 | return PTR_ERR((void *)*handle); |
| 428 | return 0; |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 429 | } |
| 430 | static void zs_zpool_free(void *pool, unsigned long handle) |
| 431 | { |
| 432 | zs_free(pool, handle); |
| 433 | } |
| 434 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 435 | static int zs_reclaim_page(struct zs_pool *pool, unsigned int retries); |
| 436 | |
| 437 | static int zs_zpool_shrink(void *pool, unsigned int pages, |
| 438 | unsigned int *reclaimed) |
| 439 | { |
| 440 | unsigned int total = 0; |
| 441 | int ret = -EINVAL; |
| 442 | |
| 443 | while (total < pages) { |
| 444 | ret = zs_reclaim_page(pool, 8); |
| 445 | if (ret < 0) |
| 446 | break; |
| 447 | total++; |
| 448 | } |
| 449 | |
| 450 | if (reclaimed) |
| 451 | *reclaimed = total; |
| 452 | |
| 453 | return ret; |
| 454 | } |
| 455 | |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 456 | static void *zs_zpool_map(void *pool, unsigned long handle, |
| 457 | enum zpool_mapmode mm) |
| 458 | { |
| 459 | enum zs_mapmode zs_mm; |
| 460 | |
| 461 | switch (mm) { |
| 462 | case ZPOOL_MM_RO: |
| 463 | zs_mm = ZS_MM_RO; |
| 464 | break; |
| 465 | case ZPOOL_MM_WO: |
| 466 | zs_mm = ZS_MM_WO; |
| 467 | break; |
Joe Perches | e4a9bc5 | 2020-04-06 20:08:39 -0700 | [diff] [blame] | 468 | case ZPOOL_MM_RW: |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 469 | default: |
| 470 | zs_mm = ZS_MM_RW; |
| 471 | break; |
| 472 | } |
| 473 | |
| 474 | return zs_map_object(pool, handle, zs_mm); |
| 475 | } |
| 476 | static void zs_zpool_unmap(void *pool, unsigned long handle) |
| 477 | { |
| 478 | zs_unmap_object(pool, handle); |
| 479 | } |
| 480 | |
| 481 | static u64 zs_zpool_total_size(void *pool) |
| 482 | { |
Minchan Kim | 722cdc1 | 2014-10-09 15:29:50 -0700 | [diff] [blame] | 483 | return zs_get_total_pages(pool) << PAGE_SHIFT; |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 484 | } |
| 485 | |
| 486 | static struct zpool_driver zs_zpool_driver = { |
Hui Zhu | c165f25 | 2019-09-23 15:39:37 -0700 | [diff] [blame] | 487 | .type = "zsmalloc", |
| 488 | .owner = THIS_MODULE, |
| 489 | .create = zs_zpool_create, |
| 490 | .destroy = zs_zpool_destroy, |
| 491 | .malloc_support_movable = true, |
| 492 | .malloc = zs_zpool_malloc, |
| 493 | .free = zs_zpool_free, |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 494 | .shrink = zs_zpool_shrink, |
Hui Zhu | c165f25 | 2019-09-23 15:39:37 -0700 | [diff] [blame] | 495 | .map = zs_zpool_map, |
| 496 | .unmap = zs_zpool_unmap, |
| 497 | .total_size = zs_zpool_total_size, |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 498 | }; |
| 499 | |
Kees Cook | 137f8cf | 2014-08-29 15:18:40 -0700 | [diff] [blame] | 500 | MODULE_ALIAS("zpool-zsmalloc"); |
Dan Streetman | c795779 | 2014-08-06 16:08:38 -0700 | [diff] [blame] | 501 | #endif /* CONFIG_ZPOOL */ |
| 502 | |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 503 | /* per-cpu VM mapping areas for zspage accesses that cross page boundaries */ |
Mike Galbraith | a372659 | 2022-01-21 22:14:17 -0800 | [diff] [blame] | 504 | static DEFINE_PER_CPU(struct mapping_area, zs_map_area) = { |
| 505 | .lock = INIT_LOCAL_LOCK(lock), |
| 506 | }; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 507 | |
Nick Desaulniers | 3457f41 | 2017-07-10 15:47:26 -0700 | [diff] [blame] | 508 | static __maybe_unused int is_first_page(struct page *page) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 509 | { |
Minchan Kim | a27545bf | 2012-04-25 15:23:09 +0900 | [diff] [blame] | 510 | return PagePrivate(page); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 511 | } |
| 512 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 513 | /* Protected by pool->lock */ |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 514 | static inline int get_zspage_inuse(struct zspage *zspage) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 515 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 516 | return zspage->inuse; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 517 | } |
| 518 | |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 519 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 520 | static inline void mod_zspage_inuse(struct zspage *zspage, int val) |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 521 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 522 | zspage->inuse += val; |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 523 | } |
| 524 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 525 | static inline struct page *get_first_page(struct zspage *zspage) |
| 526 | { |
| 527 | struct page *first_page = zspage->first_page; |
| 528 | |
| 529 | VM_BUG_ON_PAGE(!is_first_page(first_page), first_page); |
| 530 | return first_page; |
| 531 | } |
| 532 | |
Alexey Romanov | 671f2fa | 2022-09-09 11:37:22 +0300 | [diff] [blame] | 533 | static inline unsigned int get_first_obj_offset(struct page *page) |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 534 | { |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 535 | return page->page_type; |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 536 | } |
| 537 | |
Alexey Romanov | 671f2fa | 2022-09-09 11:37:22 +0300 | [diff] [blame] | 538 | static inline void set_first_obj_offset(struct page *page, unsigned int offset) |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 539 | { |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 540 | page->page_type = offset; |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 541 | } |
| 542 | |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 543 | static inline unsigned int get_freeobj(struct zspage *zspage) |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 544 | { |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 545 | return zspage->freeobj; |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 546 | } |
| 547 | |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 548 | static inline void set_freeobj(struct zspage *zspage, unsigned int obj) |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 549 | { |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 550 | zspage->freeobj = obj; |
Minchan Kim | 4f42047 | 2016-07-26 15:23:17 -0700 | [diff] [blame] | 551 | } |
| 552 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 553 | static void get_zspage_mapping(struct zspage *zspage, |
Minchan Kim | a420946 | 2016-05-20 16:59:36 -0700 | [diff] [blame] | 554 | unsigned int *class_idx, |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 555 | enum fullness_group *fullness) |
| 556 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 557 | BUG_ON(zspage->magic != ZSPAGE_MAGIC); |
| 558 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 559 | *fullness = zspage->fullness; |
| 560 | *class_idx = zspage->class; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 561 | } |
| 562 | |
Minchan Kim | 67f1c9cd | 2022-01-21 22:13:51 -0800 | [diff] [blame] | 563 | static struct size_class *zspage_class(struct zs_pool *pool, |
| 564 | struct zspage *zspage) |
| 565 | { |
| 566 | return pool->size_class[zspage->class]; |
| 567 | } |
| 568 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 569 | static void set_zspage_mapping(struct zspage *zspage, |
Minchan Kim | a420946 | 2016-05-20 16:59:36 -0700 | [diff] [blame] | 570 | unsigned int class_idx, |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 571 | enum fullness_group fullness) |
| 572 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 573 | zspage->class = class_idx; |
| 574 | zspage->fullness = fullness; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 575 | } |
| 576 | |
Nitin Cupta | c3e3e88a | 2013-12-11 11:04:37 +0900 | [diff] [blame] | 577 | /* |
| 578 | * zsmalloc divides the pool into various size classes where each |
| 579 | * class maintains a list of zspages where each zspage is divided |
| 580 | * into equal sized chunks. Each allocation falls into one of these |
| 581 | * classes depending on its size. This function returns index of the |
Shijie Luo | cb152a1 | 2021-05-06 18:05:51 -0700 | [diff] [blame] | 582 | * size class which has chunk size big enough to hold the given size. |
Nitin Cupta | c3e3e88a | 2013-12-11 11:04:37 +0900 | [diff] [blame] | 583 | */ |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 584 | static int get_size_class_index(int size) |
| 585 | { |
| 586 | int idx = 0; |
| 587 | |
| 588 | if (likely(size > ZS_MIN_ALLOC_SIZE)) |
| 589 | idx = DIV_ROUND_UP(size - ZS_MIN_ALLOC_SIZE, |
| 590 | ZS_SIZE_CLASS_DELTA); |
| 591 | |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 592 | return min_t(int, ZS_SIZE_CLASSES - 1, idx); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 593 | } |
| 594 | |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 595 | /* type can be of enum type class_stat_type or fullness_group */ |
| 596 | static inline void class_stat_inc(struct size_class *class, |
Matthias Kaehlcke | 3eb95fe | 2017-09-08 16:13:02 -0700 | [diff] [blame] | 597 | int type, unsigned long cnt) |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 598 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 599 | class->stats.objs[type] += cnt; |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 600 | } |
| 601 | |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 602 | /* type can be of enum type class_stat_type or fullness_group */ |
| 603 | static inline void class_stat_dec(struct size_class *class, |
Matthias Kaehlcke | 3eb95fe | 2017-09-08 16:13:02 -0700 | [diff] [blame] | 604 | int type, unsigned long cnt) |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 605 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 606 | class->stats.objs[type] -= cnt; |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 607 | } |
| 608 | |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 609 | /* type can be of enum type class_stat_type or fullness_group */ |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 610 | static inline unsigned long zs_stat_get(struct size_class *class, |
Matthias Kaehlcke | 3eb95fe | 2017-09-08 16:13:02 -0700 | [diff] [blame] | 611 | int type) |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 612 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 613 | return class->stats.objs[type]; |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 614 | } |
| 615 | |
Sergey Senozhatsky | 5724459 | 2015-09-08 15:04:27 -0700 | [diff] [blame] | 616 | #ifdef CONFIG_ZSMALLOC_STAT |
| 617 | |
Dan Streetman | 4abaac9 | 2016-05-26 15:16:27 -0700 | [diff] [blame] | 618 | static void __init zs_stat_init(void) |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 619 | { |
Dan Streetman | 4abaac9 | 2016-05-26 15:16:27 -0700 | [diff] [blame] | 620 | if (!debugfs_initialized()) { |
| 621 | pr_warn("debugfs not available, stat dir not created\n"); |
| 622 | return; |
| 623 | } |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 624 | |
| 625 | zs_stat_root = debugfs_create_dir("zsmalloc", NULL); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 626 | } |
| 627 | |
| 628 | static void __exit zs_stat_exit(void) |
| 629 | { |
| 630 | debugfs_remove_recursive(zs_stat_root); |
| 631 | } |
| 632 | |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 633 | static unsigned long zs_can_compact(struct size_class *class); |
| 634 | |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 635 | static int zs_stats_size_show(struct seq_file *s, void *v) |
| 636 | { |
| 637 | int i; |
| 638 | struct zs_pool *pool = s->private; |
| 639 | struct size_class *class; |
| 640 | int objs_per_zspage; |
| 641 | unsigned long class_almost_full, class_almost_empty; |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 642 | unsigned long obj_allocated, obj_used, pages_used, freeable; |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 643 | unsigned long total_class_almost_full = 0, total_class_almost_empty = 0; |
| 644 | unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0; |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 645 | unsigned long total_freeable = 0; |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 646 | |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 647 | seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n", |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 648 | "class", "size", "almost_full", "almost_empty", |
| 649 | "obj_allocated", "obj_used", "pages_used", |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 650 | "pages_per_zspage", "freeable"); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 651 | |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 652 | for (i = 0; i < ZS_SIZE_CLASSES; i++) { |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 653 | class = pool->size_class[i]; |
| 654 | |
| 655 | if (class->index != i) |
| 656 | continue; |
| 657 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 658 | spin_lock(&pool->lock); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 659 | class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL); |
| 660 | class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY); |
| 661 | obj_allocated = zs_stat_get(class, OBJ_ALLOCATED); |
| 662 | obj_used = zs_stat_get(class, OBJ_USED); |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 663 | freeable = zs_can_compact(class); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 664 | spin_unlock(&pool->lock); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 665 | |
Ganesh Mahendran | b4fd07a | 2016-07-28 15:47:49 -0700 | [diff] [blame] | 666 | objs_per_zspage = class->objs_per_zspage; |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 667 | pages_used = obj_allocated / objs_per_zspage * |
| 668 | class->pages_per_zspage; |
| 669 | |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 670 | seq_printf(s, " %5u %5u %11lu %12lu %13lu" |
| 671 | " %10lu %10lu %16d %8lu\n", |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 672 | i, class->size, class_almost_full, class_almost_empty, |
| 673 | obj_allocated, obj_used, pages_used, |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 674 | class->pages_per_zspage, freeable); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 675 | |
| 676 | total_class_almost_full += class_almost_full; |
| 677 | total_class_almost_empty += class_almost_empty; |
| 678 | total_objs += obj_allocated; |
| 679 | total_used_objs += obj_used; |
| 680 | total_pages += pages_used; |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 681 | total_freeable += freeable; |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 682 | } |
| 683 | |
| 684 | seq_puts(s, "\n"); |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 685 | seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n", |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 686 | "Total", "", total_class_almost_full, |
| 687 | total_class_almost_empty, total_objs, |
Sergey Senozhatsky | 1120ed5 | 2016-03-17 14:20:42 -0700 | [diff] [blame] | 688 | total_used_objs, total_pages, "", total_freeable); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 689 | |
| 690 | return 0; |
| 691 | } |
Andy Shevchenko | 5ad3509 | 2018-04-05 16:23:16 -0700 | [diff] [blame] | 692 | DEFINE_SHOW_ATTRIBUTE(zs_stats_size); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 693 | |
Dan Streetman | d34f615 | 2016-05-20 16:59:56 -0700 | [diff] [blame] | 694 | static void zs_pool_stat_create(struct zs_pool *pool, const char *name) |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 695 | { |
Dan Streetman | 4abaac9 | 2016-05-26 15:16:27 -0700 | [diff] [blame] | 696 | if (!zs_stat_root) { |
| 697 | pr_warn("no root stat dir, not creating <%s> stat dir\n", name); |
Dan Streetman | d34f615 | 2016-05-20 16:59:56 -0700 | [diff] [blame] | 698 | return; |
Dan Streetman | 4abaac9 | 2016-05-26 15:16:27 -0700 | [diff] [blame] | 699 | } |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 700 | |
Greg Kroah-Hartman | 4268509 | 2019-01-22 16:21:09 +0100 | [diff] [blame] | 701 | pool->stat_dentry = debugfs_create_dir(name, zs_stat_root); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 702 | |
Greg Kroah-Hartman | 4268509 | 2019-01-22 16:21:09 +0100 | [diff] [blame] | 703 | debugfs_create_file("classes", S_IFREG | 0444, pool->stat_dentry, pool, |
| 704 | &zs_stats_size_fops); |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 705 | } |
| 706 | |
| 707 | static void zs_pool_stat_destroy(struct zs_pool *pool) |
| 708 | { |
| 709 | debugfs_remove_recursive(pool->stat_dentry); |
| 710 | } |
| 711 | |
| 712 | #else /* CONFIG_ZSMALLOC_STAT */ |
Dan Streetman | 4abaac9 | 2016-05-26 15:16:27 -0700 | [diff] [blame] | 713 | static void __init zs_stat_init(void) |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 714 | { |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 715 | } |
| 716 | |
| 717 | static void __exit zs_stat_exit(void) |
| 718 | { |
| 719 | } |
| 720 | |
Dan Streetman | d34f615 | 2016-05-20 16:59:56 -0700 | [diff] [blame] | 721 | static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name) |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 722 | { |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 723 | } |
| 724 | |
| 725 | static inline void zs_pool_stat_destroy(struct zs_pool *pool) |
| 726 | { |
| 727 | } |
Minchan Kim | 248ca1b | 2015-04-15 16:15:42 -0700 | [diff] [blame] | 728 | #endif |
| 729 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 730 | |
Nitin Cupta | c3e3e88a | 2013-12-11 11:04:37 +0900 | [diff] [blame] | 731 | /* |
| 732 | * For each size class, zspages are divided into different groups |
| 733 | * depending on how "full" they are. This was done so that we could |
| 734 | * easily find empty or nearly empty zspages when we try to shrink |
| 735 | * the pool (not yet implemented). This function returns fullness |
| 736 | * status of the given page. |
| 737 | */ |
Minchan Kim | 1fc6e27 | 2016-07-26 15:23:11 -0700 | [diff] [blame] | 738 | static enum fullness_group get_fullness_group(struct size_class *class, |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 739 | struct zspage *zspage) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 740 | { |
Minchan Kim | 1fc6e27 | 2016-07-26 15:23:11 -0700 | [diff] [blame] | 741 | int inuse, objs_per_zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 742 | enum fullness_group fg; |
Minchan Kim | 830e4bc | 2016-05-20 16:59:39 -0700 | [diff] [blame] | 743 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 744 | inuse = get_zspage_inuse(zspage); |
Minchan Kim | 1fc6e27 | 2016-07-26 15:23:11 -0700 | [diff] [blame] | 745 | objs_per_zspage = class->objs_per_zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 746 | |
| 747 | if (inuse == 0) |
| 748 | fg = ZS_EMPTY; |
Minchan Kim | 1fc6e27 | 2016-07-26 15:23:11 -0700 | [diff] [blame] | 749 | else if (inuse == objs_per_zspage) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 750 | fg = ZS_FULL; |
Minchan Kim | 1fc6e27 | 2016-07-26 15:23:11 -0700 | [diff] [blame] | 751 | else if (inuse <= 3 * objs_per_zspage / fullness_threshold_frac) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 752 | fg = ZS_ALMOST_EMPTY; |
| 753 | else |
| 754 | fg = ZS_ALMOST_FULL; |
| 755 | |
| 756 | return fg; |
| 757 | } |
| 758 | |
Nitin Cupta | c3e3e88a | 2013-12-11 11:04:37 +0900 | [diff] [blame] | 759 | /* |
| 760 | * Each size class maintains various freelists and zspages are assigned |
| 761 | * to one of these freelists based on the number of live objects they |
| 762 | * have. This functions inserts the given zspage into the freelist |
| 763 | * identified by <class, fullness_group>. |
| 764 | */ |
Minchan Kim | 251cbb9 | 2016-05-20 16:59:42 -0700 | [diff] [blame] | 765 | static void insert_zspage(struct size_class *class, |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 766 | struct zspage *zspage, |
| 767 | enum fullness_group fullness) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 768 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 769 | struct zspage *head; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 770 | |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 771 | class_stat_inc(class, fullness, 1); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 772 | head = list_first_entry_or_null(&class->fullness_list[fullness], |
| 773 | struct zspage, list); |
Sergey Senozhatsky | 58f1711 | 2015-09-08 15:04:44 -0700 | [diff] [blame] | 774 | /* |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 775 | * We want to see more ZS_FULL pages and less almost empty/full. |
| 776 | * Put pages with higher ->inuse first. |
Sergey Senozhatsky | 58f1711 | 2015-09-08 15:04:44 -0700 | [diff] [blame] | 777 | */ |
Miaohe Lin | 110ceb8 | 2020-12-14 19:14:22 -0800 | [diff] [blame] | 778 | if (head && get_zspage_inuse(zspage) < get_zspage_inuse(head)) |
| 779 | list_add(&zspage->list, &head->list); |
| 780 | else |
| 781 | list_add(&zspage->list, &class->fullness_list[fullness]); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 782 | } |
| 783 | |
Nitin Cupta | c3e3e88a | 2013-12-11 11:04:37 +0900 | [diff] [blame] | 784 | /* |
| 785 | * This function removes the given zspage from the freelist identified |
| 786 | * by <class, fullness_group>. |
| 787 | */ |
Minchan Kim | 251cbb9 | 2016-05-20 16:59:42 -0700 | [diff] [blame] | 788 | static void remove_zspage(struct size_class *class, |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 789 | struct zspage *zspage, |
| 790 | enum fullness_group fullness) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 791 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 792 | VM_BUG_ON(list_empty(&class->fullness_list[fullness])); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 793 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 794 | list_del_init(&zspage->list); |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 795 | class_stat_dec(class, fullness, 1); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 796 | } |
| 797 | |
Nitin Cupta | c3e3e88a | 2013-12-11 11:04:37 +0900 | [diff] [blame] | 798 | /* |
| 799 | * Each size class maintains zspages in different fullness groups depending |
| 800 | * on the number of live objects they contain. When allocating or freeing |
| 801 | * objects, the fullness status of the page can change, say, from ALMOST_FULL |
| 802 | * to ALMOST_EMPTY when freeing an object. This function checks if such |
| 803 | * a status change has occurred for the given page and accordingly moves the |
| 804 | * page from the freelist of the old fullness group to that of the new |
| 805 | * fullness group. |
| 806 | */ |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 807 | static enum fullness_group fix_fullness_group(struct size_class *class, |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 808 | struct zspage *zspage) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 809 | { |
| 810 | int class_idx; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 811 | enum fullness_group currfg, newfg; |
| 812 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 813 | get_zspage_mapping(zspage, &class_idx, &currfg); |
| 814 | newfg = get_fullness_group(class, zspage); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 815 | if (newfg == currfg) |
| 816 | goto out; |
| 817 | |
Minchan Kim | c4549b8 | 2022-01-21 22:14:07 -0800 | [diff] [blame] | 818 | remove_zspage(class, zspage, currfg); |
| 819 | insert_zspage(class, zspage, newfg); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 820 | set_zspage_mapping(zspage, class_idx, newfg); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 821 | out: |
| 822 | return newfg; |
| 823 | } |
| 824 | |
| 825 | /* |
| 826 | * We have to decide on how many pages to link together |
| 827 | * to form a zspage for each size class. This is important |
| 828 | * to reduce wastage due to unusable space left at end of |
| 829 | * each zspage which is given as: |
Yinghao Xie | 888fa374 | 2015-04-15 16:15:49 -0700 | [diff] [blame] | 830 | * wastage = Zp % class_size |
| 831 | * usage = Zp - wastage |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 832 | * where Zp = zspage size = k * PAGE_SIZE where k = 1, 2, ... |
| 833 | * |
| 834 | * For example, for size class of 3/8 * PAGE_SIZE, we should |
| 835 | * link together 3 PAGE_SIZE sized pages to form a zspage |
| 836 | * since then we can perfectly fit in 8 such objects. |
| 837 | */ |
Minchan Kim | 2e3b6154 | 2012-05-03 15:40:39 +0900 | [diff] [blame] | 838 | static int get_pages_per_zspage(int class_size) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 839 | { |
| 840 | int i, max_usedpc = 0; |
| 841 | /* zspage order which gives maximum used size per KB */ |
| 842 | int max_usedpc_order = 1; |
| 843 | |
Seth Jennings | 84d4faa | 2012-03-05 11:33:21 -0600 | [diff] [blame] | 844 | for (i = 1; i <= ZS_MAX_PAGES_PER_ZSPAGE; i++) { |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 845 | int zspage_size; |
| 846 | int waste, usedpc; |
| 847 | |
| 848 | zspage_size = i * PAGE_SIZE; |
| 849 | waste = zspage_size % class_size; |
| 850 | usedpc = (zspage_size - waste) * 100 / zspage_size; |
| 851 | |
| 852 | if (usedpc > max_usedpc) { |
| 853 | max_usedpc = usedpc; |
| 854 | max_usedpc_order = i; |
| 855 | } |
| 856 | } |
| 857 | |
| 858 | return max_usedpc_order; |
| 859 | } |
| 860 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 861 | static struct zspage *get_zspage(struct page *page) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 862 | { |
Miaohe Lin | a6c5e0f | 2021-02-25 17:18:34 -0800 | [diff] [blame] | 863 | struct zspage *zspage = (struct zspage *)page_private(page); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 864 | |
| 865 | BUG_ON(zspage->magic != ZSPAGE_MAGIC); |
| 866 | return zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 867 | } |
| 868 | |
| 869 | static struct page *get_next_page(struct page *page) |
| 870 | { |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 871 | struct zspage *zspage = get_zspage(page); |
| 872 | |
| 873 | if (unlikely(ZsHugePage(zspage))) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 874 | return NULL; |
| 875 | |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 876 | return (struct page *)page->index; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 877 | } |
| 878 | |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 879 | /** |
| 880 | * obj_to_location - get (<page>, <obj_idx>) from encoded object value |
Mike Rapoport | e8b098f | 2018-04-05 16:24:57 -0700 | [diff] [blame] | 881 | * @obj: the encoded object value |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 882 | * @page: page object resides in zspage |
| 883 | * @obj_idx: object index |
Olav Haugan | 6729687 | 2013-11-22 09:30:41 -0800 | [diff] [blame] | 884 | */ |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 885 | static void obj_to_location(unsigned long obj, struct page **page, |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 886 | unsigned int *obj_idx) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 887 | { |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 888 | obj >>= OBJ_TAG_BITS; |
| 889 | *page = pfn_to_page(obj >> OBJ_INDEX_BITS); |
| 890 | *obj_idx = (obj & OBJ_INDEX_MASK); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 891 | } |
| 892 | |
Minchan Kim | 67f1c9cd | 2022-01-21 22:13:51 -0800 | [diff] [blame] | 893 | static void obj_to_page(unsigned long obj, struct page **page) |
| 894 | { |
| 895 | obj >>= OBJ_TAG_BITS; |
| 896 | *page = pfn_to_page(obj >> OBJ_INDEX_BITS); |
| 897 | } |
| 898 | |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 899 | /** |
| 900 | * location_to_obj - get obj value encoded from (<page>, <obj_idx>) |
| 901 | * @page: page object resides in zspage |
| 902 | * @obj_idx: object index |
| 903 | */ |
| 904 | static unsigned long location_to_obj(struct page *page, unsigned int obj_idx) |
| 905 | { |
| 906 | unsigned long obj; |
| 907 | |
| 908 | obj = page_to_pfn(page) << OBJ_INDEX_BITS; |
| 909 | obj |= obj_idx & OBJ_INDEX_MASK; |
| 910 | obj <<= OBJ_TAG_BITS; |
| 911 | |
| 912 | return obj; |
| 913 | } |
| 914 | |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 915 | static unsigned long handle_to_obj(unsigned long handle) |
| 916 | { |
| 917 | return *(unsigned long *)handle; |
| 918 | } |
| 919 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 920 | static bool obj_tagged(struct page *page, void *obj, unsigned long *phandle, |
| 921 | int tag) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 922 | { |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 923 | unsigned long handle; |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 924 | struct zspage *zspage = get_zspage(page); |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 925 | |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 926 | if (unlikely(ZsHugePage(zspage))) { |
Minchan Kim | 830e4bc | 2016-05-20 16:59:39 -0700 | [diff] [blame] | 927 | VM_BUG_ON_PAGE(!is_first_page(page), page); |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 928 | handle = page->index; |
Minchan Kim | 7b60a68 | 2015-04-15 16:15:39 -0700 | [diff] [blame] | 929 | } else |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 930 | handle = *(unsigned long *)obj; |
| 931 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 932 | if (!(handle & tag)) |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 933 | return false; |
| 934 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 935 | /* Clear all tags before returning the handle */ |
| 936 | *phandle = handle & ~OBJ_TAG_MASK; |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 937 | return true; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 938 | } |
| 939 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 940 | static inline bool obj_allocated(struct page *page, void *obj, unsigned long *phandle) |
| 941 | { |
| 942 | return obj_tagged(page, obj, phandle, OBJ_ALLOCATED_TAG); |
| 943 | } |
| 944 | |
| 945 | #ifdef CONFIG_ZPOOL |
| 946 | static bool obj_stores_deferred_handle(struct page *page, void *obj, |
| 947 | unsigned long *phandle) |
| 948 | { |
| 949 | return obj_tagged(page, obj, phandle, OBJ_DEFERRED_HANDLE_TAG); |
| 950 | } |
| 951 | #endif |
| 952 | |
Nitin Gupta | f4477e9 | 2012-04-02 09:13:56 -0500 | [diff] [blame] | 953 | static void reset_page(struct page *page) |
| 954 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 955 | __ClearPageMovable(page); |
Ganesh Mahendran | 18fd06b | 2016-07-28 15:48:00 -0700 | [diff] [blame] | 956 | ClearPagePrivate(page); |
Nitin Gupta | f4477e9 | 2012-04-02 09:13:56 -0500 | [diff] [blame] | 957 | set_page_private(page, 0); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 958 | page_mapcount_reset(page); |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 959 | page->index = 0; |
Nitin Gupta | f4477e9 | 2012-04-02 09:13:56 -0500 | [diff] [blame] | 960 | } |
| 961 | |
Colin Ian King | 4d0a540 | 2018-08-17 15:46:50 -0700 | [diff] [blame] | 962 | static int trylock_zspage(struct zspage *zspage) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 963 | { |
| 964 | struct page *cursor, *fail; |
| 965 | |
| 966 | for (cursor = get_first_page(zspage); cursor != NULL; cursor = |
| 967 | get_next_page(cursor)) { |
| 968 | if (!trylock_page(cursor)) { |
| 969 | fail = cursor; |
| 970 | goto unlock; |
| 971 | } |
| 972 | } |
| 973 | |
| 974 | return 1; |
| 975 | unlock: |
| 976 | for (cursor = get_first_page(zspage); cursor != fail; cursor = |
| 977 | get_next_page(cursor)) |
| 978 | unlock_page(cursor); |
| 979 | |
| 980 | return 0; |
| 981 | } |
| 982 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 983 | #ifdef CONFIG_ZPOOL |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 984 | static unsigned long find_deferred_handle_obj(struct size_class *class, |
| 985 | struct page *page, int *obj_idx); |
| 986 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 987 | /* |
| 988 | * Free all the deferred handles whose objects are freed in zs_free. |
| 989 | */ |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 990 | static void free_handles(struct zs_pool *pool, struct size_class *class, |
| 991 | struct zspage *zspage) |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 992 | { |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 993 | int obj_idx = 0; |
| 994 | struct page *page = get_first_page(zspage); |
| 995 | unsigned long handle; |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 996 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 997 | while (1) { |
| 998 | handle = find_deferred_handle_obj(class, page, &obj_idx); |
| 999 | if (!handle) { |
| 1000 | page = get_next_page(page); |
| 1001 | if (!page) |
| 1002 | break; |
| 1003 | obj_idx = 0; |
| 1004 | continue; |
| 1005 | } |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1006 | |
| 1007 | cache_free_handle(pool, handle); |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1008 | obj_idx++; |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1009 | } |
| 1010 | } |
| 1011 | #else |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1012 | static inline void free_handles(struct zs_pool *pool, struct size_class *class, |
| 1013 | struct zspage *zspage) {} |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1014 | #endif |
| 1015 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1016 | static void __free_zspage(struct zs_pool *pool, struct size_class *class, |
| 1017 | struct zspage *zspage) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1018 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1019 | struct page *page, *next; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1020 | enum fullness_group fg; |
| 1021 | unsigned int class_idx; |
| 1022 | |
| 1023 | get_zspage_mapping(zspage, &class_idx, &fg); |
| 1024 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1025 | assert_spin_locked(&pool->lock); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1026 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1027 | VM_BUG_ON(get_zspage_inuse(zspage)); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1028 | VM_BUG_ON(fg != ZS_EMPTY); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1029 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1030 | /* Free all deferred handles from zs_free */ |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1031 | free_handles(pool, class, zspage); |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1032 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1033 | next = page = get_first_page(zspage); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1034 | do { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1035 | VM_BUG_ON_PAGE(!PageLocked(page), page); |
| 1036 | next = get_next_page(page); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1037 | reset_page(page); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1038 | unlock_page(page); |
Minchan Kim | 91537fe | 2016-07-26 15:24:45 -0700 | [diff] [blame] | 1039 | dec_zone_page_state(page, NR_ZSPAGES); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1040 | put_page(page); |
| 1041 | page = next; |
| 1042 | } while (page != NULL); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1043 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1044 | cache_free_zspage(pool, zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1045 | |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 1046 | class_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1047 | atomic_long_sub(class->pages_per_zspage, |
| 1048 | &pool->pages_allocated); |
| 1049 | } |
| 1050 | |
| 1051 | static void free_zspage(struct zs_pool *pool, struct size_class *class, |
| 1052 | struct zspage *zspage) |
| 1053 | { |
| 1054 | VM_BUG_ON(get_zspage_inuse(zspage)); |
| 1055 | VM_BUG_ON(list_empty(&zspage->list)); |
| 1056 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1057 | /* |
| 1058 | * Since zs_free couldn't be sleepable, this function cannot call |
| 1059 | * lock_page. The page locks trylock_zspage got will be released |
| 1060 | * by __free_zspage. |
| 1061 | */ |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1062 | if (!trylock_zspage(zspage)) { |
| 1063 | kick_deferred_free(pool); |
| 1064 | return; |
| 1065 | } |
| 1066 | |
| 1067 | remove_zspage(class, zspage, ZS_EMPTY); |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 1068 | #ifdef CONFIG_ZPOOL |
| 1069 | list_del(&zspage->lru); |
| 1070 | #endif |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1071 | __free_zspage(pool, class, zspage); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1072 | } |
| 1073 | |
| 1074 | /* Initialize a newly allocated zspage */ |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1075 | static void init_zspage(struct size_class *class, struct zspage *zspage) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1076 | { |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1077 | unsigned int freeobj = 1; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1078 | unsigned long off = 0; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1079 | struct page *page = get_first_page(zspage); |
Minchan Kim | 830e4bc | 2016-05-20 16:59:39 -0700 | [diff] [blame] | 1080 | |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1081 | while (page) { |
| 1082 | struct page *next_page; |
| 1083 | struct link_free *link; |
Minchan Kim | af4ee5e | 2014-12-12 16:56:58 -0800 | [diff] [blame] | 1084 | void *vaddr; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1085 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1086 | set_first_obj_offset(page, off); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1087 | |
Minchan Kim | af4ee5e | 2014-12-12 16:56:58 -0800 | [diff] [blame] | 1088 | vaddr = kmap_atomic(page); |
| 1089 | link = (struct link_free *)vaddr + off / sizeof(*link); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1090 | |
Dan Streetman | 5538c56 | 2014-10-09 15:30:01 -0700 | [diff] [blame] | 1091 | while ((off += class->size) < PAGE_SIZE) { |
Minchan Kim | 3b1d9ca | 2016-07-26 15:23:37 -0700 | [diff] [blame] | 1092 | link->next = freeobj++ << OBJ_TAG_BITS; |
Dan Streetman | 5538c56 | 2014-10-09 15:30:01 -0700 | [diff] [blame] | 1093 | link += class->size / sizeof(*link); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1094 | } |
| 1095 | |
| 1096 | /* |
| 1097 | * We now come to the last (full or partial) object on this |
| 1098 | * page, which must point to the first object on the next |
| 1099 | * page (if present) |
| 1100 | */ |
| 1101 | next_page = get_next_page(page); |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1102 | if (next_page) { |
Minchan Kim | 3b1d9ca | 2016-07-26 15:23:37 -0700 | [diff] [blame] | 1103 | link->next = freeobj++ << OBJ_TAG_BITS; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1104 | } else { |
| 1105 | /* |
Minchan Kim | 3b1d9ca | 2016-07-26 15:23:37 -0700 | [diff] [blame] | 1106 | * Reset OBJ_TAG_BITS bit to last link to tell |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1107 | * whether it's allocated object or not. |
| 1108 | */ |
Nick Desaulniers | 01a6ad9 | 2018-01-31 16:20:15 -0800 | [diff] [blame] | 1109 | link->next = -1UL << OBJ_TAG_BITS; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1110 | } |
Minchan Kim | af4ee5e | 2014-12-12 16:56:58 -0800 | [diff] [blame] | 1111 | kunmap_atomic(vaddr); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1112 | page = next_page; |
Dan Streetman | 5538c56 | 2014-10-09 15:30:01 -0700 | [diff] [blame] | 1113 | off %= PAGE_SIZE; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1114 | } |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1115 | |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 1116 | #ifdef CONFIG_ZPOOL |
| 1117 | INIT_LIST_HEAD(&zspage->lru); |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1118 | zspage->under_reclaim = false; |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 1119 | #endif |
| 1120 | |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1121 | set_freeobj(zspage, 0); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1122 | } |
| 1123 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1124 | static void create_page_chain(struct size_class *class, struct zspage *zspage, |
| 1125 | struct page *pages[]) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1126 | { |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1127 | int i; |
| 1128 | struct page *page; |
| 1129 | struct page *prev_page = NULL; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1130 | int nr_pages = class->pages_per_zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1131 | |
| 1132 | /* |
| 1133 | * Allocate individual pages and link them together as: |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 1134 | * 1. all pages are linked together using page->index |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1135 | * 2. each sub-page point to zspage using page->private |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1136 | * |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1137 | * we set PG_private to identify the first page (i.e. no other sub-page |
Yisheng Xie | 22c5cef | 2017-02-24 14:59:42 -0800 | [diff] [blame] | 1138 | * has this flag set). |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1139 | */ |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1140 | for (i = 0; i < nr_pages; i++) { |
| 1141 | page = pages[i]; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1142 | set_page_private(page, (unsigned long)zspage); |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 1143 | page->index = 0; |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1144 | if (i == 0) { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1145 | zspage->first_page = page; |
Minchan Kim | a27545bf | 2012-04-25 15:23:09 +0900 | [diff] [blame] | 1146 | SetPagePrivate(page); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1147 | if (unlikely(class->objs_per_zspage == 1 && |
| 1148 | class->pages_per_zspage == 1)) |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 1149 | SetZsHugePage(zspage); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1150 | } else { |
Matthew Wilcox (Oracle) | ffedd09 | 2021-10-04 14:46:47 +0100 | [diff] [blame] | 1151 | prev_page->index = (unsigned long)page; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1152 | } |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1153 | prev_page = page; |
| 1154 | } |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1155 | } |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1156 | |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1157 | /* |
| 1158 | * Allocate a zspage for the given size class |
| 1159 | */ |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1160 | static struct zspage *alloc_zspage(struct zs_pool *pool, |
| 1161 | struct size_class *class, |
| 1162 | gfp_t gfp) |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1163 | { |
| 1164 | int i; |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1165 | struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE]; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1166 | struct zspage *zspage = cache_alloc_zspage(pool, gfp); |
| 1167 | |
| 1168 | if (!zspage) |
| 1169 | return NULL; |
| 1170 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1171 | zspage->magic = ZSPAGE_MAGIC; |
| 1172 | migrate_lock_init(zspage); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1173 | |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1174 | for (i = 0; i < class->pages_per_zspage; i++) { |
| 1175 | struct page *page; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1176 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1177 | page = alloc_page(gfp); |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1178 | if (!page) { |
Minchan Kim | 91537fe | 2016-07-26 15:24:45 -0700 | [diff] [blame] | 1179 | while (--i >= 0) { |
| 1180 | dec_zone_page_state(pages[i], NR_ZSPAGES); |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1181 | __free_page(pages[i]); |
Minchan Kim | 91537fe | 2016-07-26 15:24:45 -0700 | [diff] [blame] | 1182 | } |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1183 | cache_free_zspage(pool, zspage); |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1184 | return NULL; |
| 1185 | } |
Minchan Kim | 91537fe | 2016-07-26 15:24:45 -0700 | [diff] [blame] | 1186 | |
| 1187 | inc_zone_page_state(page, NR_ZSPAGES); |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1188 | pages[i] = page; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1189 | } |
| 1190 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1191 | create_page_chain(class, zspage, pages); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1192 | init_zspage(class, zspage); |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 1193 | zspage->pool = pool; |
Minchan Kim | bdb0af7 | 2016-07-26 15:23:20 -0700 | [diff] [blame] | 1194 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1195 | return zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1196 | } |
| 1197 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1198 | static struct zspage *find_get_zspage(struct size_class *class) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1199 | { |
| 1200 | int i; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1201 | struct zspage *zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1202 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1203 | for (i = ZS_ALMOST_FULL; i >= ZS_EMPTY; i--) { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1204 | zspage = list_first_entry_or_null(&class->fullness_list[i], |
| 1205 | struct zspage, list); |
| 1206 | if (zspage) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1207 | break; |
| 1208 | } |
| 1209 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1210 | return zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1211 | } |
| 1212 | |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1213 | static inline int __zs_cpu_up(struct mapping_area *area) |
| 1214 | { |
| 1215 | /* |
| 1216 | * Make sure we don't leak memory if a cpu UP notification |
| 1217 | * and zs_init() race and both call zs_cpu_up() on the same cpu |
| 1218 | */ |
| 1219 | if (area->vm_buf) |
| 1220 | return 0; |
Mahendran Ganesh | 40f9fb8 | 2014-12-12 16:57:01 -0800 | [diff] [blame] | 1221 | area->vm_buf = kmalloc(ZS_MAX_ALLOC_SIZE, GFP_KERNEL); |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1222 | if (!area->vm_buf) |
| 1223 | return -ENOMEM; |
| 1224 | return 0; |
| 1225 | } |
| 1226 | |
| 1227 | static inline void __zs_cpu_down(struct mapping_area *area) |
| 1228 | { |
Mahendran Ganesh | 40f9fb8 | 2014-12-12 16:57:01 -0800 | [diff] [blame] | 1229 | kfree(area->vm_buf); |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1230 | area->vm_buf = NULL; |
| 1231 | } |
| 1232 | |
| 1233 | static void *__zs_map_object(struct mapping_area *area, |
| 1234 | struct page *pages[2], int off, int size) |
| 1235 | { |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1236 | int sizes[2]; |
| 1237 | void *addr; |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1238 | char *buf = area->vm_buf; |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1239 | |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1240 | /* disable page faults to match kmap_atomic() return conditions */ |
| 1241 | pagefault_disable(); |
| 1242 | |
| 1243 | /* no read fastpath */ |
| 1244 | if (area->vm_mm == ZS_MM_WO) |
| 1245 | goto out; |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1246 | |
| 1247 | sizes[0] = PAGE_SIZE - off; |
| 1248 | sizes[1] = size - sizes[0]; |
| 1249 | |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1250 | /* copy object to per-cpu buffer */ |
| 1251 | addr = kmap_atomic(pages[0]); |
| 1252 | memcpy(buf, addr + off, sizes[0]); |
| 1253 | kunmap_atomic(addr); |
| 1254 | addr = kmap_atomic(pages[1]); |
| 1255 | memcpy(buf + sizes[0], addr, sizes[1]); |
| 1256 | kunmap_atomic(addr); |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1257 | out: |
| 1258 | return area->vm_buf; |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1259 | } |
| 1260 | |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1261 | static void __zs_unmap_object(struct mapping_area *area, |
| 1262 | struct page *pages[2], int off, int size) |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1263 | { |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1264 | int sizes[2]; |
| 1265 | void *addr; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1266 | char *buf; |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1267 | |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1268 | /* no write fastpath */ |
| 1269 | if (area->vm_mm == ZS_MM_RO) |
| 1270 | goto out; |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1271 | |
Minchan Kim | 7b60a68 | 2015-04-15 16:15:39 -0700 | [diff] [blame] | 1272 | buf = area->vm_buf; |
YiPing Xu | a82cbf0 | 2016-03-17 14:20:39 -0700 | [diff] [blame] | 1273 | buf = buf + ZS_HANDLE_SIZE; |
| 1274 | size -= ZS_HANDLE_SIZE; |
| 1275 | off += ZS_HANDLE_SIZE; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1276 | |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1277 | sizes[0] = PAGE_SIZE - off; |
| 1278 | sizes[1] = size - sizes[0]; |
| 1279 | |
| 1280 | /* copy per-cpu buffer to object */ |
| 1281 | addr = kmap_atomic(pages[0]); |
| 1282 | memcpy(addr + off, buf, sizes[0]); |
| 1283 | kunmap_atomic(addr); |
| 1284 | addr = kmap_atomic(pages[1]); |
| 1285 | memcpy(addr, buf + sizes[0], sizes[1]); |
| 1286 | kunmap_atomic(addr); |
Seth Jennings | f553646 | 2012-07-18 11:55:56 -0500 | [diff] [blame] | 1287 | |
| 1288 | out: |
| 1289 | /* enable page faults to match kunmap_atomic() return conditions */ |
| 1290 | pagefault_enable(); |
Seth Jennings | 5f601902 | 2012-07-02 16:15:49 -0500 | [diff] [blame] | 1291 | } |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1292 | |
Sebastian Andrzej Siewior | 215c89d | 2016-11-27 00:13:38 +0100 | [diff] [blame] | 1293 | static int zs_cpu_prepare(unsigned int cpu) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1294 | { |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1295 | struct mapping_area *area; |
| 1296 | |
Sebastian Andrzej Siewior | 215c89d | 2016-11-27 00:13:38 +0100 | [diff] [blame] | 1297 | area = &per_cpu(zs_map_area, cpu); |
| 1298 | return __zs_cpu_up(area); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1299 | } |
| 1300 | |
Sebastian Andrzej Siewior | 215c89d | 2016-11-27 00:13:38 +0100 | [diff] [blame] | 1301 | static int zs_cpu_dead(unsigned int cpu) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1302 | { |
Sebastian Andrzej Siewior | 215c89d | 2016-11-27 00:13:38 +0100 | [diff] [blame] | 1303 | struct mapping_area *area; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 1304 | |
Sebastian Andrzej Siewior | 215c89d | 2016-11-27 00:13:38 +0100 | [diff] [blame] | 1305 | area = &per_cpu(zs_map_area, cpu); |
| 1306 | __zs_cpu_down(area); |
| 1307 | return 0; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1308 | } |
| 1309 | |
Ganesh Mahendran | 64d9046 | 2016-07-28 15:47:51 -0700 | [diff] [blame] | 1310 | static bool can_merge(struct size_class *prev, int pages_per_zspage, |
| 1311 | int objs_per_zspage) |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 1312 | { |
Ganesh Mahendran | 64d9046 | 2016-07-28 15:47:51 -0700 | [diff] [blame] | 1313 | if (prev->pages_per_zspage == pages_per_zspage && |
| 1314 | prev->objs_per_zspage == objs_per_zspage) |
| 1315 | return true; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 1316 | |
Ganesh Mahendran | 64d9046 | 2016-07-28 15:47:51 -0700 | [diff] [blame] | 1317 | return false; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 1318 | } |
| 1319 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1320 | static bool zspage_full(struct size_class *class, struct zspage *zspage) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1321 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1322 | return get_zspage_inuse(zspage) == class->objs_per_zspage; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1323 | } |
| 1324 | |
Alexey Romanov | 7c2af30 | 2022-11-09 20:50:42 +0900 | [diff] [blame] | 1325 | /** |
| 1326 | * zs_lookup_class_index() - Returns index of the zsmalloc &size_class |
| 1327 | * that hold objects of the provided size. |
| 1328 | * @pool: zsmalloc pool to use |
| 1329 | * @size: object size |
| 1330 | * |
| 1331 | * Context: Any context. |
| 1332 | * |
| 1333 | * Return: the index of the zsmalloc &size_class that hold objects of the |
| 1334 | * provided size. |
| 1335 | */ |
| 1336 | unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size) |
| 1337 | { |
| 1338 | struct size_class *class; |
| 1339 | |
| 1340 | class = pool->size_class[get_size_class_index(size)]; |
| 1341 | |
| 1342 | return class->index; |
| 1343 | } |
| 1344 | EXPORT_SYMBOL_GPL(zs_lookup_class_index); |
| 1345 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1346 | unsigned long zs_get_total_pages(struct zs_pool *pool) |
| 1347 | { |
| 1348 | return atomic_long_read(&pool->pages_allocated); |
| 1349 | } |
| 1350 | EXPORT_SYMBOL_GPL(zs_get_total_pages); |
| 1351 | |
| 1352 | /** |
| 1353 | * zs_map_object - get address of allocated object from handle. |
| 1354 | * @pool: pool from which the object was allocated |
| 1355 | * @handle: handle returned from zs_malloc |
Ingo Molnar | f0953a1 | 2021-05-06 18:06:47 -0700 | [diff] [blame] | 1356 | * @mm: mapping mode to use |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1357 | * |
| 1358 | * Before using an object allocated from zs_malloc, it must be mapped using |
| 1359 | * this function. When done with the object, it must be unmapped using |
| 1360 | * zs_unmap_object. |
| 1361 | * |
| 1362 | * Only one object can be mapped per cpu at a time. There is no protection |
| 1363 | * against nested mappings. |
| 1364 | * |
| 1365 | * This function returns with preemption and page faults disabled. |
| 1366 | */ |
| 1367 | void *zs_map_object(struct zs_pool *pool, unsigned long handle, |
| 1368 | enum zs_mapmode mm) |
| 1369 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1370 | struct zspage *zspage; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1371 | struct page *page; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1372 | unsigned long obj, off; |
| 1373 | unsigned int obj_idx; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1374 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1375 | struct size_class *class; |
| 1376 | struct mapping_area *area; |
| 1377 | struct page *pages[2]; |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1378 | void *ret; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1379 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1380 | /* |
| 1381 | * Because we use per-cpu mapping areas shared among the |
| 1382 | * pools/users, we can't allow mapping in interrupt context |
| 1383 | * because it can corrupt another users mappings. |
| 1384 | */ |
Sergey Senozhatsky | 1aedcaf | 2017-11-15 17:34:03 -0800 | [diff] [blame] | 1385 | BUG_ON(in_interrupt()); |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1386 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1387 | /* It guarantees it can get zspage from handle safely */ |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1388 | spin_lock(&pool->lock); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1389 | obj = handle_to_obj(handle); |
| 1390 | obj_to_location(obj, &page, &obj_idx); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1391 | zspage = get_zspage(page); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1392 | |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 1393 | #ifdef CONFIG_ZPOOL |
| 1394 | /* |
| 1395 | * Move the zspage to front of pool's LRU. |
| 1396 | * |
| 1397 | * Note that this is swap-specific, so by definition there are no ongoing |
| 1398 | * accesses to the memory while the page is swapped out that would make |
| 1399 | * it "hot". A new entry is hot, then ages to the tail until it gets either |
| 1400 | * written back or swaps back in. |
| 1401 | * |
| 1402 | * Furthermore, map is also called during writeback. We must not put an |
| 1403 | * isolated page on the LRU mid-reclaim. |
| 1404 | * |
| 1405 | * As a result, only update the LRU when the page is mapped for write |
| 1406 | * when it's first instantiated. |
| 1407 | * |
| 1408 | * This is a deviation from the other backends, which perform this update |
| 1409 | * in the allocation function (zbud_alloc, z3fold_alloc). |
| 1410 | */ |
| 1411 | if (mm == ZS_MM_WO) { |
| 1412 | if (!list_empty(&zspage->lru)) |
| 1413 | list_del(&zspage->lru); |
| 1414 | list_add(&zspage->lru, &pool->lru); |
| 1415 | } |
| 1416 | #endif |
| 1417 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1418 | /* |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1419 | * migration cannot move any zpages in this zspage. Here, pool->lock |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1420 | * is too heavy since callers would take some time until they calls |
| 1421 | * zs_unmap_object API so delegate the locking from class to zspage |
| 1422 | * which is smaller granularity. |
| 1423 | */ |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1424 | migrate_read_lock(zspage); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1425 | spin_unlock(&pool->lock); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1426 | |
Minchan Kim | 67f1c9cd | 2022-01-21 22:13:51 -0800 | [diff] [blame] | 1427 | class = zspage_class(pool, zspage); |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1428 | off = (class->size * obj_idx) & ~PAGE_MASK; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1429 | |
Mike Galbraith | a372659 | 2022-01-21 22:14:17 -0800 | [diff] [blame] | 1430 | local_lock(&zs_map_area.lock); |
| 1431 | area = this_cpu_ptr(&zs_map_area); |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1432 | area->vm_mm = mm; |
| 1433 | if (off + class->size <= PAGE_SIZE) { |
| 1434 | /* this object is contained entirely within a page */ |
| 1435 | area->vm_addr = kmap_atomic(page); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1436 | ret = area->vm_addr + off; |
| 1437 | goto out; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1438 | } |
| 1439 | |
| 1440 | /* this object spans two pages */ |
| 1441 | pages[0] = page; |
| 1442 | pages[1] = get_next_page(page); |
| 1443 | BUG_ON(!pages[1]); |
| 1444 | |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1445 | ret = __zs_map_object(area, pages, off, class->size); |
| 1446 | out: |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 1447 | if (likely(!ZsHugePage(zspage))) |
Minchan Kim | 7b60a68 | 2015-04-15 16:15:39 -0700 | [diff] [blame] | 1448 | ret += ZS_HANDLE_SIZE; |
| 1449 | |
| 1450 | return ret; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1451 | } |
| 1452 | EXPORT_SYMBOL_GPL(zs_map_object); |
| 1453 | |
| 1454 | void zs_unmap_object(struct zs_pool *pool, unsigned long handle) |
| 1455 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1456 | struct zspage *zspage; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1457 | struct page *page; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1458 | unsigned long obj, off; |
| 1459 | unsigned int obj_idx; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1460 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1461 | struct size_class *class; |
| 1462 | struct mapping_area *area; |
| 1463 | |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1464 | obj = handle_to_obj(handle); |
| 1465 | obj_to_location(obj, &page, &obj_idx); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1466 | zspage = get_zspage(page); |
Minchan Kim | 67f1c9cd | 2022-01-21 22:13:51 -0800 | [diff] [blame] | 1467 | class = zspage_class(pool, zspage); |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1468 | off = (class->size * obj_idx) & ~PAGE_MASK; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1469 | |
| 1470 | area = this_cpu_ptr(&zs_map_area); |
| 1471 | if (off + class->size <= PAGE_SIZE) |
| 1472 | kunmap_atomic(area->vm_addr); |
| 1473 | else { |
| 1474 | struct page *pages[2]; |
| 1475 | |
| 1476 | pages[0] = page; |
| 1477 | pages[1] = get_next_page(page); |
| 1478 | BUG_ON(!pages[1]); |
| 1479 | |
| 1480 | __zs_unmap_object(area, pages, off, class->size); |
| 1481 | } |
Mike Galbraith | a372659 | 2022-01-21 22:14:17 -0800 | [diff] [blame] | 1482 | local_unlock(&zs_map_area.lock); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1483 | |
| 1484 | migrate_read_unlock(zspage); |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1485 | } |
| 1486 | EXPORT_SYMBOL_GPL(zs_unmap_object); |
| 1487 | |
Sergey Senozhatsky | 010b495 | 2018-04-05 16:24:43 -0700 | [diff] [blame] | 1488 | /** |
| 1489 | * zs_huge_class_size() - Returns the size (in bytes) of the first huge |
| 1490 | * zsmalloc &size_class. |
| 1491 | * @pool: zsmalloc pool to use |
| 1492 | * |
| 1493 | * The function returns the size of the first huge class - any object of equal |
| 1494 | * or bigger size will be stored in zspage consisting of a single physical |
| 1495 | * page. |
| 1496 | * |
| 1497 | * Context: Any context. |
| 1498 | * |
| 1499 | * Return: the size (in bytes) of the first huge zsmalloc &size_class. |
| 1500 | */ |
| 1501 | size_t zs_huge_class_size(struct zs_pool *pool) |
| 1502 | { |
| 1503 | return huge_class_size; |
| 1504 | } |
| 1505 | EXPORT_SYMBOL_GPL(zs_huge_class_size); |
| 1506 | |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1507 | static unsigned long obj_malloc(struct zs_pool *pool, |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1508 | struct zspage *zspage, unsigned long handle) |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1509 | { |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1510 | int i, nr_page, offset; |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1511 | unsigned long obj; |
| 1512 | struct link_free *link; |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1513 | struct size_class *class; |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1514 | |
| 1515 | struct page *m_page; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1516 | unsigned long m_offset; |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1517 | void *vaddr; |
| 1518 | |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1519 | class = pool->size_class[zspage->class]; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1520 | handle |= OBJ_ALLOCATED_TAG; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1521 | obj = get_freeobj(zspage); |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1522 | |
| 1523 | offset = obj * class->size; |
| 1524 | nr_page = offset >> PAGE_SHIFT; |
| 1525 | m_offset = offset & ~PAGE_MASK; |
| 1526 | m_page = get_first_page(zspage); |
| 1527 | |
| 1528 | for (i = 0; i < nr_page; i++) |
| 1529 | m_page = get_next_page(m_page); |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1530 | |
| 1531 | vaddr = kmap_atomic(m_page); |
| 1532 | link = (struct link_free *)vaddr + m_offset / sizeof(*link); |
Minchan Kim | 3b1d9ca | 2016-07-26 15:23:37 -0700 | [diff] [blame] | 1533 | set_freeobj(zspage, link->next >> OBJ_TAG_BITS); |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 1534 | if (likely(!ZsHugePage(zspage))) |
Minchan Kim | 7b60a68 | 2015-04-15 16:15:39 -0700 | [diff] [blame] | 1535 | /* record handle in the header of allocated chunk */ |
| 1536 | link->handle = handle; |
| 1537 | else |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1538 | /* record handle to page->index */ |
| 1539 | zspage->first_page->index = handle; |
| 1540 | |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1541 | kunmap_atomic(vaddr); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1542 | mod_zspage_inuse(zspage, 1); |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1543 | |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1544 | obj = location_to_obj(m_page, obj); |
| 1545 | |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1546 | return obj; |
| 1547 | } |
| 1548 | |
| 1549 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1550 | /** |
| 1551 | * zs_malloc - Allocate block of given size from pool. |
| 1552 | * @pool: pool to allocate from |
| 1553 | * @size: size of block to allocate |
Ganesh Mahendran | fd85446 | 2016-07-28 15:47:54 -0700 | [diff] [blame] | 1554 | * @gfp: gfp flags when allocating object |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1555 | * |
| 1556 | * On success, handle to the allocated object is returned, |
Hui Zhu | c7e6f17 | 2022-07-14 16:07:57 +0800 | [diff] [blame] | 1557 | * otherwise an ERR_PTR(). |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1558 | * Allocation requests with size > ZS_MAX_ALLOC_SIZE will fail. |
| 1559 | */ |
Sergey Senozhatsky | d0d8da2 | 2016-05-20 16:59:48 -0700 | [diff] [blame] | 1560 | unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp) |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1561 | { |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1562 | unsigned long handle, obj; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1563 | struct size_class *class; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1564 | enum fullness_group newfg; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1565 | struct zspage *zspage; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1566 | |
Minchan Kim | 7b60a68 | 2015-04-15 16:15:39 -0700 | [diff] [blame] | 1567 | if (unlikely(!size || size > ZS_MAX_ALLOC_SIZE)) |
Hui Zhu | c7e6f17 | 2022-07-14 16:07:57 +0800 | [diff] [blame] | 1568 | return (unsigned long)ERR_PTR(-EINVAL); |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1569 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1570 | handle = cache_alloc_handle(pool, gfp); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1571 | if (!handle) |
Hui Zhu | c7e6f17 | 2022-07-14 16:07:57 +0800 | [diff] [blame] | 1572 | return (unsigned long)ERR_PTR(-ENOMEM); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1573 | |
| 1574 | /* extra space in chunk to keep the handle */ |
| 1575 | size += ZS_HANDLE_SIZE; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1576 | class = pool->size_class[get_size_class_index(size)]; |
| 1577 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1578 | /* pool->lock effectively protects the zpage migration */ |
| 1579 | spin_lock(&pool->lock); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1580 | zspage = find_get_zspage(class); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1581 | if (likely(zspage)) { |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1582 | obj = obj_malloc(pool, zspage, handle); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1583 | /* Now move the zspage to another fullness group, if required */ |
| 1584 | fix_fullness_group(class, zspage); |
| 1585 | record_obj(handle, obj); |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1586 | class_stat_inc(class, OBJ_USED, 1); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1587 | spin_unlock(&pool->lock); |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1588 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1589 | return handle; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1590 | } |
| 1591 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1592 | spin_unlock(&pool->lock); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1593 | |
| 1594 | zspage = alloc_zspage(pool, class, gfp); |
| 1595 | if (!zspage) { |
| 1596 | cache_free_handle(pool, handle); |
Hui Zhu | c7e6f17 | 2022-07-14 16:07:57 +0800 | [diff] [blame] | 1597 | return (unsigned long)ERR_PTR(-ENOMEM); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1598 | } |
| 1599 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1600 | spin_lock(&pool->lock); |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1601 | obj = obj_malloc(pool, zspage, handle); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1602 | newfg = get_fullness_group(class, zspage); |
| 1603 | insert_zspage(class, zspage, newfg); |
| 1604 | set_zspage_mapping(zspage, class->index, newfg); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1605 | record_obj(handle, obj); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1606 | atomic_long_add(class->pages_per_zspage, |
| 1607 | &pool->pages_allocated); |
Minchan Kim | 3828a76 | 2022-01-21 22:13:54 -0800 | [diff] [blame] | 1608 | class_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage); |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1609 | class_stat_inc(class, OBJ_USED, 1); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1610 | |
| 1611 | /* We completely set up zspage so mark them as movable */ |
| 1612 | SetZsPageMovable(pool, zspage); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1613 | spin_unlock(&pool->lock); |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1614 | |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1615 | return handle; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1616 | } |
| 1617 | EXPORT_SYMBOL_GPL(zs_malloc); |
| 1618 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1619 | static void obj_free(int class_size, unsigned long obj, unsigned long *handle) |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1620 | { |
| 1621 | struct link_free *link; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1622 | struct zspage *zspage; |
| 1623 | struct page *f_page; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1624 | unsigned long f_offset; |
| 1625 | unsigned int f_objidx; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1626 | void *vaddr; |
| 1627 | |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1628 | obj_to_location(obj, &f_page, &f_objidx); |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1629 | f_offset = (class_size * f_objidx) & ~PAGE_MASK; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1630 | zspage = get_zspage(f_page); |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1631 | |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1632 | vaddr = kmap_atomic(f_page); |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1633 | link = (struct link_free *)(vaddr + f_offset); |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1634 | |
| 1635 | if (handle) { |
| 1636 | #ifdef CONFIG_ZPOOL |
| 1637 | /* Stores the (deferred) handle in the object's header */ |
| 1638 | *handle |= OBJ_DEFERRED_HANDLE_TAG; |
| 1639 | *handle &= ~OBJ_ALLOCATED_TAG; |
| 1640 | |
| 1641 | if (likely(!ZsHugePage(zspage))) |
| 1642 | link->deferred_handle = *handle; |
| 1643 | else |
| 1644 | f_page->index = *handle; |
| 1645 | #endif |
| 1646 | } else { |
| 1647 | /* Insert this object in containing zspage's freelist */ |
| 1648 | if (likely(!ZsHugePage(zspage))) |
| 1649 | link->next = get_freeobj(zspage) << OBJ_TAG_BITS; |
| 1650 | else |
| 1651 | f_page->index = 0; |
| 1652 | set_freeobj(zspage, f_objidx); |
| 1653 | } |
| 1654 | |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1655 | kunmap_atomic(vaddr); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1656 | mod_zspage_inuse(zspage, -1); |
Minchan Kim | c780626 | 2015-04-15 16:15:26 -0700 | [diff] [blame] | 1657 | } |
| 1658 | |
| 1659 | void zs_free(struct zs_pool *pool, unsigned long handle) |
| 1660 | { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1661 | struct zspage *zspage; |
| 1662 | struct page *f_page; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1663 | unsigned long obj; |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1664 | struct size_class *class; |
| 1665 | enum fullness_group fullness; |
| 1666 | |
Sergey Senozhatsky | a5d2172 | 2022-08-16 14:09:06 +0900 | [diff] [blame] | 1667 | if (IS_ERR_OR_NULL((void *)handle)) |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1668 | return; |
| 1669 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1670 | /* |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1671 | * The pool->lock protects the race with zpage's migration |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1672 | * so it's safe to get the page from handle. |
| 1673 | */ |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1674 | spin_lock(&pool->lock); |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 1675 | obj = handle_to_obj(handle); |
Minchan Kim | 67f1c9cd | 2022-01-21 22:13:51 -0800 | [diff] [blame] | 1676 | obj_to_page(obj, &f_page); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1677 | zspage = get_zspage(f_page); |
Minchan Kim | 67f1c9cd | 2022-01-21 22:13:51 -0800 | [diff] [blame] | 1678 | class = zspage_class(pool, zspage); |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1679 | |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1680 | class_stat_dec(class, OBJ_USED, 1); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1681 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1682 | #ifdef CONFIG_ZPOOL |
| 1683 | if (zspage->under_reclaim) { |
| 1684 | /* |
| 1685 | * Reclaim needs the handles during writeback. It'll free |
| 1686 | * them along with the zspage when it's done with them. |
| 1687 | * |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1688 | * Record current deferred handle in the object's header. |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1689 | */ |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1690 | obj_free(class->size, obj, &handle); |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1691 | spin_unlock(&pool->lock); |
| 1692 | return; |
| 1693 | } |
| 1694 | #endif |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1695 | obj_free(class->size, obj, NULL); |
| 1696 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1697 | fullness = fix_fullness_group(class, zspage); |
| 1698 | if (fullness == ZS_EMPTY) |
| 1699 | free_zspage(pool, class, zspage); |
| 1700 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 1701 | spin_unlock(&pool->lock); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1702 | cache_free_handle(pool, handle); |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 1703 | } |
| 1704 | EXPORT_SYMBOL_GPL(zs_free); |
| 1705 | |
Minchan Kim | 251cbb9 | 2016-05-20 16:59:42 -0700 | [diff] [blame] | 1706 | static void zs_object_copy(struct size_class *class, unsigned long dst, |
| 1707 | unsigned long src) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1708 | { |
| 1709 | struct page *s_page, *d_page; |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1710 | unsigned int s_objidx, d_objidx; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1711 | unsigned long s_off, d_off; |
| 1712 | void *s_addr, *d_addr; |
| 1713 | int s_size, d_size, size; |
| 1714 | int written = 0; |
| 1715 | |
| 1716 | s_size = d_size = class->size; |
| 1717 | |
| 1718 | obj_to_location(src, &s_page, &s_objidx); |
| 1719 | obj_to_location(dst, &d_page, &d_objidx); |
| 1720 | |
Minchan Kim | bfd093f | 2016-07-26 15:23:28 -0700 | [diff] [blame] | 1721 | s_off = (class->size * s_objidx) & ~PAGE_MASK; |
| 1722 | d_off = (class->size * d_objidx) & ~PAGE_MASK; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1723 | |
| 1724 | if (s_off + class->size > PAGE_SIZE) |
| 1725 | s_size = PAGE_SIZE - s_off; |
| 1726 | |
| 1727 | if (d_off + class->size > PAGE_SIZE) |
| 1728 | d_size = PAGE_SIZE - d_off; |
| 1729 | |
| 1730 | s_addr = kmap_atomic(s_page); |
| 1731 | d_addr = kmap_atomic(d_page); |
| 1732 | |
| 1733 | while (1) { |
| 1734 | size = min(s_size, d_size); |
| 1735 | memcpy(d_addr + d_off, s_addr + s_off, size); |
| 1736 | written += size; |
| 1737 | |
| 1738 | if (written == class->size) |
| 1739 | break; |
| 1740 | |
Sergey Senozhatsky | 495819e | 2015-04-15 16:16:15 -0700 | [diff] [blame] | 1741 | s_off += size; |
| 1742 | s_size -= size; |
| 1743 | d_off += size; |
| 1744 | d_size -= size; |
| 1745 | |
Alexey Romanov | 050a388 | 2022-08-11 18:37:54 +0300 | [diff] [blame] | 1746 | /* |
| 1747 | * Calling kunmap_atomic(d_addr) is necessary. kunmap_atomic() |
| 1748 | * calls must occurs in reverse order of calls to kmap_atomic(). |
| 1749 | * So, to call kunmap_atomic(s_addr) we should first call |
Alexey Romanov | 46e8715 | 2022-08-15 17:48:25 +0300 | [diff] [blame] | 1750 | * kunmap_atomic(d_addr). For more details see |
| 1751 | * Documentation/mm/highmem.rst. |
Alexey Romanov | 050a388 | 2022-08-11 18:37:54 +0300 | [diff] [blame] | 1752 | */ |
Sergey Senozhatsky | 495819e | 2015-04-15 16:16:15 -0700 | [diff] [blame] | 1753 | if (s_off >= PAGE_SIZE) { |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1754 | kunmap_atomic(d_addr); |
| 1755 | kunmap_atomic(s_addr); |
| 1756 | s_page = get_next_page(s_page); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1757 | s_addr = kmap_atomic(s_page); |
| 1758 | d_addr = kmap_atomic(d_page); |
| 1759 | s_size = class->size - written; |
| 1760 | s_off = 0; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1761 | } |
| 1762 | |
Sergey Senozhatsky | 495819e | 2015-04-15 16:16:15 -0700 | [diff] [blame] | 1763 | if (d_off >= PAGE_SIZE) { |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1764 | kunmap_atomic(d_addr); |
| 1765 | d_page = get_next_page(d_page); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1766 | d_addr = kmap_atomic(d_page); |
| 1767 | d_size = class->size - written; |
| 1768 | d_off = 0; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1769 | } |
| 1770 | } |
| 1771 | |
| 1772 | kunmap_atomic(d_addr); |
| 1773 | kunmap_atomic(s_addr); |
| 1774 | } |
| 1775 | |
| 1776 | /* |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1777 | * Find object with a certain tag in zspage from index object and |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1778 | * return handle. |
| 1779 | */ |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1780 | static unsigned long find_tagged_obj(struct size_class *class, |
| 1781 | struct page *page, int *obj_idx, int tag) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1782 | { |
Alexey Romanov | 671f2fa | 2022-09-09 11:37:22 +0300 | [diff] [blame] | 1783 | unsigned int offset; |
Ganesh Mahendran | cf675acb | 2016-07-28 15:47:46 -0700 | [diff] [blame] | 1784 | int index = *obj_idx; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1785 | unsigned long handle = 0; |
| 1786 | void *addr = kmap_atomic(page); |
| 1787 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1788 | offset = get_first_obj_offset(page); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1789 | offset += class->size * index; |
| 1790 | |
| 1791 | while (offset < PAGE_SIZE) { |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1792 | if (obj_tagged(page, addr + offset, &handle, tag)) |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 1793 | break; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1794 | |
| 1795 | offset += class->size; |
| 1796 | index++; |
| 1797 | } |
| 1798 | |
| 1799 | kunmap_atomic(addr); |
Ganesh Mahendran | cf675acb | 2016-07-28 15:47:46 -0700 | [diff] [blame] | 1800 | |
| 1801 | *obj_idx = index; |
| 1802 | |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1803 | return handle; |
| 1804 | } |
| 1805 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1806 | /* |
| 1807 | * Find alloced object in zspage from index object and |
| 1808 | * return handle. |
| 1809 | */ |
| 1810 | static unsigned long find_alloced_obj(struct size_class *class, |
| 1811 | struct page *page, int *obj_idx) |
| 1812 | { |
| 1813 | return find_tagged_obj(class, page, obj_idx, OBJ_ALLOCATED_TAG); |
| 1814 | } |
| 1815 | |
| 1816 | #ifdef CONFIG_ZPOOL |
| 1817 | /* |
| 1818 | * Find object storing a deferred handle in header in zspage from index object |
| 1819 | * and return handle. |
| 1820 | */ |
| 1821 | static unsigned long find_deferred_handle_obj(struct size_class *class, |
| 1822 | struct page *page, int *obj_idx) |
| 1823 | { |
| 1824 | return find_tagged_obj(class, page, obj_idx, OBJ_DEFERRED_HANDLE_TAG); |
| 1825 | } |
| 1826 | #endif |
| 1827 | |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1828 | struct zs_compact_control { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1829 | /* Source spage for migration which could be a subpage of zspage */ |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1830 | struct page *s_page; |
| 1831 | /* Destination page for migration which should be a first page |
| 1832 | * of zspage. */ |
| 1833 | struct page *d_page; |
| 1834 | /* Starting object index within @s_page which used for live object |
| 1835 | * in the subpage. */ |
Ganesh Mahendran | 41b88e1 | 2016-07-28 15:47:43 -0700 | [diff] [blame] | 1836 | int obj_idx; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1837 | }; |
| 1838 | |
| 1839 | static int migrate_zspage(struct zs_pool *pool, struct size_class *class, |
| 1840 | struct zs_compact_control *cc) |
| 1841 | { |
| 1842 | unsigned long used_obj, free_obj; |
| 1843 | unsigned long handle; |
| 1844 | struct page *s_page = cc->s_page; |
| 1845 | struct page *d_page = cc->d_page; |
Ganesh Mahendran | 41b88e1 | 2016-07-28 15:47:43 -0700 | [diff] [blame] | 1846 | int obj_idx = cc->obj_idx; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1847 | int ret = 0; |
| 1848 | |
| 1849 | while (1) { |
Ganesh Mahendran | cf675acb | 2016-07-28 15:47:46 -0700 | [diff] [blame] | 1850 | handle = find_alloced_obj(class, s_page, &obj_idx); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1851 | if (!handle) { |
| 1852 | s_page = get_next_page(s_page); |
| 1853 | if (!s_page) |
| 1854 | break; |
Ganesh Mahendran | 41b88e1 | 2016-07-28 15:47:43 -0700 | [diff] [blame] | 1855 | obj_idx = 0; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1856 | continue; |
| 1857 | } |
| 1858 | |
| 1859 | /* Stop if there is no more space */ |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1860 | if (zspage_full(class, get_zspage(d_page))) { |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1861 | ret = -ENOMEM; |
| 1862 | break; |
| 1863 | } |
| 1864 | |
| 1865 | used_obj = handle_to_obj(handle); |
Minchan Kim | 0a5f079 | 2022-01-21 22:13:57 -0800 | [diff] [blame] | 1866 | free_obj = obj_malloc(pool, get_zspage(d_page), handle); |
Minchan Kim | 251cbb9 | 2016-05-20 16:59:42 -0700 | [diff] [blame] | 1867 | zs_object_copy(class, free_obj, used_obj); |
Ganesh Mahendran | 41b88e1 | 2016-07-28 15:47:43 -0700 | [diff] [blame] | 1868 | obj_idx++; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1869 | record_obj(handle, free_obj); |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 1870 | obj_free(class->size, used_obj, NULL); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1871 | } |
| 1872 | |
| 1873 | /* Remember last position in this iteration */ |
| 1874 | cc->s_page = s_page; |
Ganesh Mahendran | 41b88e1 | 2016-07-28 15:47:43 -0700 | [diff] [blame] | 1875 | cc->obj_idx = obj_idx; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1876 | |
| 1877 | return ret; |
| 1878 | } |
| 1879 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1880 | static struct zspage *isolate_zspage(struct size_class *class, bool source) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1881 | { |
| 1882 | int i; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1883 | struct zspage *zspage; |
| 1884 | enum fullness_group fg[2] = {ZS_ALMOST_EMPTY, ZS_ALMOST_FULL}; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1885 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1886 | if (!source) { |
| 1887 | fg[0] = ZS_ALMOST_FULL; |
| 1888 | fg[1] = ZS_ALMOST_EMPTY; |
| 1889 | } |
| 1890 | |
| 1891 | for (i = 0; i < 2; i++) { |
| 1892 | zspage = list_first_entry_or_null(&class->fullness_list[fg[i]], |
| 1893 | struct zspage, list); |
| 1894 | if (zspage) { |
| 1895 | remove_zspage(class, zspage, fg[i]); |
| 1896 | return zspage; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1897 | } |
| 1898 | } |
| 1899 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1900 | return zspage; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1901 | } |
| 1902 | |
Sergey Senozhatsky | 860c707 | 2015-09-08 15:04:38 -0700 | [diff] [blame] | 1903 | /* |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1904 | * putback_zspage - add @zspage into right class's fullness list |
Sergey Senozhatsky | 860c707 | 2015-09-08 15:04:38 -0700 | [diff] [blame] | 1905 | * @class: destination class |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1906 | * @zspage: target page |
Sergey Senozhatsky | 860c707 | 2015-09-08 15:04:38 -0700 | [diff] [blame] | 1907 | * |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1908 | * Return @zspage's fullness_group |
Sergey Senozhatsky | 860c707 | 2015-09-08 15:04:38 -0700 | [diff] [blame] | 1909 | */ |
Minchan Kim | 4aa409c | 2016-07-26 15:23:26 -0700 | [diff] [blame] | 1910 | static enum fullness_group putback_zspage(struct size_class *class, |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1911 | struct zspage *zspage) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1912 | { |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1913 | enum fullness_group fullness; |
| 1914 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 1915 | fullness = get_fullness_group(class, zspage); |
| 1916 | insert_zspage(class, zspage, fullness); |
| 1917 | set_zspage_mapping(zspage, class->index, fullness); |
Minchan Kim | 839373e | 2015-04-15 16:16:18 -0700 | [diff] [blame] | 1918 | |
Sergey Senozhatsky | 860c707 | 2015-09-08 15:04:38 -0700 | [diff] [blame] | 1919 | return fullness; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 1920 | } |
| 1921 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1922 | #if defined(CONFIG_ZPOOL) || defined(CONFIG_COMPACTION) |
Colin Ian King | 4d0a540 | 2018-08-17 15:46:50 -0700 | [diff] [blame] | 1923 | /* |
| 1924 | * To prevent zspage destroy during migration, zspage freeing should |
| 1925 | * hold locks of all pages in the zspage. |
| 1926 | */ |
| 1927 | static void lock_zspage(struct zspage *zspage) |
| 1928 | { |
Sultan Alsawaf | 2505a98 | 2022-05-13 15:11:26 -0700 | [diff] [blame] | 1929 | struct page *curr_page, *page; |
Colin Ian King | 4d0a540 | 2018-08-17 15:46:50 -0700 | [diff] [blame] | 1930 | |
Sultan Alsawaf | 2505a98 | 2022-05-13 15:11:26 -0700 | [diff] [blame] | 1931 | /* |
| 1932 | * Pages we haven't locked yet can be migrated off the list while we're |
| 1933 | * trying to lock them, so we need to be careful and only attempt to |
| 1934 | * lock each page under migrate_read_lock(). Otherwise, the page we lock |
| 1935 | * may no longer belong to the zspage. This means that we may wait for |
| 1936 | * the wrong page to unlock, so we must take a reference to the page |
| 1937 | * prior to waiting for it to unlock outside migrate_read_lock(). |
| 1938 | */ |
| 1939 | while (1) { |
| 1940 | migrate_read_lock(zspage); |
| 1941 | page = get_first_page(zspage); |
| 1942 | if (trylock_page(page)) |
| 1943 | break; |
| 1944 | get_page(page); |
| 1945 | migrate_read_unlock(zspage); |
| 1946 | wait_on_page_locked(page); |
| 1947 | put_page(page); |
| 1948 | } |
| 1949 | |
| 1950 | curr_page = page; |
| 1951 | while ((page = get_next_page(curr_page))) { |
| 1952 | if (trylock_page(page)) { |
| 1953 | curr_page = page; |
| 1954 | } else { |
| 1955 | get_page(page); |
| 1956 | migrate_read_unlock(zspage); |
| 1957 | wait_on_page_locked(page); |
| 1958 | put_page(page); |
| 1959 | migrate_read_lock(zspage); |
| 1960 | } |
| 1961 | } |
| 1962 | migrate_read_unlock(zspage); |
Colin Ian King | 4d0a540 | 2018-08-17 15:46:50 -0700 | [diff] [blame] | 1963 | } |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1964 | #endif /* defined(CONFIG_ZPOOL) || defined(CONFIG_COMPACTION) */ |
| 1965 | |
| 1966 | #ifdef CONFIG_ZPOOL |
| 1967 | /* |
| 1968 | * Unlocks all the pages of the zspage. |
| 1969 | * |
| 1970 | * pool->lock must be held before this function is called |
| 1971 | * to prevent the underlying pages from migrating. |
| 1972 | */ |
| 1973 | static void unlock_zspage(struct zspage *zspage) |
| 1974 | { |
| 1975 | struct page *page = get_first_page(zspage); |
| 1976 | |
| 1977 | do { |
| 1978 | unlock_page(page); |
| 1979 | } while ((page = get_next_page(page)) != NULL); |
| 1980 | } |
| 1981 | #endif /* CONFIG_ZPOOL */ |
Colin Ian King | 4d0a540 | 2018-08-17 15:46:50 -0700 | [diff] [blame] | 1982 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1983 | static void migrate_lock_init(struct zspage *zspage) |
| 1984 | { |
| 1985 | rwlock_init(&zspage->lock); |
| 1986 | } |
| 1987 | |
Jules Irenge | cfc451cf | 2020-04-06 20:08:21 -0700 | [diff] [blame] | 1988 | static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1989 | { |
| 1990 | read_lock(&zspage->lock); |
| 1991 | } |
| 1992 | |
Jules Irenge | 8a374cc | 2020-04-06 20:08:24 -0700 | [diff] [blame] | 1993 | static void migrate_read_unlock(struct zspage *zspage) __releases(&zspage->lock) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1994 | { |
| 1995 | read_unlock(&zspage->lock); |
| 1996 | } |
| 1997 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 1998 | #ifdef CONFIG_COMPACTION |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 1999 | static void migrate_write_lock(struct zspage *zspage) |
| 2000 | { |
| 2001 | write_lock(&zspage->lock); |
| 2002 | } |
| 2003 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2004 | static void migrate_write_lock_nested(struct zspage *zspage) |
| 2005 | { |
| 2006 | write_lock_nested(&zspage->lock, SINGLE_DEPTH_NESTING); |
| 2007 | } |
| 2008 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2009 | static void migrate_write_unlock(struct zspage *zspage) |
| 2010 | { |
| 2011 | write_unlock(&zspage->lock); |
| 2012 | } |
| 2013 | |
| 2014 | /* Number of isolated subpage for *page migration* in this zspage */ |
| 2015 | static void inc_zspage_isolation(struct zspage *zspage) |
| 2016 | { |
| 2017 | zspage->isolated++; |
| 2018 | } |
| 2019 | |
| 2020 | static void dec_zspage_isolation(struct zspage *zspage) |
| 2021 | { |
Minchan Kim | c4549b8 | 2022-01-21 22:14:07 -0800 | [diff] [blame] | 2022 | VM_BUG_ON(zspage->isolated == 0); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2023 | zspage->isolated--; |
| 2024 | } |
| 2025 | |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2026 | static const struct movable_operations zsmalloc_mops; |
| 2027 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2028 | static void replace_sub_page(struct size_class *class, struct zspage *zspage, |
| 2029 | struct page *newpage, struct page *oldpage) |
| 2030 | { |
| 2031 | struct page *page; |
| 2032 | struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE] = {NULL, }; |
| 2033 | int idx = 0; |
| 2034 | |
| 2035 | page = get_first_page(zspage); |
| 2036 | do { |
| 2037 | if (page == oldpage) |
| 2038 | pages[idx] = newpage; |
| 2039 | else |
| 2040 | pages[idx] = page; |
| 2041 | idx++; |
| 2042 | } while ((page = get_next_page(page)) != NULL); |
| 2043 | |
| 2044 | create_page_chain(class, zspage, pages); |
| 2045 | set_first_obj_offset(newpage, get_first_obj_offset(oldpage)); |
Minchan Kim | a41ec88 | 2022-01-21 22:14:04 -0800 | [diff] [blame] | 2046 | if (unlikely(ZsHugePage(zspage))) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2047 | newpage->index = oldpage->index; |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2048 | __SetPageMovable(newpage, &zsmalloc_mops); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2049 | } |
| 2050 | |
Colin Ian King | 4d0a540 | 2018-08-17 15:46:50 -0700 | [diff] [blame] | 2051 | static bool zs_page_isolate(struct page *page, isolate_mode_t mode) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2052 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2053 | struct zspage *zspage; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2054 | |
| 2055 | /* |
| 2056 | * Page is locked so zspage couldn't be destroyed. For detail, look at |
| 2057 | * lock_zspage in free_zspage. |
| 2058 | */ |
| 2059 | VM_BUG_ON_PAGE(!PageMovable(page), page); |
| 2060 | VM_BUG_ON_PAGE(PageIsolated(page), page); |
| 2061 | |
| 2062 | zspage = get_zspage(page); |
Minchan Kim | c4549b8 | 2022-01-21 22:14:07 -0800 | [diff] [blame] | 2063 | migrate_write_lock(zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2064 | inc_zspage_isolation(zspage); |
Minchan Kim | c4549b8 | 2022-01-21 22:14:07 -0800 | [diff] [blame] | 2065 | migrate_write_unlock(zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2066 | |
| 2067 | return true; |
| 2068 | } |
| 2069 | |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2070 | static int zs_page_migrate(struct page *newpage, struct page *page, |
| 2071 | enum migrate_mode mode) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2072 | { |
| 2073 | struct zs_pool *pool; |
| 2074 | struct size_class *class; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2075 | struct zspage *zspage; |
| 2076 | struct page *dummy; |
| 2077 | void *s_addr, *d_addr, *addr; |
Alexey Romanov | 671f2fa | 2022-09-09 11:37:22 +0300 | [diff] [blame] | 2078 | unsigned int offset; |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 2079 | unsigned long handle; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2080 | unsigned long old_obj, new_obj; |
| 2081 | unsigned int obj_idx; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2082 | |
Jérôme Glisse | 2916ecc | 2017-09-08 16:12:06 -0700 | [diff] [blame] | 2083 | /* |
| 2084 | * We cannot support the _NO_COPY case here, because copy needs to |
| 2085 | * happen under the zs lock, which does not work with |
| 2086 | * MIGRATE_SYNC_NO_COPY workflow. |
| 2087 | */ |
| 2088 | if (mode == MIGRATE_SYNC_NO_COPY) |
| 2089 | return -EINVAL; |
| 2090 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2091 | VM_BUG_ON_PAGE(!PageMovable(page), page); |
| 2092 | VM_BUG_ON_PAGE(!PageIsolated(page), page); |
| 2093 | |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2094 | /* The page is locked, so this pointer must remain valid */ |
| 2095 | zspage = get_zspage(page); |
| 2096 | pool = zspage->pool; |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2097 | |
| 2098 | /* |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2099 | * The pool's lock protects the race between zpage migration |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2100 | * and zs_free. |
| 2101 | */ |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2102 | spin_lock(&pool->lock); |
Minchan Kim | 67f1c9cd | 2022-01-21 22:13:51 -0800 | [diff] [blame] | 2103 | class = zspage_class(pool, zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2104 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2105 | /* the migrate_write_lock protects zpage access via zs_map_object */ |
| 2106 | migrate_write_lock(zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2107 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2108 | offset = get_first_obj_offset(page); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2109 | s_addr = kmap_atomic(page); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2110 | |
| 2111 | /* |
| 2112 | * Here, any user cannot access all objects in the zspage so let's move. |
| 2113 | */ |
| 2114 | d_addr = kmap_atomic(newpage); |
| 2115 | memcpy(d_addr, s_addr, PAGE_SIZE); |
| 2116 | kunmap_atomic(d_addr); |
| 2117 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2118 | for (addr = s_addr + offset; addr < s_addr + PAGE_SIZE; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2119 | addr += class->size) { |
Minchan Kim | 3ae92ac | 2022-01-21 22:14:01 -0800 | [diff] [blame] | 2120 | if (obj_allocated(page, addr, &handle)) { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2121 | |
| 2122 | old_obj = handle_to_obj(handle); |
| 2123 | obj_to_location(old_obj, &dummy, &obj_idx); |
| 2124 | new_obj = (unsigned long)location_to_obj(newpage, |
| 2125 | obj_idx); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2126 | record_obj(handle, new_obj); |
| 2127 | } |
| 2128 | } |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2129 | kunmap_atomic(s_addr); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2130 | |
| 2131 | replace_sub_page(class, zspage, newpage, page); |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2132 | /* |
| 2133 | * Since we complete the data copy and set up new zspage structure, |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2134 | * it's okay to release the pool's lock. |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2135 | */ |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2136 | spin_unlock(&pool->lock); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2137 | dec_zspage_isolation(zspage); |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2138 | migrate_write_unlock(zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2139 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2140 | get_page(newpage); |
Chanho Min | ac8f05d | 2020-01-04 12:59:36 -0800 | [diff] [blame] | 2141 | if (page_zone(newpage) != page_zone(page)) { |
| 2142 | dec_zone_page_state(page, NR_ZSPAGES); |
| 2143 | inc_zone_page_state(newpage, NR_ZSPAGES); |
| 2144 | } |
| 2145 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2146 | reset_page(page); |
| 2147 | put_page(page); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2148 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2149 | return MIGRATEPAGE_SUCCESS; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2150 | } |
| 2151 | |
Colin Ian King | 4d0a540 | 2018-08-17 15:46:50 -0700 | [diff] [blame] | 2152 | static void zs_page_putback(struct page *page) |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2153 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2154 | struct zspage *zspage; |
| 2155 | |
| 2156 | VM_BUG_ON_PAGE(!PageMovable(page), page); |
| 2157 | VM_BUG_ON_PAGE(!PageIsolated(page), page); |
| 2158 | |
| 2159 | zspage = get_zspage(page); |
Minchan Kim | c4549b8 | 2022-01-21 22:14:07 -0800 | [diff] [blame] | 2160 | migrate_write_lock(zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2161 | dec_zspage_isolation(zspage); |
Minchan Kim | c4549b8 | 2022-01-21 22:14:07 -0800 | [diff] [blame] | 2162 | migrate_write_unlock(zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2163 | } |
| 2164 | |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2165 | static const struct movable_operations zsmalloc_mops = { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2166 | .isolate_page = zs_page_isolate, |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2167 | .migrate_page = zs_page_migrate, |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2168 | .putback_page = zs_page_putback, |
| 2169 | }; |
| 2170 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2171 | /* |
| 2172 | * Caller should hold page_lock of all pages in the zspage |
| 2173 | * In here, we cannot use zspage meta data. |
| 2174 | */ |
| 2175 | static void async_free_zspage(struct work_struct *work) |
| 2176 | { |
| 2177 | int i; |
| 2178 | struct size_class *class; |
| 2179 | unsigned int class_idx; |
| 2180 | enum fullness_group fullness; |
| 2181 | struct zspage *zspage, *tmp; |
| 2182 | LIST_HEAD(free_pages); |
| 2183 | struct zs_pool *pool = container_of(work, struct zs_pool, |
| 2184 | free_work); |
| 2185 | |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 2186 | for (i = 0; i < ZS_SIZE_CLASSES; i++) { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2187 | class = pool->size_class[i]; |
| 2188 | if (class->index != i) |
| 2189 | continue; |
| 2190 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2191 | spin_lock(&pool->lock); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2192 | list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2193 | spin_unlock(&pool->lock); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2194 | } |
| 2195 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2196 | list_for_each_entry_safe(zspage, tmp, &free_pages, list) { |
| 2197 | list_del(&zspage->list); |
| 2198 | lock_zspage(zspage); |
| 2199 | |
| 2200 | get_zspage_mapping(zspage, &class_idx, &fullness); |
| 2201 | VM_BUG_ON(fullness != ZS_EMPTY); |
| 2202 | class = pool->size_class[class_idx]; |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2203 | spin_lock(&pool->lock); |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 2204 | #ifdef CONFIG_ZPOOL |
| 2205 | list_del(&zspage->lru); |
| 2206 | #endif |
Miaohe Lin | 3384833 | 2021-06-30 18:53:04 -0700 | [diff] [blame] | 2207 | __free_zspage(pool, class, zspage); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2208 | spin_unlock(&pool->lock); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2209 | } |
| 2210 | }; |
| 2211 | |
| 2212 | static void kick_deferred_free(struct zs_pool *pool) |
| 2213 | { |
| 2214 | schedule_work(&pool->free_work); |
| 2215 | } |
| 2216 | |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2217 | static void zs_flush_migration(struct zs_pool *pool) |
| 2218 | { |
| 2219 | flush_work(&pool->free_work); |
| 2220 | } |
| 2221 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2222 | static void init_deferred_free(struct zs_pool *pool) |
| 2223 | { |
| 2224 | INIT_WORK(&pool->free_work, async_free_zspage); |
| 2225 | } |
| 2226 | |
| 2227 | static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) |
| 2228 | { |
| 2229 | struct page *page = get_first_page(zspage); |
| 2230 | |
| 2231 | do { |
| 2232 | WARN_ON(!trylock_page(page)); |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2233 | __SetPageMovable(page, &zsmalloc_mops); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2234 | unlock_page(page); |
| 2235 | } while ((page = get_next_page(page)) != NULL); |
| 2236 | } |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2237 | #else |
| 2238 | static inline void zs_flush_migration(struct zs_pool *pool) { } |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2239 | #endif |
| 2240 | |
Sergey Senozhatsky | 04f0590 | 2015-09-08 15:04:30 -0700 | [diff] [blame] | 2241 | /* |
| 2242 | * |
| 2243 | * Based on the number of unused allocated objects calculate |
| 2244 | * and return the number of pages that we can free. |
Sergey Senozhatsky | 04f0590 | 2015-09-08 15:04:30 -0700 | [diff] [blame] | 2245 | */ |
| 2246 | static unsigned long zs_can_compact(struct size_class *class) |
| 2247 | { |
| 2248 | unsigned long obj_wasted; |
Sergey Senozhatsky | 44f43e9 | 2016-05-09 16:28:49 -0700 | [diff] [blame] | 2249 | unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED); |
| 2250 | unsigned long obj_used = zs_stat_get(class, OBJ_USED); |
Sergey Senozhatsky | 04f0590 | 2015-09-08 15:04:30 -0700 | [diff] [blame] | 2251 | |
Sergey Senozhatsky | 44f43e9 | 2016-05-09 16:28:49 -0700 | [diff] [blame] | 2252 | if (obj_allocated <= obj_used) |
| 2253 | return 0; |
Sergey Senozhatsky | 04f0590 | 2015-09-08 15:04:30 -0700 | [diff] [blame] | 2254 | |
Sergey Senozhatsky | 44f43e9 | 2016-05-09 16:28:49 -0700 | [diff] [blame] | 2255 | obj_wasted = obj_allocated - obj_used; |
Ganesh Mahendran | b4fd07a | 2016-07-28 15:47:49 -0700 | [diff] [blame] | 2256 | obj_wasted /= class->objs_per_zspage; |
Sergey Senozhatsky | 04f0590 | 2015-09-08 15:04:30 -0700 | [diff] [blame] | 2257 | |
Minchan Kim | 6cbf16b | 2015-09-08 15:04:49 -0700 | [diff] [blame] | 2258 | return obj_wasted * class->pages_per_zspage; |
Sergey Senozhatsky | 04f0590 | 2015-09-08 15:04:30 -0700 | [diff] [blame] | 2259 | } |
| 2260 | |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2261 | static unsigned long __zs_compact(struct zs_pool *pool, |
| 2262 | struct size_class *class) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2263 | { |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2264 | struct zs_compact_control cc; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2265 | struct zspage *src_zspage; |
| 2266 | struct zspage *dst_zspage = NULL; |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2267 | unsigned long pages_freed = 0; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2268 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2269 | /* |
| 2270 | * protect the race between zpage migration and zs_free |
| 2271 | * as well as zpage allocation/free |
| 2272 | */ |
| 2273 | spin_lock(&pool->lock); |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2274 | while ((src_zspage = isolate_zspage(class, true))) { |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2275 | /* protect someone accessing the zspage(i.e., zs_map_object) */ |
| 2276 | migrate_write_lock(src_zspage); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2277 | |
Sergey Senozhatsky | 04f0590 | 2015-09-08 15:04:30 -0700 | [diff] [blame] | 2278 | if (!zs_can_compact(class)) |
| 2279 | break; |
| 2280 | |
Ganesh Mahendran | 41b88e1 | 2016-07-28 15:47:43 -0700 | [diff] [blame] | 2281 | cc.obj_idx = 0; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2282 | cc.s_page = get_first_page(src_zspage); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2283 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2284 | while ((dst_zspage = isolate_zspage(class, false))) { |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2285 | migrate_write_lock_nested(dst_zspage); |
| 2286 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2287 | cc.d_page = get_first_page(dst_zspage); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2288 | /* |
Sergey Senozhatsky | 0dc63d48 | 2015-09-08 15:04:33 -0700 | [diff] [blame] | 2289 | * If there is no more space in dst_page, resched |
| 2290 | * and see if anyone had allocated another zspage. |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2291 | */ |
| 2292 | if (!migrate_zspage(pool, class, &cc)) |
| 2293 | break; |
| 2294 | |
Minchan Kim | 4aa409c | 2016-07-26 15:23:26 -0700 | [diff] [blame] | 2295 | putback_zspage(class, dst_zspage); |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2296 | migrate_write_unlock(dst_zspage); |
| 2297 | dst_zspage = NULL; |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2298 | if (spin_is_contended(&pool->lock)) |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2299 | break; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2300 | } |
| 2301 | |
| 2302 | /* Stop if we couldn't find slot */ |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2303 | if (dst_zspage == NULL) |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2304 | break; |
| 2305 | |
Minchan Kim | 4aa409c | 2016-07-26 15:23:26 -0700 | [diff] [blame] | 2306 | putback_zspage(class, dst_zspage); |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2307 | migrate_write_unlock(dst_zspage); |
| 2308 | |
Minchan Kim | 4aa409c | 2016-07-26 15:23:26 -0700 | [diff] [blame] | 2309 | if (putback_zspage(class, src_zspage) == ZS_EMPTY) { |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2310 | migrate_write_unlock(src_zspage); |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2311 | free_zspage(pool, class, src_zspage); |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2312 | pages_freed += class->pages_per_zspage; |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2313 | } else |
| 2314 | migrate_write_unlock(src_zspage); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2315 | spin_unlock(&pool->lock); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2316 | cond_resched(); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2317 | spin_lock(&pool->lock); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2318 | } |
| 2319 | |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2320 | if (src_zspage) { |
Minchan Kim | 4aa409c | 2016-07-26 15:23:26 -0700 | [diff] [blame] | 2321 | putback_zspage(class, src_zspage); |
Minchan Kim | b475d42 | 2022-01-21 22:14:13 -0800 | [diff] [blame] | 2322 | migrate_write_unlock(src_zspage); |
| 2323 | } |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2324 | |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2325 | spin_unlock(&pool->lock); |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2326 | |
| 2327 | return pages_freed; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2328 | } |
| 2329 | |
| 2330 | unsigned long zs_compact(struct zs_pool *pool) |
| 2331 | { |
| 2332 | int i; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2333 | struct size_class *class; |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2334 | unsigned long pages_freed = 0; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2335 | |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 2336 | for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) { |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2337 | class = pool->size_class[i]; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2338 | if (class->index != i) |
| 2339 | continue; |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2340 | pages_freed += __zs_compact(pool, class); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2341 | } |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2342 | atomic_long_add(pages_freed, &pool->stats.pages_compacted); |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2343 | |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2344 | return pages_freed; |
Minchan Kim | 312fcae | 2015-04-15 16:15:30 -0700 | [diff] [blame] | 2345 | } |
| 2346 | EXPORT_SYMBOL_GPL(zs_compact); |
| 2347 | |
Sergey Senozhatsky | 7d3f393 | 2015-09-08 15:04:35 -0700 | [diff] [blame] | 2348 | void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats) |
| 2349 | { |
| 2350 | memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats)); |
| 2351 | } |
| 2352 | EXPORT_SYMBOL_GPL(zs_pool_stats); |
| 2353 | |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2354 | static unsigned long zs_shrinker_scan(struct shrinker *shrinker, |
| 2355 | struct shrink_control *sc) |
| 2356 | { |
| 2357 | unsigned long pages_freed; |
| 2358 | struct zs_pool *pool = container_of(shrinker, struct zs_pool, |
| 2359 | shrinker); |
| 2360 | |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2361 | /* |
| 2362 | * Compact classes and calculate compaction delta. |
| 2363 | * Can run concurrently with a manually triggered |
| 2364 | * (by user) compaction. |
| 2365 | */ |
Rokudo Yan | 2395928 | 2021-02-25 17:18:31 -0800 | [diff] [blame] | 2366 | pages_freed = zs_compact(pool); |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2367 | |
| 2368 | return pages_freed ? pages_freed : SHRINK_STOP; |
| 2369 | } |
| 2370 | |
| 2371 | static unsigned long zs_shrinker_count(struct shrinker *shrinker, |
| 2372 | struct shrink_control *sc) |
| 2373 | { |
| 2374 | int i; |
| 2375 | struct size_class *class; |
| 2376 | unsigned long pages_to_free = 0; |
| 2377 | struct zs_pool *pool = container_of(shrinker, struct zs_pool, |
| 2378 | shrinker); |
| 2379 | |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 2380 | for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) { |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2381 | class = pool->size_class[i]; |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2382 | if (class->index != i) |
| 2383 | continue; |
| 2384 | |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2385 | pages_to_free += zs_can_compact(class); |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2386 | } |
| 2387 | |
| 2388 | return pages_to_free; |
| 2389 | } |
| 2390 | |
| 2391 | static void zs_unregister_shrinker(struct zs_pool *pool) |
| 2392 | { |
Aliaksei Karaliou | 93144ca | 2018-01-31 16:18:40 -0800 | [diff] [blame] | 2393 | unregister_shrinker(&pool->shrinker); |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2394 | } |
| 2395 | |
| 2396 | static int zs_register_shrinker(struct zs_pool *pool) |
| 2397 | { |
| 2398 | pool->shrinker.scan_objects = zs_shrinker_scan; |
| 2399 | pool->shrinker.count_objects = zs_shrinker_count; |
| 2400 | pool->shrinker.batch = 0; |
| 2401 | pool->shrinker.seeks = DEFAULT_SEEKS; |
| 2402 | |
Roman Gushchin | e33c267 | 2022-05-31 20:22:24 -0700 | [diff] [blame] | 2403 | return register_shrinker(&pool->shrinker, "mm-zspool:%s", |
| 2404 | pool->name); |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2405 | } |
| 2406 | |
Davidlohr Bueso | 4bbc0bc | 2013-01-04 12:14:00 -0800 | [diff] [blame] | 2407 | /** |
| 2408 | * zs_create_pool - Creates an allocation pool to work from. |
Ganesh Mahendran | fd85446 | 2016-07-28 15:47:54 -0700 | [diff] [blame] | 2409 | * @name: pool name to be created |
Davidlohr Bueso | 4bbc0bc | 2013-01-04 12:14:00 -0800 | [diff] [blame] | 2410 | * |
| 2411 | * This function must be called before anything when using |
| 2412 | * the zsmalloc allocator. |
| 2413 | * |
| 2414 | * On success, a pointer to the newly created pool is returned, |
| 2415 | * otherwise NULL. |
| 2416 | */ |
Sergey Senozhatsky | d0d8da2 | 2016-05-20 16:59:48 -0700 | [diff] [blame] | 2417 | struct zs_pool *zs_create_pool(const char *name) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2418 | { |
Ganesh Mahendran | 1813665 | 2014-12-12 16:57:10 -0800 | [diff] [blame] | 2419 | int i; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2420 | struct zs_pool *pool; |
Ganesh Mahendran | df8b5bb | 2014-12-12 16:57:07 -0800 | [diff] [blame] | 2421 | struct size_class *prev_class = NULL; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2422 | |
Ganesh Mahendran | 1813665 | 2014-12-12 16:57:10 -0800 | [diff] [blame] | 2423 | pool = kzalloc(sizeof(*pool), GFP_KERNEL); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2424 | if (!pool) |
| 2425 | return NULL; |
| 2426 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2427 | init_deferred_free(pool); |
Nhat Pham | c0547d0 | 2022-11-28 11:16:12 -0800 | [diff] [blame] | 2428 | spin_lock_init(&pool->lock); |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2429 | |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 2430 | pool->name = kstrdup(name, GFP_KERNEL); |
| 2431 | if (!pool->name) |
| 2432 | goto err; |
| 2433 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2434 | if (create_cache(pool)) |
Minchan Kim | 2e40e16 | 2015-04-15 16:15:23 -0700 | [diff] [blame] | 2435 | goto err; |
Mahendran Ganesh | 40f9fb8 | 2014-12-12 16:57:01 -0800 | [diff] [blame] | 2436 | |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2437 | /* |
Xishi Qiu | 399d8ee | 2017-02-22 15:45:01 -0800 | [diff] [blame] | 2438 | * Iterate reversely, because, size of size_class that we want to use |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2439 | * for merging should be larger or equal to current size. |
| 2440 | */ |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 2441 | for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) { |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2442 | int size; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2443 | int pages_per_zspage; |
Ganesh Mahendran | 64d9046 | 2016-07-28 15:47:51 -0700 | [diff] [blame] | 2444 | int objs_per_zspage; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2445 | struct size_class *class; |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2446 | int fullness = 0; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2447 | |
| 2448 | size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA; |
| 2449 | if (size > ZS_MAX_ALLOC_SIZE) |
| 2450 | size = ZS_MAX_ALLOC_SIZE; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2451 | pages_per_zspage = get_pages_per_zspage(size); |
Ganesh Mahendran | 64d9046 | 2016-07-28 15:47:51 -0700 | [diff] [blame] | 2452 | objs_per_zspage = pages_per_zspage * PAGE_SIZE / size; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2453 | |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2454 | /* |
Sergey Senozhatsky | 010b495 | 2018-04-05 16:24:43 -0700 | [diff] [blame] | 2455 | * We iterate from biggest down to smallest classes, |
| 2456 | * so huge_class_size holds the size of the first huge |
| 2457 | * class. Any object bigger than or equal to that will |
| 2458 | * endup in the huge class. |
| 2459 | */ |
| 2460 | if (pages_per_zspage != 1 && objs_per_zspage != 1 && |
| 2461 | !huge_class_size) { |
| 2462 | huge_class_size = size; |
| 2463 | /* |
| 2464 | * The object uses ZS_HANDLE_SIZE bytes to store the |
| 2465 | * handle. We need to subtract it, because zs_malloc() |
| 2466 | * unconditionally adds handle size before it performs |
| 2467 | * size class search - so object may be smaller than |
| 2468 | * huge class size, yet it still can end up in the huge |
| 2469 | * class because it grows by ZS_HANDLE_SIZE extra bytes |
| 2470 | * right before class lookup. |
| 2471 | */ |
| 2472 | huge_class_size -= (ZS_HANDLE_SIZE - 1); |
| 2473 | } |
| 2474 | |
| 2475 | /* |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2476 | * size_class is used for normal zsmalloc operation such |
| 2477 | * as alloc/free for that size. Although it is natural that we |
| 2478 | * have one size_class for each size, there is a chance that we |
| 2479 | * can get more memory utilization if we use one size_class for |
| 2480 | * many different sizes whose size_class have same |
| 2481 | * characteristics. So, we makes size_class point to |
| 2482 | * previous size_class if possible. |
| 2483 | */ |
Ganesh Mahendran | df8b5bb | 2014-12-12 16:57:07 -0800 | [diff] [blame] | 2484 | if (prev_class) { |
Ganesh Mahendran | 64d9046 | 2016-07-28 15:47:51 -0700 | [diff] [blame] | 2485 | if (can_merge(prev_class, pages_per_zspage, objs_per_zspage)) { |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2486 | pool->size_class[i] = prev_class; |
| 2487 | continue; |
| 2488 | } |
| 2489 | } |
| 2490 | |
| 2491 | class = kzalloc(sizeof(struct size_class), GFP_KERNEL); |
| 2492 | if (!class) |
| 2493 | goto err; |
| 2494 | |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2495 | class->size = size; |
| 2496 | class->index = i; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2497 | class->pages_per_zspage = pages_per_zspage; |
Ganesh Mahendran | 64d9046 | 2016-07-28 15:47:51 -0700 | [diff] [blame] | 2498 | class->objs_per_zspage = objs_per_zspage; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2499 | pool->size_class[i] = class; |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2500 | for (fullness = ZS_EMPTY; fullness < NR_ZS_FULLNESS; |
| 2501 | fullness++) |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2502 | INIT_LIST_HEAD(&class->fullness_list[fullness]); |
Ganesh Mahendran | df8b5bb | 2014-12-12 16:57:07 -0800 | [diff] [blame] | 2503 | |
| 2504 | prev_class = class; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2505 | } |
| 2506 | |
Dan Streetman | d34f615 | 2016-05-20 16:59:56 -0700 | [diff] [blame] | 2507 | /* debug only, don't abort if it fails */ |
| 2508 | zs_pool_stat_create(pool, name); |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2509 | |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2510 | /* |
Aliaksei Karaliou | 93144ca | 2018-01-31 16:18:40 -0800 | [diff] [blame] | 2511 | * Not critical since shrinker is only used to trigger internal |
| 2512 | * defragmentation of the pool which is pretty optional thing. If |
| 2513 | * registration fails we still can use the pool normally and user can |
| 2514 | * trigger compaction manually. Thus, ignore return code. |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2515 | */ |
Aliaksei Karaliou | 93144ca | 2018-01-31 16:18:40 -0800 | [diff] [blame] | 2516 | zs_register_shrinker(pool); |
| 2517 | |
Nhat Pham | 64f768c | 2022-11-28 11:16:13 -0800 | [diff] [blame] | 2518 | #ifdef CONFIG_ZPOOL |
| 2519 | INIT_LIST_HEAD(&pool->lru); |
| 2520 | #endif |
| 2521 | |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2522 | return pool; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2523 | |
| 2524 | err: |
| 2525 | zs_destroy_pool(pool); |
| 2526 | return NULL; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2527 | } |
| 2528 | EXPORT_SYMBOL_GPL(zs_create_pool); |
| 2529 | |
| 2530 | void zs_destroy_pool(struct zs_pool *pool) |
| 2531 | { |
| 2532 | int i; |
| 2533 | |
Sergey Senozhatsky | ab9d306 | 2015-09-08 15:04:41 -0700 | [diff] [blame] | 2534 | zs_unregister_shrinker(pool); |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2535 | zs_flush_migration(pool); |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2536 | zs_pool_stat_destroy(pool); |
| 2537 | |
Jerome Marchand | cf8e0fed | 2017-07-10 15:50:18 -0700 | [diff] [blame] | 2538 | for (i = 0; i < ZS_SIZE_CLASSES; i++) { |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2539 | int fg; |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2540 | struct size_class *class = pool->size_class[i]; |
| 2541 | |
Alexey Romanov | 4249a05 | 2022-10-13 14:28:25 +0300 | [diff] [blame] | 2542 | if (!class) |
| 2543 | continue; |
| 2544 | |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2545 | if (class->index != i) |
| 2546 | continue; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2547 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2548 | for (fg = ZS_EMPTY; fg < NR_ZS_FULLNESS; fg++) { |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2549 | if (!list_empty(&class->fullness_list[fg])) { |
Marlies Ruck | 93ad5ab | 2013-05-15 16:56:49 -0400 | [diff] [blame] | 2550 | pr_info("Freeing non-empty class with size %db, fullness group %d\n", |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2551 | class->size, fg); |
| 2552 | } |
| 2553 | } |
Joonsoo Kim | 9eec4cd | 2014-12-12 16:56:44 -0800 | [diff] [blame] | 2554 | kfree(class); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2555 | } |
Mahendran Ganesh | 40f9fb8 | 2014-12-12 16:57:01 -0800 | [diff] [blame] | 2556 | |
Minchan Kim | 3783689 | 2016-07-26 15:23:23 -0700 | [diff] [blame] | 2557 | destroy_cache(pool); |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2558 | kfree(pool->name); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2559 | kfree(pool); |
| 2560 | } |
| 2561 | EXPORT_SYMBOL_GPL(zs_destroy_pool); |
| 2562 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 2563 | #ifdef CONFIG_ZPOOL |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 2564 | static void restore_freelist(struct zs_pool *pool, struct size_class *class, |
| 2565 | struct zspage *zspage) |
| 2566 | { |
| 2567 | unsigned int obj_idx = 0; |
| 2568 | unsigned long handle, off = 0; /* off is within-page offset */ |
| 2569 | struct page *page = get_first_page(zspage); |
| 2570 | struct link_free *prev_free = NULL; |
| 2571 | void *prev_page_vaddr = NULL; |
| 2572 | |
| 2573 | /* in case no free object found */ |
| 2574 | set_freeobj(zspage, (unsigned int)(-1UL)); |
| 2575 | |
| 2576 | while (page) { |
| 2577 | void *vaddr = kmap_atomic(page); |
| 2578 | struct page *next_page; |
| 2579 | |
| 2580 | while (off < PAGE_SIZE) { |
| 2581 | void *obj_addr = vaddr + off; |
| 2582 | |
| 2583 | /* skip allocated object */ |
| 2584 | if (obj_allocated(page, obj_addr, &handle)) { |
| 2585 | obj_idx++; |
| 2586 | off += class->size; |
| 2587 | continue; |
| 2588 | } |
| 2589 | |
| 2590 | /* free deferred handle from reclaim attempt */ |
| 2591 | if (obj_stores_deferred_handle(page, obj_addr, &handle)) |
| 2592 | cache_free_handle(pool, handle); |
| 2593 | |
| 2594 | if (prev_free) |
| 2595 | prev_free->next = obj_idx << OBJ_TAG_BITS; |
| 2596 | else /* first free object found */ |
| 2597 | set_freeobj(zspage, obj_idx); |
| 2598 | |
| 2599 | prev_free = (struct link_free *)vaddr + off / sizeof(*prev_free); |
| 2600 | /* if last free object in a previous page, need to unmap */ |
| 2601 | if (prev_page_vaddr) { |
| 2602 | kunmap_atomic(prev_page_vaddr); |
| 2603 | prev_page_vaddr = NULL; |
| 2604 | } |
| 2605 | |
| 2606 | obj_idx++; |
| 2607 | off += class->size; |
| 2608 | } |
| 2609 | |
| 2610 | /* |
| 2611 | * Handle the last (full or partial) object on this page. |
| 2612 | */ |
| 2613 | next_page = get_next_page(page); |
| 2614 | if (next_page) { |
| 2615 | if (!prev_free || prev_page_vaddr) { |
| 2616 | /* |
| 2617 | * There is no free object in this page, so we can safely |
| 2618 | * unmap it. |
| 2619 | */ |
| 2620 | kunmap_atomic(vaddr); |
| 2621 | } else { |
| 2622 | /* update prev_page_vaddr since prev_free is on this page */ |
| 2623 | prev_page_vaddr = vaddr; |
| 2624 | } |
| 2625 | } else { /* this is the last page */ |
| 2626 | if (prev_free) { |
| 2627 | /* |
| 2628 | * Reset OBJ_TAG_BITS bit to last link to tell |
| 2629 | * whether it's allocated object or not. |
| 2630 | */ |
| 2631 | prev_free->next = -1UL << OBJ_TAG_BITS; |
| 2632 | } |
| 2633 | |
| 2634 | /* unmap previous page (if not done yet) */ |
| 2635 | if (prev_page_vaddr) { |
| 2636 | kunmap_atomic(prev_page_vaddr); |
| 2637 | prev_page_vaddr = NULL; |
| 2638 | } |
| 2639 | |
| 2640 | kunmap_atomic(vaddr); |
| 2641 | } |
| 2642 | |
| 2643 | page = next_page; |
| 2644 | off %= PAGE_SIZE; |
| 2645 | } |
| 2646 | } |
| 2647 | |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 2648 | static int zs_reclaim_page(struct zs_pool *pool, unsigned int retries) |
| 2649 | { |
| 2650 | int i, obj_idx, ret = 0; |
| 2651 | unsigned long handle; |
| 2652 | struct zspage *zspage; |
| 2653 | struct page *page; |
| 2654 | enum fullness_group fullness; |
| 2655 | |
| 2656 | /* Lock LRU and fullness list */ |
| 2657 | spin_lock(&pool->lock); |
| 2658 | if (list_empty(&pool->lru)) { |
| 2659 | spin_unlock(&pool->lock); |
| 2660 | return -EINVAL; |
| 2661 | } |
| 2662 | |
| 2663 | for (i = 0; i < retries; i++) { |
| 2664 | struct size_class *class; |
| 2665 | |
| 2666 | zspage = list_last_entry(&pool->lru, struct zspage, lru); |
| 2667 | list_del(&zspage->lru); |
| 2668 | |
| 2669 | /* zs_free may free objects, but not the zspage and handles */ |
| 2670 | zspage->under_reclaim = true; |
| 2671 | |
| 2672 | class = zspage_class(pool, zspage); |
| 2673 | fullness = get_fullness_group(class, zspage); |
| 2674 | |
| 2675 | /* Lock out object allocations and object compaction */ |
| 2676 | remove_zspage(class, zspage, fullness); |
| 2677 | |
| 2678 | spin_unlock(&pool->lock); |
| 2679 | cond_resched(); |
| 2680 | |
| 2681 | /* Lock backing pages into place */ |
| 2682 | lock_zspage(zspage); |
| 2683 | |
| 2684 | obj_idx = 0; |
| 2685 | page = get_first_page(zspage); |
| 2686 | while (1) { |
| 2687 | handle = find_alloced_obj(class, page, &obj_idx); |
| 2688 | if (!handle) { |
| 2689 | page = get_next_page(page); |
| 2690 | if (!page) |
| 2691 | break; |
| 2692 | obj_idx = 0; |
| 2693 | continue; |
| 2694 | } |
| 2695 | |
| 2696 | /* |
| 2697 | * This will write the object and call zs_free. |
| 2698 | * |
| 2699 | * zs_free will free the object, but the |
| 2700 | * under_reclaim flag prevents it from freeing |
| 2701 | * the zspage altogether. This is necessary so |
| 2702 | * that we can continue working with the |
| 2703 | * zspage potentially after the last object |
| 2704 | * has been freed. |
| 2705 | */ |
| 2706 | ret = pool->zpool_ops->evict(pool->zpool, handle); |
| 2707 | if (ret) |
| 2708 | goto next; |
| 2709 | |
| 2710 | obj_idx++; |
| 2711 | } |
| 2712 | |
| 2713 | next: |
| 2714 | /* For freeing the zspage, or putting it back in the pool and LRU list. */ |
| 2715 | spin_lock(&pool->lock); |
| 2716 | zspage->under_reclaim = false; |
| 2717 | |
| 2718 | if (!get_zspage_inuse(zspage)) { |
| 2719 | /* |
| 2720 | * Fullness went stale as zs_free() won't touch it |
| 2721 | * while the page is removed from the pool. Fix it |
| 2722 | * up for the check in __free_zspage(). |
| 2723 | */ |
| 2724 | zspage->fullness = ZS_EMPTY; |
| 2725 | |
| 2726 | __free_zspage(pool, class, zspage); |
| 2727 | spin_unlock(&pool->lock); |
| 2728 | return 0; |
| 2729 | } |
| 2730 | |
Nhat Pham | 85b3258 | 2023-01-10 15:17:01 -0800 | [diff] [blame] | 2731 | /* |
| 2732 | * Eviction fails on one of the handles, so we need to restore zspage. |
| 2733 | * We need to rebuild its freelist (and free stored deferred handles), |
| 2734 | * put it back to the correct size class, and add it to the LRU list. |
| 2735 | */ |
| 2736 | restore_freelist(pool, class, zspage); |
Nhat Pham | 9997bc0 | 2022-11-28 11:16:15 -0800 | [diff] [blame] | 2737 | putback_zspage(class, zspage); |
| 2738 | list_add(&zspage->lru, &pool->lru); |
| 2739 | unlock_zspage(zspage); |
| 2740 | } |
| 2741 | |
| 2742 | spin_unlock(&pool->lock); |
| 2743 | return -EAGAIN; |
| 2744 | } |
| 2745 | #endif /* CONFIG_ZPOOL */ |
| 2746 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 2747 | static int __init zs_init(void) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2748 | { |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2749 | int ret; |
| 2750 | |
Sebastian Andrzej Siewior | 215c89d | 2016-11-27 00:13:38 +0100 | [diff] [blame] | 2751 | ret = cpuhp_setup_state(CPUHP_MM_ZS_PREPARE, "mm/zsmalloc:prepare", |
| 2752 | zs_cpu_prepare, zs_cpu_dead); |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2753 | if (ret) |
Matthew Wilcox (Oracle) | 68f2736 | 2022-06-07 15:38:48 -0400 | [diff] [blame] | 2754 | goto out; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2755 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 2756 | #ifdef CONFIG_ZPOOL |
| 2757 | zpool_register_driver(&zs_zpool_driver); |
| 2758 | #endif |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2759 | |
Dan Streetman | 4abaac9 | 2016-05-26 15:16:27 -0700 | [diff] [blame] | 2760 | zs_stat_init(); |
| 2761 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 2762 | return 0; |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2763 | |
Minchan Kim | 48b4800 | 2016-07-26 15:23:31 -0700 | [diff] [blame] | 2764 | out: |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2765 | return ret; |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2766 | } |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2767 | |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 2768 | static void __exit zs_exit(void) |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2769 | { |
Ganesh Mahendran | 66cdef6 | 2014-12-18 16:17:40 -0800 | [diff] [blame] | 2770 | #ifdef CONFIG_ZPOOL |
| 2771 | zpool_unregister_driver(&zs_zpool_driver); |
| 2772 | #endif |
Sebastian Andrzej Siewior | 215c89d | 2016-11-27 00:13:38 +0100 | [diff] [blame] | 2773 | cpuhp_remove_state(CPUHP_MM_ZS_PREPARE); |
Ganesh Mahendran | 0f050d9 | 2015-02-12 15:00:54 -0800 | [diff] [blame] | 2774 | |
| 2775 | zs_stat_exit(); |
Nitin Gupta | 61989a8 | 2012-01-09 16:51:56 -0600 | [diff] [blame] | 2776 | } |
Ben Hutchings | 069f101 | 2012-06-20 02:31:11 +0100 | [diff] [blame] | 2777 | |
| 2778 | module_init(zs_init); |
| 2779 | module_exit(zs_exit); |
| 2780 | |
| 2781 | MODULE_LICENSE("Dual BSD/GPL"); |
| 2782 | MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>"); |