Kent Overstreet | 3e3e02e | 2022-10-19 18:31:33 -0400 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 2 | |
| 3 | #include "bcachefs.h" |
Kent Overstreet | d211b40 | 2020-06-15 19:53:46 -0400 | [diff] [blame] | 4 | #include "btree_cache.h" |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 5 | #include "btree_iter.h" |
| 6 | #include "btree_key_cache.h" |
| 7 | #include "btree_locking.h" |
| 8 | #include "btree_update.h" |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 9 | #include "errcode.h" |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 10 | #include "error.h" |
| 11 | #include "journal.h" |
| 12 | #include "journal_reclaim.h" |
| 13 | #include "trace.h" |
| 14 | |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 15 | #include <linux/sched/mm.h> |
| 16 | |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 17 | static inline bool btree_uses_pcpu_readers(enum btree_id id) |
| 18 | { |
| 19 | return id == BTREE_ID_subvolumes; |
| 20 | } |
| 21 | |
Kent Overstreet | 14ba370 | 2020-11-18 14:09:33 -0500 | [diff] [blame] | 22 | static struct kmem_cache *bch2_key_cache; |
| 23 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 24 | static int bch2_btree_key_cache_cmp_fn(struct rhashtable_compare_arg *arg, |
| 25 | const void *obj) |
| 26 | { |
| 27 | const struct bkey_cached *ck = obj; |
| 28 | const struct bkey_cached_key *key = arg->key; |
| 29 | |
Kent Overstreet | e88a75e | 2022-11-24 03:12:22 -0500 | [diff] [blame] | 30 | return ck->key.btree_id != key->btree_id || |
| 31 | !bpos_eq(ck->key.pos, key->pos); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 32 | } |
| 33 | |
| 34 | static const struct rhashtable_params bch2_btree_key_cache_params = { |
| 35 | .head_offset = offsetof(struct bkey_cached, hash), |
| 36 | .key_offset = offsetof(struct bkey_cached, key), |
| 37 | .key_len = sizeof(struct bkey_cached_key), |
| 38 | .obj_cmpfn = bch2_btree_key_cache_cmp_fn, |
| 39 | }; |
| 40 | |
| 41 | __flatten |
Kent Overstreet | 8cad3e2 | 2019-09-22 19:10:21 -0400 | [diff] [blame] | 42 | inline struct bkey_cached * |
| 43 | bch2_btree_key_cache_find(struct bch_fs *c, enum btree_id btree_id, struct bpos pos) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 44 | { |
| 45 | struct bkey_cached_key key = { |
| 46 | .btree_id = btree_id, |
| 47 | .pos = pos, |
| 48 | }; |
| 49 | |
| 50 | return rhashtable_lookup_fast(&c->btree_key_cache.table, &key, |
| 51 | bch2_btree_key_cache_params); |
| 52 | } |
| 53 | |
| 54 | static bool bkey_cached_lock_for_evict(struct bkey_cached *ck) |
| 55 | { |
| 56 | if (!six_trylock_intent(&ck->c.lock)) |
| 57 | return false; |
| 58 | |
Kent Overstreet | 7af365e | 2023-01-08 00:05:30 -0500 | [diff] [blame] | 59 | if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 60 | six_unlock_intent(&ck->c.lock); |
| 61 | return false; |
| 62 | } |
| 63 | |
Kent Overstreet | 7af365e | 2023-01-08 00:05:30 -0500 | [diff] [blame] | 64 | if (!six_trylock_write(&ck->c.lock)) { |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 65 | six_unlock_intent(&ck->c.lock); |
| 66 | return false; |
| 67 | } |
| 68 | |
| 69 | return true; |
| 70 | } |
| 71 | |
| 72 | static void bkey_cached_evict(struct btree_key_cache *c, |
| 73 | struct bkey_cached *ck) |
| 74 | { |
| 75 | BUG_ON(rhashtable_remove_fast(&c->table, &ck->hash, |
| 76 | bch2_btree_key_cache_params)); |
| 77 | memset(&ck->key, ~0, sizeof(ck->key)); |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 78 | |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 79 | atomic_long_dec(&c->nr_keys); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 80 | } |
| 81 | |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 82 | static void bkey_cached_free(struct btree_key_cache *bc, |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 83 | struct bkey_cached *ck) |
| 84 | { |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 85 | struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); |
| 86 | |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 87 | BUG_ON(test_bit(BKEY_CACHED_DIRTY, &ck->flags)); |
| 88 | |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 89 | ck->btree_trans_barrier_seq = |
| 90 | start_poll_synchronize_srcu(&c->btree_trans_barrier); |
| 91 | |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 92 | if (ck->c.lock.readers) { |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 93 | list_move_tail(&ck->list, &bc->freed_pcpu); |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 94 | bc->nr_freed_pcpu++; |
| 95 | } else { |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 96 | list_move_tail(&ck->list, &bc->freed_nonpcpu); |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 97 | bc->nr_freed_nonpcpu++; |
| 98 | } |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 99 | atomic_long_inc(&bc->nr_freed); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 100 | |
| 101 | kfree(ck->k); |
| 102 | ck->k = NULL; |
| 103 | ck->u64s = 0; |
| 104 | |
| 105 | six_unlock_write(&ck->c.lock); |
| 106 | six_unlock_intent(&ck->c.lock); |
| 107 | } |
| 108 | |
Kent Overstreet | b2f83e7 | 2022-10-17 02:08:07 -0400 | [diff] [blame] | 109 | #ifdef __KERNEL__ |
| 110 | static void __bkey_cached_move_to_freelist_ordered(struct btree_key_cache *bc, |
| 111 | struct bkey_cached *ck) |
| 112 | { |
| 113 | struct bkey_cached *pos; |
| 114 | |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 115 | bc->nr_freed_nonpcpu++; |
| 116 | |
Kent Overstreet | b2f83e7 | 2022-10-17 02:08:07 -0400 | [diff] [blame] | 117 | list_for_each_entry_reverse(pos, &bc->freed_nonpcpu, list) { |
| 118 | if (ULONG_CMP_GE(ck->btree_trans_barrier_seq, |
| 119 | pos->btree_trans_barrier_seq)) { |
| 120 | list_move(&ck->list, &pos->list); |
| 121 | return; |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | list_move(&ck->list, &bc->freed_nonpcpu); |
| 126 | } |
| 127 | #endif |
| 128 | |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 129 | static void bkey_cached_move_to_freelist(struct btree_key_cache *bc, |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 130 | struct bkey_cached *ck) |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 131 | { |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 132 | BUG_ON(test_bit(BKEY_CACHED_DIRTY, &ck->flags)); |
| 133 | |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 134 | if (!ck->c.lock.readers) { |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 135 | #ifdef __KERNEL__ |
Kent Overstreet | b2f83e7 | 2022-10-17 02:08:07 -0400 | [diff] [blame] | 136 | struct btree_key_cache_freelist *f; |
| 137 | bool freed = false; |
| 138 | |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 139 | preempt_disable(); |
| 140 | f = this_cpu_ptr(bc->pcpu_freed); |
| 141 | |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 142 | if (f->nr < ARRAY_SIZE(f->objs)) { |
| 143 | f->objs[f->nr++] = ck; |
| 144 | freed = true; |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 145 | } |
| 146 | preempt_enable(); |
| 147 | |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 148 | if (!freed) { |
| 149 | mutex_lock(&bc->lock); |
| 150 | preempt_disable(); |
| 151 | f = this_cpu_ptr(bc->pcpu_freed); |
| 152 | |
| 153 | while (f->nr > ARRAY_SIZE(f->objs) / 2) { |
| 154 | struct bkey_cached *ck2 = f->objs[--f->nr]; |
| 155 | |
Kent Overstreet | b2f83e7 | 2022-10-17 02:08:07 -0400 | [diff] [blame] | 156 | __bkey_cached_move_to_freelist_ordered(bc, ck2); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 157 | } |
| 158 | preempt_enable(); |
| 159 | |
Kent Overstreet | b2f83e7 | 2022-10-17 02:08:07 -0400 | [diff] [blame] | 160 | __bkey_cached_move_to_freelist_ordered(bc, ck); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 161 | mutex_unlock(&bc->lock); |
| 162 | } |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 163 | #else |
| 164 | mutex_lock(&bc->lock); |
| 165 | list_move_tail(&ck->list, &bc->freed_nonpcpu); |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 166 | bc->nr_freed_nonpcpu++; |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 167 | mutex_unlock(&bc->lock); |
| 168 | #endif |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 169 | } else { |
| 170 | mutex_lock(&bc->lock); |
| 171 | list_move_tail(&ck->list, &bc->freed_pcpu); |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 172 | mutex_unlock(&bc->lock); |
| 173 | } |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 174 | } |
| 175 | |
| 176 | static void bkey_cached_free_fast(struct btree_key_cache *bc, |
| 177 | struct bkey_cached *ck) |
| 178 | { |
| 179 | struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); |
| 180 | |
| 181 | ck->btree_trans_barrier_seq = |
| 182 | start_poll_synchronize_srcu(&c->btree_trans_barrier); |
| 183 | |
| 184 | list_del_init(&ck->list); |
| 185 | atomic_long_inc(&bc->nr_freed); |
| 186 | |
| 187 | kfree(ck->k); |
| 188 | ck->k = NULL; |
| 189 | ck->u64s = 0; |
| 190 | |
| 191 | bkey_cached_move_to_freelist(bc, ck); |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 192 | |
| 193 | six_unlock_write(&ck->c.lock); |
| 194 | six_unlock_intent(&ck->c.lock); |
| 195 | } |
| 196 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 197 | static struct bkey_cached * |
Kent Overstreet | 061f799 | 2022-11-14 02:22:30 -0500 | [diff] [blame] | 198 | bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path, |
| 199 | bool *was_new) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 200 | { |
Kent Overstreet | 0242130 | 2022-09-03 21:14:53 -0400 | [diff] [blame] | 201 | struct bch_fs *c = trans->c; |
| 202 | struct btree_key_cache *bc = &c->btree_key_cache; |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 203 | struct bkey_cached *ck = NULL; |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 204 | bool pcpu_readers = btree_uses_pcpu_readers(path->btree_id); |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 205 | int ret; |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 206 | |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 207 | if (!pcpu_readers) { |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 208 | #ifdef __KERNEL__ |
Kent Overstreet | b2f83e7 | 2022-10-17 02:08:07 -0400 | [diff] [blame] | 209 | struct btree_key_cache_freelist *f; |
| 210 | |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 211 | preempt_disable(); |
Kent Overstreet | 0242130 | 2022-09-03 21:14:53 -0400 | [diff] [blame] | 212 | f = this_cpu_ptr(bc->pcpu_freed); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 213 | if (f->nr) |
| 214 | ck = f->objs[--f->nr]; |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 215 | preempt_enable(); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 216 | |
| 217 | if (!ck) { |
| 218 | mutex_lock(&bc->lock); |
| 219 | preempt_disable(); |
| 220 | f = this_cpu_ptr(bc->pcpu_freed); |
| 221 | |
| 222 | while (!list_empty(&bc->freed_nonpcpu) && |
| 223 | f->nr < ARRAY_SIZE(f->objs) / 2) { |
| 224 | ck = list_last_entry(&bc->freed_nonpcpu, struct bkey_cached, list); |
| 225 | list_del_init(&ck->list); |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 226 | bc->nr_freed_nonpcpu--; |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 227 | f->objs[f->nr++] = ck; |
| 228 | } |
| 229 | |
| 230 | ck = f->nr ? f->objs[--f->nr] : NULL; |
| 231 | preempt_enable(); |
| 232 | mutex_unlock(&bc->lock); |
| 233 | } |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 234 | #else |
| 235 | mutex_lock(&bc->lock); |
| 236 | if (!list_empty(&bc->freed_nonpcpu)) { |
| 237 | ck = list_last_entry(&bc->freed_nonpcpu, struct bkey_cached, list); |
| 238 | list_del_init(&ck->list); |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 239 | bc->nr_freed_nonpcpu--; |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 240 | } |
| 241 | mutex_unlock(&bc->lock); |
| 242 | #endif |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 243 | } else { |
| 244 | mutex_lock(&bc->lock); |
| 245 | if (!list_empty(&bc->freed_pcpu)) { |
| 246 | ck = list_last_entry(&bc->freed_pcpu, struct bkey_cached, list); |
| 247 | list_del_init(&ck->list); |
| 248 | } |
Kent Overstreet | 0242130 | 2022-09-03 21:14:53 -0400 | [diff] [blame] | 249 | mutex_unlock(&bc->lock); |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 250 | } |
| 251 | |
| 252 | if (ck) { |
Kent Overstreet | 94c69fa | 2023-02-04 19:39:59 -0500 | [diff] [blame] | 253 | ret = btree_node_lock_nopath(trans, &ck->c, SIX_LOCK_intent, _THIS_IP_); |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 254 | if (unlikely(ret)) { |
Kent Overstreet | 0242130 | 2022-09-03 21:14:53 -0400 | [diff] [blame] | 255 | bkey_cached_move_to_freelist(bc, ck); |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 256 | return ERR_PTR(ret); |
| 257 | } |
| 258 | |
Kent Overstreet | 1bb9123 | 2022-09-03 22:24:16 -0400 | [diff] [blame] | 259 | path->l[0].b = (void *) ck; |
Kent Overstreet | 1fb4fe6 | 2023-05-20 23:57:48 -0400 | [diff] [blame] | 260 | path->l[0].lock_seq = six_lock_seq(&ck->c.lock); |
Kent Overstreet | 5b7fbdc | 2023-09-09 21:14:54 -0400 | [diff] [blame] | 261 | mark_btree_node_locked(trans, path, 0, BTREE_NODE_INTENT_LOCKED); |
Kent Overstreet | 1bb9123 | 2022-09-03 22:24:16 -0400 | [diff] [blame] | 262 | |
| 263 | ret = bch2_btree_node_lock_write(trans, path, &ck->c); |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 264 | if (unlikely(ret)) { |
Kent Overstreet | 1bb9123 | 2022-09-03 22:24:16 -0400 | [diff] [blame] | 265 | btree_node_unlock(trans, path, 0); |
Kent Overstreet | 0242130 | 2022-09-03 21:14:53 -0400 | [diff] [blame] | 266 | bkey_cached_move_to_freelist(bc, ck); |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 267 | return ERR_PTR(ret); |
| 268 | } |
| 269 | |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 270 | return ck; |
| 271 | } |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 272 | |
Kent Overstreet | d95dd37 | 2023-05-28 03:44:38 -0400 | [diff] [blame] | 273 | ck = allocate_dropping_locks(trans, ret, |
| 274 | kmem_cache_zalloc(bch2_key_cache, _gfp)); |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 275 | if (ret) { |
| 276 | kmem_cache_free(bch2_key_cache, ck); |
| 277 | return ERR_PTR(ret); |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 278 | } |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 279 | |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 280 | if (!ck) |
| 281 | return NULL; |
Kent Overstreet | d95dd37 | 2023-05-28 03:44:38 -0400 | [diff] [blame] | 282 | |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 283 | INIT_LIST_HEAD(&ck->list); |
Kent Overstreet | 0d2234a | 2023-05-20 20:57:55 -0400 | [diff] [blame] | 284 | bch2_btree_lock_init(&ck->c, pcpu_readers ? SIX_LOCK_INIT_PCPU : 0); |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 285 | |
| 286 | ck->c.cached = true; |
| 287 | BUG_ON(!six_trylock_intent(&ck->c.lock)); |
| 288 | BUG_ON(!six_trylock_write(&ck->c.lock)); |
| 289 | *was_new = true; |
| 290 | return ck; |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 291 | } |
| 292 | |
| 293 | static struct bkey_cached * |
| 294 | bkey_cached_reuse(struct btree_key_cache *c) |
| 295 | { |
| 296 | struct bucket_table *tbl; |
| 297 | struct rhash_head *pos; |
| 298 | struct bkey_cached *ck; |
| 299 | unsigned i; |
| 300 | |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 301 | mutex_lock(&c->lock); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 302 | rcu_read_lock(); |
| 303 | tbl = rht_dereference_rcu(c->table.tbl, &c->table); |
| 304 | for (i = 0; i < tbl->size; i++) |
| 305 | rht_for_each_entry_rcu(ck, pos, tbl, i, hash) { |
| 306 | if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) && |
| 307 | bkey_cached_lock_for_evict(ck)) { |
| 308 | bkey_cached_evict(c, ck); |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 309 | goto out; |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 310 | } |
| 311 | } |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 312 | ck = NULL; |
| 313 | out: |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 314 | rcu_read_unlock(); |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 315 | mutex_unlock(&c->lock); |
| 316 | return ck; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 317 | } |
| 318 | |
| 319 | static struct bkey_cached * |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 320 | btree_key_cache_create(struct btree_trans *trans, struct btree_path *path) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 321 | { |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 322 | struct bch_fs *c = trans->c; |
Kent Overstreet | f0f41a6 | 2021-12-30 20:14:52 -0500 | [diff] [blame] | 323 | struct btree_key_cache *bc = &c->btree_key_cache; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 324 | struct bkey_cached *ck; |
Kent Overstreet | 061f799 | 2022-11-14 02:22:30 -0500 | [diff] [blame] | 325 | bool was_new = false; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 326 | |
Kent Overstreet | 061f799 | 2022-11-14 02:22:30 -0500 | [diff] [blame] | 327 | ck = bkey_cached_alloc(trans, path, &was_new); |
Kent Overstreet | 3e3e02e | 2022-10-19 18:31:33 -0400 | [diff] [blame] | 328 | if (IS_ERR(ck)) |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 329 | return ck; |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 330 | |
| 331 | if (unlikely(!ck)) { |
Kent Overstreet | f0f41a6 | 2021-12-30 20:14:52 -0500 | [diff] [blame] | 332 | ck = bkey_cached_reuse(bc); |
| 333 | if (unlikely(!ck)) { |
| 334 | bch_err(c, "error allocating memory for key cache item, btree %s", |
Kent Overstreet | 88dfe19 | 2023-10-19 22:49:08 -0400 | [diff] [blame] | 335 | bch2_btree_id_str(path->btree_id)); |
Kent Overstreet | 65d48e3 | 2023-03-14 15:35:57 -0400 | [diff] [blame] | 336 | return ERR_PTR(-BCH_ERR_ENOMEM_btree_key_cache_create); |
Kent Overstreet | f0f41a6 | 2021-12-30 20:14:52 -0500 | [diff] [blame] | 337 | } |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 338 | |
Kent Overstreet | 5b7fbdc | 2023-09-09 21:14:54 -0400 | [diff] [blame] | 339 | mark_btree_node_locked(trans, path, 0, BTREE_NODE_INTENT_LOCKED); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 340 | } |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 341 | |
| 342 | ck->c.level = 0; |
Kent Overstreet | 0242130 | 2022-09-03 21:14:53 -0400 | [diff] [blame] | 343 | ck->c.btree_id = path->btree_id; |
| 344 | ck->key.btree_id = path->btree_id; |
| 345 | ck->key.pos = path->pos; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 346 | ck->valid = false; |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 347 | ck->flags = 1U << BKEY_CACHED_ACCESSED; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 348 | |
Kent Overstreet | f0f41a6 | 2021-12-30 20:14:52 -0500 | [diff] [blame] | 349 | if (unlikely(rhashtable_lookup_insert_fast(&bc->table, |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 350 | &ck->hash, |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 351 | bch2_btree_key_cache_params))) { |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 352 | /* We raced with another fill: */ |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 353 | |
| 354 | if (likely(was_new)) { |
| 355 | six_unlock_write(&ck->c.lock); |
| 356 | six_unlock_intent(&ck->c.lock); |
| 357 | kfree(ck); |
| 358 | } else { |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 359 | bkey_cached_free_fast(bc, ck); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 360 | } |
| 361 | |
Kent Overstreet | 1bb9123 | 2022-09-03 22:24:16 -0400 | [diff] [blame] | 362 | mark_btree_node_locked(trans, path, 0, BTREE_NODE_UNLOCKED); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 363 | return NULL; |
| 364 | } |
| 365 | |
Kent Overstreet | f0f41a6 | 2021-12-30 20:14:52 -0500 | [diff] [blame] | 366 | atomic_long_inc(&bc->nr_keys); |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 367 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 368 | six_unlock_write(&ck->c.lock); |
| 369 | |
| 370 | return ck; |
| 371 | } |
| 372 | |
| 373 | static int btree_key_cache_fill(struct btree_trans *trans, |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 374 | struct btree_path *ck_path, |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 375 | struct bkey_cached *ck) |
| 376 | { |
Kent Overstreet | 1617d56 | 2022-11-22 20:15:33 -0500 | [diff] [blame] | 377 | struct btree_iter iter; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 378 | struct bkey_s_c k; |
| 379 | unsigned new_u64s = 0; |
| 380 | struct bkey_i *new_k = NULL; |
| 381 | int ret; |
| 382 | |
Kent Overstreet | b3f8e71 | 2024-03-10 20:53:17 -0400 | [diff] [blame] | 383 | bch2_trans_iter_init(trans, &iter, ck->key.btree_id, ck->key.pos, |
| 384 | BTREE_ITER_KEY_CACHE_FILL| |
| 385 | BTREE_ITER_CACHED_NOFILL); |
| 386 | iter.flags &= ~BTREE_ITER_WITH_JOURNAL; |
| 387 | k = bch2_btree_iter_peek_slot(&iter); |
Kent Overstreet | 1617d56 | 2022-11-22 20:15:33 -0500 | [diff] [blame] | 388 | ret = bkey_err(k); |
Kent Overstreet | 8d956c2 | 2021-03-19 22:54:18 -0400 | [diff] [blame] | 389 | if (ret) |
| 390 | goto err; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 391 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 392 | if (!bch2_btree_node_relock(trans, ck_path, 0)) { |
Kent Overstreet | 674cfc2 | 2022-08-27 12:48:36 -0400 | [diff] [blame] | 393 | trace_and_count(trans->c, trans_restart_relock_key_cache_fill, trans, _THIS_IP_, ck_path); |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 394 | ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_key_cache_fill); |
Kent Overstreet | 8d956c2 | 2021-03-19 22:54:18 -0400 | [diff] [blame] | 395 | goto err; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 396 | } |
| 397 | |
Kent Overstreet | bc2e5d5 | 2021-04-24 00:42:02 -0400 | [diff] [blame] | 398 | /* |
| 399 | * bch2_varint_decode can read past the end of the buffer by at |
| 400 | * most 7 bytes (it won't be used): |
| 401 | */ |
| 402 | new_u64s = k.k->u64s + 1; |
| 403 | |
Kent Overstreet | a729e48 | 2022-04-17 17:50:47 -0400 | [diff] [blame] | 404 | /* |
| 405 | * Allocate some extra space so that the transaction commit path is less |
| 406 | * likely to have to reallocate, since that requires a transaction |
| 407 | * restart: |
| 408 | */ |
| 409 | new_u64s = min(256U, (new_u64s * 3) / 2); |
| 410 | |
Kent Overstreet | bc2e5d5 | 2021-04-24 00:42:02 -0400 | [diff] [blame] | 411 | if (new_u64s > ck->u64s) { |
| 412 | new_u64s = roundup_pow_of_two(new_u64s); |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 413 | new_k = kmalloc(new_u64s * sizeof(u64), GFP_NOWAIT|__GFP_NOWARN); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 414 | if (!new_k) { |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 415 | bch2_trans_unlock(trans); |
| 416 | |
| 417 | new_k = kmalloc(new_u64s * sizeof(u64), GFP_KERNEL); |
| 418 | if (!new_k) { |
| 419 | bch_err(trans->c, "error allocating memory for key cache key, btree %s u64s %u", |
Kent Overstreet | 88dfe19 | 2023-10-19 22:49:08 -0400 | [diff] [blame] | 420 | bch2_btree_id_str(ck->key.btree_id), new_u64s); |
Kent Overstreet | 65d48e3 | 2023-03-14 15:35:57 -0400 | [diff] [blame] | 421 | ret = -BCH_ERR_ENOMEM_btree_key_cache_fill; |
Kent Overstreet | 6c36318 | 2023-01-07 05:46:52 -0500 | [diff] [blame] | 422 | goto err; |
| 423 | } |
| 424 | |
| 425 | if (!bch2_btree_node_relock(trans, ck_path, 0)) { |
| 426 | kfree(new_k); |
| 427 | trace_and_count(trans->c, trans_restart_relock_key_cache_fill, trans, _THIS_IP_, ck_path); |
| 428 | ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_key_cache_fill); |
| 429 | goto err; |
| 430 | } |
| 431 | |
| 432 | ret = bch2_trans_relock(trans); |
| 433 | if (ret) { |
| 434 | kfree(new_k); |
| 435 | goto err; |
| 436 | } |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 437 | } |
| 438 | } |
| 439 | |
Kent Overstreet | da4474f2 | 2022-09-03 21:09:54 -0400 | [diff] [blame] | 440 | ret = bch2_btree_node_lock_write(trans, ck_path, &ck_path->l[0].b->c); |
Kent Overstreet | d5024b0 | 2022-08-22 23:39:23 -0400 | [diff] [blame] | 441 | if (ret) { |
| 442 | kfree(new_k); |
| 443 | goto err; |
| 444 | } |
| 445 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 446 | if (new_k) { |
| 447 | kfree(ck->k); |
| 448 | ck->u64s = new_u64s; |
| 449 | ck->k = new_k; |
| 450 | } |
| 451 | |
| 452 | bkey_reassemble(ck->k, k); |
| 453 | ck->valid = true; |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 454 | bch2_btree_node_unlock_write(trans, ck_path, ck_path->l[0].b); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 455 | |
| 456 | /* We're not likely to need this iterator again: */ |
Kent Overstreet | 1617d56 | 2022-11-22 20:15:33 -0500 | [diff] [blame] | 457 | set_btree_iter_dontneed(&iter); |
Kent Overstreet | 8d956c2 | 2021-03-19 22:54:18 -0400 | [diff] [blame] | 458 | err: |
Kent Overstreet | 1617d56 | 2022-11-22 20:15:33 -0500 | [diff] [blame] | 459 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 8d956c2 | 2021-03-19 22:54:18 -0400 | [diff] [blame] | 460 | return ret; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 461 | } |
| 462 | |
Kent Overstreet | 3e3e02e | 2022-10-19 18:31:33 -0400 | [diff] [blame] | 463 | static noinline int |
Kent Overstreet | 99e2146 | 2022-09-26 22:34:49 -0400 | [diff] [blame] | 464 | bch2_btree_path_traverse_cached_slowpath(struct btree_trans *trans, struct btree_path *path, |
| 465 | unsigned flags) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 466 | { |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 467 | struct bch_fs *c = trans->c; |
| 468 | struct bkey_cached *ck; |
| 469 | int ret = 0; |
| 470 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 471 | BUG_ON(path->level); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 472 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 473 | path->l[1].b = NULL; |
Kent Overstreet | 5aab663 | 2021-07-14 15:13:27 -0400 | [diff] [blame] | 474 | |
Kent Overstreet | b8c5b16 | 2023-01-25 12:16:23 -0500 | [diff] [blame] | 475 | if (bch2_btree_node_relock_notrace(trans, path, 0)) { |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 476 | ck = (void *) path->l[0].b; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 477 | goto fill; |
| 478 | } |
| 479 | retry: |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 480 | ck = bch2_btree_key_cache_find(c, path->btree_id, path->pos); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 481 | if (!ck) { |
Kent Overstreet | 0242130 | 2022-09-03 21:14:53 -0400 | [diff] [blame] | 482 | ck = btree_key_cache_create(trans, path); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 483 | ret = PTR_ERR_OR_ZERO(ck); |
| 484 | if (ret) |
| 485 | goto err; |
| 486 | if (!ck) |
| 487 | goto retry; |
| 488 | |
Kent Overstreet | 5b7fbdc | 2023-09-09 21:14:54 -0400 | [diff] [blame] | 489 | mark_btree_node_locked(trans, path, 0, BTREE_NODE_INTENT_LOCKED); |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 490 | path->locks_want = 1; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 491 | } else { |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 492 | enum six_lock_type lock_want = __btree_lock_want(path, 0); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 493 | |
Kent Overstreet | 0d7009d | 2022-08-22 15:29:53 -0400 | [diff] [blame] | 494 | ret = btree_node_lock(trans, path, (void *) ck, 0, |
| 495 | lock_want, _THIS_IP_); |
| 496 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 497 | goto err; |
| 498 | |
| 499 | BUG_ON(ret); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 500 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 501 | if (ck->key.btree_id != path->btree_id || |
Kent Overstreet | e88a75e | 2022-11-24 03:12:22 -0500 | [diff] [blame] | 502 | !bpos_eq(ck->key.pos, path->pos)) { |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 503 | six_unlock_type(&ck->c.lock, lock_want); |
| 504 | goto retry; |
| 505 | } |
| 506 | |
Kent Overstreet | 5b7fbdc | 2023-09-09 21:14:54 -0400 | [diff] [blame] | 507 | mark_btree_node_locked(trans, path, 0, |
| 508 | (enum btree_node_locked_type) lock_want); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 509 | } |
| 510 | |
Kent Overstreet | 1fb4fe6 | 2023-05-20 23:57:48 -0400 | [diff] [blame] | 511 | path->l[0].lock_seq = six_lock_seq(&ck->c.lock); |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 512 | path->l[0].b = (void *) ck; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 513 | fill: |
Kent Overstreet | 6f90e6b | 2023-01-25 10:15:39 -0500 | [diff] [blame] | 514 | path->uptodate = BTREE_ITER_UPTODATE; |
| 515 | |
Kent Overstreet | 087e53c | 2022-12-20 11:26:57 -0500 | [diff] [blame] | 516 | if (!ck->valid && !(flags & BTREE_ITER_CACHED_NOFILL)) { |
Kent Overstreet | 49e401f | 2022-08-07 13:43:32 -0400 | [diff] [blame] | 517 | /* |
| 518 | * Using the underscore version because we haven't set |
| 519 | * path->uptodate yet: |
| 520 | */ |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 521 | if (!path->locks_want && |
Kent Overstreet | be9e782 | 2023-10-27 15:23:46 -0400 | [diff] [blame] | 522 | !__bch2_btree_path_upgrade(trans, path, 1, NULL)) { |
Kent Overstreet | 674cfc2 | 2022-08-27 12:48:36 -0400 | [diff] [blame] | 523 | trace_and_count(trans->c, trans_restart_key_cache_upgrade, trans, _THIS_IP_); |
Kent Overstreet | ae33e7a | 2022-08-04 12:46:37 -0400 | [diff] [blame] | 524 | ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_key_cache_upgrade); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 525 | goto err; |
| 526 | } |
| 527 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 528 | ret = btree_key_cache_fill(trans, path, ck); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 529 | if (ret) |
| 530 | goto err; |
Kent Overstreet | 6f90e6b | 2023-01-25 10:15:39 -0500 | [diff] [blame] | 531 | |
| 532 | ret = bch2_btree_path_relock(trans, path, _THIS_IP_); |
| 533 | if (ret) |
| 534 | goto err; |
| 535 | |
| 536 | path->uptodate = BTREE_ITER_UPTODATE; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 537 | } |
| 538 | |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 539 | if (!test_bit(BKEY_CACHED_ACCESSED, &ck->flags)) |
| 540 | set_bit(BKEY_CACHED_ACCESSED, &ck->flags); |
| 541 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 542 | BUG_ON(btree_node_locked_type(path, 0) != btree_lock_want(path, 0)); |
Kent Overstreet | 6f90e6b | 2023-01-25 10:15:39 -0500 | [diff] [blame] | 543 | BUG_ON(path->uptodate); |
Kent Overstreet | 53b3e3c | 2021-03-08 17:09:13 -0500 | [diff] [blame] | 544 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 545 | return ret; |
| 546 | err: |
Kent Overstreet | 6f90e6b | 2023-01-25 10:15:39 -0500 | [diff] [blame] | 547 | path->uptodate = BTREE_ITER_NEED_TRAVERSE; |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 548 | if (!bch2_err_matches(ret, BCH_ERR_transaction_restart)) { |
Daniel Hill | 8bfe14e | 2022-07-14 18:58:23 +1200 | [diff] [blame] | 549 | btree_node_unlock(trans, path, 0); |
Kent Overstreet | 315c9ba | 2022-08-10 19:08:30 -0400 | [diff] [blame] | 550 | path->l[0].b = ERR_PTR(ret); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 551 | } |
| 552 | return ret; |
| 553 | } |
| 554 | |
Kent Overstreet | 99e2146 | 2022-09-26 22:34:49 -0400 | [diff] [blame] | 555 | int bch2_btree_path_traverse_cached(struct btree_trans *trans, struct btree_path *path, |
| 556 | unsigned flags) |
| 557 | { |
| 558 | struct bch_fs *c = trans->c; |
| 559 | struct bkey_cached *ck; |
| 560 | int ret = 0; |
| 561 | |
| 562 | EBUG_ON(path->level); |
| 563 | |
| 564 | path->l[1].b = NULL; |
| 565 | |
Kent Overstreet | b8c5b16 | 2023-01-25 12:16:23 -0500 | [diff] [blame] | 566 | if (bch2_btree_node_relock_notrace(trans, path, 0)) { |
Kent Overstreet | 99e2146 | 2022-09-26 22:34:49 -0400 | [diff] [blame] | 567 | ck = (void *) path->l[0].b; |
| 568 | goto fill; |
| 569 | } |
| 570 | retry: |
| 571 | ck = bch2_btree_key_cache_find(c, path->btree_id, path->pos); |
| 572 | if (!ck) { |
| 573 | return bch2_btree_path_traverse_cached_slowpath(trans, path, flags); |
| 574 | } else { |
| 575 | enum six_lock_type lock_want = __btree_lock_want(path, 0); |
| 576 | |
| 577 | ret = btree_node_lock(trans, path, (void *) ck, 0, |
| 578 | lock_want, _THIS_IP_); |
| 579 | EBUG_ON(ret && !bch2_err_matches(ret, BCH_ERR_transaction_restart)); |
| 580 | |
| 581 | if (ret) |
| 582 | return ret; |
| 583 | |
| 584 | if (ck->key.btree_id != path->btree_id || |
Kent Overstreet | e88a75e | 2022-11-24 03:12:22 -0500 | [diff] [blame] | 585 | !bpos_eq(ck->key.pos, path->pos)) { |
Kent Overstreet | 99e2146 | 2022-09-26 22:34:49 -0400 | [diff] [blame] | 586 | six_unlock_type(&ck->c.lock, lock_want); |
| 587 | goto retry; |
| 588 | } |
| 589 | |
Kent Overstreet | 5b7fbdc | 2023-09-09 21:14:54 -0400 | [diff] [blame] | 590 | mark_btree_node_locked(trans, path, 0, |
| 591 | (enum btree_node_locked_type) lock_want); |
Kent Overstreet | 99e2146 | 2022-09-26 22:34:49 -0400 | [diff] [blame] | 592 | } |
| 593 | |
Kent Overstreet | 1fb4fe6 | 2023-05-20 23:57:48 -0400 | [diff] [blame] | 594 | path->l[0].lock_seq = six_lock_seq(&ck->c.lock); |
Kent Overstreet | 99e2146 | 2022-09-26 22:34:49 -0400 | [diff] [blame] | 595 | path->l[0].b = (void *) ck; |
| 596 | fill: |
| 597 | if (!ck->valid) |
| 598 | return bch2_btree_path_traverse_cached_slowpath(trans, path, flags); |
| 599 | |
| 600 | if (!test_bit(BKEY_CACHED_ACCESSED, &ck->flags)) |
| 601 | set_bit(BKEY_CACHED_ACCESSED, &ck->flags); |
| 602 | |
| 603 | path->uptodate = BTREE_ITER_UPTODATE; |
| 604 | EBUG_ON(!ck->valid); |
| 605 | EBUG_ON(btree_node_locked_type(path, 0) != btree_lock_want(path, 0)); |
| 606 | |
| 607 | return ret; |
| 608 | } |
| 609 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 610 | static int btree_key_cache_flush_pos(struct btree_trans *trans, |
| 611 | struct bkey_cached_key key, |
| 612 | u64 journal_seq, |
Kent Overstreet | 2940295 | 2021-04-03 16:24:13 -0400 | [diff] [blame] | 613 | unsigned commit_flags, |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 614 | bool evict) |
| 615 | { |
| 616 | struct bch_fs *c = trans->c; |
| 617 | struct journal *j = &c->journal; |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 618 | struct btree_iter c_iter, b_iter; |
Kent Overstreet | b206df6 | 2020-12-03 13:09:08 -0500 | [diff] [blame] | 619 | struct bkey_cached *ck = NULL; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 620 | int ret; |
| 621 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 622 | bch2_trans_iter_init(trans, &b_iter, key.btree_id, key.pos, |
| 623 | BTREE_ITER_SLOTS| |
Kent Overstreet | c075ff7 | 2021-03-04 22:29:25 -0500 | [diff] [blame] | 624 | BTREE_ITER_INTENT| |
| 625 | BTREE_ITER_ALL_SNAPSHOTS); |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 626 | bch2_trans_iter_init(trans, &c_iter, key.btree_id, key.pos, |
| 627 | BTREE_ITER_CACHED| |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 628 | BTREE_ITER_INTENT); |
Kent Overstreet | f7b6ca2 | 2022-02-06 23:15:12 -0500 | [diff] [blame] | 629 | b_iter.flags &= ~BTREE_ITER_WITH_KEY_CACHE; |
| 630 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 631 | ret = bch2_btree_iter_traverse(&c_iter); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 632 | if (ret) |
Kent Overstreet | a6eba44 | 2021-07-23 18:26:38 -0400 | [diff] [blame] | 633 | goto out; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 634 | |
Kent Overstreet | 07f383c | 2023-12-04 00:39:38 -0500 | [diff] [blame] | 635 | ck = (void *) btree_iter_path(trans, &c_iter)->l[0].b; |
Kent Overstreet | a9c0b12 | 2022-01-12 00:49:23 -0500 | [diff] [blame] | 636 | if (!ck) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 637 | goto out; |
| 638 | |
| 639 | if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { |
Kent Overstreet | a9c0b12 | 2022-01-12 00:49:23 -0500 | [diff] [blame] | 640 | if (evict) |
| 641 | goto evict; |
| 642 | goto out; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 643 | } |
| 644 | |
Kent Overstreet | a9c0b12 | 2022-01-12 00:49:23 -0500 | [diff] [blame] | 645 | BUG_ON(!ck->valid); |
| 646 | |
| 647 | if (journal_seq && ck->journal.seq != journal_seq) |
| 648 | goto out; |
| 649 | |
Kent Overstreet | b4b79b0 | 2023-11-13 21:12:35 -0500 | [diff] [blame] | 650 | trans->journal_res.seq = ck->journal.seq; |
| 651 | |
Kent Overstreet | f09517f | 2021-04-20 17:09:25 -0400 | [diff] [blame] | 652 | /* |
Kent Overstreet | b4b79b0 | 2023-11-13 21:12:35 -0500 | [diff] [blame] | 653 | * If we're at the end of the journal, we really want to free up space |
| 654 | * in the journal right away - we don't want to pin that old journal |
| 655 | * sequence number with a new btree node write, we want to re-journal |
| 656 | * the update |
Kent Overstreet | 3e3e02e | 2022-10-19 18:31:33 -0400 | [diff] [blame] | 657 | */ |
Kent Overstreet | b4b79b0 | 2023-11-13 21:12:35 -0500 | [diff] [blame] | 658 | if (ck->journal.seq == journal_last_seq(j)) |
| 659 | commit_flags |= BCH_WATERMARK_reclaim; |
Kent Overstreet | 0c0ba8e | 2023-12-19 20:54:11 -0500 | [diff] [blame] | 660 | |
| 661 | if (ck->journal.seq != journal_last_seq(j) || |
| 662 | j->watermark == BCH_WATERMARK_stripe) |
Kent Overstreet | b4b79b0 | 2023-11-13 21:12:35 -0500 | [diff] [blame] | 663 | commit_flags |= BCH_TRANS_COMMIT_no_journal_res; |
| 664 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 665 | ret = bch2_btree_iter_traverse(&b_iter) ?: |
| 666 | bch2_trans_update(trans, &b_iter, ck->k, |
Kent Overstreet | 12ce5b7 | 2022-01-12 01:14:47 -0500 | [diff] [blame] | 667 | BTREE_UPDATE_KEY_CACHE_RECLAIM| |
Kent Overstreet | b00fde8 | 2021-07-05 22:16:02 -0400 | [diff] [blame] | 668 | BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE| |
| 669 | BTREE_TRIGGER_NORUN) ?: |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 670 | bch2_trans_commit(trans, NULL, NULL, |
Kent Overstreet | cb52d23e | 2023-11-11 16:31:50 -0500 | [diff] [blame] | 671 | BCH_TRANS_COMMIT_no_check_rw| |
| 672 | BCH_TRANS_COMMIT_no_enospc| |
Kent Overstreet | 2940295 | 2021-04-03 16:24:13 -0400 | [diff] [blame] | 673 | commit_flags); |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 674 | |
| 675 | bch2_fs_fatal_err_on(ret && |
| 676 | !bch2_err_matches(ret, BCH_ERR_transaction_restart) && |
| 677 | !bch2_err_matches(ret, BCH_ERR_journal_reclaim_would_deadlock) && |
| 678 | !bch2_journal_error(j), c, |
Kent Overstreet | 3ed9406 | 2024-03-17 21:51:19 -0400 | [diff] [blame] | 679 | "flushing key cache: %s", bch2_err_str(ret)); |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 680 | if (ret) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 681 | goto out; |
| 682 | |
| 683 | bch2_journal_pin_drop(j, &ck->journal); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 684 | |
Kent Overstreet | 07f383c | 2023-12-04 00:39:38 -0500 | [diff] [blame] | 685 | struct btree_path *path = btree_iter_path(trans, &c_iter); |
| 686 | BUG_ON(!btree_node_locked(path, 0)); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 687 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 688 | if (!evict) { |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 689 | if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { |
| 690 | clear_bit(BKEY_CACHED_DIRTY, &ck->flags); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 691 | atomic_long_dec(&c->btree_key_cache.nr_dirty); |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 692 | } |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 693 | } else { |
Kent Overstreet | 38474c2 | 2022-09-02 22:59:39 -0400 | [diff] [blame] | 694 | struct btree_path *path2; |
Kent Overstreet | ccb7b08 | 2023-12-10 23:37:45 -0500 | [diff] [blame] | 695 | unsigned i; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 696 | evict: |
Kent Overstreet | ccb7b08 | 2023-12-10 23:37:45 -0500 | [diff] [blame] | 697 | trans_for_each_path(trans, path2, i) |
Kent Overstreet | 07f383c | 2023-12-04 00:39:38 -0500 | [diff] [blame] | 698 | if (path2 != path) |
Kent Overstreet | 38474c2 | 2022-09-02 22:59:39 -0400 | [diff] [blame] | 699 | __bch2_btree_path_unlock(trans, path2); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 700 | |
Kent Overstreet | 07f383c | 2023-12-04 00:39:38 -0500 | [diff] [blame] | 701 | bch2_btree_node_lock_write_nofail(trans, path, &ck->c); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 702 | |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 703 | if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { |
| 704 | clear_bit(BKEY_CACHED_DIRTY, &ck->flags); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 705 | atomic_long_dec(&c->btree_key_cache.nr_dirty); |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 706 | } |
| 707 | |
Kent Overstreet | 07f383c | 2023-12-04 00:39:38 -0500 | [diff] [blame] | 708 | mark_btree_node_locked_noreset(path, 0, BTREE_NODE_UNLOCKED); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 709 | bkey_cached_evict(&c->btree_key_cache, ck); |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 710 | bkey_cached_free_fast(&c->btree_key_cache, ck); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 711 | } |
| 712 | out: |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 713 | bch2_trans_iter_exit(trans, &b_iter); |
| 714 | bch2_trans_iter_exit(trans, &c_iter); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 715 | return ret; |
| 716 | } |
| 717 | |
Kent Overstreet | 241e263 | 2021-03-31 21:44:55 -0400 | [diff] [blame] | 718 | int bch2_btree_key_cache_journal_flush(struct journal *j, |
| 719 | struct journal_entry_pin *pin, u64 seq) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 720 | { |
| 721 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
| 722 | struct bkey_cached *ck = |
| 723 | container_of(pin, struct bkey_cached, journal); |
| 724 | struct bkey_cached_key key; |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 725 | struct btree_trans *trans = bch2_trans_get(c); |
Kent Overstreet | ca7d8fc | 2022-08-21 14:29:43 -0400 | [diff] [blame] | 726 | int srcu_idx = srcu_read_lock(&c->btree_trans_barrier); |
Kent Overstreet | 2940295 | 2021-04-03 16:24:13 -0400 | [diff] [blame] | 727 | int ret = 0; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 728 | |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 729 | btree_node_lock_nopath_nofail(trans, &ck->c, SIX_LOCK_read); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 730 | key = ck->key; |
| 731 | |
| 732 | if (ck->journal.seq != seq || |
| 733 | !test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { |
| 734 | six_unlock_read(&ck->c.lock); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 735 | goto unlock; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 736 | } |
Kent Overstreet | 8322a93 | 2023-01-04 04:34:16 -0500 | [diff] [blame] | 737 | |
| 738 | if (ck->seq != seq) { |
| 739 | bch2_journal_pin_update(&c->journal, ck->seq, &ck->journal, |
| 740 | bch2_btree_key_cache_journal_flush); |
| 741 | six_unlock_read(&ck->c.lock); |
| 742 | goto unlock; |
| 743 | } |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 744 | six_unlock_read(&ck->c.lock); |
| 745 | |
Kent Overstreet | cf5bacb | 2023-11-27 01:46:18 -0500 | [diff] [blame] | 746 | ret = lockrestart_do(trans, |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 747 | btree_key_cache_flush_pos(trans, key, seq, |
Kent Overstreet | cb52d23e | 2023-11-11 16:31:50 -0500 | [diff] [blame] | 748 | BCH_TRANS_COMMIT_journal_reclaim, false)); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 749 | unlock: |
| 750 | srcu_read_unlock(&c->btree_trans_barrier, srcu_idx); |
Kent Overstreet | 2940295 | 2021-04-03 16:24:13 -0400 | [diff] [blame] | 751 | |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 752 | bch2_trans_put(trans); |
Kent Overstreet | 2940295 | 2021-04-03 16:24:13 -0400 | [diff] [blame] | 753 | return ret; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 754 | } |
| 755 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 756 | bool bch2_btree_insert_key_cached(struct btree_trans *trans, |
Kent Overstreet | 30ca6ec | 2023-02-09 13:22:12 -0500 | [diff] [blame] | 757 | unsigned flags, |
Brian Foster | e53d03f | 2023-03-02 09:03:37 -0500 | [diff] [blame] | 758 | struct btree_insert_entry *insert_entry) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 759 | { |
| 760 | struct bch_fs *c = trans->c; |
Kent Overstreet | 7f9821a | 2023-12-10 16:10:24 -0500 | [diff] [blame] | 761 | struct bkey_cached *ck = (void *) (trans->paths + insert_entry->path)->l[0].b; |
Brian Foster | e53d03f | 2023-03-02 09:03:37 -0500 | [diff] [blame] | 762 | struct bkey_i *insert = insert_entry->k; |
Kent Overstreet | 8a92e54 | 2020-11-19 19:54:40 -0500 | [diff] [blame] | 763 | bool kick_reclaim = false; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 764 | |
Kent Overstreet | ac2ccdd | 2023-03-04 23:05:55 -0500 | [diff] [blame] | 765 | BUG_ON(insert->k.u64s > ck->u64s); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 766 | |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 767 | bkey_copy(ck->k, insert); |
| 768 | ck->valid = true; |
| 769 | |
| 770 | if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { |
Kent Overstreet | 3c471b6 | 2023-11-26 17:05:02 -0500 | [diff] [blame] | 771 | EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags)); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 772 | set_bit(BKEY_CACHED_DIRTY, &ck->flags); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 773 | atomic_long_inc(&c->btree_key_cache.nr_dirty); |
Kent Overstreet | 8a92e54 | 2020-11-19 19:54:40 -0500 | [diff] [blame] | 774 | |
| 775 | if (bch2_nr_btree_keys_need_flush(c)) |
| 776 | kick_reclaim = true; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 777 | } |
| 778 | |
Brian Foster | e53d03f | 2023-03-02 09:03:37 -0500 | [diff] [blame] | 779 | /* |
| 780 | * To minimize lock contention, we only add the journal pin here and |
| 781 | * defer pin updates to the flush callback via ->seq. Be careful not to |
| 782 | * update ->seq on nojournal commits because we don't want to update the |
| 783 | * pin to a seq that doesn't include journal updates on disk. Otherwise |
| 784 | * we risk losing the update after a crash. |
| 785 | * |
| 786 | * The only exception is if the pin is not active in the first place. We |
| 787 | * have to add the pin because journal reclaim drives key cache |
| 788 | * flushing. The flush callback will not proceed unless ->seq matches |
| 789 | * the latest pin, so make sure it starts with a consistent value. |
| 790 | */ |
| 791 | if (!(insert_entry->flags & BTREE_UPDATE_NOJOURNAL) || |
| 792 | !journal_pin_active(&ck->journal)) { |
| 793 | ck->seq = trans->journal_res.seq; |
| 794 | } |
Kent Overstreet | 8322a93 | 2023-01-04 04:34:16 -0500 | [diff] [blame] | 795 | bch2_journal_pin_add(&c->journal, trans->journal_res.seq, |
| 796 | &ck->journal, bch2_btree_key_cache_journal_flush); |
Kent Overstreet | 8a92e54 | 2020-11-19 19:54:40 -0500 | [diff] [blame] | 797 | |
| 798 | if (kick_reclaim) |
Kent Overstreet | b7a9bbfc | 2020-11-19 20:55:33 -0500 | [diff] [blame] | 799 | journal_reclaim_kick(&c->journal); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 800 | return true; |
| 801 | } |
| 802 | |
Kent Overstreet | 12ce5b7 | 2022-01-12 01:14:47 -0500 | [diff] [blame] | 803 | void bch2_btree_key_cache_drop(struct btree_trans *trans, |
| 804 | struct btree_path *path) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 805 | { |
Kent Overstreet | 45b033f | 2022-08-11 21:06:43 -0400 | [diff] [blame] | 806 | struct bch_fs *c = trans->c; |
Kent Overstreet | 12ce5b7 | 2022-01-12 01:14:47 -0500 | [diff] [blame] | 807 | struct bkey_cached *ck = (void *) path->l[0].b; |
| 808 | |
Kent Overstreet | 45b033f | 2022-08-11 21:06:43 -0400 | [diff] [blame] | 809 | BUG_ON(!ck->valid); |
Kent Overstreet | 12ce5b7 | 2022-01-12 01:14:47 -0500 | [diff] [blame] | 810 | |
Kent Overstreet | 45b033f | 2022-08-11 21:06:43 -0400 | [diff] [blame] | 811 | /* |
| 812 | * We just did an update to the btree, bypassing the key cache: the key |
| 813 | * cache key is now stale and must be dropped, even if dirty: |
| 814 | */ |
| 815 | if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { |
| 816 | clear_bit(BKEY_CACHED_DIRTY, &ck->flags); |
| 817 | atomic_long_dec(&c->btree_key_cache.nr_dirty); |
| 818 | bch2_journal_pin_drop(&c->journal, &ck->journal); |
| 819 | } |
| 820 | |
| 821 | ck->valid = false; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 822 | } |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 823 | |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 824 | static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, |
| 825 | struct shrink_control *sc) |
| 826 | { |
Linus Torvalds | ecae0bd | 2023-11-02 19:38:47 -1000 | [diff] [blame] | 827 | struct bch_fs *c = shrink->private_data; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 828 | struct btree_key_cache *bc = &c->btree_key_cache; |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 829 | struct bucket_table *tbl; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 830 | struct bkey_cached *ck, *t; |
| 831 | size_t scanned = 0, freed = 0, nr = sc->nr_to_scan; |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 832 | unsigned start, flags; |
| 833 | int srcu_idx; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 834 | |
Kent Overstreet | 0196eb8 | 2022-10-14 06:48:23 -0400 | [diff] [blame] | 835 | mutex_lock(&bc->lock); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 836 | srcu_idx = srcu_read_lock(&c->btree_trans_barrier); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 837 | flags = memalloc_nofs_save(); |
| 838 | |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 839 | /* |
| 840 | * Newest freed entries are at the end of the list - once we hit one |
| 841 | * that's too new to be freed, we can bail out: |
| 842 | */ |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 843 | scanned += bc->nr_freed_nonpcpu; |
| 844 | |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 845 | list_for_each_entry_safe(ck, t, &bc->freed_nonpcpu, list) { |
| 846 | if (!poll_state_synchronize_srcu(&c->btree_trans_barrier, |
| 847 | ck->btree_trans_barrier_seq)) |
| 848 | break; |
| 849 | |
| 850 | list_del(&ck->list); |
Kent Overstreet | 0d2234a | 2023-05-20 20:57:55 -0400 | [diff] [blame] | 851 | six_lock_exit(&ck->c.lock); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 852 | kmem_cache_free(bch2_key_cache, ck); |
| 853 | atomic_long_dec(&bc->nr_freed); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 854 | freed++; |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 855 | bc->nr_freed_nonpcpu--; |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 856 | } |
| 857 | |
| 858 | if (scanned >= nr) |
| 859 | goto out; |
| 860 | |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 861 | scanned += bc->nr_freed_pcpu; |
| 862 | |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 863 | list_for_each_entry_safe(ck, t, &bc->freed_pcpu, list) { |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 864 | if (!poll_state_synchronize_srcu(&c->btree_trans_barrier, |
| 865 | ck->btree_trans_barrier_seq)) |
| 866 | break; |
| 867 | |
| 868 | list_del(&ck->list); |
Kent Overstreet | 0d2234a | 2023-05-20 20:57:55 -0400 | [diff] [blame] | 869 | six_lock_exit(&ck->c.lock); |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 870 | kmem_cache_free(bch2_key_cache, ck); |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 871 | atomic_long_dec(&bc->nr_freed); |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 872 | freed++; |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 873 | bc->nr_freed_pcpu--; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 874 | } |
| 875 | |
Kent Overstreet | 1259072 | 2020-11-19 15:38:27 -0500 | [diff] [blame] | 876 | if (scanned >= nr) |
| 877 | goto out; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 878 | |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 879 | rcu_read_lock(); |
| 880 | tbl = rht_dereference_rcu(bc->table.tbl, &bc->table); |
| 881 | if (bc->shrink_iter >= tbl->size) |
| 882 | bc->shrink_iter = 0; |
| 883 | start = bc->shrink_iter; |
| 884 | |
| 885 | do { |
| 886 | struct rhash_head *pos, *next; |
| 887 | |
| 888 | pos = rht_ptr_rcu(rht_bucket(tbl, bc->shrink_iter)); |
| 889 | |
| 890 | while (!rht_is_a_nulls(pos)) { |
| 891 | next = rht_dereference_bucket_rcu(pos->next, tbl, bc->shrink_iter); |
| 892 | ck = container_of(pos, struct bkey_cached, hash); |
| 893 | |
| 894 | if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) |
| 895 | goto next; |
| 896 | |
| 897 | if (test_bit(BKEY_CACHED_ACCESSED, &ck->flags)) |
| 898 | clear_bit(BKEY_CACHED_ACCESSED, &ck->flags); |
| 899 | else if (bkey_cached_lock_for_evict(ck)) { |
| 900 | bkey_cached_evict(bc, ck); |
| 901 | bkey_cached_free(bc, ck); |
| 902 | } |
| 903 | |
| 904 | scanned++; |
| 905 | if (scanned >= nr) |
| 906 | break; |
| 907 | next: |
| 908 | pos = next; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 909 | } |
| 910 | |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 911 | bc->shrink_iter++; |
| 912 | if (bc->shrink_iter >= tbl->size) |
| 913 | bc->shrink_iter = 0; |
| 914 | } while (scanned < nr && bc->shrink_iter != start); |
| 915 | |
| 916 | rcu_read_unlock(); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 917 | out: |
| 918 | memalloc_nofs_restore(flags); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 919 | srcu_read_unlock(&c->btree_trans_barrier, srcu_idx); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 920 | mutex_unlock(&bc->lock); |
| 921 | |
| 922 | return freed; |
| 923 | } |
| 924 | |
| 925 | static unsigned long bch2_btree_key_cache_count(struct shrinker *shrink, |
| 926 | struct shrink_control *sc) |
| 927 | { |
Linus Torvalds | ecae0bd | 2023-11-02 19:38:47 -1000 | [diff] [blame] | 928 | struct bch_fs *c = shrink->private_data; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 929 | struct btree_key_cache *bc = &c->btree_key_cache; |
Kent Overstreet | baa6502 | 2021-04-27 14:02:00 -0400 | [diff] [blame] | 930 | long nr = atomic_long_read(&bc->nr_keys) - |
| 931 | atomic_long_read(&bc->nr_dirty); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 932 | |
Kent Overstreet | baa6502 | 2021-04-27 14:02:00 -0400 | [diff] [blame] | 933 | return max(0L, nr); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 934 | } |
| 935 | |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 936 | void bch2_fs_btree_key_cache_exit(struct btree_key_cache *bc) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 937 | { |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 938 | struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 939 | struct bucket_table *tbl; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 940 | struct bkey_cached *ck, *n; |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 941 | struct rhash_head *pos; |
Kent Overstreet | 5b3008b | 2023-03-02 23:51:47 -0500 | [diff] [blame] | 942 | LIST_HEAD(items); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 943 | unsigned i; |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 944 | #ifdef __KERNEL__ |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 945 | int cpu; |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 946 | #endif |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 947 | |
Linus Torvalds | ecae0bd | 2023-11-02 19:38:47 -1000 | [diff] [blame] | 948 | shrinker_free(bc->shrink); |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 949 | |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 950 | mutex_lock(&bc->lock); |
Kent Overstreet | 6a747c4 | 2020-11-09 13:01:52 -0500 | [diff] [blame] | 951 | |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 952 | /* |
| 953 | * The loop is needed to guard against racing with rehash: |
| 954 | */ |
| 955 | while (atomic_long_read(&bc->nr_keys)) { |
| 956 | rcu_read_lock(); |
| 957 | tbl = rht_dereference_rcu(bc->table.tbl, &bc->table); |
| 958 | if (tbl) |
| 959 | for (i = 0; i < tbl->size; i++) |
| 960 | rht_for_each_entry_rcu(ck, pos, tbl, i, hash) { |
| 961 | bkey_cached_evict(bc, ck); |
Kent Overstreet | 5b3008b | 2023-03-02 23:51:47 -0500 | [diff] [blame] | 962 | list_add(&ck->list, &items); |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 963 | } |
| 964 | rcu_read_unlock(); |
| 965 | } |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 966 | |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 967 | #ifdef __KERNEL__ |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 968 | for_each_possible_cpu(cpu) { |
| 969 | struct btree_key_cache_freelist *f = |
| 970 | per_cpu_ptr(bc->pcpu_freed, cpu); |
| 971 | |
| 972 | for (i = 0; i < f->nr; i++) { |
| 973 | ck = f->objs[i]; |
Kent Overstreet | 5b3008b | 2023-03-02 23:51:47 -0500 | [diff] [blame] | 974 | list_add(&ck->list, &items); |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 975 | } |
| 976 | } |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 977 | #endif |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 978 | |
Kent Overstreet | c65c13f | 2023-11-06 09:53:14 -0500 | [diff] [blame] | 979 | BUG_ON(list_count_nodes(&bc->freed_pcpu) != bc->nr_freed_pcpu); |
| 980 | BUG_ON(list_count_nodes(&bc->freed_nonpcpu) != bc->nr_freed_nonpcpu); |
| 981 | |
Kent Overstreet | 5b3008b | 2023-03-02 23:51:47 -0500 | [diff] [blame] | 982 | list_splice(&bc->freed_pcpu, &items); |
| 983 | list_splice(&bc->freed_nonpcpu, &items); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 984 | |
Kent Overstreet | 5b3008b | 2023-03-02 23:51:47 -0500 | [diff] [blame] | 985 | mutex_unlock(&bc->lock); |
| 986 | |
| 987 | list_for_each_entry_safe(ck, n, &items, list) { |
Kent Overstreet | 1d8305c | 2020-12-13 16:12:04 -0500 | [diff] [blame] | 988 | cond_resched(); |
| 989 | |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 990 | list_del(&ck->list); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 991 | kfree(ck->k); |
Kent Overstreet | 0d2234a | 2023-05-20 20:57:55 -0400 | [diff] [blame] | 992 | six_lock_exit(&ck->c.lock); |
Kent Overstreet | 14ba370 | 2020-11-18 14:09:33 -0500 | [diff] [blame] | 993 | kmem_cache_free(bch2_key_cache, ck); |
| 994 | } |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 995 | |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 996 | if (atomic_long_read(&bc->nr_dirty) && |
| 997 | !bch2_journal_error(&c->journal) && |
Kent Overstreet | 3c471b6 | 2023-11-26 17:05:02 -0500 | [diff] [blame] | 998 | test_bit(BCH_FS_was_rw, &c->flags)) |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 999 | panic("btree key cache shutdown error: nr_dirty nonzero (%li)\n", |
| 1000 | atomic_long_read(&bc->nr_dirty)); |
| 1001 | |
| 1002 | if (atomic_long_read(&bc->nr_keys)) |
| 1003 | panic("btree key cache shutdown error: nr_keys nonzero (%li)\n", |
| 1004 | atomic_long_read(&bc->nr_keys)); |
Kent Overstreet | 331194a | 2021-03-24 23:37:33 -0400 | [diff] [blame] | 1005 | |
Kent Overstreet | d002229 | 2020-11-29 23:48:20 -0500 | [diff] [blame] | 1006 | if (bc->table_init_done) |
| 1007 | rhashtable_destroy(&bc->table); |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 1008 | |
| 1009 | free_percpu(bc->pcpu_freed); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 1010 | } |
| 1011 | |
| 1012 | void bch2_fs_btree_key_cache_init_early(struct btree_key_cache *c) |
| 1013 | { |
| 1014 | mutex_init(&c->lock); |
Kent Overstreet | 3d21d48 | 2022-09-03 22:07:31 -0400 | [diff] [blame] | 1015 | INIT_LIST_HEAD(&c->freed_pcpu); |
| 1016 | INIT_LIST_HEAD(&c->freed_nonpcpu); |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 1017 | } |
| 1018 | |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 1019 | int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 1020 | { |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 1021 | struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); |
Linus Torvalds | ecae0bd | 2023-11-02 19:38:47 -1000 | [diff] [blame] | 1022 | struct shrinker *shrink; |
Kent Overstreet | 628a3ad | 2020-11-12 17:19:47 -0500 | [diff] [blame] | 1023 | |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 1024 | #ifdef __KERNEL__ |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 1025 | bc->pcpu_freed = alloc_percpu(struct btree_key_cache_freelist); |
| 1026 | if (!bc->pcpu_freed) |
Kent Overstreet | 65d48e3 | 2023-03-14 15:35:57 -0400 | [diff] [blame] | 1027 | return -BCH_ERR_ENOMEM_fs_btree_cache_init; |
Kent Overstreet | fe5b37f | 2022-10-15 00:47:21 -0400 | [diff] [blame] | 1028 | #endif |
Kent Overstreet | 8f7f566 | 2022-06-17 01:07:54 -0400 | [diff] [blame] | 1029 | |
Kent Overstreet | 65d48e3 | 2023-03-14 15:35:57 -0400 | [diff] [blame] | 1030 | if (rhashtable_init(&bc->table, &bch2_btree_key_cache_params)) |
| 1031 | return -BCH_ERR_ENOMEM_fs_btree_cache_init; |
Kent Overstreet | d002229 | 2020-11-29 23:48:20 -0500 | [diff] [blame] | 1032 | |
| 1033 | bc->table_init_done = true; |
Kent Overstreet | f72b1fd | 2021-04-05 01:23:55 -0400 | [diff] [blame] | 1034 | |
Linus Torvalds | c9d0117 | 2023-11-07 11:38:38 -0800 | [diff] [blame] | 1035 | shrink = shrinker_alloc(0, "%s-btree_key_cache", c->name); |
Linus Torvalds | ecae0bd | 2023-11-02 19:38:47 -1000 | [diff] [blame] | 1036 | if (!shrink) |
Kent Overstreet | 65d48e3 | 2023-03-14 15:35:57 -0400 | [diff] [blame] | 1037 | return -BCH_ERR_ENOMEM_fs_btree_cache_init; |
Linus Torvalds | ecae0bd | 2023-11-02 19:38:47 -1000 | [diff] [blame] | 1038 | bc->shrink = shrink; |
| 1039 | shrink->seeks = 0; |
| 1040 | shrink->count_objects = bch2_btree_key_cache_count; |
| 1041 | shrink->scan_objects = bch2_btree_key_cache_scan; |
| 1042 | shrink->private_data = c; |
| 1043 | shrinker_register(shrink); |
Kent Overstreet | 65d48e3 | 2023-03-14 15:35:57 -0400 | [diff] [blame] | 1044 | return 0; |
Kent Overstreet | 2ca88e5 | 2019-03-07 19:46:10 -0500 | [diff] [blame] | 1045 | } |
Kent Overstreet | d211b40 | 2020-06-15 19:53:46 -0400 | [diff] [blame] | 1046 | |
| 1047 | void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *c) |
| 1048 | { |
Nathan Chancellor | f7ed15e | 2023-09-12 12:15:39 -0700 | [diff] [blame] | 1049 | prt_printf(out, "nr_freed:\t%lu", atomic_long_read(&c->nr_freed)); |
Kent Overstreet | b2f83e7 | 2022-10-17 02:08:07 -0400 | [diff] [blame] | 1050 | prt_newline(out); |
| 1051 | prt_printf(out, "nr_keys:\t%lu", atomic_long_read(&c->nr_keys)); |
| 1052 | prt_newline(out); |
| 1053 | prt_printf(out, "nr_dirty:\t%lu", atomic_long_read(&c->nr_dirty)); |
| 1054 | prt_newline(out); |
Kent Overstreet | d211b40 | 2020-06-15 19:53:46 -0400 | [diff] [blame] | 1055 | } |
Kent Overstreet | 14ba370 | 2020-11-18 14:09:33 -0500 | [diff] [blame] | 1056 | |
| 1057 | void bch2_btree_key_cache_exit(void) |
| 1058 | { |
Kent Overstreet | 3e3e02e | 2022-10-19 18:31:33 -0400 | [diff] [blame] | 1059 | kmem_cache_destroy(bch2_key_cache); |
Kent Overstreet | 14ba370 | 2020-11-18 14:09:33 -0500 | [diff] [blame] | 1060 | } |
| 1061 | |
| 1062 | int __init bch2_btree_key_cache_init(void) |
| 1063 | { |
Mikulas Patocka | 5eaa76d | 2023-07-13 18:00:28 +0200 | [diff] [blame] | 1064 | bch2_key_cache = KMEM_CACHE(bkey_cached, SLAB_RECLAIM_ACCOUNT); |
Kent Overstreet | 14ba370 | 2020-11-18 14:09:33 -0500 | [diff] [blame] | 1065 | if (!bch2_key_cache) |
| 1066 | return -ENOMEM; |
| 1067 | |
| 1068 | return 0; |
| 1069 | } |