Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | |
| 3 | #include "bcachefs.h" |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 4 | #include "alloc_background.h" |
Kent Overstreet | 7b3f84e | 2018-10-06 00:46:55 -0400 | [diff] [blame] | 5 | #include "alloc_foreground.h" |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 6 | #include "backpointers.h" |
Kent Overstreet | 07a1006 | 2020-12-17 15:08:58 -0500 | [diff] [blame] | 7 | #include "bkey_buf.h" |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 8 | #include "btree_gc.h" |
Kent Overstreet | ec4edd7 | 2024-01-16 13:29:59 -0500 | [diff] [blame] | 9 | #include "btree_io.h" |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 10 | #include "btree_update.h" |
Kent Overstreet | 7ef2a73 | 2019-01-21 15:32:13 -0500 | [diff] [blame] | 11 | #include "btree_update_interior.h" |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 12 | #include "btree_write_buffer.h" |
Kent Overstreet | 189c176 | 2024-01-15 15:33:39 -0500 | [diff] [blame] | 13 | #include "compress.h" |
Kent Overstreet | 4628529 | 2018-11-04 23:10:09 -0500 | [diff] [blame] | 14 | #include "disk_groups.h" |
Kent Overstreet | 961b2d6 | 2021-10-29 16:29:13 -0400 | [diff] [blame] | 15 | #include "ec.h" |
Kent Overstreet | d4bf5ee | 2022-07-18 19:42:58 -0400 | [diff] [blame] | 16 | #include "errcode.h" |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 17 | #include "error.h" |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 18 | #include "inode.h" |
Kent Overstreet | 1809b8c | 2023-09-10 18:05:17 -0400 | [diff] [blame] | 19 | #include "io_read.h" |
| 20 | #include "io_write.h" |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 21 | #include "journal_reclaim.h" |
| 22 | #include "keylist.h" |
| 23 | #include "move.h" |
| 24 | #include "replicas.h" |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 25 | #include "snapshot.h" |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 26 | #include "super-io.h" |
| 27 | #include "trace.h" |
| 28 | |
| 29 | #include <linux/ioprio.h> |
| 30 | #include <linux/kthread.h> |
| 31 | |
Kent Overstreet | 01e9564 | 2023-11-20 18:43:48 -0500 | [diff] [blame] | 32 | const char * const bch2_data_ops_strs[] = { |
| 33 | #define x(t, n, ...) [n] = #t, |
| 34 | BCH_DATA_OPS() |
| 35 | #undef x |
| 36 | NULL |
| 37 | }; |
| 38 | |
Kent Overstreet | 189c176 | 2024-01-15 15:33:39 -0500 | [diff] [blame] | 39 | static void trace_move_extent2(struct bch_fs *c, struct bkey_s_c k, |
| 40 | struct bch_io_opts *io_opts, |
| 41 | struct data_update_opts *data_opts) |
Kent Overstreet | 5a21764 | 2023-04-20 15:24:07 -0400 | [diff] [blame] | 42 | { |
| 43 | if (trace_move_extent_enabled()) { |
| 44 | struct printbuf buf = PRINTBUF; |
| 45 | |
| 46 | bch2_bkey_val_to_text(&buf, c, k); |
Kent Overstreet | 189c176 | 2024-01-15 15:33:39 -0500 | [diff] [blame] | 47 | prt_newline(&buf); |
| 48 | bch2_data_update_opts_to_text(&buf, c, io_opts, data_opts); |
Kent Overstreet | 5a21764 | 2023-04-20 15:24:07 -0400 | [diff] [blame] | 49 | trace_move_extent(c, buf.buf); |
| 50 | printbuf_exit(&buf); |
| 51 | } |
| 52 | } |
| 53 | |
| 54 | static void trace_move_extent_read2(struct bch_fs *c, struct bkey_s_c k) |
| 55 | { |
| 56 | if (trace_move_extent_read_enabled()) { |
| 57 | struct printbuf buf = PRINTBUF; |
| 58 | |
| 59 | bch2_bkey_val_to_text(&buf, c, k); |
| 60 | trace_move_extent_read(c, buf.buf); |
| 61 | printbuf_exit(&buf); |
| 62 | } |
| 63 | } |
| 64 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 65 | struct moving_io { |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 66 | struct list_head read_list; |
| 67 | struct list_head io_list; |
Kent Overstreet | 8fcdf81 | 2023-02-27 22:58:01 -0500 | [diff] [blame] | 68 | struct move_bucket_in_flight *b; |
| 69 | struct closure cl; |
| 70 | bool read_completed; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 71 | |
Kent Overstreet | 8fcdf81 | 2023-02-27 22:58:01 -0500 | [diff] [blame] | 72 | unsigned read_sectors; |
| 73 | unsigned write_sectors; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 74 | |
Kent Overstreet | 8fcdf81 | 2023-02-27 22:58:01 -0500 | [diff] [blame] | 75 | struct bch_read_bio rbio; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 76 | |
Kent Overstreet | 8fcdf81 | 2023-02-27 22:58:01 -0500 | [diff] [blame] | 77 | struct data_update write; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 78 | /* Must be last since it is variable size */ |
Gustavo A. R. Silva | 62286a0 | 2023-11-28 12:22:55 -0600 | [diff] [blame] | 79 | struct bio_vec bi_inline_vecs[]; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 80 | }; |
| 81 | |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 82 | static void move_free(struct moving_io *io) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 83 | { |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 84 | struct moving_context *ctxt = io->write.ctxt; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 85 | |
Kent Overstreet | 8fcdf81 | 2023-02-27 22:58:01 -0500 | [diff] [blame] | 86 | if (io->b) |
| 87 | atomic_dec(&io->b->count); |
| 88 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 89 | bch2_data_update_exit(&io->write); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 90 | |
| 91 | mutex_lock(&ctxt->lock); |
| 92 | list_del(&io->io_list); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 93 | wake_up(&ctxt->wait); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 94 | mutex_unlock(&ctxt->lock); |
| 95 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 96 | kfree(io); |
| 97 | } |
| 98 | |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 99 | static void move_write_done(struct bch_write_op *op) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 100 | { |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 101 | struct moving_io *io = container_of(op, struct moving_io, write.op); |
| 102 | struct moving_context *ctxt = io->write.ctxt; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 103 | |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 104 | if (io->write.op.error) |
| 105 | ctxt->write_error = true; |
| 106 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 107 | atomic_sub(io->write_sectors, &io->write.ctxt->write_sectors); |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 108 | atomic_dec(&io->write.ctxt->write_ios); |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 109 | move_free(io); |
| 110 | closure_put(&ctxt->cl); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 111 | } |
| 112 | |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 113 | static void move_write(struct moving_io *io) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 114 | { |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 115 | if (unlikely(io->rbio.bio.bi_status || io->rbio.hole)) { |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 116 | move_free(io); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 117 | return; |
| 118 | } |
| 119 | |
Kent Overstreet | fa3185a | 2024-01-15 15:04:40 -0500 | [diff] [blame] | 120 | if (trace_move_extent_write_enabled()) { |
| 121 | struct bch_fs *c = io->write.op.c; |
| 122 | struct printbuf buf = PRINTBUF; |
| 123 | |
| 124 | bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(io->write.k.k)); |
| 125 | trace_move_extent_write(c, buf.buf); |
| 126 | printbuf_exit(&buf); |
| 127 | } |
| 128 | |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 129 | closure_get(&io->write.ctxt->cl); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 130 | atomic_add(io->write_sectors, &io->write.ctxt->write_sectors); |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 131 | atomic_inc(&io->write.ctxt->write_ios); |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 132 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 133 | bch2_data_update_read_done(&io->write, io->rbio.pick.crc); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 134 | } |
| 135 | |
Kent Overstreet | 7ffb6a7 | 2023-01-02 17:53:02 -0500 | [diff] [blame] | 136 | struct moving_io *bch2_moving_ctxt_next_pending_write(struct moving_context *ctxt) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 137 | { |
| 138 | struct moving_io *io = |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 139 | list_first_entry_or_null(&ctxt->reads, struct moving_io, read_list); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 140 | |
| 141 | return io && io->read_completed ? io : NULL; |
| 142 | } |
| 143 | |
| 144 | static void move_read_endio(struct bio *bio) |
| 145 | { |
| 146 | struct moving_io *io = container_of(bio, struct moving_io, rbio.bio); |
| 147 | struct moving_context *ctxt = io->write.ctxt; |
| 148 | |
| 149 | atomic_sub(io->read_sectors, &ctxt->read_sectors); |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 150 | atomic_dec(&ctxt->read_ios); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 151 | io->read_completed = true; |
| 152 | |
Kent Overstreet | f61816d | 2022-02-21 13:22:11 -0500 | [diff] [blame] | 153 | wake_up(&ctxt->wait); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 154 | closure_put(&ctxt->cl); |
| 155 | } |
| 156 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 157 | void bch2_moving_ctxt_do_pending_writes(struct moving_context *ctxt) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 158 | { |
| 159 | struct moving_io *io; |
| 160 | |
Kent Overstreet | 7ffb6a7 | 2023-01-02 17:53:02 -0500 | [diff] [blame] | 161 | while ((io = bch2_moving_ctxt_next_pending_write(ctxt))) { |
Kent Overstreet | f82755e | 2023-10-30 15:13:09 -0400 | [diff] [blame] | 162 | bch2_trans_unlock_long(ctxt->trans); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 163 | list_del(&io->read_list); |
Kent Overstreet | 9f311f2 | 2022-10-29 02:47:33 -0400 | [diff] [blame] | 164 | move_write(io); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 165 | } |
| 166 | } |
| 167 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 168 | void bch2_move_ctxt_wait_for_io(struct moving_context *ctxt) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 169 | { |
| 170 | unsigned sectors_pending = atomic_read(&ctxt->write_sectors); |
| 171 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 172 | move_ctxt_wait_event(ctxt, |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 173 | !atomic_read(&ctxt->write_sectors) || |
| 174 | atomic_read(&ctxt->write_sectors) != sectors_pending); |
| 175 | } |
| 176 | |
Daniel Hill | 0c06978 | 2023-11-26 19:33:31 +1300 | [diff] [blame] | 177 | void bch2_moving_ctxt_flush_all(struct moving_context *ctxt) |
Kent Overstreet | 50e029c | 2023-11-20 17:24:32 -0500 | [diff] [blame] | 178 | { |
| 179 | move_ctxt_wait_event(ctxt, list_empty(&ctxt->reads)); |
| 180 | bch2_trans_unlock_long(ctxt->trans); |
| 181 | closure_sync(&ctxt->cl); |
| 182 | } |
| 183 | |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 184 | void bch2_moving_ctxt_exit(struct moving_context *ctxt) |
| 185 | { |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 186 | struct bch_fs *c = ctxt->trans->c; |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 187 | |
Kent Overstreet | 50e029c | 2023-11-20 17:24:32 -0500 | [diff] [blame] | 188 | bch2_moving_ctxt_flush_all(ctxt); |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 189 | |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 190 | EBUG_ON(atomic_read(&ctxt->write_sectors)); |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 191 | EBUG_ON(atomic_read(&ctxt->write_ios)); |
| 192 | EBUG_ON(atomic_read(&ctxt->read_sectors)); |
| 193 | EBUG_ON(atomic_read(&ctxt->read_ios)); |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 194 | |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 195 | mutex_lock(&c->moving_context_lock); |
| 196 | list_del(&ctxt->list); |
| 197 | mutex_unlock(&c->moving_context_lock); |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 198 | |
| 199 | bch2_trans_put(ctxt->trans); |
| 200 | memset(ctxt, 0, sizeof(*ctxt)); |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 201 | } |
| 202 | |
| 203 | void bch2_moving_ctxt_init(struct moving_context *ctxt, |
| 204 | struct bch_fs *c, |
| 205 | struct bch_ratelimit *rate, |
| 206 | struct bch_move_stats *stats, |
| 207 | struct write_point_specifier wp, |
| 208 | bool wait_on_copygc) |
| 209 | { |
| 210 | memset(ctxt, 0, sizeof(*ctxt)); |
| 211 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 212 | ctxt->trans = bch2_trans_get(c); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 213 | ctxt->fn = (void *) _RET_IP_; |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 214 | ctxt->rate = rate; |
| 215 | ctxt->stats = stats; |
| 216 | ctxt->wp = wp; |
| 217 | ctxt->wait_on_copygc = wait_on_copygc; |
| 218 | |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 219 | closure_init_stack(&ctxt->cl); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 220 | |
| 221 | mutex_init(&ctxt->lock); |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 222 | INIT_LIST_HEAD(&ctxt->reads); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 223 | INIT_LIST_HEAD(&ctxt->ios); |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 224 | init_waitqueue_head(&ctxt->wait); |
| 225 | |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 226 | mutex_lock(&c->moving_context_lock); |
| 227 | list_add(&ctxt->list, &c->moving_context_list); |
| 228 | mutex_unlock(&c->moving_context_lock); |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 229 | } |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 230 | |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 231 | void bch2_move_stats_exit(struct bch_move_stats *stats, struct bch_fs *c) |
| 232 | { |
| 233 | trace_move_data(c, stats); |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 234 | } |
| 235 | |
Kent Overstreet | 01e9564 | 2023-11-20 18:43:48 -0500 | [diff] [blame] | 236 | void bch2_move_stats_init(struct bch_move_stats *stats, const char *name) |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 237 | { |
| 238 | memset(stats, 0, sizeof(*stats)); |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 239 | stats->data_type = BCH_DATA_user; |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 240 | scnprintf(stats->name, sizeof(stats->name), "%s", name); |
| 241 | } |
| 242 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 243 | int bch2_move_extent(struct moving_context *ctxt, |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 244 | struct move_bucket_in_flight *bucket_in_flight, |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 245 | struct btree_iter *iter, |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 246 | struct bkey_s_c k, |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 247 | struct bch_io_opts io_opts, |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 248 | struct data_update_opts data_opts) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 249 | { |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 250 | struct btree_trans *trans = ctxt->trans; |
Kent Overstreet | f30dd86 | 2020-10-16 21:39:16 -0400 | [diff] [blame] | 251 | struct bch_fs *c = trans->c; |
Kent Overstreet | 99aaf57 | 2019-07-25 13:52:14 -0400 | [diff] [blame] | 252 | struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 253 | struct moving_io *io; |
Kent Overstreet | 1742237 | 2018-09-27 21:08:39 -0400 | [diff] [blame] | 254 | const union bch_extent_entry *entry; |
| 255 | struct extent_ptr_decoded p; |
Kent Overstreet | 99aaf57 | 2019-07-25 13:52:14 -0400 | [diff] [blame] | 256 | unsigned sectors = k.k->size, pages; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 257 | int ret = -ENOMEM; |
| 258 | |
Kent Overstreet | 189c176 | 2024-01-15 15:33:39 -0500 | [diff] [blame] | 259 | trace_move_extent2(c, k, &io_opts, &data_opts); |
| 260 | |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 261 | if (ctxt->stats) |
| 262 | ctxt->stats->pos = BBPOS(iter->btree_id, iter->pos); |
Kent Overstreet | 5a21764 | 2023-04-20 15:24:07 -0400 | [diff] [blame] | 263 | |
Kent Overstreet | 1be8879 | 2022-10-09 03:32:17 -0400 | [diff] [blame] | 264 | bch2_data_update_opts_normalize(k, &data_opts); |
| 265 | |
| 266 | if (!data_opts.rewrite_ptrs && |
| 267 | !data_opts.extra_replicas) { |
| 268 | if (data_opts.kill_ptrs) |
| 269 | return bch2_extent_drop_ptrs(trans, iter, k, data_opts); |
| 270 | return 0; |
| 271 | } |
| 272 | |
Kent Overstreet | a8b3a67 | 2022-11-02 17:12:00 -0400 | [diff] [blame] | 273 | /* |
| 274 | * Before memory allocations & taking nocow locks in |
| 275 | * bch2_data_update_init(): |
| 276 | */ |
| 277 | bch2_trans_unlock(trans); |
| 278 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 279 | /* write path might have to decompress data: */ |
Kent Overstreet | 99aaf57 | 2019-07-25 13:52:14 -0400 | [diff] [blame] | 280 | bkey_for_each_ptr_decode(k.k, ptrs, p, entry) |
Kent Overstreet | 1742237 | 2018-09-27 21:08:39 -0400 | [diff] [blame] | 281 | sectors = max_t(unsigned, sectors, p.crc.uncompressed_size); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 282 | |
| 283 | pages = DIV_ROUND_UP(sectors, PAGE_SECTORS); |
| 284 | io = kzalloc(sizeof(struct moving_io) + |
| 285 | sizeof(struct bio_vec) * pages, GFP_KERNEL); |
| 286 | if (!io) |
| 287 | goto err; |
| 288 | |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 289 | INIT_LIST_HEAD(&io->io_list); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 290 | io->write.ctxt = ctxt; |
Kent Overstreet | 99aaf57 | 2019-07-25 13:52:14 -0400 | [diff] [blame] | 291 | io->read_sectors = k.k->size; |
| 292 | io->write_sectors = k.k->size; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 293 | |
| 294 | bio_init(&io->write.op.wbio.bio, NULL, io->bi_inline_vecs, pages, 0); |
| 295 | bio_set_prio(&io->write.op.wbio.bio, |
| 296 | IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); |
| 297 | |
| 298 | if (bch2_bio_alloc_pages(&io->write.op.wbio.bio, sectors << 9, |
| 299 | GFP_KERNEL)) |
| 300 | goto err_free; |
| 301 | |
Kent Overstreet | b50dd79 | 2019-09-07 13:16:41 -0400 | [diff] [blame] | 302 | io->rbio.c = c; |
| 303 | io->rbio.opts = io_opts; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 304 | bio_init(&io->rbio.bio, NULL, io->bi_inline_vecs, pages, 0); |
| 305 | io->rbio.bio.bi_vcnt = pages; |
| 306 | bio_set_prio(&io->rbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); |
| 307 | io->rbio.bio.bi_iter.bi_size = sectors << 9; |
| 308 | |
| 309 | io->rbio.bio.bi_opf = REQ_OP_READ; |
Kent Overstreet | 99aaf57 | 2019-07-25 13:52:14 -0400 | [diff] [blame] | 310 | io->rbio.bio.bi_iter.bi_sector = bkey_start_offset(k.k); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 311 | io->rbio.bio.bi_end_io = move_read_endio; |
| 312 | |
Kent Overstreet | 7d9f846 | 2023-11-24 21:51:45 -0500 | [diff] [blame] | 313 | ret = bch2_data_update_init(trans, iter, ctxt, &io->write, ctxt->wp, |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 314 | io_opts, data_opts, iter->btree_id, k); |
Kent Overstreet | 7d9f846 | 2023-11-24 21:51:45 -0500 | [diff] [blame] | 315 | if (ret) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 316 | goto err_free_pages; |
| 317 | |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 318 | io->write.op.end_io = move_write_done; |
| 319 | |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 320 | if (ctxt->rate) |
| 321 | bch2_ratelimit_increment(ctxt->rate, k.k->size); |
| 322 | |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 323 | if (ctxt->stats) { |
| 324 | atomic64_inc(&ctxt->stats->keys_moved); |
| 325 | atomic64_add(k.k->size, &ctxt->stats->sectors_moved); |
| 326 | } |
| 327 | |
Kent Overstreet | 8fcdf81 | 2023-02-27 22:58:01 -0500 | [diff] [blame] | 328 | if (bucket_in_flight) { |
| 329 | io->b = bucket_in_flight; |
| 330 | atomic_inc(&io->b->count); |
| 331 | } |
| 332 | |
Daniel Hill | 104c697 | 2022-03-15 21:36:33 +1300 | [diff] [blame] | 333 | this_cpu_add(c->counters[BCH_COUNTER_io_move], k.k->size); |
Kent Overstreet | 674cfc2 | 2022-08-27 12:48:36 -0400 | [diff] [blame] | 334 | this_cpu_add(c->counters[BCH_COUNTER_move_extent_read], k.k->size); |
Kent Overstreet | 5a21764 | 2023-04-20 15:24:07 -0400 | [diff] [blame] | 335 | trace_move_extent_read2(c, k); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 336 | |
| 337 | mutex_lock(&ctxt->lock); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 338 | atomic_add(io->read_sectors, &ctxt->read_sectors); |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 339 | atomic_inc(&ctxt->read_ios); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 340 | |
| 341 | list_add_tail(&io->read_list, &ctxt->reads); |
| 342 | list_add_tail(&io->io_list, &ctxt->ios); |
| 343 | mutex_unlock(&ctxt->lock); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 344 | |
| 345 | /* |
| 346 | * dropped by move_read_endio() - guards against use after free of |
| 347 | * ctxt when doing wakeup |
| 348 | */ |
| 349 | closure_get(&ctxt->cl); |
Kent Overstreet | 5ff75cc | 2021-03-14 21:30:08 -0400 | [diff] [blame] | 350 | bch2_read_extent(trans, &io->rbio, |
| 351 | bkey_start_pos(k.k), |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 352 | iter->btree_id, k, 0, |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 353 | BCH_READ_NODECODE| |
| 354 | BCH_READ_LAST_FRAGMENT); |
| 355 | return 0; |
| 356 | err_free_pages: |
| 357 | bio_free_pages(&io->write.op.wbio.bio); |
| 358 | err_free: |
| 359 | kfree(io); |
| 360 | err: |
Kent Overstreet | 7d9f846 | 2023-11-24 21:51:45 -0500 | [diff] [blame] | 361 | if (ret == -BCH_ERR_data_update_done) |
| 362 | return 0; |
| 363 | |
Kent Overstreet | 1b1bd0f | 2023-11-26 23:11:18 -0500 | [diff] [blame] | 364 | if (bch2_err_matches(ret, EROFS) || |
| 365 | bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 366 | return ret; |
| 367 | |
Kent Overstreet | 7464403 | 2023-11-27 22:37:27 -0500 | [diff] [blame] | 368 | count_event(c, move_extent_start_fail); |
| 369 | |
Kent Overstreet | ae4d612 | 2023-11-26 21:13:54 -0500 | [diff] [blame] | 370 | if (trace_move_extent_start_fail_enabled()) { |
| 371 | struct printbuf buf = PRINTBUF; |
| 372 | |
| 373 | bch2_bkey_val_to_text(&buf, c, k); |
| 374 | prt_str(&buf, ": "); |
| 375 | prt_str(&buf, bch2_err_str(ret)); |
| 376 | trace_move_extent_start_fail(c, buf.buf); |
| 377 | printbuf_exit(&buf); |
| 378 | } |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 379 | return ret; |
| 380 | } |
| 381 | |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 382 | struct bch_io_opts *bch2_move_get_io_opts(struct btree_trans *trans, |
| 383 | struct per_snapshot_io_opts *io_opts, |
| 384 | struct bkey_s_c extent_k) |
| 385 | { |
| 386 | struct bch_fs *c = trans->c; |
| 387 | u32 restart_count = trans->restart_count; |
| 388 | int ret = 0; |
| 389 | |
| 390 | if (io_opts->cur_inum != extent_k.k->p.inode) { |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 391 | io_opts->d.nr = 0; |
| 392 | |
Kent Overstreet | 5028b90 | 2023-12-07 23:33:11 -0500 | [diff] [blame] | 393 | ret = for_each_btree_key(trans, iter, BTREE_ID_inodes, POS(0, extent_k.k->p.inode), |
Kent Overstreet | 5dd8c60 | 2024-04-07 18:05:34 -0400 | [diff] [blame] | 394 | BTREE_ITER_all_snapshots, k, ({ |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 395 | if (k.k->p.offset != extent_k.k->p.inode) |
| 396 | break; |
| 397 | |
| 398 | if (!bkey_is_inode(k.k)) |
| 399 | continue; |
| 400 | |
| 401 | struct bch_inode_unpacked inode; |
| 402 | BUG_ON(bch2_inode_unpack(k, &inode)); |
| 403 | |
| 404 | struct snapshot_io_opts_entry e = { .snapshot = k.k->p.snapshot }; |
| 405 | bch2_inode_opts_get(&e.io_opts, trans->c, &inode); |
| 406 | |
Kent Overstreet | 27b2df9 | 2023-12-07 23:28:26 -0500 | [diff] [blame] | 407 | darray_push(&io_opts->d, e); |
| 408 | })); |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 409 | io_opts->cur_inum = extent_k.k->p.inode; |
| 410 | } |
| 411 | |
| 412 | ret = ret ?: trans_was_restarted(trans, restart_count); |
| 413 | if (ret) |
| 414 | return ERR_PTR(ret); |
| 415 | |
Kent Overstreet | defd9e3 | 2023-12-16 21:40:26 -0500 | [diff] [blame] | 416 | if (extent_k.k->p.snapshot) |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 417 | darray_for_each(io_opts->d, i) |
| 418 | if (bch2_snapshot_is_ancestor(c, extent_k.k->p.snapshot, i->snapshot)) |
| 419 | return &i->io_opts; |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 420 | |
| 421 | return &io_opts->fs_io_opts; |
| 422 | } |
| 423 | |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 424 | int bch2_move_get_io_opts_one(struct btree_trans *trans, |
| 425 | struct bch_io_opts *io_opts, |
| 426 | struct bkey_s_c extent_k) |
Kent Overstreet | 883d970 | 2021-03-16 18:08:10 -0400 | [diff] [blame] | 427 | { |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 428 | struct btree_iter iter; |
Kent Overstreet | 883d970 | 2021-03-16 18:08:10 -0400 | [diff] [blame] | 429 | struct bkey_s_c k; |
| 430 | int ret; |
| 431 | |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 432 | /* reflink btree? */ |
| 433 | if (!extent_k.k->p.inode) { |
| 434 | *io_opts = bch2_opts_to_inode_opts(trans->c->opts); |
| 435 | return 0; |
Kent Overstreet | 443d276 | 2021-05-23 18:42:51 -0400 | [diff] [blame] | 436 | } |
| 437 | |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 438 | k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes, |
| 439 | SPOS(0, extent_k.k->p.inode, extent_k.k->p.snapshot), |
Kent Overstreet | 5dd8c60 | 2024-04-07 18:05:34 -0400 | [diff] [blame] | 440 | BTREE_ITER_cached); |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 441 | ret = bkey_err(k); |
| 442 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 443 | return ret; |
Kent Overstreet | 883d970 | 2021-03-16 18:08:10 -0400 | [diff] [blame] | 444 | |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 445 | if (!ret && bkey_is_inode(k.k)) { |
| 446 | struct bch_inode_unpacked inode; |
| 447 | bch2_inode_unpack(k, &inode); |
| 448 | bch2_inode_opts_get(io_opts, trans->c, &inode); |
| 449 | } else { |
| 450 | *io_opts = bch2_opts_to_inode_opts(trans->c->opts); |
| 451 | } |
| 452 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 453 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 454 | return 0; |
Kent Overstreet | 883d970 | 2021-03-16 18:08:10 -0400 | [diff] [blame] | 455 | } |
| 456 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 457 | int bch2_move_ratelimit(struct moving_context *ctxt) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 458 | { |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 459 | struct bch_fs *c = ctxt->trans->c; |
Kent Overstreet | 415e510 | 2023-11-28 16:33:52 -0500 | [diff] [blame] | 460 | bool is_kthread = current->flags & PF_KTHREAD; |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 461 | u64 delay; |
| 462 | |
Kent Overstreet | 50e029c | 2023-11-20 17:24:32 -0500 | [diff] [blame] | 463 | if (ctxt->wait_on_copygc && c->copygc_running) { |
| 464 | bch2_moving_ctxt_flush_all(ctxt); |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 465 | wait_event_killable(c->copygc_running_wq, |
| 466 | !c->copygc_running || |
Kent Overstreet | 415e510 | 2023-11-28 16:33:52 -0500 | [diff] [blame] | 467 | (is_kthread && kthread_should_stop())); |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 468 | } |
| 469 | |
| 470 | do { |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 471 | delay = ctxt->rate ? bch2_ratelimit_delay(ctxt->rate) : 0; |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 472 | |
Kent Overstreet | 415e510 | 2023-11-28 16:33:52 -0500 | [diff] [blame] | 473 | if (is_kthread && kthread_should_stop()) |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 474 | return 1; |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 475 | |
| 476 | if (delay) |
Kent Overstreet | 261af2f | 2023-11-22 23:44:47 -0500 | [diff] [blame] | 477 | move_ctxt_wait_event_timeout(ctxt, |
Kent Overstreet | 415e510 | 2023-11-28 16:33:52 -0500 | [diff] [blame] | 478 | freezing(current) || |
| 479 | (is_kthread && kthread_should_stop()), |
Kent Overstreet | 261af2f | 2023-11-22 23:44:47 -0500 | [diff] [blame] | 480 | delay); |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 481 | |
| 482 | if (unlikely(freezing(current))) { |
Kent Overstreet | 50e029c | 2023-11-20 17:24:32 -0500 | [diff] [blame] | 483 | bch2_moving_ctxt_flush_all(ctxt); |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 484 | try_to_freeze(); |
| 485 | } |
| 486 | } while (delay); |
| 487 | |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 488 | /* |
| 489 | * XXX: these limits really ought to be per device, SSDs and hard drives |
| 490 | * will want different limits |
| 491 | */ |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 492 | move_ctxt_wait_event(ctxt, |
Kent Overstreet | c782c58 | 2023-01-09 01:45:18 -0500 | [diff] [blame] | 493 | atomic_read(&ctxt->write_sectors) < c->opts.move_bytes_in_flight >> 9 && |
| 494 | atomic_read(&ctxt->read_sectors) < c->opts.move_bytes_in_flight >> 9 && |
| 495 | atomic_read(&ctxt->write_ios) < c->opts.move_ios_in_flight && |
| 496 | atomic_read(&ctxt->read_ios) < c->opts.move_ios_in_flight); |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 497 | |
| 498 | return 0; |
| 499 | } |
| 500 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 501 | static int bch2_move_data_btree(struct moving_context *ctxt, |
| 502 | struct bpos start, |
| 503 | struct bpos end, |
| 504 | move_pred_fn pred, void *arg, |
| 505 | enum btree_id btree_id) |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 506 | { |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 507 | struct btree_trans *trans = ctxt->trans; |
| 508 | struct bch_fs *c = trans->c; |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 509 | struct per_snapshot_io_opts snapshot_io_opts; |
| 510 | struct bch_io_opts *io_opts; |
Kent Overstreet | 07a1006 | 2020-12-17 15:08:58 -0500 | [diff] [blame] | 511 | struct bkey_buf sk; |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 512 | struct btree_iter iter; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 513 | struct bkey_s_c k; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 514 | struct data_update_opts data_opts; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 515 | int ret = 0, ret2; |
| 516 | |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 517 | per_snapshot_io_opts_init(&snapshot_io_opts, c); |
Kent Overstreet | 07a1006 | 2020-12-17 15:08:58 -0500 | [diff] [blame] | 518 | bch2_bkey_buf_init(&sk); |
Kent Overstreet | 424eb88 | 2019-03-25 15:10:15 -0400 | [diff] [blame] | 519 | |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 520 | if (ctxt->stats) { |
| 521 | ctxt->stats->data_type = BCH_DATA_user; |
Kent Overstreet | d5eade9 | 2023-10-23 15:36:45 -0400 | [diff] [blame] | 522 | ctxt->stats->pos = BBPOS(btree_id, start); |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 523 | } |
Kent Overstreet | 424eb88 | 2019-03-25 15:10:15 -0400 | [diff] [blame] | 524 | |
Kent Overstreet | 319fef2 | 2024-06-03 18:00:48 -0400 | [diff] [blame] | 525 | bch2_trans_begin(trans); |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 526 | bch2_trans_iter_init(trans, &iter, btree_id, start, |
Kent Overstreet | 5dd8c60 | 2024-04-07 18:05:34 -0400 | [diff] [blame] | 527 | BTREE_ITER_prefetch| |
| 528 | BTREE_ITER_all_snapshots); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 529 | |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 530 | if (ctxt->rate) |
| 531 | bch2_ratelimit_reset(ctxt->rate); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 532 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 533 | while (!bch2_move_ratelimit(ctxt)) { |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 534 | bch2_trans_begin(trans); |
Kent Overstreet | 700c25b | 2021-07-24 20:24:10 -0400 | [diff] [blame] | 535 | |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 536 | k = bch2_btree_iter_peek(&iter); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 537 | if (!k.k) |
| 538 | break; |
Kent Overstreet | 8ede991 | 2022-01-09 20:52:10 -0500 | [diff] [blame] | 539 | |
Kent Overstreet | 0f23836 | 2019-03-27 22:03:30 -0400 | [diff] [blame] | 540 | ret = bkey_err(k); |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 541 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
Kent Overstreet | 8ede991 | 2022-01-09 20:52:10 -0500 | [diff] [blame] | 542 | continue; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 543 | if (ret) |
| 544 | break; |
Kent Overstreet | 8ede991 | 2022-01-09 20:52:10 -0500 | [diff] [blame] | 545 | |
Kent Overstreet | e88a75e | 2022-11-24 03:12:22 -0500 | [diff] [blame] | 546 | if (bkey_ge(bkey_start_pos(k.k), end)) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 547 | break; |
| 548 | |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 549 | if (ctxt->stats) |
Kent Overstreet | d5eade9 | 2023-10-23 15:36:45 -0400 | [diff] [blame] | 550 | ctxt->stats->pos = BBPOS(iter.btree_id, iter.pos); |
Kent Overstreet | 8ede991 | 2022-01-09 20:52:10 -0500 | [diff] [blame] | 551 | |
Kent Overstreet | 8d84260 | 2019-09-07 16:13:20 -0400 | [diff] [blame] | 552 | if (!bkey_extent_is_direct_data(k.k)) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 553 | goto next_nondata; |
| 554 | |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 555 | io_opts = bch2_move_get_io_opts(trans, &snapshot_io_opts, k); |
| 556 | ret = PTR_ERR_OR_ZERO(io_opts); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 557 | if (ret) |
| 558 | continue; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 559 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 560 | memset(&data_opts, 0, sizeof(data_opts)); |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 561 | if (!pred(c, arg, k, io_opts, &data_opts)) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 562 | goto next; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 563 | |
Kent Overstreet | eb331fe | 2022-02-15 00:06:59 -0500 | [diff] [blame] | 564 | /* |
| 565 | * The iterator gets unlocked by __bch2_read_extent - need to |
| 566 | * save a copy of @k elsewhere: |
Kent Overstreet | 3e3e02e | 2022-10-19 18:31:33 -0400 | [diff] [blame] | 567 | */ |
Kent Overstreet | 07a1006 | 2020-12-17 15:08:58 -0500 | [diff] [blame] | 568 | bch2_bkey_buf_reassemble(&sk, c, k); |
Kent Overstreet | 35189e0 | 2019-11-09 16:01:15 -0500 | [diff] [blame] | 569 | k = bkey_i_to_s_c(sk.k); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 570 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 571 | ret2 = bch2_move_extent(ctxt, NULL, &iter, k, *io_opts, data_opts); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 572 | if (ret2) { |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 573 | if (bch2_err_matches(ret2, BCH_ERR_transaction_restart)) |
Kent Overstreet | f0e7001 | 2020-12-20 21:42:19 -0500 | [diff] [blame] | 574 | continue; |
Kent Overstreet | f0e7001 | 2020-12-20 21:42:19 -0500 | [diff] [blame] | 575 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 576 | if (ret2 == -ENOMEM) { |
| 577 | /* memory allocation failure, wait for some IO to finish */ |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 578 | bch2_move_ctxt_wait_for_io(ctxt); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 579 | continue; |
| 580 | } |
| 581 | |
| 582 | /* XXX signal failure */ |
| 583 | goto next; |
| 584 | } |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 585 | next: |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 586 | if (ctxt->stats) |
| 587 | atomic64_add(k.k->size, &ctxt->stats->sectors_seen); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 588 | next_nondata: |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 589 | bch2_btree_iter_advance(&iter); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 590 | } |
Kent Overstreet | 50dc0f6 | 2021-03-19 20:29:11 -0400 | [diff] [blame] | 591 | |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 592 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 07a1006 | 2020-12-17 15:08:58 -0500 | [diff] [blame] | 593 | bch2_bkey_buf_exit(&sk, c); |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 594 | per_snapshot_io_opts_exit(&snapshot_io_opts); |
Kent Overstreet | 7642609 | 2019-08-16 09:59:56 -0400 | [diff] [blame] | 595 | |
| 596 | return ret; |
| 597 | } |
| 598 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 599 | int __bch2_move_data(struct moving_context *ctxt, |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 600 | struct bbpos start, |
| 601 | struct bbpos end, |
| 602 | move_pred_fn pred, void *arg) |
| 603 | { |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 604 | struct bch_fs *c = ctxt->trans->c; |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 605 | enum btree_id id; |
| 606 | int ret = 0; |
| 607 | |
| 608 | for (id = start.btree; |
| 609 | id <= min_t(unsigned, end.btree, btree_id_nr_alive(c) - 1); |
| 610 | id++) { |
Kent Overstreet | d5eade9 | 2023-10-23 15:36:45 -0400 | [diff] [blame] | 611 | ctxt->stats->pos = BBPOS(id, POS_MIN); |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 612 | |
| 613 | if (!btree_type_has_ptrs(id) || |
| 614 | !bch2_btree_id_root(c, id)->b) |
| 615 | continue; |
| 616 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 617 | ret = bch2_move_data_btree(ctxt, |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 618 | id == start.btree ? start.pos : POS_MIN, |
| 619 | id == end.btree ? end.pos : POS_MAX, |
| 620 | pred, arg, id); |
| 621 | if (ret) |
| 622 | break; |
| 623 | } |
| 624 | |
| 625 | return ret; |
| 626 | } |
| 627 | |
Kent Overstreet | 7642609 | 2019-08-16 09:59:56 -0400 | [diff] [blame] | 628 | int bch2_move_data(struct bch_fs *c, |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 629 | struct bbpos start, |
| 630 | struct bbpos end, |
Kent Overstreet | 7642609 | 2019-08-16 09:59:56 -0400 | [diff] [blame] | 631 | struct bch_ratelimit *rate, |
Daniel Hill | c91996c | 2022-06-16 02:06:43 +1200 | [diff] [blame] | 632 | struct bch_move_stats *stats, |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 633 | struct write_point_specifier wp, |
| 634 | bool wait_on_copygc, |
| 635 | move_pred_fn pred, void *arg) |
Kent Overstreet | 7642609 | 2019-08-16 09:59:56 -0400 | [diff] [blame] | 636 | { |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 637 | |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 638 | struct moving_context ctxt; |
Kent Overstreet | a0bfe3b | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 639 | int ret; |
Kent Overstreet | 7642609 | 2019-08-16 09:59:56 -0400 | [diff] [blame] | 640 | |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 641 | bch2_moving_ctxt_init(&ctxt, c, rate, stats, wp, wait_on_copygc); |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 642 | ret = __bch2_move_data(&ctxt, start, end, pred, arg); |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 643 | bch2_moving_ctxt_exit(&ctxt); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 644 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 645 | return ret; |
| 646 | } |
| 647 | |
Daniel Hill | 7452933 | 2023-11-26 20:26:07 +1300 | [diff] [blame] | 648 | int bch2_evacuate_bucket(struct moving_context *ctxt, |
Kent Overstreet | 8fcdf81 | 2023-02-27 22:58:01 -0500 | [diff] [blame] | 649 | struct move_bucket_in_flight *bucket_in_flight, |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 650 | struct bpos bucket, int gen, |
| 651 | struct data_update_opts _data_opts) |
| 652 | { |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 653 | struct btree_trans *trans = ctxt->trans; |
| 654 | struct bch_fs *c = trans->c; |
Kent Overstreet | 415e510 | 2023-11-28 16:33:52 -0500 | [diff] [blame] | 655 | bool is_kthread = current->flags & PF_KTHREAD; |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 656 | struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 657 | struct btree_iter iter; |
| 658 | struct bkey_buf sk; |
| 659 | struct bch_backpointer bp; |
| 660 | struct bch_alloc_v4 a_convert; |
| 661 | const struct bch_alloc_v4 *a; |
| 662 | struct bkey_s_c k; |
| 663 | struct data_update_opts data_opts; |
| 664 | unsigned dirty_sectors, bucket_size; |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 665 | u64 fragmentation; |
Kent Overstreet | 62a0355 | 2023-03-31 16:24:45 -0400 | [diff] [blame] | 666 | struct bpos bp_pos = POS_MIN; |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 667 | int ret = 0; |
| 668 | |
Kent Overstreet | cb4d340 | 2024-04-30 15:46:45 -0400 | [diff] [blame] | 669 | struct bch_dev *ca = bch2_dev_tryget(c, bucket.inode); |
| 670 | if (!ca) |
| 671 | return 0; |
| 672 | |
Kent Overstreet | 5a21764 | 2023-04-20 15:24:07 -0400 | [diff] [blame] | 673 | trace_bucket_evacuate(c, &bucket); |
| 674 | |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 675 | bch2_bkey_buf_init(&sk); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 676 | |
Kent Overstreet | 3e36e57 | 2023-03-19 14:13:17 -0400 | [diff] [blame] | 677 | /* |
| 678 | * We're not run in a context that handles transaction restarts: |
| 679 | */ |
| 680 | bch2_trans_begin(trans); |
| 681 | |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 682 | bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, |
Kent Overstreet | 5dd8c60 | 2024-04-07 18:05:34 -0400 | [diff] [blame] | 683 | bucket, BTREE_ITER_cached); |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 684 | ret = lockrestart_do(trans, |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 685 | bkey_err(k = bch2_btree_iter_peek_slot(&iter))); |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 686 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 687 | |
Kent Overstreet | cf904c8 | 2023-12-16 22:43:41 -0500 | [diff] [blame] | 688 | bch_err_msg(c, ret, "looking up alloc key"); |
| 689 | if (ret) |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 690 | goto err; |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 691 | |
| 692 | a = bch2_alloc_to_v4(k, &a_convert); |
Kent Overstreet | dafff7e | 2023-11-23 18:05:18 -0500 | [diff] [blame] | 693 | dirty_sectors = bch2_bucket_sectors_dirty(*a); |
Kent Overstreet | cb4d340 | 2024-04-30 15:46:45 -0400 | [diff] [blame] | 694 | bucket_size = ca->mi.bucket_size; |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 695 | fragmentation = a->fragmentation_lru; |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 696 | |
Kent Overstreet | cb13f47 | 2023-11-02 20:36:00 -0400 | [diff] [blame] | 697 | ret = bch2_btree_write_buffer_tryflush(trans); |
| 698 | bch_err_msg(c, ret, "flushing btree write buffer"); |
| 699 | if (ret) |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 700 | goto err; |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 701 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 702 | while (!(ret = bch2_move_ratelimit(ctxt))) { |
Kent Overstreet | 415e510 | 2023-11-28 16:33:52 -0500 | [diff] [blame] | 703 | if (is_kthread && kthread_should_stop()) |
| 704 | break; |
| 705 | |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 706 | bch2_trans_begin(trans); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 707 | |
Kent Overstreet | 633cf069 | 2024-04-30 16:50:28 -0400 | [diff] [blame] | 708 | ret = bch2_get_next_backpointer(trans, ca, bucket, gen, |
Kent Overstreet | 62a0355 | 2023-03-31 16:24:45 -0400 | [diff] [blame] | 709 | &bp_pos, &bp, |
Kent Overstreet | 5dd8c60 | 2024-04-07 18:05:34 -0400 | [diff] [blame] | 710 | BTREE_ITER_cached); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 711 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 712 | continue; |
| 713 | if (ret) |
| 714 | goto err; |
Kent Overstreet | 62a0355 | 2023-03-31 16:24:45 -0400 | [diff] [blame] | 715 | if (bkey_eq(bp_pos, POS_MAX)) |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 716 | break; |
| 717 | |
| 718 | if (!bp.level) { |
Kent Overstreet | 62a0355 | 2023-03-31 16:24:45 -0400 | [diff] [blame] | 719 | k = bch2_backpointer_get_key(trans, &iter, bp_pos, bp, 0); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 720 | ret = bkey_err(k); |
| 721 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 722 | continue; |
| 723 | if (ret) |
| 724 | goto err; |
| 725 | if (!k.k) |
| 726 | goto next; |
| 727 | |
| 728 | bch2_bkey_buf_reassemble(&sk, c, k); |
| 729 | k = bkey_i_to_s_c(sk.k); |
| 730 | |
Kent Overstreet | 8480905 | 2023-10-21 15:03:05 -0400 | [diff] [blame] | 731 | ret = bch2_move_get_io_opts_one(trans, &io_opts, k); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 732 | if (ret) { |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 733 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 734 | continue; |
| 735 | } |
| 736 | |
| 737 | data_opts = _data_opts; |
| 738 | data_opts.target = io_opts.background_target; |
| 739 | data_opts.rewrite_ptrs = 0; |
| 740 | |
Kent Overstreet | 0beebd9 | 2023-12-21 15:47:15 -0500 | [diff] [blame] | 741 | unsigned i = 0; |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 742 | bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) { |
Kent Overstreet | 3f5d3fb | 2023-03-10 18:00:10 -0500 | [diff] [blame] | 743 | if (ptr->dev == bucket.inode) { |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 744 | data_opts.rewrite_ptrs |= 1U << i; |
Kent Overstreet | 3f5d3fb | 2023-03-10 18:00:10 -0500 | [diff] [blame] | 745 | if (ptr->cached) { |
| 746 | bch2_trans_iter_exit(trans, &iter); |
| 747 | goto next; |
| 748 | } |
| 749 | } |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 750 | i++; |
| 751 | } |
| 752 | |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 753 | ret = bch2_move_extent(ctxt, bucket_in_flight, |
| 754 | &iter, k, io_opts, data_opts); |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 755 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 756 | |
| 757 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 758 | continue; |
| 759 | if (ret == -ENOMEM) { |
| 760 | /* memory allocation failure, wait for some IO to finish */ |
Kent Overstreet | 6331690 | 2023-10-20 13:32:42 -0400 | [diff] [blame] | 761 | bch2_move_ctxt_wait_for_io(ctxt); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 762 | continue; |
| 763 | } |
| 764 | if (ret) |
| 765 | goto err; |
| 766 | |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 767 | if (ctxt->stats) |
| 768 | atomic64_add(k.k->size, &ctxt->stats->sectors_seen); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 769 | } else { |
| 770 | struct btree *b; |
| 771 | |
Kent Overstreet | 62a0355 | 2023-03-31 16:24:45 -0400 | [diff] [blame] | 772 | b = bch2_backpointer_get_node(trans, &iter, bp_pos, bp); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 773 | ret = PTR_ERR_OR_ZERO(b); |
| 774 | if (ret == -BCH_ERR_backpointer_to_overwritten_btree_node) |
| 775 | continue; |
| 776 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 777 | continue; |
| 778 | if (ret) |
| 779 | goto err; |
| 780 | if (!b) |
| 781 | goto next; |
| 782 | |
Kent Overstreet | 652bc7f | 2024-05-24 18:04:22 -0400 | [diff] [blame] | 783 | unsigned sectors = btree_ptr_sectors_written(bkey_i_to_s_c(&b->key)); |
Kent Overstreet | ec4edd7 | 2024-01-16 13:29:59 -0500 | [diff] [blame] | 784 | |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 785 | ret = bch2_btree_node_rewrite(trans, &iter, b, 0); |
| 786 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 787 | |
| 788 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
| 789 | continue; |
| 790 | if (ret) |
| 791 | goto err; |
| 792 | |
| 793 | if (ctxt->rate) |
Kent Overstreet | ec4edd7 | 2024-01-16 13:29:59 -0500 | [diff] [blame] | 794 | bch2_ratelimit_increment(ctxt->rate, sectors); |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 795 | if (ctxt->stats) { |
Kent Overstreet | ec4edd7 | 2024-01-16 13:29:59 -0500 | [diff] [blame] | 796 | atomic64_add(sectors, &ctxt->stats->sectors_seen); |
| 797 | atomic64_add(sectors, &ctxt->stats->sectors_moved); |
Kent Overstreet | 2f52866 | 2023-03-04 02:51:12 -0500 | [diff] [blame] | 798 | } |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 799 | } |
| 800 | next: |
Kent Overstreet | 62a0355 | 2023-03-31 16:24:45 -0400 | [diff] [blame] | 801 | bp_pos = bpos_nosnap_successor(bp_pos); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 802 | } |
| 803 | |
Kent Overstreet | 80c3308 | 2022-12-05 10:24:19 -0500 | [diff] [blame] | 804 | trace_evacuate_bucket(c, &bucket, dirty_sectors, bucket_size, fragmentation, ret); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 805 | err: |
Kent Overstreet | cb4d340 | 2024-04-30 15:46:45 -0400 | [diff] [blame] | 806 | bch2_dev_put(ca); |
Kent Overstreet | 8e3f913 | 2022-03-18 00:42:09 -0400 | [diff] [blame] | 807 | bch2_bkey_buf_exit(&sk, c); |
| 808 | return ret; |
| 809 | } |
| 810 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 811 | typedef bool (*move_btree_pred)(struct bch_fs *, void *, |
| 812 | struct btree *, struct bch_io_opts *, |
| 813 | struct data_update_opts *); |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 814 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 815 | static int bch2_move_btree(struct bch_fs *c, |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 816 | struct bbpos start, |
| 817 | struct bbpos end, |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 818 | move_btree_pred pred, void *arg, |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 819 | struct bch_move_stats *stats) |
| 820 | { |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 821 | bool kthread = (current->flags & PF_KTHREAD) != 0; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 822 | struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts); |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 823 | struct moving_context ctxt; |
| 824 | struct btree_trans *trans; |
Kent Overstreet | 67e0dd8 | 2021-08-30 15:18:31 -0400 | [diff] [blame] | 825 | struct btree_iter iter; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 826 | struct btree *b; |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 827 | enum btree_id btree; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 828 | struct data_update_opts data_opts; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 829 | int ret = 0; |
| 830 | |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 831 | bch2_moving_ctxt_init(&ctxt, c, NULL, stats, |
| 832 | writepoint_ptr(&c->btree_write_point), |
| 833 | true); |
| 834 | trans = ctxt.trans; |
Kent Overstreet | 424eb88 | 2019-03-25 15:10:15 -0400 | [diff] [blame] | 835 | |
Kent Overstreet | 89fd25b | 2020-07-09 18:28:11 -0400 | [diff] [blame] | 836 | stats->data_type = BCH_DATA_btree; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 837 | |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 838 | for (btree = start.btree; |
| 839 | btree <= min_t(unsigned, end.btree, btree_id_nr_alive(c) - 1); |
| 840 | btree ++) { |
| 841 | stats->pos = BBPOS(btree, POS_MIN); |
Kent Overstreet | 424eb88 | 2019-03-25 15:10:15 -0400 | [diff] [blame] | 842 | |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 843 | if (!bch2_btree_id_root(c, btree)->b) |
Kent Overstreet | faa6cb6 | 2023-06-28 22:09:13 -0400 | [diff] [blame] | 844 | continue; |
| 845 | |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 846 | bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, 0, |
Kent Overstreet | 5dd8c60 | 2024-04-07 18:05:34 -0400 | [diff] [blame] | 847 | BTREE_ITER_prefetch); |
Kent Overstreet | d355c6f | 2021-10-19 14:20:50 -0400 | [diff] [blame] | 848 | retry: |
Kent Overstreet | b71717d | 2021-10-19 15:11:45 -0400 | [diff] [blame] | 849 | ret = 0; |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 850 | while (bch2_trans_begin(trans), |
Kent Overstreet | d355c6f | 2021-10-19 14:20:50 -0400 | [diff] [blame] | 851 | (b = bch2_btree_iter_peek_node(&iter)) && |
| 852 | !(ret = PTR_ERR_OR_ZERO(b))) { |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 853 | if (kthread && kthread_should_stop()) |
Kent Overstreet | 7b7278b | 2021-04-20 20:21:39 -0400 | [diff] [blame] | 854 | break; |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 855 | |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 856 | if ((cmp_int(btree, end.btree) ?: |
| 857 | bpos_cmp(b->key.k.p, end.pos)) > 0) |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 858 | break; |
| 859 | |
Kent Overstreet | d5eade9 | 2023-10-23 15:36:45 -0400 | [diff] [blame] | 860 | stats->pos = BBPOS(iter.btree_id, iter.pos); |
Kent Overstreet | 424eb88 | 2019-03-25 15:10:15 -0400 | [diff] [blame] | 861 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 862 | if (!pred(c, arg, b, &io_opts, &data_opts)) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 863 | goto next; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 864 | |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 865 | ret = bch2_btree_node_rewrite(trans, &iter, b, 0) ?: ret; |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 866 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
Kent Overstreet | f3cf099 | 2021-10-24 16:59:33 -0400 | [diff] [blame] | 867 | continue; |
| 868 | if (ret) |
| 869 | break; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 870 | next: |
Kent Overstreet | 4b09ef1 | 2021-10-07 18:08:01 -0400 | [diff] [blame] | 871 | bch2_btree_iter_next_node(&iter); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 872 | } |
Kent Overstreet | 549d173 | 2022-07-17 23:06:38 -0400 | [diff] [blame] | 873 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
Kent Overstreet | d355c6f | 2021-10-19 14:20:50 -0400 | [diff] [blame] | 874 | goto retry; |
| 875 | |
Kent Overstreet | 6bd68ec | 2023-09-12 17:16:02 -0400 | [diff] [blame] | 876 | bch2_trans_iter_exit(trans, &iter); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 877 | |
Kent Overstreet | 7b7278b | 2021-04-20 20:21:39 -0400 | [diff] [blame] | 878 | if (kthread && kthread_should_stop()) |
| 879 | break; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 880 | } |
Kent Overstreet | 7b7278b | 2021-04-20 20:21:39 -0400 | [diff] [blame] | 881 | |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 882 | bch_err_fn(c, ret); |
| 883 | bch2_moving_ctxt_exit(&ctxt); |
Kent Overstreet | c096060 | 2022-04-17 17:30:49 -0400 | [diff] [blame] | 884 | bch2_btree_interior_updates_flush(c); |
Kent Overstreet | 23af498 | 2021-10-24 17:00:33 -0400 | [diff] [blame] | 885 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 886 | return ret; |
| 887 | } |
| 888 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 889 | static bool rereplicate_pred(struct bch_fs *c, void *arg, |
| 890 | struct bkey_s_c k, |
| 891 | struct bch_io_opts *io_opts, |
| 892 | struct data_update_opts *data_opts) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 893 | { |
Kent Overstreet | 26609b6 | 2018-11-01 15:10:01 -0400 | [diff] [blame] | 894 | unsigned nr_good = bch2_bkey_durability(c, k); |
Kent Overstreet | e8bde78 | 2021-10-12 14:15:45 -0400 | [diff] [blame] | 895 | unsigned replicas = bkey_is_btree_ptr(k.k) |
| 896 | ? c->opts.metadata_replicas |
| 897 | : io_opts->data_replicas; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 898 | |
Kent Overstreet | fdccb24 | 2024-06-02 22:25:18 -0400 | [diff] [blame] | 899 | rcu_read_lock(); |
| 900 | struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); |
| 901 | unsigned i = 0; |
| 902 | bkey_for_each_ptr(ptrs, ptr) { |
| 903 | struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); |
| 904 | if (!ptr->cached && |
| 905 | (!ca || !ca->mi.durability)) |
| 906 | data_opts->kill_ptrs |= BIT(i); |
| 907 | i++; |
| 908 | } |
| 909 | rcu_read_unlock(); |
| 910 | |
| 911 | if (!data_opts->kill_ptrs && |
| 912 | (!nr_good || nr_good >= replicas)) |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 913 | return false; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 914 | |
| 915 | data_opts->target = 0; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 916 | data_opts->extra_replicas = replicas - nr_good; |
Kent Overstreet | 26609b6 | 2018-11-01 15:10:01 -0400 | [diff] [blame] | 917 | data_opts->btree_insert_flags = 0; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 918 | return true; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 919 | } |
| 920 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 921 | static bool migrate_pred(struct bch_fs *c, void *arg, |
| 922 | struct bkey_s_c k, |
| 923 | struct bch_io_opts *io_opts, |
| 924 | struct data_update_opts *data_opts) |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 925 | { |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 926 | struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 927 | struct bch_ioctl_data *op = arg; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 928 | unsigned i = 0; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 929 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 930 | data_opts->rewrite_ptrs = 0; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 931 | data_opts->target = 0; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 932 | data_opts->extra_replicas = 0; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 933 | data_opts->btree_insert_flags = 0; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 934 | |
| 935 | bkey_for_each_ptr(ptrs, ptr) { |
| 936 | if (ptr->dev == op->migrate.dev) |
| 937 | data_opts->rewrite_ptrs |= 1U << i; |
| 938 | i++; |
| 939 | } |
| 940 | |
Kent Overstreet | 3e3e02e | 2022-10-19 18:31:33 -0400 | [diff] [blame] | 941 | return data_opts->rewrite_ptrs != 0; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 942 | } |
| 943 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 944 | static bool rereplicate_btree_pred(struct bch_fs *c, void *arg, |
| 945 | struct btree *b, |
| 946 | struct bch_io_opts *io_opts, |
| 947 | struct data_update_opts *data_opts) |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 948 | { |
| 949 | return rereplicate_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts); |
| 950 | } |
| 951 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 952 | static bool migrate_btree_pred(struct bch_fs *c, void *arg, |
| 953 | struct btree *b, |
| 954 | struct bch_io_opts *io_opts, |
| 955 | struct data_update_opts *data_opts) |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 956 | { |
| 957 | return migrate_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts); |
| 958 | } |
| 959 | |
Kent Overstreet | 0ec5b3b | 2024-05-05 22:44:27 -0400 | [diff] [blame] | 960 | /* |
| 961 | * Ancient versions of bcachefs produced packed formats which could represent |
| 962 | * keys that the in memory format cannot represent; this checks for those |
| 963 | * formats so we can get rid of them. |
| 964 | */ |
Kent Overstreet | e01dacf | 2021-03-20 23:55:36 -0400 | [diff] [blame] | 965 | static bool bformat_needs_redo(struct bkey_format *f) |
| 966 | { |
Kent Overstreet | 61692c7 | 2024-05-08 10:58:26 -0400 | [diff] [blame] | 967 | for (unsigned i = 0; i < f->nr_fields; i++) |
| 968 | if (bch2_bkey_format_field_overflows(f, i)) |
Kent Overstreet | e01dacf | 2021-03-20 23:55:36 -0400 | [diff] [blame] | 969 | return true; |
| 970 | |
Kent Overstreet | e01dacf | 2021-03-20 23:55:36 -0400 | [diff] [blame] | 971 | return false; |
| 972 | } |
| 973 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 974 | static bool rewrite_old_nodes_pred(struct bch_fs *c, void *arg, |
| 975 | struct btree *b, |
| 976 | struct bch_io_opts *io_opts, |
| 977 | struct data_update_opts *data_opts) |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 978 | { |
| 979 | if (b->version_ondisk != c->sb.version || |
Kent Overstreet | e01dacf | 2021-03-20 23:55:36 -0400 | [diff] [blame] | 980 | btree_node_need_rewrite(b) || |
| 981 | bformat_needs_redo(&b->format)) { |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 982 | data_opts->target = 0; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 983 | data_opts->extra_replicas = 0; |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 984 | data_opts->btree_insert_flags = 0; |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 985 | return true; |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 986 | } |
| 987 | |
Kent Overstreet | 7f5c5d2 | 2022-06-13 19:17:45 -0400 | [diff] [blame] | 988 | return false; |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 989 | } |
| 990 | |
Kent Overstreet | a4805d6 | 2021-03-22 18:39:16 -0400 | [diff] [blame] | 991 | int bch2_scan_old_btree_nodes(struct bch_fs *c, struct bch_move_stats *stats) |
| 992 | { |
| 993 | int ret; |
| 994 | |
| 995 | ret = bch2_move_btree(c, |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 996 | BBPOS_MIN, |
| 997 | BBPOS_MAX, |
Kent Overstreet | a4805d6 | 2021-03-22 18:39:16 -0400 | [diff] [blame] | 998 | rewrite_old_nodes_pred, c, stats); |
| 999 | if (!ret) { |
| 1000 | mutex_lock(&c->sb_lock); |
Kent Overstreet | c0ebe3e | 2021-05-23 17:04:13 -0400 | [diff] [blame] | 1001 | c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_extents_above_btree_updates_done); |
| 1002 | c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_bformat_overflow_done); |
Kent Overstreet | a4805d6 | 2021-03-22 18:39:16 -0400 | [diff] [blame] | 1003 | c->disk_sb.sb->version_min = c->disk_sb.sb->version; |
| 1004 | bch2_write_super(c); |
| 1005 | mutex_unlock(&c->sb_lock); |
| 1006 | } |
| 1007 | |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 1008 | bch_err_fn(c, ret); |
Kent Overstreet | a4805d6 | 2021-03-22 18:39:16 -0400 | [diff] [blame] | 1009 | return ret; |
| 1010 | } |
| 1011 | |
Kent Overstreet | ba11c7d | 2023-11-20 19:12:40 -0500 | [diff] [blame] | 1012 | static bool drop_extra_replicas_pred(struct bch_fs *c, void *arg, |
| 1013 | struct bkey_s_c k, |
| 1014 | struct bch_io_opts *io_opts, |
| 1015 | struct data_update_opts *data_opts) |
| 1016 | { |
| 1017 | unsigned durability = bch2_bkey_durability(c, k); |
| 1018 | unsigned replicas = bkey_is_btree_ptr(k.k) |
| 1019 | ? c->opts.metadata_replicas |
| 1020 | : io_opts->data_replicas; |
| 1021 | const union bch_extent_entry *entry; |
| 1022 | struct extent_ptr_decoded p; |
| 1023 | unsigned i = 0; |
| 1024 | |
Kent Overstreet | 302c980 | 2024-04-30 20:54:20 -0400 | [diff] [blame] | 1025 | rcu_read_lock(); |
Kent Overstreet | ba11c7d | 2023-11-20 19:12:40 -0500 | [diff] [blame] | 1026 | bkey_for_each_ptr_decode(k.k, bch2_bkey_ptrs_c(k), p, entry) { |
| 1027 | unsigned d = bch2_extent_ptr_durability(c, &p); |
| 1028 | |
| 1029 | if (d && durability - d >= replicas) { |
| 1030 | data_opts->kill_ptrs |= BIT(i); |
| 1031 | durability -= d; |
| 1032 | } |
| 1033 | |
| 1034 | i++; |
| 1035 | } |
Kent Overstreet | 302c980 | 2024-04-30 20:54:20 -0400 | [diff] [blame] | 1036 | rcu_read_unlock(); |
Kent Overstreet | ba11c7d | 2023-11-20 19:12:40 -0500 | [diff] [blame] | 1037 | |
| 1038 | return data_opts->kill_ptrs != 0; |
| 1039 | } |
| 1040 | |
| 1041 | static bool drop_extra_replicas_btree_pred(struct bch_fs *c, void *arg, |
| 1042 | struct btree *b, |
| 1043 | struct bch_io_opts *io_opts, |
| 1044 | struct data_update_opts *data_opts) |
| 1045 | { |
| 1046 | return drop_extra_replicas_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts); |
| 1047 | } |
| 1048 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1049 | int bch2_data_job(struct bch_fs *c, |
| 1050 | struct bch_move_stats *stats, |
| 1051 | struct bch_ioctl_data op) |
| 1052 | { |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 1053 | struct bbpos start = BBPOS(op.start_btree, op.start_pos); |
| 1054 | struct bbpos end = BBPOS(op.end_btree, op.end_pos); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1055 | int ret = 0; |
| 1056 | |
Kent Overstreet | 01e9564 | 2023-11-20 18:43:48 -0500 | [diff] [blame] | 1057 | if (op.op >= BCH_DATA_OP_NR) |
| 1058 | return -EINVAL; |
| 1059 | |
| 1060 | bch2_move_stats_init(stats, bch2_data_ops_strs[op.op]); |
| 1061 | |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1062 | switch (op.op) { |
Kent Overstreet | 01e9564 | 2023-11-20 18:43:48 -0500 | [diff] [blame] | 1063 | case BCH_DATA_OP_rereplicate: |
Kent Overstreet | 89fd25b | 2020-07-09 18:28:11 -0400 | [diff] [blame] | 1064 | stats->data_type = BCH_DATA_journal; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1065 | ret = bch2_journal_flush_device_pins(&c->journal, -1); |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 1066 | ret = bch2_move_btree(c, start, end, |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 1067 | rereplicate_btree_pred, c, stats) ?: ret; |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 1068 | ret = bch2_move_data(c, start, end, |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 1069 | NULL, |
| 1070 | stats, |
| 1071 | writepoint_hashed((unsigned long) current), |
| 1072 | true, |
| 1073 | rereplicate_pred, c) ?: ret; |
Kent Overstreet | ae0ff7b | 2019-04-30 17:15:39 -0400 | [diff] [blame] | 1074 | ret = bch2_replicas_gc2(c) ?: ret; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1075 | break; |
Kent Overstreet | 01e9564 | 2023-11-20 18:43:48 -0500 | [diff] [blame] | 1076 | case BCH_DATA_OP_migrate: |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1077 | if (op.migrate.dev >= c->sb.nr_devices) |
| 1078 | return -EINVAL; |
| 1079 | |
Kent Overstreet | 89fd25b | 2020-07-09 18:28:11 -0400 | [diff] [blame] | 1080 | stats->data_type = BCH_DATA_journal; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1081 | ret = bch2_journal_flush_device_pins(&c->journal, op.migrate.dev); |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 1082 | ret = bch2_move_btree(c, start, end, |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 1083 | migrate_btree_pred, &op, stats) ?: ret; |
Kent Overstreet | 3c843a6 | 2023-11-20 18:52:33 -0500 | [diff] [blame] | 1084 | ret = bch2_move_data(c, start, end, |
Kent Overstreet | 0337cc7 | 2022-06-20 15:40:26 -0400 | [diff] [blame] | 1085 | NULL, |
| 1086 | stats, |
| 1087 | writepoint_hashed((unsigned long) current), |
| 1088 | true, |
| 1089 | migrate_pred, &op) ?: ret; |
Kent Overstreet | ae0ff7b | 2019-04-30 17:15:39 -0400 | [diff] [blame] | 1090 | ret = bch2_replicas_gc2(c) ?: ret; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1091 | break; |
Kent Overstreet | 01e9564 | 2023-11-20 18:43:48 -0500 | [diff] [blame] | 1092 | case BCH_DATA_OP_rewrite_old_nodes: |
Kent Overstreet | a4805d6 | 2021-03-22 18:39:16 -0400 | [diff] [blame] | 1093 | ret = bch2_scan_old_btree_nodes(c, stats); |
Kent Overstreet | 1889ad5 | 2021-03-14 19:01:14 -0400 | [diff] [blame] | 1094 | break; |
Kent Overstreet | ba11c7d | 2023-11-20 19:12:40 -0500 | [diff] [blame] | 1095 | case BCH_DATA_OP_drop_extra_replicas: |
| 1096 | ret = bch2_move_btree(c, start, end, |
| 1097 | drop_extra_replicas_btree_pred, c, stats) ?: ret; |
| 1098 | ret = bch2_move_data(c, start, end, NULL, stats, |
| 1099 | writepoint_hashed((unsigned long) current), |
| 1100 | true, |
| 1101 | drop_extra_replicas_pred, c) ?: ret; |
| 1102 | ret = bch2_replicas_gc2(c) ?: ret; |
| 1103 | break; |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1104 | default: |
| 1105 | ret = -EINVAL; |
| 1106 | } |
| 1107 | |
Kent Overstreet | 01e9564 | 2023-11-20 18:43:48 -0500 | [diff] [blame] | 1108 | bch2_move_stats_exit(stats, c); |
Kent Overstreet | 1c6fdbd | 2017-03-16 22:18:50 -0800 | [diff] [blame] | 1109 | return ret; |
| 1110 | } |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1111 | |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 1112 | void bch2_move_stats_to_text(struct printbuf *out, struct bch_move_stats *stats) |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1113 | { |
Kent Overstreet | e58f963 | 2024-01-06 20:57:43 -0500 | [diff] [blame] | 1114 | prt_printf(out, "%s: data type==", stats->name); |
| 1115 | bch2_prt_data_type(out, stats->data_type); |
| 1116 | prt_str(out, " pos="); |
Kent Overstreet | d5eade9 | 2023-10-23 15:36:45 -0400 | [diff] [blame] | 1117 | bch2_bbpos_to_text(out, stats->pos); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1118 | prt_newline(out); |
| 1119 | printbuf_indent_add(out, 2); |
| 1120 | |
Kent Overstreet | 7423330 | 2024-04-10 16:08:24 -0400 | [diff] [blame] | 1121 | prt_printf(out, "keys moved: %llu\n", atomic64_read(&stats->keys_moved)); |
| 1122 | prt_printf(out, "keys raced: %llu\n", atomic64_read(&stats->keys_raced)); |
| 1123 | prt_printf(out, "bytes seen: "); |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 1124 | prt_human_readable_u64(out, atomic64_read(&stats->sectors_seen) << 9); |
| 1125 | prt_newline(out); |
| 1126 | |
Kent Overstreet | 7423330 | 2024-04-10 16:08:24 -0400 | [diff] [blame] | 1127 | prt_printf(out, "bytes moved: "); |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 1128 | prt_human_readable_u64(out, atomic64_read(&stats->sectors_moved) << 9); |
| 1129 | prt_newline(out); |
| 1130 | |
Kent Overstreet | 7423330 | 2024-04-10 16:08:24 -0400 | [diff] [blame] | 1131 | prt_printf(out, "bytes raced: "); |
Kent Overstreet | 96a363a | 2023-10-23 16:21:54 -0400 | [diff] [blame] | 1132 | prt_human_readable_u64(out, atomic64_read(&stats->sectors_raced) << 9); |
| 1133 | prt_newline(out); |
| 1134 | |
| 1135 | printbuf_indent_sub(out, 2); |
| 1136 | } |
| 1137 | |
| 1138 | static void bch2_moving_ctxt_to_text(struct printbuf *out, struct bch_fs *c, struct moving_context *ctxt) |
| 1139 | { |
| 1140 | struct moving_io *io; |
| 1141 | |
| 1142 | bch2_move_stats_to_text(out, ctxt->stats); |
| 1143 | printbuf_indent_add(out, 2); |
| 1144 | |
Kent Overstreet | 7423330 | 2024-04-10 16:08:24 -0400 | [diff] [blame] | 1145 | prt_printf(out, "reads: ios %u/%u sectors %u/%u\n", |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1146 | atomic_read(&ctxt->read_ios), |
Kent Overstreet | 9d2a7bd | 2023-08-23 21:20:42 -0400 | [diff] [blame] | 1147 | c->opts.move_ios_in_flight, |
| 1148 | atomic_read(&ctxt->read_sectors), |
| 1149 | c->opts.move_bytes_in_flight >> 9); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1150 | |
Kent Overstreet | 7423330 | 2024-04-10 16:08:24 -0400 | [diff] [blame] | 1151 | prt_printf(out, "writes: ios %u/%u sectors %u/%u\n", |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1152 | atomic_read(&ctxt->write_ios), |
Kent Overstreet | 9d2a7bd | 2023-08-23 21:20:42 -0400 | [diff] [blame] | 1153 | c->opts.move_ios_in_flight, |
| 1154 | atomic_read(&ctxt->write_sectors), |
| 1155 | c->opts.move_bytes_in_flight >> 9); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1156 | |
| 1157 | printbuf_indent_add(out, 2); |
| 1158 | |
| 1159 | mutex_lock(&ctxt->lock); |
Kent Overstreet | 9d2a7bd | 2023-08-23 21:20:42 -0400 | [diff] [blame] | 1160 | list_for_each_entry(io, &ctxt->ios, io_list) |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1161 | bch2_write_op_to_text(out, &io->write.op); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1162 | mutex_unlock(&ctxt->lock); |
| 1163 | |
| 1164 | printbuf_indent_sub(out, 4); |
| 1165 | } |
| 1166 | |
| 1167 | void bch2_fs_moving_ctxts_to_text(struct printbuf *out, struct bch_fs *c) |
| 1168 | { |
| 1169 | struct moving_context *ctxt; |
| 1170 | |
| 1171 | mutex_lock(&c->moving_context_lock); |
| 1172 | list_for_each_entry(ctxt, &c->moving_context_list, list) |
Kent Overstreet | 9d2a7bd | 2023-08-23 21:20:42 -0400 | [diff] [blame] | 1173 | bch2_moving_ctxt_to_text(out, c, ctxt); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1174 | mutex_unlock(&c->moving_context_lock); |
| 1175 | } |
| 1176 | |
| 1177 | void bch2_fs_move_init(struct bch_fs *c) |
| 1178 | { |
| 1179 | INIT_LIST_HEAD(&c->moving_context_list); |
| 1180 | mutex_init(&c->moving_context_lock); |
Kent Overstreet | b9fa375 | 2023-03-11 20:38:46 -0500 | [diff] [blame] | 1181 | } |