blob: 7d3920e03742deebce5798731c98a36ef1e9369d [file] [log] [blame]
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001// SPDX-License-Identifier: GPL-2.0
2
3#include "bcachefs.h"
Kent Overstreet8e3f9132022-03-18 00:42:09 -04004#include "alloc_background.h"
Kent Overstreet7b3f84e2018-10-06 00:46:55 -04005#include "alloc_foreground.h"
Kent Overstreet8e3f9132022-03-18 00:42:09 -04006#include "backpointers.h"
Kent Overstreet07a10062020-12-17 15:08:58 -05007#include "bkey_buf.h"
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08008#include "btree_gc.h"
Kent Overstreetec4edd72024-01-16 13:29:59 -05009#include "btree_io.h"
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080010#include "btree_update.h"
Kent Overstreet7ef2a732019-01-21 15:32:13 -050011#include "btree_update_interior.h"
Kent Overstreet8e3f9132022-03-18 00:42:09 -040012#include "btree_write_buffer.h"
Kent Overstreet189c1762024-01-15 15:33:39 -050013#include "compress.h"
Kent Overstreet46285292018-11-04 23:10:09 -050014#include "disk_groups.h"
Kent Overstreet961b2d62021-10-29 16:29:13 -040015#include "ec.h"
Kent Overstreetd4bf5ee2022-07-18 19:42:58 -040016#include "errcode.h"
Kent Overstreet8e3f9132022-03-18 00:42:09 -040017#include "error.h"
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080018#include "inode.h"
Kent Overstreet1809b8c2023-09-10 18:05:17 -040019#include "io_read.h"
20#include "io_write.h"
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080021#include "journal_reclaim.h"
22#include "keylist.h"
23#include "move.h"
24#include "replicas.h"
Kent Overstreet84809052023-10-21 15:03:05 -040025#include "snapshot.h"
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080026#include "super-io.h"
27#include "trace.h"
28
29#include <linux/ioprio.h>
30#include <linux/kthread.h>
31
Kent Overstreet01e95642023-11-20 18:43:48 -050032const char * const bch2_data_ops_strs[] = {
33#define x(t, n, ...) [n] = #t,
34 BCH_DATA_OPS()
35#undef x
36 NULL
37};
38
Kent Overstreet189c1762024-01-15 15:33:39 -050039static void trace_move_extent2(struct bch_fs *c, struct bkey_s_c k,
40 struct bch_io_opts *io_opts,
41 struct data_update_opts *data_opts)
Kent Overstreet5a217642023-04-20 15:24:07 -040042{
43 if (trace_move_extent_enabled()) {
44 struct printbuf buf = PRINTBUF;
45
46 bch2_bkey_val_to_text(&buf, c, k);
Kent Overstreet189c1762024-01-15 15:33:39 -050047 prt_newline(&buf);
48 bch2_data_update_opts_to_text(&buf, c, io_opts, data_opts);
Kent Overstreet5a217642023-04-20 15:24:07 -040049 trace_move_extent(c, buf.buf);
50 printbuf_exit(&buf);
51 }
52}
53
54static void trace_move_extent_read2(struct bch_fs *c, struct bkey_s_c k)
55{
56 if (trace_move_extent_read_enabled()) {
57 struct printbuf buf = PRINTBUF;
58
59 bch2_bkey_val_to_text(&buf, c, k);
60 trace_move_extent_read(c, buf.buf);
61 printbuf_exit(&buf);
62 }
63}
64
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080065struct moving_io {
Kent Overstreetb9fa3752023-03-11 20:38:46 -050066 struct list_head read_list;
67 struct list_head io_list;
Kent Overstreet8fcdf812023-02-27 22:58:01 -050068 struct move_bucket_in_flight *b;
69 struct closure cl;
70 bool read_completed;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080071
Kent Overstreet8fcdf812023-02-27 22:58:01 -050072 unsigned read_sectors;
73 unsigned write_sectors;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080074
Kent Overstreet8fcdf812023-02-27 22:58:01 -050075 struct bch_read_bio rbio;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080076
Kent Overstreet8fcdf812023-02-27 22:58:01 -050077 struct data_update write;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080078 /* Must be last since it is variable size */
Gustavo A. R. Silva62286a02023-11-28 12:22:55 -060079 struct bio_vec bi_inline_vecs[];
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080080};
81
Kent Overstreet9f311f22022-10-29 02:47:33 -040082static void move_free(struct moving_io *io)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080083{
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080084 struct moving_context *ctxt = io->write.ctxt;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080085
Kent Overstreet8fcdf812023-02-27 22:58:01 -050086 if (io->b)
87 atomic_dec(&io->b->count);
88
Kent Overstreet7f5c5d22022-06-13 19:17:45 -040089 bch2_data_update_exit(&io->write);
Kent Overstreetb9fa3752023-03-11 20:38:46 -050090
91 mutex_lock(&ctxt->lock);
92 list_del(&io->io_list);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080093 wake_up(&ctxt->wait);
Kent Overstreetb9fa3752023-03-11 20:38:46 -050094 mutex_unlock(&ctxt->lock);
95
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -080096 kfree(io);
97}
98
Kent Overstreet9f311f22022-10-29 02:47:33 -040099static void move_write_done(struct bch_write_op *op)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800100{
Kent Overstreet9f311f22022-10-29 02:47:33 -0400101 struct moving_io *io = container_of(op, struct moving_io, write.op);
102 struct moving_context *ctxt = io->write.ctxt;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800103
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400104 if (io->write.op.error)
105 ctxt->write_error = true;
106
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800107 atomic_sub(io->write_sectors, &io->write.ctxt->write_sectors);
Kent Overstreetc782c582023-01-09 01:45:18 -0500108 atomic_dec(&io->write.ctxt->write_ios);
Kent Overstreet9f311f22022-10-29 02:47:33 -0400109 move_free(io);
110 closure_put(&ctxt->cl);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800111}
112
Kent Overstreet9f311f22022-10-29 02:47:33 -0400113static void move_write(struct moving_io *io)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800114{
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800115 if (unlikely(io->rbio.bio.bi_status || io->rbio.hole)) {
Kent Overstreet9f311f22022-10-29 02:47:33 -0400116 move_free(io);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800117 return;
118 }
119
Kent Overstreetfa3185a2024-01-15 15:04:40 -0500120 if (trace_move_extent_write_enabled()) {
121 struct bch_fs *c = io->write.op.c;
122 struct printbuf buf = PRINTBUF;
123
124 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(io->write.k.k));
125 trace_move_extent_write(c, buf.buf);
126 printbuf_exit(&buf);
127 }
128
Kent Overstreet9f311f22022-10-29 02:47:33 -0400129 closure_get(&io->write.ctxt->cl);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800130 atomic_add(io->write_sectors, &io->write.ctxt->write_sectors);
Kent Overstreetc782c582023-01-09 01:45:18 -0500131 atomic_inc(&io->write.ctxt->write_ios);
Kent Overstreet9f311f22022-10-29 02:47:33 -0400132
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400133 bch2_data_update_read_done(&io->write, io->rbio.pick.crc);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800134}
135
Kent Overstreet7ffb6a72023-01-02 17:53:02 -0500136struct moving_io *bch2_moving_ctxt_next_pending_write(struct moving_context *ctxt)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800137{
138 struct moving_io *io =
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500139 list_first_entry_or_null(&ctxt->reads, struct moving_io, read_list);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800140
141 return io && io->read_completed ? io : NULL;
142}
143
144static void move_read_endio(struct bio *bio)
145{
146 struct moving_io *io = container_of(bio, struct moving_io, rbio.bio);
147 struct moving_context *ctxt = io->write.ctxt;
148
149 atomic_sub(io->read_sectors, &ctxt->read_sectors);
Kent Overstreetc782c582023-01-09 01:45:18 -0500150 atomic_dec(&ctxt->read_ios);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800151 io->read_completed = true;
152
Kent Overstreetf61816d2022-02-21 13:22:11 -0500153 wake_up(&ctxt->wait);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800154 closure_put(&ctxt->cl);
155}
156
Kent Overstreet63316902023-10-20 13:32:42 -0400157void bch2_moving_ctxt_do_pending_writes(struct moving_context *ctxt)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800158{
159 struct moving_io *io;
160
Kent Overstreet7ffb6a72023-01-02 17:53:02 -0500161 while ((io = bch2_moving_ctxt_next_pending_write(ctxt))) {
Kent Overstreetf82755e2023-10-30 15:13:09 -0400162 bch2_trans_unlock_long(ctxt->trans);
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500163 list_del(&io->read_list);
Kent Overstreet9f311f22022-10-29 02:47:33 -0400164 move_write(io);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800165 }
166}
167
Kent Overstreet63316902023-10-20 13:32:42 -0400168void bch2_move_ctxt_wait_for_io(struct moving_context *ctxt)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800169{
170 unsigned sectors_pending = atomic_read(&ctxt->write_sectors);
171
Kent Overstreet63316902023-10-20 13:32:42 -0400172 move_ctxt_wait_event(ctxt,
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800173 !atomic_read(&ctxt->write_sectors) ||
174 atomic_read(&ctxt->write_sectors) != sectors_pending);
175}
176
Daniel Hill0c069782023-11-26 19:33:31 +1300177void bch2_moving_ctxt_flush_all(struct moving_context *ctxt)
Kent Overstreet50e029c2023-11-20 17:24:32 -0500178{
179 move_ctxt_wait_event(ctxt, list_empty(&ctxt->reads));
180 bch2_trans_unlock_long(ctxt->trans);
181 closure_sync(&ctxt->cl);
182}
183
Kent Overstreet0337cc72022-06-20 15:40:26 -0400184void bch2_moving_ctxt_exit(struct moving_context *ctxt)
185{
Kent Overstreet63316902023-10-20 13:32:42 -0400186 struct bch_fs *c = ctxt->trans->c;
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500187
Kent Overstreet50e029c2023-11-20 17:24:32 -0500188 bch2_moving_ctxt_flush_all(ctxt);
Kent Overstreetc782c582023-01-09 01:45:18 -0500189
Kent Overstreet0337cc72022-06-20 15:40:26 -0400190 EBUG_ON(atomic_read(&ctxt->write_sectors));
Kent Overstreetc782c582023-01-09 01:45:18 -0500191 EBUG_ON(atomic_read(&ctxt->write_ios));
192 EBUG_ON(atomic_read(&ctxt->read_sectors));
193 EBUG_ON(atomic_read(&ctxt->read_ios));
Kent Overstreet0337cc72022-06-20 15:40:26 -0400194
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500195 mutex_lock(&c->moving_context_lock);
196 list_del(&ctxt->list);
197 mutex_unlock(&c->moving_context_lock);
Kent Overstreet63316902023-10-20 13:32:42 -0400198
199 bch2_trans_put(ctxt->trans);
200 memset(ctxt, 0, sizeof(*ctxt));
Kent Overstreet0337cc72022-06-20 15:40:26 -0400201}
202
203void bch2_moving_ctxt_init(struct moving_context *ctxt,
204 struct bch_fs *c,
205 struct bch_ratelimit *rate,
206 struct bch_move_stats *stats,
207 struct write_point_specifier wp,
208 bool wait_on_copygc)
209{
210 memset(ctxt, 0, sizeof(*ctxt));
211
Kent Overstreet63316902023-10-20 13:32:42 -0400212 ctxt->trans = bch2_trans_get(c);
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500213 ctxt->fn = (void *) _RET_IP_;
Kent Overstreet0337cc72022-06-20 15:40:26 -0400214 ctxt->rate = rate;
215 ctxt->stats = stats;
216 ctxt->wp = wp;
217 ctxt->wait_on_copygc = wait_on_copygc;
218
Kent Overstreet0337cc72022-06-20 15:40:26 -0400219 closure_init_stack(&ctxt->cl);
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500220
221 mutex_init(&ctxt->lock);
Kent Overstreet0337cc72022-06-20 15:40:26 -0400222 INIT_LIST_HEAD(&ctxt->reads);
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500223 INIT_LIST_HEAD(&ctxt->ios);
Kent Overstreet0337cc72022-06-20 15:40:26 -0400224 init_waitqueue_head(&ctxt->wait);
225
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500226 mutex_lock(&c->moving_context_lock);
227 list_add(&ctxt->list, &c->moving_context_list);
228 mutex_unlock(&c->moving_context_lock);
Kent Overstreet96a363a2023-10-23 16:21:54 -0400229}
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500230
Kent Overstreet96a363a2023-10-23 16:21:54 -0400231void bch2_move_stats_exit(struct bch_move_stats *stats, struct bch_fs *c)
232{
233 trace_move_data(c, stats);
Kent Overstreet0337cc72022-06-20 15:40:26 -0400234}
235
Kent Overstreet01e95642023-11-20 18:43:48 -0500236void bch2_move_stats_init(struct bch_move_stats *stats, const char *name)
Kent Overstreet0337cc72022-06-20 15:40:26 -0400237{
238 memset(stats, 0, sizeof(*stats));
Kent Overstreet96a363a2023-10-23 16:21:54 -0400239 stats->data_type = BCH_DATA_user;
Kent Overstreet0337cc72022-06-20 15:40:26 -0400240 scnprintf(stats->name, sizeof(stats->name), "%s", name);
241}
242
Kent Overstreet63316902023-10-20 13:32:42 -0400243int bch2_move_extent(struct moving_context *ctxt,
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400244 struct move_bucket_in_flight *bucket_in_flight,
Kent Overstreet63316902023-10-20 13:32:42 -0400245 struct btree_iter *iter,
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400246 struct bkey_s_c k,
Kent Overstreet63316902023-10-20 13:32:42 -0400247 struct bch_io_opts io_opts,
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400248 struct data_update_opts data_opts)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800249{
Kent Overstreet63316902023-10-20 13:32:42 -0400250 struct btree_trans *trans = ctxt->trans;
Kent Overstreetf30dd862020-10-16 21:39:16 -0400251 struct bch_fs *c = trans->c;
Kent Overstreet99aaf572019-07-25 13:52:14 -0400252 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800253 struct moving_io *io;
Kent Overstreet17422372018-09-27 21:08:39 -0400254 const union bch_extent_entry *entry;
255 struct extent_ptr_decoded p;
Kent Overstreet99aaf572019-07-25 13:52:14 -0400256 unsigned sectors = k.k->size, pages;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800257 int ret = -ENOMEM;
258
Kent Overstreet189c1762024-01-15 15:33:39 -0500259 trace_move_extent2(c, k, &io_opts, &data_opts);
260
Kent Overstreet96a363a2023-10-23 16:21:54 -0400261 if (ctxt->stats)
262 ctxt->stats->pos = BBPOS(iter->btree_id, iter->pos);
Kent Overstreet5a217642023-04-20 15:24:07 -0400263
Kent Overstreet1be88792022-10-09 03:32:17 -0400264 bch2_data_update_opts_normalize(k, &data_opts);
265
266 if (!data_opts.rewrite_ptrs &&
267 !data_opts.extra_replicas) {
268 if (data_opts.kill_ptrs)
269 return bch2_extent_drop_ptrs(trans, iter, k, data_opts);
270 return 0;
271 }
272
Kent Overstreeta8b3a672022-11-02 17:12:00 -0400273 /*
274 * Before memory allocations & taking nocow locks in
275 * bch2_data_update_init():
276 */
277 bch2_trans_unlock(trans);
278
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800279 /* write path might have to decompress data: */
Kent Overstreet99aaf572019-07-25 13:52:14 -0400280 bkey_for_each_ptr_decode(k.k, ptrs, p, entry)
Kent Overstreet17422372018-09-27 21:08:39 -0400281 sectors = max_t(unsigned, sectors, p.crc.uncompressed_size);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800282
283 pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
284 io = kzalloc(sizeof(struct moving_io) +
285 sizeof(struct bio_vec) * pages, GFP_KERNEL);
286 if (!io)
287 goto err;
288
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500289 INIT_LIST_HEAD(&io->io_list);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800290 io->write.ctxt = ctxt;
Kent Overstreet99aaf572019-07-25 13:52:14 -0400291 io->read_sectors = k.k->size;
292 io->write_sectors = k.k->size;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800293
294 bio_init(&io->write.op.wbio.bio, NULL, io->bi_inline_vecs, pages, 0);
295 bio_set_prio(&io->write.op.wbio.bio,
296 IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
297
298 if (bch2_bio_alloc_pages(&io->write.op.wbio.bio, sectors << 9,
299 GFP_KERNEL))
300 goto err_free;
301
Kent Overstreetb50dd792019-09-07 13:16:41 -0400302 io->rbio.c = c;
303 io->rbio.opts = io_opts;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800304 bio_init(&io->rbio.bio, NULL, io->bi_inline_vecs, pages, 0);
305 io->rbio.bio.bi_vcnt = pages;
306 bio_set_prio(&io->rbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
307 io->rbio.bio.bi_iter.bi_size = sectors << 9;
308
309 io->rbio.bio.bi_opf = REQ_OP_READ;
Kent Overstreet99aaf572019-07-25 13:52:14 -0400310 io->rbio.bio.bi_iter.bi_sector = bkey_start_offset(k.k);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800311 io->rbio.bio.bi_end_io = move_read_endio;
312
Kent Overstreet7d9f8462023-11-24 21:51:45 -0500313 ret = bch2_data_update_init(trans, iter, ctxt, &io->write, ctxt->wp,
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400314 io_opts, data_opts, iter->btree_id, k);
Kent Overstreet7d9f8462023-11-24 21:51:45 -0500315 if (ret)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800316 goto err_free_pages;
317
Kent Overstreet2f528662023-03-04 02:51:12 -0500318 io->write.op.end_io = move_write_done;
319
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400320 if (ctxt->rate)
321 bch2_ratelimit_increment(ctxt->rate, k.k->size);
322
Kent Overstreet2f528662023-03-04 02:51:12 -0500323 if (ctxt->stats) {
324 atomic64_inc(&ctxt->stats->keys_moved);
325 atomic64_add(k.k->size, &ctxt->stats->sectors_moved);
326 }
327
Kent Overstreet8fcdf812023-02-27 22:58:01 -0500328 if (bucket_in_flight) {
329 io->b = bucket_in_flight;
330 atomic_inc(&io->b->count);
331 }
332
Daniel Hill104c6972022-03-15 21:36:33 +1300333 this_cpu_add(c->counters[BCH_COUNTER_io_move], k.k->size);
Kent Overstreet674cfc22022-08-27 12:48:36 -0400334 this_cpu_add(c->counters[BCH_COUNTER_move_extent_read], k.k->size);
Kent Overstreet5a217642023-04-20 15:24:07 -0400335 trace_move_extent_read2(c, k);
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500336
337 mutex_lock(&ctxt->lock);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800338 atomic_add(io->read_sectors, &ctxt->read_sectors);
Kent Overstreetc782c582023-01-09 01:45:18 -0500339 atomic_inc(&ctxt->read_ios);
Kent Overstreetb9fa3752023-03-11 20:38:46 -0500340
341 list_add_tail(&io->read_list, &ctxt->reads);
342 list_add_tail(&io->io_list, &ctxt->ios);
343 mutex_unlock(&ctxt->lock);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800344
345 /*
346 * dropped by move_read_endio() - guards against use after free of
347 * ctxt when doing wakeup
348 */
349 closure_get(&ctxt->cl);
Kent Overstreet5ff75cc2021-03-14 21:30:08 -0400350 bch2_read_extent(trans, &io->rbio,
351 bkey_start_pos(k.k),
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400352 iter->btree_id, k, 0,
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800353 BCH_READ_NODECODE|
354 BCH_READ_LAST_FRAGMENT);
355 return 0;
356err_free_pages:
357 bio_free_pages(&io->write.op.wbio.bio);
358err_free:
359 kfree(io);
360err:
Kent Overstreet7d9f8462023-11-24 21:51:45 -0500361 if (ret == -BCH_ERR_data_update_done)
362 return 0;
363
Kent Overstreet1b1bd0f2023-11-26 23:11:18 -0500364 if (bch2_err_matches(ret, EROFS) ||
365 bch2_err_matches(ret, BCH_ERR_transaction_restart))
366 return ret;
367
Kent Overstreet74644032023-11-27 22:37:27 -0500368 count_event(c, move_extent_start_fail);
369
Kent Overstreetae4d6122023-11-26 21:13:54 -0500370 if (trace_move_extent_start_fail_enabled()) {
371 struct printbuf buf = PRINTBUF;
372
373 bch2_bkey_val_to_text(&buf, c, k);
374 prt_str(&buf, ": ");
375 prt_str(&buf, bch2_err_str(ret));
376 trace_move_extent_start_fail(c, buf.buf);
377 printbuf_exit(&buf);
378 }
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800379 return ret;
380}
381
Kent Overstreet84809052023-10-21 15:03:05 -0400382struct bch_io_opts *bch2_move_get_io_opts(struct btree_trans *trans,
383 struct per_snapshot_io_opts *io_opts,
384 struct bkey_s_c extent_k)
385{
386 struct bch_fs *c = trans->c;
387 u32 restart_count = trans->restart_count;
388 int ret = 0;
389
390 if (io_opts->cur_inum != extent_k.k->p.inode) {
Kent Overstreet84809052023-10-21 15:03:05 -0400391 io_opts->d.nr = 0;
392
Kent Overstreet5028b902023-12-07 23:33:11 -0500393 ret = for_each_btree_key(trans, iter, BTREE_ID_inodes, POS(0, extent_k.k->p.inode),
Kent Overstreet5dd8c602024-04-07 18:05:34 -0400394 BTREE_ITER_all_snapshots, k, ({
Kent Overstreet84809052023-10-21 15:03:05 -0400395 if (k.k->p.offset != extent_k.k->p.inode)
396 break;
397
398 if (!bkey_is_inode(k.k))
399 continue;
400
401 struct bch_inode_unpacked inode;
402 BUG_ON(bch2_inode_unpack(k, &inode));
403
404 struct snapshot_io_opts_entry e = { .snapshot = k.k->p.snapshot };
405 bch2_inode_opts_get(&e.io_opts, trans->c, &inode);
406
Kent Overstreet27b2df92023-12-07 23:28:26 -0500407 darray_push(&io_opts->d, e);
408 }));
Kent Overstreet84809052023-10-21 15:03:05 -0400409 io_opts->cur_inum = extent_k.k->p.inode;
410 }
411
412 ret = ret ?: trans_was_restarted(trans, restart_count);
413 if (ret)
414 return ERR_PTR(ret);
415
Kent Overstreetdefd9e32023-12-16 21:40:26 -0500416 if (extent_k.k->p.snapshot)
Kent Overstreet84809052023-10-21 15:03:05 -0400417 darray_for_each(io_opts->d, i)
418 if (bch2_snapshot_is_ancestor(c, extent_k.k->p.snapshot, i->snapshot))
419 return &i->io_opts;
Kent Overstreet84809052023-10-21 15:03:05 -0400420
421 return &io_opts->fs_io_opts;
422}
423
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400424int bch2_move_get_io_opts_one(struct btree_trans *trans,
425 struct bch_io_opts *io_opts,
426 struct bkey_s_c extent_k)
Kent Overstreet883d9702021-03-16 18:08:10 -0400427{
Kent Overstreet67e0dd82021-08-30 15:18:31 -0400428 struct btree_iter iter;
Kent Overstreet883d9702021-03-16 18:08:10 -0400429 struct bkey_s_c k;
430 int ret;
431
Kent Overstreet84809052023-10-21 15:03:05 -0400432 /* reflink btree? */
433 if (!extent_k.k->p.inode) {
434 *io_opts = bch2_opts_to_inode_opts(trans->c->opts);
435 return 0;
Kent Overstreet443d2762021-05-23 18:42:51 -0400436 }
437
Kent Overstreet84809052023-10-21 15:03:05 -0400438 k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes,
439 SPOS(0, extent_k.k->p.inode, extent_k.k->p.snapshot),
Kent Overstreet5dd8c602024-04-07 18:05:34 -0400440 BTREE_ITER_cached);
Kent Overstreet84809052023-10-21 15:03:05 -0400441 ret = bkey_err(k);
442 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
443 return ret;
Kent Overstreet883d9702021-03-16 18:08:10 -0400444
Kent Overstreet84809052023-10-21 15:03:05 -0400445 if (!ret && bkey_is_inode(k.k)) {
446 struct bch_inode_unpacked inode;
447 bch2_inode_unpack(k, &inode);
448 bch2_inode_opts_get(io_opts, trans->c, &inode);
449 } else {
450 *io_opts = bch2_opts_to_inode_opts(trans->c->opts);
451 }
452
Kent Overstreet67e0dd82021-08-30 15:18:31 -0400453 bch2_trans_iter_exit(trans, &iter);
Kent Overstreet84809052023-10-21 15:03:05 -0400454 return 0;
Kent Overstreet883d9702021-03-16 18:08:10 -0400455}
456
Kent Overstreet63316902023-10-20 13:32:42 -0400457int bch2_move_ratelimit(struct moving_context *ctxt)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800458{
Kent Overstreet63316902023-10-20 13:32:42 -0400459 struct bch_fs *c = ctxt->trans->c;
Kent Overstreet415e5102023-11-28 16:33:52 -0500460 bool is_kthread = current->flags & PF_KTHREAD;
Daniel Hillc91996c2022-06-16 02:06:43 +1200461 u64 delay;
462
Kent Overstreet50e029c2023-11-20 17:24:32 -0500463 if (ctxt->wait_on_copygc && c->copygc_running) {
464 bch2_moving_ctxt_flush_all(ctxt);
Daniel Hillc91996c2022-06-16 02:06:43 +1200465 wait_event_killable(c->copygc_running_wq,
466 !c->copygc_running ||
Kent Overstreet415e5102023-11-28 16:33:52 -0500467 (is_kthread && kthread_should_stop()));
Daniel Hillc91996c2022-06-16 02:06:43 +1200468 }
469
470 do {
Kent Overstreet0337cc72022-06-20 15:40:26 -0400471 delay = ctxt->rate ? bch2_ratelimit_delay(ctxt->rate) : 0;
Daniel Hillc91996c2022-06-16 02:06:43 +1200472
Kent Overstreet415e5102023-11-28 16:33:52 -0500473 if (is_kthread && kthread_should_stop())
Daniel Hillc91996c2022-06-16 02:06:43 +1200474 return 1;
Daniel Hillc91996c2022-06-16 02:06:43 +1200475
476 if (delay)
Kent Overstreet261af2f2023-11-22 23:44:47 -0500477 move_ctxt_wait_event_timeout(ctxt,
Kent Overstreet415e5102023-11-28 16:33:52 -0500478 freezing(current) ||
479 (is_kthread && kthread_should_stop()),
Kent Overstreet261af2f2023-11-22 23:44:47 -0500480 delay);
Daniel Hillc91996c2022-06-16 02:06:43 +1200481
482 if (unlikely(freezing(current))) {
Kent Overstreet50e029c2023-11-20 17:24:32 -0500483 bch2_moving_ctxt_flush_all(ctxt);
Daniel Hillc91996c2022-06-16 02:06:43 +1200484 try_to_freeze();
485 }
486 } while (delay);
487
Kent Overstreetc782c582023-01-09 01:45:18 -0500488 /*
489 * XXX: these limits really ought to be per device, SSDs and hard drives
490 * will want different limits
491 */
Kent Overstreet63316902023-10-20 13:32:42 -0400492 move_ctxt_wait_event(ctxt,
Kent Overstreetc782c582023-01-09 01:45:18 -0500493 atomic_read(&ctxt->write_sectors) < c->opts.move_bytes_in_flight >> 9 &&
494 atomic_read(&ctxt->read_sectors) < c->opts.move_bytes_in_flight >> 9 &&
495 atomic_read(&ctxt->write_ios) < c->opts.move_ios_in_flight &&
496 atomic_read(&ctxt->read_ios) < c->opts.move_ios_in_flight);
Daniel Hillc91996c2022-06-16 02:06:43 +1200497
498 return 0;
499}
500
Kent Overstreet63316902023-10-20 13:32:42 -0400501static int bch2_move_data_btree(struct moving_context *ctxt,
502 struct bpos start,
503 struct bpos end,
504 move_pred_fn pred, void *arg,
505 enum btree_id btree_id)
Daniel Hillc91996c2022-06-16 02:06:43 +1200506{
Kent Overstreet63316902023-10-20 13:32:42 -0400507 struct btree_trans *trans = ctxt->trans;
508 struct bch_fs *c = trans->c;
Kent Overstreet84809052023-10-21 15:03:05 -0400509 struct per_snapshot_io_opts snapshot_io_opts;
510 struct bch_io_opts *io_opts;
Kent Overstreet07a10062020-12-17 15:08:58 -0500511 struct bkey_buf sk;
Kent Overstreet67e0dd82021-08-30 15:18:31 -0400512 struct btree_iter iter;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800513 struct bkey_s_c k;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400514 struct data_update_opts data_opts;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800515 int ret = 0, ret2;
516
Kent Overstreet84809052023-10-21 15:03:05 -0400517 per_snapshot_io_opts_init(&snapshot_io_opts, c);
Kent Overstreet07a10062020-12-17 15:08:58 -0500518 bch2_bkey_buf_init(&sk);
Kent Overstreet424eb882019-03-25 15:10:15 -0400519
Kent Overstreet2f528662023-03-04 02:51:12 -0500520 if (ctxt->stats) {
521 ctxt->stats->data_type = BCH_DATA_user;
Kent Overstreetd5eade92023-10-23 15:36:45 -0400522 ctxt->stats->pos = BBPOS(btree_id, start);
Kent Overstreet2f528662023-03-04 02:51:12 -0500523 }
Kent Overstreet424eb882019-03-25 15:10:15 -0400524
Kent Overstreet319fef22024-06-03 18:00:48 -0400525 bch2_trans_begin(trans);
Kent Overstreet6bd68ec2023-09-12 17:16:02 -0400526 bch2_trans_iter_init(trans, &iter, btree_id, start,
Kent Overstreet5dd8c602024-04-07 18:05:34 -0400527 BTREE_ITER_prefetch|
528 BTREE_ITER_all_snapshots);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800529
Kent Overstreet0337cc72022-06-20 15:40:26 -0400530 if (ctxt->rate)
531 bch2_ratelimit_reset(ctxt->rate);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800532
Kent Overstreet63316902023-10-20 13:32:42 -0400533 while (!bch2_move_ratelimit(ctxt)) {
Kent Overstreet6bd68ec2023-09-12 17:16:02 -0400534 bch2_trans_begin(trans);
Kent Overstreet700c25b2021-07-24 20:24:10 -0400535
Kent Overstreet67e0dd82021-08-30 15:18:31 -0400536 k = bch2_btree_iter_peek(&iter);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800537 if (!k.k)
538 break;
Kent Overstreet8ede9912022-01-09 20:52:10 -0500539
Kent Overstreet0f238362019-03-27 22:03:30 -0400540 ret = bkey_err(k);
Kent Overstreet549d1732022-07-17 23:06:38 -0400541 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
Kent Overstreet8ede9912022-01-09 20:52:10 -0500542 continue;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800543 if (ret)
544 break;
Kent Overstreet8ede9912022-01-09 20:52:10 -0500545
Kent Overstreete88a75e2022-11-24 03:12:22 -0500546 if (bkey_ge(bkey_start_pos(k.k), end))
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800547 break;
548
Kent Overstreet2f528662023-03-04 02:51:12 -0500549 if (ctxt->stats)
Kent Overstreetd5eade92023-10-23 15:36:45 -0400550 ctxt->stats->pos = BBPOS(iter.btree_id, iter.pos);
Kent Overstreet8ede9912022-01-09 20:52:10 -0500551
Kent Overstreet8d842602019-09-07 16:13:20 -0400552 if (!bkey_extent_is_direct_data(k.k))
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800553 goto next_nondata;
554
Kent Overstreet84809052023-10-21 15:03:05 -0400555 io_opts = bch2_move_get_io_opts(trans, &snapshot_io_opts, k);
556 ret = PTR_ERR_OR_ZERO(io_opts);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400557 if (ret)
558 continue;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800559
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400560 memset(&data_opts, 0, sizeof(data_opts));
Kent Overstreet84809052023-10-21 15:03:05 -0400561 if (!pred(c, arg, k, io_opts, &data_opts))
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800562 goto next;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800563
Kent Overstreeteb331fe2022-02-15 00:06:59 -0500564 /*
565 * The iterator gets unlocked by __bch2_read_extent - need to
566 * save a copy of @k elsewhere:
Kent Overstreet3e3e02e2022-10-19 18:31:33 -0400567 */
Kent Overstreet07a10062020-12-17 15:08:58 -0500568 bch2_bkey_buf_reassemble(&sk, c, k);
Kent Overstreet35189e02019-11-09 16:01:15 -0500569 k = bkey_i_to_s_c(sk.k);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800570
Kent Overstreet63316902023-10-20 13:32:42 -0400571 ret2 = bch2_move_extent(ctxt, NULL, &iter, k, *io_opts, data_opts);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800572 if (ret2) {
Kent Overstreet549d1732022-07-17 23:06:38 -0400573 if (bch2_err_matches(ret2, BCH_ERR_transaction_restart))
Kent Overstreetf0e70012020-12-20 21:42:19 -0500574 continue;
Kent Overstreetf0e70012020-12-20 21:42:19 -0500575
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800576 if (ret2 == -ENOMEM) {
577 /* memory allocation failure, wait for some IO to finish */
Kent Overstreet63316902023-10-20 13:32:42 -0400578 bch2_move_ctxt_wait_for_io(ctxt);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800579 continue;
580 }
581
582 /* XXX signal failure */
583 goto next;
584 }
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800585next:
Kent Overstreet2f528662023-03-04 02:51:12 -0500586 if (ctxt->stats)
587 atomic64_add(k.k->size, &ctxt->stats->sectors_seen);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800588next_nondata:
Kent Overstreet67e0dd82021-08-30 15:18:31 -0400589 bch2_btree_iter_advance(&iter);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800590 }
Kent Overstreet50dc0f62021-03-19 20:29:11 -0400591
Kent Overstreet6bd68ec2023-09-12 17:16:02 -0400592 bch2_trans_iter_exit(trans, &iter);
Kent Overstreet07a10062020-12-17 15:08:58 -0500593 bch2_bkey_buf_exit(&sk, c);
Kent Overstreet84809052023-10-21 15:03:05 -0400594 per_snapshot_io_opts_exit(&snapshot_io_opts);
Kent Overstreet76426092019-08-16 09:59:56 -0400595
596 return ret;
597}
598
Kent Overstreet63316902023-10-20 13:32:42 -0400599int __bch2_move_data(struct moving_context *ctxt,
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400600 struct bbpos start,
601 struct bbpos end,
602 move_pred_fn pred, void *arg)
603{
Kent Overstreet63316902023-10-20 13:32:42 -0400604 struct bch_fs *c = ctxt->trans->c;
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400605 enum btree_id id;
606 int ret = 0;
607
608 for (id = start.btree;
609 id <= min_t(unsigned, end.btree, btree_id_nr_alive(c) - 1);
610 id++) {
Kent Overstreetd5eade92023-10-23 15:36:45 -0400611 ctxt->stats->pos = BBPOS(id, POS_MIN);
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400612
613 if (!btree_type_has_ptrs(id) ||
614 !bch2_btree_id_root(c, id)->b)
615 continue;
616
Kent Overstreet63316902023-10-20 13:32:42 -0400617 ret = bch2_move_data_btree(ctxt,
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400618 id == start.btree ? start.pos : POS_MIN,
619 id == end.btree ? end.pos : POS_MAX,
620 pred, arg, id);
621 if (ret)
622 break;
623 }
624
625 return ret;
626}
627
Kent Overstreet76426092019-08-16 09:59:56 -0400628int bch2_move_data(struct bch_fs *c,
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400629 struct bbpos start,
630 struct bbpos end,
Kent Overstreet76426092019-08-16 09:59:56 -0400631 struct bch_ratelimit *rate,
Daniel Hillc91996c2022-06-16 02:06:43 +1200632 struct bch_move_stats *stats,
Kent Overstreet0337cc72022-06-20 15:40:26 -0400633 struct write_point_specifier wp,
634 bool wait_on_copygc,
635 move_pred_fn pred, void *arg)
Kent Overstreet76426092019-08-16 09:59:56 -0400636{
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400637
Kent Overstreet0337cc72022-06-20 15:40:26 -0400638 struct moving_context ctxt;
Kent Overstreeta0bfe3b2023-10-20 13:32:42 -0400639 int ret;
Kent Overstreet76426092019-08-16 09:59:56 -0400640
Kent Overstreet0337cc72022-06-20 15:40:26 -0400641 bch2_moving_ctxt_init(&ctxt, c, rate, stats, wp, wait_on_copygc);
Kent Overstreet63316902023-10-20 13:32:42 -0400642 ret = __bch2_move_data(&ctxt, start, end, pred, arg);
Kent Overstreet0337cc72022-06-20 15:40:26 -0400643 bch2_moving_ctxt_exit(&ctxt);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800644
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800645 return ret;
646}
647
Daniel Hill74529332023-11-26 20:26:07 +1300648int bch2_evacuate_bucket(struct moving_context *ctxt,
Kent Overstreet8fcdf812023-02-27 22:58:01 -0500649 struct move_bucket_in_flight *bucket_in_flight,
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400650 struct bpos bucket, int gen,
651 struct data_update_opts _data_opts)
652{
Kent Overstreet63316902023-10-20 13:32:42 -0400653 struct btree_trans *trans = ctxt->trans;
654 struct bch_fs *c = trans->c;
Kent Overstreet415e5102023-11-28 16:33:52 -0500655 bool is_kthread = current->flags & PF_KTHREAD;
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400656 struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400657 struct btree_iter iter;
658 struct bkey_buf sk;
659 struct bch_backpointer bp;
660 struct bch_alloc_v4 a_convert;
661 const struct bch_alloc_v4 *a;
662 struct bkey_s_c k;
663 struct data_update_opts data_opts;
664 unsigned dirty_sectors, bucket_size;
Kent Overstreet80c33082022-12-05 10:24:19 -0500665 u64 fragmentation;
Kent Overstreet62a03552023-03-31 16:24:45 -0400666 struct bpos bp_pos = POS_MIN;
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400667 int ret = 0;
668
Kent Overstreetcb4d3402024-04-30 15:46:45 -0400669 struct bch_dev *ca = bch2_dev_tryget(c, bucket.inode);
670 if (!ca)
671 return 0;
672
Kent Overstreet5a217642023-04-20 15:24:07 -0400673 trace_bucket_evacuate(c, &bucket);
674
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400675 bch2_bkey_buf_init(&sk);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400676
Kent Overstreet3e36e572023-03-19 14:13:17 -0400677 /*
678 * We're not run in a context that handles transaction restarts:
679 */
680 bch2_trans_begin(trans);
681
Kent Overstreet80c33082022-12-05 10:24:19 -0500682 bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc,
Kent Overstreet5dd8c602024-04-07 18:05:34 -0400683 bucket, BTREE_ITER_cached);
Kent Overstreet80c33082022-12-05 10:24:19 -0500684 ret = lockrestart_do(trans,
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400685 bkey_err(k = bch2_btree_iter_peek_slot(&iter)));
Kent Overstreet80c33082022-12-05 10:24:19 -0500686 bch2_trans_iter_exit(trans, &iter);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400687
Kent Overstreetcf904c82023-12-16 22:43:41 -0500688 bch_err_msg(c, ret, "looking up alloc key");
689 if (ret)
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400690 goto err;
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400691
692 a = bch2_alloc_to_v4(k, &a_convert);
Kent Overstreetdafff7e2023-11-23 18:05:18 -0500693 dirty_sectors = bch2_bucket_sectors_dirty(*a);
Kent Overstreetcb4d3402024-04-30 15:46:45 -0400694 bucket_size = ca->mi.bucket_size;
Kent Overstreet80c33082022-12-05 10:24:19 -0500695 fragmentation = a->fragmentation_lru;
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400696
Kent Overstreetcb13f472023-11-02 20:36:00 -0400697 ret = bch2_btree_write_buffer_tryflush(trans);
698 bch_err_msg(c, ret, "flushing btree write buffer");
699 if (ret)
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400700 goto err;
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400701
Kent Overstreet63316902023-10-20 13:32:42 -0400702 while (!(ret = bch2_move_ratelimit(ctxt))) {
Kent Overstreet415e5102023-11-28 16:33:52 -0500703 if (is_kthread && kthread_should_stop())
704 break;
705
Kent Overstreet80c33082022-12-05 10:24:19 -0500706 bch2_trans_begin(trans);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400707
Kent Overstreet633cf0692024-04-30 16:50:28 -0400708 ret = bch2_get_next_backpointer(trans, ca, bucket, gen,
Kent Overstreet62a03552023-03-31 16:24:45 -0400709 &bp_pos, &bp,
Kent Overstreet5dd8c602024-04-07 18:05:34 -0400710 BTREE_ITER_cached);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400711 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
712 continue;
713 if (ret)
714 goto err;
Kent Overstreet62a03552023-03-31 16:24:45 -0400715 if (bkey_eq(bp_pos, POS_MAX))
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400716 break;
717
718 if (!bp.level) {
Kent Overstreet62a03552023-03-31 16:24:45 -0400719 k = bch2_backpointer_get_key(trans, &iter, bp_pos, bp, 0);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400720 ret = bkey_err(k);
721 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
722 continue;
723 if (ret)
724 goto err;
725 if (!k.k)
726 goto next;
727
728 bch2_bkey_buf_reassemble(&sk, c, k);
729 k = bkey_i_to_s_c(sk.k);
730
Kent Overstreet84809052023-10-21 15:03:05 -0400731 ret = bch2_move_get_io_opts_one(trans, &io_opts, k);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400732 if (ret) {
Kent Overstreet80c33082022-12-05 10:24:19 -0500733 bch2_trans_iter_exit(trans, &iter);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400734 continue;
735 }
736
737 data_opts = _data_opts;
738 data_opts.target = io_opts.background_target;
739 data_opts.rewrite_ptrs = 0;
740
Kent Overstreet0beebd92023-12-21 15:47:15 -0500741 unsigned i = 0;
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400742 bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) {
Kent Overstreet3f5d3fb2023-03-10 18:00:10 -0500743 if (ptr->dev == bucket.inode) {
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400744 data_opts.rewrite_ptrs |= 1U << i;
Kent Overstreet3f5d3fb2023-03-10 18:00:10 -0500745 if (ptr->cached) {
746 bch2_trans_iter_exit(trans, &iter);
747 goto next;
748 }
749 }
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400750 i++;
751 }
752
Kent Overstreet63316902023-10-20 13:32:42 -0400753 ret = bch2_move_extent(ctxt, bucket_in_flight,
754 &iter, k, io_opts, data_opts);
Kent Overstreet80c33082022-12-05 10:24:19 -0500755 bch2_trans_iter_exit(trans, &iter);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400756
757 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
758 continue;
759 if (ret == -ENOMEM) {
760 /* memory allocation failure, wait for some IO to finish */
Kent Overstreet63316902023-10-20 13:32:42 -0400761 bch2_move_ctxt_wait_for_io(ctxt);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400762 continue;
763 }
764 if (ret)
765 goto err;
766
Kent Overstreet2f528662023-03-04 02:51:12 -0500767 if (ctxt->stats)
768 atomic64_add(k.k->size, &ctxt->stats->sectors_seen);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400769 } else {
770 struct btree *b;
771
Kent Overstreet62a03552023-03-31 16:24:45 -0400772 b = bch2_backpointer_get_node(trans, &iter, bp_pos, bp);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400773 ret = PTR_ERR_OR_ZERO(b);
774 if (ret == -BCH_ERR_backpointer_to_overwritten_btree_node)
775 continue;
776 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
777 continue;
778 if (ret)
779 goto err;
780 if (!b)
781 goto next;
782
Kent Overstreet652bc7f2024-05-24 18:04:22 -0400783 unsigned sectors = btree_ptr_sectors_written(bkey_i_to_s_c(&b->key));
Kent Overstreetec4edd72024-01-16 13:29:59 -0500784
Kent Overstreet80c33082022-12-05 10:24:19 -0500785 ret = bch2_btree_node_rewrite(trans, &iter, b, 0);
786 bch2_trans_iter_exit(trans, &iter);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400787
788 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
789 continue;
790 if (ret)
791 goto err;
792
793 if (ctxt->rate)
Kent Overstreetec4edd72024-01-16 13:29:59 -0500794 bch2_ratelimit_increment(ctxt->rate, sectors);
Kent Overstreet2f528662023-03-04 02:51:12 -0500795 if (ctxt->stats) {
Kent Overstreetec4edd72024-01-16 13:29:59 -0500796 atomic64_add(sectors, &ctxt->stats->sectors_seen);
797 atomic64_add(sectors, &ctxt->stats->sectors_moved);
Kent Overstreet2f528662023-03-04 02:51:12 -0500798 }
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400799 }
800next:
Kent Overstreet62a03552023-03-31 16:24:45 -0400801 bp_pos = bpos_nosnap_successor(bp_pos);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400802 }
803
Kent Overstreet80c33082022-12-05 10:24:19 -0500804 trace_evacuate_bucket(c, &bucket, dirty_sectors, bucket_size, fragmentation, ret);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400805err:
Kent Overstreetcb4d3402024-04-30 15:46:45 -0400806 bch2_dev_put(ca);
Kent Overstreet8e3f9132022-03-18 00:42:09 -0400807 bch2_bkey_buf_exit(&sk, c);
808 return ret;
809}
810
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400811typedef bool (*move_btree_pred)(struct bch_fs *, void *,
812 struct btree *, struct bch_io_opts *,
813 struct data_update_opts *);
Kent Overstreet1889ad52021-03-14 19:01:14 -0400814
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800815static int bch2_move_btree(struct bch_fs *c,
Kent Overstreet3c843a62023-11-20 18:52:33 -0500816 struct bbpos start,
817 struct bbpos end,
Kent Overstreet1889ad52021-03-14 19:01:14 -0400818 move_btree_pred pred, void *arg,
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800819 struct bch_move_stats *stats)
820{
Kent Overstreet1889ad52021-03-14 19:01:14 -0400821 bool kthread = (current->flags & PF_KTHREAD) != 0;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800822 struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
Kent Overstreet96a363a2023-10-23 16:21:54 -0400823 struct moving_context ctxt;
824 struct btree_trans *trans;
Kent Overstreet67e0dd82021-08-30 15:18:31 -0400825 struct btree_iter iter;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800826 struct btree *b;
Kent Overstreet3c843a62023-11-20 18:52:33 -0500827 enum btree_id btree;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400828 struct data_update_opts data_opts;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800829 int ret = 0;
830
Kent Overstreet96a363a2023-10-23 16:21:54 -0400831 bch2_moving_ctxt_init(&ctxt, c, NULL, stats,
832 writepoint_ptr(&c->btree_write_point),
833 true);
834 trans = ctxt.trans;
Kent Overstreet424eb882019-03-25 15:10:15 -0400835
Kent Overstreet89fd25b2020-07-09 18:28:11 -0400836 stats->data_type = BCH_DATA_btree;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800837
Kent Overstreet3c843a62023-11-20 18:52:33 -0500838 for (btree = start.btree;
839 btree <= min_t(unsigned, end.btree, btree_id_nr_alive(c) - 1);
840 btree ++) {
841 stats->pos = BBPOS(btree, POS_MIN);
Kent Overstreet424eb882019-03-25 15:10:15 -0400842
Kent Overstreet3c843a62023-11-20 18:52:33 -0500843 if (!bch2_btree_id_root(c, btree)->b)
Kent Overstreetfaa6cb62023-06-28 22:09:13 -0400844 continue;
845
Kent Overstreet3c843a62023-11-20 18:52:33 -0500846 bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, 0,
Kent Overstreet5dd8c602024-04-07 18:05:34 -0400847 BTREE_ITER_prefetch);
Kent Overstreetd355c6f2021-10-19 14:20:50 -0400848retry:
Kent Overstreetb71717d2021-10-19 15:11:45 -0400849 ret = 0;
Kent Overstreet6bd68ec2023-09-12 17:16:02 -0400850 while (bch2_trans_begin(trans),
Kent Overstreetd355c6f2021-10-19 14:20:50 -0400851 (b = bch2_btree_iter_peek_node(&iter)) &&
852 !(ret = PTR_ERR_OR_ZERO(b))) {
Kent Overstreet1889ad52021-03-14 19:01:14 -0400853 if (kthread && kthread_should_stop())
Kent Overstreet7b7278b2021-04-20 20:21:39 -0400854 break;
Kent Overstreet1889ad52021-03-14 19:01:14 -0400855
Kent Overstreet3c843a62023-11-20 18:52:33 -0500856 if ((cmp_int(btree, end.btree) ?:
857 bpos_cmp(b->key.k.p, end.pos)) > 0)
Kent Overstreet1889ad52021-03-14 19:01:14 -0400858 break;
859
Kent Overstreetd5eade92023-10-23 15:36:45 -0400860 stats->pos = BBPOS(iter.btree_id, iter.pos);
Kent Overstreet424eb882019-03-25 15:10:15 -0400861
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400862 if (!pred(c, arg, b, &io_opts, &data_opts))
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800863 goto next;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800864
Kent Overstreet6bd68ec2023-09-12 17:16:02 -0400865 ret = bch2_btree_node_rewrite(trans, &iter, b, 0) ?: ret;
Kent Overstreet549d1732022-07-17 23:06:38 -0400866 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
Kent Overstreetf3cf0992021-10-24 16:59:33 -0400867 continue;
868 if (ret)
869 break;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800870next:
Kent Overstreet4b09ef12021-10-07 18:08:01 -0400871 bch2_btree_iter_next_node(&iter);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800872 }
Kent Overstreet549d1732022-07-17 23:06:38 -0400873 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
Kent Overstreetd355c6f2021-10-19 14:20:50 -0400874 goto retry;
875
Kent Overstreet6bd68ec2023-09-12 17:16:02 -0400876 bch2_trans_iter_exit(trans, &iter);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800877
Kent Overstreet7b7278b2021-04-20 20:21:39 -0400878 if (kthread && kthread_should_stop())
879 break;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800880 }
Kent Overstreet7b7278b2021-04-20 20:21:39 -0400881
Kent Overstreet96a363a2023-10-23 16:21:54 -0400882 bch_err_fn(c, ret);
883 bch2_moving_ctxt_exit(&ctxt);
Kent Overstreetc0960602022-04-17 17:30:49 -0400884 bch2_btree_interior_updates_flush(c);
Kent Overstreet23af4982021-10-24 17:00:33 -0400885
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800886 return ret;
887}
888
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400889static bool rereplicate_pred(struct bch_fs *c, void *arg,
890 struct bkey_s_c k,
891 struct bch_io_opts *io_opts,
892 struct data_update_opts *data_opts)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800893{
Kent Overstreet26609b62018-11-01 15:10:01 -0400894 unsigned nr_good = bch2_bkey_durability(c, k);
Kent Overstreete8bde782021-10-12 14:15:45 -0400895 unsigned replicas = bkey_is_btree_ptr(k.k)
896 ? c->opts.metadata_replicas
897 : io_opts->data_replicas;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800898
Kent Overstreetfdccb242024-06-02 22:25:18 -0400899 rcu_read_lock();
900 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
901 unsigned i = 0;
902 bkey_for_each_ptr(ptrs, ptr) {
903 struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev);
904 if (!ptr->cached &&
905 (!ca || !ca->mi.durability))
906 data_opts->kill_ptrs |= BIT(i);
907 i++;
908 }
909 rcu_read_unlock();
910
911 if (!data_opts->kill_ptrs &&
912 (!nr_good || nr_good >= replicas))
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400913 return false;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800914
915 data_opts->target = 0;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400916 data_opts->extra_replicas = replicas - nr_good;
Kent Overstreet26609b62018-11-01 15:10:01 -0400917 data_opts->btree_insert_flags = 0;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400918 return true;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800919}
920
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400921static bool migrate_pred(struct bch_fs *c, void *arg,
922 struct bkey_s_c k,
923 struct bch_io_opts *io_opts,
924 struct data_update_opts *data_opts)
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800925{
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400926 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800927 struct bch_ioctl_data *op = arg;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400928 unsigned i = 0;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800929
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400930 data_opts->rewrite_ptrs = 0;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800931 data_opts->target = 0;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400932 data_opts->extra_replicas = 0;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800933 data_opts->btree_insert_flags = 0;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400934
935 bkey_for_each_ptr(ptrs, ptr) {
936 if (ptr->dev == op->migrate.dev)
937 data_opts->rewrite_ptrs |= 1U << i;
938 i++;
939 }
940
Kent Overstreet3e3e02e2022-10-19 18:31:33 -0400941 return data_opts->rewrite_ptrs != 0;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -0800942}
943
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400944static bool rereplicate_btree_pred(struct bch_fs *c, void *arg,
945 struct btree *b,
946 struct bch_io_opts *io_opts,
947 struct data_update_opts *data_opts)
Kent Overstreet1889ad52021-03-14 19:01:14 -0400948{
949 return rereplicate_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts);
950}
951
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400952static bool migrate_btree_pred(struct bch_fs *c, void *arg,
953 struct btree *b,
954 struct bch_io_opts *io_opts,
955 struct data_update_opts *data_opts)
Kent Overstreet1889ad52021-03-14 19:01:14 -0400956{
957 return migrate_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts);
958}
959
Kent Overstreet0ec5b3b2024-05-05 22:44:27 -0400960/*
961 * Ancient versions of bcachefs produced packed formats which could represent
962 * keys that the in memory format cannot represent; this checks for those
963 * formats so we can get rid of them.
964 */
Kent Overstreete01dacf2021-03-20 23:55:36 -0400965static bool bformat_needs_redo(struct bkey_format *f)
966{
Kent Overstreet61692c72024-05-08 10:58:26 -0400967 for (unsigned i = 0; i < f->nr_fields; i++)
968 if (bch2_bkey_format_field_overflows(f, i))
Kent Overstreete01dacf2021-03-20 23:55:36 -0400969 return true;
970
Kent Overstreete01dacf2021-03-20 23:55:36 -0400971 return false;
972}
973
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400974static bool rewrite_old_nodes_pred(struct bch_fs *c, void *arg,
975 struct btree *b,
976 struct bch_io_opts *io_opts,
977 struct data_update_opts *data_opts)
Kent Overstreet1889ad52021-03-14 19:01:14 -0400978{
979 if (b->version_ondisk != c->sb.version ||
Kent Overstreete01dacf2021-03-20 23:55:36 -0400980 btree_node_need_rewrite(b) ||
981 bformat_needs_redo(&b->format)) {
Kent Overstreet1889ad52021-03-14 19:01:14 -0400982 data_opts->target = 0;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400983 data_opts->extra_replicas = 0;
Kent Overstreet1889ad52021-03-14 19:01:14 -0400984 data_opts->btree_insert_flags = 0;
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400985 return true;
Kent Overstreet1889ad52021-03-14 19:01:14 -0400986 }
987
Kent Overstreet7f5c5d22022-06-13 19:17:45 -0400988 return false;
Kent Overstreet1889ad52021-03-14 19:01:14 -0400989}
990
Kent Overstreeta4805d62021-03-22 18:39:16 -0400991int bch2_scan_old_btree_nodes(struct bch_fs *c, struct bch_move_stats *stats)
992{
993 int ret;
994
995 ret = bch2_move_btree(c,
Kent Overstreet3c843a62023-11-20 18:52:33 -0500996 BBPOS_MIN,
997 BBPOS_MAX,
Kent Overstreeta4805d62021-03-22 18:39:16 -0400998 rewrite_old_nodes_pred, c, stats);
999 if (!ret) {
1000 mutex_lock(&c->sb_lock);
Kent Overstreetc0ebe3e2021-05-23 17:04:13 -04001001 c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_extents_above_btree_updates_done);
1002 c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_bformat_overflow_done);
Kent Overstreeta4805d62021-03-22 18:39:16 -04001003 c->disk_sb.sb->version_min = c->disk_sb.sb->version;
1004 bch2_write_super(c);
1005 mutex_unlock(&c->sb_lock);
1006 }
1007
Kent Overstreet96a363a2023-10-23 16:21:54 -04001008 bch_err_fn(c, ret);
Kent Overstreeta4805d62021-03-22 18:39:16 -04001009 return ret;
1010}
1011
Kent Overstreetba11c7d2023-11-20 19:12:40 -05001012static bool drop_extra_replicas_pred(struct bch_fs *c, void *arg,
1013 struct bkey_s_c k,
1014 struct bch_io_opts *io_opts,
1015 struct data_update_opts *data_opts)
1016{
1017 unsigned durability = bch2_bkey_durability(c, k);
1018 unsigned replicas = bkey_is_btree_ptr(k.k)
1019 ? c->opts.metadata_replicas
1020 : io_opts->data_replicas;
1021 const union bch_extent_entry *entry;
1022 struct extent_ptr_decoded p;
1023 unsigned i = 0;
1024
Kent Overstreet302c9802024-04-30 20:54:20 -04001025 rcu_read_lock();
Kent Overstreetba11c7d2023-11-20 19:12:40 -05001026 bkey_for_each_ptr_decode(k.k, bch2_bkey_ptrs_c(k), p, entry) {
1027 unsigned d = bch2_extent_ptr_durability(c, &p);
1028
1029 if (d && durability - d >= replicas) {
1030 data_opts->kill_ptrs |= BIT(i);
1031 durability -= d;
1032 }
1033
1034 i++;
1035 }
Kent Overstreet302c9802024-04-30 20:54:20 -04001036 rcu_read_unlock();
Kent Overstreetba11c7d2023-11-20 19:12:40 -05001037
1038 return data_opts->kill_ptrs != 0;
1039}
1040
1041static bool drop_extra_replicas_btree_pred(struct bch_fs *c, void *arg,
1042 struct btree *b,
1043 struct bch_io_opts *io_opts,
1044 struct data_update_opts *data_opts)
1045{
1046 return drop_extra_replicas_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts);
1047}
1048
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001049int bch2_data_job(struct bch_fs *c,
1050 struct bch_move_stats *stats,
1051 struct bch_ioctl_data op)
1052{
Kent Overstreet3c843a62023-11-20 18:52:33 -05001053 struct bbpos start = BBPOS(op.start_btree, op.start_pos);
1054 struct bbpos end = BBPOS(op.end_btree, op.end_pos);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001055 int ret = 0;
1056
Kent Overstreet01e95642023-11-20 18:43:48 -05001057 if (op.op >= BCH_DATA_OP_NR)
1058 return -EINVAL;
1059
1060 bch2_move_stats_init(stats, bch2_data_ops_strs[op.op]);
1061
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001062 switch (op.op) {
Kent Overstreet01e95642023-11-20 18:43:48 -05001063 case BCH_DATA_OP_rereplicate:
Kent Overstreet89fd25b2020-07-09 18:28:11 -04001064 stats->data_type = BCH_DATA_journal;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001065 ret = bch2_journal_flush_device_pins(&c->journal, -1);
Kent Overstreet3c843a62023-11-20 18:52:33 -05001066 ret = bch2_move_btree(c, start, end,
Kent Overstreet1889ad52021-03-14 19:01:14 -04001067 rereplicate_btree_pred, c, stats) ?: ret;
Kent Overstreet3c843a62023-11-20 18:52:33 -05001068 ret = bch2_move_data(c, start, end,
Kent Overstreet0337cc72022-06-20 15:40:26 -04001069 NULL,
1070 stats,
1071 writepoint_hashed((unsigned long) current),
1072 true,
1073 rereplicate_pred, c) ?: ret;
Kent Overstreetae0ff7b2019-04-30 17:15:39 -04001074 ret = bch2_replicas_gc2(c) ?: ret;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001075 break;
Kent Overstreet01e95642023-11-20 18:43:48 -05001076 case BCH_DATA_OP_migrate:
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001077 if (op.migrate.dev >= c->sb.nr_devices)
1078 return -EINVAL;
1079
Kent Overstreet89fd25b2020-07-09 18:28:11 -04001080 stats->data_type = BCH_DATA_journal;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001081 ret = bch2_journal_flush_device_pins(&c->journal, op.migrate.dev);
Kent Overstreet3c843a62023-11-20 18:52:33 -05001082 ret = bch2_move_btree(c, start, end,
Kent Overstreet1889ad52021-03-14 19:01:14 -04001083 migrate_btree_pred, &op, stats) ?: ret;
Kent Overstreet3c843a62023-11-20 18:52:33 -05001084 ret = bch2_move_data(c, start, end,
Kent Overstreet0337cc72022-06-20 15:40:26 -04001085 NULL,
1086 stats,
1087 writepoint_hashed((unsigned long) current),
1088 true,
1089 migrate_pred, &op) ?: ret;
Kent Overstreetae0ff7b2019-04-30 17:15:39 -04001090 ret = bch2_replicas_gc2(c) ?: ret;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001091 break;
Kent Overstreet01e95642023-11-20 18:43:48 -05001092 case BCH_DATA_OP_rewrite_old_nodes:
Kent Overstreeta4805d62021-03-22 18:39:16 -04001093 ret = bch2_scan_old_btree_nodes(c, stats);
Kent Overstreet1889ad52021-03-14 19:01:14 -04001094 break;
Kent Overstreetba11c7d2023-11-20 19:12:40 -05001095 case BCH_DATA_OP_drop_extra_replicas:
1096 ret = bch2_move_btree(c, start, end,
1097 drop_extra_replicas_btree_pred, c, stats) ?: ret;
1098 ret = bch2_move_data(c, start, end, NULL, stats,
1099 writepoint_hashed((unsigned long) current),
1100 true,
1101 drop_extra_replicas_pred, c) ?: ret;
1102 ret = bch2_replicas_gc2(c) ?: ret;
1103 break;
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001104 default:
1105 ret = -EINVAL;
1106 }
1107
Kent Overstreet01e95642023-11-20 18:43:48 -05001108 bch2_move_stats_exit(stats, c);
Kent Overstreet1c6fdbd2017-03-16 22:18:50 -08001109 return ret;
1110}
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001111
Kent Overstreet96a363a2023-10-23 16:21:54 -04001112void bch2_move_stats_to_text(struct printbuf *out, struct bch_move_stats *stats)
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001113{
Kent Overstreete58f9632024-01-06 20:57:43 -05001114 prt_printf(out, "%s: data type==", stats->name);
1115 bch2_prt_data_type(out, stats->data_type);
1116 prt_str(out, " pos=");
Kent Overstreetd5eade92023-10-23 15:36:45 -04001117 bch2_bbpos_to_text(out, stats->pos);
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001118 prt_newline(out);
1119 printbuf_indent_add(out, 2);
1120
Kent Overstreet74233302024-04-10 16:08:24 -04001121 prt_printf(out, "keys moved: %llu\n", atomic64_read(&stats->keys_moved));
1122 prt_printf(out, "keys raced: %llu\n", atomic64_read(&stats->keys_raced));
1123 prt_printf(out, "bytes seen: ");
Kent Overstreet96a363a2023-10-23 16:21:54 -04001124 prt_human_readable_u64(out, atomic64_read(&stats->sectors_seen) << 9);
1125 prt_newline(out);
1126
Kent Overstreet74233302024-04-10 16:08:24 -04001127 prt_printf(out, "bytes moved: ");
Kent Overstreet96a363a2023-10-23 16:21:54 -04001128 prt_human_readable_u64(out, atomic64_read(&stats->sectors_moved) << 9);
1129 prt_newline(out);
1130
Kent Overstreet74233302024-04-10 16:08:24 -04001131 prt_printf(out, "bytes raced: ");
Kent Overstreet96a363a2023-10-23 16:21:54 -04001132 prt_human_readable_u64(out, atomic64_read(&stats->sectors_raced) << 9);
1133 prt_newline(out);
1134
1135 printbuf_indent_sub(out, 2);
1136}
1137
1138static void bch2_moving_ctxt_to_text(struct printbuf *out, struct bch_fs *c, struct moving_context *ctxt)
1139{
1140 struct moving_io *io;
1141
1142 bch2_move_stats_to_text(out, ctxt->stats);
1143 printbuf_indent_add(out, 2);
1144
Kent Overstreet74233302024-04-10 16:08:24 -04001145 prt_printf(out, "reads: ios %u/%u sectors %u/%u\n",
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001146 atomic_read(&ctxt->read_ios),
Kent Overstreet9d2a7bd2023-08-23 21:20:42 -04001147 c->opts.move_ios_in_flight,
1148 atomic_read(&ctxt->read_sectors),
1149 c->opts.move_bytes_in_flight >> 9);
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001150
Kent Overstreet74233302024-04-10 16:08:24 -04001151 prt_printf(out, "writes: ios %u/%u sectors %u/%u\n",
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001152 atomic_read(&ctxt->write_ios),
Kent Overstreet9d2a7bd2023-08-23 21:20:42 -04001153 c->opts.move_ios_in_flight,
1154 atomic_read(&ctxt->write_sectors),
1155 c->opts.move_bytes_in_flight >> 9);
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001156
1157 printbuf_indent_add(out, 2);
1158
1159 mutex_lock(&ctxt->lock);
Kent Overstreet9d2a7bd2023-08-23 21:20:42 -04001160 list_for_each_entry(io, &ctxt->ios, io_list)
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001161 bch2_write_op_to_text(out, &io->write.op);
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001162 mutex_unlock(&ctxt->lock);
1163
1164 printbuf_indent_sub(out, 4);
1165}
1166
1167void bch2_fs_moving_ctxts_to_text(struct printbuf *out, struct bch_fs *c)
1168{
1169 struct moving_context *ctxt;
1170
1171 mutex_lock(&c->moving_context_lock);
1172 list_for_each_entry(ctxt, &c->moving_context_list, list)
Kent Overstreet9d2a7bd2023-08-23 21:20:42 -04001173 bch2_moving_ctxt_to_text(out, c, ctxt);
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001174 mutex_unlock(&c->moving_context_lock);
1175}
1176
1177void bch2_fs_move_init(struct bch_fs *c)
1178{
1179 INIT_LIST_HEAD(&c->moving_context_list);
1180 mutex_init(&c->moving_context_lock);
Kent Overstreetb9fa3752023-03-11 20:38:46 -05001181}