blob: 64563bfdf0da02b1c2eb3b709d59ac3157e599da [file] [log] [blame]
Thomas Gleixnerc6ae4c02019-05-22 09:51:37 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Philipp Reisnerb411b362009-09-25 16:07:19 -07002/*
3 drbd_worker.c
4
5 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6
7 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
8 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
9 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10
Philipp Reisnerb411b362009-09-25 16:07:19 -070011
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +020012*/
Philipp Reisnerb411b362009-09-25 16:07:19 -070013
Philipp Reisnerb411b362009-09-25 16:07:19 -070014#include <linux/module.h>
Philipp Reisnerb411b362009-09-25 16:07:19 -070015#include <linux/drbd.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010016#include <linux/sched/signal.h>
Philipp Reisnerb411b362009-09-25 16:07:19 -070017#include <linux/wait.h>
18#include <linux/mm.h>
19#include <linux/memcontrol.h>
20#include <linux/mm_inline.h>
21#include <linux/slab.h>
22#include <linux/random.h>
Philipp Reisnerb411b362009-09-25 16:07:19 -070023#include <linux/string.h>
24#include <linux/scatterlist.h>
Christoph Hellwigc6a564ff2020-03-25 16:48:42 +010025#include <linux/part_stat.h>
Philipp Reisnerb411b362009-09-25 16:07:19 -070026
27#include "drbd_int.h"
Andreas Gruenbachera3603a62011-05-30 11:47:37 +020028#include "drbd_protocol.h"
Philipp Reisnerb411b362009-09-25 16:07:19 -070029#include "drbd_req.h"
Philipp Reisnerb411b362009-09-25 16:07:19 -070030
Andreas Gruenbacherd448a2e2011-08-25 16:59:58 +020031static int make_ov_request(struct drbd_device *, int);
32static int make_resync_request(struct drbd_device *, int);
Philipp Reisnerb411b362009-09-25 16:07:19 -070033
Andreas Gruenbacherc5a91612011-01-25 17:33:38 +010034/* endio handlers:
Andreas Gruenbachered15b792014-09-11 14:29:06 +020035 * drbd_md_endio (defined here)
Andreas Gruenbacherfcefa622011-02-17 16:46:59 +010036 * drbd_request_endio (defined here)
37 * drbd_peer_request_endio (defined here)
Andreas Gruenbachered15b792014-09-11 14:29:06 +020038 * drbd_bm_endio (defined in drbd_bitmap.c)
Andreas Gruenbacherc5a91612011-01-25 17:33:38 +010039 *
Philipp Reisnerb411b362009-09-25 16:07:19 -070040 * For all these callbacks, note the following:
41 * The callbacks will be called in irq context by the IDE drivers,
42 * and in Softirqs/Tasklets/BH context by the SCSI drivers.
43 * Try to get the locking right :)
44 *
45 */
46
Philipp Reisnerb411b362009-09-25 16:07:19 -070047/* used for synchronous meta data and bitmap IO
48 * submitted by drbd_md_sync_page_io()
49 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +020050void drbd_md_endio(struct bio *bio)
Philipp Reisnerb411b362009-09-25 16:07:19 -070051{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020052 struct drbd_device *device;
Philipp Reisnerb411b362009-09-25 16:07:19 -070053
Lars Ellenberge37d2432014-04-01 23:53:30 +020054 device = bio->bi_private;
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +020055 device->md_io.error = blk_status_to_errno(bio->bi_status);
Philipp Reisnerb411b362009-09-25 16:07:19 -070056
Lars Ellenberg7c752ed2017-08-29 10:20:39 +020057 /* special case: drbd_md_read() during drbd_adm_attach() */
58 if (device->ldev)
59 put_ldev(device);
60 bio_put(bio);
61
Philipp Reisner0cfac5d2011-11-10 12:12:52 +010062 /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able
63 * to timeout on the lower level device, and eventually detach from it.
64 * If this io completion runs after that timeout expired, this
65 * drbd_md_put_buffer() may allow us to finally try and re-attach.
66 * During normal operation, this only puts that extra reference
67 * down to 1 again.
68 * Make sure we first drop the reference, and only then signal
69 * completion, or we may (in drbd_al_read_log()) cycle so fast into the
70 * next drbd_md_sync_page_io(), that we trigger the
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020071 * ASSERT(atomic_read(&device->md_io_in_use) == 1) there.
Philipp Reisner0cfac5d2011-11-10 12:12:52 +010072 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020073 drbd_md_put_buffer(device);
Lars Ellenberge37d2432014-04-01 23:53:30 +020074 device->md_io.done = 1;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020075 wake_up(&device->misc_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -070076}
77
78/* reads on behalf of the partner,
79 * "submitted" by the receiver
80 */
Rashika Kheriaa186e472013-12-19 15:06:10 +053081static void drbd_endio_read_sec_final(struct drbd_peer_request *peer_req) __releases(local)
Philipp Reisnerb411b362009-09-25 16:07:19 -070082{
83 unsigned long flags = 0;
Andreas Gruenbacher67801392011-09-13 10:39:41 +020084 struct drbd_peer_device *peer_device = peer_req->peer_device;
85 struct drbd_device *device = peer_device->device;
Philipp Reisnerb411b362009-09-25 16:07:19 -070086
Andreas Gruenbacher05008132011-07-07 14:19:42 +020087 spin_lock_irqsave(&device->resource->req_lock, flags);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020088 device->read_cnt += peer_req->i.size >> 9;
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +020089 list_del(&peer_req->w.list);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020090 if (list_empty(&device->read_ee))
91 wake_up(&device->ee_wait);
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +010092 if (test_bit(__EE_WAS_ERROR, &peer_req->flags))
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020093 __drbd_chk_io_error(device, DRBD_READ_ERROR);
Andreas Gruenbacher05008132011-07-07 14:19:42 +020094 spin_unlock_irqrestore(&device->resource->req_lock, flags);
Philipp Reisnerb411b362009-09-25 16:07:19 -070095
Andreas Gruenbacher67801392011-09-13 10:39:41 +020096 drbd_queue_work(&peer_device->connection->sender_work, &peer_req->w);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020097 put_ldev(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -070098}
99
100/* writes on behalf of the partner, or resync writes,
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200101 * "submitted" by the receiver, final stage. */
Lars Ellenberga0fb3c42014-04-28 18:43:23 +0200102void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(local)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700103{
104 unsigned long flags = 0;
Andreas Gruenbacher67801392011-09-13 10:39:41 +0200105 struct drbd_peer_device *peer_device = peer_req->peer_device;
106 struct drbd_device *device = peer_device->device;
Philipp Reisner668700b2015-03-16 16:08:29 +0100107 struct drbd_connection *connection = peer_device->connection;
Lars Ellenberg181286a2011-03-31 15:18:56 +0200108 struct drbd_interval i;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700109 int do_wake;
Andreas Gruenbacher579b57e2011-01-13 18:40:57 +0100110 u64 block_id;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700111 int do_al_complete_io;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700112
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100113 /* after we moved peer_req to done_ee,
Philipp Reisnerb411b362009-09-25 16:07:19 -0700114 * we may no longer access it,
115 * it may be freed/reused already!
116 * (as soon as we release the req_lock) */
Lars Ellenberg181286a2011-03-31 15:18:56 +0200117 i = peer_req->i;
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100118 do_al_complete_io = peer_req->flags & EE_CALL_AL_COMPLETE_IO;
119 block_id = peer_req->block_id;
Lars Ellenberg21ae5d72014-05-05 23:42:24 +0200120 peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700121
Lars Ellenberge1fbc4c2017-08-29 10:20:35 +0200122 if (peer_req->flags & EE_WAS_ERROR) {
123 /* In protocol != C, we usually do not send write acks.
124 * In case of a write error, send the neg ack anyways. */
125 if (!__test_and_set_bit(__EE_SEND_WRITE_ACK, &peer_req->flags))
126 inc_unacked(device);
127 drbd_set_out_of_sync(device, peer_req->i.sector, peer_req->i.size);
128 }
129
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200130 spin_lock_irqsave(&device->resource->req_lock, flags);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200131 device->writ_cnt += peer_req->i.size >> 9;
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +0200132 list_move_tail(&peer_req->w.list, &device->done_ee);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700133
Andreas Gruenbacherbb3bfe92011-01-21 15:59:23 +0100134 /*
Andreas Gruenbacher5e472262011-01-27 14:42:51 +0100135 * Do not remove from the write_requests tree here: we did not send the
Andreas Gruenbacherbb3bfe92011-01-21 15:59:23 +0100136 * Ack yet and did not wake possibly waiting conflicting requests.
137 * Removed from the tree from "drbd_process_done_ee" within the
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200138 * appropriate dw.cb (e_end_block/e_end_resync_block) or from
Andreas Gruenbacherbb3bfe92011-01-21 15:59:23 +0100139 * _drbd_clear_done_ee.
140 */
Philipp Reisnerb411b362009-09-25 16:07:19 -0700141
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200142 do_wake = list_empty(block_id == ID_SYNCER ? &device->sync_ee : &device->active_ee);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700143
Bart Van Assche93054552018-10-03 13:56:25 -0700144 /* FIXME do we want to detach for failed REQ_OP_DISCARD?
Lars Ellenbergf31e5832018-12-20 17:23:42 +0100145 * ((peer_req->flags & (EE_WAS_ERROR|EE_TRIM)) == EE_WAS_ERROR) */
Lars Ellenberga0fb3c42014-04-28 18:43:23 +0200146 if (peer_req->flags & EE_WAS_ERROR)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200147 __drbd_chk_io_error(device, DRBD_WRITE_ERROR);
Philipp Reisner668700b2015-03-16 16:08:29 +0100148
149 if (connection->cstate >= C_WF_REPORT_PARAMS) {
150 kref_get(&device->kref); /* put is in drbd_send_acks_wf() */
151 if (!queue_work(connection->ack_sender, &peer_device->send_acks_work))
152 kref_put(&device->kref, drbd_destroy_device);
153 }
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200154 spin_unlock_irqrestore(&device->resource->req_lock, flags);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700155
Andreas Gruenbacher579b57e2011-01-13 18:40:57 +0100156 if (block_id == ID_SYNCER)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200157 drbd_rs_complete_io(device, i.sector);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700158
159 if (do_wake)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200160 wake_up(&device->ee_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700161
162 if (do_al_complete_io)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200163 drbd_al_complete_io(device, &i);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700164
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200165 put_ldev(device);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200166}
Philipp Reisnerb411b362009-09-25 16:07:19 -0700167
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200168/* writes on behalf of the partner, or resync writes,
169 * "submitted" by the receiver.
170 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200171void drbd_peer_request_endio(struct bio *bio)
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200172{
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100173 struct drbd_peer_request *peer_req = bio->bi_private;
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +0200174 struct drbd_device *device = peer_req->peer_device->device;
Fabian Frederick7e5fec32016-06-14 00:26:35 +0200175 bool is_write = bio_data_dir(bio) == WRITE;
Christoph Hellwig45c21792017-04-05 19:21:22 +0200176 bool is_discard = bio_op(bio) == REQ_OP_WRITE_ZEROES ||
177 bio_op(bio) == REQ_OP_DISCARD;
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200178
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200179 if (bio->bi_status && __ratelimit(&drbd_ratelimit_state))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200180 drbd_warn(device, "%s: error=%d s=%llus\n",
Lars Ellenberga0fb3c42014-04-28 18:43:23 +0200181 is_write ? (is_discard ? "discard" : "write")
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200182 : "read", bio->bi_status,
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100183 (unsigned long long)peer_req->i.sector);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200184
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200185 if (bio->bi_status)
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100186 set_bit(__EE_WAS_ERROR, &peer_req->flags);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200187
188 bio_put(bio); /* no need for the bio anymore */
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100189 if (atomic_dec_and_test(&peer_req->pending_bios)) {
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200190 if (is_write)
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100191 drbd_endio_write_sec_final(peer_req);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200192 else
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100193 drbd_endio_read_sec_final(peer_req);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200194 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700195}
196
Baoyou Xie1ffa7bf2017-08-29 10:20:36 +0200197static void
198drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device)
Lars Ellenberg142207f2015-02-19 13:48:59 +0100199{
200 panic("drbd%u %s/%u potential random memory corruption caused by delayed completion of aborted local request\n",
201 device->minor, device->resource->name, device->vnr);
202}
203
Philipp Reisnerb411b362009-09-25 16:07:19 -0700204/* read, readA or write requests on R_PRIMARY coming from drbd_make_request
205 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200206void drbd_request_endio(struct bio *bio)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700207{
Lars Ellenberga1154132010-11-13 20:42:29 +0100208 unsigned long flags;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700209 struct drbd_request *req = bio->bi_private;
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200210 struct drbd_device *device = req->device;
Lars Ellenberga1154132010-11-13 20:42:29 +0100211 struct bio_and_error m;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700212 enum drbd_req_event what;
Philipp Reisner1b6dd252012-09-04 15:16:20 +0200213
214 /* If this request was aborted locally before,
215 * but now was completed "successfully",
216 * chances are that this caused arbitrary data corruption.
217 *
218 * "aborting" requests, or force-detaching the disk, is intended for
219 * completely blocked/hung local backing devices which do no longer
220 * complete requests at all, not even do error completions. In this
221 * situation, usually a hard-reset and failover is the only way out.
222 *
223 * By "aborting", basically faking a local error-completion,
224 * we allow for a more graceful swichover by cleanly migrating services.
225 * Still the affected node has to be rebooted "soon".
226 *
227 * By completing these requests, we allow the upper layers to re-use
228 * the associated data pages.
229 *
230 * If later the local backing device "recovers", and now DMAs some data
231 * from disk into the original request pages, in the best case it will
232 * just put random data into unused pages; but typically it will corrupt
233 * meanwhile completely unrelated data, causing all sorts of damage.
234 *
235 * Which means delayed successful completion,
236 * especially for READ requests,
237 * is a reason to panic().
238 *
239 * We assume that a delayed *error* completion is OK,
240 * though we still will complain noisily about it.
241 */
242 if (unlikely(req->rq_state & RQ_LOCAL_ABORTED)) {
243 if (__ratelimit(&drbd_ratelimit_state))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200244 drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n");
Philipp Reisner1b6dd252012-09-04 15:16:20 +0200245
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200246 if (!bio->bi_status)
Lars Ellenberg142207f2015-02-19 13:48:59 +0100247 drbd_panic_after_delayed_completion_of_aborted_request(device);
Philipp Reisner1b6dd252012-09-04 15:16:20 +0200248 }
249
Philipp Reisnerb411b362009-09-25 16:07:19 -0700250 /* to avoid recursion in __req_mod */
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200251 if (unlikely(bio->bi_status)) {
Christoph Hellwig70246282016-07-19 11:28:41 +0200252 switch (bio_op(bio)) {
Christoph Hellwig45c21792017-04-05 19:21:22 +0200253 case REQ_OP_WRITE_ZEROES:
Christoph Hellwig70246282016-07-19 11:28:41 +0200254 case REQ_OP_DISCARD:
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200255 if (bio->bi_status == BLK_STS_NOTSUPP)
Christoph Hellwig70246282016-07-19 11:28:41 +0200256 what = DISCARD_COMPLETED_NOTSUPP;
257 else
258 what = DISCARD_COMPLETED_WITH_ERROR;
259 break;
260 case REQ_OP_READ:
Jens Axboe1eff9d32016-08-05 15:35:16 -0600261 if (bio->bi_opf & REQ_RAHEAD)
Christoph Hellwig70246282016-07-19 11:28:41 +0200262 what = READ_AHEAD_COMPLETED_WITH_ERROR;
263 else
264 what = READ_COMPLETED_WITH_ERROR;
265 break;
266 default:
267 what = WRITE_COMPLETED_WITH_ERROR;
268 break;
269 }
270 } else {
Andreas Gruenbacher8554df12011-01-25 15:37:43 +0100271 what = COMPLETED_OK;
Christoph Hellwig70246282016-07-19 11:28:41 +0200272 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700273
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200274 req->private_bio = ERR_PTR(blk_status_to_errno(bio->bi_status));
Lars Ellenberg64dafbc2018-06-25 11:39:52 +0200275 bio_put(bio);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700276
Lars Ellenberga1154132010-11-13 20:42:29 +0100277 /* not req_mod(), we need irqsave here! */
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200278 spin_lock_irqsave(&device->resource->req_lock, flags);
Lars Ellenberga1154132010-11-13 20:42:29 +0100279 __req_mod(req, what, &m);
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200280 spin_unlock_irqrestore(&device->resource->req_lock, flags);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200281 put_ldev(device);
Lars Ellenberga1154132010-11-13 20:42:29 +0100282
283 if (m.bio)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200284 complete_master_bio(device, &m);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700285}
286
Kees Cook3d0e6372018-08-06 16:32:16 -0700287void drbd_csum_ee(struct crypto_shash *tfm, struct drbd_peer_request *peer_req, void *digest)
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200288{
Kees Cook3d0e6372018-08-06 16:32:16 -0700289 SHASH_DESC_ON_STACK(desc, tfm);
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100290 struct page *page = peer_req->pages;
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200291 struct page *tmp;
292 unsigned len;
Kees Cook3d0e6372018-08-06 16:32:16 -0700293 void *src;
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200294
Kees Cook3d0e6372018-08-06 16:32:16 -0700295 desc->tfm = tfm;
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200296
Kees Cook3d0e6372018-08-06 16:32:16 -0700297 crypto_shash_init(desc);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200298
Kees Cook3d0e6372018-08-06 16:32:16 -0700299 src = kmap_atomic(page);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200300 while ((tmp = page_chain_next(page))) {
301 /* all but the last page will be fully used */
Kees Cook3d0e6372018-08-06 16:32:16 -0700302 crypto_shash_update(desc, src, PAGE_SIZE);
303 kunmap_atomic(src);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200304 page = tmp;
Kees Cook3d0e6372018-08-06 16:32:16 -0700305 src = kmap_atomic(page);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200306 }
307 /* and now the last, possibly only partially used page */
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100308 len = peer_req->i.size & (PAGE_SIZE - 1);
Kees Cook3d0e6372018-08-06 16:32:16 -0700309 crypto_shash_update(desc, src, len ?: PAGE_SIZE);
310 kunmap_atomic(src);
311
312 crypto_shash_final(desc, digest);
313 shash_desc_zero(desc);
Lars Ellenberg45bb9122010-05-14 17:10:48 +0200314}
315
Kees Cook3d0e6372018-08-06 16:32:16 -0700316void drbd_csum_bio(struct crypto_shash *tfm, struct bio *bio, void *digest)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700317{
Kees Cook3d0e6372018-08-06 16:32:16 -0700318 SHASH_DESC_ON_STACK(desc, tfm);
Kent Overstreet79886132013-11-23 17:19:00 -0800319 struct bio_vec bvec;
320 struct bvec_iter iter;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700321
Kees Cook3d0e6372018-08-06 16:32:16 -0700322 desc->tfm = tfm;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700323
Kees Cook3d0e6372018-08-06 16:32:16 -0700324 crypto_shash_init(desc);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700325
Kent Overstreet79886132013-11-23 17:19:00 -0800326 bio_for_each_segment(bvec, bio, iter) {
Kees Cook3d0e6372018-08-06 16:32:16 -0700327 u8 *src;
328
329 src = kmap_atomic(bvec.bv_page);
330 crypto_shash_update(desc, src + bvec.bv_offset, bvec.bv_len);
331 kunmap_atomic(src);
332
Lars Ellenberg9104d312016-06-14 00:26:31 +0200333 /* REQ_OP_WRITE_SAME has only one segment,
334 * checksum the payload only once. */
335 if (bio_op(bio) == REQ_OP_WRITE_SAME)
336 break;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700337 }
Kees Cook3d0e6372018-08-06 16:32:16 -0700338 crypto_shash_final(desc, digest);
339 shash_desc_zero(desc);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700340}
341
Lars Ellenberg9676c762011-02-22 14:02:31 +0100342/* MAYBE merge common code with w_e_end_ov_req */
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100343static int w_e_send_csum(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700344{
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +0200345 struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
Andreas Gruenbacher67801392011-09-13 10:39:41 +0200346 struct drbd_peer_device *peer_device = peer_req->peer_device;
347 struct drbd_device *device = peer_device->device;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700348 int digest_size;
349 void *digest;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100350 int err = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700351
Lars Ellenberg53ea4332011-03-08 17:11:40 +0100352 if (unlikely(cancel))
353 goto out;
354
Lars Ellenberg9676c762011-02-22 14:02:31 +0100355 if (unlikely((peer_req->flags & EE_WAS_ERROR) != 0))
Lars Ellenberg53ea4332011-03-08 17:11:40 +0100356 goto out;
357
Kees Cook3d0e6372018-08-06 16:32:16 -0700358 digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm);
Lars Ellenberg53ea4332011-03-08 17:11:40 +0100359 digest = kmalloc(digest_size, GFP_NOIO);
360 if (digest) {
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100361 sector_t sector = peer_req->i.sector;
362 unsigned int size = peer_req->i.size;
Andreas Gruenbacher67801392011-09-13 10:39:41 +0200363 drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest);
Lars Ellenberg9676c762011-02-22 14:02:31 +0100364 /* Free peer_req and pages before send.
Lars Ellenberg53ea4332011-03-08 17:11:40 +0100365 * In case we block on congestion, we could otherwise run into
366 * some distributed deadlock, if the other side blocks on
367 * congestion as well, because our receiver blocks in
Andreas Gruenbacherc37c8ec2011-04-07 21:02:09 +0200368 * drbd_alloc_pages due to pp_in_use > max_buffers. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200369 drbd_free_peer_req(device, peer_req);
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100370 peer_req = NULL;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200371 inc_rs_pending(device);
Andreas Gruenbacher67801392011-09-13 10:39:41 +0200372 err = drbd_send_drequest_csum(peer_device, sector, size,
Andreas Gruenbacherdb1b0b72011-03-16 01:37:21 +0100373 digest, digest_size,
374 P_CSUM_RS_REQUEST);
Lars Ellenberg53ea4332011-03-08 17:11:40 +0100375 kfree(digest);
376 } else {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200377 drbd_err(device, "kmalloc() of digest failed.\n");
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100378 err = -ENOMEM;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700379 }
380
Lars Ellenberg53ea4332011-03-08 17:11:40 +0100381out:
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100382 if (peer_req)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200383 drbd_free_peer_req(device, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700384
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100385 if (unlikely(err))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200386 drbd_err(device, "drbd_send_drequest(..., csum) failed\n");
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100387 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700388}
389
390#define GFP_TRY (__GFP_HIGHMEM | __GFP_NOWARN)
391
Andreas Gruenbacher69a22772011-08-09 00:47:13 +0200392static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, int size)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700393{
Andreas Gruenbacher69a22772011-08-09 00:47:13 +0200394 struct drbd_device *device = peer_device->device;
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100395 struct drbd_peer_request *peer_req;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700396
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200397 if (!get_ldev(device))
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200398 return -EIO;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700399
400 /* GFP_TRY, because if there is no memory available right now, this may
401 * be rescheduled for later. It is "only" background resync, after all. */
Andreas Gruenbacher69a22772011-08-09 00:47:13 +0200402 peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER /* unused */, sector,
Lars Ellenberg9104d312016-06-14 00:26:31 +0200403 size, size, GFP_TRY);
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +0100404 if (!peer_req)
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200405 goto defer;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700406
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +0200407 peer_req->w.cb = w_e_send_csum;
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200408 spin_lock_irq(&device->resource->req_lock);
Lars Ellenbergb9ed7082014-04-23 12:15:35 +0200409 list_add_tail(&peer_req->w.list, &device->read_ee);
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200410 spin_unlock_irq(&device->resource->req_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700411
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200412 atomic_add(size >> 9, &device->rs_sect_ev);
Mike Christiebb3cc852016-06-05 14:32:06 -0500413 if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0,
414 DRBD_FAULT_RS_RD) == 0)
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200415 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700416
Lars Ellenberg10f6d9922011-01-24 14:47:09 +0100417 /* If it failed because of ENOMEM, retry should help. If it failed
418 * because bio_add_page failed (probably broken lower level driver),
419 * retry may or may not help.
420 * If it does not, you may need to force disconnect. */
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200421 spin_lock_irq(&device->resource->req_lock);
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +0200422 list_del(&peer_req->w.list);
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200423 spin_unlock_irq(&device->resource->req_lock);
Lars Ellenberg22cc37a2010-09-14 20:40:41 +0200424
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200425 drbd_free_peer_req(device, peer_req);
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200426defer:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200427 put_ldev(device);
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200428 return -EAGAIN;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700429}
430
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100431int w_resync_timer(struct drbd_work *w, int cancel)
Philipp Reisner794abb72010-12-27 11:51:23 +0100432{
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200433 struct drbd_device *device =
434 container_of(w, struct drbd_device, resync_work);
435
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200436 switch (device->state.conn) {
Philipp Reisner794abb72010-12-27 11:51:23 +0100437 case C_VERIFY_S:
Andreas Gruenbacherd448a2e2011-08-25 16:59:58 +0200438 make_ov_request(device, cancel);
Philipp Reisner794abb72010-12-27 11:51:23 +0100439 break;
440 case C_SYNC_TARGET:
Andreas Gruenbacherd448a2e2011-08-25 16:59:58 +0200441 make_resync_request(device, cancel);
Philipp Reisner794abb72010-12-27 11:51:23 +0100442 break;
443 }
444
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100445 return 0;
Philipp Reisner794abb72010-12-27 11:51:23 +0100446}
447
Kees Cook2bccef32017-10-17 20:33:01 -0700448void resync_timer_fn(struct timer_list *t)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700449{
Kees Cook2bccef32017-10-17 20:33:01 -0700450 struct drbd_device *device = from_timer(device, t, resync_timer);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700451
Lars Ellenberg15e26f6a2014-04-28 11:43:21 +0200452 drbd_queue_work_if_unqueued(
453 &first_peer_device(device)->connection->sender_work,
454 &device->resync_work);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700455}
456
Philipp Reisner778f2712010-07-06 11:14:00 +0200457static void fifo_set(struct fifo_buffer *fb, int value)
458{
459 int i;
460
461 for (i = 0; i < fb->size; i++)
Philipp Reisnerf10f2622010-10-05 16:50:17 +0200462 fb->values[i] = value;
Philipp Reisner778f2712010-07-06 11:14:00 +0200463}
464
465static int fifo_push(struct fifo_buffer *fb, int value)
466{
467 int ov;
468
469 ov = fb->values[fb->head_index];
470 fb->values[fb->head_index++] = value;
471
472 if (fb->head_index >= fb->size)
473 fb->head_index = 0;
474
475 return ov;
476}
477
478static void fifo_add_val(struct fifo_buffer *fb, int value)
479{
480 int i;
481
482 for (i = 0; i < fb->size; i++)
483 fb->values[i] += value;
484}
485
Stephen Kitt6a365872020-01-24 21:03:07 +0100486struct fifo_buffer *fifo_alloc(unsigned int fifo_size)
Philipp Reisner9958c852011-05-03 16:19:31 +0200487{
488 struct fifo_buffer *fb;
489
Stephen Kitt6a365872020-01-24 21:03:07 +0100490 fb = kzalloc(struct_size(fb, values, fifo_size), GFP_NOIO);
Philipp Reisner9958c852011-05-03 16:19:31 +0200491 if (!fb)
492 return NULL;
493
494 fb->head_index = 0;
495 fb->size = fifo_size;
496 fb->total = 0;
497
498 return fb;
499}
500
Lars Ellenberg0e49d7b2014-04-28 18:43:18 +0200501static int drbd_rs_controller(struct drbd_device *device, unsigned int sect_in)
Philipp Reisner778f2712010-07-06 11:14:00 +0200502{
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +0200503 struct disk_conf *dc;
Lars Ellenberg7f34f612014-04-22 16:37:16 +0200504 unsigned int want; /* The number of sectors we want in-flight */
Philipp Reisner778f2712010-07-06 11:14:00 +0200505 int req_sect; /* Number of sectors to request in this turn */
Lars Ellenberg7f34f612014-04-22 16:37:16 +0200506 int correction; /* Number of sectors more we need in-flight */
Philipp Reisner778f2712010-07-06 11:14:00 +0200507 int cps; /* correction per invocation of drbd_rs_controller() */
508 int steps; /* Number of time steps to plan ahead */
509 int curr_corr;
510 int max_sect;
Philipp Reisner813472c2011-05-03 16:47:02 +0200511 struct fifo_buffer *plan;
Philipp Reisner778f2712010-07-06 11:14:00 +0200512
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200513 dc = rcu_dereference(device->ldev->disk_conf);
514 plan = rcu_dereference(device->rs_plan_s);
Philipp Reisner778f2712010-07-06 11:14:00 +0200515
Philipp Reisner813472c2011-05-03 16:47:02 +0200516 steps = plan->size; /* (dc->c_plan_ahead * 10 * SLEEP_TIME) / HZ; */
Philipp Reisner778f2712010-07-06 11:14:00 +0200517
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200518 if (device->rs_in_flight + sect_in == 0) { /* At start of resync */
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +0200519 want = ((dc->resync_rate * 2 * SLEEP_TIME) / HZ) * steps;
Philipp Reisner778f2712010-07-06 11:14:00 +0200520 } else { /* normal path */
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +0200521 want = dc->c_fill_target ? dc->c_fill_target :
522 sect_in * dc->c_delay_target * HZ / (SLEEP_TIME * 10);
Philipp Reisner778f2712010-07-06 11:14:00 +0200523 }
524
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200525 correction = want - device->rs_in_flight - plan->total;
Philipp Reisner778f2712010-07-06 11:14:00 +0200526
527 /* Plan ahead */
528 cps = correction / steps;
Philipp Reisner813472c2011-05-03 16:47:02 +0200529 fifo_add_val(plan, cps);
530 plan->total += cps * steps;
Philipp Reisner778f2712010-07-06 11:14:00 +0200531
532 /* What we do in this step */
Philipp Reisner813472c2011-05-03 16:47:02 +0200533 curr_corr = fifo_push(plan, 0);
534 plan->total -= curr_corr;
Philipp Reisner778f2712010-07-06 11:14:00 +0200535
536 req_sect = sect_in + curr_corr;
537 if (req_sect < 0)
538 req_sect = 0;
539
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +0200540 max_sect = (dc->c_max_rate * 2 * SLEEP_TIME) / HZ;
Philipp Reisner778f2712010-07-06 11:14:00 +0200541 if (req_sect > max_sect)
542 req_sect = max_sect;
543
544 /*
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200545 drbd_warn(device, "si=%u if=%d wa=%u co=%d st=%d cps=%d pl=%d cc=%d rs=%d\n",
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200546 sect_in, device->rs_in_flight, want, correction,
547 steps, cps, device->rs_planed, curr_corr, req_sect);
Philipp Reisner778f2712010-07-06 11:14:00 +0200548 */
549
550 return req_sect;
551}
552
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200553static int drbd_rs_number_requests(struct drbd_device *device)
Lars Ellenberge65f4402010-11-05 10:04:07 +0100554{
Lars Ellenberg0e49d7b2014-04-28 18:43:18 +0200555 unsigned int sect_in; /* Number of sectors that came in since the last turn */
556 int number, mxb;
557
558 sect_in = atomic_xchg(&device->rs_sect_in, 0);
559 device->rs_in_flight -= sect_in;
Philipp Reisner813472c2011-05-03 16:47:02 +0200560
561 rcu_read_lock();
Lars Ellenberg0e49d7b2014-04-28 18:43:18 +0200562 mxb = drbd_get_max_buffers(device) / 2;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200563 if (rcu_dereference(device->rs_plan_s)->size) {
Lars Ellenberg0e49d7b2014-04-28 18:43:18 +0200564 number = drbd_rs_controller(device, sect_in) >> (BM_BLOCK_SHIFT - 9);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200565 device->c_sync_rate = number * HZ * (BM_BLOCK_SIZE / 1024) / SLEEP_TIME;
Lars Ellenberge65f4402010-11-05 10:04:07 +0100566 } else {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200567 device->c_sync_rate = rcu_dereference(device->ldev->disk_conf)->resync_rate;
568 number = SLEEP_TIME * device->c_sync_rate / ((BM_BLOCK_SIZE / 1024) * HZ);
Lars Ellenberge65f4402010-11-05 10:04:07 +0100569 }
Philipp Reisner813472c2011-05-03 16:47:02 +0200570 rcu_read_unlock();
Lars Ellenberge65f4402010-11-05 10:04:07 +0100571
Lars Ellenberg0e49d7b2014-04-28 18:43:18 +0200572 /* Don't have more than "max-buffers"/2 in-flight.
573 * Otherwise we may cause the remote site to stall on drbd_alloc_pages(),
574 * potentially causing a distributed deadlock on congestion during
575 * online-verify or (checksum-based) resync, if max-buffers,
576 * socket buffer sizes and resync rate settings are mis-configured. */
Lars Ellenberg7f34f612014-04-22 16:37:16 +0200577
578 /* note that "number" is in units of "BM_BLOCK_SIZE" (which is 4k),
579 * mxb (as used here, and in drbd_alloc_pages on the peer) is
580 * "number of pages" (typically also 4k),
581 * but "rs_in_flight" is in "sectors" (512 Byte). */
582 if (mxb - device->rs_in_flight/8 < number)
583 number = mxb - device->rs_in_flight/8;
Lars Ellenberg0e49d7b2014-04-28 18:43:18 +0200584
Lars Ellenberge65f4402010-11-05 10:04:07 +0100585 return number;
586}
587
Lars Ellenberg44a4d552013-11-22 12:40:58 +0100588static int make_resync_request(struct drbd_device *const device, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700589{
Lars Ellenberg44a4d552013-11-22 12:40:58 +0100590 struct drbd_peer_device *const peer_device = first_peer_device(device);
591 struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700592 unsigned long bit;
593 sector_t sector;
Christoph Hellwig155bd9d2020-09-25 18:06:17 +0200594 const sector_t capacity = get_capacity(device->vdisk);
Lars Ellenberg1816a2b2010-11-11 15:19:07 +0100595 int max_bio_size;
Lars Ellenberge65f4402010-11-05 10:04:07 +0100596 int number, rollback_i, size;
Lars Ellenberg506afb62014-01-31 14:55:12 +0100597 int align, requeue = 0;
Lars Ellenberg0f0601f2010-08-11 23:40:24 +0200598 int i = 0;
Philipp Reisner92d94ae2016-06-14 00:26:15 +0200599 int discard_granularity = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700600
601 if (unlikely(cancel))
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100602 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700603
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200604 if (device->rs_total == 0) {
Lars Ellenbergaf85e8e2010-10-07 16:07:55 +0200605 /* empty resync? */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200606 drbd_resync_finished(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100607 return 0;
Lars Ellenbergaf85e8e2010-10-07 16:07:55 +0200608 }
609
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200610 if (!get_ldev(device)) {
611 /* Since we only need to access device->rsync a
612 get_ldev_if_state(device,D_FAILED) would be sufficient, but
Philipp Reisnerb411b362009-09-25 16:07:19 -0700613 to continue resync with a broken disk makes no sense at
614 all */
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200615 drbd_err(device, "Disk broke down during resync!\n");
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100616 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700617 }
618
Lars Ellenberg9104d312016-06-14 00:26:31 +0200619 if (connection->agreed_features & DRBD_FF_THIN_RESYNC) {
Philipp Reisner92d94ae2016-06-14 00:26:15 +0200620 rcu_read_lock();
621 discard_granularity = rcu_dereference(device->ldev->disk_conf)->rs_discard_granularity;
622 rcu_read_unlock();
623 }
624
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200625 max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9;
626 number = drbd_rs_number_requests(device);
Lars Ellenberg0e49d7b2014-04-28 18:43:18 +0200627 if (number <= 0)
Lars Ellenberg0f0601f2010-08-11 23:40:24 +0200628 goto requeue;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700629
Philipp Reisnerb411b362009-09-25 16:07:19 -0700630 for (i = 0; i < number; i++) {
Lars Ellenberg506afb62014-01-31 14:55:12 +0100631 /* Stop generating RS requests when half of the send buffer is filled,
632 * but notify TCP that we'd like to have more space. */
Lars Ellenberg44a4d552013-11-22 12:40:58 +0100633 mutex_lock(&connection->data.mutex);
634 if (connection->data.socket) {
Lars Ellenberg506afb62014-01-31 14:55:12 +0100635 struct sock *sk = connection->data.socket->sk;
636 int queued = sk->sk_wmem_queued;
637 int sndbuf = sk->sk_sndbuf;
638 if (queued > sndbuf / 2) {
639 requeue = 1;
640 if (sk->sk_socket)
641 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
642 }
643 } else
644 requeue = 1;
Lars Ellenberg44a4d552013-11-22 12:40:58 +0100645 mutex_unlock(&connection->data.mutex);
Lars Ellenberg506afb62014-01-31 14:55:12 +0100646 if (requeue)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700647 goto requeue;
648
649next_sector:
650 size = BM_BLOCK_SIZE;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200651 bit = drbd_bm_find_next(device, device->bm_resync_fo);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700652
Lars Ellenberg4b0715f2010-12-14 15:13:04 +0100653 if (bit == DRBD_END_OF_BITMAP) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200654 device->bm_resync_fo = drbd_bm_bits(device);
655 put_ldev(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100656 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700657 }
658
659 sector = BM_BIT_TO_SECT(bit);
660
Lars Ellenbergad3fee72013-12-20 11:22:13 +0100661 if (drbd_try_rs_begin_io(device, sector)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200662 device->bm_resync_fo = bit;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700663 goto requeue;
664 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200665 device->bm_resync_fo = bit + 1;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700666
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200667 if (unlikely(drbd_bm_test_bit(device, bit) == 0)) {
668 drbd_rs_complete_io(device, sector);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700669 goto next_sector;
670 }
671
Lars Ellenberg1816a2b2010-11-11 15:19:07 +0100672#if DRBD_MAX_BIO_SIZE > BM_BLOCK_SIZE
Philipp Reisnerb411b362009-09-25 16:07:19 -0700673 /* try to find some adjacent bits.
674 * we stop if we have already the maximum req size.
675 *
676 * Additionally always align bigger requests, in order to
677 * be prepared for all stripe sizes of software RAIDs.
Philipp Reisnerb411b362009-09-25 16:07:19 -0700678 */
679 align = 1;
Philipp Reisnerd2074502010-07-22 15:27:27 +0200680 rollback_i = i;
Lars Ellenberg6377b922014-04-28 18:43:17 +0200681 while (i < number) {
Lars Ellenberg1816a2b2010-11-11 15:19:07 +0100682 if (size + BM_BLOCK_SIZE > max_bio_size)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700683 break;
684
685 /* Be always aligned */
686 if (sector & ((1<<(align+3))-1))
687 break;
688
Philipp Reisner92d94ae2016-06-14 00:26:15 +0200689 if (discard_granularity && size == discard_granularity)
690 break;
691
Philipp Reisnerb411b362009-09-25 16:07:19 -0700692 /* do not cross extent boundaries */
693 if (((bit+1) & BM_BLOCKS_PER_BM_EXT_MASK) == 0)
694 break;
695 /* now, is it actually dirty, after all?
696 * caution, drbd_bm_test_bit is tri-state for some
697 * obscure reason; ( b == 0 ) would get the out-of-band
698 * only accidentally right because of the "oddly sized"
699 * adjustment below */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200700 if (drbd_bm_test_bit(device, bit+1) != 1)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700701 break;
702 bit++;
703 size += BM_BLOCK_SIZE;
704 if ((BM_BLOCK_SIZE << align) <= size)
705 align++;
706 i++;
707 }
708 /* if we merged some,
709 * reset the offset to start the next drbd_bm_find_next from */
710 if (size > BM_BLOCK_SIZE)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200711 device->bm_resync_fo = bit + 1;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700712#endif
713
714 /* adjust very last sectors, in case we are oddly sized */
715 if (sector + (size>>9) > capacity)
716 size = (capacity-sector)<<9;
Lars Ellenbergaaaba342014-03-18 12:30:09 +0100717
718 if (device->use_csums) {
Lars Ellenberg44a4d552013-11-22 12:40:58 +0100719 switch (read_for_csum(peer_device, sector, size)) {
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200720 case -EIO: /* Disk failure */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200721 put_ldev(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100722 return -EIO;
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200723 case -EAGAIN: /* allocation failed, or ldev busy */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200724 drbd_rs_complete_io(device, sector);
725 device->bm_resync_fo = BM_SECT_TO_BIT(sector);
Philipp Reisnerd2074502010-07-22 15:27:27 +0200726 i = rollback_i;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700727 goto requeue;
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200728 case 0:
729 /* everything ok */
730 break;
731 default:
732 BUG();
Philipp Reisnerb411b362009-09-25 16:07:19 -0700733 }
734 } else {
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100735 int err;
736
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200737 inc_rs_pending(device);
Philipp Reisner92d94ae2016-06-14 00:26:15 +0200738 err = drbd_send_drequest(peer_device,
739 size == discard_granularity ? P_RS_THIN_REQ : P_RS_DATA_REQUEST,
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100740 sector, size, ID_SYNCER);
741 if (err) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200742 drbd_err(device, "drbd_send_drequest() failed, aborting...\n");
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200743 dec_rs_pending(device);
744 put_ldev(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100745 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700746 }
747 }
748 }
749
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200750 if (device->bm_resync_fo >= drbd_bm_bits(device)) {
Philipp Reisnerb411b362009-09-25 16:07:19 -0700751 /* last syncer _request_ was sent,
752 * but the P_RS_DATA_REPLY not yet received. sync will end (and
753 * next sync group will resume), as soon as we receive the last
754 * resync data block, and the last bit is cleared.
755 * until then resync "work" is "inactive" ...
756 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200757 put_ldev(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100758 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700759 }
760
761 requeue:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200762 device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9));
763 mod_timer(&device->resync_timer, jiffies + SLEEP_TIME);
764 put_ldev(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100765 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700766}
767
Andreas Gruenbacherd448a2e2011-08-25 16:59:58 +0200768static int make_ov_request(struct drbd_device *device, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700769{
770 int number, i, size;
771 sector_t sector;
Christoph Hellwig155bd9d2020-09-25 18:06:17 +0200772 const sector_t capacity = get_capacity(device->vdisk);
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200773 bool stop_sector_reached = false;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700774
775 if (unlikely(cancel))
776 return 1;
777
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200778 number = drbd_rs_number_requests(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700779
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200780 sector = device->ov_position;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700781 for (i = 0; i < number; i++) {
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200782 if (sector >= capacity)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700783 return 1;
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200784
785 /* We check for "finished" only in the reply path:
786 * w_e_end_ov_reply().
787 * We need to send at least one request out. */
788 stop_sector_reached = i > 0
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200789 && verify_can_do_stop_sector(device)
790 && sector >= device->ov_stop_sector;
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200791 if (stop_sector_reached)
792 break;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700793
794 size = BM_BLOCK_SIZE;
795
Lars Ellenbergad3fee72013-12-20 11:22:13 +0100796 if (drbd_try_rs_begin_io(device, sector)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200797 device->ov_position = sector;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700798 goto requeue;
799 }
800
801 if (sector + (size>>9) > capacity)
802 size = (capacity-sector)<<9;
803
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200804 inc_rs_pending(device);
Andreas Gruenbacher69a22772011-08-09 00:47:13 +0200805 if (drbd_send_ov_request(first_peer_device(device), sector, size)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200806 dec_rs_pending(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700807 return 0;
808 }
809 sector += BM_SECT_PER_BIT;
810 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200811 device->ov_position = sector;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700812
813 requeue:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200814 device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9));
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200815 if (i == 0 || !stop_sector_reached)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200816 mod_timer(&device->resync_timer, jiffies + SLEEP_TIME);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700817 return 1;
818}
819
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100820int w_ov_finished(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700821{
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200822 struct drbd_device_work *dw =
823 container_of(w, struct drbd_device_work, w);
824 struct drbd_device *device = dw->device;
825 kfree(dw);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200826 ov_out_of_sync_print(device);
827 drbd_resync_finished(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700828
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100829 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700830}
831
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100832static int w_resync_finished(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700833{
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200834 struct drbd_device_work *dw =
835 container_of(w, struct drbd_device_work, w);
836 struct drbd_device *device = dw->device;
837 kfree(dw);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700838
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200839 drbd_resync_finished(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700840
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +0100841 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700842}
843
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200844static void ping_peer(struct drbd_device *device)
Lars Ellenbergaf85e8e2010-10-07 16:07:55 +0200845{
Andreas Gruenbachera6b32bc32011-05-31 14:33:49 +0200846 struct drbd_connection *connection = first_peer_device(device)->connection;
Philipp Reisner2a67d8b2011-02-09 14:10:32 +0100847
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +0200848 clear_bit(GOT_PING_ACK, &connection->flags);
849 request_ping(connection);
850 wait_event(connection->ping_wait,
851 test_bit(GOT_PING_ACK, &connection->flags) || device->state.conn < C_CONNECTED);
Lars Ellenbergaf85e8e2010-10-07 16:07:55 +0200852}
853
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200854int drbd_resync_finished(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700855{
Lars Ellenberg26a96112016-06-14 00:26:25 +0200856 struct drbd_connection *connection = first_peer_device(device)->connection;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700857 unsigned long db, dt, dbdt;
858 unsigned long n_oos;
859 union drbd_state os, ns;
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200860 struct drbd_device_work *dw;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700861 char *khelper_cmd = NULL;
Lars Ellenberg26525612010-11-05 09:56:33 +0100862 int verify_done = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700863
864 /* Remove all elements from the resync LRU. Since future actions
865 * might set bits in the (main) bitmap, then the entries in the
866 * resync LRU would be wrong. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200867 if (drbd_rs_del_all(device)) {
Philipp Reisnerb411b362009-09-25 16:07:19 -0700868 /* In case this is not possible now, most probably because
869 * there are P_RS_DATA_REPLY Packets lingering on the worker's
870 * queue (or even the read operations for those packets
871 * is not finished by now). Retry in 100ms. */
872
Philipp Reisner20ee6392011-01-18 15:28:59 +0100873 schedule_timeout_interruptible(HZ / 10);
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200874 dw = kmalloc(sizeof(struct drbd_device_work), GFP_ATOMIC);
875 if (dw) {
876 dw->w.cb = w_resync_finished;
877 dw->device = device;
Lars Ellenberg26a96112016-06-14 00:26:25 +0200878 drbd_queue_work(&connection->sender_work, &dw->w);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700879 return 1;
880 }
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200881 drbd_err(device, "Warn failed to drbd_rs_del_all() and to kmalloc(dw).\n");
Philipp Reisnerb411b362009-09-25 16:07:19 -0700882 }
883
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200884 dt = (jiffies - device->rs_start - device->rs_paused) / HZ;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700885 if (dt <= 0)
886 dt = 1;
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +0200887
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200888 db = device->rs_total;
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200889 /* adjust for verify start and stop sectors, respective reached position */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200890 if (device->state.conn == C_VERIFY_S || device->state.conn == C_VERIFY_T)
891 db -= device->ov_left;
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200892
Philipp Reisnerb411b362009-09-25 16:07:19 -0700893 dbdt = Bit2KB(db/dt);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200894 device->rs_paused /= HZ;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700895
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200896 if (!get_ldev(device))
Philipp Reisnerb411b362009-09-25 16:07:19 -0700897 goto out;
898
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200899 ping_peer(device);
Lars Ellenbergaf85e8e2010-10-07 16:07:55 +0200900
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200901 spin_lock_irq(&device->resource->req_lock);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200902 os = drbd_read_state(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700903
Lars Ellenberg26525612010-11-05 09:56:33 +0100904 verify_done = (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T);
905
Philipp Reisnerb411b362009-09-25 16:07:19 -0700906 /* This protects us against multiple calls (that can happen in the presence
907 of application IO), and against connectivity loss just before we arrive here. */
908 if (os.conn <= C_CONNECTED)
909 goto out_unlock;
910
911 ns = os;
912 ns.conn = C_CONNECTED;
913
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200914 drbd_info(device, "%s done (total %lu sec; paused %lu sec; %lu K/sec)\n",
Lars Ellenberg58ffa582012-07-26 14:09:49 +0200915 verify_done ? "Online verify" : "Resync",
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200916 dt + device->rs_paused, device->rs_paused, dbdt);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700917
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200918 n_oos = drbd_bm_total_weight(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700919
920 if (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) {
921 if (n_oos) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200922 drbd_alert(device, "Online verify found %lu %dk block out of sync!\n",
Philipp Reisnerb411b362009-09-25 16:07:19 -0700923 n_oos, Bit2KB(1));
924 khelper_cmd = "out-of-sync";
925 }
926 } else {
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +0200927 D_ASSERT(device, (n_oos - device->rs_failed) == 0);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700928
929 if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T)
930 khelper_cmd = "after-resync-target";
931
Lars Ellenbergaaaba342014-03-18 12:30:09 +0100932 if (device->use_csums && device->rs_total) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200933 const unsigned long s = device->rs_same_csum;
934 const unsigned long t = device->rs_total;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700935 const int ratio =
936 (t == 0) ? 0 :
937 (t < 100000) ? ((s*100)/t) : (s/(t/100));
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200938 drbd_info(device, "%u %% had equal checksums, eliminated: %luK; "
Philipp Reisnerb411b362009-09-25 16:07:19 -0700939 "transferred %luK total %luK\n",
940 ratio,
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200941 Bit2KB(device->rs_same_csum),
942 Bit2KB(device->rs_total - device->rs_same_csum),
943 Bit2KB(device->rs_total));
Philipp Reisnerb411b362009-09-25 16:07:19 -0700944 }
945 }
946
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200947 if (device->rs_failed) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200948 drbd_info(device, " %lu failed blocks\n", device->rs_failed);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700949
950 if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) {
951 ns.disk = D_INCONSISTENT;
952 ns.pdsk = D_UP_TO_DATE;
953 } else {
954 ns.disk = D_UP_TO_DATE;
955 ns.pdsk = D_INCONSISTENT;
956 }
957 } else {
958 ns.disk = D_UP_TO_DATE;
959 ns.pdsk = D_UP_TO_DATE;
960
961 if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200962 if (device->p_uuid) {
Philipp Reisnerb411b362009-09-25 16:07:19 -0700963 int i;
964 for (i = UI_BITMAP ; i <= UI_HISTORY_END ; i++)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200965 _drbd_uuid_set(device, i, device->p_uuid[i]);
966 drbd_uuid_set(device, UI_BITMAP, device->ldev->md.uuid[UI_CURRENT]);
967 _drbd_uuid_set(device, UI_CURRENT, device->p_uuid[UI_CURRENT]);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700968 } else {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200969 drbd_err(device, "device->p_uuid is NULL! BUG\n");
Philipp Reisnerb411b362009-09-25 16:07:19 -0700970 }
971 }
972
Lars Ellenberg62b0da32011-01-20 13:25:21 +0100973 if (!(os.conn == C_VERIFY_S || os.conn == C_VERIFY_T)) {
974 /* for verify runs, we don't update uuids here,
975 * so there would be nothing to report. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200976 drbd_uuid_set_bm(device, 0UL);
977 drbd_print_uuids(device, "updated UUIDs");
978 if (device->p_uuid) {
Lars Ellenberg62b0da32011-01-20 13:25:21 +0100979 /* Now the two UUID sets are equal, update what we
980 * know of the peer. */
981 int i;
982 for (i = UI_CURRENT ; i <= UI_HISTORY_END ; i++)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200983 device->p_uuid[i] = device->ldev->md.uuid[i];
Lars Ellenberg62b0da32011-01-20 13:25:21 +0100984 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700985 }
986 }
987
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200988 _drbd_set_state(device, ns, CS_VERBOSE, NULL);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700989out_unlock:
Andreas Gruenbacher05008132011-07-07 14:19:42 +0200990 spin_unlock_irq(&device->resource->req_lock);
Lars Ellenberg26a96112016-06-14 00:26:25 +0200991
992 /* If we have been sync source, and have an effective fencing-policy,
993 * once *all* volumes are back in sync, call "unfence". */
994 if (os.conn == C_SYNC_SOURCE) {
995 enum drbd_disk_state disk_state = D_MASK;
996 enum drbd_disk_state pdsk_state = D_MASK;
997 enum drbd_fencing_p fp = FP_DONT_CARE;
998
999 rcu_read_lock();
1000 fp = rcu_dereference(device->ldev->disk_conf)->fencing;
1001 if (fp != FP_DONT_CARE) {
1002 struct drbd_peer_device *peer_device;
1003 int vnr;
1004 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
1005 struct drbd_device *device = peer_device->device;
1006 disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk);
1007 pdsk_state = min_t(enum drbd_disk_state, pdsk_state, device->state.pdsk);
1008 }
1009 }
1010 rcu_read_unlock();
1011 if (disk_state == D_UP_TO_DATE && pdsk_state == D_UP_TO_DATE)
1012 conn_khelper(connection, "unfence-peer");
1013 }
1014
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001015 put_ldev(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001016out:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001017 device->rs_total = 0;
1018 device->rs_failed = 0;
1019 device->rs_paused = 0;
Lars Ellenberg58ffa582012-07-26 14:09:49 +02001020
1021 /* reset start sector, if we reached end of device */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001022 if (verify_done && device->ov_left == 0)
1023 device->ov_start_sector = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001024
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001025 drbd_md_sync(device);
Lars Ellenberg13d42682010-10-13 17:37:54 +02001026
Philipp Reisnerb411b362009-09-25 16:07:19 -07001027 if (khelper_cmd)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001028 drbd_khelper(device, khelper_cmd);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001029
1030 return 1;
1031}
1032
1033/* helper */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001034static void move_to_net_ee_or_free(struct drbd_device *device, struct drbd_peer_request *peer_req)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001035{
Andreas Gruenbacher045417f2011-04-07 21:34:24 +02001036 if (drbd_peer_req_has_active_page(peer_req)) {
Philipp Reisnerb411b362009-09-25 16:07:19 -07001037 /* This might happen if sendpage() has not finished */
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001038 int i = (peer_req->i.size + PAGE_SIZE -1) >> PAGE_SHIFT;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001039 atomic_add(i, &device->pp_in_use_by_net);
1040 atomic_sub(i, &device->pp_in_use);
Andreas Gruenbacher05008132011-07-07 14:19:42 +02001041 spin_lock_irq(&device->resource->req_lock);
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +02001042 list_add_tail(&peer_req->w.list, &device->net_ee);
Andreas Gruenbacher05008132011-07-07 14:19:42 +02001043 spin_unlock_irq(&device->resource->req_lock);
Lars Ellenberg435f0742010-09-06 12:30:25 +02001044 wake_up(&drbd_pp_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001045 } else
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001046 drbd_free_peer_req(device, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001047}
1048
1049/**
1050 * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST
Philipp Reisnerb411b362009-09-25 16:07:19 -07001051 * @w: work object.
1052 * @cancel: The connection will be closed anyways
1053 */
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001054int w_e_end_data_req(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001055{
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +02001056 struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001057 struct drbd_peer_device *peer_device = peer_req->peer_device;
1058 struct drbd_device *device = peer_device->device;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001059 int err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001060
1061 if (unlikely(cancel)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001062 drbd_free_peer_req(device, peer_req);
1063 dec_unacked(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001064 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001065 }
1066
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001067 if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001068 err = drbd_send_block(peer_device, P_DATA_REPLY, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001069 } else {
1070 if (__ratelimit(&drbd_ratelimit_state))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001071 drbd_err(device, "Sending NegDReply. sector=%llus.\n",
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001072 (unsigned long long)peer_req->i.sector);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001073
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001074 err = drbd_send_ack(peer_device, P_NEG_DREPLY, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001075 }
1076
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001077 dec_unacked(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001078
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001079 move_to_net_ee_or_free(device, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001080
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001081 if (unlikely(err))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001082 drbd_err(device, "drbd_send_block() failed\n");
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001083 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001084}
1085
Philipp Reisner700ca8c2016-06-14 00:26:13 +02001086static bool all_zero(struct drbd_peer_request *peer_req)
1087{
1088 struct page *page = peer_req->pages;
1089 unsigned int len = peer_req->i.size;
1090
1091 page_chain_for_each(page) {
1092 unsigned int l = min_t(unsigned int, len, PAGE_SIZE);
1093 unsigned int i, words = l / sizeof(long);
1094 unsigned long *d;
1095
1096 d = kmap_atomic(page);
1097 for (i = 0; i < words; i++) {
1098 if (d[i]) {
1099 kunmap_atomic(d);
1100 return false;
1101 }
1102 }
1103 kunmap_atomic(d);
1104 len -= l;
1105 }
1106
1107 return true;
1108}
1109
Philipp Reisnerb411b362009-09-25 16:07:19 -07001110/**
Andreas Gruenbachera209b4a2011-08-17 12:43:25 +02001111 * w_e_end_rsdata_req() - Worker callback to send a P_RS_DATA_REPLY packet in response to a P_RS_DATA_REQUEST
Philipp Reisnerb411b362009-09-25 16:07:19 -07001112 * @w: work object.
1113 * @cancel: The connection will be closed anyways
1114 */
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001115int w_e_end_rsdata_req(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001116{
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +02001117 struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001118 struct drbd_peer_device *peer_device = peer_req->peer_device;
1119 struct drbd_device *device = peer_device->device;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001120 int err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001121
1122 if (unlikely(cancel)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001123 drbd_free_peer_req(device, peer_req);
1124 dec_unacked(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001125 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001126 }
1127
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001128 if (get_ldev_if_state(device, D_FAILED)) {
1129 drbd_rs_complete_io(device, peer_req->i.sector);
1130 put_ldev(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001131 }
1132
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001133 if (device->state.conn == C_AHEAD) {
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001134 err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req);
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001135 } else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001136 if (likely(device->state.pdsk >= D_INCONSISTENT)) {
1137 inc_rs_pending(device);
Philipp Reisner700ca8c2016-06-14 00:26:13 +02001138 if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req))
1139 err = drbd_send_rs_deallocated(peer_device, peer_req);
1140 else
1141 err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001142 } else {
1143 if (__ratelimit(&drbd_ratelimit_state))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001144 drbd_err(device, "Not sending RSDataReply, "
Philipp Reisnerb411b362009-09-25 16:07:19 -07001145 "partner DISKLESS!\n");
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001146 err = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001147 }
1148 } else {
1149 if (__ratelimit(&drbd_ratelimit_state))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001150 drbd_err(device, "Sending NegRSDReply. sector %llus.\n",
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001151 (unsigned long long)peer_req->i.sector);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001152
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001153 err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001154
1155 /* update resync data with failure */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001156 drbd_rs_failed_io(device, peer_req->i.sector, peer_req->i.size);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001157 }
1158
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001159 dec_unacked(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001160
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001161 move_to_net_ee_or_free(device, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001162
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001163 if (unlikely(err))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001164 drbd_err(device, "drbd_send_block() failed\n");
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001165 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001166}
1167
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001168int w_e_end_csum_rs_req(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001169{
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +02001170 struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001171 struct drbd_peer_device *peer_device = peer_req->peer_device;
1172 struct drbd_device *device = peer_device->device;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001173 struct digest_info *di;
1174 int digest_size;
1175 void *digest = NULL;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001176 int err, eq = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001177
1178 if (unlikely(cancel)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001179 drbd_free_peer_req(device, peer_req);
1180 dec_unacked(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001181 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001182 }
1183
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001184 if (get_ldev(device)) {
1185 drbd_rs_complete_io(device, peer_req->i.sector);
1186 put_ldev(device);
Lars Ellenberg1d53f092010-09-05 01:13:24 +02001187 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07001188
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001189 di = peer_req->digest;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001190
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001191 if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
Philipp Reisnerb411b362009-09-25 16:07:19 -07001192 /* quick hack to try to avoid a race against reconfiguration.
1193 * a real fix would be much more involved,
1194 * introducing more locking mechanisms */
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001195 if (peer_device->connection->csums_tfm) {
Kees Cook3d0e6372018-08-06 16:32:16 -07001196 digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm);
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001197 D_ASSERT(device, digest_size == di->digest_size);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001198 digest = kmalloc(digest_size, GFP_NOIO);
1199 }
1200 if (digest) {
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001201 drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001202 eq = !memcmp(digest, di->digest, digest_size);
1203 kfree(digest);
1204 }
1205
1206 if (eq) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001207 drbd_set_in_sync(device, peer_req->i.sector, peer_req->i.size);
Lars Ellenberg676396d2010-03-03 02:08:22 +01001208 /* rs_same_csums unit is BM_BLOCK_SIZE */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001209 device->rs_same_csum += peer_req->i.size >> BM_BLOCK_SHIFT;
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001210 err = drbd_send_ack(peer_device, P_RS_IS_IN_SYNC, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001211 } else {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001212 inc_rs_pending(device);
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001213 peer_req->block_id = ID_SYNCER; /* By setting block_id, digest pointer becomes invalid! */
1214 peer_req->flags &= ~EE_HAS_DIGEST; /* This peer request no longer has a digest pointer */
Philipp Reisner204bba92010-08-23 16:17:13 +02001215 kfree(di);
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001216 err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001217 }
1218 } else {
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001219 err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001220 if (__ratelimit(&drbd_ratelimit_state))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001221 drbd_err(device, "Sending NegDReply. I guess it gets messy.\n");
Philipp Reisnerb411b362009-09-25 16:07:19 -07001222 }
1223
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001224 dec_unacked(device);
1225 move_to_net_ee_or_free(device, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001226
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001227 if (unlikely(err))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001228 drbd_err(device, "drbd_send_block/ack() failed\n");
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001229 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001230}
1231
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001232int w_e_end_ov_req(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001233{
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +02001234 struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001235 struct drbd_peer_device *peer_device = peer_req->peer_device;
1236 struct drbd_device *device = peer_device->device;
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001237 sector_t sector = peer_req->i.sector;
1238 unsigned int size = peer_req->i.size;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001239 int digest_size;
1240 void *digest;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001241 int err = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001242
1243 if (unlikely(cancel))
1244 goto out;
1245
Kees Cook3d0e6372018-08-06 16:32:16 -07001246 digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001247 digest = kmalloc(digest_size, GFP_NOIO);
Philipp Reisner8f214202011-03-01 15:52:35 +01001248 if (!digest) {
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001249 err = 1; /* terminate the connection in case the allocation failed */
Philipp Reisner8f214202011-03-01 15:52:35 +01001250 goto out;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001251 }
1252
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001253 if (likely(!(peer_req->flags & EE_WAS_ERROR)))
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001254 drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest);
Philipp Reisner8f214202011-03-01 15:52:35 +01001255 else
1256 memset(digest, 0, digest_size);
1257
Lars Ellenberg53ea4332011-03-08 17:11:40 +01001258 /* Free e and pages before send.
1259 * In case we block on congestion, we could otherwise run into
1260 * some distributed deadlock, if the other side blocks on
1261 * congestion as well, because our receiver blocks in
Andreas Gruenbacherc37c8ec2011-04-07 21:02:09 +02001262 * drbd_alloc_pages due to pp_in_use > max_buffers. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001263 drbd_free_peer_req(device, peer_req);
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001264 peer_req = NULL;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001265 inc_rs_pending(device);
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001266 err = drbd_send_drequest_csum(peer_device, sector, size, digest, digest_size, P_OV_REPLY);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001267 if (err)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001268 dec_rs_pending(device);
Philipp Reisner8f214202011-03-01 15:52:35 +01001269 kfree(digest);
1270
Philipp Reisnerb411b362009-09-25 16:07:19 -07001271out:
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001272 if (peer_req)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001273 drbd_free_peer_req(device, peer_req);
1274 dec_unacked(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001275 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001276}
1277
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001278void drbd_ov_out_of_sync_found(struct drbd_device *device, sector_t sector, int size)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001279{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001280 if (device->ov_last_oos_start + device->ov_last_oos_size == sector) {
1281 device->ov_last_oos_size += size>>9;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001282 } else {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001283 device->ov_last_oos_start = sector;
1284 device->ov_last_oos_size = size>>9;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001285 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001286 drbd_set_out_of_sync(device, sector, size);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001287}
1288
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001289int w_e_end_ov_reply(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001290{
Andreas Gruenbachera8cd15b2011-08-25 15:49:40 +02001291 struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001292 struct drbd_peer_device *peer_device = peer_req->peer_device;
1293 struct drbd_device *device = peer_device->device;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001294 struct digest_info *di;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001295 void *digest;
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001296 sector_t sector = peer_req->i.sector;
1297 unsigned int size = peer_req->i.size;
Lars Ellenberg53ea4332011-03-08 17:11:40 +01001298 int digest_size;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001299 int err, eq = 0;
Lars Ellenberg58ffa582012-07-26 14:09:49 +02001300 bool stop_sector_reached = false;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001301
1302 if (unlikely(cancel)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001303 drbd_free_peer_req(device, peer_req);
1304 dec_unacked(device);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001305 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001306 }
1307
1308 /* after "cancel", because after drbd_disconnect/drbd_rs_cancel_all
1309 * the resync lru has been cleaned up already */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001310 if (get_ldev(device)) {
1311 drbd_rs_complete_io(device, peer_req->i.sector);
1312 put_ldev(device);
Lars Ellenberg1d53f092010-09-05 01:13:24 +02001313 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07001314
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001315 di = peer_req->digest;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001316
Andreas Gruenbacherdb830c42011-02-04 15:57:48 +01001317 if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
Kees Cook3d0e6372018-08-06 16:32:16 -07001318 digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001319 digest = kmalloc(digest_size, GFP_NOIO);
1320 if (digest) {
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001321 drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001322
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001323 D_ASSERT(device, digest_size == di->digest_size);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001324 eq = !memcmp(digest, di->digest, digest_size);
1325 kfree(digest);
1326 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07001327 }
1328
Lars Ellenberg9676c762011-02-22 14:02:31 +01001329 /* Free peer_req and pages before send.
1330 * In case we block on congestion, we could otherwise run into
1331 * some distributed deadlock, if the other side blocks on
1332 * congestion as well, because our receiver blocks in
Andreas Gruenbacherc37c8ec2011-04-07 21:02:09 +02001333 * drbd_alloc_pages due to pp_in_use > max_buffers. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001334 drbd_free_peer_req(device, peer_req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001335 if (!eq)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001336 drbd_ov_out_of_sync_found(device, sector, size);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001337 else
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001338 ov_out_of_sync_print(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001339
Andreas Gruenbacher67801392011-09-13 10:39:41 +02001340 err = drbd_send_ack_ex(peer_device, P_OV_RESULT, sector, size,
Andreas Gruenbacherfa79abd2011-03-16 01:31:39 +01001341 eq ? ID_IN_SYNC : ID_OUT_OF_SYNC);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001342
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001343 dec_unacked(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001344
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001345 --device->ov_left;
Lars Ellenbergea5442a2010-11-05 09:48:01 +01001346
1347 /* let's advance progress step marks only for every other megabyte */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001348 if ((device->ov_left & 0x200) == 0x200)
1349 drbd_advance_rs_marks(device, device->ov_left);
Lars Ellenbergea5442a2010-11-05 09:48:01 +01001350
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001351 stop_sector_reached = verify_can_do_stop_sector(device) &&
1352 (sector + (size>>9)) >= device->ov_stop_sector;
Lars Ellenberg58ffa582012-07-26 14:09:49 +02001353
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001354 if (device->ov_left == 0 || stop_sector_reached) {
1355 ov_out_of_sync_print(device);
1356 drbd_resync_finished(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001357 }
1358
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001359 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001360}
1361
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001362/* FIXME
1363 * We need to track the number of pending barrier acks,
1364 * and to be able to wait for them.
1365 * See also comment in drbd_adm_attach before drbd_suspend_io.
1366 */
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001367static int drbd_send_barrier(struct drbd_connection *connection)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001368{
Andreas Gruenbacher9f5bdc32011-03-28 14:23:08 +02001369 struct p_barrier *p;
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001370 struct drbd_socket *sock;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001371
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001372 sock = &connection->data;
1373 p = conn_prepare_command(connection, sock);
Andreas Gruenbacher9f5bdc32011-03-28 14:23:08 +02001374 if (!p)
1375 return -EIO;
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001376 p->barrier = connection->send.current_epoch_nr;
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001377 p->pad = 0;
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001378 connection->send.current_epoch_writes = 0;
Lars Ellenberg84d34f22015-02-19 13:54:11 +01001379 connection->send.last_sent_barrier_jif = jiffies;
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001380
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001381 return conn_send_command(connection, sock, P_BARRIER, sizeof(*p), NULL, 0);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001382}
1383
Lars Ellenbergc51a0ef2017-08-29 10:20:32 +02001384static int pd_send_unplug_remote(struct drbd_peer_device *pd)
1385{
1386 struct drbd_socket *sock = &pd->connection->data;
1387 if (!drbd_prepare_command(pd, sock))
1388 return -EIO;
1389 return drbd_send_command(pd, sock, P_UNPLUG_REMOTE, 0, NULL, 0);
1390}
1391
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001392int w_send_write_hint(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001393{
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +02001394 struct drbd_device *device =
1395 container_of(w, struct drbd_device, unplug_work);
Andreas Gruenbacher9f5bdc32011-03-28 14:23:08 +02001396
Philipp Reisnerb411b362009-09-25 16:07:19 -07001397 if (cancel)
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001398 return 0;
Lars Ellenbergc51a0ef2017-08-29 10:20:32 +02001399 return pd_send_unplug_remote(first_peer_device(device));
Philipp Reisnerb411b362009-09-25 16:07:19 -07001400}
1401
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001402static void re_init_if_first_write(struct drbd_connection *connection, unsigned int epoch)
Lars Ellenberg4eb9b3c2012-08-20 11:05:23 +02001403{
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001404 if (!connection->send.seen_any_write_yet) {
1405 connection->send.seen_any_write_yet = true;
1406 connection->send.current_epoch_nr = epoch;
1407 connection->send.current_epoch_writes = 0;
Lars Ellenberg84d34f22015-02-19 13:54:11 +01001408 connection->send.last_sent_barrier_jif = jiffies;
Lars Ellenberg4eb9b3c2012-08-20 11:05:23 +02001409 }
1410}
1411
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001412static void maybe_send_barrier(struct drbd_connection *connection, unsigned int epoch)
Lars Ellenberg4eb9b3c2012-08-20 11:05:23 +02001413{
1414 /* re-init if first write on this connection */
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001415 if (!connection->send.seen_any_write_yet)
Lars Ellenberg4eb9b3c2012-08-20 11:05:23 +02001416 return;
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001417 if (connection->send.current_epoch_nr != epoch) {
1418 if (connection->send.current_epoch_writes)
1419 drbd_send_barrier(connection);
1420 connection->send.current_epoch_nr = epoch;
Lars Ellenberg4eb9b3c2012-08-20 11:05:23 +02001421 }
1422}
1423
Andreas Gruenbacher8f7bed72010-12-19 23:53:14 +01001424int w_send_out_of_sync(struct drbd_work *w, int cancel)
Philipp Reisner73a01a12010-10-27 14:33:00 +02001425{
1426 struct drbd_request *req = container_of(w, struct drbd_request, w);
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +02001427 struct drbd_device *device = req->device;
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001428 struct drbd_peer_device *const peer_device = first_peer_device(device);
1429 struct drbd_connection *const connection = peer_device->connection;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001430 int err;
Philipp Reisner73a01a12010-10-27 14:33:00 +02001431
1432 if (unlikely(cancel)) {
Andreas Gruenbacher8554df12011-01-25 15:37:43 +01001433 req_mod(req, SEND_CANCELED);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001434 return 0;
Philipp Reisner73a01a12010-10-27 14:33:00 +02001435 }
Lars Ellenberge5f891b2013-11-22 12:32:01 +01001436 req->pre_send_jif = jiffies;
Philipp Reisner73a01a12010-10-27 14:33:00 +02001437
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001438 /* this time, no connection->send.current_epoch_writes++;
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001439 * If it was sent, it was the closing barrier for the last
1440 * replicated epoch, before we went into AHEAD mode.
1441 * No more barriers will be sent, until we leave AHEAD mode again. */
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001442 maybe_send_barrier(connection, req->epoch);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001443
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001444 err = drbd_send_out_of_sync(peer_device, req);
Andreas Gruenbacher8554df12011-01-25 15:37:43 +01001445 req_mod(req, OOS_HANDED_TO_NETWORK);
Philipp Reisner73a01a12010-10-27 14:33:00 +02001446
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001447 return err;
Philipp Reisner73a01a12010-10-27 14:33:00 +02001448}
1449
Philipp Reisnerb411b362009-09-25 16:07:19 -07001450/**
1451 * w_send_dblock() - Worker callback to send a P_DATA packet in order to mirror a write request
Philipp Reisnerb411b362009-09-25 16:07:19 -07001452 * @w: work object.
1453 * @cancel: The connection will be closed anyways
1454 */
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001455int w_send_dblock(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001456{
1457 struct drbd_request *req = container_of(w, struct drbd_request, w);
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +02001458 struct drbd_device *device = req->device;
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001459 struct drbd_peer_device *const peer_device = first_peer_device(device);
1460 struct drbd_connection *connection = peer_device->connection;
Lars Ellenbergc51a0ef2017-08-29 10:20:32 +02001461 bool do_send_unplug = req->rq_state & RQ_UNPLUG;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001462 int err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001463
1464 if (unlikely(cancel)) {
Andreas Gruenbacher8554df12011-01-25 15:37:43 +01001465 req_mod(req, SEND_CANCELED);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001466 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001467 }
Lars Ellenberge5f891b2013-11-22 12:32:01 +01001468 req->pre_send_jif = jiffies;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001469
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001470 re_init_if_first_write(connection, req->epoch);
1471 maybe_send_barrier(connection, req->epoch);
1472 connection->send.current_epoch_writes++;
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001473
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001474 err = drbd_send_dblock(peer_device, req);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001475 req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001476
Lars Ellenbergc51a0ef2017-08-29 10:20:32 +02001477 if (do_send_unplug && !err)
1478 pd_send_unplug_remote(peer_device);
1479
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001480 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001481}
1482
1483/**
1484 * w_send_read_req() - Worker callback to send a read request (P_DATA_REQUEST) packet
Philipp Reisnerb411b362009-09-25 16:07:19 -07001485 * @w: work object.
1486 * @cancel: The connection will be closed anyways
1487 */
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001488int w_send_read_req(struct drbd_work *w, int cancel)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001489{
1490 struct drbd_request *req = container_of(w, struct drbd_request, w);
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +02001491 struct drbd_device *device = req->device;
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001492 struct drbd_peer_device *const peer_device = first_peer_device(device);
1493 struct drbd_connection *connection = peer_device->connection;
Lars Ellenbergc51a0ef2017-08-29 10:20:32 +02001494 bool do_send_unplug = req->rq_state & RQ_UNPLUG;
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001495 int err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001496
1497 if (unlikely(cancel)) {
Andreas Gruenbacher8554df12011-01-25 15:37:43 +01001498 req_mod(req, SEND_CANCELED);
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001499 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001500 }
Lars Ellenberge5f891b2013-11-22 12:32:01 +01001501 req->pre_send_jif = jiffies;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001502
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001503 /* Even read requests may close a write epoch,
1504 * if there was any yet. */
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02001505 maybe_send_barrier(connection, req->epoch);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01001506
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001507 err = drbd_send_drequest(peer_device, P_DATA_REQUEST, req->i.sector, req->i.size,
Andreas Gruenbacher6c1005e2011-03-16 01:34:24 +01001508 (unsigned long)req);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001509
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001510 req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001511
Lars Ellenbergc51a0ef2017-08-29 10:20:32 +02001512 if (do_send_unplug && !err)
1513 pd_send_unplug_remote(peer_device);
1514
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001515 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001516}
1517
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001518int w_restart_disk_io(struct drbd_work *w, int cancel)
Philipp Reisner265be2d2010-05-31 10:14:17 +02001519{
1520 struct drbd_request *req = container_of(w, struct drbd_request, w);
Andreas Gruenbacher84b8c062011-07-28 15:27:51 +02001521 struct drbd_device *device = req->device;
Philipp Reisner265be2d2010-05-31 10:14:17 +02001522
Philipp Reisner07782862010-08-31 12:00:50 +02001523 if (bio_data_dir(req->master_bio) == WRITE && req->rq_state & RQ_IN_ACT_LOG)
Lars Ellenberg4dd726f2014-02-11 11:15:36 +01001524 drbd_al_begin_io(device, &req->i);
Philipp Reisner265be2d2010-05-31 10:14:17 +02001525
Christoph Hellwigae7153f2021-01-26 15:52:40 +01001526 req->private_bio = bio_clone_fast(req->master_bio, GFP_NOIO,
1527 &drbd_io_bio_set);
Christoph Hellwig74d46992017-08-23 19:10:32 +02001528 bio_set_dev(req->private_bio, device->ldev->backing_bdev);
Christoph Hellwigae7153f2021-01-26 15:52:40 +01001529 req->private_bio->bi_private = req;
1530 req->private_bio->bi_end_io = drbd_request_endio;
Christoph Hellwiged00aab2020-07-01 10:59:44 +02001531 submit_bio_noacct(req->private_bio);
Philipp Reisner265be2d2010-05-31 10:14:17 +02001532
Andreas Gruenbacher99920dc2011-03-16 15:31:39 +01001533 return 0;
Philipp Reisner265be2d2010-05-31 10:14:17 +02001534}
1535
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001536static int _drbd_may_sync_now(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001537{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001538 struct drbd_device *odev = device;
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001539 int resync_after;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001540
1541 while (1) {
Lars Ellenberga3f8f7d2013-03-27 14:08:43 +01001542 if (!odev->ldev || odev->state.disk == D_DISKLESS)
Philipp Reisner438c8372011-03-28 14:48:01 +02001543 return 1;
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +02001544 rcu_read_lock();
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001545 resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after;
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +02001546 rcu_read_unlock();
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001547 if (resync_after == -1)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001548 return 1;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001549 odev = minor_to_device(resync_after);
Lars Ellenberga3f8f7d2013-03-27 14:08:43 +01001550 if (!odev)
Andreas Gruenbacher841ce242010-12-15 19:31:20 +01001551 return 1;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001552 if ((odev->state.conn >= C_SYNC_SOURCE &&
1553 odev->state.conn <= C_PAUSED_SYNC_T) ||
1554 odev->state.aftr_isp || odev->state.peer_isp ||
1555 odev->state.user_isp)
1556 return 0;
1557 }
1558}
1559
1560/**
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001561 * drbd_pause_after() - Pause resync on all devices that may not resync now
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001562 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -07001563 *
1564 * Called from process context only (admin command and after_state_ch).
1565 */
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001566static bool drbd_pause_after(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001567{
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001568 bool changed = false;
Andreas Gruenbacher54761692011-05-30 16:15:21 +02001569 struct drbd_device *odev;
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001570 int i;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001571
Philipp Reisner695d08f2011-04-11 22:53:32 -07001572 rcu_read_lock();
Andreas Gruenbacher05a10ec2011-06-07 22:54:17 +02001573 idr_for_each_entry(&drbd_devices, odev, i) {
Philipp Reisnerb411b362009-09-25 16:07:19 -07001574 if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
1575 continue;
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001576 if (!_drbd_may_sync_now(odev) &&
1577 _drbd_set_state(_NS(odev, aftr_isp, 1),
1578 CS_HARD, NULL) != SS_NOTHING_TO_DO)
1579 changed = true;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001580 }
Philipp Reisner695d08f2011-04-11 22:53:32 -07001581 rcu_read_unlock();
Philipp Reisnerb411b362009-09-25 16:07:19 -07001582
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001583 return changed;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001584}
1585
1586/**
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001587 * drbd_resume_next() - Resume resync on all devices that may resync now
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001588 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -07001589 *
1590 * Called from process context only (admin command and worker).
1591 */
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001592static bool drbd_resume_next(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001593{
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001594 bool changed = false;
Andreas Gruenbacher54761692011-05-30 16:15:21 +02001595 struct drbd_device *odev;
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001596 int i;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001597
Philipp Reisner695d08f2011-04-11 22:53:32 -07001598 rcu_read_lock();
Andreas Gruenbacher05a10ec2011-06-07 22:54:17 +02001599 idr_for_each_entry(&drbd_devices, odev, i) {
Philipp Reisnerb411b362009-09-25 16:07:19 -07001600 if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
1601 continue;
1602 if (odev->state.aftr_isp) {
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001603 if (_drbd_may_sync_now(odev) &&
1604 _drbd_set_state(_NS(odev, aftr_isp, 0),
1605 CS_HARD, NULL) != SS_NOTHING_TO_DO)
1606 changed = true;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001607 }
1608 }
Philipp Reisner695d08f2011-04-11 22:53:32 -07001609 rcu_read_unlock();
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001610 return changed;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001611}
1612
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001613void resume_next_sg(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001614{
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001615 lock_all_resources();
1616 drbd_resume_next(device);
1617 unlock_all_resources();
Philipp Reisnerb411b362009-09-25 16:07:19 -07001618}
1619
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001620void suspend_other_sg(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001621{
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001622 lock_all_resources();
1623 drbd_pause_after(device);
1624 unlock_all_resources();
Philipp Reisnerb411b362009-09-25 16:07:19 -07001625}
1626
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001627/* caller must lock_all_resources() */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001628enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001629{
Andreas Gruenbacher54761692011-05-30 16:15:21 +02001630 struct drbd_device *odev;
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001631 int resync_after;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001632
1633 if (o_minor == -1)
1634 return NO_ERROR;
Lars Ellenberga3f8f7d2013-03-27 14:08:43 +01001635 if (o_minor < -1 || o_minor > MINORMASK)
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001636 return ERR_RESYNC_AFTER;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001637
1638 /* check for loops */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001639 odev = minor_to_device(o_minor);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001640 while (1) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001641 if (odev == device)
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001642 return ERR_RESYNC_AFTER_CYCLE;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001643
Lars Ellenberga3f8f7d2013-03-27 14:08:43 +01001644 /* You are free to depend on diskless, non-existing,
1645 * or not yet/no longer existing minors.
1646 * We only reject dependency loops.
1647 * We cannot follow the dependency chain beyond a detached or
1648 * missing minor.
1649 */
1650 if (!odev || !odev->ldev || odev->state.disk == D_DISKLESS)
1651 return NO_ERROR;
1652
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +02001653 rcu_read_lock();
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001654 resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after;
Philipp Reisnerdaeda1c2011-05-03 15:00:55 +02001655 rcu_read_unlock();
Philipp Reisnerb411b362009-09-25 16:07:19 -07001656 /* dependency chain ends here, no cycles. */
Andreas Gruenbacher95f8efd2011-05-12 11:15:34 +02001657 if (resync_after == -1)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001658 return NO_ERROR;
1659
1660 /* follow the dependency chain */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001661 odev = minor_to_device(resync_after);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001662 }
1663}
1664
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001665/* caller must lock_all_resources() */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001666void drbd_resync_after_changed(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001667{
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001668 int changed;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001669
Philipp Reisnerdc97b702011-05-03 14:27:15 +02001670 do {
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001671 changed = drbd_pause_after(device);
1672 changed |= drbd_resume_next(device);
1673 } while (changed);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001674}
1675
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001676void drbd_rs_controller_reset(struct drbd_device *device)
Lars Ellenberg9bd28d32010-11-05 09:55:18 +01001677{
Christoph Hellwig8c40c7c2020-09-03 07:41:00 +02001678 struct gendisk *disk = device->ldev->backing_bdev->bd_disk;
Philipp Reisner813472c2011-05-03 16:47:02 +02001679 struct fifo_buffer *plan;
1680
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001681 atomic_set(&device->rs_sect_in, 0);
1682 atomic_set(&device->rs_sect_ev, 0);
1683 device->rs_in_flight = 0;
Christoph Hellwigcb8432d2020-11-26 18:47:17 +01001684 device->rs_last_events =
Christoph Hellwig8446fe92020-11-24 09:36:54 +01001685 (int)part_stat_read_accum(disk->part0, sectors);
Philipp Reisner813472c2011-05-03 16:47:02 +02001686
1687 /* Updating the RCU protected object in place is necessary since
1688 this function gets called from atomic context.
1689 It is valid since all other updates also lead to an completely
1690 empty fifo */
1691 rcu_read_lock();
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001692 plan = rcu_dereference(device->rs_plan_s);
Philipp Reisner813472c2011-05-03 16:47:02 +02001693 plan->total = 0;
1694 fifo_set(plan, 0);
1695 rcu_read_unlock();
Lars Ellenberg9bd28d32010-11-05 09:55:18 +01001696}
1697
Kees Cook2bccef32017-10-17 20:33:01 -07001698void start_resync_timer_fn(struct timer_list *t)
Philipp Reisner1f04af32011-02-07 11:33:59 +01001699{
Kees Cook2bccef32017-10-17 20:33:01 -07001700 struct drbd_device *device = from_timer(device, t, start_resync_timer);
Lars Ellenbergac0acb92014-02-11 09:47:58 +01001701 drbd_device_post_work(device, RS_START);
Philipp Reisner1f04af32011-02-07 11:33:59 +01001702}
1703
Lars Ellenbergac0acb92014-02-11 09:47:58 +01001704static void do_start_resync(struct drbd_device *device)
Philipp Reisner1f04af32011-02-07 11:33:59 +01001705{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001706 if (atomic_read(&device->unacked_cnt) || atomic_read(&device->rs_pending_cnt)) {
Lars Ellenbergac0acb92014-02-11 09:47:58 +01001707 drbd_warn(device, "postponing start_resync ...\n");
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001708 device->start_resync_timer.expires = jiffies + HZ/10;
1709 add_timer(&device->start_resync_timer);
Lars Ellenbergac0acb92014-02-11 09:47:58 +01001710 return;
Philipp Reisner1f04af32011-02-07 11:33:59 +01001711 }
1712
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001713 drbd_start_resync(device, C_SYNC_SOURCE);
1714 clear_bit(AHEAD_TO_SYNC_SOURCE, &device->flags);
Philipp Reisner1f04af32011-02-07 11:33:59 +01001715}
1716
Lars Ellenbergaaaba342014-03-18 12:30:09 +01001717static bool use_checksum_based_resync(struct drbd_connection *connection, struct drbd_device *device)
1718{
1719 bool csums_after_crash_only;
1720 rcu_read_lock();
1721 csums_after_crash_only = rcu_dereference(connection->net_conf)->csums_after_crash_only;
1722 rcu_read_unlock();
1723 return connection->agreed_pro_version >= 89 && /* supported? */
1724 connection->csums_tfm && /* configured? */
Fabian Frederick7e5fec32016-06-14 00:26:35 +02001725 (csums_after_crash_only == false /* use for each resync? */
Lars Ellenbergaaaba342014-03-18 12:30:09 +01001726 || test_bit(CRASHED_PRIMARY, &device->flags)); /* or only after Primary crash? */
1727}
1728
Philipp Reisnerb411b362009-09-25 16:07:19 -07001729/**
1730 * drbd_start_resync() - Start the resync process
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001731 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -07001732 * @side: Either C_SYNC_SOURCE or C_SYNC_TARGET
1733 *
1734 * This function might bring you directly into one of the
1735 * C_PAUSED_SYNC_* states.
1736 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001737void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001738{
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001739 struct drbd_peer_device *peer_device = first_peer_device(device);
1740 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001741 union drbd_state ns;
1742 int r;
1743
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001744 if (device->state.conn >= C_SYNC_SOURCE && device->state.conn < C_AHEAD) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001745 drbd_err(device, "Resync already running!\n");
Philipp Reisnerb411b362009-09-25 16:07:19 -07001746 return;
1747 }
1748
Roland Kammererd3d29482017-08-29 10:20:47 +02001749 if (!connection) {
1750 drbd_err(device, "No connection to peer, aborting!\n");
1751 return;
1752 }
1753
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001754 if (!test_bit(B_RS_H_DONE, &device->flags)) {
Philipp Reisnere64a3292011-02-05 17:34:11 +01001755 if (side == C_SYNC_TARGET) {
1756 /* Since application IO was locked out during C_WF_BITMAP_T and
1757 C_WF_SYNC_UUID we are still unmodified. Before going to C_SYNC_TARGET
1758 we check that we might make the data inconsistent. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001759 r = drbd_khelper(device, "before-resync-target");
Philipp Reisnere64a3292011-02-05 17:34:11 +01001760 r = (r >> 8) & 0xff;
1761 if (r > 0) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001762 drbd_info(device, "before-resync-target handler returned %d, "
Philipp Reisner09b9e792010-12-03 16:04:24 +01001763 "dropping connection.\n", r);
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001764 conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
Philipp Reisner09b9e792010-12-03 16:04:24 +01001765 return;
1766 }
Philipp Reisnere64a3292011-02-05 17:34:11 +01001767 } else /* C_SYNC_SOURCE */ {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001768 r = drbd_khelper(device, "before-resync-source");
Philipp Reisnere64a3292011-02-05 17:34:11 +01001769 r = (r >> 8) & 0xff;
1770 if (r > 0) {
1771 if (r == 3) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001772 drbd_info(device, "before-resync-source handler returned %d, "
Philipp Reisnere64a3292011-02-05 17:34:11 +01001773 "ignoring. Old userland tools?", r);
1774 } else {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001775 drbd_info(device, "before-resync-source handler returned %d, "
Philipp Reisnere64a3292011-02-05 17:34:11 +01001776 "dropping connection.\n", r);
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001777 conn_request_state(connection,
Andreas Gruenbachera6b32bc32011-05-31 14:33:49 +02001778 NS(conn, C_DISCONNECTING), CS_HARD);
Philipp Reisnere64a3292011-02-05 17:34:11 +01001779 return;
1780 }
1781 }
Philipp Reisner09b9e792010-12-03 16:04:24 +01001782 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07001783 }
1784
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001785 if (current == connection->worker.task) {
Philipp Reisnerdad20552011-02-11 19:43:55 +01001786 /* The worker should not sleep waiting for state_mutex,
Philipp Reisnere64a3292011-02-05 17:34:11 +01001787 that can take long */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001788 if (!mutex_trylock(device->state_mutex)) {
1789 set_bit(B_RS_H_DONE, &device->flags);
1790 device->start_resync_timer.expires = jiffies + HZ/5;
1791 add_timer(&device->start_resync_timer);
Philipp Reisnere64a3292011-02-05 17:34:11 +01001792 return;
1793 }
1794 } else {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001795 mutex_lock(device->state_mutex);
Philipp Reisnere64a3292011-02-05 17:34:11 +01001796 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07001797
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001798 lock_all_resources();
1799 clear_bit(B_RS_H_DONE, &device->flags);
Philipp Reisnera7004712013-03-27 14:08:35 +01001800 /* Did some connection breakage or IO error race with us? */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001801 if (device->state.conn < C_CONNECTED
1802 || !get_ldev_if_state(device, D_NEGOTIATING)) {
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001803 unlock_all_resources();
1804 goto out;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001805 }
1806
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001807 ns = drbd_read_state(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001808
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001809 ns.aftr_isp = !_drbd_may_sync_now(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001810
1811 ns.conn = side;
1812
1813 if (side == C_SYNC_TARGET)
1814 ns.disk = D_INCONSISTENT;
1815 else /* side == C_SYNC_SOURCE */
1816 ns.pdsk = D_INCONSISTENT;
1817
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001818 r = _drbd_set_state(device, ns, CS_VERBOSE, NULL);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001819 ns = drbd_read_state(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001820
1821 if (ns.conn < C_CONNECTED)
1822 r = SS_UNKNOWN_ERROR;
1823
1824 if (r == SS_SUCCESS) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001825 unsigned long tw = drbd_bm_total_weight(device);
Lars Ellenberg1d7734a2010-08-11 21:21:50 +02001826 unsigned long now = jiffies;
1827 int i;
1828
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001829 device->rs_failed = 0;
1830 device->rs_paused = 0;
1831 device->rs_same_csum = 0;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001832 device->rs_last_sect_ev = 0;
1833 device->rs_total = tw;
1834 device->rs_start = now;
Lars Ellenberg1d7734a2010-08-11 21:21:50 +02001835 for (i = 0; i < DRBD_SYNC_MARKS; i++) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001836 device->rs_mark_left[i] = tw;
1837 device->rs_mark_time[i] = now;
Lars Ellenberg1d7734a2010-08-11 21:21:50 +02001838 }
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001839 drbd_pause_after(device);
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01001840 /* Forget potentially stale cached per resync extent bit-counts.
1841 * Open coded drbd_rs_cancel_all(device), we already have IRQs
1842 * disabled, and know the disk state is ok. */
1843 spin_lock(&device->al_lock);
1844 lc_reset(device->resync);
1845 device->resync_locked = 0;
1846 device->resync_wenr = LC_FREE;
1847 spin_unlock(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001848 }
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001849 unlock_all_resources();
Lars Ellenberg5a22db82010-12-17 21:14:23 +01001850
Philipp Reisnerb411b362009-09-25 16:07:19 -07001851 if (r == SS_SUCCESS) {
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01001852 wake_up(&device->al_wait); /* for lc_reset() above */
Philipp Reisner328e0f12012-10-19 14:37:47 +02001853 /* reset rs_last_bcast when a resync or verify is started,
1854 * to deal with potential jiffies wrap. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001855 device->rs_last_bcast = jiffies - HZ;
Philipp Reisner328e0f12012-10-19 14:37:47 +02001856
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001857 drbd_info(device, "Began resync as %s (will sync %lu KB [%lu bits set]).\n",
Philipp Reisnerb411b362009-09-25 16:07:19 -07001858 drbd_conn_str(ns.conn),
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001859 (unsigned long) device->rs_total << (BM_BLOCK_SHIFT-10),
1860 (unsigned long) device->rs_total);
Lars Ellenbergaaaba342014-03-18 12:30:09 +01001861 if (side == C_SYNC_TARGET) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001862 device->bm_resync_fo = 0;
Lars Ellenbergaaaba342014-03-18 12:30:09 +01001863 device->use_csums = use_checksum_based_resync(connection, device);
1864 } else {
Fabian Frederick7e5fec32016-06-14 00:26:35 +02001865 device->use_csums = false;
Lars Ellenbergaaaba342014-03-18 12:30:09 +01001866 }
Lars Ellenberg6c922ed2011-01-12 11:51:13 +01001867
1868 /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid
1869 * with w_send_oos, or the sync target will get confused as to
1870 * how much bits to resync. We cannot do that always, because for an
1871 * empty resync and protocol < 95, we need to do it here, as we call
1872 * drbd_resync_finished from here in that case.
1873 * We drbd_gen_and_send_sync_uuid here for protocol < 96,
1874 * and from after_state_ch otherwise. */
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001875 if (side == C_SYNC_SOURCE && connection->agreed_pro_version < 96)
1876 drbd_gen_and_send_sync_uuid(peer_device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001877
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001878 if (connection->agreed_pro_version < 95 && device->rs_total == 0) {
Lars Ellenbergaf85e8e2010-10-07 16:07:55 +02001879 /* This still has a race (about when exactly the peers
1880 * detect connection loss) that can lead to a full sync
1881 * on next handshake. In 8.3.9 we fixed this with explicit
1882 * resync-finished notifications, but the fix
1883 * introduces a protocol change. Sleeping for some
1884 * time longer than the ping interval + timeout on the
1885 * SyncSource, to give the SyncTarget the chance to
1886 * detect connection loss, then waiting for a ping
1887 * response (implicit in drbd_resync_finished) reduces
1888 * the race considerably, but does not solve it. */
Philipp Reisner44ed1672011-04-19 17:10:19 +02001889 if (side == C_SYNC_SOURCE) {
1890 struct net_conf *nc;
1891 int timeo;
1892
1893 rcu_read_lock();
Lars Ellenberg44a4d552013-11-22 12:40:58 +01001894 nc = rcu_dereference(connection->net_conf);
Philipp Reisner44ed1672011-04-19 17:10:19 +02001895 timeo = nc->ping_int * HZ + nc->ping_timeo * HZ / 9;
1896 rcu_read_unlock();
1897 schedule_timeout_interruptible(timeo);
1898 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001899 drbd_resync_finished(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001900 }
1901
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001902 drbd_rs_controller_reset(device);
1903 /* ns.conn may already be != device->state.conn,
Philipp Reisnerb411b362009-09-25 16:07:19 -07001904 * we may have been paused in between, or become paused until
1905 * the timer triggers.
1906 * No matter, that is handled in resync_timer_fn() */
1907 if (ns.conn == C_SYNC_TARGET)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001908 mod_timer(&device->resync_timer, jiffies);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001909
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001910 drbd_md_sync(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001911 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001912 put_ldev(device);
Andreas Gruenbacher28bc3b82014-08-14 18:33:30 +02001913out:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001914 mutex_unlock(device->state_mutex);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001915}
1916
Lars Ellenberge334f552014-02-11 09:30:49 +01001917static void update_on_disk_bitmap(struct drbd_device *device, bool resync_done)
Lars Ellenbergc7a58db2013-12-20 11:39:48 +01001918{
1919 struct sib_info sib = { .sib_reason = SIB_SYNC_PROGRESS, };
1920 device->rs_last_bcast = jiffies;
1921
1922 if (!get_ldev(device))
1923 return;
1924
1925 drbd_bm_write_lazy(device, 0);
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01001926 if (resync_done && is_sync_state(device->state.conn))
Lars Ellenbergc7a58db2013-12-20 11:39:48 +01001927 drbd_resync_finished(device);
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01001928
Lars Ellenbergc7a58db2013-12-20 11:39:48 +01001929 drbd_bcast_event(device, &sib);
1930 /* update timestamp, in case it took a while to write out stuff */
1931 device->rs_last_bcast = jiffies;
1932 put_ldev(device);
1933}
1934
Lars Ellenberge334f552014-02-11 09:30:49 +01001935static void drbd_ldev_destroy(struct drbd_device *device)
1936{
1937 lc_destroy(device->resync);
1938 device->resync = NULL;
1939 lc_destroy(device->act_log);
1940 device->act_log = NULL;
Andreas Gruenbacherd1b80852014-09-11 14:29:09 +02001941
1942 __acquire(local);
Lars Ellenberg63a7c8a2015-03-26 20:53:55 +01001943 drbd_backing_dev_free(device, device->ldev);
Andreas Gruenbacherd1b80852014-09-11 14:29:09 +02001944 device->ldev = NULL;
1945 __release(local);
1946
Lars Ellenberge334f552014-02-11 09:30:49 +01001947 clear_bit(GOING_DISKLESS, &device->flags);
1948 wake_up(&device->misc_wait);
1949}
1950
1951static void go_diskless(struct drbd_device *device)
1952{
1953 D_ASSERT(device, device->state.disk == D_FAILED);
1954 /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
1955 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
1956 * the protected members anymore, though, so once put_ldev reaches zero
1957 * again, it will be safe to free them. */
1958
1959 /* Try to write changed bitmap pages, read errors may have just
1960 * set some bits outside the area covered by the activity log.
1961 *
1962 * If we have an IO error during the bitmap writeout,
1963 * we will want a full sync next time, just in case.
1964 * (Do we want a specific meta data flag for this?)
1965 *
1966 * If that does not make it to stable storage either,
1967 * we cannot do anything about that anymore.
1968 *
1969 * We still need to check if both bitmap and ldev are present, we may
1970 * end up here after a failed attach, before ldev was even assigned.
1971 */
1972 if (device->bitmap && device->ldev) {
1973 /* An interrupted resync or similar is allowed to recounts bits
1974 * while we detach.
1975 * Any modifications would not be expected anymore, though.
1976 */
1977 if (drbd_bitmap_io_from_worker(device, drbd_bm_write,
1978 "detach", BM_LOCKED_TEST_ALLOWED)) {
1979 if (test_bit(WAS_READ_ERROR, &device->flags)) {
1980 drbd_md_set_flag(device, MDF_FULL_SYNC);
1981 drbd_md_sync(device);
1982 }
1983 }
1984 }
1985
1986 drbd_force_state(device, NS(disk, D_DISKLESS));
1987}
1988
Lars Ellenbergac0acb92014-02-11 09:47:58 +01001989static int do_md_sync(struct drbd_device *device)
1990{
1991 drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
1992 drbd_md_sync(device);
1993 return 0;
1994}
1995
Lars Ellenberg944410e2014-05-06 15:02:05 +02001996/* only called from drbd_worker thread, no locking */
1997void __update_timing_details(
1998 struct drbd_thread_timing_details *tdp,
1999 unsigned int *cb_nr,
2000 void *cb,
2001 const char *fn, const unsigned int line)
2002{
2003 unsigned int i = *cb_nr % DRBD_THREAD_DETAILS_HIST;
2004 struct drbd_thread_timing_details *td = tdp + i;
2005
2006 td->start_jif = jiffies;
2007 td->cb_addr = cb;
2008 td->caller_fn = fn;
2009 td->line = line;
2010 td->cb_nr = *cb_nr;
2011
2012 i = (i+1) % DRBD_THREAD_DETAILS_HIST;
2013 td = tdp + i;
2014 memset(td, 0, sizeof(*td));
2015
2016 ++(*cb_nr);
2017}
2018
Lars Ellenberge334f552014-02-11 09:30:49 +01002019static void do_device_work(struct drbd_device *device, const unsigned long todo)
2020{
Andreas Gruenbacherb47a06d2014-09-11 14:29:10 +02002021 if (test_bit(MD_SYNC, &todo))
Lars Ellenbergac0acb92014-02-11 09:47:58 +01002022 do_md_sync(device);
Andreas Gruenbacherb47a06d2014-09-11 14:29:10 +02002023 if (test_bit(RS_DONE, &todo) ||
2024 test_bit(RS_PROGRESS, &todo))
2025 update_on_disk_bitmap(device, test_bit(RS_DONE, &todo));
2026 if (test_bit(GO_DISKLESS, &todo))
Lars Ellenberge334f552014-02-11 09:30:49 +01002027 go_diskless(device);
Andreas Gruenbacherb47a06d2014-09-11 14:29:10 +02002028 if (test_bit(DESTROY_DISK, &todo))
Lars Ellenberge334f552014-02-11 09:30:49 +01002029 drbd_ldev_destroy(device);
Andreas Gruenbacherb47a06d2014-09-11 14:29:10 +02002030 if (test_bit(RS_START, &todo))
Lars Ellenbergac0acb92014-02-11 09:47:58 +01002031 do_start_resync(device);
Lars Ellenberge334f552014-02-11 09:30:49 +01002032}
2033
2034#define DRBD_DEVICE_WORK_MASK \
2035 ((1UL << GO_DISKLESS) \
2036 |(1UL << DESTROY_DISK) \
Lars Ellenbergac0acb92014-02-11 09:47:58 +01002037 |(1UL << MD_SYNC) \
2038 |(1UL << RS_START) \
Lars Ellenberge334f552014-02-11 09:30:49 +01002039 |(1UL << RS_PROGRESS) \
2040 |(1UL << RS_DONE) \
2041 )
2042
2043static unsigned long get_work_bits(unsigned long *flags)
2044{
2045 unsigned long old, new;
2046 do {
2047 old = *flags;
2048 new = old & ~DRBD_DEVICE_WORK_MASK;
2049 } while (cmpxchg(flags, old, new) != old);
2050 return old & DRBD_DEVICE_WORK_MASK;
2051}
2052
2053static void do_unqueued_work(struct drbd_connection *connection)
Lars Ellenbergc7a58db2013-12-20 11:39:48 +01002054{
2055 struct drbd_peer_device *peer_device;
2056 int vnr;
2057
2058 rcu_read_lock();
2059 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
2060 struct drbd_device *device = peer_device->device;
Lars Ellenberge334f552014-02-11 09:30:49 +01002061 unsigned long todo = get_work_bits(&device->flags);
2062 if (!todo)
Lars Ellenbergc7a58db2013-12-20 11:39:48 +01002063 continue;
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01002064
Lars Ellenbergc7a58db2013-12-20 11:39:48 +01002065 kref_get(&device->kref);
2066 rcu_read_unlock();
Lars Ellenberge334f552014-02-11 09:30:49 +01002067 do_device_work(device, todo);
Lars Ellenbergc7a58db2013-12-20 11:39:48 +01002068 kref_put(&device->kref, drbd_destroy_device);
2069 rcu_read_lock();
2070 }
2071 rcu_read_unlock();
2072}
2073
Rashika Kheriaa186e472013-12-19 15:06:10 +05302074static bool dequeue_work_batch(struct drbd_work_queue *queue, struct list_head *work_list)
Lars Ellenberg8c0785a2011-10-19 11:50:57 +02002075{
2076 spin_lock_irq(&queue->q_lock);
Lars Ellenberg15e26f6a2014-04-28 11:43:21 +02002077 list_splice_tail_init(&queue->q, work_list);
Lars Ellenberg8c0785a2011-10-19 11:50:57 +02002078 spin_unlock_irq(&queue->q_lock);
2079 return !list_empty(work_list);
2080}
2081
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02002082static void wait_for_work(struct drbd_connection *connection, struct list_head *work_list)
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002083{
2084 DEFINE_WAIT(wait);
2085 struct net_conf *nc;
2086 int uncork, cork;
2087
Lars Ellenbergabde9cc2014-09-11 14:29:11 +02002088 dequeue_work_batch(&connection->sender_work, work_list);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002089 if (!list_empty(work_list))
2090 return;
2091
2092 /* Still nothing to do?
2093 * Maybe we still need to close the current epoch,
2094 * even if no new requests are queued yet.
2095 *
2096 * Also, poke TCP, just in case.
2097 * Then wait for new work (or signal). */
2098 rcu_read_lock();
2099 nc = rcu_dereference(connection->net_conf);
2100 uncork = nc ? nc->tcp_cork : 0;
2101 rcu_read_unlock();
2102 if (uncork) {
2103 mutex_lock(&connection->data.mutex);
2104 if (connection->data.socket)
Christoph Hellwigdb105382020-05-28 07:12:18 +02002105 tcp_sock_set_cork(connection->data.socket->sk, false);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002106 mutex_unlock(&connection->data.mutex);
2107 }
2108
2109 for (;;) {
2110 int send_barrier;
2111 prepare_to_wait(&connection->sender_work.q_wait, &wait, TASK_INTERRUPTIBLE);
Andreas Gruenbacher05008132011-07-07 14:19:42 +02002112 spin_lock_irq(&connection->resource->req_lock);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002113 spin_lock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */
Lars Ellenbergbc317a92012-08-22 11:47:14 +02002114 if (!list_empty(&connection->sender_work.q))
Lars Ellenberg4dd726f2014-02-11 11:15:36 +01002115 list_splice_tail_init(&connection->sender_work.q, work_list);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002116 spin_unlock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */
2117 if (!list_empty(work_list) || signal_pending(current)) {
Andreas Gruenbacher05008132011-07-07 14:19:42 +02002118 spin_unlock_irq(&connection->resource->req_lock);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002119 break;
2120 }
Lars Ellenbergf9c78122014-04-28 18:43:29 +02002121
2122 /* We found nothing new to do, no to-be-communicated request,
2123 * no other work item. We may still need to close the last
2124 * epoch. Next incoming request epoch will be connection ->
2125 * current transfer log epoch number. If that is different
2126 * from the epoch of the last request we communicated, it is
2127 * safe to send the epoch separating barrier now.
2128 */
2129 send_barrier =
2130 atomic_read(&connection->current_tle_nr) !=
2131 connection->send.current_epoch_nr;
Andreas Gruenbacher05008132011-07-07 14:19:42 +02002132 spin_unlock_irq(&connection->resource->req_lock);
Lars Ellenbergf9c78122014-04-28 18:43:29 +02002133
2134 if (send_barrier)
2135 maybe_send_barrier(connection,
2136 connection->send.current_epoch_nr + 1);
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01002137
Lars Ellenberge334f552014-02-11 09:30:49 +01002138 if (test_bit(DEVICE_WORK_PENDING, &connection->flags))
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01002139 break;
2140
Lars Ellenberga80ca1a2013-12-27 17:17:25 +01002141 /* drbd_send() may have called flush_signals() */
2142 if (get_t_state(&connection->worker) != RUNNING)
2143 break;
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01002144
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002145 schedule();
2146 /* may be woken up for other things but new work, too,
2147 * e.g. if the current epoch got closed.
2148 * In which case we send the barrier above. */
2149 }
2150 finish_wait(&connection->sender_work.q_wait, &wait);
2151
2152 /* someone may have changed the config while we have been waiting above. */
2153 rcu_read_lock();
2154 nc = rcu_dereference(connection->net_conf);
2155 cork = nc ? nc->tcp_cork : 0;
2156 rcu_read_unlock();
2157 mutex_lock(&connection->data.mutex);
2158 if (connection->data.socket) {
2159 if (cork)
Christoph Hellwigdb105382020-05-28 07:12:18 +02002160 tcp_sock_set_cork(connection->data.socket->sk, true);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002161 else if (!uncork)
Christoph Hellwigdb105382020-05-28 07:12:18 +02002162 tcp_sock_set_cork(connection->data.socket->sk, false);
Lars Ellenbergb6dd1a82011-11-28 15:04:49 +01002163 }
2164 mutex_unlock(&connection->data.mutex);
2165}
2166
Philipp Reisnerb411b362009-09-25 16:07:19 -07002167int drbd_worker(struct drbd_thread *thi)
2168{
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02002169 struct drbd_connection *connection = thi->connection;
Andreas Gruenbacher6db7e502011-08-26 23:50:08 +02002170 struct drbd_work *w = NULL;
Andreas Gruenbacherc06ece62011-06-21 17:23:59 +02002171 struct drbd_peer_device *peer_device;
Philipp Reisnerb411b362009-09-25 16:07:19 -07002172 LIST_HEAD(work_list);
Lars Ellenberg8c0785a2011-10-19 11:50:57 +02002173 int vnr;
Philipp Reisnerb411b362009-09-25 16:07:19 -07002174
Andreas Gruenbachere77a0a52011-01-25 15:43:39 +01002175 while (get_t_state(thi) == RUNNING) {
Philipp Reisner80822282011-02-08 12:46:30 +01002176 drbd_thread_current_set_cpu(thi);
Philipp Reisnerb411b362009-09-25 16:07:19 -07002177
Lars Ellenberg944410e2014-05-06 15:02:05 +02002178 if (list_empty(&work_list)) {
2179 update_worker_timing_details(connection, wait_for_work);
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02002180 wait_for_work(connection, &work_list);
Lars Ellenberg944410e2014-05-06 15:02:05 +02002181 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07002182
Lars Ellenberg944410e2014-05-06 15:02:05 +02002183 if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) {
2184 update_worker_timing_details(connection, do_unqueued_work);
Lars Ellenberge334f552014-02-11 09:30:49 +01002185 do_unqueued_work(connection);
Lars Ellenberg944410e2014-05-06 15:02:05 +02002186 }
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +01002187
Lars Ellenberg8c0785a2011-10-19 11:50:57 +02002188 if (signal_pending(current)) {
Philipp Reisnerb411b362009-09-25 16:07:19 -07002189 flush_signals(current);
Philipp Reisner19393e12011-02-09 10:09:07 +01002190 if (get_t_state(thi) == RUNNING) {
Andreas Gruenbacher1ec861e2011-07-06 11:01:44 +02002191 drbd_warn(connection, "Worker got an unexpected signal\n");
Philipp Reisnerb411b362009-09-25 16:07:19 -07002192 continue;
Philipp Reisner19393e12011-02-09 10:09:07 +01002193 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07002194 break;
2195 }
2196
Andreas Gruenbachere77a0a52011-01-25 15:43:39 +01002197 if (get_t_state(thi) != RUNNING)
Philipp Reisnerb411b362009-09-25 16:07:19 -07002198 break;
Philipp Reisnerb411b362009-09-25 16:07:19 -07002199
Lars Ellenberg729e8b82014-09-11 14:29:12 +02002200 if (!list_empty(&work_list)) {
Andreas Gruenbacher6db7e502011-08-26 23:50:08 +02002201 w = list_first_entry(&work_list, struct drbd_work, list);
2202 list_del_init(&w->list);
Lars Ellenberg944410e2014-05-06 15:02:05 +02002203 update_worker_timing_details(connection, w->cb);
Andreas Gruenbacher6db7e502011-08-26 23:50:08 +02002204 if (w->cb(w, connection->cstate < C_WF_REPORT_PARAMS) == 0)
Lars Ellenberg8c0785a2011-10-19 11:50:57 +02002205 continue;
Andreas Gruenbacherbde89a92011-05-30 16:32:41 +02002206 if (connection->cstate >= C_WF_REPORT_PARAMS)
2207 conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD);
Philipp Reisnerb411b362009-09-25 16:07:19 -07002208 }
2209 }
Philipp Reisnerb411b362009-09-25 16:07:19 -07002210
Lars Ellenberg8c0785a2011-10-19 11:50:57 +02002211 do {
Lars Ellenberg944410e2014-05-06 15:02:05 +02002212 if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) {
2213 update_worker_timing_details(connection, do_unqueued_work);
Lars Ellenberge334f552014-02-11 09:30:49 +01002214 do_unqueued_work(connection);
Lars Ellenberg944410e2014-05-06 15:02:05 +02002215 }
Lars Ellenberg729e8b82014-09-11 14:29:12 +02002216 if (!list_empty(&work_list)) {
Andreas Gruenbacher6db7e502011-08-26 23:50:08 +02002217 w = list_first_entry(&work_list, struct drbd_work, list);
2218 list_del_init(&w->list);
Lars Ellenberg944410e2014-05-06 15:02:05 +02002219 update_worker_timing_details(connection, w->cb);
Andreas Gruenbacher6db7e502011-08-26 23:50:08 +02002220 w->cb(w, 1);
Lars Ellenberg729e8b82014-09-11 14:29:12 +02002221 } else
2222 dequeue_work_batch(&connection->sender_work, &work_list);
Lars Ellenberge334f552014-02-11 09:30:49 +01002223 } while (!list_empty(&work_list) || test_bit(DEVICE_WORK_PENDING, &connection->flags));
Philipp Reisnerb411b362009-09-25 16:07:19 -07002224
Philipp Reisnerc141ebd2011-05-05 16:13:10 +02002225 rcu_read_lock();
Andreas Gruenbacherc06ece62011-06-21 17:23:59 +02002226 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
2227 struct drbd_device *device = peer_device->device;
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02002228 D_ASSERT(device, device->state.disk == D_DISKLESS && device->state.conn == C_STANDALONE);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02002229 kref_get(&device->kref);
Philipp Reisnerc141ebd2011-05-05 16:13:10 +02002230 rcu_read_unlock();
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02002231 drbd_device_cleanup(device);
Andreas Gruenbacher05a10ec2011-06-07 22:54:17 +02002232 kref_put(&device->kref, drbd_destroy_device);
Philipp Reisnerc141ebd2011-05-05 16:13:10 +02002233 rcu_read_lock();
Philipp Reisner0e29d162011-02-18 14:23:11 +01002234 }
Philipp Reisnerc141ebd2011-05-05 16:13:10 +02002235 rcu_read_unlock();
Philipp Reisnerb411b362009-09-25 16:07:19 -07002236
2237 return 0;
2238}