fs/netfs/read_retry.c

// SPDX-License-Identifier: GPL-2.0-only
/* Network filesystem read subrequest retrying.
 *
 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/fs.h>
#include <linux/slab.h>
#include "internal.h"

static void netfs_reissue_read(struct netfs_io_request *rreq,
			       struct netfs_io_subrequest *subreq)
{
	struct iov_iter *io_iter = &subreq->io_iter;

	if (iov_iter_is_folioq(io_iter)) {
		subreq->curr_folioq = (struct folio_queue *)io_iter->folioq;
		subreq->curr_folioq_slot = io_iter->folioq_slot;
		subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];
	}

	atomic_inc(&rreq->nr_outstanding);
	__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
	netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
	subreq->rreq->netfs_ops->issue_read(subreq);
}

/*
 * Go through the list of failed/short reads, retrying all retryable ones.  We
 * need to switch failed cache reads to network downloads.
 */
static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
{
	struct netfs_io_subrequest *subreq;
	struct netfs_io_stream *stream0 = &rreq->io_streams[0];
	LIST_HEAD(sublist);
	LIST_HEAD(queue);

	_enter("R=%x", rreq->debug_id);

	if (list_empty(&rreq->subrequests))
		return;

	if (rreq->netfs_ops->retry_request)
		rreq->netfs_ops->retry_request(rreq, NULL);

	/* If there's no renegotiation to do, just resend each retryable subreq
	 * up to the first permanently failed one.
	 */
	if (!rreq->netfs_ops->prepare_read &&
	    !test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags)) {
		struct netfs_io_subrequest *subreq;

		list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
			if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
				break;
			if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
				netfs_reset_iter(subreq);
				netfs_reissue_read(rreq, subreq);
			}
		}
		return;
	}

	/* Okay, we need to renegotiate all the download requests and flip any
	 * failed cache reads over to being download requests and negotiate
	 * those also.  All fully successful subreqs have been removed from the
	 * list and any spare data from those has been donated.
	 *
	 * What we do is decant the list and rebuild it one subreq at a time so
	 * that we don't end up with donations jumping over a gap we're busy
	 * populating with smaller subrequests.  In the event that the subreq
	 * we just launched finishes before we insert the next subreq, it'll
	 * fill in rreq->prev_donated instead.

	 * Note: Alternatively, we could split the tail subrequest right before
	 * we reissue it and fix up the donations under lock.
	 */
	list_splice_init(&rreq->subrequests, &queue);

	do {
		struct netfs_io_subrequest *from;
		struct iov_iter source;
		unsigned long long start, len;
		size_t part, deferred_next_donated = 0;
		bool boundary = false;

		/* Go through the subreqs and find the next span of contiguous
		 * buffer that we then rejig (cifs, for example, needs the
		 * rsize renegotiating) and reissue.
		 */
		from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link);
		list_move_tail(&from->rreq_link, &sublist);
		start = from->start + from->transferred;
		len   = from->len   - from->transferred;

		_debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx",
		       rreq->debug_id, from->debug_index,
		       from->start, from->consumed, from->transferred, from->len);

		if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
		    !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
			goto abandon;

		deferred_next_donated = from->next_donated;
		while ((subreq = list_first_entry_or_null(
				&queue, struct netfs_io_subrequest, rreq_link))) {
			if (subreq->start != start + len ||
			    subreq->transferred > 0 ||
			    !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
				break;
			list_move_tail(&subreq->rreq_link, &sublist);
			len += subreq->len;
			deferred_next_donated = subreq->next_donated;
			if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags))
				break;
		}

		_debug(" - range: %llx-%llx %llx", start, start + len - 1, len);

		/* Determine the set of buffers we're going to use.  Each
		 * subreq gets a subset of a single overall contiguous buffer.
		 */
		netfs_reset_iter(from);
		source = from->io_iter;
		source.count = len;

		/* Work through the sublist. */
		while ((subreq = list_first_entry_or_null(
				&sublist, struct netfs_io_subrequest, rreq_link))) {
			list_del(&subreq->rreq_link);

			subreq->source	= NETFS_DOWNLOAD_FROM_SERVER;
			subreq->start	= start - subreq->transferred;
			subreq->len	= len   + subreq->transferred;
			stream0->sreq_max_len = subreq->len;

			__clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
			__set_bit(NETFS_SREQ_RETRYING, &subreq->flags);

			spin_lock_bh(&rreq->lock);
			list_add_tail(&subreq->rreq_link, &rreq->subrequests);
			subreq->prev_donated += rreq->prev_donated;
			rreq->prev_donated = 0;
			trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
			spin_unlock_bh(&rreq->lock);

			BUG_ON(!len);

			/* Renegotiate max_len (rsize) */
			if (rreq->netfs_ops->prepare_read(subreq) < 0) {
				trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
				__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
			}

			part = umin(len, stream0->sreq_max_len);
			if (unlikely(rreq->io_streams[0].sreq_max_segs))
				part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs);
			subreq->len = subreq->transferred + part;
			subreq->io_iter = source;
			iov_iter_truncate(&subreq->io_iter, part);
			iov_iter_advance(&source, part);
			len -= part;
			start += part;
			if (!len) {
				if (boundary)
					__set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
				subreq->next_donated = deferred_next_donated;
			} else {
				__clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
				subreq->next_donated = 0;
			}

			netfs_reissue_read(rreq, subreq);
			if (!len)
				break;

			/* If we ran out of subrequests, allocate another. */
			if (list_empty(&sublist)) {
				subreq = netfs_alloc_subrequest(rreq);
				if (!subreq)
					goto abandon;
				subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
				subreq->start = start;

				/* We get two refs, but need just one. */
				netfs_put_subrequest(subreq, false, netfs_sreq_trace_new);
				trace_netfs_sreq(subreq, netfs_sreq_trace_split);
				list_add_tail(&subreq->rreq_link, &sublist);
			}
		}

		/* If we managed to use fewer subreqs, we can discard the
		 * excess.
		 */
		while ((subreq = list_first_entry_or_null(
				&sublist, struct netfs_io_subrequest, rreq_link))) {
			trace_netfs_sreq(subreq, netfs_sreq_trace_discard);
			list_del(&subreq->rreq_link);
			netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done);
		}

	} while (!list_empty(&queue));

	return;

	/* If we hit ENOMEM, fail all remaining subrequests */
abandon:
	list_splice_init(&sublist, &queue);
	list_for_each_entry(subreq, &queue, rreq_link) {
		if (!subreq->error)
			subreq->error = -ENOMEM;
		__clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
		__clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
		__clear_bit(NETFS_SREQ_RETRYING, &subreq->flags);
	}
	spin_lock_bh(&rreq->lock);
	list_splice_tail_init(&queue, &rreq->subrequests);
	spin_unlock_bh(&rreq->lock);
}

/*
 * Retry reads.
 */
void netfs_retry_reads(struct netfs_io_request *rreq)
{
	trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);

	atomic_inc(&rreq->nr_outstanding);

	netfs_retry_read_subrequests(rreq);

	if (atomic_dec_and_test(&rreq->nr_outstanding))
		netfs_rreq_terminated(rreq, false);
}

/*
 * Unlock any the pages that haven't been unlocked yet due to abandoned
 * subrequests.
 */
void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
{
	struct folio_queue *p;

	for (p = rreq->buffer; p; p = p->next) {
		for (int slot = 0; slot < folioq_count(p); slot++) {
			struct folio *folio = folioq_folio(p, slot);

			if (folio && !folioq_is_marked2(p, slot)) {
				trace_netfs_folio(folio, netfs_folio_trace_abandon);
				folio_unlock(folio);
			}
		}
	}
}