fs/erofs/data.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2017-2018 HUAWEI, Inc.
  *             https://www.huawei.com/
  * Created by Gao Xiang <gaoxiang25@huawei.com>
  */
 #include "internal.h"
 #include <linux/prefetch.h>

 #include <trace/events/erofs.h>

 static void erofs_readendio(struct bio *bio)
 {
 	struct bio_vec *bvec;
 	blk_status_t err = bio->bi_status;
 	struct bvec_iter_all iter_all;

 	bio_for_each_segment_all(bvec, bio, iter_all) {
 		struct page *page = bvec->bv_page;

 		/* page is already locked */
 		DBG_BUGON(PageUptodate(page));

 		if (err)
 			SetPageError(page);
 		else
 			SetPageUptodate(page);

 		unlock_page(page);
 		/* page could be reclaimed now */
 	}
 	bio_put(bio);
 }

 struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr)
 {
 	struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping;
 	struct page *page;

 	page = read_cache_page_gfp(mapping, blkaddr,
 				   mapping_gfp_constraint(mapping, ~__GFP_FS));
 	/* should already be PageUptodate */
 	if (!IS_ERR(page))
 		lock_page(page);
 	return page;
 }

 static int erofs_map_blocks_flatmode(struct inode *inode,
 				     struct erofs_map_blocks *map,
 				     int flags)
 {
 	int err = 0;
 	erofs_blk_t nblocks, lastblk;
 	u64 offset = map->m_la;
 	struct erofs_inode *vi = EROFS_I(inode);
 	bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);

 	trace_erofs_map_blocks_flatmode_enter(inode, map, flags);

 	nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
 	lastblk = nblocks - tailendpacking;

 	if (offset >= inode->i_size) {
 		/* leave out-of-bound access unmapped */
 		map->m_flags = 0;
 		map->m_plen = 0;
 		goto out;
 	}

 	/* there is no hole in flatmode */
 	map->m_flags = EROFS_MAP_MAPPED;

 	if (offset < blknr_to_addr(lastblk)) {
 		map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
 		map->m_plen = blknr_to_addr(lastblk) - offset;
 	} else if (tailendpacking) {
 		/* 2 - inode inline B: inode, [xattrs], inline last blk... */
 		struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);

 		map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize +
 			vi->xattr_isize + erofs_blkoff(map->m_la);
 		map->m_plen = inode->i_size - offset;

 		/* inline data should be located in one meta block */
 		if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) {
 			erofs_err(inode->i_sb,
 				  "inline data cross block boundary @ nid %llu",
 				  vi->nid);
 			DBG_BUGON(1);
 			err = -EFSCORRUPTED;
 			goto err_out;
 		}

 		map->m_flags |= EROFS_MAP_META;
 	} else {
 		erofs_err(inode->i_sb,
 			  "internal error @ nid: %llu (size %llu), m_la 0x%llx",
 			  vi->nid, inode->i_size, map->m_la);
 		DBG_BUGON(1);
 		err = -EIO;
 		goto err_out;
 	}

 out:
 	map->m_llen = map->m_plen;

 err_out:
 	trace_erofs_map_blocks_flatmode_exit(inode, map, flags, 0);
 	return err;
 }

 int erofs_map_blocks(struct inode *inode,
 		     struct erofs_map_blocks *map, int flags)
 {
 	if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) {
 		int err = z_erofs_map_blocks_iter(inode, map, flags);

 		if (map->mpage) {
 			put_page(map->mpage);
 			map->mpage = NULL;
 		}
 		return err;
 	}
 	return erofs_map_blocks_flatmode(inode, map, flags);
 }

 static inline struct bio *erofs_read_raw_page(struct bio *bio,
 					      struct address_space *mapping,
 					      struct page *page,
 					      erofs_off_t *last_block,
 					      unsigned int nblocks,
 					      bool ra)
 {
 	struct inode *const inode = mapping->host;
 	struct super_block *const sb = inode->i_sb;
 	erofs_off_t current_block = (erofs_off_t)page->index;
 	int err;

 	DBG_BUGON(!nblocks);

 	if (PageUptodate(page)) {
 		err = 0;
 		goto has_updated;
 	}

 	/* note that for readpage case, bio also equals to NULL */
 	if (bio &&
 	    /* not continuous */
 	    *last_block + 1 != current_block) {
 submit_bio_retry:
 		submit_bio(bio);
 		bio = NULL;
 	}

 	if (!bio) {
 		struct erofs_map_blocks map = {
 			.m_la = blknr_to_addr(current_block),
 		};
 		erofs_blk_t blknr;
 		unsigned int blkoff;

 		err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
 		if (err)
 			goto err_out;

 		/* zero out the holed page */
 		if (!(map.m_flags & EROFS_MAP_MAPPED)) {
 			zero_user_segment(page, 0, PAGE_SIZE);
 			SetPageUptodate(page);

 			/* imply err = 0, see erofs_map_blocks */
 			goto has_updated;
 		}

 		/* for RAW access mode, m_plen must be equal to m_llen */
 		DBG_BUGON(map.m_plen != map.m_llen);

 		blknr = erofs_blknr(map.m_pa);
 		blkoff = erofs_blkoff(map.m_pa);

 		/* deal with inline page */
 		if (map.m_flags & EROFS_MAP_META) {
 			void *vsrc, *vto;
 			struct page *ipage;

 			DBG_BUGON(map.m_plen > PAGE_SIZE);

 			ipage = erofs_get_meta_page(inode->i_sb, blknr);

 			if (IS_ERR(ipage)) {
 				err = PTR_ERR(ipage);
 				goto err_out;
 			}

 			vsrc = kmap_atomic(ipage);
 			vto = kmap_atomic(page);
 			memcpy(vto, vsrc + blkoff, map.m_plen);
 			memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen);
 			kunmap_atomic(vto);
 			kunmap_atomic(vsrc);
 			flush_dcache_page(page);

 			SetPageUptodate(page);
 			/* TODO: could we unlock the page earlier? */
 			unlock_page(ipage);
 			put_page(ipage);

 			/* imply err = 0, see erofs_map_blocks */
 			goto has_updated;
 		}

 		/* pa must be block-aligned for raw reading */
 		DBG_BUGON(erofs_blkoff(map.m_pa));

 		/* max # of continuous pages */
 		if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE))
 			nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE);

 		bio = bio_alloc(GFP_NOIO, bio_max_segs(nblocks));

 		bio->bi_end_io = erofs_readendio;
 		bio_set_dev(bio, sb->s_bdev);
 		bio->bi_iter.bi_sector = (sector_t)blknr <<
 			LOG_SECTORS_PER_BLOCK;
 		bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0);
 	}

 	err = bio_add_page(bio, page, PAGE_SIZE, 0);
 	/* out of the extent or bio is full */
 	if (err < PAGE_SIZE)
 		goto submit_bio_retry;

 	*last_block = current_block;

 	/* shift in advance in case of it followed by too many gaps */
 	if (bio->bi_iter.bi_size >= bio->bi_max_vecs * PAGE_SIZE) {
 		/* err should reassign to 0 after submitting */
 		err = 0;
 		goto submit_bio_out;
 	}

 	return bio;

 err_out:
 	/* for sync reading, set page error immediately */
 	if (!ra) {
 		SetPageError(page);
 		ClearPageUptodate(page);
 	}
 has_updated:
 	unlock_page(page);

 	/* if updated manually, continuous pages has a gap */
 	if (bio)
 submit_bio_out:
 		submit_bio(bio);
 	return err ? ERR_PTR(err) : NULL;
 }

 /*
  * since we dont have write or truncate flows, so no inode
  * locking needs to be held at the moment.
  */
 static int erofs_raw_access_readpage(struct file *file, struct page *page)
 {
 	erofs_off_t last_block;
 	struct bio *bio;

 	trace_erofs_readpage(page, true);

 	bio = erofs_read_raw_page(NULL, page->mapping,
 				  page, &last_block, 1, false);

 	if (IS_ERR(bio))
 		return PTR_ERR(bio);

 	DBG_BUGON(bio);	/* since we have only one bio -- must be NULL */
 	return 0;
 }

 static void erofs_raw_access_readahead(struct readahead_control *rac)
 {
 	erofs_off_t last_block;
 	struct bio *bio = NULL;
 	struct page *page;

 	trace_erofs_readpages(rac->mapping->host, readahead_index(rac),
 			readahead_count(rac), true);

 	while ((page = readahead_page(rac))) {
 		prefetchw(&page->flags);

 		bio = erofs_read_raw_page(bio, rac->mapping, page, &last_block,
 				readahead_count(rac), true);

 		/* all the page errors are ignored when readahead */
 		if (IS_ERR(bio)) {
 			pr_err("%s, readahead error at page %lu of nid %llu\n",
 			       __func__, page->index,
 			       EROFS_I(rac->mapping->host)->nid);

 			bio = NULL;
 		}

 		put_page(page);
 	}

 	/* the rare case (end in gaps) */
 	if (bio)
 		submit_bio(bio);
 }

 static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
 {
 	struct inode *inode = mapping->host;
 	struct erofs_map_blocks map = {
 		.m_la = blknr_to_addr(block),
 	};

 	if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) {
 		erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE;

 		if (block >> LOG_SECTORS_PER_BLOCK >= blks)
 			return 0;
 	}

 	if (!erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW))
 		return erofs_blknr(map.m_pa);

 	return 0;
 }

 /* for uncompressed (aligned) files and raw access for other files */
 const struct address_space_operations erofs_raw_access_aops = {
 	.readpage = erofs_raw_access_readpage,
 	.readahead = erofs_raw_access_readahead,
 	.bmap = erofs_bmap,
 };
	// SPDX-License-Identifier: GPL-2.0-only
	/*
	* Copyright (C) 2017-2018 HUAWEI, Inc.
	* https://www.huawei.com/
	* Created by Gao Xiang <gaoxiang25@huawei.com>
	*/
	#include "internal.h"
	#include <linux/prefetch.h>

	#include <trace/events/erofs.h>

	static void erofs_readendio(struct bio *bio)
	{
	struct bio_vec *bvec;
	blk_status_t err = bio->bi_status;
	struct bvec_iter_all iter_all;

	bio_for_each_segment_all(bvec, bio, iter_all) {
	struct page *page = bvec->bv_page;

	/* page is already locked */
	DBG_BUGON(PageUptodate(page));

	if (err)
	SetPageError(page);
	else
	SetPageUptodate(page);

	unlock_page(page);
	/* page could be reclaimed now */
	}
	bio_put(bio);
	}

	struct page erofs_get_meta_page(struct super_block sb, erofs_blk_t blkaddr)
	{
	struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping;
	struct page *page;

	page = read_cache_page_gfp(mapping, blkaddr,
	mapping_gfp_constraint(mapping, ~__GFP_FS));
	/* should already be PageUptodate */
	if (!IS_ERR(page))
	lock_page(page);
	return page;
	}

	static int erofs_map_blocks_flatmode(struct inode *inode,
	struct erofs_map_blocks *map,
	int flags)
	{
	int err = 0;
	erofs_blk_t nblocks, lastblk;
	u64 offset = map->m_la;
	struct erofs_inode *vi = EROFS_I(inode);
	bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);

	trace_erofs_map_blocks_flatmode_enter(inode, map, flags);

	nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
	lastblk = nblocks - tailendpacking;

	if (offset >= inode->i_size) {
	/* leave out-of-bound access unmapped */
	map->m_flags = 0;
	map->m_plen = 0;
	goto out;
	}

	/* there is no hole in flatmode */
	map->m_flags = EROFS_MAP_MAPPED;

	if (offset < blknr_to_addr(lastblk)) {
	map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
	map->m_plen = blknr_to_addr(lastblk) - offset;
	} else if (tailendpacking) {
	/* 2 - inode inline B: inode, [xattrs], inline last blk... */
	struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);

	map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize +
	vi->xattr_isize + erofs_blkoff(map->m_la);
	map->m_plen = inode->i_size - offset;

	/* inline data should be located in one meta block */
	if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) {
	erofs_err(inode->i_sb,
	"inline data cross block boundary @ nid %llu",
	vi->nid);
	DBG_BUGON(1);
	err = -EFSCORRUPTED;
	goto err_out;
	}

	map->m_flags \|= EROFS_MAP_META;
	} else {
	erofs_err(inode->i_sb,
	"internal error @ nid: %llu (size %llu), m_la 0x%llx",
	vi->nid, inode->i_size, map->m_la);
	DBG_BUGON(1);
	err = -EIO;
	goto err_out;
	}

	out:
	map->m_llen = map->m_plen;

	err_out:
	trace_erofs_map_blocks_flatmode_exit(inode, map, flags, 0);
	return err;
	}

	int erofs_map_blocks(struct inode *inode,
	struct erofs_map_blocks *map, int flags)
	{
	if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) {
	int err = z_erofs_map_blocks_iter(inode, map, flags);

	if (map->mpage) {
	put_page(map->mpage);
	map->mpage = NULL;
	}
	return err;
	}
	return erofs_map_blocks_flatmode(inode, map, flags);
	}

	static inline struct bio erofs_read_raw_page(struct bio bio,
	struct address_space *mapping,
	struct page *page,
	erofs_off_t *last_block,
	unsigned int nblocks,
	bool ra)
	{
	struct inode *const inode = mapping->host;
	struct super_block *const sb = inode->i_sb;
	erofs_off_t current_block = (erofs_off_t)page->index;
	int err;

	DBG_BUGON(!nblocks);

	if (PageUptodate(page)) {
	err = 0;
	goto has_updated;
	}

	/* note that for readpage case, bio also equals to NULL */
	if (bio &&
	/* not continuous */
	*last_block + 1 != current_block) {
	submit_bio_retry:
	submit_bio(bio);
	bio = NULL;
	}

	if (!bio) {
	struct erofs_map_blocks map = {
	.m_la = blknr_to_addr(current_block),
	};
	erofs_blk_t blknr;
	unsigned int blkoff;

	err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
	if (err)
	goto err_out;

	/* zero out the holed page */
	if (!(map.m_flags & EROFS_MAP_MAPPED)) {
	zero_user_segment(page, 0, PAGE_SIZE);
	SetPageUptodate(page);

	/* imply err = 0, see erofs_map_blocks */
	goto has_updated;
	}

	/* for RAW access mode, m_plen must be equal to m_llen */
	DBG_BUGON(map.m_plen != map.m_llen);

	blknr = erofs_blknr(map.m_pa);
	blkoff = erofs_blkoff(map.m_pa);

	/* deal with inline page */
	if (map.m_flags & EROFS_MAP_META) {
	void vsrc, vto;
	struct page *ipage;

	DBG_BUGON(map.m_plen > PAGE_SIZE);

	ipage = erofs_get_meta_page(inode->i_sb, blknr);

	if (IS_ERR(ipage)) {
	err = PTR_ERR(ipage);
	goto err_out;
	}

	vsrc = kmap_atomic(ipage);
	vto = kmap_atomic(page);
	memcpy(vto, vsrc + blkoff, map.m_plen);
	memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen);
	kunmap_atomic(vto);
	kunmap_atomic(vsrc);
	flush_dcache_page(page);

	SetPageUptodate(page);
	/* TODO: could we unlock the page earlier? */
	unlock_page(ipage);
	put_page(ipage);

	/* imply err = 0, see erofs_map_blocks */
	goto has_updated;
	}

	/* pa must be block-aligned for raw reading */
	DBG_BUGON(erofs_blkoff(map.m_pa));

	/* max # of continuous pages */
	if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE))
	nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE);

	bio = bio_alloc(GFP_NOIO, bio_max_segs(nblocks));

	bio->bi_end_io = erofs_readendio;
	bio_set_dev(bio, sb->s_bdev);
	bio->bi_iter.bi_sector = (sector_t)blknr <<
	LOG_SECTORS_PER_BLOCK;
	bio->bi_opf = REQ_OP_READ \| (ra ? REQ_RAHEAD : 0);
	}

	err = bio_add_page(bio, page, PAGE_SIZE, 0);
	/* out of the extent or bio is full */
	if (err < PAGE_SIZE)
	goto submit_bio_retry;

	*last_block = current_block;

	/* shift in advance in case of it followed by too many gaps */
	if (bio->bi_iter.bi_size >= bio->bi_max_vecs * PAGE_SIZE) {
	/* err should reassign to 0 after submitting */
	err = 0;
	goto submit_bio_out;
	}

	return bio;

	err_out:
	/* for sync reading, set page error immediately */
	if (!ra) {
	SetPageError(page);
	ClearPageUptodate(page);
	}
	has_updated:
	unlock_page(page);

	/* if updated manually, continuous pages has a gap */
	if (bio)
	submit_bio_out:
	submit_bio(bio);
	return err ? ERR_PTR(err) : NULL;
	}

	/*
	* since we dont have write or truncate flows, so no inode
	* locking needs to be held at the moment.
	*/
	static int erofs_raw_access_readpage(struct file file, struct page page)
	{
	erofs_off_t last_block;
	struct bio *bio;

	trace_erofs_readpage(page, true);

	bio = erofs_read_raw_page(NULL, page->mapping,
	page, &last_block, 1, false);

	if (IS_ERR(bio))
	return PTR_ERR(bio);

	DBG_BUGON(bio); /* since we have only one bio -- must be NULL */
	return 0;
	}

	static void erofs_raw_access_readahead(struct readahead_control *rac)
	{
	erofs_off_t last_block;
	struct bio *bio = NULL;
	struct page *page;

	trace_erofs_readpages(rac->mapping->host, readahead_index(rac),
	readahead_count(rac), true);

	while ((page = readahead_page(rac))) {
	prefetchw(&page->flags);

	bio = erofs_read_raw_page(bio, rac->mapping, page, &last_block,
	readahead_count(rac), true);

	/* all the page errors are ignored when readahead */
	if (IS_ERR(bio)) {
	pr_err("%s, readahead error at page %lu of nid %llu\n",
	__func__, page->index,
	EROFS_I(rac->mapping->host)->nid);

	bio = NULL;
	}

	put_page(page);
	}

	/* the rare case (end in gaps) */
	if (bio)
	submit_bio(bio);
	}

	static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
	{
	struct inode *inode = mapping->host;
	struct erofs_map_blocks map = {
	.m_la = blknr_to_addr(block),
	};

	if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) {
	erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE;

	if (block >> LOG_SECTORS_PER_BLOCK >= blks)
	return 0;
	}

	if (!erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW))
	return erofs_blknr(map.m_pa);

	return 0;
	}

	/* for uncompressed (aligned) files and raw access for other files */
	const struct address_space_operations erofs_raw_access_aops = {
	.readpage = erofs_raw_access_readpage,
	.readahead = erofs_raw_access_readahead,
	.bmap = erofs_bmap,
	};