blob: 03c2253005f000722151a33c7324e8a83678b36b [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Dave Kleikampac27a0e2006-10-11 01:20:50 -07002/*
Mingming Cao617ba132006-10-11 01:20:53 -07003 * linux/fs/ext4/inode.c
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004 *
5 * Copyright (C) 1992, 1993, 1994, 1995
6 * Remy Card (card@masi.ibp.fr)
7 * Laboratoire MASI - Institut Blaise Pascal
8 * Universite Pierre et Marie Curie (Paris VI)
9 *
10 * from
11 *
12 * linux/fs/minix/inode.c
13 *
14 * Copyright (C) 1991, 1992 Linus Torvalds
15 *
Dave Kleikampac27a0e2006-10-11 01:20:50 -070016 * 64-bit file support on 64-bit platforms by Jakub Jelinek
17 * (jj@sunsite.ms.mff.cuni.cz)
18 *
Mingming Cao617ba132006-10-11 01:20:53 -070019 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
Dave Kleikampac27a0e2006-10-11 01:20:50 -070020 */
21
Dave Kleikampac27a0e2006-10-11 01:20:50 -070022#include <linux/fs.h>
23#include <linux/time.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070024#include <linux/highuid.h>
25#include <linux/pagemap.h>
Matthew Wilcoxc94c2ac2015-09-08 14:58:40 -070026#include <linux/dax.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070027#include <linux/quotaops.h>
28#include <linux/string.h>
29#include <linux/buffer_head.h>
30#include <linux/writeback.h>
Alex Tomas64769242008-07-11 19:27:31 -040031#include <linux/pagevec.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070032#include <linux/mpage.h>
Duane Griffine83c1392008-12-19 20:47:15 +000033#include <linux/namei.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070034#include <linux/uio.h>
35#include <linux/bio.h>
Mingming Cao4c0425f2009-09-28 15:48:41 -040036#include <linux/workqueue.h>
Jiaying Zhang744692d2010-03-04 16:14:02 -050037#include <linux/kernel.h>
Andrew Morton6db26ff2011-01-12 16:59:13 -080038#include <linux/printk.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090039#include <linux/slab.h>
Theodore Ts'o00a1a052014-03-30 10:20:01 -040040#include <linux/bitops.h>
Jan Kara364443c2016-11-20 17:36:06 -050041#include <linux/iomap.h>
Jeff Laytonae5e1652018-01-29 06:41:30 -050042#include <linux/iversion.h>
Theodore Ts'o9bffad12009-06-17 11:48:11 -040043
Christoph Hellwig3dcf5452008-04-29 18:13:32 -040044#include "ext4_jbd2.h"
Dave Kleikampac27a0e2006-10-11 01:20:50 -070045#include "xattr.h"
46#include "acl.h"
Theodore Ts'o9f125d62011-06-27 19:16:04 -040047#include "truncate.h"
Dave Kleikampac27a0e2006-10-11 01:20:50 -070048
Theodore Ts'o9bffad12009-06-17 11:48:11 -040049#include <trace/events/ext4.h>
50
Darrick J. Wong814525f2012-04-29 18:31:10 -040051static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
52 struct ext4_inode_info *ei)
53{
54 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Darrick J. Wong814525f2012-04-29 18:31:10 -040055 __u32 csum;
Daeho Jeongb47820e2016-07-03 17:51:39 -040056 __u16 dummy_csum = 0;
57 int offset = offsetof(struct ext4_inode, i_checksum_lo);
58 unsigned int csum_size = sizeof(dummy_csum);
Darrick J. Wong814525f2012-04-29 18:31:10 -040059
Daeho Jeongb47820e2016-07-03 17:51:39 -040060 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
61 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
62 offset += csum_size;
63 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
64 EXT4_GOOD_OLD_INODE_SIZE - offset);
65
66 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
67 offset = offsetof(struct ext4_inode, i_checksum_hi);
68 csum = ext4_chksum(sbi, csum, (__u8 *)raw +
69 EXT4_GOOD_OLD_INODE_SIZE,
70 offset - EXT4_GOOD_OLD_INODE_SIZE);
71 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
72 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
73 csum_size);
74 offset += csum_size;
Daeho Jeongb47820e2016-07-03 17:51:39 -040075 }
Daeho Jeong05ac5aa2016-12-01 11:49:12 -050076 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
77 EXT4_INODE_SIZE(inode->i_sb) - offset);
Darrick J. Wong814525f2012-04-29 18:31:10 -040078 }
79
Darrick J. Wong814525f2012-04-29 18:31:10 -040080 return csum;
81}
82
83static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
84 struct ext4_inode_info *ei)
85{
86 __u32 provided, calculated;
87
88 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
89 cpu_to_le32(EXT4_OS_LINUX) ||
Dmitry Monakhov9aa5d32b2014-10-13 03:36:16 -040090 !ext4_has_metadata_csum(inode->i_sb))
Darrick J. Wong814525f2012-04-29 18:31:10 -040091 return 1;
92
93 provided = le16_to_cpu(raw->i_checksum_lo);
94 calculated = ext4_inode_csum(inode, raw, ei);
95 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
96 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
97 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
98 else
99 calculated &= 0xFFFF;
100
101 return provided == calculated;
102}
103
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700104void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
105 struct ext4_inode_info *ei)
Darrick J. Wong814525f2012-04-29 18:31:10 -0400106{
107 __u32 csum;
108
109 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
110 cpu_to_le32(EXT4_OS_LINUX) ||
Dmitry Monakhov9aa5d32b2014-10-13 03:36:16 -0400111 !ext4_has_metadata_csum(inode->i_sb))
Darrick J. Wong814525f2012-04-29 18:31:10 -0400112 return;
113
114 csum = ext4_inode_csum(inode, raw, ei);
115 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
116 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
117 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
118 raw->i_checksum_hi = cpu_to_le16(csum >> 16);
119}
120
Jan Kara678aaf42008-07-11 19:27:31 -0400121static inline int ext4_begin_ordered_truncate(struct inode *inode,
122 loff_t new_size)
123{
Theodore Ts'o7ff9c072010-11-08 13:51:33 -0500124 trace_ext4_begin_ordered_truncate(inode, new_size);
Theodore Ts'o8aefcd52011-01-10 12:29:43 -0500125 /*
126 * If jinode is zero, then we never opened the file for
127 * writing, so there's no need to call
128 * jbd2_journal_begin_ordered_truncate() since there's no
129 * outstanding writes we need to flush.
130 */
131 if (!EXT4_I(inode)->jinode)
132 return 0;
133 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
134 EXT4_I(inode)->jinode,
135 new_size);
Jan Kara678aaf42008-07-11 19:27:31 -0400136}
137
Lukas Czernerd47992f2013-05-21 23:17:23 -0400138static void ext4_invalidatepage(struct page *page, unsigned int offset,
139 unsigned int length);
Theodore Ts'ocb20d512010-10-27 21:30:09 -0400140static int __ext4_journalled_writepage(struct page *page, unsigned int len);
141static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh);
Tahsin Erdogandec214d2017-06-22 11:44:55 -0400142static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
143 int pextents);
Alex Tomas64769242008-07-11 19:27:31 -0400144
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700145/*
146 * Test whether an inode is a fast symlink.
Tahsin Erdogan407cd7f2017-07-04 00:11:21 -0400147 * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700148 */
Theodore Ts'of348c252015-04-16 01:55:00 -0400149int ext4_inode_is_fast_symlink(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700150{
Andi Kleenfc822282017-12-03 20:38:01 -0500151 if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
152 int ea_blocks = EXT4_I(inode)->i_file_acl ?
153 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
154
155 if (ext4_has_inline_data(inode))
156 return 0;
157
158 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
159 }
Tahsin Erdogan407cd7f2017-07-04 00:11:21 -0400160 return S_ISLNK(inode->i_mode) && inode->i_size &&
161 (inode->i_size < EXT4_N_BLOCKS * 4);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700162}
163
164/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700165 * Called at the last iput() if i_nlink is zero.
166 */
Al Viro0930fcc2010-06-07 13:16:22 -0400167void ext4_evict_inode(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700168{
169 handle_t *handle;
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400170 int err;
Jan Kara65db8692019-11-05 17:44:12 +0100171 /*
172 * Credits for final inode cleanup and freeing:
173 * sb + inode (ext4_orphan_del()), block bitmap, group descriptor
174 * (xattr block freeing), bitmap, group descriptor (inode freeing)
175 */
176 int extra_credits = 6;
Tahsin Erdogan0421a182017-06-22 10:26:31 -0400177 struct ext4_xattr_inode_array *ea_inode_array = NULL;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700178
Theodore Ts'o7ff9c072010-11-08 13:51:33 -0500179 trace_ext4_evict_inode(inode);
Jiaying Zhang2581fdc2011-08-13 12:17:13 -0400180
Al Viro0930fcc2010-06-07 13:16:22 -0400181 if (inode->i_nlink) {
Jan Kara2d859db2011-07-26 09:07:11 -0400182 /*
183 * When journalling data dirty buffers are tracked only in the
184 * journal. So although mm thinks everything is clean and
185 * ready for reaping the inode might still have some pages to
186 * write in the running transaction or waiting to be
187 * checkpointed. Thus calling jbd2_journal_invalidatepage()
188 * (via truncate_inode_pages()) to discard these buffers can
189 * cause data loss. Also even if we did not discard these
190 * buffers, we would have no way to find them after the inode
191 * is reaped and thus user could see stale data if he tries to
192 * read them before the transaction is checkpointed. So be
193 * careful and force everything to disk here... We use
194 * ei->i_datasync_tid to store the newest transaction
195 * containing inode's data.
196 *
197 * Note that directories do not have this problem because they
198 * don't use page cache.
199 */
Vegard Nossum6a7fd522016-07-04 11:03:00 -0400200 if (inode->i_ino != EXT4_JOURNAL_INO &&
201 ext4_should_journal_data(inode) &&
Jan Kara3abb1a02017-06-22 23:49:46 -0400202 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode)) &&
203 inode->i_data.nrpages) {
Jan Kara2d859db2011-07-26 09:07:11 -0400204 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
205 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
206
Theodore Ts'od76a3a772013-04-03 22:02:52 -0400207 jbd2_complete_transaction(journal, commit_tid);
Jan Kara2d859db2011-07-26 09:07:11 -0400208 filemap_write_and_wait(&inode->i_data);
209 }
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700210 truncate_inode_pages_final(&inode->i_data);
Jan Kara5dc23bd2013-06-04 14:46:12 -0400211
Al Viro0930fcc2010-06-07 13:16:22 -0400212 goto no_delete;
213 }
214
Theodore Ts'oe2bfb082014-10-05 22:47:07 -0400215 if (is_bad_inode(inode))
216 goto no_delete;
217 dquot_initialize(inode);
Christoph Hellwig907f4552010-03-03 09:05:06 -0500218
Jan Kara678aaf42008-07-11 19:27:31 -0400219 if (ext4_should_order_data(inode))
220 ext4_begin_ordered_truncate(inode, 0);
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700221 truncate_inode_pages_final(&inode->i_data);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700222
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200223 /*
Jan Karaceff86f2020-04-21 10:54:45 +0200224 * For inodes with journalled data, transaction commit could have
225 * dirtied the inode. Flush worker is ignoring it because of I_FREEING
226 * flag but we still need to remove the inode from the writeback lists.
227 */
228 if (!list_empty_careful(&inode->i_io_list)) {
229 WARN_ON_ONCE(!ext4_should_journal_data(inode));
230 inode_io_list_del(inode);
231 }
232
233 /*
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200234 * Protect us against freezing - iput() caller didn't have to have any
235 * protection against it
236 */
237 sb_start_intwrite(inode->i_sb);
Andreas Dilgere50e5122017-06-21 21:10:32 -0400238
Tahsin Erdogan30a7eb92017-06-22 11:42:09 -0400239 if (!IS_NOQUOTA(inode))
240 extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
241
Jan Kara65db8692019-11-05 17:44:12 +0100242 /*
243 * Block bitmap, group descriptor, and inode are accounted in both
244 * ext4_blocks_for_truncate() and extra_credits. So subtract 3.
245 */
Tahsin Erdogan30a7eb92017-06-22 11:42:09 -0400246 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
Jan Kara65db8692019-11-05 17:44:12 +0100247 ext4_blocks_for_truncate(inode) + extra_credits - 3);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700248 if (IS_ERR(handle)) {
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400249 ext4_std_error(inode->i_sb, PTR_ERR(handle));
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700250 /*
251 * If we're going to skip the normal cleanup, we still need to
252 * make sure that the in-core orphan linked list is properly
253 * cleaned up.
254 */
Mingming Cao617ba132006-10-11 01:20:53 -0700255 ext4_orphan_del(NULL, inode);
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200256 sb_end_intwrite(inode->i_sb);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700257 goto no_delete;
258 }
Tahsin Erdogan30a7eb92017-06-22 11:42:09 -0400259
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700260 if (IS_SYNC(inode))
Frank Mayhar03901312009-01-07 00:06:22 -0500261 ext4_handle_sync(handle);
Tahsin Erdogan407cd7f2017-07-04 00:11:21 -0400262
263 /*
264 * Set inode->i_size to 0 before calling ext4_truncate(). We need
265 * special handling of symlinks here because i_size is used to
266 * determine whether ext4_inode_info->i_data contains symlink data or
267 * block mappings. Setting i_size to 0 will remove its fast symlink
268 * status. Erase i_data so that it becomes a valid empty block map.
269 */
270 if (ext4_inode_is_fast_symlink(inode))
271 memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700272 inode->i_size = 0;
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400273 err = ext4_mark_inode_dirty(handle, inode);
274 if (err) {
Eric Sandeen12062dd2010-02-15 14:19:27 -0500275 ext4_warning(inode->i_sb,
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400276 "couldn't mark inode dirty (err %d)", err);
277 goto stop_handle;
278 }
Theodore Ts'o2c98eb52016-11-13 22:02:26 -0500279 if (inode->i_blocks) {
280 err = ext4_truncate(inode);
281 if (err) {
Theodore Ts'o54d3adb2020-03-28 19:33:43 -0400282 ext4_error_err(inode->i_sb, -err,
283 "couldn't truncate inode %lu (err %d)",
284 inode->i_ino, err);
Theodore Ts'o2c98eb52016-11-13 22:02:26 -0500285 goto stop_handle;
286 }
287 }
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400288
Tahsin Erdogan30a7eb92017-06-22 11:42:09 -0400289 /* Remove xattr references. */
290 err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
291 extra_credits);
292 if (err) {
293 ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
294stop_handle:
295 ext4_journal_stop(handle);
296 ext4_orphan_del(NULL, inode);
297 sb_end_intwrite(inode->i_sb);
298 ext4_xattr_inode_array_free(ea_inode_array);
299 goto no_delete;
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400300 }
301
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700302 /*
Mingming Cao617ba132006-10-11 01:20:53 -0700303 * Kill off the orphan record which ext4_truncate created.
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700304 * AKPM: I think this can be inside the above `if'.
Mingming Cao617ba132006-10-11 01:20:53 -0700305 * Note that ext4_orphan_del() has to be able to cope with the
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700306 * deletion of a non-existent orphan - this is because we don't
Mingming Cao617ba132006-10-11 01:20:53 -0700307 * know if ext4_truncate() actually created an orphan record.
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700308 * (Well, we could do this if we need to, but heck - it works)
309 */
Mingming Cao617ba132006-10-11 01:20:53 -0700310 ext4_orphan_del(handle, inode);
Arnd Bergmann5ffff832018-07-29 15:50:00 -0400311 EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds();
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700312
313 /*
314 * One subtle ordering requirement: if anything has gone wrong
315 * (transaction abort, IO errors, whatever), then we can still
316 * do these next steps (the fs will already have been marked as
317 * having errors), but we can't free the inode if the mark_dirty
318 * fails.
319 */
Mingming Cao617ba132006-10-11 01:20:53 -0700320 if (ext4_mark_inode_dirty(handle, inode))
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700321 /* If that failed, just do the required in-core inode clear. */
Al Viro0930fcc2010-06-07 13:16:22 -0400322 ext4_clear_inode(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700323 else
Mingming Cao617ba132006-10-11 01:20:53 -0700324 ext4_free_inode(handle, inode);
325 ext4_journal_stop(handle);
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200326 sb_end_intwrite(inode->i_sb);
Tahsin Erdogan0421a182017-06-22 10:26:31 -0400327 ext4_xattr_inode_array_free(ea_inode_array);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700328 return;
329no_delete:
Al Viro0930fcc2010-06-07 13:16:22 -0400330 ext4_clear_inode(inode); /* We must guarantee clearing of inode... */
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700331}
332
Dmitry Monakhova9e7f442009-12-14 15:21:14 +0300333#ifdef CONFIG_QUOTA
334qsize_t *ext4_get_reserved_space(struct inode *inode)
Mingming Cao60e58e02009-01-22 18:13:05 +0100335{
Dmitry Monakhova9e7f442009-12-14 15:21:14 +0300336 return &EXT4_I(inode)->i_reserved_quota;
Mingming Cao60e58e02009-01-22 18:13:05 +0100337}
Dmitry Monakhova9e7f442009-12-14 15:21:14 +0300338#endif
Theodore Ts'o9d0be502010-01-01 02:41:30 -0500339
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400340/*
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500341 * Called with i_data_sem down, which is important since we can call
342 * ext4_discard_preallocations() from here.
343 */
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500344void ext4_da_update_reserve_space(struct inode *inode,
345 int used, int quota_claim)
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400346{
347 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500348 struct ext4_inode_info *ei = EXT4_I(inode);
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400349
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500350 spin_lock(&ei->i_block_reservation_lock);
Aditya Kalid8990242011-09-09 19:18:51 -0400351 trace_ext4_da_update_reserve_space(inode, used, quota_claim);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500352 if (unlikely(used > ei->i_reserved_data_blocks)) {
Theodore Ts'o8de5c322013-02-14 15:11:41 -0500353 ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
Theodore Ts'o1084f252012-03-19 23:13:43 -0400354 "with only %d reserved data blocks",
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500355 __func__, inode->i_ino, used,
356 ei->i_reserved_data_blocks);
357 WARN_ON(1);
358 used = ei->i_reserved_data_blocks;
Aneesh Kumar K.V6bc6e632008-10-10 09:39:00 -0400359 }
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400360
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500361 /* Update per-inode reservations */
362 ei->i_reserved_data_blocks -= used;
Theodore Ts'o71d4f7d2014-07-15 06:02:38 -0400363 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500364
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400365 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
Mingming Cao60e58e02009-01-22 18:13:05 +0100366
Eric Sandeen72b8ab92010-05-16 11:00:00 -0400367 /* Update quota subsystem for data blocks */
368 if (quota_claim)
Aditya Kali7b415bf2011-09-09 19:04:51 -0400369 dquot_claim_block(inode, EXT4_C2B(sbi, used));
Eric Sandeen72b8ab92010-05-16 11:00:00 -0400370 else {
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500371 /*
372 * We did fallocate with an offset that is already delayed
373 * allocated. So on delayed allocated writeback we should
Eric Sandeen72b8ab92010-05-16 11:00:00 -0400374 * not re-claim the quota for fallocated blocks.
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500375 */
Aditya Kali7b415bf2011-09-09 19:04:51 -0400376 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500377 }
Aneesh Kumar K.Vd6014302009-03-27 22:36:43 -0400378
379 /*
380 * If we have done all the pending block allocations and if
381 * there aren't any writers on the inode, we can discard the
382 * inode's preallocations.
383 */
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500384 if ((ei->i_reserved_data_blocks == 0) &&
Nikolay Borisov82dd1242019-02-10 23:04:16 -0500385 !inode_is_open_for_write(inode))
brookxu27bc4462020-08-17 15:36:15 +0800386 ext4_discard_preallocations(inode, 0);
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400387}
388
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400389static int __check_block_validity(struct inode *inode, const char *func,
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400390 unsigned int line,
391 struct ext4_map_blocks *map)
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400392{
Theodore Ts'o345c0db2019-04-09 23:37:08 -0400393 if (ext4_has_feature_journal(inode->i_sb) &&
394 (inode->i_ino ==
395 le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum)))
396 return 0;
Jan Karace9f24c2020-07-28 15:04:34 +0200397 if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400398 ext4_error_inode(inode, func, line, map->m_pblk,
Theodore Ts'obdbd6ce2018-06-15 12:27:16 -0400399 "lblock %lu mapped to illegal pblock %llu "
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400400 "(length %d)", (unsigned long) map->m_lblk,
Theodore Ts'obdbd6ce2018-06-15 12:27:16 -0400401 map->m_pblk, map->m_len);
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400402 return -EFSCORRUPTED;
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400403 }
404 return 0;
405}
406
Jan Kara53085fa2015-12-07 15:09:35 -0500407int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
408 ext4_lblk_t len)
409{
410 int ret;
411
Eric Biggers33b4cc22019-12-26 10:10:22 -0600412 if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
Jaegeuk Kima7550b32016-07-10 14:01:03 -0400413 return fscrypt_zeroout_range(inode, lblk, pblk, len);
Jan Kara53085fa2015-12-07 15:09:35 -0500414
415 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
416 if (ret > 0)
417 ret = 0;
418
419 return ret;
420}
421
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400422#define check_block_validity(inode, map) \
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400423 __check_block_validity((inode), __func__, __LINE__, (map))
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400424
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400425#ifdef ES_AGGRESSIVE_TEST
426static void ext4_map_blocks_es_recheck(handle_t *handle,
427 struct inode *inode,
428 struct ext4_map_blocks *es_map,
429 struct ext4_map_blocks *map,
430 int flags)
431{
432 int retval;
433
434 map->m_flags = 0;
435 /*
436 * There is a race window that the result is not the same.
437 * e.g. xfstests #223 when dioread_nolock enables. The reason
438 * is that we lookup a block mapping in extent status tree with
439 * out taking i_data_sem. So at the time the unwritten extent
440 * could be converted.
441 */
Jan Kara2dcba472015-12-07 15:04:57 -0500442 down_read(&EXT4_I(inode)->i_data_sem);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400443 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
Eric Whitney9e524842020-04-15 16:31:39 -0400444 retval = ext4_ext_map_blocks(handle, inode, map, 0);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400445 } else {
Eric Whitney9e524842020-04-15 16:31:39 -0400446 retval = ext4_ind_map_blocks(handle, inode, map, 0);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400447 }
Jan Kara2dcba472015-12-07 15:04:57 -0500448 up_read((&EXT4_I(inode)->i_data_sem));
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400449
450 /*
451 * We don't check m_len because extent will be collpased in status
452 * tree. So the m_len might not equal.
453 */
454 if (es_map->m_lblk != map->m_lblk ||
455 es_map->m_flags != map->m_flags ||
456 es_map->m_pblk != map->m_pblk) {
Theodore Ts'obdafe422013-07-13 00:40:31 -0400457 printk("ES cache assertion failed for inode: %lu "
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400458 "es_cached ex [%d/%d/%llu/%x] != "
459 "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
460 inode->i_ino, es_map->m_lblk, es_map->m_len,
461 es_map->m_pblk, es_map->m_flags, map->m_lblk,
462 map->m_len, map->m_pblk, map->m_flags,
463 retval, flags);
464 }
465}
466#endif /* ES_AGGRESSIVE_TEST */
467
Theodore Ts'o55138e0b2009-09-29 13:31:31 -0400468/*
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400469 * The ext4_map_blocks() function tries to look up the requested blocks,
Theodore Ts'o2b2d6d02008-07-26 16:15:44 -0400470 * and returns if the blocks are already mapped.
Mingming Caof5ab0d12008-02-25 15:29:55 -0500471 *
Mingming Caof5ab0d12008-02-25 15:29:55 -0500472 * Otherwise it takes the write lock of the i_data_sem and allocate blocks
473 * and store the allocated blocks in the result buffer head and mark it
474 * mapped.
475 *
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400476 * If file type is extents based, it will call ext4_ext_map_blocks(),
477 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
Mingming Caof5ab0d12008-02-25 15:29:55 -0500478 * based files
479 *
Jan Karafacab4d2016-03-09 22:54:00 -0500480 * On success, it returns the number of blocks being mapped or allocated. if
481 * create==0 and the blocks are pre-allocated and unwritten, the resulting @map
482 * is marked as unwritten. If the create == 1, it will mark @map as mapped.
Mingming Caof5ab0d12008-02-25 15:29:55 -0500483 *
484 * It returns 0 if plain look up failed (blocks have not been allocated), in
Jan Karafacab4d2016-03-09 22:54:00 -0500485 * that case, @map is returned as unmapped but we still do fill map->m_len to
486 * indicate the length of a hole starting at map->m_lblk.
Mingming Caof5ab0d12008-02-25 15:29:55 -0500487 *
488 * It returns the error in case of allocation failure.
489 */
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400490int ext4_map_blocks(handle_t *handle, struct inode *inode,
491 struct ext4_map_blocks *map, int flags)
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500492{
Zheng Liud100eef2013-02-18 00:29:59 -0500493 struct extent_status es;
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500494 int retval;
Lukas Czernerb8a86842014-03-18 18:05:35 -0400495 int ret = 0;
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400496#ifdef ES_AGGRESSIVE_TEST
497 struct ext4_map_blocks orig_map;
498
499 memcpy(&orig_map, map, sizeof(*map));
500#endif
Mingming Caof5ab0d12008-02-25 15:29:55 -0500501
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400502 map->m_flags = 0;
Ritesh Harjani70aa1552020-05-10 11:54:55 +0530503 ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
504 flags, map->m_len, (unsigned long) map->m_lblk);
Zheng Liud100eef2013-02-18 00:29:59 -0500505
Theodore Ts'oe861b5e2014-02-20 12:54:05 -0500506 /*
507 * ext4_map_blocks returns an int, and m_len is an unsigned int
508 */
509 if (unlikely(map->m_len > INT_MAX))
510 map->m_len = INT_MAX;
511
Kazuya Mio4adb6ab2014-04-07 10:53:28 -0400512 /* We can handle the block number less than EXT_MAX_BLOCKS */
513 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400514 return -EFSCORRUPTED;
Kazuya Mio4adb6ab2014-04-07 10:53:28 -0400515
Zheng Liud100eef2013-02-18 00:29:59 -0500516 /* Lookup extent status tree firstly */
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700517 if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) &&
518 ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
Zheng Liud100eef2013-02-18 00:29:59 -0500519 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
520 map->m_pblk = ext4_es_pblock(&es) +
521 map->m_lblk - es.es_lblk;
522 map->m_flags |= ext4_es_is_written(&es) ?
523 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
524 retval = es.es_len - (map->m_lblk - es.es_lblk);
525 if (retval > map->m_len)
526 retval = map->m_len;
527 map->m_len = retval;
528 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
Jan Karafacab4d2016-03-09 22:54:00 -0500529 map->m_pblk = 0;
530 retval = es.es_len - (map->m_lblk - es.es_lblk);
531 if (retval > map->m_len)
532 retval = map->m_len;
533 map->m_len = retval;
Zheng Liud100eef2013-02-18 00:29:59 -0500534 retval = 0;
535 } else {
Arnd Bergmann1e83bc82019-04-07 12:24:43 -0400536 BUG();
Zheng Liud100eef2013-02-18 00:29:59 -0500537 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400538#ifdef ES_AGGRESSIVE_TEST
539 ext4_map_blocks_es_recheck(handle, inode, map,
540 &orig_map, flags);
541#endif
Zheng Liud100eef2013-02-18 00:29:59 -0500542 goto found;
543 }
544
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500545 /*
Theodore Ts'ob920c752009-05-14 00:54:29 -0400546 * Try to see if we can get the block without requesting a new
547 * file system block.
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500548 */
Jan Kara2dcba472015-12-07 15:04:57 -0500549 down_read(&EXT4_I(inode)->i_data_sem);
Dmitry Monakhov12e9b892010-05-16 22:00:00 -0400550 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
Eric Whitney9e524842020-04-15 16:31:39 -0400551 retval = ext4_ext_map_blocks(handle, inode, map, 0);
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500552 } else {
Eric Whitney9e524842020-04-15 16:31:39 -0400553 retval = ext4_ind_map_blocks(handle, inode, map, 0);
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500554 }
Zheng Liuf7fec032013-02-18 00:28:47 -0500555 if (retval > 0) {
Theodore Ts'o3be78c72013-08-16 21:22:41 -0400556 unsigned int status;
Zheng Liuf7fec032013-02-18 00:28:47 -0500557
Zheng Liu44fb851d2013-07-29 12:51:42 -0400558 if (unlikely(retval != map->m_len)) {
559 ext4_warning(inode->i_sb,
560 "ES len assertion failed for inode "
561 "%lu: retval %d != map->m_len %d",
562 inode->i_ino, retval, map->m_len);
563 WARN_ON(1);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400564 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400565
Zheng Liuf7fec032013-02-18 00:28:47 -0500566 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
567 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
568 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
Lukas Czernerd2dc3172015-05-02 21:36:55 -0400569 !(status & EXTENT_STATUS_WRITTEN) &&
Eric Whitneyad431022018-10-01 14:10:39 -0400570 ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
571 map->m_lblk + map->m_len - 1))
Zheng Liuf7fec032013-02-18 00:28:47 -0500572 status |= EXTENT_STATUS_DELAYED;
573 ret = ext4_es_insert_extent(inode, map->m_lblk,
574 map->m_len, map->m_pblk, status);
575 if (ret < 0)
576 retval = ret;
577 }
Jan Kara2dcba472015-12-07 15:04:57 -0500578 up_read((&EXT4_I(inode)->i_data_sem));
Mingming Caof5ab0d12008-02-25 15:29:55 -0500579
Zheng Liud100eef2013-02-18 00:29:59 -0500580found:
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400581 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
Lukas Czernerb8a86842014-03-18 18:05:35 -0400582 ret = check_block_validity(inode, map);
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400583 if (ret != 0)
584 return ret;
585 }
586
Mingming Caof5ab0d12008-02-25 15:29:55 -0500587 /* If it is only a block(s) look up */
Theodore Ts'oc2177052009-05-14 00:58:52 -0400588 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500589 return retval;
590
591 /*
Mingming Caof5ab0d12008-02-25 15:29:55 -0500592 * Returns if the blocks have already allocated
593 *
594 * Note that if blocks have been preallocated
Tao Madf3ab172011-10-08 15:53:49 -0400595 * ext4_ext_get_block() returns the create = 0
Mingming Caof5ab0d12008-02-25 15:29:55 -0500596 * with buffer head unmapped.
597 */
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400598 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
Lukas Czernerb8a86842014-03-18 18:05:35 -0400599 /*
600 * If we need to convert extent to unwritten
601 * we continue and do the actual work in
602 * ext4_ext_map_blocks()
603 */
604 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
605 return retval;
Mingming Caof5ab0d12008-02-25 15:29:55 -0500606
607 /*
Zheng Liua25a4e12013-02-18 00:28:04 -0500608 * Here we clear m_flags because after allocating an new extent,
609 * it will be set again.
Aneesh Kumar K.V2a8964d2009-05-14 17:05:39 -0400610 */
Zheng Liua25a4e12013-02-18 00:28:04 -0500611 map->m_flags &= ~EXT4_MAP_FLAGS;
Aneesh Kumar K.V2a8964d2009-05-14 17:05:39 -0400612
613 /*
Lukas Czerner556615d2014-04-20 23:45:47 -0400614 * New blocks allocate and/or writing to unwritten extent
Mingming Caof5ab0d12008-02-25 15:29:55 -0500615 * will possibly result in updating i_data, so we take
Seunghun Leed91bd2c2014-09-01 22:15:30 -0400616 * the write lock of i_data_sem, and call get_block()
Mingming Caof5ab0d12008-02-25 15:29:55 -0500617 * with create == 1 flag.
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500618 */
Lukas Czernerc8b459f2014-05-12 12:55:07 -0400619 down_write(&EXT4_I(inode)->i_data_sem);
Mingming Caod2a17632008-07-14 17:52:37 -0400620
621 /*
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500622 * We need to check for EXT4 here because migrate
623 * could have changed the inode type in between
624 */
Dmitry Monakhov12e9b892010-05-16 22:00:00 -0400625 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400626 retval = ext4_ext_map_blocks(handle, inode, map, flags);
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500627 } else {
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400628 retval = ext4_ind_map_blocks(handle, inode, map, flags);
Aneesh Kumar K.V267e4db2008-04-29 08:11:12 -0400629
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400630 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
Aneesh Kumar K.V267e4db2008-04-29 08:11:12 -0400631 /*
632 * We allocated new blocks which will result in
633 * i_data's format changing. Force the migrate
634 * to fail by clearing migrate flags
635 */
Theodore Ts'o19f5fb72010-01-24 14:34:07 -0500636 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
Aneesh Kumar K.V267e4db2008-04-29 08:11:12 -0400637 }
Mingming Caod2a17632008-07-14 17:52:37 -0400638
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500639 /*
640 * Update reserved blocks/metadata blocks after successful
641 * block allocation which had been deferred till now. We don't
642 * support fallocate for non extent files. So we can update
643 * reserve space here.
644 */
645 if ((retval > 0) &&
Aneesh Kumar K.V1296cc82010-01-15 01:27:59 -0500646 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500647 ext4_da_update_reserve_space(inode, retval, 1);
648 }
Theodore Ts'o2ac3b6e2009-05-14 13:57:08 -0400649
Zheng Liuf7fec032013-02-18 00:28:47 -0500650 if (retval > 0) {
Theodore Ts'o3be78c72013-08-16 21:22:41 -0400651 unsigned int status;
Zheng Liuf7fec032013-02-18 00:28:47 -0500652
Zheng Liu44fb851d2013-07-29 12:51:42 -0400653 if (unlikely(retval != map->m_len)) {
654 ext4_warning(inode->i_sb,
655 "ES len assertion failed for inode "
656 "%lu: retval %d != map->m_len %d",
657 inode->i_ino, retval, map->m_len);
658 WARN_ON(1);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400659 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400660
Zheng Liuadb23552013-03-10 21:13:05 -0400661 /*
Jan Karac86d8db2015-12-07 15:10:26 -0500662 * We have to zeroout blocks before inserting them into extent
663 * status tree. Otherwise someone could look them up there and
Jan Kara9b623df2016-09-30 02:02:29 -0400664 * use them before they are really zeroed. We also have to
665 * unmap metadata before zeroing as otherwise writeback can
666 * overwrite zeros with stale data from block device.
Jan Karac86d8db2015-12-07 15:10:26 -0500667 */
668 if (flags & EXT4_GET_BLOCKS_ZERO &&
669 map->m_flags & EXT4_MAP_MAPPED &&
670 map->m_flags & EXT4_MAP_NEW) {
671 ret = ext4_issue_zeroout(inode, map->m_lblk,
672 map->m_pblk, map->m_len);
673 if (ret) {
674 retval = ret;
675 goto out_sem;
676 }
677 }
678
679 /*
Zheng Liuadb23552013-03-10 21:13:05 -0400680 * If the extent has been zeroed out, we don't need to update
681 * extent status tree.
682 */
683 if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
Theodore Ts'obb5835e2019-08-11 16:32:41 -0400684 ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
Zheng Liuadb23552013-03-10 21:13:05 -0400685 if (ext4_es_is_written(&es))
Jan Karac86d8db2015-12-07 15:10:26 -0500686 goto out_sem;
Zheng Liuadb23552013-03-10 21:13:05 -0400687 }
Zheng Liuf7fec032013-02-18 00:28:47 -0500688 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
689 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
690 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
Lukas Czernerd2dc3172015-05-02 21:36:55 -0400691 !(status & EXTENT_STATUS_WRITTEN) &&
Eric Whitneyad431022018-10-01 14:10:39 -0400692 ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
693 map->m_lblk + map->m_len - 1))
Zheng Liuf7fec032013-02-18 00:28:47 -0500694 status |= EXTENT_STATUS_DELAYED;
695 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
696 map->m_pblk, status);
Jan Karac86d8db2015-12-07 15:10:26 -0500697 if (ret < 0) {
Zheng Liuf7fec032013-02-18 00:28:47 -0500698 retval = ret;
Jan Karac86d8db2015-12-07 15:10:26 -0500699 goto out_sem;
700 }
Aditya Kali5356f2612011-09-09 19:20:51 -0400701 }
702
Jan Karac86d8db2015-12-07 15:10:26 -0500703out_sem:
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500704 up_write((&EXT4_I(inode)->i_data_sem));
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400705 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
Lukas Czernerb8a86842014-03-18 18:05:35 -0400706 ret = check_block_validity(inode, map);
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400707 if (ret != 0)
708 return ret;
Jan Kara06bd3c32016-04-24 00:56:03 -0400709
710 /*
711 * Inodes with freshly allocated blocks where contents will be
712 * visible after transaction commit must be on transaction's
713 * ordered data list.
714 */
715 if (map->m_flags & EXT4_MAP_NEW &&
716 !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
717 !(flags & EXT4_GET_BLOCKS_ZERO) &&
Tahsin Erdogan02749a42017-06-22 11:31:25 -0400718 !ext4_is_quota_file(inode) &&
Jan Kara06bd3c32016-04-24 00:56:03 -0400719 ext4_should_order_data(inode)) {
Ross Zwisler73131fb2019-06-20 17:26:26 -0400720 loff_t start_byte =
721 (loff_t)map->m_lblk << inode->i_blkbits;
722 loff_t length = (loff_t)map->m_len << inode->i_blkbits;
723
Jan Karaee0876b2016-04-24 00:56:08 -0400724 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
Ross Zwisler73131fb2019-06-20 17:26:26 -0400725 ret = ext4_jbd2_inode_add_wait(handle, inode,
726 start_byte, length);
Jan Karaee0876b2016-04-24 00:56:08 -0400727 else
Ross Zwisler73131fb2019-06-20 17:26:26 -0400728 ret = ext4_jbd2_inode_add_write(handle, inode,
729 start_byte, length);
Jan Kara06bd3c32016-04-24 00:56:03 -0400730 if (ret)
731 return ret;
732 }
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -0700733 ext4_fc_track_range(inode, map->m_lblk,
734 map->m_lblk + map->m_len - 1);
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400735 }
Ritesh Harjaniec8c60b2020-05-10 11:54:52 +0530736
737 if (retval < 0)
Ritesh Harjani70aa1552020-05-10 11:54:55 +0530738 ext_debug(inode, "failed with err %d\n", retval);
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500739 return retval;
740}
741
Jan Karaed8ad832016-02-19 00:18:25 -0500742/*
743 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
744 * we have to be careful as someone else may be manipulating b_state as well.
745 */
746static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
747{
748 unsigned long old_state;
749 unsigned long new_state;
750
751 flags &= EXT4_MAP_FLAGS;
752
753 /* Dummy buffer_head? Set non-atomically. */
754 if (!bh->b_page) {
755 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
756 return;
757 }
758 /*
759 * Someone else may be modifying b_state. Be careful! This is ugly but
760 * once we get rid of using bh as a container for mapping information
761 * to pass to / from get_block functions, this can go away.
762 */
763 do {
764 old_state = READ_ONCE(bh->b_state);
765 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
766 } while (unlikely(
767 cmpxchg(&bh->b_state, old_state, new_state) != old_state));
768}
769
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400770static int _ext4_get_block(struct inode *inode, sector_t iblock,
771 struct buffer_head *bh, int flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700772{
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400773 struct ext4_map_blocks map;
Jan Karaefe70c22016-03-08 23:35:46 -0500774 int ret = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700775
Tao Ma46c7f252012-12-10 14:04:52 -0500776 if (ext4_has_inline_data(inode))
777 return -ERANGE;
778
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400779 map.m_lblk = iblock;
780 map.m_len = bh->b_size >> inode->i_blkbits;
781
Jan Karaefe70c22016-03-08 23:35:46 -0500782 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
783 flags);
Jan Kara7fb54092008-02-10 01:08:38 -0500784 if (ret > 0) {
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400785 map_bh(bh, inode->i_sb, map.m_pblk);
Jan Karaed8ad832016-02-19 00:18:25 -0500786 ext4_update_bh_state(bh, map.m_flags);
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400787 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
Jan Kara7fb54092008-02-10 01:08:38 -0500788 ret = 0;
Ross Zwisler547edce2016-11-08 11:30:58 +1100789 } else if (ret == 0) {
790 /* hole case, need to fill in bh->b_size */
791 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700792 }
793 return ret;
794}
795
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400796int ext4_get_block(struct inode *inode, sector_t iblock,
797 struct buffer_head *bh, int create)
798{
799 return _ext4_get_block(inode, iblock, bh,
800 create ? EXT4_GET_BLOCKS_CREATE : 0);
801}
802
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700803/*
Jan Kara705965b2016-03-08 23:08:10 -0500804 * Get block function used when preparing for buffered write if we require
805 * creating an unwritten extent if blocks haven't been allocated. The extent
806 * will be converted to written after the IO is complete.
807 */
808int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
809 struct buffer_head *bh_result, int create)
810{
811 ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
812 inode->i_ino, create);
813 return _ext4_get_block(inode, iblock, bh_result,
814 EXT4_GET_BLOCKS_IO_CREATE_EXT);
815}
816
Jan Karaefe70c22016-03-08 23:35:46 -0500817/* Maximum number of blocks we map for direct IO at once. */
818#define DIO_MAX_BLOCKS 4096
819
Jan Karae84dfbe2016-04-01 02:07:22 -0400820/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700821 * `handle' can be NULL if create is zero
822 */
Mingming Cao617ba132006-10-11 01:20:53 -0700823struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400824 ext4_lblk_t block, int map_flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700825{
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400826 struct ext4_map_blocks map;
827 struct buffer_head *bh;
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400828 int create = map_flags & EXT4_GET_BLOCKS_CREATE;
Theodore Ts'o10560082014-08-29 20:51:32 -0400829 int err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700830
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700831 J_ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
832 || handle != NULL || create == 0);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700833
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400834 map.m_lblk = block;
835 map.m_len = 1;
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400836 err = ext4_map_blocks(handle, inode, &map, map_flags);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700837
Theodore Ts'o10560082014-08-29 20:51:32 -0400838 if (err == 0)
839 return create ? ERR_PTR(-ENOSPC) : NULL;
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400840 if (err < 0)
Theodore Ts'o10560082014-08-29 20:51:32 -0400841 return ERR_PTR(err);
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400842
843 bh = sb_getblk(inode->i_sb, map.m_pblk);
Theodore Ts'o10560082014-08-29 20:51:32 -0400844 if (unlikely(!bh))
845 return ERR_PTR(-ENOMEM);
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400846 if (map.m_flags & EXT4_MAP_NEW) {
847 J_ASSERT(create != 0);
Harshad Shirwadkar8016e292020-10-15 13:37:59 -0700848 J_ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
849 || (handle != NULL));
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400850
851 /*
852 * Now that we do not always journal data, we should
853 * keep in mind whether this should always journal the
854 * new buffer as metadata. For now, regular file
855 * writes use ext4_get_block instead, so it's not a
856 * problem.
857 */
858 lock_buffer(bh);
859 BUFFER_TRACE(bh, "call get_create_access");
Theodore Ts'o10560082014-08-29 20:51:32 -0400860 err = ext4_journal_get_create_access(handle, bh);
861 if (unlikely(err)) {
862 unlock_buffer(bh);
863 goto errout;
864 }
865 if (!buffer_uptodate(bh)) {
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400866 memset(bh->b_data, 0, inode->i_sb->s_blocksize);
867 set_buffer_uptodate(bh);
868 }
869 unlock_buffer(bh);
870 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
871 err = ext4_handle_dirty_metadata(handle, inode, bh);
Theodore Ts'o10560082014-08-29 20:51:32 -0400872 if (unlikely(err))
873 goto errout;
874 } else
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400875 BUFFER_TRACE(bh, "not a new buffer");
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400876 return bh;
Theodore Ts'o10560082014-08-29 20:51:32 -0400877errout:
878 brelse(bh);
879 return ERR_PTR(err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700880}
881
Mingming Cao617ba132006-10-11 01:20:53 -0700882struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400883 ext4_lblk_t block, int map_flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700884{
Theodore Ts'oaf5bc922008-09-08 22:25:24 -0400885 struct buffer_head *bh;
zhangyi (F)2d069c02020-09-24 15:33:33 +0800886 int ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700887
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400888 bh = ext4_getblk(handle, inode, block, map_flags);
Theodore Ts'o1c215022014-08-29 20:52:15 -0400889 if (IS_ERR(bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700890 return bh;
ZhangXiaoxu7963e5a2019-08-22 23:00:32 -0400891 if (!bh || ext4_buffer_uptodate(bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700892 return bh;
zhangyi (F)2d069c02020-09-24 15:33:33 +0800893
894 ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);
895 if (ret) {
896 put_bh(bh);
897 return ERR_PTR(ret);
898 }
899 return bh;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700900}
901
Tahsin Erdogan9699d4f2017-08-06 00:07:01 -0400902/* Read a contiguous batch of blocks. */
903int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
904 bool wait, struct buffer_head **bhs)
905{
906 int i, err;
907
908 for (i = 0; i < bh_count; i++) {
909 bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);
910 if (IS_ERR(bhs[i])) {
911 err = PTR_ERR(bhs[i]);
912 bh_count = i;
913 goto out_brelse;
914 }
915 }
916
917 for (i = 0; i < bh_count; i++)
918 /* Note that NULL bhs[i] is valid because of holes. */
zhangyi (F)2d069c02020-09-24 15:33:33 +0800919 if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))
920 ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);
Tahsin Erdogan9699d4f2017-08-06 00:07:01 -0400921
922 if (!wait)
923 return 0;
924
925 for (i = 0; i < bh_count; i++)
926 if (bhs[i])
927 wait_on_buffer(bhs[i]);
928
929 for (i = 0; i < bh_count; i++) {
930 if (bhs[i] && !buffer_uptodate(bhs[i])) {
931 err = -EIO;
932 goto out_brelse;
933 }
934 }
935 return 0;
936
937out_brelse:
938 for (i = 0; i < bh_count; i++) {
939 brelse(bhs[i]);
940 bhs[i] = NULL;
941 }
942 return err;
943}
944
Tao Maf19d5872012-12-10 14:05:51 -0500945int ext4_walk_page_buffers(handle_t *handle,
946 struct buffer_head *head,
947 unsigned from,
948 unsigned to,
949 int *partial,
950 int (*fn)(handle_t *handle,
951 struct buffer_head *bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700952{
953 struct buffer_head *bh;
954 unsigned block_start, block_end;
955 unsigned blocksize = head->b_size;
956 int err, ret = 0;
957 struct buffer_head *next;
958
Theodore Ts'oaf5bc922008-09-08 22:25:24 -0400959 for (bh = head, block_start = 0;
960 ret == 0 && (bh != head || !block_start);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -0400961 block_start = block_end, bh = next) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700962 next = bh->b_this_page;
963 block_end = block_start + blocksize;
964 if (block_end <= from || block_start >= to) {
965 if (partial && !buffer_uptodate(bh))
966 *partial = 1;
967 continue;
968 }
969 err = (*fn)(handle, bh);
970 if (!ret)
971 ret = err;
972 }
973 return ret;
974}
975
976/*
977 * To preserve ordering, it is essential that the hole instantiation and
978 * the data write be encapsulated in a single transaction. We cannot
Mingming Cao617ba132006-10-11 01:20:53 -0700979 * close off a transaction and start a new one between the ext4_get_block()
Mingming Caodab291a2006-10-11 01:21:01 -0700980 * and the commit_write(). So doing the jbd2_journal_start at the start of
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700981 * prepare_write() is the right place.
982 *
Jan Kara36ade452013-01-28 09:30:52 -0500983 * Also, this function can nest inside ext4_writepage(). In that case, we
984 * *know* that ext4_writepage() has generated enough buffer credits to do the
985 * whole page. So we won't block on the journal in that case, which is good,
986 * because the caller may be PF_MEMALLOC.
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700987 *
Mingming Cao617ba132006-10-11 01:20:53 -0700988 * By accident, ext4 can be reentered when a transaction is open via
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700989 * quota file writes. If we were to commit the transaction while thus
990 * reentered, there can be a deadlock - we would be holding a quota
991 * lock, and the commit would never complete if another thread had a
992 * transaction open and was blocking on the quota lock - a ranking
993 * violation.
994 *
Mingming Caodab291a2006-10-11 01:21:01 -0700995 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700996 * will _not_ run commit under these circumstances because handle->h_ref
997 * is elevated. We'll still have enough credits for the tiny quotafile
998 * write.
999 */
Tao Maf19d5872012-12-10 14:05:51 -05001000int do_journal_get_write_access(handle_t *handle,
1001 struct buffer_head *bh)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001002{
Jan Kara56d35a42010-08-05 14:41:42 -04001003 int dirty = buffer_dirty(bh);
1004 int ret;
1005
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001006 if (!buffer_mapped(bh) || buffer_freed(bh))
1007 return 0;
Jan Kara56d35a42010-08-05 14:41:42 -04001008 /*
Christoph Hellwigebdec242010-10-06 10:47:23 +02001009 * __block_write_begin() could have dirtied some buffers. Clean
Jan Kara56d35a42010-08-05 14:41:42 -04001010 * the dirty bit as jbd2_journal_get_write_access() could complain
1011 * otherwise about fs integrity issues. Setting of the dirty bit
Christoph Hellwigebdec242010-10-06 10:47:23 +02001012 * by __block_write_begin() isn't a real problem here as we clear
Jan Kara56d35a42010-08-05 14:41:42 -04001013 * the bit before releasing a page lock and thus writeback cannot
1014 * ever write the buffer.
1015 */
1016 if (dirty)
1017 clear_buffer_dirty(bh);
liang xie5d601252014-05-12 22:06:43 -04001018 BUFFER_TRACE(bh, "get write access");
Jan Kara56d35a42010-08-05 14:41:42 -04001019 ret = ext4_journal_get_write_access(handle, bh);
1020 if (!ret && dirty)
1021 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1022 return ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001023}
1024
Chandan Rajendra643fa962018-12-12 15:20:12 +05301025#ifdef CONFIG_FS_ENCRYPTION
Michael Halcrow2058f832015-04-12 00:55:10 -04001026static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
1027 get_block_t *get_block)
1028{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001029 unsigned from = pos & (PAGE_SIZE - 1);
Michael Halcrow2058f832015-04-12 00:55:10 -04001030 unsigned to = from + len;
1031 struct inode *inode = page->mapping->host;
1032 unsigned block_start, block_end;
1033 sector_t block;
1034 int err = 0;
1035 unsigned blocksize = inode->i_sb->s_blocksize;
1036 unsigned bbits;
Chandan Rajendra0b578f32019-05-20 09:29:50 -07001037 struct buffer_head *bh, *head, *wait[2];
1038 int nr_wait = 0;
1039 int i;
Michael Halcrow2058f832015-04-12 00:55:10 -04001040
1041 BUG_ON(!PageLocked(page));
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001042 BUG_ON(from > PAGE_SIZE);
1043 BUG_ON(to > PAGE_SIZE);
Michael Halcrow2058f832015-04-12 00:55:10 -04001044 BUG_ON(from > to);
1045
1046 if (!page_has_buffers(page))
1047 create_empty_buffers(page, blocksize, 0);
1048 head = page_buffers(page);
1049 bbits = ilog2(blocksize);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001050 block = (sector_t)page->index << (PAGE_SHIFT - bbits);
Michael Halcrow2058f832015-04-12 00:55:10 -04001051
1052 for (bh = head, block_start = 0; bh != head || !block_start;
1053 block++, block_start = block_end, bh = bh->b_this_page) {
1054 block_end = block_start + blocksize;
1055 if (block_end <= from || block_start >= to) {
1056 if (PageUptodate(page)) {
1057 if (!buffer_uptodate(bh))
1058 set_buffer_uptodate(bh);
1059 }
1060 continue;
1061 }
1062 if (buffer_new(bh))
1063 clear_buffer_new(bh);
1064 if (!buffer_mapped(bh)) {
1065 WARN_ON(bh->b_size != blocksize);
1066 err = get_block(inode, block, bh, 1);
1067 if (err)
1068 break;
1069 if (buffer_new(bh)) {
Michael Halcrow2058f832015-04-12 00:55:10 -04001070 if (PageUptodate(page)) {
1071 clear_buffer_new(bh);
1072 set_buffer_uptodate(bh);
1073 mark_buffer_dirty(bh);
1074 continue;
1075 }
1076 if (block_end > to || block_start < from)
1077 zero_user_segments(page, to, block_end,
1078 block_start, from);
1079 continue;
1080 }
1081 }
1082 if (PageUptodate(page)) {
1083 if (!buffer_uptodate(bh))
1084 set_buffer_uptodate(bh);
1085 continue;
1086 }
1087 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1088 !buffer_unwritten(bh) &&
1089 (block_start < from || block_end > to)) {
zhangyi (F)2d069c02020-09-24 15:33:33 +08001090 ext4_read_bh_lock(bh, 0, false);
Chandan Rajendra0b578f32019-05-20 09:29:50 -07001091 wait[nr_wait++] = bh;
Michael Halcrow2058f832015-04-12 00:55:10 -04001092 }
1093 }
1094 /*
1095 * If we issued read requests, let them complete.
1096 */
Chandan Rajendra0b578f32019-05-20 09:29:50 -07001097 for (i = 0; i < nr_wait; i++) {
1098 wait_on_buffer(wait[i]);
1099 if (!buffer_uptodate(wait[i]))
Michael Halcrow2058f832015-04-12 00:55:10 -04001100 err = -EIO;
1101 }
Chandan Rajendra7e0785f2019-05-20 09:29:49 -07001102 if (unlikely(err)) {
Michael Halcrow2058f832015-04-12 00:55:10 -04001103 page_zero_new_buffers(page, from, to);
Eric Biggers4f74d152020-07-02 01:56:07 +00001104 } else if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
Chandan Rajendra0b578f32019-05-20 09:29:50 -07001105 for (i = 0; i < nr_wait; i++) {
1106 int err2;
1107
1108 err2 = fscrypt_decrypt_pagecache_blocks(page, blocksize,
1109 bh_offset(wait[i]));
1110 if (err2) {
1111 clear_buffer_uptodate(wait[i]);
1112 err = err2;
1113 }
1114 }
Chandan Rajendra7e0785f2019-05-20 09:29:49 -07001115 }
1116
Michael Halcrow2058f832015-04-12 00:55:10 -04001117 return err;
1118}
1119#endif
1120
Nick Pigginbfc1af62007-10-16 01:25:05 -07001121static int ext4_write_begin(struct file *file, struct address_space *mapping,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001122 loff_t pos, unsigned len, unsigned flags,
1123 struct page **pagep, void **fsdata)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001124{
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001125 struct inode *inode = mapping->host;
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001126 int ret, needed_blocks;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001127 handle_t *handle;
1128 int retries = 0;
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001129 struct page *page;
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001130 pgoff_t index;
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001131 unsigned from, to;
Nick Pigginbfc1af62007-10-16 01:25:05 -07001132
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05001133 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
1134 return -EIO;
1135
Theodore Ts'o9bffad12009-06-17 11:48:11 -04001136 trace_ext4_write_begin(inode, pos, len, flags);
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001137 /*
1138 * Reserve one block more for addition to orphan list in case
1139 * we allocate blocks but write fails for some reason
1140 */
1141 needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001142 index = pos >> PAGE_SHIFT;
1143 from = pos & (PAGE_SIZE - 1);
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001144 to = from + len;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001145
Tao Maf19d5872012-12-10 14:05:51 -05001146 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1147 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1148 flags, pagep);
1149 if (ret < 0)
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001150 return ret;
1151 if (ret == 1)
1152 return 0;
Tao Maf19d5872012-12-10 14:05:51 -05001153 }
1154
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001155 /*
1156 * grab_cache_page_write_begin() can take a long time if the
1157 * system is thrashing due to memory pressure, or if the page
1158 * is being written back. So grab it first before we start
1159 * the transaction handle. This also allows us to allocate
1160 * the page (if needed) without using GFP_NOFS.
1161 */
1162retry_grab:
Nick Piggin54566b22009-01-04 12:00:53 -08001163 page = grab_cache_page_write_begin(mapping, index, flags);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001164 if (!page)
1165 return -ENOMEM;
1166 unlock_page(page);
1167
1168retry_journal:
Theodore Ts'o9924a922013-02-08 21:59:22 -05001169 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001170 if (IS_ERR(handle)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001171 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001172 return PTR_ERR(handle);
Jan Karacf108bc2008-07-11 19:27:31 -04001173 }
Tao Maf19d5872012-12-10 14:05:51 -05001174
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001175 lock_page(page);
1176 if (page->mapping != mapping) {
1177 /* The page got truncated from under us */
1178 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001179 put_page(page);
Jan Karacf108bc2008-07-11 19:27:31 -04001180 ext4_journal_stop(handle);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001181 goto retry_grab;
Jan Karacf108bc2008-07-11 19:27:31 -04001182 }
Dmitry Monakhov7afe5aa2013-08-28 14:30:47 -04001183 /* In case writeback began while the page was unlocked */
1184 wait_for_stable_page(page);
Jan Karacf108bc2008-07-11 19:27:31 -04001185
Chandan Rajendra643fa962018-12-12 15:20:12 +05301186#ifdef CONFIG_FS_ENCRYPTION
Michael Halcrow2058f832015-04-12 00:55:10 -04001187 if (ext4_should_dioread_nolock(inode))
1188 ret = ext4_block_write_begin(page, pos, len,
Jan Kara705965b2016-03-08 23:08:10 -05001189 ext4_get_block_unwritten);
Michael Halcrow2058f832015-04-12 00:55:10 -04001190 else
1191 ret = ext4_block_write_begin(page, pos, len,
1192 ext4_get_block);
1193#else
Jiaying Zhang744692d2010-03-04 16:14:02 -05001194 if (ext4_should_dioread_nolock(inode))
Jan Kara705965b2016-03-08 23:08:10 -05001195 ret = __block_write_begin(page, pos, len,
1196 ext4_get_block_unwritten);
Jiaying Zhang744692d2010-03-04 16:14:02 -05001197 else
Christoph Hellwig6e1db882010-06-04 11:29:57 +02001198 ret = __block_write_begin(page, pos, len, ext4_get_block);
Michael Halcrow2058f832015-04-12 00:55:10 -04001199#endif
Nick Pigginbfc1af62007-10-16 01:25:05 -07001200 if (!ret && ext4_should_journal_data(inode)) {
Tao Maf19d5872012-12-10 14:05:51 -05001201 ret = ext4_walk_page_buffers(handle, page_buffers(page),
1202 from, to, NULL,
1203 do_journal_get_write_access);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001204 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001205
1206 if (ret) {
Eric Biggersc93d8f82019-07-22 09:26:24 -07001207 bool extended = (pos + len > inode->i_size) &&
1208 !ext4_verity_in_progress(inode);
1209
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001210 unlock_page(page);
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04001211 /*
Christoph Hellwig6e1db882010-06-04 11:29:57 +02001212 * __block_write_begin may have instantiated a few blocks
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04001213 * outside i_size. Trim these off again. Don't need
1214 * i_size_read because we hold i_mutex.
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001215 *
1216 * Add inode to orphan list in case we crash before
1217 * truncate finishes
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04001218 */
Eric Biggersc93d8f82019-07-22 09:26:24 -07001219 if (extended && ext4_can_truncate(inode))
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001220 ext4_orphan_add(handle, inode);
1221
1222 ext4_journal_stop(handle);
Eric Biggersc93d8f82019-07-22 09:26:24 -07001223 if (extended) {
Jan Karab9a42072009-12-08 21:24:33 -05001224 ext4_truncate_failed_write(inode);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001225 /*
Jan Karaffacfa72009-07-13 16:22:22 -04001226 * If truncate failed early the inode might
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001227 * still be on the orphan list; we need to
1228 * make sure the inode is removed from the
1229 * orphan list in that case.
1230 */
1231 if (inode->i_nlink)
1232 ext4_orphan_del(NULL, inode);
1233 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001234
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001235 if (ret == -ENOSPC &&
1236 ext4_should_retry_alloc(inode->i_sb, &retries))
1237 goto retry_journal;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001238 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001239 return ret;
1240 }
1241 *pagep = page;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001242 return ret;
1243}
1244
Nick Pigginbfc1af62007-10-16 01:25:05 -07001245/* For write_end() in data=journal mode */
1246static int write_end_fn(handle_t *handle, struct buffer_head *bh)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001247{
Theodore Ts'o13fca322013-04-21 16:45:54 -04001248 int ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001249 if (!buffer_mapped(bh) || buffer_freed(bh))
1250 return 0;
1251 set_buffer_uptodate(bh);
Theodore Ts'o13fca322013-04-21 16:45:54 -04001252 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1253 clear_buffer_meta(bh);
1254 clear_buffer_prio(bh);
1255 return ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001256}
1257
Zheng Liueed43332013-04-03 12:41:17 -04001258/*
1259 * We need to pick up the new inode size which generic_commit_write gave us
1260 * `file' can be NULL - eg, when called from page_symlink().
1261 *
1262 * ext4 never places buffers on inode->i_mapping->private_list. metadata
1263 * buffers are managed internally.
1264 */
1265static int ext4_write_end(struct file *file,
1266 struct address_space *mapping,
1267 loff_t pos, unsigned len, unsigned copied,
1268 struct page *page, void *fsdata)
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001269{
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001270 handle_t *handle = ext4_journal_current_handle();
Zheng Liueed43332013-04-03 12:41:17 -04001271 struct inode *inode = mapping->host;
Xiaoguang Wang05726392015-02-12 23:00:17 -05001272 loff_t old_size = inode->i_size;
Zheng Liueed43332013-04-03 12:41:17 -04001273 int ret = 0, ret2;
1274 int i_size_changed = 0;
Theodore Ts'o362eca72018-07-10 01:07:43 -04001275 int inline_data = ext4_has_inline_data(inode);
Eric Biggersc93d8f82019-07-22 09:26:24 -07001276 bool verity = ext4_verity_in_progress(inode);
Zheng Liueed43332013-04-03 12:41:17 -04001277
1278 trace_ext4_write_end(inode, pos, len, copied);
Theodore Ts'o362eca72018-07-10 01:07:43 -04001279 if (inline_data) {
Theodore Ts'o42c832d2013-07-01 08:12:39 -04001280 ret = ext4_write_inline_data_end(inode, pos, len,
1281 copied, page);
Theodore Ts'oeb5efbc2017-02-04 23:04:00 -05001282 if (ret < 0) {
1283 unlock_page(page);
1284 put_page(page);
Theodore Ts'o42c832d2013-07-01 08:12:39 -04001285 goto errout;
Theodore Ts'oeb5efbc2017-02-04 23:04:00 -05001286 }
Theodore Ts'o42c832d2013-07-01 08:12:39 -04001287 copied = ret;
1288 } else
Tao Maf19d5872012-12-10 14:05:51 -05001289 copied = block_write_end(file, mapping, pos,
1290 len, copied, page, fsdata);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001291 /*
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001292 * it's important to update i_size while still holding page lock:
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001293 * page writeout could otherwise come in and zero beyond i_size.
Eric Biggersc93d8f82019-07-22 09:26:24 -07001294 *
1295 * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree
1296 * blocks are being written past EOF, so skip the i_size update.
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001297 */
Eric Biggersc93d8f82019-07-22 09:26:24 -07001298 if (!verity)
1299 i_size_changed = ext4_update_inode_size(inode, pos + copied);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001300 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001301 put_page(page);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001302
Eric Biggersc93d8f82019-07-22 09:26:24 -07001303 if (old_size < pos && !verity)
Xiaoguang Wang05726392015-02-12 23:00:17 -05001304 pagecache_isize_extended(inode, old_size, pos);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001305 /*
1306 * Don't mark the inode dirty under page lock. First, it unnecessarily
1307 * makes the holding time of page lock longer. Second, it forces lock
1308 * ordering of page lock and transaction start for journaling
1309 * filesystems.
1310 */
Theodore Ts'o362eca72018-07-10 01:07:43 -04001311 if (i_size_changed || inline_data)
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07001312 ret = ext4_mark_inode_dirty(handle, inode);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001313
Eric Biggersc93d8f82019-07-22 09:26:24 -07001314 if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001315 /* if we have allocated more blocks and copied
1316 * less. We will have blocks allocated outside
1317 * inode->i_size. So truncate them
1318 */
1319 ext4_orphan_add(handle, inode);
Theodore Ts'o74d553a2013-04-03 12:39:17 -04001320errout:
Mingming Cao617ba132006-10-11 01:20:53 -07001321 ret2 = ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001322 if (!ret)
1323 ret = ret2;
Nick Pigginbfc1af62007-10-16 01:25:05 -07001324
Eric Biggersc93d8f82019-07-22 09:26:24 -07001325 if (pos + len > inode->i_size && !verity) {
Jan Karab9a42072009-12-08 21:24:33 -05001326 ext4_truncate_failed_write(inode);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001327 /*
Jan Karaffacfa72009-07-13 16:22:22 -04001328 * If truncate failed early the inode might still be
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001329 * on the orphan list; we need to make sure the inode
1330 * is removed from the orphan list in that case.
1331 */
1332 if (inode->i_nlink)
1333 ext4_orphan_del(NULL, inode);
1334 }
1335
Nick Pigginbfc1af62007-10-16 01:25:05 -07001336 return ret ? ret : copied;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001337}
1338
Theodore Ts'ob90197b2015-10-15 10:29:05 -04001339/*
1340 * This is a private version of page_zero_new_buffers() which doesn't
1341 * set the buffer to be dirty, since in data=journalled mode we need
1342 * to call ext4_handle_dirty_metadata() instead.
1343 */
Jan Kara3b136492017-01-27 14:35:38 -05001344static void ext4_journalled_zero_new_buffers(handle_t *handle,
1345 struct page *page,
1346 unsigned from, unsigned to)
Theodore Ts'ob90197b2015-10-15 10:29:05 -04001347{
1348 unsigned int block_start = 0, block_end;
1349 struct buffer_head *head, *bh;
1350
1351 bh = head = page_buffers(page);
1352 do {
1353 block_end = block_start + bh->b_size;
1354 if (buffer_new(bh)) {
1355 if (block_end > from && block_start < to) {
1356 if (!PageUptodate(page)) {
1357 unsigned start, size;
1358
1359 start = max(from, block_start);
1360 size = min(to, block_end) - start;
1361
1362 zero_user(page, start, size);
Jan Kara3b136492017-01-27 14:35:38 -05001363 write_end_fn(handle, bh);
Theodore Ts'ob90197b2015-10-15 10:29:05 -04001364 }
1365 clear_buffer_new(bh);
1366 }
1367 }
1368 block_start = block_end;
1369 bh = bh->b_this_page;
1370 } while (bh != head);
1371}
1372
Nick Pigginbfc1af62007-10-16 01:25:05 -07001373static int ext4_journalled_write_end(struct file *file,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001374 struct address_space *mapping,
1375 loff_t pos, unsigned len, unsigned copied,
1376 struct page *page, void *fsdata)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001377{
Mingming Cao617ba132006-10-11 01:20:53 -07001378 handle_t *handle = ext4_journal_current_handle();
Nick Pigginbfc1af62007-10-16 01:25:05 -07001379 struct inode *inode = mapping->host;
Xiaoguang Wang05726392015-02-12 23:00:17 -05001380 loff_t old_size = inode->i_size;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001381 int ret = 0, ret2;
1382 int partial = 0;
Nick Pigginbfc1af62007-10-16 01:25:05 -07001383 unsigned from, to;
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001384 int size_changed = 0;
Theodore Ts'o362eca72018-07-10 01:07:43 -04001385 int inline_data = ext4_has_inline_data(inode);
Eric Biggersc93d8f82019-07-22 09:26:24 -07001386 bool verity = ext4_verity_in_progress(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001387
Theodore Ts'o9bffad12009-06-17 11:48:11 -04001388 trace_ext4_journalled_write_end(inode, pos, len, copied);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001389 from = pos & (PAGE_SIZE - 1);
Nick Pigginbfc1af62007-10-16 01:25:05 -07001390 to = from + len;
1391
Curt Wohlgemuth441c8502011-08-13 11:25:18 -04001392 BUG_ON(!ext4_handle_valid(handle));
1393
Theodore Ts'o362eca72018-07-10 01:07:43 -04001394 if (inline_data) {
Theodore Ts'oeb5efbc2017-02-04 23:04:00 -05001395 ret = ext4_write_inline_data_end(inode, pos, len,
1396 copied, page);
1397 if (ret < 0) {
1398 unlock_page(page);
1399 put_page(page);
1400 goto errout;
1401 }
1402 copied = ret;
1403 } else if (unlikely(copied < len) && !PageUptodate(page)) {
Jan Kara3b136492017-01-27 14:35:38 -05001404 copied = 0;
1405 ext4_journalled_zero_new_buffers(handle, page, from, to);
1406 } else {
1407 if (unlikely(copied < len))
1408 ext4_journalled_zero_new_buffers(handle, page,
1409 from + copied, to);
Tao Ma3fdcfb62012-12-10 14:05:57 -05001410 ret = ext4_walk_page_buffers(handle, page_buffers(page), from,
Jan Kara3b136492017-01-27 14:35:38 -05001411 from + copied, &partial,
1412 write_end_fn);
Tao Ma3fdcfb62012-12-10 14:05:57 -05001413 if (!partial)
1414 SetPageUptodate(page);
1415 }
Eric Biggersc93d8f82019-07-22 09:26:24 -07001416 if (!verity)
1417 size_changed = ext4_update_inode_size(inode, pos + copied);
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05001418 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
Jan Kara2d859db2011-07-26 09:07:11 -04001419 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001420 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001421 put_page(page);
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001422
Eric Biggersc93d8f82019-07-22 09:26:24 -07001423 if (old_size < pos && !verity)
Xiaoguang Wang05726392015-02-12 23:00:17 -05001424 pagecache_isize_extended(inode, old_size, pos);
1425
Theodore Ts'o362eca72018-07-10 01:07:43 -04001426 if (size_changed || inline_data) {
Mingming Cao617ba132006-10-11 01:20:53 -07001427 ret2 = ext4_mark_inode_dirty(handle, inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001428 if (!ret)
1429 ret = ret2;
1430 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001431
Eric Biggersc93d8f82019-07-22 09:26:24 -07001432 if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001433 /* if we have allocated more blocks and copied
1434 * less. We will have blocks allocated outside
1435 * inode->i_size. So truncate them
1436 */
1437 ext4_orphan_add(handle, inode);
1438
Theodore Ts'oeb5efbc2017-02-04 23:04:00 -05001439errout:
Mingming Cao617ba132006-10-11 01:20:53 -07001440 ret2 = ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001441 if (!ret)
1442 ret = ret2;
Eric Biggersc93d8f82019-07-22 09:26:24 -07001443 if (pos + len > inode->i_size && !verity) {
Jan Karab9a42072009-12-08 21:24:33 -05001444 ext4_truncate_failed_write(inode);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001445 /*
Jan Karaffacfa72009-07-13 16:22:22 -04001446 * If truncate failed early the inode might still be
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001447 * on the orphan list; we need to make sure the inode
1448 * is removed from the orphan list in that case.
1449 */
1450 if (inode->i_nlink)
1451 ext4_orphan_del(NULL, inode);
1452 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001453
1454 return ret ? ret : copied;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001455}
Mingming Caod2a17632008-07-14 17:52:37 -04001456
Theodore Ts'o9d0be502010-01-01 02:41:30 -05001457/*
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001458 * Reserve space for a single cluster
Theodore Ts'o9d0be502010-01-01 02:41:30 -05001459 */
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001460static int ext4_da_reserve_space(struct inode *inode)
Mingming Caod2a17632008-07-14 17:52:37 -04001461{
Mingming Cao60e58e02009-01-22 18:13:05 +01001462 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001463 struct ext4_inode_info *ei = EXT4_I(inode);
Christoph Hellwig5dd40562010-03-03 09:05:00 -05001464 int ret;
Mingming Caod2a17632008-07-14 17:52:37 -04001465
Mingming Cao60e58e02009-01-22 18:13:05 +01001466 /*
Eric Sandeen72b8ab92010-05-16 11:00:00 -04001467 * We will charge metadata quota at writeout time; this saves
1468 * us from metadata over-estimation, though we may go over by
1469 * a small amount in the end. Here we just reserve for data.
Mingming Cao60e58e02009-01-22 18:13:05 +01001470 */
Aditya Kali7b415bf2011-09-09 19:04:51 -04001471 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
Christoph Hellwig5dd40562010-03-03 09:05:00 -05001472 if (ret)
1473 return ret;
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001474
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001475 spin_lock(&ei->i_block_reservation_lock);
Theodore Ts'o71d4f7d2014-07-15 06:02:38 -04001476 if (ext4_claim_free_clusters(sbi, 1, 0)) {
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001477 spin_unlock(&ei->i_block_reservation_lock);
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001478 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
Mingming Caod2a17632008-07-14 17:52:37 -04001479 return -ENOSPC;
1480 }
Theodore Ts'o9d0be502010-01-01 02:41:30 -05001481 ei->i_reserved_data_blocks++;
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001482 trace_ext4_da_reserve_space(inode);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001483 spin_unlock(&ei->i_block_reservation_lock);
Dmitry Monakhov39bc6802009-12-10 16:36:27 +00001484
Mingming Caod2a17632008-07-14 17:52:37 -04001485 return 0; /* success */
1486}
1487
Eric Whitneyf4567672018-10-01 14:33:24 -04001488void ext4_da_release_space(struct inode *inode, int to_free)
Mingming Caod2a17632008-07-14 17:52:37 -04001489{
1490 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001491 struct ext4_inode_info *ei = EXT4_I(inode);
Mingming Caod2a17632008-07-14 17:52:37 -04001492
Mingming Caocd213222008-08-19 22:16:59 -04001493 if (!to_free)
1494 return; /* Nothing to release, exit */
1495
Mingming Caod2a17632008-07-14 17:52:37 -04001496 spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
Mingming Caocd213222008-08-19 22:16:59 -04001497
Li Zefan5a58ec872010-05-17 02:00:00 -04001498 trace_ext4_da_release_space(inode, to_free);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001499 if (unlikely(to_free > ei->i_reserved_data_blocks)) {
Mingming Caocd213222008-08-19 22:16:59 -04001500 /*
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001501 * if there aren't enough reserved blocks, then the
1502 * counter is messed up somewhere. Since this
1503 * function is called from invalidate page, it's
1504 * harmless to return without any action.
Mingming Caocd213222008-08-19 22:16:59 -04001505 */
Theodore Ts'o8de5c322013-02-14 15:11:41 -05001506 ext4_warning(inode->i_sb, "ext4_da_release_space: "
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001507 "ino %lu, to_free %d with only %d reserved "
Theodore Ts'o1084f252012-03-19 23:13:43 -04001508 "data blocks", inode->i_ino, to_free,
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001509 ei->i_reserved_data_blocks);
1510 WARN_ON(1);
1511 to_free = ei->i_reserved_data_blocks;
1512 }
1513 ei->i_reserved_data_blocks -= to_free;
1514
Eric Sandeen72b8ab92010-05-16 11:00:00 -04001515 /* update fs dirty data blocks counter */
Theodore Ts'o57042652011-09-09 18:56:51 -04001516 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
Mingming Caod2a17632008-07-14 17:52:37 -04001517
Mingming Caod2a17632008-07-14 17:52:37 -04001518 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
Mingming Cao60e58e02009-01-22 18:13:05 +01001519
Aditya Kali7b415bf2011-09-09 19:04:51 -04001520 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
Mingming Caod2a17632008-07-14 17:52:37 -04001521}
1522
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001523/*
Alex Tomas64769242008-07-11 19:27:31 -04001524 * Delayed allocation stuff
1525 */
1526
Jan Kara4e7ea812013-06-04 13:17:40 -04001527struct mpage_da_data {
1528 struct inode *inode;
1529 struct writeback_control *wbc;
Jan Kara6b523df2013-06-04 13:21:11 -04001530
Jan Kara4e7ea812013-06-04 13:17:40 -04001531 pgoff_t first_page; /* The first page to write */
1532 pgoff_t next_page; /* Current page to examine */
1533 pgoff_t last_page; /* Last page to examine */
Aneesh Kumar K.V791b7f02009-01-05 21:50:43 -05001534 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04001535 * Extent to map - this can be after first_page because that can be
1536 * fully mapped. We somewhat abuse m_flags to store whether the extent
1537 * is delalloc or unwritten.
Aneesh Kumar K.V791b7f02009-01-05 21:50:43 -05001538 */
Jan Kara4e7ea812013-06-04 13:17:40 -04001539 struct ext4_map_blocks map;
1540 struct ext4_io_submit io_submit; /* IO submission data */
Jan Karadddbd6a2017-04-30 18:29:10 -04001541 unsigned int do_map:1;
Jan Kara6b8ed622020-05-25 10:12:15 +02001542 unsigned int scanned_until_end:1;
Jan Kara4e7ea812013-06-04 13:17:40 -04001543};
Alex Tomas64769242008-07-11 19:27:31 -04001544
Jan Kara4e7ea812013-06-04 13:17:40 -04001545static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1546 bool invalidate)
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001547{
1548 int nr_pages, i;
1549 pgoff_t index, end;
1550 struct pagevec pvec;
1551 struct inode *inode = mpd->inode;
1552 struct address_space *mapping = inode->i_mapping;
Jan Kara4e7ea812013-06-04 13:17:40 -04001553
1554 /* This is necessary when next_page == 0. */
1555 if (mpd->first_page >= mpd->next_page)
1556 return;
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001557
Jan Kara6b8ed622020-05-25 10:12:15 +02001558 mpd->scanned_until_end = 0;
Curt Wohlgemuthc7f59382011-02-26 12:27:52 -05001559 index = mpd->first_page;
1560 end = mpd->next_page - 1;
Jan Kara4e7ea812013-06-04 13:17:40 -04001561 if (invalidate) {
1562 ext4_lblk_t start, last;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001563 start = index << (PAGE_SHIFT - inode->i_blkbits);
1564 last = end << (PAGE_SHIFT - inode->i_blkbits);
Jan Kara4e7ea812013-06-04 13:17:40 -04001565 ext4_es_remove_extent(inode, start, last - start + 1);
1566 }
Zheng Liu51865fd2012-11-08 21:57:32 -05001567
Mel Gorman86679822017-11-15 17:37:52 -08001568 pagevec_init(&pvec);
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001569 while (index <= end) {
Jan Kara397162f2017-09-06 16:21:43 -07001570 nr_pages = pagevec_lookup_range(&pvec, mapping, &index, end);
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001571 if (nr_pages == 0)
1572 break;
1573 for (i = 0; i < nr_pages; i++) {
1574 struct page *page = pvec.pages[i];
Jan Kara2b85a612017-09-06 16:21:30 -07001575
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001576 BUG_ON(!PageLocked(page));
1577 BUG_ON(PageWriteback(page));
Jan Kara4e7ea812013-06-04 13:17:40 -04001578 if (invalidate) {
wangguang4e800c02016-09-15 11:32:46 -04001579 if (page_mapped(page))
1580 clear_page_dirty_for_io(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001581 block_invalidatepage(page, 0, PAGE_SIZE);
Jan Kara4e7ea812013-06-04 13:17:40 -04001582 ClearPageUptodate(page);
1583 }
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001584 unlock_page(page);
1585 }
Jan Kara9b1d09982010-03-03 16:19:32 -05001586 pagevec_release(&pvec);
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001587 }
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001588}
1589
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04001590static void ext4_print_free_blocks(struct inode *inode)
1591{
1592 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o92b978162012-03-19 23:41:49 -04001593 struct super_block *sb = inode->i_sb;
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001594 struct ext4_inode_info *ei = EXT4_I(inode);
Theodore Ts'o92b978162012-03-19 23:41:49 -04001595
1596 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
Theodore Ts'o5dee5432011-09-09 19:10:51 -04001597 EXT4_C2B(EXT4_SB(inode->i_sb),
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001598 ext4_count_free_clusters(sb)));
Theodore Ts'o92b978162012-03-19 23:41:49 -04001599 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1600 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001601 (long long) EXT4_C2B(EXT4_SB(sb),
Theodore Ts'o57042652011-09-09 18:56:51 -04001602 percpu_counter_sum(&sbi->s_freeclusters_counter)));
Theodore Ts'o92b978162012-03-19 23:41:49 -04001603 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001604 (long long) EXT4_C2B(EXT4_SB(sb),
Aditya Kali7b415bf2011-09-09 19:04:51 -04001605 percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
Theodore Ts'o92b978162012-03-19 23:41:49 -04001606 ext4_msg(sb, KERN_CRIT, "Block reservation details");
1607 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001608 ei->i_reserved_data_blocks);
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04001609 return;
1610}
1611
Aneesh Kumar K.Vc364b222009-06-14 17:57:10 -04001612static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh)
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04001613{
Aneesh Kumar K.Vc364b222009-06-14 17:57:10 -04001614 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04001615}
1616
Alex Tomas64769242008-07-11 19:27:31 -04001617/*
Eric Whitney0b02f4c2018-10-01 14:19:37 -04001618 * ext4_insert_delayed_block - adds a delayed block to the extents status
1619 * tree, incrementing the reserved cluster/block
1620 * count or making a pending reservation
1621 * where needed
1622 *
1623 * @inode - file containing the newly added block
1624 * @lblk - logical block to be added
1625 *
1626 * Returns 0 on success, negative error code on failure.
1627 */
1628static int ext4_insert_delayed_block(struct inode *inode, ext4_lblk_t lblk)
1629{
1630 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1631 int ret;
1632 bool allocated = false;
1633
1634 /*
1635 * If the cluster containing lblk is shared with a delayed,
1636 * written, or unwritten extent in a bigalloc file system, it's
1637 * already been accounted for and does not need to be reserved.
1638 * A pending reservation must be made for the cluster if it's
1639 * shared with a written or unwritten extent and doesn't already
1640 * have one. Written and unwritten extents can be purged from the
1641 * extents status tree if the system is under memory pressure, so
1642 * it's necessary to examine the extent tree if a search of the
1643 * extents status tree doesn't get a match.
1644 */
1645 if (sbi->s_cluster_ratio == 1) {
1646 ret = ext4_da_reserve_space(inode);
1647 if (ret != 0) /* ENOSPC */
1648 goto errout;
1649 } else { /* bigalloc */
1650 if (!ext4_es_scan_clu(inode, &ext4_es_is_delonly, lblk)) {
1651 if (!ext4_es_scan_clu(inode,
1652 &ext4_es_is_mapped, lblk)) {
1653 ret = ext4_clu_mapped(inode,
1654 EXT4_B2C(sbi, lblk));
1655 if (ret < 0)
1656 goto errout;
1657 if (ret == 0) {
1658 ret = ext4_da_reserve_space(inode);
1659 if (ret != 0) /* ENOSPC */
1660 goto errout;
1661 } else {
1662 allocated = true;
1663 }
1664 } else {
1665 allocated = true;
1666 }
1667 }
1668 }
1669
1670 ret = ext4_es_insert_delayed_block(inode, lblk, allocated);
1671
1672errout:
1673 return ret;
1674}
1675
1676/*
Aditya Kali5356f2612011-09-09 19:20:51 -04001677 * This function is grabs code from the very beginning of
1678 * ext4_map_blocks, but assumes that the caller is from delayed write
1679 * time. This function looks up the requested blocks and sets the
1680 * buffer delay bit under the protection of i_data_sem.
1681 */
1682static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1683 struct ext4_map_blocks *map,
1684 struct buffer_head *bh)
1685{
Zheng Liud100eef2013-02-18 00:29:59 -05001686 struct extent_status es;
Aditya Kali5356f2612011-09-09 19:20:51 -04001687 int retval;
1688 sector_t invalid_block = ~((sector_t) 0xffff);
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001689#ifdef ES_AGGRESSIVE_TEST
1690 struct ext4_map_blocks orig_map;
1691
1692 memcpy(&orig_map, map, sizeof(*map));
1693#endif
Aditya Kali5356f2612011-09-09 19:20:51 -04001694
1695 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
1696 invalid_block = ~0;
1697
1698 map->m_flags = 0;
Ritesh Harjani70aa1552020-05-10 11:54:55 +05301699 ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
Aditya Kali5356f2612011-09-09 19:20:51 -04001700 (unsigned long) map->m_lblk);
Zheng Liud100eef2013-02-18 00:29:59 -05001701
1702 /* Lookup extent status tree firstly */
Theodore Ts'obb5835e2019-08-11 16:32:41 -04001703 if (ext4_es_lookup_extent(inode, iblock, NULL, &es)) {
Zheng Liud100eef2013-02-18 00:29:59 -05001704 if (ext4_es_is_hole(&es)) {
1705 retval = 0;
Lukas Czernerc8b459f2014-05-12 12:55:07 -04001706 down_read(&EXT4_I(inode)->i_data_sem);
Zheng Liud100eef2013-02-18 00:29:59 -05001707 goto add_delayed;
1708 }
1709
1710 /*
1711 * Delayed extent could be allocated by fallocate.
1712 * So we need to check it.
1713 */
1714 if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
1715 map_bh(bh, inode->i_sb, invalid_block);
1716 set_buffer_new(bh);
1717 set_buffer_delay(bh);
1718 return 0;
1719 }
1720
1721 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
1722 retval = es.es_len - (iblock - es.es_lblk);
1723 if (retval > map->m_len)
1724 retval = map->m_len;
1725 map->m_len = retval;
1726 if (ext4_es_is_written(&es))
1727 map->m_flags |= EXT4_MAP_MAPPED;
1728 else if (ext4_es_is_unwritten(&es))
1729 map->m_flags |= EXT4_MAP_UNWRITTEN;
1730 else
Arnd Bergmann1e83bc82019-04-07 12:24:43 -04001731 BUG();
Zheng Liud100eef2013-02-18 00:29:59 -05001732
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001733#ifdef ES_AGGRESSIVE_TEST
1734 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1735#endif
Zheng Liud100eef2013-02-18 00:29:59 -05001736 return retval;
1737 }
1738
Aditya Kali5356f2612011-09-09 19:20:51 -04001739 /*
1740 * Try to see if we can get the block without requesting a new
1741 * file system block.
1742 */
Lukas Czernerc8b459f2014-05-12 12:55:07 -04001743 down_read(&EXT4_I(inode)->i_data_sem);
Jan Karacbd75842014-11-25 11:41:49 -05001744 if (ext4_has_inline_data(inode))
Tao Ma9c3569b2012-12-10 14:05:57 -05001745 retval = 0;
Jan Karacbd75842014-11-25 11:41:49 -05001746 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
Zheng Liu2f8e0a72014-11-25 11:44:37 -05001747 retval = ext4_ext_map_blocks(NULL, inode, map, 0);
Aditya Kali5356f2612011-09-09 19:20:51 -04001748 else
Zheng Liu2f8e0a72014-11-25 11:44:37 -05001749 retval = ext4_ind_map_blocks(NULL, inode, map, 0);
Aditya Kali5356f2612011-09-09 19:20:51 -04001750
Zheng Liud100eef2013-02-18 00:29:59 -05001751add_delayed:
Aditya Kali5356f2612011-09-09 19:20:51 -04001752 if (retval == 0) {
Zheng Liuf7fec032013-02-18 00:28:47 -05001753 int ret;
Eric Whitneyad431022018-10-01 14:10:39 -04001754
Aditya Kali5356f2612011-09-09 19:20:51 -04001755 /*
1756 * XXX: __block_prepare_write() unmaps passed block,
1757 * is it OK?
1758 */
Aditya Kali5356f2612011-09-09 19:20:51 -04001759
Eric Whitney0b02f4c2018-10-01 14:19:37 -04001760 ret = ext4_insert_delayed_block(inode, map->m_lblk);
1761 if (ret != 0) {
Zheng Liuf7fec032013-02-18 00:28:47 -05001762 retval = ret;
Zheng Liu51865fd2012-11-08 21:57:32 -05001763 goto out_unlock;
Zheng Liuf7fec032013-02-18 00:28:47 -05001764 }
Zheng Liu51865fd2012-11-08 21:57:32 -05001765
Aditya Kali5356f2612011-09-09 19:20:51 -04001766 map_bh(bh, inode->i_sb, invalid_block);
1767 set_buffer_new(bh);
1768 set_buffer_delay(bh);
Zheng Liuf7fec032013-02-18 00:28:47 -05001769 } else if (retval > 0) {
1770 int ret;
Theodore Ts'o3be78c72013-08-16 21:22:41 -04001771 unsigned int status;
Zheng Liuf7fec032013-02-18 00:28:47 -05001772
Zheng Liu44fb851d2013-07-29 12:51:42 -04001773 if (unlikely(retval != map->m_len)) {
1774 ext4_warning(inode->i_sb,
1775 "ES len assertion failed for inode "
1776 "%lu: retval %d != map->m_len %d",
1777 inode->i_ino, retval, map->m_len);
1778 WARN_ON(1);
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001779 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001780
Zheng Liuf7fec032013-02-18 00:28:47 -05001781 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1782 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1783 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1784 map->m_pblk, status);
1785 if (ret != 0)
1786 retval = ret;
Aditya Kali5356f2612011-09-09 19:20:51 -04001787 }
1788
1789out_unlock:
1790 up_read((&EXT4_I(inode)->i_data_sem));
1791
1792 return retval;
1793}
1794
1795/*
Seunghun Leed91bd2c2014-09-01 22:15:30 -04001796 * This is a special get_block_t callback which is used by
Theodore Ts'ob920c752009-05-14 00:54:29 -04001797 * ext4_da_write_begin(). It will either return mapped block or
1798 * reserve space for a single block.
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04001799 *
1800 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1801 * We also have b_blocknr = -1 and b_bdev initialized properly
1802 *
1803 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1804 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1805 * initialized properly.
Alex Tomas64769242008-07-11 19:27:31 -04001806 */
Tao Ma9c3569b2012-12-10 14:05:57 -05001807int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1808 struct buffer_head *bh, int create)
Alex Tomas64769242008-07-11 19:27:31 -04001809{
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001810 struct ext4_map_blocks map;
Alex Tomas64769242008-07-11 19:27:31 -04001811 int ret = 0;
1812
1813 BUG_ON(create == 0);
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001814 BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1815
1816 map.m_lblk = iblock;
1817 map.m_len = 1;
Alex Tomas64769242008-07-11 19:27:31 -04001818
1819 /*
1820 * first, we need to know whether the block is allocated already
1821 * preallocated blocks are unmapped but should treated
1822 * the same as allocated blocks.
1823 */
Aditya Kali5356f2612011-09-09 19:20:51 -04001824 ret = ext4_da_map_blocks(inode, iblock, &map, bh);
1825 if (ret <= 0)
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001826 return ret;
Alex Tomas64769242008-07-11 19:27:31 -04001827
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001828 map_bh(bh, inode->i_sb, map.m_pblk);
Jan Karaed8ad832016-02-19 00:18:25 -05001829 ext4_update_bh_state(bh, map.m_flags);
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001830
1831 if (buffer_unwritten(bh)) {
1832 /* A delayed write to unwritten bh should be marked
1833 * new and mapped. Mapped ensures that we don't do
1834 * get_block multiple times when we write to the same
1835 * offset and new ensures that we do proper zero out
1836 * for partial write.
1837 */
1838 set_buffer_new(bh);
Theodore Ts'oc8205632011-04-10 22:30:07 -04001839 set_buffer_mapped(bh);
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001840 }
1841 return 0;
Alex Tomas64769242008-07-11 19:27:31 -04001842}
Mingming Cao61628a32008-07-11 19:27:31 -04001843
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001844static int bget_one(handle_t *handle, struct buffer_head *bh)
1845{
1846 get_bh(bh);
1847 return 0;
1848}
1849
1850static int bput_one(handle_t *handle, struct buffer_head *bh)
1851{
1852 put_bh(bh);
1853 return 0;
1854}
1855
1856static int __ext4_journalled_writepage(struct page *page,
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001857 unsigned int len)
1858{
1859 struct address_space *mapping = page->mapping;
1860 struct inode *inode = mapping->host;
Tao Ma3fdcfb62012-12-10 14:05:57 -05001861 struct buffer_head *page_bufs = NULL;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001862 handle_t *handle = NULL;
Tao Ma3fdcfb62012-12-10 14:05:57 -05001863 int ret = 0, err = 0;
1864 int inline_data = ext4_has_inline_data(inode);
1865 struct buffer_head *inode_bh = NULL;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001866
Theodore Ts'ocb20d512010-10-27 21:30:09 -04001867 ClearPageChecked(page);
Tao Ma3fdcfb62012-12-10 14:05:57 -05001868
1869 if (inline_data) {
1870 BUG_ON(page->index != 0);
1871 BUG_ON(len > ext4_get_max_inline_size(inode));
1872 inode_bh = ext4_journalled_write_inline_data(inode, len, page);
1873 if (inode_bh == NULL)
1874 goto out;
1875 } else {
1876 page_bufs = page_buffers(page);
1877 if (!page_bufs) {
1878 BUG();
1879 goto out;
1880 }
1881 ext4_walk_page_buffers(handle, page_bufs, 0, len,
1882 NULL, bget_one);
1883 }
Theodore Ts'obdf96832015-06-12 23:45:33 -04001884 /*
1885 * We need to release the page lock before we start the
1886 * journal, so grab a reference so the page won't disappear
1887 * out from under us.
1888 */
1889 get_page(page);
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001890 unlock_page(page);
1891
Theodore Ts'o9924a922013-02-08 21:59:22 -05001892 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
1893 ext4_writepage_trans_blocks(inode));
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001894 if (IS_ERR(handle)) {
1895 ret = PTR_ERR(handle);
Theodore Ts'obdf96832015-06-12 23:45:33 -04001896 put_page(page);
1897 goto out_no_pagelock;
1898 }
1899 BUG_ON(!ext4_handle_valid(handle));
1900
1901 lock_page(page);
1902 put_page(page);
1903 if (page->mapping != mapping) {
1904 /* The page got truncated from under us */
1905 ext4_journal_stop(handle);
1906 ret = 0;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001907 goto out;
1908 }
1909
Tao Ma3fdcfb62012-12-10 14:05:57 -05001910 if (inline_data) {
Theodore Ts'o362eca72018-07-10 01:07:43 -04001911 ret = ext4_mark_inode_dirty(handle, inode);
Tao Ma3fdcfb62012-12-10 14:05:57 -05001912 } else {
1913 ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
1914 do_journal_get_write_access);
1915
1916 err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
1917 write_end_fn);
1918 }
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001919 if (ret == 0)
1920 ret = err;
Mauricio Faria de Oliveiraafb585a2020-10-05 21:48:41 -03001921 err = ext4_jbd2_inode_add_write(handle, inode, 0, len);
1922 if (ret == 0)
1923 ret = err;
Jan Kara2d859db2011-07-26 09:07:11 -04001924 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001925 err = ext4_journal_stop(handle);
1926 if (!ret)
1927 ret = err;
1928
Tao Ma3fdcfb62012-12-10 14:05:57 -05001929 if (!ext4_has_inline_data(inode))
Theodore Ts'o8c9367f2014-01-07 13:08:03 -05001930 ext4_walk_page_buffers(NULL, page_bufs, 0, len,
Tao Ma3fdcfb62012-12-10 14:05:57 -05001931 NULL, bput_one);
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05001932 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001933out:
Theodore Ts'obdf96832015-06-12 23:45:33 -04001934 unlock_page(page);
1935out_no_pagelock:
Tao Ma3fdcfb62012-12-10 14:05:57 -05001936 brelse(inode_bh);
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001937 return ret;
1938}
1939
Mingming Cao61628a32008-07-11 19:27:31 -04001940/*
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001941 * Note that we don't need to start a transaction unless we're journaling data
1942 * because we should have holes filled from ext4_page_mkwrite(). We even don't
1943 * need to file the inode to the transaction's list in ordered mode because if
1944 * we are writing back data added by write(), the inode is already there and if
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001945 * we are writing back data modified via mmap(), no one guarantees in which
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001946 * transaction the data will hit the disk. In case we are journaling data, we
1947 * cannot start transaction directly because transaction start ranks above page
1948 * lock so we have to do some magic.
1949 *
Theodore Ts'ob920c752009-05-14 00:54:29 -04001950 * This function can get called via...
Theodore Ts'o20970ba2013-06-06 14:00:46 -04001951 * - ext4_writepages after taking page lock (have journal handle)
Theodore Ts'ob920c752009-05-14 00:54:29 -04001952 * - journal_submit_inode_data_buffers (no journal handle)
Artem Bityutskiyf6463b02012-07-25 18:12:04 +03001953 * - shrink_page_list via the kswapd/direct reclaim (no journal handle)
Theodore Ts'ob920c752009-05-14 00:54:29 -04001954 * - grab_page_cache when doing write_begin (have journal handle)
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001955 *
1956 * We don't do any block allocation in this function. If we have page with
1957 * multiple blocks we need to write those buffer_heads that are mapped. This
1958 * is important for mmaped based write. So if we do with blocksize 1K
1959 * truncate(f, 1024);
1960 * a = mmap(f, 0, 4096);
1961 * a[0] = 'a';
1962 * truncate(f, 4096);
1963 * we have in the page first buffer_head mapped via page_mkwrite call back
Paul Bolle90802ed2011-12-05 13:00:34 +01001964 * but other buffer_heads would be unmapped but dirty (dirty done via the
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001965 * do_wp_page). So writepage should write the first block. If we modify
1966 * the mmap area beyond 1024 we will again get a page_fault and the
1967 * page_mkwrite callback will do the block allocation and mark the
1968 * buffer_heads mapped.
1969 *
1970 * We redirty the page if we have any buffer_heads that is either delay or
1971 * unwritten in the page.
1972 *
1973 * We can get recursively called as show below.
1974 *
1975 * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
1976 * ext4_writepage()
1977 *
1978 * But since we don't do any block allocation we should not deadlock.
1979 * Page also have the dirty flag cleared so we don't get recurive page_lock.
Mingming Cao61628a32008-07-11 19:27:31 -04001980 */
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001981static int ext4_writepage(struct page *page,
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001982 struct writeback_control *wbc)
Alex Tomas64769242008-07-11 19:27:31 -04001983{
Jan Karaf8bec372013-01-28 12:55:08 -05001984 int ret = 0;
Mingming Cao61628a32008-07-11 19:27:31 -04001985 loff_t size;
Theodore Ts'o498e5f22008-11-05 00:14:04 -05001986 unsigned int len;
Jiaying Zhang744692d2010-03-04 16:14:02 -05001987 struct buffer_head *page_bufs = NULL;
Mingming Cao61628a32008-07-11 19:27:31 -04001988 struct inode *inode = page->mapping->host;
Jan Kara36ade452013-01-28 09:30:52 -05001989 struct ext4_io_submit io_submit;
Namjae Jeon1c8349a2014-05-12 08:12:25 -04001990 bool keep_towrite = false;
Alex Tomas64769242008-07-11 19:27:31 -04001991
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05001992 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
yangerkunc2a559b2020-02-26 12:10:02 +08001993 inode->i_mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05001994 unlock_page(page);
1995 return -EIO;
1996 }
1997
Lukas Czernera9c667f2011-06-06 09:51:52 -04001998 trace_ext4_writepage(page);
Aneesh Kumar K.Vf0e6c982008-07-11 19:27:31 -04001999 size = i_size_read(inode);
Eric Biggersc93d8f82019-07-22 09:26:24 -07002000 if (page->index == size >> PAGE_SHIFT &&
2001 !ext4_verity_in_progress(inode))
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002002 len = size & ~PAGE_MASK;
Aneesh Kumar K.Vf0e6c982008-07-11 19:27:31 -04002003 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002004 len = PAGE_SIZE;
Alex Tomas64769242008-07-11 19:27:31 -04002005
Theodore Ts'oa42afc52010-10-27 21:30:09 -04002006 page_bufs = page_buffers(page);
Aneesh Kumar K.Vc364b222009-06-14 17:57:10 -04002007 /*
Jan Karafe386132013-01-28 21:06:42 -05002008 * We cannot do block allocation or other extent handling in this
2009 * function. If there are buffers needing that, we have to redirty
2010 * the page. But we may reach here when we do a journal commit via
2011 * journal_submit_inode_data_buffers() and in that case we must write
2012 * allocated buffers to achieve data=ordered mode guarantees.
Theodore Ts'occcd1472015-10-03 10:49:23 -04002013 *
2014 * Also, if there is only one buffer per page (the fs block
2015 * size == the page size), if one buffer needs block
2016 * allocation or needs to modify the extent tree to clear the
2017 * unwritten flag, we know that the page can't be written at
2018 * all, so we might as well refuse the write immediately.
2019 * Unfortunately if the block size != page size, we can't as
2020 * easily detect this case using ext4_walk_page_buffers(), but
2021 * for the extremely common case, this is an optimization that
2022 * skips a useless round trip through ext4_bio_write_page().
Aneesh Kumar K.Vcd1aac32008-07-11 19:27:31 -04002023 */
Tao Maf19d5872012-12-10 14:05:51 -05002024 if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL,
2025 ext4_bh_delay_or_unwritten)) {
Jan Karaf8bec372013-01-28 12:55:08 -05002026 redirty_page_for_writepage(wbc, page);
Theodore Ts'occcd1472015-10-03 10:49:23 -04002027 if ((current->flags & PF_MEMALLOC) ||
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002028 (inode->i_sb->s_blocksize == PAGE_SIZE)) {
Jan Karafe386132013-01-28 21:06:42 -05002029 /*
2030 * For memory cleaning there's no point in writing only
2031 * some buffers. So just bail out. Warn if we came here
2032 * from direct reclaim.
2033 */
2034 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
2035 == PF_MEMALLOC);
Aneesh Kumar K.Vf0e6c982008-07-11 19:27:31 -04002036 unlock_page(page);
2037 return 0;
2038 }
Namjae Jeon1c8349a2014-05-12 08:12:25 -04002039 keep_towrite = true;
Theodore Ts'oa42afc52010-10-27 21:30:09 -04002040 }
Alex Tomas64769242008-07-11 19:27:31 -04002041
Theodore Ts'ocb20d512010-10-27 21:30:09 -04002042 if (PageChecked(page) && ext4_should_journal_data(inode))
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04002043 /*
2044 * It's mmapped pagecache. Add buffers and journal it. There
2045 * doesn't seem much point in redirtying the page here.
2046 */
Wu Fengguang3f0ca302009-11-24 11:15:44 -05002047 return __ext4_journalled_writepage(page, len);
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04002048
Jan Kara97a851e2013-06-04 11:58:58 -04002049 ext4_io_submit_init(&io_submit, wbc);
2050 io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
2051 if (!io_submit.io_end) {
2052 redirty_page_for_writepage(wbc, page);
2053 unlock_page(page);
2054 return -ENOMEM;
2055 }
Namjae Jeon1c8349a2014-05-12 08:12:25 -04002056 ret = ext4_bio_write_page(&io_submit, page, len, wbc, keep_towrite);
Jan Kara36ade452013-01-28 09:30:52 -05002057 ext4_io_submit(&io_submit);
Jan Kara97a851e2013-06-04 11:58:58 -04002058 /* Drop io_end reference we got from init */
2059 ext4_put_io_end_defer(io_submit.io_end);
Alex Tomas64769242008-07-11 19:27:31 -04002060 return ret;
2061}
2062
Jan Kara5f1132b2013-08-17 10:02:33 -04002063static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
2064{
2065 int len;
Jan Karaa056bda2017-05-26 17:45:45 -04002066 loff_t size;
Jan Kara5f1132b2013-08-17 10:02:33 -04002067 int err;
2068
2069 BUG_ON(page->index != mpd->first_page);
Jan Karaa056bda2017-05-26 17:45:45 -04002070 clear_page_dirty_for_io(page);
2071 /*
2072 * We have to be very careful here! Nothing protects writeback path
2073 * against i_size changes and the page can be writeably mapped into
2074 * page tables. So an application can be growing i_size and writing
2075 * data through mmap while writeback runs. clear_page_dirty_for_io()
2076 * write-protects our page in page tables and the page cannot get
2077 * written to again until we release page lock. So only after
2078 * clear_page_dirty_for_io() we are safe to sample i_size for
2079 * ext4_bio_write_page() to zero-out tail of the written page. We rely
2080 * on the barrier provided by TestClearPageDirty in
2081 * clear_page_dirty_for_io() to make sure i_size is really sampled only
2082 * after page tables are updated.
2083 */
2084 size = i_size_read(mpd->inode);
Eric Biggersc93d8f82019-07-22 09:26:24 -07002085 if (page->index == size >> PAGE_SHIFT &&
2086 !ext4_verity_in_progress(mpd->inode))
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002087 len = size & ~PAGE_MASK;
Jan Kara5f1132b2013-08-17 10:02:33 -04002088 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002089 len = PAGE_SIZE;
Namjae Jeon1c8349a2014-05-12 08:12:25 -04002090 err = ext4_bio_write_page(&mpd->io_submit, page, len, mpd->wbc, false);
Jan Kara5f1132b2013-08-17 10:02:33 -04002091 if (!err)
2092 mpd->wbc->nr_to_write--;
2093 mpd->first_page++;
2094
2095 return err;
2096}
2097
Ritesh Harjani6db07462020-05-10 11:54:51 +05302098#define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
Jan Kara4e7ea812013-06-04 13:17:40 -04002099
Mingming Cao61628a32008-07-11 19:27:31 -04002100/*
Jan Karafffb2732013-06-04 13:01:11 -04002101 * mballoc gives us at most this number of blocks...
2102 * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
Anatol Pomozov70261f52013-08-28 14:40:12 -04002103 * The rest of mballoc seems to handle chunks up to full group size.
Mingming Cao61628a32008-07-11 19:27:31 -04002104 */
Jan Karafffb2732013-06-04 13:01:11 -04002105#define MAX_WRITEPAGES_EXTENT_LEN 2048
Mingming Cao525f4ed2008-08-19 22:15:58 -04002106
Jan Karafffb2732013-06-04 13:01:11 -04002107/*
Jan Kara4e7ea812013-06-04 13:17:40 -04002108 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2109 *
2110 * @mpd - extent of blocks
2111 * @lblk - logical number of the block in the file
Jan Kara09930042013-08-17 09:57:56 -04002112 * @bh - buffer head we want to add to the extent
Jan Kara4e7ea812013-06-04 13:17:40 -04002113 *
Jan Kara09930042013-08-17 09:57:56 -04002114 * The function is used to collect contig. blocks in the same state. If the
2115 * buffer doesn't require mapping for writeback and we haven't started the
2116 * extent of buffers to map yet, the function returns 'true' immediately - the
2117 * caller can write the buffer right away. Otherwise the function returns true
2118 * if the block has been added to the extent, false if the block couldn't be
2119 * added.
Jan Kara4e7ea812013-06-04 13:17:40 -04002120 */
Jan Kara09930042013-08-17 09:57:56 -04002121static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2122 struct buffer_head *bh)
Jan Kara4e7ea812013-06-04 13:17:40 -04002123{
2124 struct ext4_map_blocks *map = &mpd->map;
2125
Jan Kara09930042013-08-17 09:57:56 -04002126 /* Buffer that doesn't need mapping for writeback? */
2127 if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2128 (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2129 /* So far no extent to map => we write the buffer right away */
2130 if (map->m_len == 0)
2131 return true;
2132 return false;
2133 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002134
2135 /* First block in the extent? */
2136 if (map->m_len == 0) {
Jan Karadddbd6a2017-04-30 18:29:10 -04002137 /* We cannot map unless handle is started... */
2138 if (!mpd->do_map)
2139 return false;
Jan Kara4e7ea812013-06-04 13:17:40 -04002140 map->m_lblk = lblk;
2141 map->m_len = 1;
Jan Kara09930042013-08-17 09:57:56 -04002142 map->m_flags = bh->b_state & BH_FLAGS;
2143 return true;
Jan Kara4e7ea812013-06-04 13:17:40 -04002144 }
2145
Jan Kara09930042013-08-17 09:57:56 -04002146 /* Don't go larger than mballoc is willing to allocate */
2147 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2148 return false;
2149
Jan Kara4e7ea812013-06-04 13:17:40 -04002150 /* Can we merge the block to our big extent? */
2151 if (lblk == map->m_lblk + map->m_len &&
Jan Kara09930042013-08-17 09:57:56 -04002152 (bh->b_state & BH_FLAGS) == map->m_flags) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002153 map->m_len++;
Jan Kara09930042013-08-17 09:57:56 -04002154 return true;
Jan Kara4e7ea812013-06-04 13:17:40 -04002155 }
Jan Kara09930042013-08-17 09:57:56 -04002156 return false;
Jan Kara4e7ea812013-06-04 13:17:40 -04002157}
2158
Jan Kara5f1132b2013-08-17 10:02:33 -04002159/*
2160 * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2161 *
2162 * @mpd - extent of blocks for mapping
2163 * @head - the first buffer in the page
2164 * @bh - buffer we should start processing from
2165 * @lblk - logical number of the block in the file corresponding to @bh
2166 *
2167 * Walk through page buffers from @bh upto @head (exclusive) and either submit
2168 * the page for IO if all buffers in this page were mapped and there's no
2169 * accumulated extent of buffers to map or add buffers in the page to the
2170 * extent of buffers to map. The function returns 1 if the caller can continue
2171 * by processing the next page, 0 if it should stop adding buffers to the
2172 * extent to map because we cannot extend it anymore. It can also return value
2173 * < 0 in case of error during IO submission.
2174 */
2175static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2176 struct buffer_head *head,
2177 struct buffer_head *bh,
2178 ext4_lblk_t lblk)
Jan Kara4e7ea812013-06-04 13:17:40 -04002179{
2180 struct inode *inode = mpd->inode;
Jan Kara5f1132b2013-08-17 10:02:33 -04002181 int err;
Fabian Frederick93407472017-02-27 14:28:32 -08002182 ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
Jan Kara4e7ea812013-06-04 13:17:40 -04002183 >> inode->i_blkbits;
2184
Eric Biggersc93d8f82019-07-22 09:26:24 -07002185 if (ext4_verity_in_progress(inode))
2186 blocks = EXT_MAX_BLOCKS;
2187
Jan Kara4e7ea812013-06-04 13:17:40 -04002188 do {
2189 BUG_ON(buffer_locked(bh));
2190
Jan Kara09930042013-08-17 09:57:56 -04002191 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002192 /* Found extent to map? */
2193 if (mpd->map.m_len)
Jan Kara5f1132b2013-08-17 10:02:33 -04002194 return 0;
Jan Karadddbd6a2017-04-30 18:29:10 -04002195 /* Buffer needs mapping and handle is not started? */
2196 if (!mpd->do_map)
2197 return 0;
Jan Kara09930042013-08-17 09:57:56 -04002198 /* Everything mapped so far and we hit EOF */
Jan Kara5f1132b2013-08-17 10:02:33 -04002199 break;
Jan Kara4e7ea812013-06-04 13:17:40 -04002200 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002201 } while (lblk++, (bh = bh->b_this_page) != head);
Jan Kara5f1132b2013-08-17 10:02:33 -04002202 /* So far everything mapped? Submit the page for IO. */
2203 if (mpd->map.m_len == 0) {
2204 err = mpage_submit_page(mpd, head->b_page);
2205 if (err < 0)
2206 return err;
2207 }
Jan Kara6b8ed622020-05-25 10:12:15 +02002208 if (lblk >= blocks) {
2209 mpd->scanned_until_end = 1;
2210 return 0;
2211 }
2212 return 1;
Jan Kara4e7ea812013-06-04 13:17:40 -04002213}
2214
2215/*
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302216 * mpage_process_page - update page buffers corresponding to changed extent and
2217 * may submit fully mapped page for IO
2218 *
2219 * @mpd - description of extent to map, on return next extent to map
2220 * @m_lblk - logical block mapping.
2221 * @m_pblk - corresponding physical mapping.
2222 * @map_bh - determines on return whether this page requires any further
2223 * mapping or not.
2224 * Scan given page buffers corresponding to changed extent and update buffer
2225 * state according to new extent state.
2226 * We map delalloc buffers to their physical location, clear unwritten bits.
2227 * If the given page is not fully mapped, we update @map to the next extent in
2228 * the given page that needs mapping & return @map_bh as true.
2229 */
2230static int mpage_process_page(struct mpage_da_data *mpd, struct page *page,
2231 ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
2232 bool *map_bh)
2233{
2234 struct buffer_head *head, *bh;
2235 ext4_io_end_t *io_end = mpd->io_submit.io_end;
2236 ext4_lblk_t lblk = *m_lblk;
2237 ext4_fsblk_t pblock = *m_pblk;
2238 int err = 0;
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302239 int blkbits = mpd->inode->i_blkbits;
2240 ssize_t io_end_size = 0;
2241 struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302242
2243 bh = head = page_buffers(page);
2244 do {
2245 if (lblk < mpd->map.m_lblk)
2246 continue;
2247 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2248 /*
2249 * Buffer after end of mapped extent.
2250 * Find next buffer in the page to map.
2251 */
2252 mpd->map.m_len = 0;
2253 mpd->map.m_flags = 0;
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302254 io_end_vec->size += io_end_size;
2255 io_end_size = 0;
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302256
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302257 err = mpage_process_page_bufs(mpd, head, bh, lblk);
2258 if (err > 0)
2259 err = 0;
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302260 if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2261 io_end_vec = ext4_alloc_io_end_vec(io_end);
Ritesh Harjani4d06bfb2019-11-06 15:08:09 +05302262 if (IS_ERR(io_end_vec)) {
2263 err = PTR_ERR(io_end_vec);
2264 goto out;
2265 }
Ritesh Harjanid1e18b82020-10-08 20:32:48 +05302266 io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302267 }
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302268 *map_bh = true;
2269 goto out;
2270 }
2271 if (buffer_delay(bh)) {
2272 clear_buffer_delay(bh);
2273 bh->b_blocknr = pblock++;
2274 }
2275 clear_buffer_unwritten(bh);
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302276 io_end_size += (1 << blkbits);
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302277 } while (lblk++, (bh = bh->b_this_page) != head);
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302278
2279 io_end_vec->size += io_end_size;
2280 io_end_size = 0;
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302281 *map_bh = false;
2282out:
2283 *m_lblk = lblk;
2284 *m_pblk = pblock;
2285 return err;
2286}
2287
2288/*
Jan Kara4e7ea812013-06-04 13:17:40 -04002289 * mpage_map_buffers - update buffers corresponding to changed extent and
2290 * submit fully mapped pages for IO
2291 *
2292 * @mpd - description of extent to map, on return next extent to map
2293 *
2294 * Scan buffers corresponding to changed extent (we expect corresponding pages
2295 * to be already locked) and update buffer state according to new extent state.
2296 * We map delalloc buffers to their physical location, clear unwritten bits,
Lukas Czerner556615d2014-04-20 23:45:47 -04002297 * and mark buffers as uninit when we perform writes to unwritten extents
Jan Kara4e7ea812013-06-04 13:17:40 -04002298 * and do extent conversion after IO is finished. If the last page is not fully
2299 * mapped, we update @map to the next extent in the last page that needs
2300 * mapping. Otherwise we submit the page for IO.
2301 */
2302static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2303{
2304 struct pagevec pvec;
2305 int nr_pages, i;
2306 struct inode *inode = mpd->inode;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002307 int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
Jan Kara4e7ea812013-06-04 13:17:40 -04002308 pgoff_t start, end;
2309 ext4_lblk_t lblk;
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302310 ext4_fsblk_t pblock;
Jan Kara4e7ea812013-06-04 13:17:40 -04002311 int err;
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302312 bool map_bh = false;
Jan Kara4e7ea812013-06-04 13:17:40 -04002313
2314 start = mpd->map.m_lblk >> bpp_bits;
2315 end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2316 lblk = start << bpp_bits;
2317 pblock = mpd->map.m_pblk;
2318
Mel Gorman86679822017-11-15 17:37:52 -08002319 pagevec_init(&pvec);
Jan Kara4e7ea812013-06-04 13:17:40 -04002320 while (start <= end) {
Jan Kara2b85a612017-09-06 16:21:30 -07002321 nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping,
Jan Kara397162f2017-09-06 16:21:43 -07002322 &start, end);
Jan Kara4e7ea812013-06-04 13:17:40 -04002323 if (nr_pages == 0)
2324 break;
2325 for (i = 0; i < nr_pages; i++) {
2326 struct page *page = pvec.pages[i];
2327
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302328 err = mpage_process_page(mpd, page, &lblk, &pblock,
2329 &map_bh);
Jan Kara4e7ea812013-06-04 13:17:40 -04002330 /*
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302331 * If map_bh is true, means page may require further bh
2332 * mapping, or maybe the page was submitted for IO.
2333 * So we return to call further extent mapping.
Jan Kara4e7ea812013-06-04 13:17:40 -04002334 */
Jason Yan39c0ae12020-04-20 12:29:18 +08002335 if (err < 0 || map_bh)
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302336 goto out;
Jan Kara4e7ea812013-06-04 13:17:40 -04002337 /* Page fully mapped - let IO run! */
2338 err = mpage_submit_page(mpd, page);
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302339 if (err < 0)
2340 goto out;
Jan Kara4e7ea812013-06-04 13:17:40 -04002341 }
2342 pagevec_release(&pvec);
2343 }
2344 /* Extent fully mapped and matches with page boundary. We are done. */
2345 mpd->map.m_len = 0;
2346 mpd->map.m_flags = 0;
2347 return 0;
Ritesh Harjani2943fdb2019-10-16 13:07:09 +05302348out:
2349 pagevec_release(&pvec);
2350 return err;
Jan Kara4e7ea812013-06-04 13:17:40 -04002351}
2352
2353static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2354{
2355 struct inode *inode = mpd->inode;
2356 struct ext4_map_blocks *map = &mpd->map;
2357 int get_blocks_flags;
Lukas Czerner090f32e2014-04-20 23:44:47 -04002358 int err, dioread_nolock;
Jan Kara4e7ea812013-06-04 13:17:40 -04002359
2360 trace_ext4_da_write_pages_extent(inode, map);
2361 /*
2362 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
Lukas Czerner556615d2014-04-20 23:45:47 -04002363 * to convert an unwritten extent to be initialized (in the case
Jan Kara4e7ea812013-06-04 13:17:40 -04002364 * where we have written into one or more preallocated blocks). It is
2365 * possible that we're going to need more metadata blocks than
2366 * previously reserved. However we must not fail because we're in
2367 * writeback and there is nothing we can do about it so it might result
2368 * in data loss. So use reserved blocks to allocate metadata if
2369 * possible.
2370 *
Theodore Ts'o754cfed2014-09-04 18:08:22 -04002371 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
2372 * the blocks in question are delalloc blocks. This indicates
2373 * that the blocks and quotas has already been checked when
2374 * the data was copied into the page cache.
Jan Kara4e7ea812013-06-04 13:17:40 -04002375 */
2376 get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
Jan Karaee0876b2016-04-24 00:56:08 -04002377 EXT4_GET_BLOCKS_METADATA_NOFAIL |
2378 EXT4_GET_BLOCKS_IO_SUBMIT;
Lukas Czerner090f32e2014-04-20 23:44:47 -04002379 dioread_nolock = ext4_should_dioread_nolock(inode);
2380 if (dioread_nolock)
Jan Kara4e7ea812013-06-04 13:17:40 -04002381 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
Ritesh Harjani6db07462020-05-10 11:54:51 +05302382 if (map->m_flags & BIT(BH_Delay))
Jan Kara4e7ea812013-06-04 13:17:40 -04002383 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2384
2385 err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2386 if (err < 0)
2387 return err;
Lukas Czerner090f32e2014-04-20 23:44:47 -04002388 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
Jan Kara6b523df2013-06-04 13:21:11 -04002389 if (!mpd->io_submit.io_end->handle &&
2390 ext4_handle_valid(handle)) {
2391 mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2392 handle->h_rsv_handle = NULL;
2393 }
Jan Kara3613d222013-06-04 13:19:34 -04002394 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
Jan Kara6b523df2013-06-04 13:21:11 -04002395 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002396
2397 BUG_ON(map->m_len == 0);
Jan Kara4e7ea812013-06-04 13:17:40 -04002398 return 0;
2399}
2400
2401/*
2402 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2403 * mpd->len and submit pages underlying it for IO
2404 *
2405 * @handle - handle for journal operations
2406 * @mpd - extent to map
Jan Kara7534e852013-10-16 08:26:08 -04002407 * @give_up_on_write - we set this to true iff there is a fatal error and there
2408 * is no hope of writing the data. The caller should discard
2409 * dirty pages to avoid infinite loops.
Jan Kara4e7ea812013-06-04 13:17:40 -04002410 *
2411 * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2412 * delayed, blocks are allocated, if it is unwritten, we may need to convert
2413 * them to initialized or split the described range from larger unwritten
2414 * extent. Note that we need not map all the described range since allocation
2415 * can return less blocks or the range is covered by more unwritten extents. We
2416 * cannot map more because we are limited by reserved transaction credits. On
2417 * the other hand we always make sure that the last touched page is fully
2418 * mapped so that it can be written out (and thus forward progress is
2419 * guaranteed). After mapping we submit all mapped pages for IO.
2420 */
2421static int mpage_map_and_submit_extent(handle_t *handle,
Theodore Ts'ocb530542013-07-01 08:12:40 -04002422 struct mpage_da_data *mpd,
2423 bool *give_up_on_write)
Jan Kara4e7ea812013-06-04 13:17:40 -04002424{
2425 struct inode *inode = mpd->inode;
2426 struct ext4_map_blocks *map = &mpd->map;
2427 int err;
2428 loff_t disksize;
Dmitry Monakhov66031202014-08-27 18:40:03 -04002429 int progress = 0;
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302430 ext4_io_end_t *io_end = mpd->io_submit.io_end;
Ritesh Harjani4d06bfb2019-11-06 15:08:09 +05302431 struct ext4_io_end_vec *io_end_vec;
Jan Kara4e7ea812013-06-04 13:17:40 -04002432
Ritesh Harjani4d06bfb2019-11-06 15:08:09 +05302433 io_end_vec = ext4_alloc_io_end_vec(io_end);
2434 if (IS_ERR(io_end_vec))
2435 return PTR_ERR(io_end_vec);
Ritesh Harjanic8cc8812019-10-16 13:07:10 +05302436 io_end_vec->offset = ((loff_t)map->m_lblk) << inode->i_blkbits;
Jan Kara27d7c4e2013-07-05 21:57:22 -04002437 do {
Jan Kara4e7ea812013-06-04 13:17:40 -04002438 err = mpage_map_one_extent(handle, mpd);
2439 if (err < 0) {
2440 struct super_block *sb = inode->i_sb;
2441
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05002442 if (ext4_forced_shutdown(EXT4_SB(sb)) ||
2443 EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)
Theodore Ts'ocb530542013-07-01 08:12:40 -04002444 goto invalidate_dirty_pages;
Jan Kara4e7ea812013-06-04 13:17:40 -04002445 /*
Theodore Ts'ocb530542013-07-01 08:12:40 -04002446 * Let the uper layers retry transient errors.
2447 * In the case of ENOSPC, if ext4_count_free_blocks()
2448 * is non-zero, a commit should free up blocks.
Jan Kara4e7ea812013-06-04 13:17:40 -04002449 */
Theodore Ts'ocb530542013-07-01 08:12:40 -04002450 if ((err == -ENOMEM) ||
Dmitry Monakhov66031202014-08-27 18:40:03 -04002451 (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2452 if (progress)
2453 goto update_disksize;
Theodore Ts'ocb530542013-07-01 08:12:40 -04002454 return err;
Dmitry Monakhov66031202014-08-27 18:40:03 -04002455 }
Theodore Ts'ocb530542013-07-01 08:12:40 -04002456 ext4_msg(sb, KERN_CRIT,
2457 "Delayed block allocation failed for "
2458 "inode %lu at logical offset %llu with"
2459 " max blocks %u with error %d",
2460 inode->i_ino,
2461 (unsigned long long)map->m_lblk,
2462 (unsigned)map->m_len, -err);
2463 ext4_msg(sb, KERN_CRIT,
2464 "This should not happen!! Data will "
2465 "be lost\n");
2466 if (err == -ENOSPC)
2467 ext4_print_free_blocks(inode);
2468 invalidate_dirty_pages:
2469 *give_up_on_write = true;
Jan Kara4e7ea812013-06-04 13:17:40 -04002470 return err;
2471 }
Dmitry Monakhov66031202014-08-27 18:40:03 -04002472 progress = 1;
Jan Kara4e7ea812013-06-04 13:17:40 -04002473 /*
2474 * Update buffer state, submit mapped pages, and get us new
2475 * extent to map
2476 */
2477 err = mpage_map_and_submit_buffers(mpd);
2478 if (err < 0)
Dmitry Monakhov66031202014-08-27 18:40:03 -04002479 goto update_disksize;
Jan Kara27d7c4e2013-07-05 21:57:22 -04002480 } while (map->m_len);
Jan Kara4e7ea812013-06-04 13:17:40 -04002481
Dmitry Monakhov66031202014-08-27 18:40:03 -04002482update_disksize:
Theodore Ts'o622cad12014-04-11 10:35:17 -04002483 /*
2484 * Update on-disk size after IO is submitted. Races with
2485 * truncate are avoided by checking i_size under i_data_sem.
2486 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002487 disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
Qian Cai35df4292020-02-07 09:29:11 -05002488 if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002489 int err2;
Theodore Ts'o622cad12014-04-11 10:35:17 -04002490 loff_t i_size;
Jan Kara4e7ea812013-06-04 13:17:40 -04002491
Theodore Ts'o622cad12014-04-11 10:35:17 -04002492 down_write(&EXT4_I(inode)->i_data_sem);
2493 i_size = i_size_read(inode);
2494 if (disksize > i_size)
2495 disksize = i_size;
2496 if (disksize > EXT4_I(inode)->i_disksize)
2497 EXT4_I(inode)->i_disksize = disksize;
Theodore Ts'o622cad12014-04-11 10:35:17 -04002498 up_write(&EXT4_I(inode)->i_data_sem);
Theodore Ts'ob907f2d2017-01-11 22:14:49 -05002499 err2 = ext4_mark_inode_dirty(handle, inode);
Theodore Ts'o878520a2019-11-19 21:54:15 -05002500 if (err2) {
Theodore Ts'o54d3adb2020-03-28 19:33:43 -04002501 ext4_error_err(inode->i_sb, -err2,
2502 "Failed to mark inode %lu dirty",
2503 inode->i_ino);
Theodore Ts'o878520a2019-11-19 21:54:15 -05002504 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002505 if (!err)
2506 err = err2;
2507 }
2508 return err;
2509}
2510
2511/*
Jan Karafffb2732013-06-04 13:01:11 -04002512 * Calculate the total number of credits to reserve for one writepages
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002513 * iteration. This is called from ext4_writepages(). We map an extent of
Anatol Pomozov70261f52013-08-28 14:40:12 -04002514 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
Jan Karafffb2732013-06-04 13:01:11 -04002515 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
2516 * bpp - 1 blocks in bpp different extents.
2517 */
Mingming Cao525f4ed2008-08-19 22:15:58 -04002518static int ext4_da_writepages_trans_blocks(struct inode *inode)
2519{
Jan Karafffb2732013-06-04 13:01:11 -04002520 int bpp = ext4_journal_blocks_per_page(inode);
Mingming Cao525f4ed2008-08-19 22:15:58 -04002521
Jan Karafffb2732013-06-04 13:01:11 -04002522 return ext4_meta_trans_blocks(inode,
2523 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
Mingming Cao525f4ed2008-08-19 22:15:58 -04002524}
Mingming Cao61628a32008-07-11 19:27:31 -04002525
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002526/*
Jan Kara4e7ea812013-06-04 13:17:40 -04002527 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2528 * and underlying extent to map
2529 *
2530 * @mpd - where to look for pages
2531 *
2532 * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2533 * IO immediately. When we find a page which isn't mapped we start accumulating
2534 * extent of buffers underlying these pages that needs mapping (formed by
2535 * either delayed or unwritten buffers). We also lock the pages containing
2536 * these buffers. The extent found is returned in @mpd structure (starting at
2537 * mpd->lblk with length mpd->len blocks).
2538 *
2539 * Note that this function can attach bios to one io_end structure which are
2540 * neither logically nor physically contiguous. Although it may seem as an
2541 * unnecessary complication, it is actually inevitable in blocksize < pagesize
2542 * case as we need to track IO to all buffers underlying a page in one io_end.
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002543 */
Jan Kara4e7ea812013-06-04 13:17:40 -04002544static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002545{
Jan Kara4e7ea812013-06-04 13:17:40 -04002546 struct address_space *mapping = mpd->inode->i_mapping;
2547 struct pagevec pvec;
2548 unsigned int nr_pages;
Ming Leiaeac5892013-10-17 18:56:16 -04002549 long left = mpd->wbc->nr_to_write;
Jan Kara4e7ea812013-06-04 13:17:40 -04002550 pgoff_t index = mpd->first_page;
2551 pgoff_t end = mpd->last_page;
Matthew Wilcox10bbd232017-12-05 17:30:38 -05002552 xa_mark_t tag;
Jan Kara4e7ea812013-06-04 13:17:40 -04002553 int i, err = 0;
2554 int blkbits = mpd->inode->i_blkbits;
2555 ext4_lblk_t lblk;
2556 struct buffer_head *head;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002557
Jan Kara4e7ea812013-06-04 13:17:40 -04002558 if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
Eric Sandeen5b41d922010-10-27 21:30:13 -04002559 tag = PAGECACHE_TAG_TOWRITE;
2560 else
2561 tag = PAGECACHE_TAG_DIRTY;
2562
Mel Gorman86679822017-11-15 17:37:52 -08002563 pagevec_init(&pvec);
Jan Kara4e7ea812013-06-04 13:17:40 -04002564 mpd->map.m_len = 0;
2565 mpd->next_page = index;
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002566 while (index <= end) {
Jan Karadc7f3e82017-11-15 17:34:44 -08002567 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
Jan Kara67fd7072017-11-15 17:35:19 -08002568 tag);
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002569 if (nr_pages == 0)
Jan Kara6b8ed622020-05-25 10:12:15 +02002570 break;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002571
2572 for (i = 0; i < nr_pages; i++) {
2573 struct page *page = pvec.pages[i];
2574
2575 /*
Ming Leiaeac5892013-10-17 18:56:16 -04002576 * Accumulated enough dirty pages? This doesn't apply
2577 * to WB_SYNC_ALL mode. For integrity sync we have to
2578 * keep going because someone may be concurrently
2579 * dirtying pages, and we might have synced a lot of
2580 * newly appeared dirty pages, but have not synced all
2581 * of the old dirty pages.
2582 */
2583 if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
2584 goto out;
2585
Jan Kara4e7ea812013-06-04 13:17:40 -04002586 /* If we can't merge this page, we are done. */
2587 if (mpd->map.m_len > 0 && mpd->next_page != page->index)
2588 goto out;
Theodore Ts'o78aaced2011-02-26 14:09:14 -05002589
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002590 lock_page(page);
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002591 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04002592 * If the page is no longer dirty, or its mapping no
2593 * longer corresponds to inode we are writing (which
2594 * means it has been truncated or invalidated), or the
2595 * page is already under writeback and we are not doing
2596 * a data integrity writeback, skip the page
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002597 */
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002598 if (!PageDirty(page) ||
2599 (PageWriteback(page) &&
Jan Kara4e7ea812013-06-04 13:17:40 -04002600 (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002601 unlikely(page->mapping != mapping)) {
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002602 unlock_page(page);
2603 continue;
2604 }
2605
Darrick J. Wong7cb1a532011-05-18 13:53:20 -04002606 wait_on_page_writeback(page);
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002607 BUG_ON(PageWriteback(page));
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002608
Jan Kara4e7ea812013-06-04 13:17:40 -04002609 if (mpd->map.m_len == 0)
Theodore Ts'o8eb9e5c2011-02-26 14:07:31 -05002610 mpd->first_page = page->index;
Theodore Ts'o8eb9e5c2011-02-26 14:07:31 -05002611 mpd->next_page = page->index + 1;
Jan Karaf8bec372013-01-28 12:55:08 -05002612 /* Add all dirty buffers to mpd */
Jan Kara4e7ea812013-06-04 13:17:40 -04002613 lblk = ((ext4_lblk_t)page->index) <<
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002614 (PAGE_SHIFT - blkbits);
Jan Karaf8bec372013-01-28 12:55:08 -05002615 head = page_buffers(page);
Jan Kara5f1132b2013-08-17 10:02:33 -04002616 err = mpage_process_page_bufs(mpd, head, head, lblk);
2617 if (err <= 0)
Jan Kara4e7ea812013-06-04 13:17:40 -04002618 goto out;
Jan Kara5f1132b2013-08-17 10:02:33 -04002619 err = 0;
Ming Leiaeac5892013-10-17 18:56:16 -04002620 left--;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002621 }
2622 pagevec_release(&pvec);
2623 cond_resched();
2624 }
Jan Kara6b8ed622020-05-25 10:12:15 +02002625 mpd->scanned_until_end = 1;
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002626 return 0;
Theodore Ts'o8eb9e5c2011-02-26 14:07:31 -05002627out:
2628 pagevec_release(&pvec);
Jan Kara4e7ea812013-06-04 13:17:40 -04002629 return err;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002630}
2631
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002632static int ext4_writepages(struct address_space *mapping,
2633 struct writeback_control *wbc)
Alex Tomas64769242008-07-11 19:27:31 -04002634{
Jan Kara4e7ea812013-06-04 13:17:40 -04002635 pgoff_t writeback_index = 0;
2636 long nr_to_write = wbc->nr_to_write;
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002637 int range_whole = 0;
Jan Kara4e7ea812013-06-04 13:17:40 -04002638 int cycled = 1;
Mingming Cao61628a32008-07-11 19:27:31 -04002639 handle_t *handle = NULL;
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04002640 struct mpage_da_data mpd;
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04002641 struct inode *inode = mapping->host;
Jan Kara6b523df2013-06-04 13:21:11 -04002642 int needed_blocks, rsv_blocks = 0, ret = 0;
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04002643 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
Shaohua Li1bce63d12011-10-18 10:55:51 -04002644 struct blk_plug plug;
Theodore Ts'ocb530542013-07-01 08:12:40 -04002645 bool give_up_on_write = false;
Mingming Cao61628a32008-07-11 19:27:31 -04002646
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05002647 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2648 return -EIO;
2649
Eric Biggersbbd559372020-02-19 10:30:46 -08002650 percpu_down_read(&sbi->s_writepages_rwsem);
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002651 trace_ext4_writepages(inode, wbc);
Theodore Ts'oba80b102009-01-03 20:03:21 -05002652
Mingming Cao61628a32008-07-11 19:27:31 -04002653 /*
2654 * No pages to write? This is mainly a kludge to avoid starting
2655 * a transaction for special inodes like journal inode on last iput()
2656 * because that could violate lock ordering on umount
2657 */
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002658 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
Ming Leibbf023c72013-10-30 07:27:16 -04002659 goto out_writepages;
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002660
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002661 if (ext4_should_journal_data(inode)) {
Goldwyn Rodrigues043d20d2018-03-26 01:32:50 -04002662 ret = generic_writepages(mapping, wbc);
Ming Leibbf023c72013-10-30 07:27:16 -04002663 goto out_writepages;
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002664 }
2665
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002666 /*
2667 * If the filesystem has aborted, it is read-only, so return
2668 * right away instead of dumping stack traces later on that
2669 * will obscure the real source of the problem. We test
Linus Torvalds1751e8a2017-11-27 13:05:09 -08002670 * EXT4_MF_FS_ABORTED instead of sb->s_flag's SB_RDONLY because
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002671 * the latter could be true if the filesystem is mounted
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002672 * read-only, and in that case, ext4_writepages should
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002673 * *never* be called, so if that ever happens, we would want
2674 * the stack trace.
2675 */
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05002676 if (unlikely(ext4_forced_shutdown(EXT4_SB(mapping->host->i_sb)) ||
2677 sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) {
Ming Leibbf023c72013-10-30 07:27:16 -04002678 ret = -EROFS;
2679 goto out_writepages;
2680 }
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002681
Jan Kara4e7ea812013-06-04 13:17:40 -04002682 /*
2683 * If we have inline data and arrive here, it means that
2684 * we will soon create the block for the 1st page, so
2685 * we'd better clear the inline data here.
2686 */
2687 if (ext4_has_inline_data(inode)) {
2688 /* Just inode will be modified... */
2689 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2690 if (IS_ERR(handle)) {
2691 ret = PTR_ERR(handle);
2692 goto out_writepages;
2693 }
2694 BUG_ON(ext4_test_inode_state(inode,
2695 EXT4_STATE_MAY_INLINE_DATA));
2696 ext4_destroy_inline_data(handle, inode);
2697 ext4_journal_stop(handle);
2698 }
2699
yangerkun4e343232019-08-11 16:27:41 -04002700 if (ext4_should_dioread_nolock(inode)) {
2701 /*
2702 * We may need to convert up to one extent per block in
2703 * the page and we may dirty the inode.
2704 */
2705 rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,
2706 PAGE_SIZE >> inode->i_blkbits);
2707 }
2708
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002709 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2710 range_whole = 1;
Mingming Cao61628a32008-07-11 19:27:31 -04002711
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002712 if (wbc->range_cyclic) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002713 writeback_index = mapping->writeback_index;
2714 if (writeback_index)
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002715 cycled = 0;
Jan Kara4e7ea812013-06-04 13:17:40 -04002716 mpd.first_page = writeback_index;
2717 mpd.last_page = -1;
Eric Sandeen5b41d922010-10-27 21:30:13 -04002718 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002719 mpd.first_page = wbc->range_start >> PAGE_SHIFT;
2720 mpd.last_page = wbc->range_end >> PAGE_SHIFT;
Eric Sandeen5b41d922010-10-27 21:30:13 -04002721 }
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002722
Jan Kara4e7ea812013-06-04 13:17:40 -04002723 mpd.inode = inode;
2724 mpd.wbc = wbc;
2725 ext4_io_submit_init(&mpd.io_submit, wbc);
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002726retry:
Wu Fengguang6e6938b2010-06-06 10:38:15 -06002727 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Jan Kara4e7ea812013-06-04 13:17:40 -04002728 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
Shaohua Li1bce63d12011-10-18 10:55:51 -04002729 blk_start_plug(&plug);
Jan Karadddbd6a2017-04-30 18:29:10 -04002730
2731 /*
2732 * First writeback pages that don't need mapping - we can avoid
2733 * starting a transaction unnecessarily and also avoid being blocked
2734 * in the block layer on device congestion while having transaction
2735 * started.
2736 */
2737 mpd.do_map = 0;
Jan Kara6b8ed622020-05-25 10:12:15 +02002738 mpd.scanned_until_end = 0;
Jan Karadddbd6a2017-04-30 18:29:10 -04002739 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2740 if (!mpd.io_submit.io_end) {
2741 ret = -ENOMEM;
2742 goto unplug;
2743 }
2744 ret = mpage_prepare_extent_to_map(&mpd);
Xiaoguang Wanga297b2f2019-02-10 23:53:21 -05002745 /* Unlock pages we didn't use */
2746 mpage_release_unused_pages(&mpd, false);
Jan Karadddbd6a2017-04-30 18:29:10 -04002747 /* Submit prepared bio */
2748 ext4_io_submit(&mpd.io_submit);
2749 ext4_put_io_end_defer(mpd.io_submit.io_end);
2750 mpd.io_submit.io_end = NULL;
Jan Karadddbd6a2017-04-30 18:29:10 -04002751 if (ret < 0)
2752 goto unplug;
2753
Jan Kara6b8ed622020-05-25 10:12:15 +02002754 while (!mpd.scanned_until_end && wbc->nr_to_write > 0) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002755 /* For each extent of pages we use new io_end */
2756 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2757 if (!mpd.io_submit.io_end) {
2758 ret = -ENOMEM;
2759 break;
2760 }
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002761
2762 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04002763 * We have two constraints: We find one extent to map and we
2764 * must always write out whole page (makes a difference when
2765 * blocksize < pagesize) so that we don't block on IO when we
2766 * try to write out the rest of the page. Journalled mode is
2767 * not supported by delalloc.
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002768 */
2769 BUG_ON(ext4_should_journal_data(inode));
Mingming Cao525f4ed2008-08-19 22:15:58 -04002770 needed_blocks = ext4_da_writepages_trans_blocks(inode);
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002771
Jan Kara4e7ea812013-06-04 13:17:40 -04002772 /* start a new transaction */
Jan Kara6b523df2013-06-04 13:21:11 -04002773 handle = ext4_journal_start_with_reserve(inode,
2774 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
Mingming Cao61628a32008-07-11 19:27:31 -04002775 if (IS_ERR(handle)) {
2776 ret = PTR_ERR(handle);
Theodore Ts'o16939182009-09-26 17:43:59 -04002777 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
Curt Wohlgemuthfbe845d2010-05-16 13:00:00 -04002778 "%ld pages, ino %lu; err %d", __func__,
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002779 wbc->nr_to_write, inode->i_ino, ret);
Jan Kara4e7ea812013-06-04 13:17:40 -04002780 /* Release allocated io_end */
2781 ext4_put_io_end(mpd.io_submit.io_end);
Jan Karadddbd6a2017-04-30 18:29:10 -04002782 mpd.io_submit.io_end = NULL;
Jan Kara4e7ea812013-06-04 13:17:40 -04002783 break;
Mingming Cao61628a32008-07-11 19:27:31 -04002784 }
Jan Karadddbd6a2017-04-30 18:29:10 -04002785 mpd.do_map = 1;
Theodore Ts'of63e60052009-02-23 16:42:39 -05002786
Jan Kara4e7ea812013-06-04 13:17:40 -04002787 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
2788 ret = mpage_prepare_extent_to_map(&mpd);
Jan Kara6b8ed622020-05-25 10:12:15 +02002789 if (!ret && mpd.map.m_len)
2790 ret = mpage_map_and_submit_extent(handle, &mpd,
Theodore Ts'ocb530542013-07-01 08:12:40 -04002791 &give_up_on_write);
Jan Kara646caa92016-07-04 10:14:01 -04002792 /*
2793 * Caution: If the handle is synchronous,
2794 * ext4_journal_stop() can wait for transaction commit
2795 * to finish which may depend on writeback of pages to
2796 * complete or on page lock to be released. In that
Randy Dunlapb483bb72020-08-04 19:48:50 -07002797 * case, we have to wait until after we have
Jan Kara646caa92016-07-04 10:14:01 -04002798 * submitted all the IO, released page locks we hold,
2799 * and dropped io_end reference (for extent conversion
2800 * to be able to complete) before stopping the handle.
2801 */
2802 if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2803 ext4_journal_stop(handle);
2804 handle = NULL;
Jan Karadddbd6a2017-04-30 18:29:10 -04002805 mpd.do_map = 0;
Jan Kara646caa92016-07-04 10:14:01 -04002806 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002807 /* Unlock pages we didn't use */
Theodore Ts'ocb530542013-07-01 08:12:40 -04002808 mpage_release_unused_pages(&mpd, give_up_on_write);
Xiaoguang Wanga297b2f2019-02-10 23:53:21 -05002809 /* Submit prepared bio */
2810 ext4_io_submit(&mpd.io_submit);
2811
Jan Kara646caa92016-07-04 10:14:01 -04002812 /*
2813 * Drop our io_end reference we got from init. We have
2814 * to be careful and use deferred io_end finishing if
2815 * we are still holding the transaction as we can
2816 * release the last reference to io_end which may end
2817 * up doing unwritten extent conversion.
2818 */
2819 if (handle) {
2820 ext4_put_io_end_defer(mpd.io_submit.io_end);
2821 ext4_journal_stop(handle);
2822 } else
2823 ext4_put_io_end(mpd.io_submit.io_end);
Jan Karadddbd6a2017-04-30 18:29:10 -04002824 mpd.io_submit.io_end = NULL;
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04002825
Jan Kara4e7ea812013-06-04 13:17:40 -04002826 if (ret == -ENOSPC && sbi->s_journal) {
2827 /*
2828 * Commit the transaction which would
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002829 * free blocks released in the transaction
2830 * and try again
2831 */
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04002832 jbd2_journal_force_commit_nested(sbi->s_journal);
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002833 ret = 0;
Jan Kara4e7ea812013-06-04 13:17:40 -04002834 continue;
2835 }
2836 /* Fatal error - ENOMEM, EIO... */
2837 if (ret)
Mingming Cao61628a32008-07-11 19:27:31 -04002838 break;
Mingming Cao61628a32008-07-11 19:27:31 -04002839 }
Jan Karadddbd6a2017-04-30 18:29:10 -04002840unplug:
Shaohua Li1bce63d12011-10-18 10:55:51 -04002841 blk_finish_plug(&plug);
Jan Kara9c12a832013-09-16 08:24:26 -04002842 if (!ret && !cycled && wbc->nr_to_write > 0) {
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002843 cycled = 1;
Jan Kara4e7ea812013-06-04 13:17:40 -04002844 mpd.last_page = writeback_index - 1;
2845 mpd.first_page = 0;
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002846 goto retry;
2847 }
Mingming Cao61628a32008-07-11 19:27:31 -04002848
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002849 /* Update index */
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002850 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2851 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04002852 * Set the writeback_index so that range_cyclic
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002853 * mode will write it back later
2854 */
Jan Kara4e7ea812013-06-04 13:17:40 -04002855 mapping->writeback_index = mpd.first_page;
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002856
Mingming Cao61628a32008-07-11 19:27:31 -04002857out_writepages:
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002858 trace_ext4_writepages_result(inode, wbc, ret,
2859 nr_to_write - wbc->nr_to_write);
Eric Biggersbbd559372020-02-19 10:30:46 -08002860 percpu_up_read(&sbi->s_writepages_rwsem);
Mingming Cao61628a32008-07-11 19:27:31 -04002861 return ret;
Alex Tomas64769242008-07-11 19:27:31 -04002862}
2863
Dan Williams5f0663b2017-12-21 12:25:11 -08002864static int ext4_dax_writepages(struct address_space *mapping,
2865 struct writeback_control *wbc)
2866{
2867 int ret;
2868 long nr_to_write = wbc->nr_to_write;
2869 struct inode *inode = mapping->host;
2870 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2871
2872 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2873 return -EIO;
2874
Eric Biggersbbd559372020-02-19 10:30:46 -08002875 percpu_down_read(&sbi->s_writepages_rwsem);
Dan Williams5f0663b2017-12-21 12:25:11 -08002876 trace_ext4_writepages(inode, wbc);
2877
Vivek Goyal3f666c52020-01-03 13:33:07 -05002878 ret = dax_writeback_mapping_range(mapping, sbi->s_daxdev, wbc);
Dan Williams5f0663b2017-12-21 12:25:11 -08002879 trace_ext4_writepages_result(inode, wbc, ret,
2880 nr_to_write - wbc->nr_to_write);
Eric Biggersbbd559372020-02-19 10:30:46 -08002881 percpu_up_read(&sbi->s_writepages_rwsem);
Dan Williams5f0663b2017-12-21 12:25:11 -08002882 return ret;
2883}
2884
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002885static int ext4_nonda_switch(struct super_block *sb)
2886{
Eric Whitney5c1ff332013-04-09 09:27:31 -04002887 s64 free_clusters, dirty_clusters;
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002888 struct ext4_sb_info *sbi = EXT4_SB(sb);
2889
2890 /*
2891 * switch to non delalloc mode if we are running low
2892 * on free block. The free block accounting via percpu
Eric Dumazet179f7eb2009-01-06 14:41:04 -08002893 * counters can get slightly wrong with percpu_counter_batch getting
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002894 * accumulated on each CPU without updating global counters
2895 * Delalloc need an accurate free block accounting. So switch
2896 * to non delalloc when we are near to error range.
2897 */
Eric Whitney5c1ff332013-04-09 09:27:31 -04002898 free_clusters =
2899 percpu_counter_read_positive(&sbi->s_freeclusters_counter);
2900 dirty_clusters =
2901 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
Theodore Ts'o00d4e732012-09-19 22:42:36 -04002902 /*
2903 * Start pushing delalloc when 1/2 of free blocks are dirty.
2904 */
Eric Whitney5c1ff332013-04-09 09:27:31 -04002905 if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
Miao Xie10ee27a2013-01-10 13:47:57 +08002906 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
Theodore Ts'o00d4e732012-09-19 22:42:36 -04002907
Eric Whitney5c1ff332013-04-09 09:27:31 -04002908 if (2 * free_clusters < 3 * dirty_clusters ||
2909 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002910 /*
Eric Sandeenc8afb442009-12-23 07:58:12 -05002911 * free block count is less than 150% of dirty blocks
2912 * or free blocks is less than watermark
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002913 */
2914 return 1;
2915 }
2916 return 0;
2917}
2918
Eric Sandeen0ff89472014-10-11 19:51:17 -04002919/* We always reserve for an inode update; the superblock could be there too */
2920static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len)
2921{
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04002922 if (likely(ext4_has_feature_large_file(inode->i_sb)))
Eric Sandeen0ff89472014-10-11 19:51:17 -04002923 return 1;
2924
2925 if (pos + len <= 0x7fffffffULL)
2926 return 1;
2927
2928 /* We might need to update the superblock to set LARGE_FILE */
2929 return 2;
2930}
2931
Alex Tomas64769242008-07-11 19:27:31 -04002932static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04002933 loff_t pos, unsigned len, unsigned flags,
2934 struct page **pagep, void **fsdata)
Alex Tomas64769242008-07-11 19:27:31 -04002935{
Eric Sandeen72b8ab92010-05-16 11:00:00 -04002936 int ret, retries = 0;
Alex Tomas64769242008-07-11 19:27:31 -04002937 struct page *page;
2938 pgoff_t index;
Alex Tomas64769242008-07-11 19:27:31 -04002939 struct inode *inode = mapping->host;
2940 handle_t *handle;
2941
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05002942 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2943 return -EIO;
2944
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002945 index = pos >> PAGE_SHIFT;
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002946
Eric Biggersc93d8f82019-07-22 09:26:24 -07002947 if (ext4_nonda_switch(inode->i_sb) || S_ISLNK(inode->i_mode) ||
2948 ext4_verity_in_progress(inode)) {
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002949 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
2950 return ext4_write_begin(file, mapping, pos,
2951 len, flags, pagep, fsdata);
2952 }
2953 *fsdata = (void *)0;
Theodore Ts'o9bffad12009-06-17 11:48:11 -04002954 trace_ext4_da_write_begin(inode, pos, len, flags);
Tao Ma9c3569b2012-12-10 14:05:57 -05002955
2956 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
2957 ret = ext4_da_write_inline_data_begin(mapping, inode,
2958 pos, len, flags,
2959 pagep, fsdata);
2960 if (ret < 0)
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002961 return ret;
2962 if (ret == 1)
2963 return 0;
Tao Ma9c3569b2012-12-10 14:05:57 -05002964 }
2965
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002966 /*
2967 * grab_cache_page_write_begin() can take a long time if the
2968 * system is thrashing due to memory pressure, or if the page
2969 * is being written back. So grab it first before we start
2970 * the transaction handle. This also allows us to allocate
2971 * the page (if needed) without using GFP_NOFS.
2972 */
2973retry_grab:
2974 page = grab_cache_page_write_begin(mapping, index, flags);
2975 if (!page)
2976 return -ENOMEM;
2977 unlock_page(page);
2978
Alex Tomas64769242008-07-11 19:27:31 -04002979 /*
2980 * With delayed allocation, we don't log the i_disksize update
2981 * if there is delayed block allocation. But we still need
2982 * to journalling the i_disksize update if writes to the end
2983 * of file which has an already mapped buffer.
2984 */
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002985retry_journal:
Eric Sandeen0ff89472014-10-11 19:51:17 -04002986 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
2987 ext4_da_write_credits(inode, pos, len));
Alex Tomas64769242008-07-11 19:27:31 -04002988 if (IS_ERR(handle)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002989 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002990 return PTR_ERR(handle);
Alex Tomas64769242008-07-11 19:27:31 -04002991 }
2992
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002993 lock_page(page);
2994 if (page->mapping != mapping) {
2995 /* The page got truncated from under us */
2996 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002997 put_page(page);
Eric Sandeend5a0d4f2008-08-02 18:51:06 -04002998 ext4_journal_stop(handle);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002999 goto retry_grab;
Eric Sandeend5a0d4f2008-08-02 18:51:06 -04003000 }
Theodore Ts'o47564bf2013-02-09 09:24:14 -05003001 /* In case writeback began while the page was unlocked */
Dmitry Monakhov7afe5aa2013-08-28 14:30:47 -04003002 wait_for_stable_page(page);
Alex Tomas64769242008-07-11 19:27:31 -04003003
Chandan Rajendra643fa962018-12-12 15:20:12 +05303004#ifdef CONFIG_FS_ENCRYPTION
Michael Halcrow2058f832015-04-12 00:55:10 -04003005 ret = ext4_block_write_begin(page, pos, len,
3006 ext4_da_get_block_prep);
3007#else
Christoph Hellwig6e1db882010-06-04 11:29:57 +02003008 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
Michael Halcrow2058f832015-04-12 00:55:10 -04003009#endif
Alex Tomas64769242008-07-11 19:27:31 -04003010 if (ret < 0) {
3011 unlock_page(page);
3012 ext4_journal_stop(handle);
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04003013 /*
3014 * block_write_begin may have instantiated a few blocks
3015 * outside i_size. Trim these off again. Don't need
3016 * i_size_read because we hold i_mutex.
3017 */
3018 if (pos + len > inode->i_size)
Jan Karab9a42072009-12-08 21:24:33 -05003019 ext4_truncate_failed_write(inode);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05003020
3021 if (ret == -ENOSPC &&
3022 ext4_should_retry_alloc(inode->i_sb, &retries))
3023 goto retry_journal;
3024
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003025 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05003026 return ret;
Alex Tomas64769242008-07-11 19:27:31 -04003027 }
3028
Theodore Ts'o47564bf2013-02-09 09:24:14 -05003029 *pagep = page;
Alex Tomas64769242008-07-11 19:27:31 -04003030 return ret;
3031}
3032
Mingming Cao632eaea2008-07-11 19:27:31 -04003033/*
3034 * Check if we should update i_disksize
3035 * when write to the end of file but not require block allocation
3036 */
3037static int ext4_da_should_update_i_disksize(struct page *page,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04003038 unsigned long offset)
Mingming Cao632eaea2008-07-11 19:27:31 -04003039{
3040 struct buffer_head *bh;
3041 struct inode *inode = page->mapping->host;
3042 unsigned int idx;
3043 int i;
3044
3045 bh = page_buffers(page);
3046 idx = offset >> inode->i_blkbits;
3047
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04003048 for (i = 0; i < idx; i++)
Mingming Cao632eaea2008-07-11 19:27:31 -04003049 bh = bh->b_this_page;
3050
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04003051 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
Mingming Cao632eaea2008-07-11 19:27:31 -04003052 return 0;
3053 return 1;
3054}
3055
Alex Tomas64769242008-07-11 19:27:31 -04003056static int ext4_da_write_end(struct file *file,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04003057 struct address_space *mapping,
3058 loff_t pos, unsigned len, unsigned copied,
3059 struct page *page, void *fsdata)
Alex Tomas64769242008-07-11 19:27:31 -04003060{
3061 struct inode *inode = mapping->host;
3062 int ret = 0, ret2;
3063 handle_t *handle = ext4_journal_current_handle();
3064 loff_t new_i_size;
Mingming Cao632eaea2008-07-11 19:27:31 -04003065 unsigned long start, end;
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04003066 int write_mode = (int)(unsigned long)fsdata;
3067
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003068 if (write_mode == FALL_BACK_TO_NONDELALLOC)
3069 return ext4_write_end(file, mapping, pos,
3070 len, copied, page, fsdata);
Mingming Cao632eaea2008-07-11 19:27:31 -04003071
Theodore Ts'o9bffad12009-06-17 11:48:11 -04003072 trace_ext4_da_write_end(inode, pos, len, copied);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003073 start = pos & (PAGE_SIZE - 1);
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04003074 end = start + copied - 1;
Alex Tomas64769242008-07-11 19:27:31 -04003075
3076 /*
3077 * generic_write_end() will run mark_inode_dirty() if i_size
3078 * changes. So let's piggyback the i_disksize mark_inode_dirty
3079 * into that.
3080 */
Alex Tomas64769242008-07-11 19:27:31 -04003081 new_i_size = pos + copied;
Andrea Arcangeliea51d132011-12-13 21:41:15 -05003082 if (copied && new_i_size > EXT4_I(inode)->i_disksize) {
Tao Ma9c3569b2012-12-10 14:05:57 -05003083 if (ext4_has_inline_data(inode) ||
3084 ext4_da_should_update_i_disksize(page, end)) {
Dmitry Monakhovee124d22014-08-30 23:34:06 -04003085 ext4_update_i_disksize(inode, new_i_size);
Aneesh Kumar K.Vcf17fea2008-09-13 13:06:18 -04003086 /* We need to mark inode dirty even if
3087 * new_i_size is less that inode->i_size
3088 * bu greater than i_disksize.(hint delalloc)
3089 */
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07003090 ret = ext4_mark_inode_dirty(handle, inode);
Alex Tomas64769242008-07-11 19:27:31 -04003091 }
Mingming Cao632eaea2008-07-11 19:27:31 -04003092 }
Tao Ma9c3569b2012-12-10 14:05:57 -05003093
3094 if (write_mode != CONVERT_INLINE_DATA &&
3095 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3096 ext4_has_inline_data(inode))
3097 ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied,
3098 page);
3099 else
3100 ret2 = generic_write_end(file, mapping, pos, len, copied,
Alex Tomas64769242008-07-11 19:27:31 -04003101 page, fsdata);
Tao Ma9c3569b2012-12-10 14:05:57 -05003102
Alex Tomas64769242008-07-11 19:27:31 -04003103 copied = ret2;
3104 if (ret2 < 0)
3105 ret = ret2;
3106 ret2 = ext4_journal_stop(handle);
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07003107 if (unlikely(ret2 && !ret))
Alex Tomas64769242008-07-11 19:27:31 -04003108 ret = ret2;
3109
3110 return ret ? ret : copied;
3111}
3112
Theodore Ts'occd25062009-02-26 01:04:07 -05003113/*
3114 * Force all delayed allocation blocks to be allocated for a given inode.
3115 */
3116int ext4_alloc_da_blocks(struct inode *inode)
3117{
Theodore Ts'ofb40ba02009-09-16 19:30:40 -04003118 trace_ext4_alloc_da_blocks(inode);
3119
Theodore Ts'o71d4f7d2014-07-15 06:02:38 -04003120 if (!EXT4_I(inode)->i_reserved_data_blocks)
Theodore Ts'occd25062009-02-26 01:04:07 -05003121 return 0;
3122
3123 /*
3124 * We do something simple for now. The filemap_flush() will
3125 * also start triggering a write of the data blocks, which is
3126 * not strictly speaking necessary (and for users of
3127 * laptop_mode, not even desirable). However, to do otherwise
3128 * would require replicating code paths in:
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04003129 *
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003130 * ext4_writepages() ->
Theodore Ts'occd25062009-02-26 01:04:07 -05003131 * write_cache_pages() ---> (via passed in callback function)
3132 * __mpage_da_writepage() -->
3133 * mpage_add_bh_to_extent()
3134 * mpage_da_map_blocks()
3135 *
3136 * The problem is that write_cache_pages(), located in
3137 * mm/page-writeback.c, marks pages clean in preparation for
3138 * doing I/O, which is not desirable if we're not planning on
3139 * doing I/O at all.
3140 *
3141 * We could call write_cache_pages(), and then redirty all of
Wu Fengguang380cf092010-11-11 19:23:29 +08003142 * the pages by calling redirty_page_for_writepage() but that
Theodore Ts'occd25062009-02-26 01:04:07 -05003143 * would be ugly in the extreme. So instead we would need to
3144 * replicate parts of the code in the above functions,
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003145 * simplifying them because we wouldn't actually intend to
Theodore Ts'occd25062009-02-26 01:04:07 -05003146 * write out the pages, but rather only collect contiguous
3147 * logical block extents, call the multi-block allocator, and
3148 * then update the buffer heads with the block allocations.
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04003149 *
Theodore Ts'occd25062009-02-26 01:04:07 -05003150 * For now, though, we'll cheat by calling filemap_flush(),
3151 * which will map the blocks, and start the I/O, but not
3152 * actually wait for the I/O to complete.
3153 */
3154 return filemap_flush(inode->i_mapping);
3155}
Alex Tomas64769242008-07-11 19:27:31 -04003156
3157/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003158 * bmap() is special. It gets used by applications such as lilo and by
3159 * the swapper to find the on-disk block of a specific piece of data.
3160 *
3161 * Naturally, this is dangerous if the block concerned is still in the
Mingming Cao617ba132006-10-11 01:20:53 -07003162 * journal. If somebody makes a swapfile on an ext4 data-journaling
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003163 * filesystem and enables swap, then they may get a nasty shock when the
3164 * data getting swapped to that swapfile suddenly gets overwritten by
3165 * the original zero's written out previously to the journal and
3166 * awaiting writeback in the kernel's buffer cache.
3167 *
3168 * So, if we see any bmap calls here on a modified, data-journaled file,
3169 * take extra steps to flush any blocks which might be in the cache.
3170 */
Mingming Cao617ba132006-10-11 01:20:53 -07003171static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003172{
3173 struct inode *inode = mapping->host;
3174 journal_t *journal;
3175 int err;
3176
Tao Ma46c7f252012-12-10 14:04:52 -05003177 /*
3178 * We can get here for an inline file via the FIBMAP ioctl
3179 */
3180 if (ext4_has_inline_data(inode))
3181 return 0;
3182
Alex Tomas64769242008-07-11 19:27:31 -04003183 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3184 test_opt(inode->i_sb, DELALLOC)) {
3185 /*
3186 * With delalloc we want to sync the file
3187 * so that we can make sure we allocate
3188 * blocks for file
3189 */
3190 filemap_write_and_wait(mapping);
3191 }
3192
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05003193 if (EXT4_JOURNAL(inode) &&
3194 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003195 /*
3196 * This is a REALLY heavyweight approach, but the use of
3197 * bmap on dirty files is expected to be extremely rare:
3198 * only if we run lilo or swapon on a freshly made file
3199 * do we expect this to happen.
3200 *
3201 * (bmap requires CAP_SYS_RAWIO so this does not
3202 * represent an unprivileged user DOS attack --- we'd be
3203 * in trouble if mortal users could trigger this path at
3204 * will.)
3205 *
Mingming Cao617ba132006-10-11 01:20:53 -07003206 * NB. EXT4_STATE_JDATA is not set on files other than
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003207 * regular files. If somebody wants to bmap a directory
3208 * or symlink and gets confused because the buffer
3209 * hasn't yet been flushed to disk, they deserve
3210 * everything they get.
3211 */
3212
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05003213 ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
Mingming Cao617ba132006-10-11 01:20:53 -07003214 journal = EXT4_JOURNAL(inode);
Mingming Caodab291a2006-10-11 01:21:01 -07003215 jbd2_journal_lock_updates(journal);
3216 err = jbd2_journal_flush(journal);
3217 jbd2_journal_unlock_updates(journal);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003218
3219 if (err)
3220 return 0;
3221 }
3222
Ritesh Harjaniac58e4f2020-02-28 14:56:56 +05303223 return iomap_bmap(mapping, block, &ext4_iomap_ops);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003224}
3225
Mingming Cao617ba132006-10-11 01:20:53 -07003226static int ext4_readpage(struct file *file, struct page *page)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003227{
Tao Ma46c7f252012-12-10 14:04:52 -05003228 int ret = -EAGAIN;
3229 struct inode *inode = page->mapping->host;
3230
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003231 trace_ext4_readpage(page);
Tao Ma46c7f252012-12-10 14:04:52 -05003232
3233 if (ext4_has_inline_data(inode))
3234 ret = ext4_readpage_inline(inode, page);
3235
3236 if (ret == -EAGAIN)
Matthew Wilcox (Oracle)a07f6242020-06-01 21:47:20 -07003237 return ext4_mpage_readpages(inode, NULL, page);
Tao Ma46c7f252012-12-10 14:04:52 -05003238
3239 return ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003240}
3241
Matthew Wilcox (Oracle)6311f91f2020-06-01 21:47:16 -07003242static void ext4_readahead(struct readahead_control *rac)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003243{
Matthew Wilcox (Oracle)6311f91f2020-06-01 21:47:16 -07003244 struct inode *inode = rac->mapping->host;
Tao Ma46c7f252012-12-10 14:04:52 -05003245
Matthew Wilcox (Oracle)6311f91f2020-06-01 21:47:16 -07003246 /* If the file has inline data, no need to do readahead. */
Tao Ma46c7f252012-12-10 14:04:52 -05003247 if (ext4_has_inline_data(inode))
Matthew Wilcox (Oracle)6311f91f2020-06-01 21:47:16 -07003248 return;
Tao Ma46c7f252012-12-10 14:04:52 -05003249
Matthew Wilcox (Oracle)a07f6242020-06-01 21:47:20 -07003250 ext4_mpage_readpages(inode, rac, NULL);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003251}
3252
Lukas Czernerd47992f2013-05-21 23:17:23 -04003253static void ext4_invalidatepage(struct page *page, unsigned int offset,
3254 unsigned int length)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003255{
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003256 trace_ext4_invalidatepage(page, offset, length);
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003257
Jan Kara4520fb32012-12-25 13:28:54 -05003258 /* No journalling happens on data buffers when this function is used */
3259 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page)));
3260
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003261 block_invalidatepage(page, offset, length);
Jan Kara4520fb32012-12-25 13:28:54 -05003262}
3263
Jan Kara53e87262012-12-25 13:29:52 -05003264static int __ext4_journalled_invalidatepage(struct page *page,
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003265 unsigned int offset,
3266 unsigned int length)
Jan Kara4520fb32012-12-25 13:28:54 -05003267{
3268 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3269
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003270 trace_ext4_journalled_invalidatepage(page, offset, length);
Jan Kara4520fb32012-12-25 13:28:54 -05003271
Jiaying Zhang744692d2010-03-04 16:14:02 -05003272 /*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003273 * If it's a full truncate we just forget about the pending dirtying
3274 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003275 if (offset == 0 && length == PAGE_SIZE)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003276 ClearPageChecked(page);
3277
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003278 return jbd2_journal_invalidatepage(journal, page, offset, length);
Jan Kara53e87262012-12-25 13:29:52 -05003279}
3280
3281/* Wrapper for aops... */
3282static void ext4_journalled_invalidatepage(struct page *page,
Lukas Czernerd47992f2013-05-21 23:17:23 -04003283 unsigned int offset,
3284 unsigned int length)
Jan Kara53e87262012-12-25 13:29:52 -05003285{
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003286 WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003287}
3288
Mingming Cao617ba132006-10-11 01:20:53 -07003289static int ext4_releasepage(struct page *page, gfp_t wait)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003290{
Mingming Cao617ba132006-10-11 01:20:53 -07003291 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003292
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003293 trace_ext4_releasepage(page);
3294
Jan Karae1c36592013-03-10 22:19:00 -04003295 /* Page has dirty journalled data -> cannot release */
3296 if (PageChecked(page))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003297 return 0;
Frank Mayhar03901312009-01-07 00:06:22 -05003298 if (journal)
zhangyi (F)529a7812020-06-20 10:54:27 +08003299 return jbd2_journal_try_to_free_buffers(journal, page);
Frank Mayhar03901312009-01-07 00:06:22 -05003300 else
3301 return try_to_free_buffers(page);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003302}
3303
Jan Karab8a61762017-11-01 16:36:45 +01003304static bool ext4_inode_datasync_dirty(struct inode *inode)
3305{
3306 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3307
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07003308 if (journal) {
3309 if (jbd2_transaction_committed(journal,
3310 EXT4_I(inode)->i_datasync_tid))
3311 return true;
3312 return atomic_read(&EXT4_SB(inode->i_sb)->s_fc_subtid) >=
3313 EXT4_I(inode)->i_fc_committed_subtid;
3314 }
3315
Jan Karab8a61762017-11-01 16:36:45 +01003316 /* Any metadata buffers to write? */
3317 if (!list_empty(&inode->i_mapping->private_list))
3318 return true;
3319 return inode->i_state & I_DIRTY_DATASYNC;
3320}
3321
Matthew Bobrowskic8fdfe292019-11-05 22:59:56 +11003322static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
3323 struct ext4_map_blocks *map, loff_t offset,
3324 loff_t length)
Jan Kara364443c2016-11-20 17:36:06 -05003325{
Matthew Bobrowskic8fdfe292019-11-05 22:59:56 +11003326 u8 blkbits = inode->i_blkbits;
3327
3328 /*
3329 * Writes that span EOF might trigger an I/O size update on completion,
3330 * so consider them to be dirty for the purpose of O_DSYNC, even if
3331 * there is no other metadata changes being made or are pending.
3332 */
3333 iomap->flags = 0;
3334 if (ext4_inode_datasync_dirty(inode) ||
3335 offset + length > i_size_read(inode))
3336 iomap->flags |= IOMAP_F_DIRTY;
3337
3338 if (map->m_flags & EXT4_MAP_NEW)
3339 iomap->flags |= IOMAP_F_NEW;
3340
3341 iomap->bdev = inode->i_sb->s_bdev;
3342 iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
3343 iomap->offset = (u64) map->m_lblk << blkbits;
3344 iomap->length = (u64) map->m_len << blkbits;
3345
Ritesh Harjani63867222020-02-28 14:56:54 +05303346 if ((map->m_flags & EXT4_MAP_MAPPED) &&
3347 !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3348 iomap->flags |= IOMAP_F_MERGED;
3349
Matthew Bobrowskic8fdfe292019-11-05 22:59:56 +11003350 /*
3351 * Flags passed to ext4_map_blocks() for direct I/O writes can result
3352 * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits
3353 * set. In order for any allocated unwritten extents to be converted
3354 * into written extents correctly within the ->end_io() handler, we
3355 * need to ensure that the iomap->type is set appropriately. Hence, the
3356 * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has
3357 * been set first.
3358 */
3359 if (map->m_flags & EXT4_MAP_UNWRITTEN) {
3360 iomap->type = IOMAP_UNWRITTEN;
3361 iomap->addr = (u64) map->m_pblk << blkbits;
3362 } else if (map->m_flags & EXT4_MAP_MAPPED) {
3363 iomap->type = IOMAP_MAPPED;
3364 iomap->addr = (u64) map->m_pblk << blkbits;
3365 } else {
3366 iomap->type = IOMAP_HOLE;
3367 iomap->addr = IOMAP_NULL_ADDR;
3368 }
3369}
3370
Matthew Bobrowskif063db52019-11-05 23:00:14 +11003371static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
3372 unsigned int flags)
3373{
3374 handle_t *handle;
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11003375 u8 blkbits = inode->i_blkbits;
3376 int ret, dio_credits, m_flags = 0, retries = 0;
Matthew Bobrowskif063db52019-11-05 23:00:14 +11003377
3378 /*
3379 * Trim the mapping request to the maximum value that we can map at
3380 * once for direct I/O.
3381 */
3382 if (map->m_len > DIO_MAX_BLOCKS)
3383 map->m_len = DIO_MAX_BLOCKS;
3384 dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);
3385
3386retry:
3387 /*
3388 * Either we allocate blocks and then don't get an unwritten extent, so
3389 * in that case we have reserved enough credits. Or, the blocks are
3390 * already allocated and unwritten. In that case, the extent conversion
3391 * fits into the credits as well.
3392 */
3393 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
3394 if (IS_ERR(handle))
3395 return PTR_ERR(handle);
3396
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11003397 /*
3398 * DAX and direct I/O are the only two operations that are currently
3399 * supported with IOMAP_WRITE.
3400 */
3401 WARN_ON(!IS_DAX(inode) && !(flags & IOMAP_DIRECT));
3402 if (IS_DAX(inode))
3403 m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3404 /*
3405 * We use i_size instead of i_disksize here because delalloc writeback
3406 * can complete at any point during the I/O and subsequently push the
3407 * i_disksize out to i_size. This could be beyond where direct I/O is
3408 * happening and thus expose allocated blocks to direct I/O reads.
3409 */
3410 else if ((map->m_lblk * (1 << blkbits)) >= i_size_read(inode))
3411 m_flags = EXT4_GET_BLOCKS_CREATE;
3412 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3413 m_flags = EXT4_GET_BLOCKS_IO_CREATE_EXT;
3414
3415 ret = ext4_map_blocks(handle, inode, map, m_flags);
3416
3417 /*
3418 * We cannot fill holes in indirect tree based inodes as that could
3419 * expose stale data in the case of a crash. Use the magic error code
3420 * to fallback to buffered I/O.
3421 */
3422 if (!m_flags && !ret)
3423 ret = -ENOTBLK;
Matthew Bobrowskif063db52019-11-05 23:00:14 +11003424
Matthew Bobrowskif063db52019-11-05 23:00:14 +11003425 ext4_journal_stop(handle);
3426 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
3427 goto retry;
3428
3429 return ret;
3430}
3431
3432
Jan Kara364443c2016-11-20 17:36:06 -05003433static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
Goldwyn Rodriguesc039b992019-10-18 16:44:10 -07003434 unsigned flags, struct iomap *iomap, struct iomap *srcmap)
Jan Kara364443c2016-11-20 17:36:06 -05003435{
Jan Kara364443c2016-11-20 17:36:06 -05003436 int ret;
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003437 struct ext4_map_blocks map;
3438 u8 blkbits = inode->i_blkbits;
Jan Kara364443c2016-11-20 17:36:06 -05003439
Theodore Ts'obcd8e912018-09-01 12:45:04 -04003440 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3441 return -EINVAL;
Andreas Gruenbacher7046ae32017-10-01 17:57:54 -04003442
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003443 if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3444 return -ERANGE;
Jan Kara364443c2016-11-20 17:36:06 -05003445
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003446 /*
3447 * Calculate the first and last logical blocks respectively.
3448 */
3449 map.m_lblk = offset >> blkbits;
3450 map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3451 EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
Jan Kara364443c2016-11-20 17:36:06 -05003452
Ritesh Harjani9faac622020-09-18 10:36:35 +05303453 if (flags & IOMAP_WRITE) {
3454 /*
3455 * We check here if the blocks are already allocated, then we
3456 * don't need to start a journal txn and we can directly return
3457 * the mapping information. This could boost performance
3458 * especially in multi-threaded overwrite requests.
3459 */
3460 if (offset + length <= i_size_read(inode)) {
3461 ret = ext4_map_blocks(NULL, inode, &map, 0);
3462 if (ret > 0 && (map.m_flags & EXT4_MAP_MAPPED))
3463 goto out;
3464 }
Matthew Bobrowskif063db52019-11-05 23:00:14 +11003465 ret = ext4_iomap_alloc(inode, &map, flags);
Ritesh Harjani9faac622020-09-18 10:36:35 +05303466 } else {
Jan Kara776722e2016-11-20 18:09:11 -05003467 ret = ext4_map_blocks(NULL, inode, &map, 0);
Ritesh Harjani9faac622020-09-18 10:36:35 +05303468 }
Christoph Hellwig545052e2017-10-01 17:58:54 -04003469
Matthew Bobrowskif063db52019-11-05 23:00:14 +11003470 if (ret < 0)
3471 return ret;
Ritesh Harjani9faac622020-09-18 10:36:35 +05303472out:
Matthew Bobrowskic8fdfe292019-11-05 22:59:56 +11003473 ext4_set_iomap(inode, iomap, &map, offset, length);
Christoph Hellwig545052e2017-10-01 17:58:54 -04003474
Jan Kara364443c2016-11-20 17:36:06 -05003475 return 0;
3476}
3477
Jan Kara8cd115b2019-12-18 18:44:33 +01003478static int ext4_iomap_overwrite_begin(struct inode *inode, loff_t offset,
3479 loff_t length, unsigned flags, struct iomap *iomap,
3480 struct iomap *srcmap)
3481{
3482 int ret;
3483
3484 /*
3485 * Even for writes we don't need to allocate blocks, so just pretend
3486 * we are reading to save overhead of starting a transaction.
3487 */
3488 flags &= ~IOMAP_WRITE;
3489 ret = ext4_iomap_begin(inode, offset, length, flags, iomap, srcmap);
3490 WARN_ON_ONCE(iomap->type != IOMAP_MAPPED);
3491 return ret;
3492}
3493
Jan Kara776722e2016-11-20 18:09:11 -05003494static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3495 ssize_t written, unsigned flags, struct iomap *iomap)
3496{
Jan Kara776722e2016-11-20 18:09:11 -05003497 /*
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11003498 * Check to see whether an error occurred while writing out the data to
3499 * the allocated blocks. If so, return the magic error code so that we
3500 * fallback to buffered I/O and attempt to complete the remainder of
3501 * the I/O. Any blocks that may have been allocated in preparation for
3502 * the direct I/O will be reused during buffered I/O.
Jan Kara776722e2016-11-20 18:09:11 -05003503 */
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11003504 if (flags & (IOMAP_WRITE | IOMAP_DIRECT) && written == 0)
3505 return -ENOTBLK;
Jan Kara776722e2016-11-20 18:09:11 -05003506
Matthew Bobrowski569342d2019-11-05 23:01:51 +11003507 return 0;
Jan Kara776722e2016-11-20 18:09:11 -05003508}
3509
Christoph Hellwig8ff6daa2017-01-27 23:20:26 -08003510const struct iomap_ops ext4_iomap_ops = {
Jan Kara364443c2016-11-20 17:36:06 -05003511 .iomap_begin = ext4_iomap_begin,
Jan Kara776722e2016-11-20 18:09:11 -05003512 .iomap_end = ext4_iomap_end,
Jan Kara364443c2016-11-20 17:36:06 -05003513};
3514
Jan Kara8cd115b2019-12-18 18:44:33 +01003515const struct iomap_ops ext4_iomap_overwrite_ops = {
3516 .iomap_begin = ext4_iomap_overwrite_begin,
3517 .iomap_end = ext4_iomap_end,
3518};
3519
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003520static bool ext4_iomap_is_delalloc(struct inode *inode,
3521 struct ext4_map_blocks *map)
Mingming Cao4c0425f2009-09-28 15:48:41 -04003522{
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003523 struct extent_status es;
3524 ext4_lblk_t offset = 0, end = map->m_lblk + map->m_len - 1;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003525
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003526 ext4_es_find_extent_range(inode, &ext4_es_is_delayed,
3527 map->m_lblk, end, &es);
Mingming4b70df12009-11-03 14:44:54 -05003528
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003529 if (!es.es_len || es.es_lblk > end)
3530 return false;
3531
3532 if (es.es_lblk > map->m_lblk) {
3533 map->m_len = es.es_lblk - map->m_lblk;
3534 return false;
3535 }
3536
3537 offset = map->m_lblk - es.es_lblk;
3538 map->m_len = es.es_len - offset;
3539
3540 return true;
3541}
3542
3543static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
3544 loff_t length, unsigned int flags,
3545 struct iomap *iomap, struct iomap *srcmap)
3546{
3547 int ret;
3548 bool delalloc = false;
3549 struct ext4_map_blocks map;
3550 u8 blkbits = inode->i_blkbits;
3551
3552 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3553 return -EINVAL;
3554
3555 if (ext4_has_inline_data(inode)) {
3556 ret = ext4_inline_data_iomap(inode, iomap);
3557 if (ret != -EAGAIN) {
3558 if (ret == 0 && offset >= iomap->length)
3559 ret = -ENOENT;
3560 return ret;
3561 }
3562 }
Mingming Cao8d5d02e2009-09-28 15:48:29 -04003563
Jan Kara74c66bc2016-02-29 08:36:38 +11003564 /*
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003565 * Calculate the first and last logical block respectively.
Jan Kara74c66bc2016-02-29 08:36:38 +11003566 */
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003567 map.m_lblk = offset >> blkbits;
3568 map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3569 EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3570
Ritesh Harjanib2c57642020-02-28 14:56:57 +05303571 /*
3572 * Fiemap callers may call for offset beyond s_bitmap_maxbytes.
3573 * So handle it here itself instead of querying ext4_map_blocks().
3574 * Since ext4_map_blocks() will warn about it and will return
3575 * -EIO error.
3576 */
3577 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
3578 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3579
3580 if (offset >= sbi->s_bitmap_maxbytes) {
3581 map.m_flags = 0;
3582 goto set_iomap;
3583 }
3584 }
3585
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003586 ret = ext4_map_blocks(NULL, inode, &map, 0);
3587 if (ret < 0)
3588 return ret;
3589 if (ret == 0)
3590 delalloc = ext4_iomap_is_delalloc(inode, &map);
3591
Ritesh Harjanib2c57642020-02-28 14:56:57 +05303592set_iomap:
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003593 ext4_set_iomap(inode, iomap, &map, offset, length);
3594 if (delalloc && iomap->type == IOMAP_HOLE)
3595 iomap->type = IOMAP_DELALLOC;
Christoph Hellwig187372a2016-02-08 14:40:51 +11003596
3597 return 0;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003598}
Jiaying Zhangc7064ef2010-03-02 13:28:44 -05003599
Matthew Bobrowski09edf4d2019-11-05 23:03:31 +11003600const struct iomap_ops ext4_iomap_report_ops = {
3601 .iomap_begin = ext4_iomap_begin_report,
3602};
Mingming Cao4c0425f2009-09-28 15:48:41 -04003603
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003604/*
Mingming Cao617ba132006-10-11 01:20:53 -07003605 * Pages can be marked dirty completely asynchronously from ext4's journalling
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003606 * activity. By filemap_sync_pte(), try_to_unmap_one(), etc. We cannot do
3607 * much here because ->set_page_dirty is called under VFS locks. The page is
3608 * not necessarily locked.
3609 *
3610 * We cannot just dirty the page and leave attached buffers clean, because the
3611 * buffers' dirty state is "definitive". We cannot just set the buffers dirty
3612 * or jbddirty because all the journalling code will explode.
3613 *
3614 * So what we do is to mark the page "pending dirty" and next time writepage
3615 * is called, propagate that into the buffers appropriately.
3616 */
Mingming Cao617ba132006-10-11 01:20:53 -07003617static int ext4_journalled_set_page_dirty(struct page *page)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003618{
3619 SetPageChecked(page);
3620 return __set_page_dirty_nobuffers(page);
3621}
3622
Jan Kara6dcc6932016-12-01 11:46:40 -05003623static int ext4_set_page_dirty(struct page *page)
3624{
3625 WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page));
3626 WARN_ON_ONCE(!page_has_buffers(page));
3627 return __set_page_dirty_buffers(page);
3628}
3629
Ritesh Harjani0e6895b2020-09-04 14:46:53 +05303630static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
3631 struct file *file, sector_t *span)
3632{
3633 return iomap_swapfile_activate(sis, file, span,
3634 &ext4_iomap_report_ops);
3635}
3636
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003637static const struct address_space_operations ext4_aops = {
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003638 .readpage = ext4_readpage,
Matthew Wilcox (Oracle)6311f91f2020-06-01 21:47:16 -07003639 .readahead = ext4_readahead,
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04003640 .writepage = ext4_writepage,
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003641 .writepages = ext4_writepages,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003642 .write_begin = ext4_write_begin,
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003643 .write_end = ext4_write_end,
Jan Kara6dcc6932016-12-01 11:46:40 -05003644 .set_page_dirty = ext4_set_page_dirty,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003645 .bmap = ext4_bmap,
3646 .invalidatepage = ext4_invalidatepage,
3647 .releasepage = ext4_releasepage,
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11003648 .direct_IO = noop_direct_IO,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003649 .migratepage = buffer_migrate_page,
3650 .is_partially_uptodate = block_is_partially_uptodate,
Andi Kleenaa261f52009-09-16 11:50:16 +02003651 .error_remove_page = generic_error_remove_page,
Ritesh Harjani0e6895b2020-09-04 14:46:53 +05303652 .swap_activate = ext4_iomap_swap_activate,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003653};
3654
Mingming Cao617ba132006-10-11 01:20:53 -07003655static const struct address_space_operations ext4_journalled_aops = {
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003656 .readpage = ext4_readpage,
Matthew Wilcox (Oracle)6311f91f2020-06-01 21:47:16 -07003657 .readahead = ext4_readahead,
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04003658 .writepage = ext4_writepage,
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003659 .writepages = ext4_writepages,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003660 .write_begin = ext4_write_begin,
3661 .write_end = ext4_journalled_write_end,
3662 .set_page_dirty = ext4_journalled_set_page_dirty,
3663 .bmap = ext4_bmap,
Jan Kara4520fb32012-12-25 13:28:54 -05003664 .invalidatepage = ext4_journalled_invalidatepage,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003665 .releasepage = ext4_releasepage,
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11003666 .direct_IO = noop_direct_IO,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003667 .is_partially_uptodate = block_is_partially_uptodate,
Andi Kleenaa261f52009-09-16 11:50:16 +02003668 .error_remove_page = generic_error_remove_page,
Ritesh Harjani0e6895b2020-09-04 14:46:53 +05303669 .swap_activate = ext4_iomap_swap_activate,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003670};
3671
Alex Tomas64769242008-07-11 19:27:31 -04003672static const struct address_space_operations ext4_da_aops = {
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003673 .readpage = ext4_readpage,
Matthew Wilcox (Oracle)6311f91f2020-06-01 21:47:16 -07003674 .readahead = ext4_readahead,
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04003675 .writepage = ext4_writepage,
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003676 .writepages = ext4_writepages,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003677 .write_begin = ext4_da_write_begin,
3678 .write_end = ext4_da_write_end,
Jan Kara6dcc6932016-12-01 11:46:40 -05003679 .set_page_dirty = ext4_set_page_dirty,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003680 .bmap = ext4_bmap,
Eric Whitney8fcc3a52019-08-22 23:22:14 -04003681 .invalidatepage = ext4_invalidatepage,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003682 .releasepage = ext4_releasepage,
Matthew Bobrowski378f32b2019-11-05 23:02:39 +11003683 .direct_IO = noop_direct_IO,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003684 .migratepage = buffer_migrate_page,
3685 .is_partially_uptodate = block_is_partially_uptodate,
Andi Kleenaa261f52009-09-16 11:50:16 +02003686 .error_remove_page = generic_error_remove_page,
Ritesh Harjani0e6895b2020-09-04 14:46:53 +05303687 .swap_activate = ext4_iomap_swap_activate,
Alex Tomas64769242008-07-11 19:27:31 -04003688};
3689
Dan Williams5f0663b2017-12-21 12:25:11 -08003690static const struct address_space_operations ext4_dax_aops = {
3691 .writepages = ext4_dax_writepages,
3692 .direct_IO = noop_direct_IO,
3693 .set_page_dirty = noop_set_page_dirty,
Toshi Kani94dbb632018-09-15 21:23:41 -04003694 .bmap = ext4_bmap,
Dan Williams5f0663b2017-12-21 12:25:11 -08003695 .invalidatepage = noop_invalidatepage,
Ritesh Harjani0e6895b2020-09-04 14:46:53 +05303696 .swap_activate = ext4_iomap_swap_activate,
Dan Williams5f0663b2017-12-21 12:25:11 -08003697};
3698
Mingming Cao617ba132006-10-11 01:20:53 -07003699void ext4_set_aops(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003700{
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003701 switch (ext4_inode_journal_mode(inode)) {
3702 case EXT4_INODE_ORDERED_DATA_MODE:
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003703 case EXT4_INODE_WRITEBACK_DATA_MODE:
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003704 break;
3705 case EXT4_INODE_JOURNAL_DATA_MODE:
Mingming Cao617ba132006-10-11 01:20:53 -07003706 inode->i_mapping->a_ops = &ext4_journalled_aops;
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003707 return;
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003708 default:
3709 BUG();
3710 }
Dan Williams5f0663b2017-12-21 12:25:11 -08003711 if (IS_DAX(inode))
3712 inode->i_mapping->a_ops = &ext4_dax_aops;
3713 else if (test_opt(inode->i_sb, DELALLOC))
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003714 inode->i_mapping->a_ops = &ext4_da_aops;
3715 else
3716 inode->i_mapping->a_ops = &ext4_aops;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003717}
3718
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003719static int __ext4_block_zero_page_range(handle_t *handle,
Lukas Czernerd863dc32013-05-27 23:32:35 -04003720 struct address_space *mapping, loff_t from, loff_t length)
3721{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003722 ext4_fsblk_t index = from >> PAGE_SHIFT;
3723 unsigned offset = from & (PAGE_SIZE-1);
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003724 unsigned blocksize, pos;
Lukas Czernerd863dc32013-05-27 23:32:35 -04003725 ext4_lblk_t iblock;
3726 struct inode *inode = mapping->host;
3727 struct buffer_head *bh;
3728 struct page *page;
3729 int err = 0;
3730
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003731 page = find_or_create_page(mapping, from >> PAGE_SHIFT,
Michal Hockoc62d2552015-11-06 16:28:49 -08003732 mapping_gfp_constraint(mapping, ~__GFP_FS));
Lukas Czernerd863dc32013-05-27 23:32:35 -04003733 if (!page)
3734 return -ENOMEM;
3735
3736 blocksize = inode->i_sb->s_blocksize;
Lukas Czernerd863dc32013-05-27 23:32:35 -04003737
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003738 iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
Lukas Czernerd863dc32013-05-27 23:32:35 -04003739
3740 if (!page_has_buffers(page))
3741 create_empty_buffers(page, blocksize, 0);
3742
3743 /* Find the buffer that contains "offset" */
3744 bh = page_buffers(page);
3745 pos = blocksize;
3746 while (offset >= pos) {
3747 bh = bh->b_this_page;
3748 iblock++;
3749 pos += blocksize;
3750 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003751 if (buffer_freed(bh)) {
3752 BUFFER_TRACE(bh, "freed: skip");
3753 goto unlock;
3754 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003755 if (!buffer_mapped(bh)) {
3756 BUFFER_TRACE(bh, "unmapped");
3757 ext4_get_block(inode, iblock, bh, 0);
3758 /* unmapped? It's a hole - nothing to do */
3759 if (!buffer_mapped(bh)) {
3760 BUFFER_TRACE(bh, "still unmapped");
3761 goto unlock;
3762 }
3763 }
3764
3765 /* Ok, it's mapped. Make sure it's up-to-date */
3766 if (PageUptodate(page))
3767 set_buffer_uptodate(bh);
3768
3769 if (!buffer_uptodate(bh)) {
zhangyi (F)2d069c02020-09-24 15:33:33 +08003770 err = ext4_read_bh_lock(bh, 0, true);
3771 if (err)
Lukas Czernerd863dc32013-05-27 23:32:35 -04003772 goto unlock;
Eric Biggers4f74d152020-07-02 01:56:07 +00003773 if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
Michael Halcrowc9c74292015-04-12 00:56:10 -04003774 /* We expect the key to be set. */
Jaegeuk Kima7550b32016-07-10 14:01:03 -04003775 BUG_ON(!fscrypt_has_encryption_key(inode));
Eric Biggers834f1562019-12-26 09:41:05 -06003776 err = fscrypt_decrypt_pagecache_blocks(page, blocksize,
3777 bh_offset(bh));
3778 if (err) {
3779 clear_buffer_uptodate(bh);
3780 goto unlock;
3781 }
Michael Halcrowc9c74292015-04-12 00:56:10 -04003782 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003783 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003784 if (ext4_should_journal_data(inode)) {
3785 BUFFER_TRACE(bh, "get write access");
3786 err = ext4_journal_get_write_access(handle, bh);
3787 if (err)
3788 goto unlock;
3789 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003790 zero_user(page, offset, length);
Lukas Czernerd863dc32013-05-27 23:32:35 -04003791 BUFFER_TRACE(bh, "zeroed end of block");
3792
Lukas Czernerd863dc32013-05-27 23:32:35 -04003793 if (ext4_should_journal_data(inode)) {
3794 err = ext4_handle_dirty_metadata(handle, inode, bh);
Lukas Czerner0713ed02013-05-27 23:32:35 -04003795 } else {
jon ernst353eefd2013-07-01 08:12:39 -04003796 err = 0;
Lukas Czernerd863dc32013-05-27 23:32:35 -04003797 mark_buffer_dirty(bh);
Jan Kara3957ef52016-04-24 00:56:05 -04003798 if (ext4_should_order_data(inode))
Ross Zwisler73131fb2019-06-20 17:26:26 -04003799 err = ext4_jbd2_inode_add_write(handle, inode, from,
3800 length);
Lukas Czerner0713ed02013-05-27 23:32:35 -04003801 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003802
3803unlock:
3804 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003805 put_page(page);
Lukas Czernerd863dc32013-05-27 23:32:35 -04003806 return err;
3807}
3808
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003809/*
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003810 * ext4_block_zero_page_range() zeros out a mapping of length 'length'
3811 * starting from file offset 'from'. The range to be zero'd must
3812 * be contained with in one block. If the specified range exceeds
3813 * the end of the block it will be shortened to end of the block
3814 * that cooresponds to 'from'
3815 */
3816static int ext4_block_zero_page_range(handle_t *handle,
3817 struct address_space *mapping, loff_t from, loff_t length)
3818{
3819 struct inode *inode = mapping->host;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003820 unsigned offset = from & (PAGE_SIZE-1);
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003821 unsigned blocksize = inode->i_sb->s_blocksize;
3822 unsigned max = blocksize - (offset & (blocksize - 1));
3823
3824 /*
3825 * correct length if it does not fall between
3826 * 'from' and the end of the block
3827 */
3828 if (length > max || length < 0)
3829 length = max;
3830
Jan Kara47e69352016-11-20 18:08:05 -05003831 if (IS_DAX(inode)) {
3832 return iomap_zero_range(inode, from, length, NULL,
3833 &ext4_iomap_ops);
3834 }
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003835 return __ext4_block_zero_page_range(handle, mapping, from, length);
3836}
3837
3838/*
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003839 * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
3840 * up to the end of the block which corresponds to `from'.
3841 * This required during truncate. We need to physically zero the tail end
3842 * of that block so it doesn't yield old data if the file is later grown.
3843 */
Stephen Hemmingerc1978552014-05-12 10:50:23 -04003844static int ext4_block_truncate_page(handle_t *handle,
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003845 struct address_space *mapping, loff_t from)
3846{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003847 unsigned offset = from & (PAGE_SIZE-1);
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003848 unsigned length;
3849 unsigned blocksize;
3850 struct inode *inode = mapping->host;
3851
Theodore Ts'o0d068632017-02-14 11:31:15 -05003852 /* If we are processing an encrypted inode during orphan list handling */
Chandan Rajendra592ddec2018-12-12 15:20:10 +05303853 if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
Theodore Ts'o0d068632017-02-14 11:31:15 -05003854 return 0;
3855
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003856 blocksize = inode->i_sb->s_blocksize;
3857 length = blocksize - (offset & (blocksize - 1));
3858
3859 return ext4_block_zero_page_range(handle, mapping, from, length);
3860}
3861
Lukas Czernera87dd182013-05-27 23:32:35 -04003862int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
3863 loff_t lstart, loff_t length)
3864{
3865 struct super_block *sb = inode->i_sb;
3866 struct address_space *mapping = inode->i_mapping;
Lukas Czernere1be3a92013-07-01 08:12:39 -04003867 unsigned partial_start, partial_end;
Lukas Czernera87dd182013-05-27 23:32:35 -04003868 ext4_fsblk_t start, end;
3869 loff_t byte_end = (lstart + length - 1);
3870 int err = 0;
3871
Lukas Czernere1be3a92013-07-01 08:12:39 -04003872 partial_start = lstart & (sb->s_blocksize - 1);
3873 partial_end = byte_end & (sb->s_blocksize - 1);
3874
Lukas Czernera87dd182013-05-27 23:32:35 -04003875 start = lstart >> sb->s_blocksize_bits;
3876 end = byte_end >> sb->s_blocksize_bits;
3877
3878 /* Handle partial zero within the single block */
Lukas Czernere1be3a92013-07-01 08:12:39 -04003879 if (start == end &&
3880 (partial_start || (partial_end != sb->s_blocksize - 1))) {
Lukas Czernera87dd182013-05-27 23:32:35 -04003881 err = ext4_block_zero_page_range(handle, mapping,
3882 lstart, length);
3883 return err;
3884 }
3885 /* Handle partial zero out on the start of the range */
Lukas Czernere1be3a92013-07-01 08:12:39 -04003886 if (partial_start) {
Lukas Czernera87dd182013-05-27 23:32:35 -04003887 err = ext4_block_zero_page_range(handle, mapping,
3888 lstart, sb->s_blocksize);
3889 if (err)
3890 return err;
3891 }
3892 /* Handle partial zero out on the end of the range */
Lukas Czernere1be3a92013-07-01 08:12:39 -04003893 if (partial_end != sb->s_blocksize - 1)
Lukas Czernera87dd182013-05-27 23:32:35 -04003894 err = ext4_block_zero_page_range(handle, mapping,
Lukas Czernere1be3a92013-07-01 08:12:39 -04003895 byte_end - partial_end,
3896 partial_end + 1);
Lukas Czernera87dd182013-05-27 23:32:35 -04003897 return err;
3898}
3899
Duane Griffin91ef4ca2008-07-11 19:27:31 -04003900int ext4_can_truncate(struct inode *inode)
3901{
Duane Griffin91ef4ca2008-07-11 19:27:31 -04003902 if (S_ISREG(inode->i_mode))
3903 return 1;
3904 if (S_ISDIR(inode->i_mode))
3905 return 1;
3906 if (S_ISLNK(inode->i_mode))
3907 return !ext4_inode_is_fast_symlink(inode);
3908 return 0;
3909}
3910
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003911/*
Jan Kara01127842015-12-07 14:34:49 -05003912 * We have to make sure i_disksize gets properly updated before we truncate
3913 * page cache due to hole punching or zero range. Otherwise i_disksize update
3914 * can get lost as it may have been postponed to submission of writeback but
3915 * that will never happen after we truncate page cache.
3916 */
3917int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
3918 loff_t len)
3919{
3920 handle_t *handle;
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07003921 int ret;
3922
Jan Kara01127842015-12-07 14:34:49 -05003923 loff_t size = i_size_read(inode);
3924
Al Viro59551022016-01-22 15:40:57 -05003925 WARN_ON(!inode_is_locked(inode));
Jan Kara01127842015-12-07 14:34:49 -05003926 if (offset > size || offset + len < size)
3927 return 0;
3928
3929 if (EXT4_I(inode)->i_disksize >= size)
3930 return 0;
3931
3932 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
3933 if (IS_ERR(handle))
3934 return PTR_ERR(handle);
3935 ext4_update_i_disksize(inode, size);
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07003936 ret = ext4_mark_inode_dirty(handle, inode);
Jan Kara01127842015-12-07 14:34:49 -05003937 ext4_journal_stop(handle);
3938
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07003939 return ret;
Jan Kara01127842015-12-07 14:34:49 -05003940}
3941
Ross Zwislerb1f38212018-09-11 13:31:16 -04003942static void ext4_wait_dax_page(struct ext4_inode_info *ei)
Ross Zwisler430657b2018-07-29 17:00:22 -04003943{
Ross Zwisler430657b2018-07-29 17:00:22 -04003944 up_write(&ei->i_mmap_sem);
3945 schedule();
3946 down_write(&ei->i_mmap_sem);
3947}
3948
3949int ext4_break_layouts(struct inode *inode)
3950{
3951 struct ext4_inode_info *ei = EXT4_I(inode);
3952 struct page *page;
Ross Zwisler430657b2018-07-29 17:00:22 -04003953 int error;
3954
3955 if (WARN_ON_ONCE(!rwsem_is_locked(&ei->i_mmap_sem)))
3956 return -EINVAL;
3957
3958 do {
Ross Zwisler430657b2018-07-29 17:00:22 -04003959 page = dax_layout_busy_page(inode->i_mapping);
3960 if (!page)
3961 return 0;
3962
3963 error = ___wait_var_event(&page->_refcount,
3964 atomic_read(&page->_refcount) == 1,
3965 TASK_INTERRUPTIBLE, 0, 0,
Ross Zwislerb1f38212018-09-11 13:31:16 -04003966 ext4_wait_dax_page(ei));
3967 } while (error == 0);
Ross Zwisler430657b2018-07-29 17:00:22 -04003968
3969 return error;
3970}
3971
Jan Kara01127842015-12-07 14:34:49 -05003972/*
Ross Zwislercca32b72016-09-22 11:49:38 -04003973 * ext4_punch_hole: punches a hole in a file by releasing the blocks
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003974 * associated with the given offset and length
3975 *
3976 * @inode: File inode
3977 * @offset: The offset where the hole will begin
3978 * @len: The length of the hole
3979 *
Anatol Pomozov4907cb72012-09-01 10:31:09 -07003980 * Returns: 0 on success or negative on failure
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003981 */
3982
Ashish Sangwanaeb28172013-07-01 08:12:38 -04003983int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003984{
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003985 struct super_block *sb = inode->i_sb;
3986 ext4_lblk_t first_block, stop_block;
3987 struct address_space *mapping = inode->i_mapping;
Lukas Czernera87dd182013-05-27 23:32:35 -04003988 loff_t first_block_offset, last_block_offset;
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003989 handle_t *handle;
3990 unsigned int credits;
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07003991 int ret = 0, ret2 = 0;
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003992
Lukas Czernerb8a86842014-03-18 18:05:35 -04003993 trace_ext4_punch_hole(inode, offset, length, 0);
Zheng Liuaaddea82013-01-16 20:21:26 -05003994
Theodore Ts'oc1e82202019-08-23 22:38:00 -04003995 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
3996 if (ext4_has_inline_data(inode)) {
3997 down_write(&EXT4_I(inode)->i_mmap_sem);
3998 ret = ext4_convert_inline_data(inode);
3999 up_write(&EXT4_I(inode)->i_mmap_sem);
4000 if (ret)
4001 return ret;
4002 }
4003
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004004 /*
4005 * Write out all dirty pages to avoid race conditions
4006 * Then release them.
4007 */
Ross Zwislercca32b72016-09-22 11:49:38 -04004008 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004009 ret = filemap_write_and_wait_range(mapping, offset,
4010 offset + length - 1);
4011 if (ret)
4012 return ret;
4013 }
4014
Al Viro59551022016-01-22 15:40:57 -05004015 inode_lock(inode);
Lukas Czerner9ef06ce2014-04-12 09:47:00 -04004016
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004017 /* No need to punch hole beyond i_size */
4018 if (offset >= inode->i_size)
4019 goto out_mutex;
4020
4021 /*
4022 * If the hole extends beyond i_size, set the hole
4023 * to end after the page that contains i_size
4024 */
4025 if (offset + length > inode->i_size) {
4026 length = inode->i_size +
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004027 PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004028 offset;
4029 }
4030
Jan Karaa3612932013-08-16 21:19:41 -04004031 if (offset & (sb->s_blocksize - 1) ||
4032 (offset + length) & (sb->s_blocksize - 1)) {
4033 /*
4034 * Attach jinode to inode for jbd2 if we do any zeroing of
4035 * partial block
4036 */
4037 ret = ext4_inode_attach_jinode(inode);
4038 if (ret < 0)
4039 goto out_mutex;
4040
4041 }
4042
Jan Karaea3d7202015-12-07 14:28:03 -05004043 /* Wait all existing dio workers, newcomers will block on i_mutex */
Jan Karaea3d7202015-12-07 14:28:03 -05004044 inode_dio_wait(inode);
4045
4046 /*
4047 * Prevent page faults from reinstantiating pages we have released from
4048 * page cache.
4049 */
4050 down_write(&EXT4_I(inode)->i_mmap_sem);
Ross Zwisler430657b2018-07-29 17:00:22 -04004051
4052 ret = ext4_break_layouts(inode);
4053 if (ret)
4054 goto out_dio;
4055
Lukas Czernera87dd182013-05-27 23:32:35 -04004056 first_block_offset = round_up(offset, sb->s_blocksize);
4057 last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004058
Lukas Czernera87dd182013-05-27 23:32:35 -04004059 /* Now release the pages and zero block aligned part of pages*/
Jan Kara01127842015-12-07 14:34:49 -05004060 if (last_block_offset > first_block_offset) {
4061 ret = ext4_update_disksize_before_punch(inode, offset, length);
4062 if (ret)
4063 goto out_dio;
Lukas Czernera87dd182013-05-27 23:32:35 -04004064 truncate_pagecache_range(inode, first_block_offset,
4065 last_block_offset);
Jan Kara01127842015-12-07 14:34:49 -05004066 }
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004067
4068 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4069 credits = ext4_writepage_trans_blocks(inode);
4070 else
4071 credits = ext4_blocks_for_truncate(inode);
4072 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4073 if (IS_ERR(handle)) {
4074 ret = PTR_ERR(handle);
4075 ext4_std_error(sb, ret);
4076 goto out_dio;
4077 }
4078
Lukas Czernera87dd182013-05-27 23:32:35 -04004079 ret = ext4_zero_partial_blocks(handle, inode, offset,
4080 length);
4081 if (ret)
4082 goto out_stop;
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004083
4084 first_block = (offset + sb->s_blocksize - 1) >>
4085 EXT4_BLOCK_SIZE_BITS(sb);
4086 stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
4087
Lukas Czernereee597a2018-05-13 19:28:35 -04004088 /* If there are blocks to remove, do it */
4089 if (stop_block > first_block) {
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004090
Lukas Czernereee597a2018-05-13 19:28:35 -04004091 down_write(&EXT4_I(inode)->i_data_sem);
brookxu27bc4462020-08-17 15:36:15 +08004092 ext4_discard_preallocations(inode, 0);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004093
Lukas Czernereee597a2018-05-13 19:28:35 -04004094 ret = ext4_es_remove_extent(inode, first_block,
4095 stop_block - first_block);
4096 if (ret) {
4097 up_write(&EXT4_I(inode)->i_data_sem);
4098 goto out_stop;
4099 }
4100
4101 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4102 ret = ext4_ext_remove_space(inode, first_block,
4103 stop_block - 1);
4104 else
4105 ret = ext4_ind_remove_space(handle, inode, first_block,
4106 stop_block);
4107
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004108 up_write(&EXT4_I(inode)->i_data_sem);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004109 }
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07004110 ext4_fc_track_range(inode, first_block, stop_block);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004111 if (IS_SYNC(inode))
4112 ext4_handle_sync(handle);
Maxim Patlasove251f9b2014-02-20 16:58:05 -05004113
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05004114 inode->i_mtime = inode->i_ctime = current_time(inode);
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07004115 ret2 = ext4_mark_inode_dirty(handle, inode);
4116 if (unlikely(ret2))
4117 ret = ret2;
Jan Kara67a7d5f2017-05-29 13:24:55 -04004118 if (ret >= 0)
4119 ext4_update_inode_fsync_trans(handle, inode, 1);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004120out_stop:
4121 ext4_journal_stop(handle);
4122out_dio:
Jan Karaea3d7202015-12-07 14:28:03 -05004123 up_write(&EXT4_I(inode)->i_mmap_sem);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004124out_mutex:
Al Viro59551022016-01-22 15:40:57 -05004125 inode_unlock(inode);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004126 return ret;
Allison Hendersona4bb6b62011-05-25 07:41:50 -04004127}
4128
Jan Karaa3612932013-08-16 21:19:41 -04004129int ext4_inode_attach_jinode(struct inode *inode)
4130{
4131 struct ext4_inode_info *ei = EXT4_I(inode);
4132 struct jbd2_inode *jinode;
4133
4134 if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
4135 return 0;
4136
4137 jinode = jbd2_alloc_inode(GFP_KERNEL);
4138 spin_lock(&inode->i_lock);
4139 if (!ei->jinode) {
4140 if (!jinode) {
4141 spin_unlock(&inode->i_lock);
4142 return -ENOMEM;
4143 }
4144 ei->jinode = jinode;
4145 jbd2_journal_init_jbd_inode(ei->jinode, inode);
4146 jinode = NULL;
4147 }
4148 spin_unlock(&inode->i_lock);
4149 if (unlikely(jinode != NULL))
4150 jbd2_free_inode(jinode);
4151 return 0;
4152}
4153
Allison Hendersona4bb6b62011-05-25 07:41:50 -04004154/*
Mingming Cao617ba132006-10-11 01:20:53 -07004155 * ext4_truncate()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004156 *
Mingming Cao617ba132006-10-11 01:20:53 -07004157 * We block out ext4_get_block() block instantiations across the entire
4158 * transaction, and VFS/VM ensures that ext4_truncate() cannot run
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004159 * simultaneously on behalf of the same inode.
4160 *
Justin P. Mattock42b2aa82011-11-28 20:31:00 -08004161 * As we work through the truncate and commit bits of it to the journal there
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004162 * is one core, guiding principle: the file's tree must always be consistent on
4163 * disk. We must be able to restart the truncate after a crash.
4164 *
4165 * The file's tree may be transiently inconsistent in memory (although it
4166 * probably isn't), but whenever we close off and commit a journal transaction,
4167 * the contents of (the filesystem + the journal) must be consistent and
4168 * restartable. It's pretty simple, really: bottom up, right to left (although
4169 * left-to-right works OK too).
4170 *
4171 * Note that at recovery time, journal replay occurs *before* the restart of
4172 * truncate against the orphan inode list.
4173 *
4174 * The committed inode has the new, desired i_size (which is the same as
Mingming Cao617ba132006-10-11 01:20:53 -07004175 * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004176 * that this inode's truncate did not complete and it will again call
Mingming Cao617ba132006-10-11 01:20:53 -07004177 * ext4_truncate() to have another go. So there will be instantiated blocks
4178 * to the right of the truncation point in a crashed ext4 filesystem. But
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004179 * that's fine - as long as they are linked from the inode, the post-crash
Mingming Cao617ba132006-10-11 01:20:53 -07004180 * ext4_truncate() run will find them and release them.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004181 */
Theodore Ts'o2c98eb52016-11-13 22:02:26 -05004182int ext4_truncate(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004183{
Theodore Ts'o819c4922013-04-03 12:47:17 -04004184 struct ext4_inode_info *ei = EXT4_I(inode);
4185 unsigned int credits;
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07004186 int err = 0, err2;
Theodore Ts'o819c4922013-04-03 12:47:17 -04004187 handle_t *handle;
4188 struct address_space *mapping = inode->i_mapping;
Theodore Ts'o819c4922013-04-03 12:47:17 -04004189
Theodore Ts'o19b5ef62013-04-03 21:58:52 -04004190 /*
4191 * There is a possibility that we're either freeing the inode
Matthew Wilcoxe04027e2014-03-24 15:15:07 -04004192 * or it's a completely new inode. In those cases we might not
Theodore Ts'o19b5ef62013-04-03 21:58:52 -04004193 * have i_mutex locked because it's not necessary.
4194 */
4195 if (!(inode->i_state & (I_NEW|I_FREEING)))
Al Viro59551022016-01-22 15:40:57 -05004196 WARN_ON(!inode_is_locked(inode));
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04004197 trace_ext4_truncate_enter(inode);
4198
Duane Griffin91ef4ca2008-07-11 19:27:31 -04004199 if (!ext4_can_truncate(inode))
zhengliang9a5d2652020-07-01 16:30:27 +08004200 goto out_trace;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004201
Theodore Ts'o5534fb52009-09-17 09:34:16 -04004202 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05004203 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
Theodore Ts'o7d8f9f72009-02-24 08:21:14 -05004204
Tao Maaef1c852012-12-10 14:06:02 -05004205 if (ext4_has_inline_data(inode)) {
4206 int has_inline = 1;
4207
Theodore Ts'o01daf942017-01-22 19:35:49 -05004208 err = ext4_inline_data_truncate(inode, &has_inline);
zhengliang9a5d2652020-07-01 16:30:27 +08004209 if (err || has_inline)
4210 goto out_trace;
Tao Maaef1c852012-12-10 14:06:02 -05004211 }
4212
Jan Karaa3612932013-08-16 21:19:41 -04004213 /* If we zero-out tail of the page, we have to create jinode for jbd2 */
4214 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4215 if (ext4_inode_attach_jinode(inode) < 0)
zhengliang9a5d2652020-07-01 16:30:27 +08004216 goto out_trace;
Jan Karaa3612932013-08-16 21:19:41 -04004217 }
4218
Amir Goldsteinff9893d2011-06-27 16:36:31 -04004219 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
Theodore Ts'o819c4922013-04-03 12:47:17 -04004220 credits = ext4_writepage_trans_blocks(inode);
Amir Goldsteinff9893d2011-06-27 16:36:31 -04004221 else
Theodore Ts'o819c4922013-04-03 12:47:17 -04004222 credits = ext4_blocks_for_truncate(inode);
4223
4224 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
zhengliang9a5d2652020-07-01 16:30:27 +08004225 if (IS_ERR(handle)) {
4226 err = PTR_ERR(handle);
4227 goto out_trace;
4228 }
Theodore Ts'o819c4922013-04-03 12:47:17 -04004229
Lukas Czernereb3544c2013-05-27 23:32:35 -04004230 if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4231 ext4_block_truncate_page(handle, mapping, inode->i_size);
Theodore Ts'o819c4922013-04-03 12:47:17 -04004232
4233 /*
4234 * We add the inode to the orphan list, so that if this
4235 * truncate spans multiple transactions, and we crash, we will
4236 * resume the truncate when the filesystem recovers. It also
4237 * marks the inode dirty, to catch the new size.
4238 *
4239 * Implication: the file must always be in a sane, consistent
4240 * truncatable state while each transaction commits.
4241 */
Theodore Ts'o2c98eb52016-11-13 22:02:26 -05004242 err = ext4_orphan_add(handle, inode);
4243 if (err)
Theodore Ts'o819c4922013-04-03 12:47:17 -04004244 goto out_stop;
4245
4246 down_write(&EXT4_I(inode)->i_data_sem);
4247
brookxu27bc4462020-08-17 15:36:15 +08004248 ext4_discard_preallocations(inode, 0);
Theodore Ts'o819c4922013-04-03 12:47:17 -04004249
4250 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
Theodore Ts'od0abb362016-11-13 22:02:28 -05004251 err = ext4_ext_truncate(handle, inode);
Theodore Ts'o819c4922013-04-03 12:47:17 -04004252 else
4253 ext4_ind_truncate(handle, inode);
4254
4255 up_write(&ei->i_data_sem);
Theodore Ts'od0abb362016-11-13 22:02:28 -05004256 if (err)
4257 goto out_stop;
Theodore Ts'o819c4922013-04-03 12:47:17 -04004258
4259 if (IS_SYNC(inode))
4260 ext4_handle_sync(handle);
4261
4262out_stop:
4263 /*
4264 * If this was a simple ftruncate() and the file will remain alive,
4265 * then we need to clear up the orphan record which we created above.
4266 * However, if this was a real unlink then we were called by
Wang Shilong58d86a52014-11-25 16:17:29 -05004267 * ext4_evict_inode(), and we allow that function to clean up the
Theodore Ts'o819c4922013-04-03 12:47:17 -04004268 * orphan info for us.
4269 */
4270 if (inode->i_nlink)
4271 ext4_orphan_del(handle, inode);
4272
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05004273 inode->i_mtime = inode->i_ctime = current_time(inode);
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07004274 err2 = ext4_mark_inode_dirty(handle, inode);
4275 if (unlikely(err2 && !err))
4276 err = err2;
Theodore Ts'o819c4922013-04-03 12:47:17 -04004277 ext4_journal_stop(handle);
Alex Tomasa86c6182006-10-11 01:21:03 -07004278
zhengliang9a5d2652020-07-01 16:30:27 +08004279out_trace:
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04004280 trace_ext4_truncate_exit(inode);
Theodore Ts'o2c98eb52016-11-13 22:02:26 -05004281 return err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004282}
4283
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004284/*
Mingming Cao617ba132006-10-11 01:20:53 -07004285 * ext4_get_inode_loc returns with an extra refcount against the inode's
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004286 * underlying buffer_head on success. If 'in_mem' is true, we have all
4287 * data in memory that is needed to recreate the on-disk version of this
4288 * inode.
4289 */
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004290static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
4291 struct ext4_iloc *iloc, int in_mem,
4292 ext4_fsblk_t *ret_block)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004293{
Theodore Ts'o240799c2008-10-09 23:53:47 -04004294 struct ext4_group_desc *gdp;
4295 struct buffer_head *bh;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004296 ext4_fsblk_t block;
Linus Torvalds02f03c42019-09-29 17:59:23 -07004297 struct blk_plug plug;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004298 int inodes_per_block, inode_offset;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004299
Aneesh Kumar K.V3a06d772008-11-22 15:04:59 -05004300 iloc->bh = NULL;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004301 if (ino < EXT4_ROOT_INO ||
4302 ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004303 return -EFSCORRUPTED;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004304
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004305 iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004306 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
4307 if (!gdp)
4308 return -EIO;
4309
4310 /*
4311 * Figure out the offset within the block group inode table
4312 */
Tao Ma00d09882011-05-09 10:26:41 -04004313 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004314 inode_offset = ((ino - 1) %
Theodore Ts'o240799c2008-10-09 23:53:47 -04004315 EXT4_INODES_PER_GROUP(sb));
4316 block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block);
4317 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4318
4319 bh = sb_getblk(sb, block);
Wang Shilongaebf0242013-01-12 16:28:47 -05004320 if (unlikely(!bh))
Theodore Ts'o860d21e2013-01-12 16:19:36 -05004321 return -ENOMEM;
Theodore Ts'o46f870d62019-11-21 13:09:43 -05004322 if (ext4_simulate_fail(sb, EXT4_SIM_INODE_EIO))
4323 goto simulate_eio;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004324 if (!buffer_uptodate(bh)) {
4325 lock_buffer(bh);
Hidehiro Kawai9c83a922008-07-26 16:39:26 -04004326
zhangyi (F)60c776e2020-09-24 15:33:34 +08004327 if (ext4_buffer_uptodate(bh)) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004328 /* someone brought it uptodate while we waited */
4329 unlock_buffer(bh);
4330 goto has_buffer;
4331 }
4332
4333 /*
4334 * If we have all information of the inode in memory and this
4335 * is the only valid inode in the block, we need not read the
4336 * block.
4337 */
4338 if (in_mem) {
4339 struct buffer_head *bitmap_bh;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004340 int i, start;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004341
Theodore Ts'o240799c2008-10-09 23:53:47 -04004342 start = inode_offset & ~(inodes_per_block - 1);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004343
4344 /* Is the inode bitmap in cache? */
Theodore Ts'o240799c2008-10-09 23:53:47 -04004345 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
Wang Shilongaebf0242013-01-12 16:28:47 -05004346 if (unlikely(!bitmap_bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004347 goto make_io;
4348
4349 /*
4350 * If the inode bitmap isn't in cache then the
4351 * optimisation may end up performing two reads instead
4352 * of one, so skip it.
4353 */
4354 if (!buffer_uptodate(bitmap_bh)) {
4355 brelse(bitmap_bh);
4356 goto make_io;
4357 }
Theodore Ts'o240799c2008-10-09 23:53:47 -04004358 for (i = start; i < start + inodes_per_block; i++) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004359 if (i == inode_offset)
4360 continue;
Mingming Cao617ba132006-10-11 01:20:53 -07004361 if (ext4_test_bit(i, bitmap_bh->b_data))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004362 break;
4363 }
4364 brelse(bitmap_bh);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004365 if (i == start + inodes_per_block) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004366 /* all other inodes are free, so skip I/O */
4367 memset(bh->b_data, 0, bh->b_size);
4368 set_buffer_uptodate(bh);
4369 unlock_buffer(bh);
4370 goto has_buffer;
4371 }
4372 }
4373
4374make_io:
4375 /*
Theodore Ts'o240799c2008-10-09 23:53:47 -04004376 * If we need to do any I/O, try to pre-readahead extra
4377 * blocks from the inode table.
4378 */
Linus Torvalds02f03c42019-09-29 17:59:23 -07004379 blk_start_plug(&plug);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004380 if (EXT4_SB(sb)->s_inode_readahead_blks) {
4381 ext4_fsblk_t b, end, table;
4382 unsigned num;
Theodore Ts'o0d606e2c2013-04-23 08:59:35 -04004383 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004384
4385 table = ext4_inode_table(sb, gdp);
Theodore Ts'ob713a5e2009-03-31 09:11:14 -04004386 /* s_inode_readahead_blks is always a power of 2 */
Theodore Ts'o0d606e2c2013-04-23 08:59:35 -04004387 b = block & ~((ext4_fsblk_t) ra_blks - 1);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004388 if (table > b)
4389 b = table;
Theodore Ts'o0d606e2c2013-04-23 08:59:35 -04004390 end = b + ra_blks;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004391 num = EXT4_INODES_PER_GROUP(sb);
Darrick J. Wongfeb0ab32012-04-29 18:45:10 -04004392 if (ext4_has_group_desc_csum(sb))
Aneesh Kumar K.V560671a2009-01-05 22:20:24 -05004393 num -= ext4_itable_unused_count(sb, gdp);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004394 table += num / inodes_per_block;
4395 if (end > table)
4396 end = table;
4397 while (b <= end)
zhangyi (F)5df1d412020-09-24 15:33:35 +08004398 ext4_sb_breadahead_unmovable(sb, b++);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004399 }
4400
4401 /*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004402 * There are other valid inodes in the buffer, this inode
4403 * has in-inode xattrs, or we don't have this inode in memory.
4404 * Read the block from disk.
4405 */
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004406 trace_ext4_load_inode(sb, ino);
zhangyi (F)2d069c02020-09-24 15:33:33 +08004407 ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL);
Linus Torvalds02f03c42019-09-29 17:59:23 -07004408 blk_finish_plug(&plug);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004409 wait_on_buffer(bh);
4410 if (!buffer_uptodate(bh)) {
Theodore Ts'o46f870d62019-11-21 13:09:43 -05004411 simulate_eio:
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004412 if (ret_block)
4413 *ret_block = block;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004414 brelse(bh);
4415 return -EIO;
4416 }
4417 }
4418has_buffer:
4419 iloc->bh = bh;
4420 return 0;
4421}
4422
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004423static int __ext4_get_inode_loc_noinmem(struct inode *inode,
4424 struct ext4_iloc *iloc)
4425{
4426 ext4_fsblk_t err_blk;
4427 int ret;
4428
4429 ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, iloc, 0,
4430 &err_blk);
4431
4432 if (ret == -EIO)
4433 ext4_error_inode_block(inode, err_blk, EIO,
4434 "unable to read itable block");
4435
4436 return ret;
4437}
4438
Mingming Cao617ba132006-10-11 01:20:53 -07004439int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004440{
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004441 ext4_fsblk_t err_blk;
4442 int ret;
4443
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004444 /* We have all inode data except xattrs in memory here. */
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004445 ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, iloc,
4446 !ext4_test_inode_state(inode, EXT4_STATE_XATTR), &err_blk);
4447
4448 if (ret == -EIO)
4449 ext4_error_inode_block(inode, err_blk, EIO,
4450 "unable to read itable block");
4451
4452 return ret;
4453}
4454
4455
4456int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
4457 struct ext4_iloc *iloc)
4458{
4459 return __ext4_get_inode_loc(sb, ino, iloc, 0, NULL);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004460}
4461
Ira Weinya8ab6d32020-05-28 07:59:58 -07004462static bool ext4_should_enable_dax(struct inode *inode)
Ross Zwisler66425862017-10-12 12:00:59 -04004463{
Ira Weinya8ab6d32020-05-28 07:59:58 -07004464 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4465
Ira Weiny9cb20f92020-05-28 08:00:00 -07004466 if (test_opt2(inode->i_sb, DAX_NEVER))
Ross Zwisler66425862017-10-12 12:00:59 -04004467 return false;
4468 if (!S_ISREG(inode->i_mode))
4469 return false;
4470 if (ext4_should_journal_data(inode))
4471 return false;
4472 if (ext4_has_inline_data(inode))
4473 return false;
Chandan Rajendra592ddec2018-12-12 15:20:10 +05304474 if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))
Ross Zwisler66425862017-10-12 12:00:59 -04004475 return false;
Eric Biggersc93d8f82019-07-22 09:26:24 -07004476 if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))
4477 return false;
Ira Weinya8ab6d32020-05-28 07:59:58 -07004478 if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
4479 return false;
4480 if (test_opt(inode->i_sb, DAX_ALWAYS))
4481 return true;
4482
Ira Weinyb383a732020-05-28 08:00:02 -07004483 return ext4_test_inode_flag(inode, EXT4_INODE_DAX);
Ross Zwisler66425862017-10-12 12:00:59 -04004484}
4485
Ira Weiny043546e2020-05-28 07:59:59 -07004486void ext4_set_inode_flags(struct inode *inode, bool init)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004487{
Mingming Cao617ba132006-10-11 01:20:53 -07004488 unsigned int flags = EXT4_I(inode)->i_flags;
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004489 unsigned int new_fl = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004490
Ira Weiny043546e2020-05-28 07:59:59 -07004491 WARN_ON_ONCE(IS_DAX(inode) && init);
4492
Mingming Cao617ba132006-10-11 01:20:53 -07004493 if (flags & EXT4_SYNC_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004494 new_fl |= S_SYNC;
Mingming Cao617ba132006-10-11 01:20:53 -07004495 if (flags & EXT4_APPEND_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004496 new_fl |= S_APPEND;
Mingming Cao617ba132006-10-11 01:20:53 -07004497 if (flags & EXT4_IMMUTABLE_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004498 new_fl |= S_IMMUTABLE;
Mingming Cao617ba132006-10-11 01:20:53 -07004499 if (flags & EXT4_NOATIME_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004500 new_fl |= S_NOATIME;
Mingming Cao617ba132006-10-11 01:20:53 -07004501 if (flags & EXT4_DIRSYNC_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004502 new_fl |= S_DIRSYNC;
Ira Weiny043546e2020-05-28 07:59:59 -07004503
4504 /* Because of the way inode_set_flags() works we must preserve S_DAX
4505 * here if already set. */
4506 new_fl |= (inode->i_flags & S_DAX);
4507 if (init && ext4_should_enable_dax(inode))
Ross Zwisler923ae0f2015-02-16 15:59:38 -08004508 new_fl |= S_DAX;
Ira Weiny043546e2020-05-28 07:59:59 -07004509
Eric Biggers2ee6a572017-10-09 12:15:35 -07004510 if (flags & EXT4_ENCRYPT_FL)
4511 new_fl |= S_ENCRYPTED;
Gabriel Krisman Bertazib886ee32019-04-25 14:12:08 -04004512 if (flags & EXT4_CASEFOLD_FL)
4513 new_fl |= S_CASEFOLD;
Eric Biggersc93d8f82019-07-22 09:26:24 -07004514 if (flags & EXT4_VERITY_FL)
4515 new_fl |= S_VERITY;
Theodore Ts'o5f16f322014-03-24 14:43:12 -04004516 inode_set_flags(inode, new_fl,
Eric Biggers2ee6a572017-10-09 12:15:35 -07004517 S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
Eric Biggersc93d8f82019-07-22 09:26:24 -07004518 S_ENCRYPTED|S_CASEFOLD|S_VERITY);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004519}
4520
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004521static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04004522 struct ext4_inode_info *ei)
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004523{
4524 blkcnt_t i_blocks ;
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004525 struct inode *inode = &(ei->vfs_inode);
4526 struct super_block *sb = inode->i_sb;
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004527
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004528 if (ext4_has_feature_huge_file(sb)) {
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004529 /* we are using combined 48 bit field */
4530 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
4531 le32_to_cpu(raw_inode->i_blocks_lo);
Theodore Ts'o07a03822010-06-14 09:54:48 -04004532 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004533 /* i_blocks represent file system block size */
4534 return i_blocks << (inode->i_blkbits - 9);
4535 } else {
4536 return i_blocks;
4537 }
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004538 } else {
4539 return le32_to_cpu(raw_inode->i_blocks_lo);
4540 }
4541}
Jan Karaff9ddf72007-07-18 09:24:20 -04004542
Theodore Ts'oeb9b5f02018-05-22 17:14:07 -04004543static inline int ext4_iget_extra_inode(struct inode *inode,
Tao Ma152a7b02012-12-02 11:13:24 -05004544 struct ext4_inode *raw_inode,
4545 struct ext4_inode_info *ei)
4546{
4547 __le32 *magic = (void *)raw_inode +
4548 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
Theodore Ts'oeb9b5f02018-05-22 17:14:07 -04004549
Eric Biggers290ab232016-12-01 14:51:58 -05004550 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <=
4551 EXT4_INODE_SIZE(inode->i_sb) &&
4552 *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
Tao Ma152a7b02012-12-02 11:13:24 -05004553 ext4_set_inode_state(inode, EXT4_STATE_XATTR);
Theodore Ts'oeb9b5f02018-05-22 17:14:07 -04004554 return ext4_find_inline_data_nolock(inode);
Tao Maf19d5872012-12-10 14:05:51 -05004555 } else
4556 EXT4_I(inode)->i_inline_off = 0;
Theodore Ts'oeb9b5f02018-05-22 17:14:07 -04004557 return 0;
Tao Ma152a7b02012-12-02 11:13:24 -05004558}
4559
Li Xi040cb372016-01-08 16:01:21 -05004560int ext4_get_projid(struct inode *inode, kprojid_t *projid)
4561{
Kaho Ng0b7b7772016-09-05 23:11:58 -04004562 if (!ext4_has_feature_project(inode->i_sb))
Li Xi040cb372016-01-08 16:01:21 -05004563 return -EOPNOTSUPP;
4564 *projid = EXT4_I(inode)->i_projid;
4565 return 0;
4566}
4567
Eryu Guane254d1a2018-05-10 11:55:31 -04004568/*
4569 * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
4570 * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
4571 * set.
4572 */
4573static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
4574{
4575 if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4576 inode_set_iversion_raw(inode, val);
4577 else
4578 inode_set_iversion_queried(inode, val);
4579}
4580static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
4581{
4582 if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4583 return inode_peek_iversion_raw(inode);
4584 else
4585 return inode_peek_iversion(inode);
4586}
4587
Theodore Ts'o8a363972018-12-19 12:29:13 -05004588struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
4589 ext4_iget_flags flags, const char *function,
4590 unsigned int line)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004591{
Mingming Cao617ba132006-10-11 01:20:53 -07004592 struct ext4_iloc iloc;
4593 struct ext4_inode *raw_inode;
David Howells1d1fe1e2008-02-07 00:15:37 -08004594 struct ext4_inode_info *ei;
David Howells1d1fe1e2008-02-07 00:15:37 -08004595 struct inode *inode;
Jan Karab436b9b2009-12-08 23:51:10 -05004596 journal_t *journal = EXT4_SB(sb)->s_journal;
David Howells1d1fe1e2008-02-07 00:15:37 -08004597 long ret;
Darrick J. Wong7e6e1ef2016-12-10 09:55:01 -05004598 loff_t size;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004599 int block;
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004600 uid_t i_uid;
4601 gid_t i_gid;
Li Xi040cb372016-01-08 16:01:21 -05004602 projid_t i_projid;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004603
Theodore Ts'o191ce1782018-12-31 22:34:31 -05004604 if ((!(flags & EXT4_IGET_SPECIAL) &&
Theodore Ts'o8a363972018-12-19 12:29:13 -05004605 (ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO)) ||
4606 (ino < EXT4_ROOT_INO) ||
4607 (ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))) {
4608 if (flags & EXT4_IGET_HANDLE)
4609 return ERR_PTR(-ESTALE);
Theodore Ts'o54d3adb2020-03-28 19:33:43 -04004610 __ext4_error(sb, function, line, EFSCORRUPTED, 0,
Theodore Ts'o8a363972018-12-19 12:29:13 -05004611 "inode #%lu: comm %s: iget: illegal inode #",
4612 ino, current->comm);
4613 return ERR_PTR(-EFSCORRUPTED);
4614 }
4615
David Howells1d1fe1e2008-02-07 00:15:37 -08004616 inode = iget_locked(sb, ino);
4617 if (!inode)
4618 return ERR_PTR(-ENOMEM);
4619 if (!(inode->i_state & I_NEW))
4620 return inode;
4621
4622 ei = EXT4_I(inode);
Peter Huewe7dc57612011-02-21 21:01:42 -05004623 iloc.bh = NULL;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004624
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004625 ret = __ext4_get_inode_loc_noinmem(inode, &iloc);
David Howells1d1fe1e2008-02-07 00:15:37 -08004626 if (ret < 0)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004627 goto bad_inode;
Mingming Cao617ba132006-10-11 01:20:53 -07004628 raw_inode = ext4_raw_inode(&iloc);
Darrick J. Wong814525f2012-04-29 18:31:10 -04004629
Theodore Ts'o8e4b5ea2018-03-29 21:56:09 -04004630 if ((ino == EXT4_ROOT_INO) && (raw_inode->i_links_count == 0)) {
Theodore Ts'o8a363972018-12-19 12:29:13 -05004631 ext4_error_inode(inode, function, line, 0,
4632 "iget: root inode unallocated");
Theodore Ts'o8e4b5ea2018-03-29 21:56:09 -04004633 ret = -EFSCORRUPTED;
4634 goto bad_inode;
4635 }
4636
Theodore Ts'o8a363972018-12-19 12:29:13 -05004637 if ((flags & EXT4_IGET_HANDLE) &&
4638 (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
4639 ret = -ESTALE;
4640 goto bad_inode;
4641 }
4642
Darrick J. Wong814525f2012-04-29 18:31:10 -04004643 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4644 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
4645 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
Eric Biggers2dc8d9e2016-12-01 14:43:33 -05004646 EXT4_INODE_SIZE(inode->i_sb) ||
4647 (ei->i_extra_isize & 3)) {
Theodore Ts'o8a363972018-12-19 12:29:13 -05004648 ext4_error_inode(inode, function, line, 0,
4649 "iget: bad extra_isize %u "
4650 "(inode size %u)",
Eric Biggers2dc8d9e2016-12-01 14:43:33 -05004651 ei->i_extra_isize,
4652 EXT4_INODE_SIZE(inode->i_sb));
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004653 ret = -EFSCORRUPTED;
Darrick J. Wong814525f2012-04-29 18:31:10 -04004654 goto bad_inode;
4655 }
4656 } else
4657 ei->i_extra_isize = 0;
4658
4659 /* Precompute checksum seed for inode metadata */
Dmitry Monakhov9aa5d32b2014-10-13 03:36:16 -04004660 if (ext4_has_metadata_csum(sb)) {
Darrick J. Wong814525f2012-04-29 18:31:10 -04004661 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4662 __u32 csum;
4663 __le32 inum = cpu_to_le32(inode->i_ino);
4664 __le32 gen = raw_inode->i_generation;
4665 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,
4666 sizeof(inum));
4667 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,
4668 sizeof(gen));
4669 }
4670
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004671 if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||
4672 ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
4673 (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
4674 ext4_error_inode_err(inode, function, line, 0,
4675 EFSBADCRC, "iget: checksum invalid");
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004676 ret = -EFSBADCRC;
Darrick J. Wong814525f2012-04-29 18:31:10 -04004677 goto bad_inode;
4678 }
4679
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004680 inode->i_mode = le16_to_cpu(raw_inode->i_mode);
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004681 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
4682 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
Kaho Ng0b7b7772016-09-05 23:11:58 -04004683 if (ext4_has_feature_project(sb) &&
Li Xi040cb372016-01-08 16:01:21 -05004684 EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4685 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4686 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
4687 else
4688 i_projid = EXT4_DEF_PROJID;
4689
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04004690 if (!(test_opt(inode->i_sb, NO_UID32))) {
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004691 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
4692 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004693 }
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004694 i_uid_write(inode, i_uid);
4695 i_gid_write(inode, i_gid);
Li Xi040cb372016-01-08 16:01:21 -05004696 ei->i_projid = make_kprojid(&init_user_ns, i_projid);
Miklos Szeredibfe86842011-10-28 14:13:29 +02004697 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004698
Theodore Ts'o353eb832011-01-10 12:18:25 -05004699 ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */
Tao Ma67cf5b02012-12-10 14:04:46 -05004700 ei->i_inline_off = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004701 ei->i_dir_start_lookup = 0;
4702 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4703 /* We now have enough fields to check if the inode was active or not.
4704 * This is needed because nfsd might try to access dead inodes
4705 * the test is that same one that e2fsck uses
4706 * NeilBrown 1999oct15
4707 */
4708 if (inode->i_nlink == 0) {
Dr. Tilmann Bubeck393d1d12013-04-08 12:54:05 -04004709 if ((inode->i_mode == 0 ||
4710 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
4711 ino != EXT4_BOOT_LOADER_INO) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004712 /* this inode is deleted */
David Howells1d1fe1e2008-02-07 00:15:37 -08004713 ret = -ESTALE;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004714 goto bad_inode;
4715 }
4716 /* The only unlinked inodes we let through here have
4717 * valid i_mode and are being read by the orphan
4718 * recovery code: that's fine, we're about to complete
Dr. Tilmann Bubeck393d1d12013-04-08 12:54:05 -04004719 * the process of deleting those.
4720 * OR it is the EXT4_BOOT_LOADER_INO which is
4721 * not initialized on a new filesystem. */
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004722 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004723 ei->i_flags = le32_to_cpu(raw_inode->i_flags);
Ira Weiny043546e2020-05-28 07:59:59 -07004724 ext4_set_inode_flags(inode, true);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004725 inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
Aneesh Kumar K.V7973c0c2008-01-28 23:58:27 -05004726 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004727 if (ext4_has_feature_64bit(sb))
Badari Pulavartya1ddeb72006-10-11 01:21:09 -07004728 ei->i_file_acl |=
4729 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
Artem Blagodarenkoe08ac992017-06-21 21:09:57 -04004730 inode->i_size = ext4_isize(sb, raw_inode);
Darrick J. Wong7e6e1ef2016-12-10 09:55:01 -05004731 if ((size = i_size_read(inode)) < 0) {
Theodore Ts'o8a363972018-12-19 12:29:13 -05004732 ext4_error_inode(inode, function, line, 0,
4733 "iget: bad i_size value: %lld", size);
Darrick J. Wong7e6e1ef2016-12-10 09:55:01 -05004734 ret = -EFSCORRUPTED;
4735 goto bad_inode;
4736 }
Jan Kara48a34312020-02-10 15:43:16 +01004737 /*
4738 * If dir_index is not enabled but there's dir with INDEX flag set,
4739 * we'd normally treat htree data as empty space. But with metadata
4740 * checksumming that corrupts checksums so forbid that.
4741 */
4742 if (!ext4_has_feature_dir_index(sb) && ext4_has_metadata_csum(sb) &&
4743 ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
4744 ext4_error_inode(inode, function, line, 0,
4745 "iget: Dir with htree data on filesystem without dir_index feature.");
4746 ret = -EFSCORRUPTED;
4747 goto bad_inode;
4748 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004749 ei->i_disksize = inode->i_size;
Dmitry Monakhova9e7f442009-12-14 15:21:14 +03004750#ifdef CONFIG_QUOTA
4751 ei->i_reserved_quota = 0;
4752#endif
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004753 inode->i_generation = le32_to_cpu(raw_inode->i_generation);
4754 ei->i_block_group = iloc.block_group;
Theodore Ts'oa4912122009-03-12 12:18:34 -04004755 ei->i_last_alloc_group = ~0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004756 /*
4757 * NOTE! The in-memory inode i_data array is in little-endian order
4758 * even on big-endian machines: we do NOT byteswap the block numbers!
4759 */
Mingming Cao617ba132006-10-11 01:20:53 -07004760 for (block = 0; block < EXT4_N_BLOCKS; block++)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004761 ei->i_data[block] = raw_inode->i_block[block];
4762 INIT_LIST_HEAD(&ei->i_orphan);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07004763 ext4_fc_init_inode(&ei->vfs_inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004764
Jan Karab436b9b2009-12-08 23:51:10 -05004765 /*
4766 * Set transaction id's of transactions that have to be committed
4767 * to finish f[data]sync. We set them to currently running transaction
4768 * as we cannot be sure that the inode or some of its metadata isn't
4769 * part of the transaction - the inode could have been reclaimed and
4770 * now it is reread from disk.
4771 */
4772 if (journal) {
4773 transaction_t *transaction;
4774 tid_t tid;
4775
Theodore Ts'oa931da62010-08-03 21:35:12 -04004776 read_lock(&journal->j_state_lock);
Jan Karab436b9b2009-12-08 23:51:10 -05004777 if (journal->j_running_transaction)
4778 transaction = journal->j_running_transaction;
4779 else
4780 transaction = journal->j_committing_transaction;
4781 if (transaction)
4782 tid = transaction->t_tid;
4783 else
4784 tid = journal->j_commit_sequence;
Theodore Ts'oa931da62010-08-03 21:35:12 -04004785 read_unlock(&journal->j_state_lock);
Jan Karab436b9b2009-12-08 23:51:10 -05004786 ei->i_sync_tid = tid;
4787 ei->i_datasync_tid = tid;
4788 }
4789
Eric Sandeen0040d982008-02-05 22:36:43 -05004790 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004791 if (ei->i_extra_isize == 0) {
4792 /* The extra space is currently unused. Use it. */
Eric Biggers2dc8d9e2016-12-01 14:43:33 -05004793 BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
Mingming Cao617ba132006-10-11 01:20:53 -07004794 ei->i_extra_isize = sizeof(struct ext4_inode) -
4795 EXT4_GOOD_OLD_INODE_SIZE;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004796 } else {
Theodore Ts'oeb9b5f02018-05-22 17:14:07 -04004797 ret = ext4_iget_extra_inode(inode, raw_inode, ei);
4798 if (ret)
4799 goto bad_inode;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004800 }
Darrick J. Wong814525f2012-04-29 18:31:10 -04004801 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004802
Kalpak Shahef7f3832007-07-18 09:15:20 -04004803 EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
4804 EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
4805 EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
4806 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
4807
Theodore Ts'oed3654e2014-03-24 14:09:06 -04004808 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
Jeff Laytonee73f9a2018-01-09 08:21:39 -05004809 u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
4810
Theodore Ts'oc4f65702014-03-20 00:32:57 -04004811 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4812 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
Jeff Laytonee73f9a2018-01-09 08:21:39 -05004813 ivers |=
Theodore Ts'oc4f65702014-03-20 00:32:57 -04004814 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
4815 }
Eryu Guane254d1a2018-05-10 11:55:31 -04004816 ext4_inode_set_iversion_queried(inode, ivers);
Jean Noel Cordenner25ec56b2008-01-28 23:58:27 -05004817 }
4818
Theodore Ts'oc4b5a612009-04-24 18:45:35 -04004819 ret = 0;
Theodore Ts'o485c26e2009-04-24 13:43:20 -04004820 if (ei->i_file_acl &&
Jan Karace9f24c2020-07-28 15:04:34 +02004821 !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
Theodore Ts'o8a363972018-12-19 12:29:13 -05004822 ext4_error_inode(inode, function, line, 0,
4823 "iget: bad extended attribute block %llu",
Theodore Ts'o24676da2010-05-16 21:00:00 -04004824 ei->i_file_acl);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004825 ret = -EFSCORRUPTED;
Theodore Ts'o485c26e2009-04-24 13:43:20 -04004826 goto bad_inode;
Tao Maf19d5872012-12-10 14:05:51 -05004827 } else if (!ext4_has_inline_data(inode)) {
Liu Songbc716522018-08-02 00:11:16 -04004828 /* validate the block references in the inode */
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07004829 if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
4830 (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4831 (S_ISLNK(inode->i_mode) &&
4832 !ext4_inode_is_fast_symlink(inode)))) {
Liu Songbc716522018-08-02 00:11:16 -04004833 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
Tao Maf19d5872012-12-10 14:05:51 -05004834 ret = ext4_ext_check_inode(inode);
Liu Songbc716522018-08-02 00:11:16 -04004835 else
4836 ret = ext4_ind_check_inode(inode);
Tao Maf19d5872012-12-10 14:05:51 -05004837 }
Thiemo Nagelfe2c8192009-03-31 08:36:10 -04004838 }
Theodore Ts'o567f3e92009-11-14 08:19:05 -05004839 if (ret)
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04004840 goto bad_inode;
Aneesh Kumar K.V7a262f72009-03-27 16:39:58 -04004841
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004842 if (S_ISREG(inode->i_mode)) {
Mingming Cao617ba132006-10-11 01:20:53 -07004843 inode->i_op = &ext4_file_inode_operations;
Boaz Harroshbe64f882015-04-15 16:15:17 -07004844 inode->i_fop = &ext4_file_operations;
Mingming Cao617ba132006-10-11 01:20:53 -07004845 ext4_set_aops(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004846 } else if (S_ISDIR(inode->i_mode)) {
Mingming Cao617ba132006-10-11 01:20:53 -07004847 inode->i_op = &ext4_dir_inode_operations;
4848 inode->i_fop = &ext4_dir_operations;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004849 } else if (S_ISLNK(inode->i_mode)) {
Luis R. Rodriguez6390d332018-05-13 16:45:56 -04004850 /* VFS does not allow setting these so must be corruption */
4851 if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
Theodore Ts'o8a363972018-12-19 12:29:13 -05004852 ext4_error_inode(inode, function, line, 0,
4853 "iget: immutable or append flags "
4854 "not allowed on symlinks");
Luis R. Rodriguez6390d332018-05-13 16:45:56 -04004855 ret = -EFSCORRUPTED;
4856 goto bad_inode;
4857 }
Chandan Rajendra592ddec2018-12-12 15:20:10 +05304858 if (IS_ENCRYPTED(inode)) {
Al Viroa7a67e82015-04-27 17:51:30 -04004859 inode->i_op = &ext4_encrypted_symlink_inode_operations;
4860 ext4_set_aops(inode);
4861 } else if (ext4_inode_is_fast_symlink(inode)) {
Al Viro75e75662015-05-02 10:13:58 -04004862 inode->i_link = (char *)ei->i_data;
Mingming Cao617ba132006-10-11 01:20:53 -07004863 inode->i_op = &ext4_fast_symlink_inode_operations;
Duane Griffine83c1392008-12-19 20:47:15 +00004864 nd_terminate_link(ei->i_data, inode->i_size,
4865 sizeof(ei->i_data) - 1);
4866 } else {
Mingming Cao617ba132006-10-11 01:20:53 -07004867 inode->i_op = &ext4_symlink_inode_operations;
4868 ext4_set_aops(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004869 }
Al Viro21fc61c2015-11-17 01:07:57 -05004870 inode_nohighmem(inode);
Theodore Ts'o563bdd62009-03-26 00:06:19 -04004871 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
4872 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
Mingming Cao617ba132006-10-11 01:20:53 -07004873 inode->i_op = &ext4_special_inode_operations;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004874 if (raw_inode->i_block[0])
4875 init_special_inode(inode, inode->i_mode,
4876 old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
4877 else
4878 init_special_inode(inode, inode->i_mode,
4879 new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
Dr. Tilmann Bubeck393d1d12013-04-08 12:54:05 -04004880 } else if (ino == EXT4_BOOT_LOADER_INO) {
4881 make_bad_inode(inode);
Theodore Ts'o563bdd62009-03-26 00:06:19 -04004882 } else {
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004883 ret = -EFSCORRUPTED;
Theodore Ts'o8a363972018-12-19 12:29:13 -05004884 ext4_error_inode(inode, function, line, 0,
4885 "iget: bogus i_mode (%o)", inode->i_mode);
Theodore Ts'o563bdd62009-03-26 00:06:19 -04004886 goto bad_inode;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004887 }
Theodore Ts'o6456ca62019-09-03 01:43:17 -04004888 if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb))
4889 ext4_error_inode(inode, function, line, 0,
4890 "casefold flag without casefold feature");
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04004891 brelse(iloc.bh);
Tahsin Erdogandec214d2017-06-22 11:44:55 -04004892
David Howells1d1fe1e2008-02-07 00:15:37 -08004893 unlock_new_inode(inode);
4894 return inode;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004895
4896bad_inode:
Theodore Ts'o567f3e92009-11-14 08:19:05 -05004897 brelse(iloc.bh);
David Howells1d1fe1e2008-02-07 00:15:37 -08004898 iget_failed(inode);
4899 return ERR_PTR(ret);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004900}
4901
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004902static int ext4_inode_blocks_set(handle_t *handle,
4903 struct ext4_inode *raw_inode,
4904 struct ext4_inode_info *ei)
4905{
4906 struct inode *inode = &(ei->vfs_inode);
Qian Cai28936b62020-02-21 23:32:58 -05004907 u64 i_blocks = READ_ONCE(inode->i_blocks);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004908 struct super_block *sb = inode->i_sb;
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004909
4910 if (i_blocks <= ~0U) {
4911 /*
Anatol Pomozov4907cb72012-09-01 10:31:09 -07004912 * i_blocks can be represented in a 32 bit variable
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004913 * as multiple of 512 bytes
4914 */
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004915 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004916 raw_inode->i_blocks_high = 0;
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004917 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
Theodore Ts'of287a1a2008-10-16 22:50:48 -04004918 return 0;
4919 }
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004920 if (!ext4_has_feature_huge_file(sb))
Theodore Ts'of287a1a2008-10-16 22:50:48 -04004921 return -EFBIG;
4922
4923 if (i_blocks <= 0xffffffffffffULL) {
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004924 /*
4925 * i_blocks can be represented in a 48 bit variable
4926 * as multiple of 512 bytes
4927 */
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004928 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004929 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004930 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004931 } else {
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004932 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004933 /* i_block is stored in file system block size */
4934 i_blocks = i_blocks >> (inode->i_blkbits - 9);
4935 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4936 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004937 }
Theodore Ts'of287a1a2008-10-16 22:50:48 -04004938 return 0;
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004939}
4940
David Howells3f19b2a2017-12-01 11:40:16 +00004941static void __ext4_update_other_inode_time(struct super_block *sb,
4942 unsigned long orig_ino,
4943 unsigned long ino,
4944 struct ext4_inode *raw_inode)
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004945{
David Howells3f19b2a2017-12-01 11:40:16 +00004946 struct inode *inode;
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004947
David Howells3f19b2a2017-12-01 11:40:16 +00004948 inode = find_inode_by_ino_rcu(sb, ino);
4949 if (!inode)
4950 return;
4951
4952 if ((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
Christoph Hellwig0e11f642018-02-21 07:54:49 -08004953 I_DIRTY_INODE)) ||
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004954 ((inode->i_state & I_DIRTY_TIME) == 0))
David Howells3f19b2a2017-12-01 11:40:16 +00004955 return;
4956
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004957 spin_lock(&inode->i_lock);
4958 if (((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
Christoph Hellwig0e11f642018-02-21 07:54:49 -08004959 I_DIRTY_INODE)) == 0) &&
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004960 (inode->i_state & I_DIRTY_TIME)) {
4961 struct ext4_inode_info *ei = EXT4_I(inode);
4962
Jan Kara5fcd5752020-05-29 16:24:43 +02004963 inode->i_state &= ~I_DIRTY_TIME;
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004964 spin_unlock(&inode->i_lock);
4965
4966 spin_lock(&ei->i_raw_lock);
David Howells3f19b2a2017-12-01 11:40:16 +00004967 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
4968 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
4969 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
4970 ext4_inode_csum_set(inode, raw_inode, ei);
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004971 spin_unlock(&ei->i_raw_lock);
David Howells3f19b2a2017-12-01 11:40:16 +00004972 trace_ext4_other_inode_update_time(inode, orig_ino);
4973 return;
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004974 }
4975 spin_unlock(&inode->i_lock);
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004976}
4977
4978/*
4979 * Opportunistically update the other time fields for other inodes in
4980 * the same inode table block.
4981 */
4982static void ext4_update_other_inodes_time(struct super_block *sb,
4983 unsigned long orig_ino, char *buf)
4984{
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004985 unsigned long ino;
4986 int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4987 int inode_size = EXT4_INODE_SIZE(sb);
4988
Theodore Ts'o0f0ff9a2015-07-01 23:37:46 -04004989 /*
4990 * Calculate the first inode in the inode table block. Inode
4991 * numbers are one-based. That is, the first inode in a block
4992 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
4993 */
4994 ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
David Howells3f19b2a2017-12-01 11:40:16 +00004995 rcu_read_lock();
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004996 for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
4997 if (ino == orig_ino)
4998 continue;
David Howells3f19b2a2017-12-01 11:40:16 +00004999 __ext4_update_other_inode_time(sb, orig_ino, ino,
5000 (struct ext4_inode *)buf);
Theodore Ts'oa26f4992015-02-02 00:37:02 -05005001 }
David Howells3f19b2a2017-12-01 11:40:16 +00005002 rcu_read_unlock();
Theodore Ts'oa26f4992015-02-02 00:37:02 -05005003}
5004
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005005/*
5006 * Post the struct inode info into an on-disk inode location in the
5007 * buffer-cache. This gobbles the caller's reference to the
5008 * buffer_head in the inode location struct.
5009 *
5010 * The caller must have write access to iloc->bh.
5011 */
Mingming Cao617ba132006-10-11 01:20:53 -07005012static int ext4_do_update_inode(handle_t *handle,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005013 struct inode *inode,
Frank Mayhar830156c2009-09-29 10:07:47 -04005014 struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005015{
Mingming Cao617ba132006-10-11 01:20:53 -07005016 struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
5017 struct ext4_inode_info *ei = EXT4_I(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005018 struct buffer_head *bh = iloc->bh;
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005019 struct super_block *sb = inode->i_sb;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005020 int err = 0, rc, block;
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005021 int need_datasync = 0, set_large_file = 0;
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005022 uid_t i_uid;
5023 gid_t i_gid;
Li Xi040cb372016-01-08 16:01:21 -05005024 projid_t i_projid;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005025
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005026 spin_lock(&ei->i_raw_lock);
5027
5028 /* For fields not tracked in the in-memory inode,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005029 * initialise them to zero for new inodes. */
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05005030 if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
Mingming Cao617ba132006-10-11 01:20:53 -07005031 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005032
Luo Meng13221812020-10-20 09:36:31 +08005033 err = ext4_inode_blocks_set(handle, raw_inode, ei);
5034 if (err) {
5035 spin_unlock(&ei->i_raw_lock);
5036 goto out_brelse;
5037 }
5038
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005039 raw_inode->i_mode = cpu_to_le16(inode->i_mode);
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005040 i_uid = i_uid_read(inode);
5041 i_gid = i_gid_read(inode);
Li Xi040cb372016-01-08 16:01:21 -05005042 i_projid = from_kprojid(&init_user_ns, ei->i_projid);
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04005043 if (!(test_opt(inode->i_sb, NO_UID32))) {
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005044 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
5045 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005046/*
5047 * Fix up interoperability with old kernels. Otherwise, old inodes get
5048 * re-used with the upper 16 bits of the uid/gid intact
5049 */
Daeho Jeong93e3b4e2016-09-05 22:56:10 -04005050 if (ei->i_dtime && list_empty(&ei->i_orphan)) {
5051 raw_inode->i_uid_high = 0;
5052 raw_inode->i_gid_high = 0;
5053 } else {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005054 raw_inode->i_uid_high =
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005055 cpu_to_le16(high_16_bits(i_uid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005056 raw_inode->i_gid_high =
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005057 cpu_to_le16(high_16_bits(i_gid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005058 }
5059 } else {
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005060 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
5061 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005062 raw_inode->i_uid_high = 0;
5063 raw_inode->i_gid_high = 0;
5064 }
5065 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
Kalpak Shahef7f3832007-07-18 09:15:20 -04005066
5067 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
5068 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
5069 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
5070 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
5071
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005072 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
Theodore Ts'o353eb832011-01-10 12:18:25 -05005073 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
Theodore Ts'oed3654e2014-03-24 14:09:06 -04005074 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
Badari Pulavartya1ddeb72006-10-11 01:21:09 -07005075 raw_inode->i_file_acl_high =
5076 cpu_to_le16(ei->i_file_acl >> 32);
Aneesh Kumar K.V7973c0c2008-01-28 23:58:27 -05005077 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
Qiujun Huangdce8e232020-02-24 23:02:46 +08005078 if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode)) {
Jan Karab71fc072012-09-26 21:52:20 -04005079 ext4_isize_set(raw_inode, ei->i_disksize);
5080 need_datasync = 1;
5081 }
Aneesh Kumar K.Va48380f2008-01-28 23:58:27 -05005082 if (ei->i_disksize > 0x7fffffffULL) {
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04005083 if (!ext4_has_feature_large_file(sb) ||
Aneesh Kumar K.Va48380f2008-01-28 23:58:27 -05005084 EXT4_SB(sb)->s_es->s_rev_level ==
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005085 cpu_to_le32(EXT4_GOOD_OLD_REV))
5086 set_large_file = 1;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005087 }
5088 raw_inode->i_generation = cpu_to_le32(inode->i_generation);
5089 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
5090 if (old_valid_dev(inode->i_rdev)) {
5091 raw_inode->i_block[0] =
5092 cpu_to_le32(old_encode_dev(inode->i_rdev));
5093 raw_inode->i_block[1] = 0;
5094 } else {
5095 raw_inode->i_block[0] = 0;
5096 raw_inode->i_block[1] =
5097 cpu_to_le32(new_encode_dev(inode->i_rdev));
5098 raw_inode->i_block[2] = 0;
5099 }
Tao Maf19d5872012-12-10 14:05:51 -05005100 } else if (!ext4_has_inline_data(inode)) {
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04005101 for (block = 0; block < EXT4_N_BLOCKS; block++)
5102 raw_inode->i_block[block] = ei->i_data[block];
Tao Maf19d5872012-12-10 14:05:51 -05005103 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005104
Theodore Ts'oed3654e2014-03-24 14:09:06 -04005105 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
Eryu Guane254d1a2018-05-10 11:55:31 -04005106 u64 ivers = ext4_inode_peek_iversion(inode);
Jeff Laytonee73f9a2018-01-09 08:21:39 -05005107
5108 raw_inode->i_disk_version = cpu_to_le32(ivers);
Theodore Ts'oc4f65702014-03-20 00:32:57 -04005109 if (ei->i_extra_isize) {
5110 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
5111 raw_inode->i_version_hi =
Jeff Laytonee73f9a2018-01-09 08:21:39 -05005112 cpu_to_le32(ivers >> 32);
Theodore Ts'oc4f65702014-03-20 00:32:57 -04005113 raw_inode->i_extra_isize =
5114 cpu_to_le16(ei->i_extra_isize);
5115 }
Jean Noel Cordenner25ec56b2008-01-28 23:58:27 -05005116 }
Li Xi040cb372016-01-08 16:01:21 -05005117
Kaho Ng0b7b7772016-09-05 23:11:58 -04005118 BUG_ON(!ext4_has_feature_project(inode->i_sb) &&
Li Xi040cb372016-01-08 16:01:21 -05005119 i_projid != EXT4_DEF_PROJID);
5120
5121 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
5122 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
5123 raw_inode->i_projid = cpu_to_le32(i_projid);
5124
Darrick J. Wong814525f2012-04-29 18:31:10 -04005125 ext4_inode_csum_set(inode, raw_inode, ei);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005126 spin_unlock(&ei->i_raw_lock);
Linus Torvalds1751e8a2017-11-27 13:05:09 -08005127 if (inode->i_sb->s_flags & SB_LAZYTIME)
Theodore Ts'oa26f4992015-02-02 00:37:02 -05005128 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
5129 bh->b_data);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005130
Frank Mayhar830156c2009-09-29 10:07:47 -04005131 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
Curt Wohlgemuth73b50c12010-02-16 15:06:29 -05005132 rc = ext4_handle_dirty_metadata(handle, NULL, bh);
Frank Mayhar830156c2009-09-29 10:07:47 -04005133 if (!err)
5134 err = rc;
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05005135 ext4_clear_inode_state(inode, EXT4_STATE_NEW);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005136 if (set_large_file) {
liang xie5d601252014-05-12 22:06:43 -04005137 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005138 err = ext4_journal_get_write_access(handle, EXT4_SB(sb)->s_sbh);
5139 if (err)
5140 goto out_brelse;
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04005141 ext4_set_feature_large_file(sb);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04005142 ext4_handle_sync(handle);
5143 err = ext4_handle_dirty_super(handle, sb);
5144 }
Jan Karab71fc072012-09-26 21:52:20 -04005145 ext4_update_inode_fsync_trans(handle, inode, need_datasync);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005146out_brelse:
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04005147 brelse(bh);
Mingming Cao617ba132006-10-11 01:20:53 -07005148 ext4_std_error(inode->i_sb, err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005149 return err;
5150}
5151
5152/*
Mingming Cao617ba132006-10-11 01:20:53 -07005153 * ext4_write_inode()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005154 *
5155 * We are called from a few places:
5156 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04005157 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005158 * Here, there will be no transaction running. We wait for any running
Anatol Pomozov4907cb72012-09-01 10:31:09 -07005159 * transaction to commit.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005160 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04005161 * - Within flush work (sys_sync(), kupdate and such).
5162 * We wait on commit, if told to.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005163 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04005164 * - Within iput_final() -> write_inode_now()
5165 * We wait on commit, if told to.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005166 *
5167 * In all cases it is actually safe for us to return without doing anything,
5168 * because the inode has been copied into a raw inode buffer in
Theodore Ts'o87f7e412014-04-08 11:38:28 -04005169 * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL
5170 * writeback.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005171 *
5172 * Note that we are absolutely dependent upon all inode dirtiers doing the
5173 * right thing: they *must* call mark_inode_dirty() after dirtying info in
5174 * which we are interested.
5175 *
5176 * It would be a bug for them to not do this. The code:
5177 *
5178 * mark_inode_dirty(inode)
5179 * stuff();
5180 * inode->i_size = expr;
5181 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04005182 * is in error because write_inode() could occur while `stuff()' is running,
5183 * and the new i_size will be lost. Plus the inode will no longer be on the
5184 * superblock's dirty inode list.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005185 */
Christoph Hellwiga9185b42010-03-05 09:21:37 +01005186int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005187{
Frank Mayhar91ac6f42009-09-09 22:33:47 -04005188 int err;
5189
Theodore Ts'o18f2c4f2018-12-19 14:36:58 -05005190 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC) ||
5191 sb_rdonly(inode->i_sb))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005192 return 0;
5193
Theodore Ts'o18f2c4f2018-12-19 14:36:58 -05005194 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5195 return -EIO;
5196
Frank Mayhar91ac6f42009-09-09 22:33:47 -04005197 if (EXT4_SB(inode->i_sb)->s_journal) {
5198 if (ext4_journal_current_handle()) {
5199 jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n");
5200 dump_stack();
5201 return -EIO;
5202 }
5203
Jan Kara10542c22014-03-04 10:50:50 -05005204 /*
5205 * No need to force transaction in WB_SYNC_NONE mode. Also
5206 * ext4_sync_fs() will force the commit after everything is
5207 * written.
5208 */
5209 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
Frank Mayhar91ac6f42009-09-09 22:33:47 -04005210 return 0;
5211
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005212 err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
Theodore Ts'o18f2c4f2018-12-19 14:36:58 -05005213 EXT4_I(inode)->i_sync_tid);
Frank Mayhar91ac6f42009-09-09 22:33:47 -04005214 } else {
5215 struct ext4_iloc iloc;
5216
Harshad Shirwadkar8016e292020-10-15 13:37:59 -07005217 err = __ext4_get_inode_loc_noinmem(inode, &iloc);
Frank Mayhar91ac6f42009-09-09 22:33:47 -04005218 if (err)
5219 return err;
Jan Kara10542c22014-03-04 10:50:50 -05005220 /*
5221 * sync(2) will flush the whole buffer cache. No need to do
5222 * it here separately for each inode.
5223 */
5224 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
Frank Mayhar830156c2009-09-29 10:07:47 -04005225 sync_dirty_buffer(iloc.bh);
5226 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
Theodore Ts'o54d3adb2020-03-28 19:33:43 -04005227 ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
5228 "IO error syncing inode");
Frank Mayhar830156c2009-09-29 10:07:47 -04005229 err = -EIO;
5230 }
Curt Wohlgemuthfd2dd9f2010-04-03 17:44:16 -04005231 brelse(iloc.bh);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005232 }
Frank Mayhar91ac6f42009-09-09 22:33:47 -04005233 return err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005234}
5235
5236/*
Jan Kara53e87262012-12-25 13:29:52 -05005237 * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate
5238 * buffers that are attached to a page stradding i_size and are undergoing
5239 * commit. In that case we have to wait for commit to finish and try again.
5240 */
5241static void ext4_wait_for_tail_page_commit(struct inode *inode)
5242{
5243 struct page *page;
5244 unsigned offset;
5245 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
5246 tid_t commit_tid = 0;
5247 int ret;
5248
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005249 offset = inode->i_size & (PAGE_SIZE - 1);
Jan Kara53e87262012-12-25 13:29:52 -05005250 /*
yangerkun565333a2019-09-19 14:35:08 +08005251 * If the page is fully truncated, we don't need to wait for any commit
5252 * (and we even should not as __ext4_journalled_invalidatepage() may
5253 * strip all buffers from the page but keep the page dirty which can then
5254 * confuse e.g. concurrent ext4_writepage() seeing dirty page without
5255 * buffers). Also we don't need to wait for any commit if all buffers in
5256 * the page remain valid. This is most beneficial for the common case of
5257 * blocksize == PAGESIZE.
Jan Kara53e87262012-12-25 13:29:52 -05005258 */
yangerkun565333a2019-09-19 14:35:08 +08005259 if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
Jan Kara53e87262012-12-25 13:29:52 -05005260 return;
5261 while (1) {
5262 page = find_lock_page(inode->i_mapping,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005263 inode->i_size >> PAGE_SHIFT);
Jan Kara53e87262012-12-25 13:29:52 -05005264 if (!page)
5265 return;
Lukas Czernerca99fdd2013-05-21 23:25:01 -04005266 ret = __ext4_journalled_invalidatepage(page, offset,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005267 PAGE_SIZE - offset);
Jan Kara53e87262012-12-25 13:29:52 -05005268 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005269 put_page(page);
Jan Kara53e87262012-12-25 13:29:52 -05005270 if (ret != -EBUSY)
5271 return;
5272 commit_tid = 0;
5273 read_lock(&journal->j_state_lock);
5274 if (journal->j_committing_transaction)
5275 commit_tid = journal->j_committing_transaction->t_tid;
5276 read_unlock(&journal->j_state_lock);
5277 if (commit_tid)
5278 jbd2_log_wait_commit(journal, commit_tid);
5279 }
5280}
5281
5282/*
Mingming Cao617ba132006-10-11 01:20:53 -07005283 * ext4_setattr()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005284 *
5285 * Called from notify_change.
5286 *
5287 * We want to trap VFS attempts to truncate the file as soon as
5288 * possible. In particular, we want to make sure that when the VFS
5289 * shrinks i_size, we put the inode on the orphan list and modify
5290 * i_disksize immediately, so that during the subsequent flushing of
5291 * dirty pages and freeing of disk blocks, we can guarantee that any
5292 * commit will leave the blocks being flushed in an unused state on
5293 * disk. (On recovery, the inode will get truncated and the blocks will
5294 * be freed, so we have a strong guarantee that no future commit will
5295 * leave these blocks visible to the user.)
5296 *
Jan Kara678aaf42008-07-11 19:27:31 -04005297 * Another thing we have to assure is that if we are in ordered mode
5298 * and inode is still attached to the committing transaction, we must
5299 * we start writeout of all the dirty pages which are being truncated.
5300 * This way we are sure that all the data written in the previous
5301 * transaction are already on disk (truncate waits for pages under
5302 * writeback).
5303 *
5304 * Called with inode->i_mutex down.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005305 */
Mingming Cao617ba132006-10-11 01:20:53 -07005306int ext4_setattr(struct dentry *dentry, struct iattr *attr)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005307{
David Howells2b0143b2015-03-17 22:25:59 +00005308 struct inode *inode = d_inode(dentry);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005309 int error, rc = 0;
Dmitry Monakhov3d287de2010-10-27 22:08:46 -04005310 int orphan = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005311 const unsigned int ia_valid = attr->ia_valid;
5312
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05005313 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5314 return -EIO;
5315
Theodore Ts'o02b016c2019-06-09 22:04:33 -04005316 if (unlikely(IS_IMMUTABLE(inode)))
5317 return -EPERM;
5318
5319 if (unlikely(IS_APPEND(inode) &&
5320 (ia_valid & (ATTR_MODE | ATTR_UID |
5321 ATTR_GID | ATTR_TIMES_SET))))
5322 return -EPERM;
5323
Jan Kara31051c82016-05-26 16:55:18 +02005324 error = setattr_prepare(dentry, attr);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005325 if (error)
5326 return error;
5327
Eric Biggers3ce2b8d2017-10-18 20:21:58 -04005328 error = fscrypt_prepare_setattr(dentry, attr);
5329 if (error)
5330 return error;
5331
Eric Biggersc93d8f82019-07-22 09:26:24 -07005332 error = fsverity_prepare_setattr(dentry, attr);
5333 if (error)
5334 return error;
5335
Jan Karaa7cdade2015-06-29 16:22:54 +02005336 if (is_quota_modification(inode, attr)) {
5337 error = dquot_initialize(inode);
5338 if (error)
5339 return error;
5340 }
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005341 ext4_fc_start_update(inode);
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005342 if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) ||
5343 (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005344 handle_t *handle;
5345
5346 /* (user+group)*(old+new) structure, inode write (sb,
5347 * inode block, ? - but truncate inode update has it) */
Theodore Ts'o9924a922013-02-08 21:59:22 -05005348 handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5349 (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5350 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005351 if (IS_ERR(handle)) {
5352 error = PTR_ERR(handle);
5353 goto err_out;
5354 }
Tahsin Erdogan7a9ca532017-06-22 11:46:48 -04005355
5356 /* dquot_transfer() calls back ext4_get_inode_usage() which
5357 * counts xattr inode references.
5358 */
5359 down_read(&EXT4_I(inode)->xattr_sem);
Christoph Hellwigb43fa822010-03-03 09:05:03 -05005360 error = dquot_transfer(inode, attr);
Tahsin Erdogan7a9ca532017-06-22 11:46:48 -04005361 up_read(&EXT4_I(inode)->xattr_sem);
5362
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005363 if (error) {
Mingming Cao617ba132006-10-11 01:20:53 -07005364 ext4_journal_stop(handle);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005365 ext4_fc_stop_update(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005366 return error;
5367 }
5368 /* Update corresponding info in inode so that everything is in
5369 * one transaction */
5370 if (attr->ia_valid & ATTR_UID)
5371 inode->i_uid = attr->ia_uid;
5372 if (attr->ia_valid & ATTR_GID)
5373 inode->i_gid = attr->ia_gid;
Mingming Cao617ba132006-10-11 01:20:53 -07005374 error = ext4_mark_inode_dirty(handle, inode);
5375 ext4_journal_stop(handle);
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07005376 if (unlikely(error))
5377 return error;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005378 }
5379
Josef Bacik3da40c7b02015-06-22 00:31:26 -04005380 if (attr->ia_valid & ATTR_SIZE) {
Jan Kara52083862013-08-17 10:07:17 -04005381 handle_t *handle;
Josef Bacik3da40c7b02015-06-22 00:31:26 -04005382 loff_t oldsize = inode->i_size;
Jan Karab9c1c262019-05-30 11:56:23 -04005383 int shrink = (attr->ia_size < inode->i_size);
Christoph Hellwig562c72aa52011-06-24 14:29:45 -04005384
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04005385 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
Eric Sandeene2b46572008-01-28 23:58:27 -05005386 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5387
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005388 if (attr->ia_size > sbi->s_bitmap_maxbytes) {
5389 ext4_fc_stop_update(inode);
Theodore Ts'o0c095c72010-07-27 11:56:06 -04005390 return -EFBIG;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005391 }
Eric Sandeene2b46572008-01-28 23:58:27 -05005392 }
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005393 if (!S_ISREG(inode->i_mode)) {
5394 ext4_fc_stop_update(inode);
Josef Bacik3da40c7b02015-06-22 00:31:26 -04005395 return -EINVAL;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005396 }
Christoph Hellwigdff6efc2013-11-19 07:17:07 -08005397
5398 if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size)
5399 inode_inc_iversion(inode);
5400
Jan Karab9c1c262019-05-30 11:56:23 -04005401 if (shrink) {
5402 if (ext4_should_order_data(inode)) {
5403 error = ext4_begin_ordered_truncate(inode,
Jan Kara678aaf42008-07-11 19:27:31 -04005404 attr->ia_size);
Jan Karab9c1c262019-05-30 11:56:23 -04005405 if (error)
5406 goto err_out;
5407 }
5408 /*
5409 * Blocks are going to be removed from the inode. Wait
5410 * for dio in flight.
5411 */
5412 inode_dio_wait(inode);
Josef Bacik3da40c7b02015-06-22 00:31:26 -04005413 }
Jan Karab9c1c262019-05-30 11:56:23 -04005414
5415 down_write(&EXT4_I(inode)->i_mmap_sem);
5416
5417 rc = ext4_break_layouts(inode);
5418 if (rc) {
5419 up_write(&EXT4_I(inode)->i_mmap_sem);
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005420 goto err_out;
Jan Karab9c1c262019-05-30 11:56:23 -04005421 }
5422
Josef Bacik3da40c7b02015-06-22 00:31:26 -04005423 if (attr->ia_size != inode->i_size) {
Jan Kara52083862013-08-17 10:07:17 -04005424 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5425 if (IS_ERR(handle)) {
5426 error = PTR_ERR(handle);
Jan Karab9c1c262019-05-30 11:56:23 -04005427 goto out_mmap_sem;
Jan Kara52083862013-08-17 10:07:17 -04005428 }
Josef Bacik3da40c7b02015-06-22 00:31:26 -04005429 if (ext4_handle_valid(handle) && shrink) {
Jan Kara52083862013-08-17 10:07:17 -04005430 error = ext4_orphan_add(handle, inode);
5431 orphan = 1;
5432 }
Eryu Guan911af572015-07-28 15:08:41 -04005433 /*
5434 * Update c/mtime on truncate up, ext4_truncate() will
5435 * update c/mtime in shrink case below
5436 */
5437 if (!shrink) {
Deepa Dinamanieeca7ea2016-11-14 21:40:10 -05005438 inode->i_mtime = current_time(inode);
Eryu Guan911af572015-07-28 15:08:41 -04005439 inode->i_ctime = inode->i_mtime;
5440 }
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005441
5442 if (shrink)
5443 ext4_fc_track_range(inode,
5444 (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5445 inode->i_sb->s_blocksize_bits,
5446 (oldsize > 0 ? oldsize - 1 : 0) >>
5447 inode->i_sb->s_blocksize_bits);
5448 else
5449 ext4_fc_track_range(
5450 inode,
5451 (oldsize > 0 ? oldsize - 1 : oldsize) >>
5452 inode->i_sb->s_blocksize_bits,
5453 (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5454 inode->i_sb->s_blocksize_bits);
5455
Jan Kara90e775b2013-08-17 10:09:31 -04005456 down_write(&EXT4_I(inode)->i_data_sem);
Jan Kara52083862013-08-17 10:07:17 -04005457 EXT4_I(inode)->i_disksize = attr->ia_size;
5458 rc = ext4_mark_inode_dirty(handle, inode);
5459 if (!error)
5460 error = rc;
Jan Kara90e775b2013-08-17 10:09:31 -04005461 /*
5462 * We have to update i_size under i_data_sem together
5463 * with i_disksize to avoid races with writeback code
5464 * running ext4_wb_update_i_disksize().
5465 */
5466 if (!error)
5467 i_size_write(inode, attr->ia_size);
5468 up_write(&EXT4_I(inode)->i_data_sem);
Jan Kara52083862013-08-17 10:07:17 -04005469 ext4_journal_stop(handle);
Jan Karab9c1c262019-05-30 11:56:23 -04005470 if (error)
5471 goto out_mmap_sem;
5472 if (!shrink) {
5473 pagecache_isize_extended(inode, oldsize,
5474 inode->i_size);
5475 } else if (ext4_should_journal_data(inode)) {
5476 ext4_wait_for_tail_page_commit(inode);
Jan Kara678aaf42008-07-11 19:27:31 -04005477 }
Jan Karad6320cb2014-10-01 21:49:46 -04005478 }
Ross Zwisler430657b2018-07-29 17:00:22 -04005479
Jan Kara52083862013-08-17 10:07:17 -04005480 /*
5481 * Truncate pagecache after we've waited for commit
5482 * in data=journal mode to make pages freeable.
5483 */
Ross Zwisler923ae0f2015-02-16 15:59:38 -08005484 truncate_pagecache(inode, inode->i_size);
Jan Karab9c1c262019-05-30 11:56:23 -04005485 /*
5486 * Call ext4_truncate() even if i_size didn't change to
5487 * truncate possible preallocated blocks.
5488 */
5489 if (attr->ia_size <= oldsize) {
Theodore Ts'o2c98eb52016-11-13 22:02:26 -05005490 rc = ext4_truncate(inode);
5491 if (rc)
5492 error = rc;
5493 }
Jan Karab9c1c262019-05-30 11:56:23 -04005494out_mmap_sem:
Jan Karaea3d7202015-12-07 14:28:03 -05005495 up_write(&EXT4_I(inode)->i_mmap_sem);
Theodore Ts'o072bd7e2011-05-23 15:13:02 -04005496 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005497
Theodore Ts'o2c98eb52016-11-13 22:02:26 -05005498 if (!error) {
Christoph Hellwig10257742010-06-04 11:30:02 +02005499 setattr_copy(inode, attr);
5500 mark_inode_dirty(inode);
5501 }
5502
5503 /*
5504 * If the call to ext4_truncate failed to get a transaction handle at
5505 * all, we need to clean up the in-core orphan list manually.
5506 */
Dmitry Monakhov3d287de2010-10-27 22:08:46 -04005507 if (orphan && inode->i_nlink)
Mingming Cao617ba132006-10-11 01:20:53 -07005508 ext4_orphan_del(NULL, inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005509
Theodore Ts'o2c98eb52016-11-13 22:02:26 -05005510 if (!error && (ia_valid & ATTR_MODE))
Christoph Hellwig64e178a2013-12-20 05:16:44 -08005511 rc = posix_acl_chmod(inode, inode->i_mode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005512
5513err_out:
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005514 if (error)
5515 ext4_std_error(inode->i_sb, error);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005516 if (!error)
5517 error = rc;
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005518 ext4_fc_stop_update(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005519 return error;
5520}
5521
David Howellsa528d352017-01-31 16:46:22 +00005522int ext4_getattr(const struct path *path, struct kstat *stat,
5523 u32 request_mask, unsigned int query_flags)
Mingming Cao3e3398a2008-07-11 19:27:31 -04005524{
David Howells99652ea2017-03-31 18:31:56 +01005525 struct inode *inode = d_inode(path->dentry);
5526 struct ext4_inode *raw_inode;
5527 struct ext4_inode_info *ei = EXT4_I(inode);
5528 unsigned int flags;
Mingming Cao3e3398a2008-07-11 19:27:31 -04005529
Theodore Ts'od4c5e962019-11-28 22:26:51 -05005530 if ((request_mask & STATX_BTIME) &&
5531 EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
David Howells99652ea2017-03-31 18:31:56 +01005532 stat->result_mask |= STATX_BTIME;
5533 stat->btime.tv_sec = ei->i_crtime.tv_sec;
5534 stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
5535 }
5536
5537 flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
5538 if (flags & EXT4_APPEND_FL)
5539 stat->attributes |= STATX_ATTR_APPEND;
5540 if (flags & EXT4_COMPR_FL)
5541 stat->attributes |= STATX_ATTR_COMPRESSED;
5542 if (flags & EXT4_ENCRYPT_FL)
5543 stat->attributes |= STATX_ATTR_ENCRYPTED;
5544 if (flags & EXT4_IMMUTABLE_FL)
5545 stat->attributes |= STATX_ATTR_IMMUTABLE;
5546 if (flags & EXT4_NODUMP_FL)
5547 stat->attributes |= STATX_ATTR_NODUMP;
Eric Biggers1f607192019-10-29 13:41:39 -07005548 if (flags & EXT4_VERITY_FL)
5549 stat->attributes |= STATX_ATTR_VERITY;
David Howells99652ea2017-03-31 18:31:56 +01005550
David Howells3209f682017-03-31 18:32:17 +01005551 stat->attributes_mask |= (STATX_ATTR_APPEND |
5552 STATX_ATTR_COMPRESSED |
5553 STATX_ATTR_ENCRYPTED |
5554 STATX_ATTR_IMMUTABLE |
Eric Biggers1f607192019-10-29 13:41:39 -07005555 STATX_ATTR_NODUMP |
5556 STATX_ATTR_VERITY);
David Howells3209f682017-03-31 18:32:17 +01005557
Mingming Cao3e3398a2008-07-11 19:27:31 -04005558 generic_fillattr(inode, stat);
David Howells99652ea2017-03-31 18:31:56 +01005559 return 0;
5560}
5561
5562int ext4_file_getattr(const struct path *path, struct kstat *stat,
5563 u32 request_mask, unsigned int query_flags)
5564{
5565 struct inode *inode = d_inode(path->dentry);
5566 u64 delalloc_blocks;
5567
5568 ext4_getattr(path, stat, request_mask, query_flags);
Mingming Cao3e3398a2008-07-11 19:27:31 -04005569
5570 /*
Andreas Dilger9206c5612013-11-11 22:38:12 -05005571 * If there is inline data in the inode, the inode will normally not
5572 * have data blocks allocated (it may have an external xattr block).
5573 * Report at least one sector for such files, so tools like tar, rsync,
Theodore Ts'od67d64f2017-03-25 17:33:31 -04005574 * others don't incorrectly think the file is completely sparse.
Andreas Dilger9206c5612013-11-11 22:38:12 -05005575 */
5576 if (unlikely(ext4_has_inline_data(inode)))
5577 stat->blocks += (stat->size + 511) >> 9;
5578
5579 /*
Mingming Cao3e3398a2008-07-11 19:27:31 -04005580 * We can't update i_blocks if the block allocation is delayed
5581 * otherwise in the case of system crash before the real block
5582 * allocation is done, we will have i_blocks inconsistent with
5583 * on-disk file blocks.
5584 * We always keep i_blocks updated together with real
5585 * allocation. But to not confuse with user, stat
5586 * will return the blocks that include the delayed allocation
5587 * blocks for this file.
5588 */
Tao Ma96607552012-05-31 22:54:16 -04005589 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
Andreas Dilger9206c5612013-11-11 22:38:12 -05005590 EXT4_I(inode)->i_reserved_data_blocks);
5591 stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
Mingming Cao3e3398a2008-07-11 19:27:31 -04005592 return 0;
5593}
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005594
Jan Karafffb2732013-06-04 13:01:11 -04005595static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
5596 int pextents)
Mingming Caoa02908f2008-08-19 22:16:07 -04005597{
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04005598 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
Jan Karafffb2732013-06-04 13:01:11 -04005599 return ext4_ind_trans_blocks(inode, lblocks);
5600 return ext4_ext_index_trans_blocks(inode, pextents);
Mingming Caoa02908f2008-08-19 22:16:07 -04005601}
Theodore Ts'oac51d832008-11-06 16:49:36 -05005602
Mingming Caoa02908f2008-08-19 22:16:07 -04005603/*
5604 * Account for index blocks, block groups bitmaps and block group
5605 * descriptor blocks if modify datablocks and index blocks
5606 * worse case, the indexs blocks spread over different block groups
5607 *
5608 * If datablocks are discontiguous, they are possible to spread over
Anatol Pomozov4907cb72012-09-01 10:31:09 -07005609 * different block groups too. If they are contiguous, with flexbg,
Mingming Caoa02908f2008-08-19 22:16:07 -04005610 * they could still across block group boundary.
5611 *
5612 * Also account for superblock, inode, quota and xattr blocks
5613 */
Tahsin Erdogandec214d2017-06-22 11:44:55 -04005614static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
Jan Karafffb2732013-06-04 13:01:11 -04005615 int pextents)
Mingming Caoa02908f2008-08-19 22:16:07 -04005616{
Theodore Ts'o8df96752009-05-01 08:50:38 -04005617 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
5618 int gdpblocks;
Mingming Caoa02908f2008-08-19 22:16:07 -04005619 int idxblocks;
5620 int ret = 0;
5621
5622 /*
Jan Karafffb2732013-06-04 13:01:11 -04005623 * How many index blocks need to touch to map @lblocks logical blocks
5624 * to @pextents physical extents?
Mingming Caoa02908f2008-08-19 22:16:07 -04005625 */
Jan Karafffb2732013-06-04 13:01:11 -04005626 idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
Mingming Caoa02908f2008-08-19 22:16:07 -04005627
5628 ret = idxblocks;
5629
5630 /*
5631 * Now let's see how many group bitmaps and group descriptors need
5632 * to account
5633 */
Jan Karafffb2732013-06-04 13:01:11 -04005634 groups = idxblocks + pextents;
Mingming Caoa02908f2008-08-19 22:16:07 -04005635 gdpblocks = groups;
Theodore Ts'o8df96752009-05-01 08:50:38 -04005636 if (groups > ngroups)
5637 groups = ngroups;
Mingming Caoa02908f2008-08-19 22:16:07 -04005638 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
5639 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
5640
5641 /* bitmaps and block group descriptor blocks */
5642 ret += groups + gdpblocks;
5643
5644 /* Blocks for super block, inode, quota and xattr blocks */
5645 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005646
5647 return ret;
5648}
5649
5650/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03005651 * Calculate the total number of credits to reserve to fit
Mingming Caof3bd1f32008-08-19 22:16:03 -04005652 * the modification of a single pages into a single transaction,
5653 * which may include multiple chunks of block allocations.
Mingming Caoa02908f2008-08-19 22:16:07 -04005654 *
Mingming Cao525f4ed2008-08-19 22:15:58 -04005655 * This could be called via ext4_write_begin()
Mingming Caoa02908f2008-08-19 22:16:07 -04005656 *
Mingming Cao525f4ed2008-08-19 22:15:58 -04005657 * We need to consider the worse case, when
Mingming Caoa02908f2008-08-19 22:16:07 -04005658 * one new block per extent.
Mingming Caoa02908f2008-08-19 22:16:07 -04005659 */
5660int ext4_writepage_trans_blocks(struct inode *inode)
5661{
5662 int bpp = ext4_journal_blocks_per_page(inode);
5663 int ret;
5664
Jan Karafffb2732013-06-04 13:01:11 -04005665 ret = ext4_meta_trans_blocks(inode, bpp, bpp);
Mingming Caoa02908f2008-08-19 22:16:07 -04005666
5667 /* Account for data blocks for journalled mode */
5668 if (ext4_should_journal_data(inode))
5669 ret += bpp;
5670 return ret;
5671}
Mingming Caof3bd1f32008-08-19 22:16:03 -04005672
5673/*
5674 * Calculate the journal credits for a chunk of data modification.
5675 *
5676 * This is called from DIO, fallocate or whoever calling
Eric Sandeen79e83032010-07-27 11:56:07 -04005677 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
Mingming Caof3bd1f32008-08-19 22:16:03 -04005678 *
5679 * journal buffers for data blocks are not included here, as DIO
5680 * and fallocate do no need to journal data buffers.
5681 */
5682int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
5683{
5684 return ext4_meta_trans_blocks(inode, nrblocks, 1);
5685}
5686
Mingming Caoa02908f2008-08-19 22:16:07 -04005687/*
Mingming Cao617ba132006-10-11 01:20:53 -07005688 * The caller must have previously called ext4_reserve_inode_write().
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005689 * Give this, we know that the caller already has write access to iloc->bh.
5690 */
Mingming Cao617ba132006-10-11 01:20:53 -07005691int ext4_mark_iloc_dirty(handle_t *handle,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04005692 struct inode *inode, struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005693{
5694 int err = 0;
5695
Vasily Averina6758302018-11-06 16:49:50 -05005696 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
5697 put_bh(iloc->bh);
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05005698 return -EIO;
Vasily Averina6758302018-11-06 16:49:50 -05005699 }
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07005700 ext4_fc_track_inode(inode);
5701
Theodore Ts'oc64db502012-03-02 12:23:11 -05005702 if (IS_I_VERSION(inode))
Jean Noel Cordenner25ec56b2008-01-28 23:58:27 -05005703 inode_inc_iversion(inode);
5704
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005705 /* the do_update_inode consumes one bh->b_count */
5706 get_bh(iloc->bh);
5707
Mingming Caodab291a2006-10-11 01:21:01 -07005708 /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
Frank Mayhar830156c2009-09-29 10:07:47 -04005709 err = ext4_do_update_inode(handle, inode, iloc);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005710 put_bh(iloc->bh);
5711 return err;
5712}
5713
5714/*
5715 * On success, We end up with an outstanding reference count against
5716 * iloc->bh. This _must_ be cleaned up later.
5717 */
5718
5719int
Mingming Cao617ba132006-10-11 01:20:53 -07005720ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
5721 struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005722{
Frank Mayhar03901312009-01-07 00:06:22 -05005723 int err;
5724
Theodore Ts'o0db1ff22017-02-05 01:28:48 -05005725 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5726 return -EIO;
5727
Frank Mayhar03901312009-01-07 00:06:22 -05005728 err = ext4_get_inode_loc(inode, iloc);
5729 if (!err) {
5730 BUFFER_TRACE(iloc->bh, "get_write_access");
5731 err = ext4_journal_get_write_access(handle, iloc->bh);
5732 if (err) {
5733 brelse(iloc->bh);
5734 iloc->bh = NULL;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005735 }
5736 }
Mingming Cao617ba132006-10-11 01:20:53 -07005737 ext4_std_error(inode->i_sb, err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005738 return err;
5739}
5740
Miao Xiec03b45b2017-08-06 01:00:49 -04005741static int __ext4_expand_extra_isize(struct inode *inode,
5742 unsigned int new_extra_isize,
5743 struct ext4_iloc *iloc,
5744 handle_t *handle, int *no_expand)
5745{
5746 struct ext4_inode *raw_inode;
5747 struct ext4_xattr_ibody_header *header;
Theodore Ts'o4ea99932019-11-07 21:43:41 -05005748 unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
5749 struct ext4_inode_info *ei = EXT4_I(inode);
Miao Xiec03b45b2017-08-06 01:00:49 -04005750 int error;
5751
Theodore Ts'o4ea99932019-11-07 21:43:41 -05005752 /* this was checked at iget time, but double check for good measure */
5753 if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
5754 (ei->i_extra_isize & 3)) {
5755 EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
5756 ei->i_extra_isize,
5757 EXT4_INODE_SIZE(inode->i_sb));
5758 return -EFSCORRUPTED;
5759 }
5760 if ((new_extra_isize < ei->i_extra_isize) ||
5761 (new_extra_isize < 4) ||
5762 (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
5763 return -EINVAL; /* Should never happen */
5764
Miao Xiec03b45b2017-08-06 01:00:49 -04005765 raw_inode = ext4_raw_inode(iloc);
5766
5767 header = IHDR(inode, raw_inode);
5768
5769 /* No extended attributes present */
5770 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5771 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
5772 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
5773 EXT4_I(inode)->i_extra_isize, 0,
5774 new_extra_isize - EXT4_I(inode)->i_extra_isize);
5775 EXT4_I(inode)->i_extra_isize = new_extra_isize;
5776 return 0;
5777 }
5778
5779 /* try to expand with EAs present */
5780 error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
5781 raw_inode, handle);
5782 if (error) {
5783 /*
5784 * Inode size expansion failed; don't try again
5785 */
5786 *no_expand = 1;
5787 }
5788
5789 return error;
5790}
5791
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005792/*
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005793 * Expand an inode by new_extra_isize bytes.
5794 * Returns 0 on success or negative error number on failure.
5795 */
Miao Xiecf0a5e82017-08-06 00:40:01 -04005796static int ext4_try_to_expand_extra_isize(struct inode *inode,
5797 unsigned int new_extra_isize,
5798 struct ext4_iloc iloc,
5799 handle_t *handle)
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005800{
Miao Xie3b10fdc2017-08-06 00:27:38 -04005801 int no_expand;
5802 int error;
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005803
Miao Xiecf0a5e82017-08-06 00:40:01 -04005804 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
5805 return -EOVERFLOW;
5806
5807 /*
5808 * In nojournal mode, we can immediately attempt to expand
5809 * the inode. When journaled, we first need to obtain extra
5810 * buffer credits since we may write into the EA block
5811 * with this same handle. If journal_extend fails, then it will
5812 * only result in a minor loss of functionality for that inode.
5813 * If this is felt to be critical, then e2fsck should be run to
5814 * force a large enough s_min_extra_isize.
5815 */
Jan Kara6cb367c2019-11-05 17:44:14 +01005816 if (ext4_journal_extend(handle,
Jan Kara83448bd2019-11-05 17:44:29 +01005817 EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)
Miao Xiecf0a5e82017-08-06 00:40:01 -04005818 return -ENOSPC;
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005819
Miao Xie3b10fdc2017-08-06 00:27:38 -04005820 if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
Miao Xiecf0a5e82017-08-06 00:40:01 -04005821 return -EBUSY;
Miao Xie3b10fdc2017-08-06 00:27:38 -04005822
Miao Xiec03b45b2017-08-06 01:00:49 -04005823 error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
5824 handle, &no_expand);
Miao Xie3b10fdc2017-08-06 00:27:38 -04005825 ext4_write_unlock_xattr(inode, &no_expand);
Miao Xiecf0a5e82017-08-06 00:40:01 -04005826
Miao Xie3b10fdc2017-08-06 00:27:38 -04005827 return error;
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005828}
5829
Miao Xiec03b45b2017-08-06 01:00:49 -04005830int ext4_expand_extra_isize(struct inode *inode,
5831 unsigned int new_extra_isize,
5832 struct ext4_iloc *iloc)
5833{
5834 handle_t *handle;
5835 int no_expand;
5836 int error, rc;
5837
5838 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
5839 brelse(iloc->bh);
5840 return -EOVERFLOW;
5841 }
5842
5843 handle = ext4_journal_start(inode, EXT4_HT_INODE,
5844 EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
5845 if (IS_ERR(handle)) {
5846 error = PTR_ERR(handle);
5847 brelse(iloc->bh);
5848 return error;
5849 }
5850
5851 ext4_write_lock_xattr(inode, &no_expand);
5852
zhangyi (F)ddccb6d2019-02-21 11:29:10 -05005853 BUFFER_TRACE(iloc->bh, "get_write_access");
Miao Xiec03b45b2017-08-06 01:00:49 -04005854 error = ext4_journal_get_write_access(handle, iloc->bh);
5855 if (error) {
5856 brelse(iloc->bh);
Dan Carpenter7f420d642019-12-13 21:50:11 +03005857 goto out_unlock;
Miao Xiec03b45b2017-08-06 01:00:49 -04005858 }
5859
5860 error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
5861 handle, &no_expand);
5862
5863 rc = ext4_mark_iloc_dirty(handle, inode, iloc);
5864 if (!error)
5865 error = rc;
5866
Dan Carpenter7f420d642019-12-13 21:50:11 +03005867out_unlock:
Miao Xiec03b45b2017-08-06 01:00:49 -04005868 ext4_write_unlock_xattr(inode, &no_expand);
Miao Xiec03b45b2017-08-06 01:00:49 -04005869 ext4_journal_stop(handle);
5870 return error;
5871}
5872
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005873/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005874 * What we do here is to mark the in-core inode as clean with respect to inode
5875 * dirtiness (it may still be data-dirty).
5876 * This means that the in-core inode may be reaped by prune_icache
5877 * without having to perform any I/O. This is a very good thing,
5878 * because *any* task may call prune_icache - even ones which
5879 * have a transaction open against a different journal.
5880 *
5881 * Is this cheating? Not really. Sure, we haven't written the
5882 * inode out, but prune_icache isn't a user-visible syncing function.
5883 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
5884 * we start and wait on commits.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005885 */
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07005886int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
5887 const char *func, unsigned int line)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005888{
Mingming Cao617ba132006-10-11 01:20:53 -07005889 struct ext4_iloc iloc;
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005890 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Miao Xiecf0a5e82017-08-06 00:40:01 -04005891 int err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005892
5893 might_sleep();
Theodore Ts'o7ff9c072010-11-08 13:51:33 -05005894 trace_ext4_mark_inode_dirty(inode, _RET_IP_);
Mingming Cao617ba132006-10-11 01:20:53 -07005895 err = ext4_reserve_inode_write(handle, inode, &iloc);
Eryu Guan5e1021f2016-03-12 21:40:32 -05005896 if (err)
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07005897 goto out;
Miao Xiecf0a5e82017-08-06 00:40:01 -04005898
5899 if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
5900 ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
5901 iloc, handle);
5902
Harshad Shirwadkar4209ae12020-04-26 18:34:37 -07005903 err = ext4_mark_iloc_dirty(handle, inode, &iloc);
5904out:
5905 if (unlikely(err))
5906 ext4_error_inode_err(inode, func, line, 0, err,
5907 "mark_inode_dirty error");
5908 return err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005909}
5910
5911/*
Mingming Cao617ba132006-10-11 01:20:53 -07005912 * ext4_dirty_inode() is called from __mark_inode_dirty()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005913 *
5914 * We're really interested in the case where a file is being extended.
5915 * i_size has been changed by generic_commit_write() and we thus need
5916 * to include the updated inode in the current transaction.
5917 *
Christoph Hellwig5dd40562010-03-03 09:05:00 -05005918 * Also, dquot_alloc_block() will always dirty the inode when blocks
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005919 * are allocated to the file.
5920 *
5921 * If the inode is marked synchronous, we don't honour that here - doing
5922 * so would cause a commit on atime updates, which we don't bother doing.
5923 * We handle synchronous inodes at the highest possible level.
Theodore Ts'o0ae45f62015-02-02 00:37:00 -05005924 *
5925 * If only the I_DIRTY_TIME flag is set, we can skip everything. If
5926 * I_DIRTY_TIME and I_DIRTY_SYNC is set, the only inode fields we need
5927 * to copy into the on-disk inode structure are the timestamp files.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005928 */
Christoph Hellwigaa385722011-05-27 06:53:02 -04005929void ext4_dirty_inode(struct inode *inode, int flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005930{
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005931 handle_t *handle;
5932
Theodore Ts'o0ae45f62015-02-02 00:37:00 -05005933 if (flags == I_DIRTY_TIME)
5934 return;
Theodore Ts'o9924a922013-02-08 21:59:22 -05005935 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005936 if (IS_ERR(handle))
5937 goto out;
Curt Wohlgemuthf3dc2722009-09-29 16:06:01 -04005938
Curt Wohlgemuthf3dc2722009-09-29 16:06:01 -04005939 ext4_mark_inode_dirty(handle, inode);
5940
Mingming Cao617ba132006-10-11 01:20:53 -07005941 ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005942out:
5943 return;
5944}
5945
Mingming Cao617ba132006-10-11 01:20:53 -07005946int ext4_change_inode_journal_flag(struct inode *inode, int val)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005947{
5948 journal_t *journal;
5949 handle_t *handle;
5950 int err;
Daeho Jeongc8585c62016-04-25 23:22:35 -04005951 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005952
5953 /*
5954 * We have to be very careful here: changing a data block's
5955 * journaling status dynamically is dangerous. If we write a
5956 * data block to the journal, change the status and then delete
5957 * that block, we risk forgetting to revoke the old log record
5958 * from the journal and so a subsequent replay can corrupt data.
5959 * So, first we make sure that the journal is empty and that
5960 * nobody is changing anything.
5961 */
5962
Mingming Cao617ba132006-10-11 01:20:53 -07005963 journal = EXT4_JOURNAL(inode);
Frank Mayhar03901312009-01-07 00:06:22 -05005964 if (!journal)
5965 return 0;
Dave Hansend6995942007-07-18 08:33:51 -04005966 if (is_journal_aborted(journal))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005967 return -EROFS;
5968
Dmitry Monakhov17335dc2012-09-29 00:41:21 -04005969 /* Wait for all existing dio workers */
Dmitry Monakhov17335dc2012-09-29 00:41:21 -04005970 inode_dio_wait(inode);
5971
Daeho Jeong4c546592016-04-25 23:21:00 -04005972 /*
5973 * Before flushing the journal and switching inode's aops, we have
5974 * to flush all dirty data the inode has. There can be outstanding
5975 * delayed allocations, there can be unwritten extents created by
5976 * fallocate or buffered writes in dioread_nolock mode covered by
5977 * dirty data which can be converted only after flushing the dirty
5978 * data (and journalled aops don't know how to handle these cases).
5979 */
5980 if (val) {
5981 down_write(&EXT4_I(inode)->i_mmap_sem);
5982 err = filemap_write_and_wait(inode->i_mapping);
5983 if (err < 0) {
5984 up_write(&EXT4_I(inode)->i_mmap_sem);
Daeho Jeong4c546592016-04-25 23:21:00 -04005985 return err;
5986 }
5987 }
5988
Eric Biggersbbd559372020-02-19 10:30:46 -08005989 percpu_down_write(&sbi->s_writepages_rwsem);
Mingming Caodab291a2006-10-11 01:21:01 -07005990 jbd2_journal_lock_updates(journal);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005991
5992 /*
5993 * OK, there are no updates running now, and all cached data is
5994 * synced to disk. We are now in a completely consistent state
5995 * which doesn't have anything in the journal, and we know that
5996 * no filesystem updates are running, so it is safe to modify
5997 * the inode's in-core data-journaling state flag now.
5998 */
5999
6000 if (val)
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04006001 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
Yongqiang Yang5872dda2011-12-28 13:55:51 -05006002 else {
Jan Kara4f879ca2014-10-30 10:53:17 -04006003 err = jbd2_journal_flush(journal);
6004 if (err < 0) {
6005 jbd2_journal_unlock_updates(journal);
Eric Biggersbbd559372020-02-19 10:30:46 -08006006 percpu_up_write(&sbi->s_writepages_rwsem);
Jan Kara4f879ca2014-10-30 10:53:17 -04006007 return err;
6008 }
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04006009 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
Yongqiang Yang5872dda2011-12-28 13:55:51 -05006010 }
Mingming Cao617ba132006-10-11 01:20:53 -07006011 ext4_set_aops(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07006012
Mingming Caodab291a2006-10-11 01:21:01 -07006013 jbd2_journal_unlock_updates(journal);
Eric Biggersbbd559372020-02-19 10:30:46 -08006014 percpu_up_write(&sbi->s_writepages_rwsem);
Daeho Jeongc8585c62016-04-25 23:22:35 -04006015
Daeho Jeong4c546592016-04-25 23:21:00 -04006016 if (val)
6017 up_write(&EXT4_I(inode)->i_mmap_sem);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07006018
6019 /* Finally we can mark the inode as dirty. */
6020
Theodore Ts'o9924a922013-02-08 21:59:22 -05006021 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07006022 if (IS_ERR(handle))
6023 return PTR_ERR(handle);
6024
Harshad Shirwadkaraa75f4d2020-10-15 13:37:57 -07006025 ext4_fc_mark_ineligible(inode->i_sb,
6026 EXT4_FC_REASON_JOURNAL_FLAG_CHANGE);
Mingming Cao617ba132006-10-11 01:20:53 -07006027 err = ext4_mark_inode_dirty(handle, inode);
Frank Mayhar03901312009-01-07 00:06:22 -05006028 ext4_handle_sync(handle);
Mingming Cao617ba132006-10-11 01:20:53 -07006029 ext4_journal_stop(handle);
6030 ext4_std_error(inode->i_sb, err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07006031
6032 return err;
6033}
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006034
6035static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh)
6036{
6037 return !buffer_mapped(bh);
6038}
6039
Souptick Joarder401b25a2018-10-02 22:20:50 -04006040vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006041{
Dave Jiang11bac802017-02-24 14:56:41 -08006042 struct vm_area_struct *vma = vmf->vma;
Nick Pigginc2ec1752009-03-31 15:23:21 -07006043 struct page *page = vmf->page;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006044 loff_t size;
6045 unsigned long len;
Souptick Joarder401b25a2018-10-02 22:20:50 -04006046 int err;
6047 vm_fault_t ret;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006048 struct file *file = vma->vm_file;
Al Viro496ad9a2013-01-23 17:07:38 -05006049 struct inode *inode = file_inode(file);
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006050 struct address_space *mapping = inode->i_mapping;
Jan Kara9ea7df52011-06-24 14:29:41 -04006051 handle_t *handle;
6052 get_block_t *get_block;
6053 int retries = 0;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006054
Theodore Ts'o02b016c2019-06-09 22:04:33 -04006055 if (unlikely(IS_IMMUTABLE(inode)))
6056 return VM_FAULT_SIGBUS;
6057
Jan Kara8e8ad8a2012-06-12 16:20:38 +02006058 sb_start_pagefault(inode->i_sb);
Theodore Ts'o041bbb6d2012-09-30 23:04:56 -04006059 file_update_time(vma->vm_file);
Jan Karaea3d7202015-12-07 14:28:03 -05006060
6061 down_read(&EXT4_I(inode)->i_mmap_sem);
Eric Biggers7b4cc972017-04-30 00:10:50 -04006062
Souptick Joarder401b25a2018-10-02 22:20:50 -04006063 err = ext4_convert_inline_data(inode);
6064 if (err)
Eric Biggers7b4cc972017-04-30 00:10:50 -04006065 goto out_ret;
6066
Mauricio Faria de Oliveira64a9f142020-10-05 21:48:40 -03006067 /*
6068 * On data journalling we skip straight to the transaction handle:
6069 * there's no delalloc; page truncated will be checked later; the
6070 * early return w/ all buffers mapped (calculates size/len) can't
6071 * be used; and there's no dioread_nolock, so only ext4_get_block.
6072 */
6073 if (ext4_should_journal_data(inode))
6074 goto retry_alloc;
6075
Jan Kara9ea7df52011-06-24 14:29:41 -04006076 /* Delalloc case is easy... */
6077 if (test_opt(inode->i_sb, DELALLOC) &&
Jan Kara9ea7df52011-06-24 14:29:41 -04006078 !ext4_nonda_switch(inode->i_sb)) {
6079 do {
Souptick Joarder401b25a2018-10-02 22:20:50 -04006080 err = block_page_mkwrite(vma, vmf,
Jan Kara9ea7df52011-06-24 14:29:41 -04006081 ext4_da_get_block_prep);
Souptick Joarder401b25a2018-10-02 22:20:50 -04006082 } while (err == -ENOSPC &&
Jan Kara9ea7df52011-06-24 14:29:41 -04006083 ext4_should_retry_alloc(inode->i_sb, &retries));
6084 goto out_ret;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006085 }
Darrick J. Wong0e499892011-05-18 13:55:20 -04006086
6087 lock_page(page);
Jan Kara9ea7df52011-06-24 14:29:41 -04006088 size = i_size_read(inode);
6089 /* Page got truncated from under us? */
6090 if (page->mapping != mapping || page_offset(page) > size) {
6091 unlock_page(page);
6092 ret = VM_FAULT_NOPAGE;
6093 goto out;
Darrick J. Wong0e499892011-05-18 13:55:20 -04006094 }
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006095
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03006096 if (page->index == size >> PAGE_SHIFT)
6097 len = size & ~PAGE_MASK;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006098 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03006099 len = PAGE_SIZE;
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04006100 /*
Jan Kara9ea7df52011-06-24 14:29:41 -04006101 * Return if we have all the buffers mapped. This avoids the need to do
6102 * journal_start/journal_stop which can block and take a long time
Mauricio Faria de Oliveira64a9f142020-10-05 21:48:40 -03006103 *
6104 * This cannot be done for data journalling, as we have to add the
6105 * inode to the transaction's list to writeprotect pages on commit.
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04006106 */
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006107 if (page_has_buffers(page)) {
Tao Maf19d5872012-12-10 14:05:51 -05006108 if (!ext4_walk_page_buffers(NULL, page_buffers(page),
6109 0, len, NULL,
6110 ext4_bh_unmapped)) {
Jan Kara9ea7df52011-06-24 14:29:41 -04006111 /* Wait so that we don't change page under IO */
Darrick J. Wong1d1d1a72013-02-21 16:42:51 -08006112 wait_for_stable_page(page);
Jan Kara9ea7df52011-06-24 14:29:41 -04006113 ret = VM_FAULT_LOCKED;
6114 goto out;
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04006115 }
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006116 }
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04006117 unlock_page(page);
Jan Kara9ea7df52011-06-24 14:29:41 -04006118 /* OK, we need to fill the hole... */
6119 if (ext4_should_dioread_nolock(inode))
Jan Kara705965b2016-03-08 23:08:10 -05006120 get_block = ext4_get_block_unwritten;
Jan Kara9ea7df52011-06-24 14:29:41 -04006121 else
6122 get_block = ext4_get_block;
6123retry_alloc:
Theodore Ts'o9924a922013-02-08 21:59:22 -05006124 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
6125 ext4_writepage_trans_blocks(inode));
Jan Kara9ea7df52011-06-24 14:29:41 -04006126 if (IS_ERR(handle)) {
Nick Pigginc2ec1752009-03-31 15:23:21 -07006127 ret = VM_FAULT_SIGBUS;
Jan Kara9ea7df52011-06-24 14:29:41 -04006128 goto out;
6129 }
Mauricio Faria de Oliveira64a9f142020-10-05 21:48:40 -03006130 /*
6131 * Data journalling can't use block_page_mkwrite() because it
6132 * will set_buffer_dirty() before do_journal_get_write_access()
6133 * thus might hit warning messages for dirty metadata buffers.
6134 */
6135 if (!ext4_should_journal_data(inode)) {
6136 err = block_page_mkwrite(vma, vmf, get_block);
6137 } else {
6138 lock_page(page);
6139 size = i_size_read(inode);
6140 /* Page got truncated from under us? */
6141 if (page->mapping != mapping || page_offset(page) > size) {
Mauricio Faria de Oliveira64a9f142020-10-05 21:48:40 -03006142 ret = VM_FAULT_NOPAGE;
Mauricio Faria de Oliveiraafb585a2020-10-05 21:48:41 -03006143 goto out_error;
Jan Kara9ea7df52011-06-24 14:29:41 -04006144 }
Mauricio Faria de Oliveira64a9f142020-10-05 21:48:40 -03006145
6146 if (page->index == size >> PAGE_SHIFT)
6147 len = size & ~PAGE_MASK;
6148 else
6149 len = PAGE_SIZE;
6150
6151 err = __block_write_begin(page, 0, len, ext4_get_block);
6152 if (!err) {
Mauricio Faria de Oliveiraafb585a2020-10-05 21:48:41 -03006153 ret = VM_FAULT_SIGBUS;
Mauricio Faria de Oliveira64a9f142020-10-05 21:48:40 -03006154 if (ext4_walk_page_buffers(handle, page_buffers(page),
Mauricio Faria de Oliveiraafb585a2020-10-05 21:48:41 -03006155 0, len, NULL, do_journal_get_write_access))
6156 goto out_error;
6157 if (ext4_walk_page_buffers(handle, page_buffers(page),
6158 0, len, NULL, write_end_fn))
6159 goto out_error;
6160 if (ext4_jbd2_inode_add_write(handle, inode, 0, len))
6161 goto out_error;
Mauricio Faria de Oliveira64a9f142020-10-05 21:48:40 -03006162 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
6163 } else {
6164 unlock_page(page);
6165 }
Jan Kara9ea7df52011-06-24 14:29:41 -04006166 }
6167 ext4_journal_stop(handle);
Souptick Joarder401b25a2018-10-02 22:20:50 -04006168 if (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
Jan Kara9ea7df52011-06-24 14:29:41 -04006169 goto retry_alloc;
6170out_ret:
Souptick Joarder401b25a2018-10-02 22:20:50 -04006171 ret = block_page_mkwrite_return(err);
Jan Kara9ea7df52011-06-24 14:29:41 -04006172out:
Jan Karaea3d7202015-12-07 14:28:03 -05006173 up_read(&EXT4_I(inode)->i_mmap_sem);
Jan Kara8e8ad8a2012-06-12 16:20:38 +02006174 sb_end_pagefault(inode->i_sb);
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006175 return ret;
Mauricio Faria de Oliveiraafb585a2020-10-05 21:48:41 -03006176out_error:
6177 unlock_page(page);
6178 ext4_journal_stop(handle);
6179 goto out;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04006180}
Jan Karaea3d7202015-12-07 14:28:03 -05006181
Souptick Joarder401b25a2018-10-02 22:20:50 -04006182vm_fault_t ext4_filemap_fault(struct vm_fault *vmf)
Jan Karaea3d7202015-12-07 14:28:03 -05006183{
Dave Jiang11bac802017-02-24 14:56:41 -08006184 struct inode *inode = file_inode(vmf->vma->vm_file);
Souptick Joarder401b25a2018-10-02 22:20:50 -04006185 vm_fault_t ret;
Jan Karaea3d7202015-12-07 14:28:03 -05006186
6187 down_read(&EXT4_I(inode)->i_mmap_sem);
Souptick Joarder401b25a2018-10-02 22:20:50 -04006188 ret = filemap_fault(vmf);
Jan Karaea3d7202015-12-07 14:28:03 -05006189 up_read(&EXT4_I(inode)->i_mmap_sem);
6190
Souptick Joarder401b25a2018-10-02 22:20:50 -04006191 return ret;
Jan Karaea3d7202015-12-07 14:28:03 -05006192}