blob: 382c1ef9a9e4d6e9df05e39031ce0f5958e51663 [file] [log] [blame]
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +09001/*
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/buffer_head.h>
14#include <linux/mpage.h>
15#include <linux/writeback.h>
16#include <linux/backing-dev.h>
Chao Yu8f46dca2015-07-14 18:56:10 +080017#include <linux/pagevec.h>
Jaegeuk Kimeb47b802012-11-02 17:10:12 +090018#include <linux/blkdev.h>
19#include <linux/bio.h>
Geert Uytterhoeven690e4a32012-12-19 22:19:30 +010020#include <linux/prefetch.h>
Christoph Hellwige2e40f22015-02-22 08:58:50 -080021#include <linux/uio.h>
Jaegeuk Kimf1e88662015-04-09 11:20:42 -070022#include <linux/cleancache.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010023#include <linux/sched/signal.h>
Jaegeuk Kimeb47b802012-11-02 17:10:12 +090024
25#include "f2fs.h"
26#include "node.h"
27#include "segment.h"
Jaegeuk Kimdb9f7c12014-12-17 20:04:08 -080028#include "trace.h"
Namjae Jeon848753a2013-04-23 16:38:02 +090029#include <trace/events/f2fs.h>
Jaegeuk Kimeb47b802012-11-02 17:10:12 +090030
Eric Biggers6dbb1792018-04-18 11:09:48 -070031#define NUM_PREALLOC_POST_READ_CTXS 128
32
33static struct kmem_cache *bio_post_read_ctx_cache;
34static mempool_t *bio_post_read_ctx_pool;
35
Chao Yu36951b32016-11-16 10:41:20 +080036static bool __is_cp_guaranteed(struct page *page)
37{
38 struct address_space *mapping = page->mapping;
39 struct inode *inode;
40 struct f2fs_sb_info *sbi;
41
42 if (!mapping)
43 return false;
44
45 inode = mapping->host;
46 sbi = F2FS_I_SB(inode);
47
48 if (inode->i_ino == F2FS_META_INO(sbi) ||
49 inode->i_ino == F2FS_NODE_INO(sbi) ||
50 S_ISDIR(inode->i_mode) ||
Chao Yue7a4feb2018-05-08 14:06:03 +080051 (S_ISREG(inode->i_mode) &&
52 is_inode_flag_set(inode, FI_ATOMIC_FILE)) ||
Chao Yu36951b32016-11-16 10:41:20 +080053 is_cold_data(page))
54 return true;
55 return false;
56}
57
Eric Biggers6dbb1792018-04-18 11:09:48 -070058/* postprocessing steps for read bios */
59enum bio_post_read_step {
60 STEP_INITIAL = 0,
61 STEP_DECRYPT,
62};
63
64struct bio_post_read_ctx {
65 struct bio *bio;
66 struct work_struct work;
67 unsigned int cur_step;
68 unsigned int enabled_steps;
69};
70
71static void __read_end_io(struct bio *bio)
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +090072{
Eric Biggers6dbb1792018-04-18 11:09:48 -070073 struct page *page;
74 struct bio_vec *bv;
Linus Torvaldsf5688492014-01-30 11:19:05 -080075 int i;
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +090076
Eric Biggers6dbb1792018-04-18 11:09:48 -070077 bio_for_each_segment_all(bv, bio, i) {
78 page = bv->bv_page;
79
80 /* PG_error was set if any post_read step failed */
81 if (bio->bi_status || PageError(page)) {
82 ClearPageUptodate(page);
83 SetPageError(page);
84 } else {
85 SetPageUptodate(page);
86 }
87 unlock_page(page);
88 }
89 if (bio->bi_private)
90 mempool_free(bio->bi_private, bio_post_read_ctx_pool);
91 bio_put(bio);
92}
93
94static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
95
96static void decrypt_work(struct work_struct *work)
97{
98 struct bio_post_read_ctx *ctx =
99 container_of(work, struct bio_post_read_ctx, work);
100
101 fscrypt_decrypt_bio(ctx->bio);
102
103 bio_post_read_processing(ctx);
104}
105
106static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
107{
108 switch (++ctx->cur_step) {
109 case STEP_DECRYPT:
110 if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {
111 INIT_WORK(&ctx->work, decrypt_work);
112 fscrypt_enqueue_decrypt_work(&ctx->work);
113 return;
114 }
115 ctx->cur_step++;
116 /* fall-through */
117 default:
118 __read_end_io(ctx->bio);
119 }
120}
121
122static bool f2fs_bio_post_read_required(struct bio *bio)
123{
124 return bio->bi_private && !bio->bi_status;
125}
126
127static void f2fs_read_end_io(struct bio *bio)
128{
Ming Lei263663c2017-12-18 20:22:04 +0800129 if (time_to_inject(F2FS_P_SB(bio_first_page_all(bio)), FAULT_IO)) {
Chao Yu55523512017-02-25 11:08:28 +0800130 f2fs_show_injection_info(FAULT_IO);
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200131 bio->bi_status = BLK_STS_IOERR;
Chao Yu55523512017-02-25 11:08:28 +0800132 }
Chao Yu8b038c72016-09-18 23:30:07 +0800133
Eric Biggers6dbb1792018-04-18 11:09:48 -0700134 if (f2fs_bio_post_read_required(bio)) {
135 struct bio_post_read_ctx *ctx = bio->bi_private;
136
137 ctx->cur_step = STEP_INITIAL;
138 bio_post_read_processing(ctx);
139 return;
Chao Yu12377022015-05-25 18:03:38 +0800140 }
141
Eric Biggers6dbb1792018-04-18 11:09:48 -0700142 __read_end_io(bio);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900143}
144
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200145static void f2fs_write_end_io(struct bio *bio)
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900146{
Jaegeuk Kim1b1f5592014-02-03 10:50:22 +0900147 struct f2fs_sb_info *sbi = bio->bi_private;
Linus Torvaldsf5688492014-01-30 11:19:05 -0800148 struct bio_vec *bvec;
149 int i;
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900150
Linus Torvaldsf5688492014-01-30 11:19:05 -0800151 bio_for_each_segment_all(bvec, bio, i) {
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900152 struct page *page = bvec->bv_page;
Chao Yu36951b32016-11-16 10:41:20 +0800153 enum count_type type = WB_DATA_TYPE(page);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900154
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800155 if (IS_DUMMY_WRITTEN_PAGE(page)) {
156 set_page_private(page, (unsigned long)NULL);
157 ClearPagePrivate(page);
158 unlock_page(page);
159 mempool_free(page, sbi->write_io_dummy);
160
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200161 if (unlikely(bio->bi_status))
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800162 f2fs_stop_checkpoint(sbi, true);
163 continue;
164 }
165
Jaegeuk Kim0b81d072015-05-15 16:26:10 -0700166 fscrypt_pullback_bio_page(&page, true);
Jaegeuk Kim4375a332015-04-23 12:04:33 -0700167
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200168 if (unlikely(bio->bi_status)) {
Michal Hocko5114a972016-10-11 13:56:01 -0700169 mapping_set_error(page->mapping, -EIO);
Jaegeuk Kimb1ca3212017-12-31 16:26:38 -0800170 if (type == F2FS_WB_CP_DATA)
171 f2fs_stop_checkpoint(sbi, true);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900172 }
Yunlei He7dff55d2018-01-11 14:19:32 +0800173
174 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
175 page->index != nid_of_node(page));
176
Chao Yu36951b32016-11-16 10:41:20 +0800177 dec_page_count(sbi, type);
Chao Yu50fa53e2018-08-02 23:03:19 +0800178 if (f2fs_in_warm_node_list(sbi, page))
179 f2fs_del_fsync_node_entry(sbi, page);
Chao Yu36951b32016-11-16 10:41:20 +0800180 clear_cold_data(page);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900181 end_page_writeback(page);
Linus Torvaldsf5688492014-01-30 11:19:05 -0800182 }
Chao Yu36951b32016-11-16 10:41:20 +0800183 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
Jaegeuk Kimf5730182016-05-17 16:23:36 -0700184 wq_has_sleeper(&sbi->cp_wait))
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900185 wake_up(&sbi->cp_wait);
186
187 bio_put(bio);
188}
189
Gu Zheng940a6d32013-12-20 17:39:59 +0800190/*
Jaegeuk Kim3c62be12016-10-06 19:02:05 -0700191 * Return true, if pre_bio's bdev is same as its target device.
192 */
193struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
194 block_t blk_addr, struct bio *bio)
195{
196 struct block_device *bdev = sbi->sb->s_bdev;
197 int i;
198
199 for (i = 0; i < sbi->s_ndevs; i++) {
200 if (FDEV(i).start_blk <= blk_addr &&
201 FDEV(i).end_blk >= blk_addr) {
202 blk_addr -= FDEV(i).start_blk;
203 bdev = FDEV(i).bdev;
204 break;
205 }
206 }
207 if (bio) {
Christoph Hellwig74d46992017-08-23 19:10:32 +0200208 bio_set_dev(bio, bdev);
Jaegeuk Kim3c62be12016-10-06 19:02:05 -0700209 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
210 }
211 return bdev;
212}
213
214int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
215{
216 int i;
217
218 for (i = 0; i < sbi->s_ndevs; i++)
219 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
220 return i;
221 return 0;
222}
223
224static bool __same_bdev(struct f2fs_sb_info *sbi,
225 block_t blk_addr, struct bio *bio)
226{
Christoph Hellwig74d46992017-08-23 19:10:32 +0200227 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
228 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
Jaegeuk Kim3c62be12016-10-06 19:02:05 -0700229}
230
231/*
Gu Zheng940a6d32013-12-20 17:39:59 +0800232 * Low-level block read/write IO operations.
233 */
234static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
Yufen Yu578c6472018-01-09 19:33:39 +0800235 struct writeback_control *wbc,
Hyunchul Lee0cdd3192018-01-31 11:36:57 +0900236 int npages, bool is_read,
237 enum page_type type, enum temp_type temp)
Gu Zheng940a6d32013-12-20 17:39:59 +0800238{
239 struct bio *bio;
240
Chao Yud62fe972017-10-28 16:52:31 +0800241 bio = f2fs_bio_alloc(sbi, npages, true);
Gu Zheng940a6d32013-12-20 17:39:59 +0800242
Jaegeuk Kim3c62be12016-10-06 19:02:05 -0700243 f2fs_target_device(sbi, blk_addr, bio);
Hyunchul Lee0cdd3192018-01-31 11:36:57 +0900244 if (is_read) {
245 bio->bi_end_io = f2fs_read_end_io;
246 bio->bi_private = NULL;
247 } else {
248 bio->bi_end_io = f2fs_write_end_io;
249 bio->bi_private = sbi;
Chao Yu4d57b862018-05-30 00:20:41 +0800250 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, type, temp);
Hyunchul Lee0cdd3192018-01-31 11:36:57 +0900251 }
Yufen Yu578c6472018-01-09 19:33:39 +0800252 if (wbc)
253 wbc_init_bio(wbc, bio);
Gu Zheng940a6d32013-12-20 17:39:59 +0800254
255 return bio;
256}
257
Linus Torvalds4fc29c12016-07-27 10:36:31 -0700258static inline void __submit_bio(struct f2fs_sb_info *sbi,
259 struct bio *bio, enum page_type type)
Jaegeuk Kimf5730182016-05-17 16:23:36 -0700260{
Linus Torvalds4fc29c12016-07-27 10:36:31 -0700261 if (!is_read_io(bio_op(bio))) {
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800262 unsigned int start;
263
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800264 if (type != DATA && type != NODE)
265 goto submit_io;
266
Yunlong Song66415ce2018-07-12 23:09:28 +0800267 if (test_opt(sbi, LFS) && current->plug)
Tiezhu Yang3bb09a02018-02-06 08:21:45 +0800268 blk_finish_plug(current->plug);
269
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800270 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
271 start %= F2FS_IO_SIZE(sbi);
272
273 if (start == 0)
274 goto submit_io;
275
276 /* fill dummy pages */
277 for (; start < F2FS_IO_SIZE(sbi); start++) {
278 struct page *page =
279 mempool_alloc(sbi->write_io_dummy,
280 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
281 f2fs_bug_on(sbi, !page);
282
283 SetPagePrivate(page);
284 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
285 lock_page(page);
286 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
287 f2fs_bug_on(sbi, 1);
288 }
289 /*
290 * In the NODE case, we lose next block address chain. So, we
291 * need to do checkpoint in f2fs_sync_file.
292 */
293 if (type == NODE)
294 set_sbi_flag(sbi, SBI_NEED_CP);
Jaegeuk Kim19a5f5e2016-06-04 14:25:24 -0700295 }
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800296submit_io:
Jaegeuk Kim554b5122016-12-21 12:13:03 -0800297 if (is_read_io(bio_op(bio)))
298 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
299 else
300 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
Mike Christie4e49ea42016-06-05 14:31:41 -0500301 submit_bio(bio);
Jaegeuk Kimf5730182016-05-17 16:23:36 -0700302}
303
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900304static void __submit_merged_bio(struct f2fs_bio_info *io)
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900305{
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900306 struct f2fs_io_info *fio = &io->fio;
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900307
308 if (!io->bio)
309 return;
310
Mike Christie04d328d2016-06-05 14:31:55 -0500311 bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
312
Jaegeuk Kim554b5122016-12-21 12:13:03 -0800313 if (is_read_io(fio->op))
314 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
315 else
316 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
317
Linus Torvalds4fc29c12016-07-27 10:36:31 -0700318 __submit_bio(io->sbi, io->bio, fio->type);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900319 io->bio = NULL;
320}
321
Jaegeuk Kim942fd312017-02-01 16:51:22 -0800322static bool __has_merged_page(struct f2fs_bio_info *io,
323 struct inode *inode, nid_t ino, pgoff_t idx)
Chao Yu0fd785e2016-01-18 18:24:59 +0800324{
Chao Yu0fd785e2016-01-18 18:24:59 +0800325 struct bio_vec *bvec;
326 struct page *target;
327 int i;
328
Chao Yu0c3a5792016-01-18 18:28:11 +0800329 if (!io->bio)
Chao Yu0fd785e2016-01-18 18:24:59 +0800330 return false;
Chao Yu0c3a5792016-01-18 18:28:11 +0800331
Jaegeuk Kim942fd312017-02-01 16:51:22 -0800332 if (!inode && !ino)
Chao Yu0c3a5792016-01-18 18:28:11 +0800333 return true;
Chao Yu0fd785e2016-01-18 18:24:59 +0800334
335 bio_for_each_segment_all(bvec, io->bio, i) {
336
Jaegeuk Kim0b81d072015-05-15 16:26:10 -0700337 if (bvec->bv_page->mapping)
Chao Yu0fd785e2016-01-18 18:24:59 +0800338 target = bvec->bv_page;
Jaegeuk Kim0b81d072015-05-15 16:26:10 -0700339 else
340 target = fscrypt_control_page(bvec->bv_page);
Chao Yu0fd785e2016-01-18 18:24:59 +0800341
Jaegeuk Kim942fd312017-02-01 16:51:22 -0800342 if (idx != target->index)
343 continue;
344
Chao Yu0c3a5792016-01-18 18:28:11 +0800345 if (inode && inode == target->mapping->host)
Chao Yu0fd785e2016-01-18 18:24:59 +0800346 return true;
Chao Yu0c3a5792016-01-18 18:28:11 +0800347 if (ino && ino == ino_of_node(target))
348 return true;
Chao Yu0fd785e2016-01-18 18:24:59 +0800349 }
350
Chao Yu0fd785e2016-01-18 18:24:59 +0800351 return false;
352}
353
Chao Yu0c3a5792016-01-18 18:28:11 +0800354static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
Jaegeuk Kim942fd312017-02-01 16:51:22 -0800355 nid_t ino, pgoff_t idx, enum page_type type)
Chao Yu0c3a5792016-01-18 18:28:11 +0800356{
357 enum page_type btype = PAGE_TYPE_OF_BIO(type);
Jaegeuk Kima912b542017-05-10 11:18:25 -0700358 enum temp_type temp;
359 struct f2fs_bio_info *io;
360 bool ret = false;
Chao Yu0c3a5792016-01-18 18:28:11 +0800361
Jaegeuk Kima912b542017-05-10 11:18:25 -0700362 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
363 io = sbi->write_io[btype] + temp;
364
365 down_read(&io->io_rwsem);
366 ret = __has_merged_page(io, inode, ino, idx);
367 up_read(&io->io_rwsem);
368
369 /* TODO: use HOT temp only for meta pages now. */
370 if (ret || btype == META)
371 break;
372 }
Chao Yu0c3a5792016-01-18 18:28:11 +0800373 return ret;
374}
375
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700376static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
Jaegeuk Kima912b542017-05-10 11:18:25 -0700377 enum page_type type, enum temp_type temp)
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900378{
379 enum page_type btype = PAGE_TYPE_OF_BIO(type);
Jaegeuk Kima912b542017-05-10 11:18:25 -0700380 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900381
Chao Yudf0f8dc2014-03-22 14:57:23 +0800382 down_write(&io->io_rwsem);
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900383
384 /* change META to META_FLUSH in the checkpoint procedure */
385 if (type >= META_FLUSH) {
386 io->fio.type = META_FLUSH;
Mike Christie04d328d2016-06-05 14:31:55 -0500387 io->fio.op = REQ_OP_WRITE;
Jan Kara3adc5fcb2017-05-02 17:03:47 +0200388 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
Christoph Hellwig70fd7612016-11-01 07:40:10 -0600389 if (!test_opt(sbi, NOBARRIER))
Jaegeuk Kim7f54f512017-02-06 13:57:58 -0800390 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900391 }
392 __submit_merged_bio(io);
Chao Yudf0f8dc2014-03-22 14:57:23 +0800393 up_write(&io->io_rwsem);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900394}
395
Jaegeuk Kima912b542017-05-10 11:18:25 -0700396static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
Jaegeuk Kim942fd312017-02-01 16:51:22 -0800397 struct inode *inode, nid_t ino, pgoff_t idx,
Jaegeuk Kima912b542017-05-10 11:18:25 -0700398 enum page_type type, bool force)
Chao Yu0c3a5792016-01-18 18:28:11 +0800399{
Jaegeuk Kima912b542017-05-10 11:18:25 -0700400 enum temp_type temp;
401
402 if (!force && !has_merged_page(sbi, inode, ino, idx, type))
403 return;
404
405 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
406
407 __f2fs_submit_merged_write(sbi, type, temp);
408
409 /* TODO: use HOT temp only for meta pages now. */
410 if (type >= META)
411 break;
412 }
Chao Yu0c3a5792016-01-18 18:28:11 +0800413}
414
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700415void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
Chao Yu406657d2016-02-24 17:17:55 +0800416{
Jaegeuk Kima912b542017-05-10 11:18:25 -0700417 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900418}
419
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700420void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900421 struct inode *inode, nid_t ino, pgoff_t idx,
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700422 enum page_type type)
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900423{
Jaegeuk Kima912b542017-05-10 11:18:25 -0700424 __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900425}
426
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700427void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
Chao Yu406657d2016-02-24 17:17:55 +0800428{
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700429 f2fs_submit_merged_write(sbi, DATA);
430 f2fs_submit_merged_write(sbi, NODE);
431 f2fs_submit_merged_write(sbi, META);
Chao Yu406657d2016-02-24 17:17:55 +0800432}
433
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900434/*
435 * Fill the locked page with data located in the block address.
Tomohiro Kusumi771a9a72017-04-05 22:49:44 +0300436 * A caller needs to unlock the page on failure.
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900437 */
Jaegeuk Kim05ca3632015-04-23 14:38:15 -0700438int f2fs_submit_page_bio(struct f2fs_io_info *fio)
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900439{
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900440 struct bio *bio;
Jaegeuk Kim0b81d072015-05-15 16:26:10 -0700441 struct page *page = fio->encrypted_page ?
442 fio->encrypted_page : fio->page;
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900443
Chao Yuc9b60782018-08-01 19:13:44 +0800444 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
445 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
446 return -EFAULT;
447
Chao Yu2ace38e2014-12-24 16:08:14 +0800448 trace_f2fs_submit_page_bio(page, fio);
Jaegeuk Kim05ca3632015-04-23 14:38:15 -0700449 f2fs_trace_ios(fio, 0);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900450
451 /* Allocate a new bio */
Yufen Yu578c6472018-01-09 19:33:39 +0800452 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
Hyunchul Lee0cdd3192018-01-31 11:36:57 +0900453 1, is_read_io(fio->op), fio->type, fio->temp);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900454
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300455 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900456 bio_put(bio);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900457 return -EFAULT;
458 }
Mike Christie04d328d2016-06-05 14:31:55 -0500459 bio_set_op_attrs(bio, fio->op, fio->op_flags);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900460
Linus Torvalds4fc29c12016-07-27 10:36:31 -0700461 __submit_bio(fio->sbi, bio, fio->type);
Jaegeuk Kimd1b3e722017-03-30 21:02:46 -0700462
463 if (!is_read_io(fio->op))
464 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900465 return 0;
466}
467
Chao Yufe16efe2018-05-28 23:47:18 +0800468void f2fs_submit_page_write(struct f2fs_io_info *fio)
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900469{
Jaegeuk Kim05ca3632015-04-23 14:38:15 -0700470 struct f2fs_sb_info *sbi = fio->sbi;
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900471 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
Jaegeuk Kima912b542017-05-10 11:18:25 -0700472 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
Jaegeuk Kim4375a332015-04-23 12:04:33 -0700473 struct page *bio_page;
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900474
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700475 f2fs_bug_on(sbi, is_read_io(fio->op));
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900476
Chao Yufb830fc2017-05-19 23:37:01 +0800477 down_write(&io->io_rwsem);
478next:
479 if (fio->in_list) {
480 spin_lock(&io->io_lock);
481 if (list_empty(&io->io_list)) {
482 spin_unlock(&io->io_lock);
Chao Yufe16efe2018-05-28 23:47:18 +0800483 goto out;
Chao Yufb830fc2017-05-19 23:37:01 +0800484 }
485 fio = list_first_entry(&io->io_list,
486 struct f2fs_io_info, list);
487 list_del(&fio->list);
488 spin_unlock(&io->io_lock);
489 }
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900490
Chao Yue1da7872018-06-05 17:44:11 +0800491 if (__is_valid_data_blkaddr(fio->old_blkaddr))
Yunlei He08337212018-03-08 16:29:13 +0800492 verify_block_addr(fio, fio->old_blkaddr);
493 verify_block_addr(fio, fio->new_blkaddr);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900494
Chao Yu36951b32016-11-16 10:41:20 +0800495 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
496
Thomas Meyerebf7c522017-10-07 16:02:21 +0200497 /* set submitted = true as a return value */
498 fio->submitted = true;
Jaegeuk Kimd68f7352017-02-03 17:44:04 -0800499
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700500 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900501
Chao Yu7a9d7542016-02-22 18:36:38 +0800502 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
Jaegeuk Kim3c62be12016-10-06 19:02:05 -0700503 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
504 !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900505 __submit_merged_bio(io);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900506alloc_new:
507 if (io->bio == NULL) {
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800508 if ((fio->type == DATA || fio->type == NODE) &&
509 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
Jaegeuk Kimb9109b02017-05-10 11:28:38 -0700510 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
Chao Yufe16efe2018-05-28 23:47:18 +0800511 fio->retry = true;
512 goto skip;
Jaegeuk Kim0a595eb2016-12-14 10:12:56 -0800513 }
Yufen Yu578c6472018-01-09 19:33:39 +0800514 io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc,
Hyunchul Lee0cdd3192018-01-31 11:36:57 +0900515 BIO_MAX_PAGES, false,
516 fio->type, fio->temp);
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900517 io->fio = *fio;
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900518 }
519
Jaegeuk Kima912b542017-05-10 11:18:25 -0700520 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
Jaegeuk Kim458e6192013-12-11 13:54:01 +0900521 __submit_merged_bio(io);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900522 goto alloc_new;
523 }
524
Yufen Yu578c6472018-01-09 19:33:39 +0800525 if (fio->io_wbc)
526 wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);
527
Chao Yu7a9d7542016-02-22 18:36:38 +0800528 io->last_block_in_bio = fio->new_blkaddr;
Jaegeuk Kim05ca3632015-04-23 14:38:15 -0700529 f2fs_trace_ios(fio, 0);
Chao Yufb830fc2017-05-19 23:37:01 +0800530
531 trace_f2fs_submit_page_write(fio->page, fio);
Chao Yufe16efe2018-05-28 23:47:18 +0800532skip:
Chao Yufb830fc2017-05-19 23:37:01 +0800533 if (fio->in_list)
534 goto next;
Chao Yufe16efe2018-05-28 23:47:18 +0800535out:
Chao Yudf0f8dc2014-03-22 14:57:23 +0800536 up_write(&io->io_rwsem);
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900537}
538
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700539static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
Jaegeuk Kime2e59412018-06-21 11:29:43 -0700540 unsigned nr_pages, unsigned op_flag)
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700541{
542 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700543 struct bio *bio;
Eric Biggers6dbb1792018-04-18 11:09:48 -0700544 struct bio_post_read_ctx *ctx;
545 unsigned int post_read_steps = 0;
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700546
Chao Yu91291e92018-07-10 23:01:45 +0800547 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC))
548 return ERR_PTR(-EFAULT);
549
Eric Biggers6dbb1792018-04-18 11:09:48 -0700550 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
551 if (!bio)
552 return ERR_PTR(-ENOMEM);
553 f2fs_target_device(sbi, blkaddr, bio);
554 bio->bi_end_io = f2fs_read_end_io;
Jaegeuk Kime2e59412018-06-21 11:29:43 -0700555 bio_set_op_attrs(bio, REQ_OP_READ, op_flag);
Eric Biggers6dbb1792018-04-18 11:09:48 -0700556
557 if (f2fs_encrypted_file(inode))
558 post_read_steps |= 1 << STEP_DECRYPT;
559 if (post_read_steps) {
560 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
561 if (!ctx) {
562 bio_put(bio);
563 return ERR_PTR(-ENOMEM);
564 }
565 ctx->bio = bio;
566 ctx->enabled_steps = post_read_steps;
567 bio->bi_private = ctx;
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700568
569 /* wait the page to be moved by cleaning */
570 f2fs_wait_on_block_writeback(sbi, blkaddr);
571 }
572
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700573 return bio;
574}
575
576/* This can handle encryption stuffs */
577static int f2fs_submit_page_read(struct inode *inode, struct page *page,
578 block_t blkaddr)
579{
Jaegeuk Kime2e59412018-06-21 11:29:43 -0700580 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0);
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700581
582 if (IS_ERR(bio))
583 return PTR_ERR(bio);
584
585 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
586 bio_put(bio);
587 return -EFAULT;
588 }
589 __submit_bio(F2FS_I_SB(inode), bio, DATA);
590 return 0;
591}
592
Chao Yu46008c62016-05-09 19:56:30 +0800593static void __set_data_blkaddr(struct dnode_of_data *dn)
594{
595 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
596 __le32 *addr_array;
Chao Yu7a2af762017-07-19 00:19:06 +0800597 int base = 0;
598
599 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
600 base = get_extra_isize(dn->inode);
Chao Yu46008c62016-05-09 19:56:30 +0800601
602 /* Get physical address of data block */
603 addr_array = blkaddr_in_node(rn);
Chao Yu7a2af762017-07-19 00:19:06 +0800604 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
Chao Yu46008c62016-05-09 19:56:30 +0800605}
606
Jaegeuk Kim93dfe2a2013-11-30 12:51:14 +0900607/*
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900608 * Lock ordering for the change of data block address:
609 * ->data_page
610 * ->node_page
611 * update block addresses in the node page
612 */
Chao Yu4d57b862018-05-30 00:20:41 +0800613void f2fs_set_data_blkaddr(struct dnode_of_data *dn)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900614{
Chao Yu46008c62016-05-09 19:56:30 +0800615 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
616 __set_data_blkaddr(dn);
617 if (set_page_dirty(dn->node_page))
Jaegeuk Kim12719ae2016-01-07 13:23:12 -0800618 dn->node_changed = true;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900619}
620
Chao Yuf28b3432016-02-24 17:16:47 +0800621void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
622{
623 dn->data_blkaddr = blkaddr;
Chao Yu4d57b862018-05-30 00:20:41 +0800624 f2fs_set_data_blkaddr(dn);
Chao Yuf28b3432016-02-24 17:16:47 +0800625 f2fs_update_extent_cache(dn);
626}
627
Chao Yu46008c62016-05-09 19:56:30 +0800628/* dn->ofs_in_node will be returned with up-to-date last block pointer */
Chao Yu4d57b862018-05-30 00:20:41 +0800629int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900630{
Jaegeuk Kim40813632014-09-02 15:31:18 -0700631 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
Chao Yu0abd6752017-07-09 00:13:07 +0800632 int err;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900633
Chao Yu46008c62016-05-09 19:56:30 +0800634 if (!count)
635 return 0;
636
Jaegeuk Kim91942322016-05-20 10:13:22 -0700637 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900638 return -EPERM;
Chao Yu0abd6752017-07-09 00:13:07 +0800639 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
640 return err;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900641
Chao Yu46008c62016-05-09 19:56:30 +0800642 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
643 dn->ofs_in_node, count);
Namjae Jeonc01e2852013-04-23 17:00:52 +0900644
Chao Yu46008c62016-05-09 19:56:30 +0800645 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
646
647 for (; count > 0; dn->ofs_in_node++) {
Chao Yu7a2af762017-07-19 00:19:06 +0800648 block_t blkaddr = datablock_addr(dn->inode,
649 dn->node_page, dn->ofs_in_node);
Chao Yu46008c62016-05-09 19:56:30 +0800650 if (blkaddr == NULL_ADDR) {
651 dn->data_blkaddr = NEW_ADDR;
652 __set_data_blkaddr(dn);
653 count--;
654 }
655 }
656
657 if (set_page_dirty(dn->node_page))
658 dn->node_changed = true;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900659 return 0;
660}
661
Chao Yu46008c62016-05-09 19:56:30 +0800662/* Should keep dn->ofs_in_node unchanged */
Chao Yu4d57b862018-05-30 00:20:41 +0800663int f2fs_reserve_new_block(struct dnode_of_data *dn)
Chao Yu46008c62016-05-09 19:56:30 +0800664{
665 unsigned int ofs_in_node = dn->ofs_in_node;
666 int ret;
667
Chao Yu4d57b862018-05-30 00:20:41 +0800668 ret = f2fs_reserve_new_blocks(dn, 1);
Chao Yu46008c62016-05-09 19:56:30 +0800669 dn->ofs_in_node = ofs_in_node;
670 return ret;
671}
672
Huajun Lib6009652013-11-10 23:13:18 +0800673int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
674{
675 bool need_put = dn->inode_page ? false : true;
676 int err;
677
Chao Yu4d57b862018-05-30 00:20:41 +0800678 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
Huajun Lib6009652013-11-10 23:13:18 +0800679 if (err)
680 return err;
Jaegeuk Kima8865372013-12-27 17:04:17 +0900681
Huajun Lib6009652013-11-10 23:13:18 +0800682 if (dn->data_blkaddr == NULL_ADDR)
Chao Yu4d57b862018-05-30 00:20:41 +0800683 err = f2fs_reserve_new_block(dn);
Jaegeuk Kima8865372013-12-27 17:04:17 +0900684 if (err || need_put)
Huajun Lib6009652013-11-10 23:13:18 +0800685 f2fs_put_dnode(dn);
686 return err;
687}
688
Fan Li759af1c2015-08-05 15:52:16 +0800689int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900690{
Hou Pengyange15882b2017-02-23 09:18:05 +0000691 struct extent_info ei = {0,0,0};
Fan Li759af1c2015-08-05 15:52:16 +0800692 struct inode *inode = dn->inode;
Chao Yu028a41e2015-03-19 19:26:02 +0800693
Fan Li759af1c2015-08-05 15:52:16 +0800694 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
695 dn->data_blkaddr = ei.blk + index - ei.fofs;
696 return 0;
Chao Yu429511c2015-02-05 17:54:31 +0800697 }
698
Fan Li759af1c2015-08-05 15:52:16 +0800699 return f2fs_reserve_block(dn, index);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900700}
701
Chao Yu4d57b862018-05-30 00:20:41 +0800702struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
Mike Christie04d328d2016-06-05 14:31:55 -0500703 int op_flags, bool for_write)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900704{
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900705 struct address_space *mapping = inode->i_mapping;
706 struct dnode_of_data dn;
707 struct page *page;
Hou Pengyange15882b2017-02-23 09:18:05 +0000708 struct extent_info ei = {0,0,0};
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900709 int err;
Jaegeuk Kim4375a332015-04-23 12:04:33 -0700710
Jaegeuk Kima56c7c62015-10-09 15:11:38 -0700711 page = f2fs_grab_cache_page(mapping, index, for_write);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900712 if (!page)
713 return ERR_PTR(-ENOMEM);
714
Chao Yucb3bc9e2015-02-05 18:03:40 +0800715 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
716 dn.data_blkaddr = ei.blk + index - ei.fofs;
717 goto got_it;
718 }
719
Jaegeuk Kim650495d2013-05-13 08:38:35 +0900720 set_new_dnode(&dn, inode, NULL, NULL, 0);
Chao Yu4d57b862018-05-30 00:20:41 +0800721 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
Jaegeuk Kim86531d62015-07-15 13:08:21 -0700722 if (err)
723 goto put_err;
Jaegeuk Kim650495d2013-05-13 08:38:35 +0900724 f2fs_put_dnode(&dn);
725
Jaegeuk Kim6bacf522013-12-06 15:00:58 +0900726 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
Jaegeuk Kim86531d62015-07-15 13:08:21 -0700727 err = -ENOENT;
728 goto put_err;
Jaegeuk Kim650495d2013-05-13 08:38:35 +0900729 }
Chao Yucb3bc9e2015-02-05 18:03:40 +0800730got_it:
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700731 if (PageUptodate(page)) {
732 unlock_page(page);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900733 return page;
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700734 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900735
Jaegeuk Kimd59ff4d2013-08-20 19:13:07 +0900736 /*
737 * A new dentry page is allocated but not able to be written, since its
738 * new inode page couldn't be allocated due to -ENOSPC.
739 * In such the case, its blkaddr can be remained as NEW_ADDR.
Chao Yu4d57b862018-05-30 00:20:41 +0800740 * see, f2fs_add_link -> f2fs_get_new_data_page ->
741 * f2fs_init_inode_metadata.
Jaegeuk Kimd59ff4d2013-08-20 19:13:07 +0900742 */
743 if (dn.data_blkaddr == NEW_ADDR) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300744 zero_user_segment(page, 0, PAGE_SIZE);
Jaegeuk Kim237c0792016-06-30 18:49:15 -0700745 if (!PageUptodate(page))
746 SetPageUptodate(page);
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700747 unlock_page(page);
Jaegeuk Kimd59ff4d2013-08-20 19:13:07 +0900748 return page;
749 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900750
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -0700751 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
Jaegeuk Kim393ff912013-03-08 21:29:23 +0900752 if (err)
Jaegeuk Kim86531d62015-07-15 13:08:21 -0700753 goto put_err;
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700754 return page;
Jaegeuk Kim86531d62015-07-15 13:08:21 -0700755
756put_err:
757 f2fs_put_page(page, 1);
758 return ERR_PTR(err);
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700759}
Jaegeuk Kim393ff912013-03-08 21:29:23 +0900760
Chao Yu4d57b862018-05-30 00:20:41 +0800761struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index)
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700762{
763 struct address_space *mapping = inode->i_mapping;
764 struct page *page;
765
766 page = find_get_page(mapping, index);
767 if (page && PageUptodate(page))
768 return page;
769 f2fs_put_page(page, 0);
770
Chao Yu4d57b862018-05-30 00:20:41 +0800771 page = f2fs_get_read_data_page(inode, index, 0, false);
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700772 if (IS_ERR(page))
773 return page;
774
775 if (PageUptodate(page))
776 return page;
777
778 wait_on_page_locked(page);
779 if (unlikely(!PageUptodate(page))) {
780 f2fs_put_page(page, 0);
781 return ERR_PTR(-EIO);
782 }
783 return page;
784}
785
786/*
787 * If it tries to access a hole, return an error.
788 * Because, the callers, functions in dir.c and GC, should be able to know
789 * whether this page exists or not.
790 */
Chao Yu4d57b862018-05-30 00:20:41 +0800791struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
Jaegeuk Kima56c7c62015-10-09 15:11:38 -0700792 bool for_write)
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700793{
794 struct address_space *mapping = inode->i_mapping;
795 struct page *page;
796repeat:
Chao Yu4d57b862018-05-30 00:20:41 +0800797 page = f2fs_get_read_data_page(inode, index, 0, for_write);
Jaegeuk Kim43f3eae2015-04-30 17:00:33 -0700798 if (IS_ERR(page))
799 return page;
800
801 /* wait for read completion */
Jaegeuk Kim393ff912013-03-08 21:29:23 +0900802 lock_page(page);
Jaegeuk Kim6bacf522013-12-06 15:00:58 +0900803 if (unlikely(page->mapping != mapping)) {
Jaegeuk Kimafcb7ca02013-04-26 11:55:17 +0900804 f2fs_put_page(page, 1);
805 goto repeat;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900806 }
Chao Yu1563ac72016-07-03 22:05:12 +0800807 if (unlikely(!PageUptodate(page))) {
808 f2fs_put_page(page, 1);
809 return ERR_PTR(-EIO);
810 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900811 return page;
812}
813
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +0900814/*
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900815 * Caller ensures that this data page is never allocated.
816 * A new zero-filled data page is allocated in the page cache.
Jaegeuk Kim39936832012-11-22 16:21:29 +0900817 *
Chao Yu4f4124d2013-12-21 18:02:14 +0800818 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
819 * f2fs_unlock_op().
Chao Yu470f00e2015-07-14 18:14:06 +0800820 * Note that, ipage is set only by make_empty_dir, and if any error occur,
821 * ipage should be released by this function.
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900822 */
Chao Yu4d57b862018-05-30 00:20:41 +0800823struct page *f2fs_get_new_data_page(struct inode *inode,
Jaegeuk Kima8865372013-12-27 17:04:17 +0900824 struct page *ipage, pgoff_t index, bool new_i_size)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900825{
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900826 struct address_space *mapping = inode->i_mapping;
827 struct page *page;
828 struct dnode_of_data dn;
829 int err;
Jaegeuk Kim76121182016-01-01 22:03:47 -0800830
Jaegeuk Kima56c7c62015-10-09 15:11:38 -0700831 page = f2fs_grab_cache_page(mapping, index, true);
Chao Yu470f00e2015-07-14 18:14:06 +0800832 if (!page) {
833 /*
834 * before exiting, we should make sure ipage will be released
835 * if any error occur.
836 */
837 f2fs_put_page(ipage, 1);
Jaegeuk Kim01f28612015-04-29 11:18:42 -0700838 return ERR_PTR(-ENOMEM);
Chao Yu470f00e2015-07-14 18:14:06 +0800839 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900840
Jaegeuk Kima8865372013-12-27 17:04:17 +0900841 set_new_dnode(&dn, inode, ipage, NULL, 0);
Huajun Lib6009652013-11-10 23:13:18 +0800842 err = f2fs_reserve_block(&dn, index);
Jaegeuk Kim01f28612015-04-29 11:18:42 -0700843 if (err) {
844 f2fs_put_page(page, 1);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900845 return ERR_PTR(err);
Jaegeuk Kima8865372013-12-27 17:04:17 +0900846 }
Jaegeuk Kim01f28612015-04-29 11:18:42 -0700847 if (!ipage)
848 f2fs_put_dnode(&dn);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900849
850 if (PageUptodate(page))
Jaegeuk Kim01f28612015-04-29 11:18:42 -0700851 goto got_it;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900852
853 if (dn.data_blkaddr == NEW_ADDR) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300854 zero_user_segment(page, 0, PAGE_SIZE);
Jaegeuk Kim237c0792016-06-30 18:49:15 -0700855 if (!PageUptodate(page))
856 SetPageUptodate(page);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900857 } else {
Jaegeuk Kim4375a332015-04-23 12:04:33 -0700858 f2fs_put_page(page, 1);
Jaegeuk Kima8865372013-12-27 17:04:17 +0900859
Jaegeuk Kim76121182016-01-01 22:03:47 -0800860 /* if ipage exists, blkaddr should be NEW_ADDR */
861 f2fs_bug_on(F2FS_I_SB(inode), ipage);
Chao Yu4d57b862018-05-30 00:20:41 +0800862 page = f2fs_get_lock_data_page(inode, index, true);
Jaegeuk Kim4375a332015-04-23 12:04:33 -0700863 if (IS_ERR(page))
Jaegeuk Kim76121182016-01-01 22:03:47 -0800864 return page;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900865 }
Jaegeuk Kim01f28612015-04-29 11:18:42 -0700866got_it:
Chao Yu9edcdab2015-09-11 14:43:52 +0800867 if (new_i_size && i_size_read(inode) <
Jaegeuk Kimee6d1822016-05-20 16:32:49 -0700868 ((loff_t)(index + 1) << PAGE_SHIFT))
Jaegeuk Kimfc9581c2016-05-20 09:22:03 -0700869 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900870 return page;
871}
872
Hyunchul Leed5097be2017-11-28 09:23:00 +0900873static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900874{
Jaegeuk Kim40813632014-09-02 15:31:18 -0700875 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900876 struct f2fs_summary sum;
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900877 struct node_info ni;
Chao Yu6aa58d82018-08-14 22:37:25 +0800878 block_t old_blkaddr;
Jaegeuk Kim976e4c52014-09-15 19:32:16 -0700879 pgoff_t fofs;
Chao Yu46008c62016-05-09 19:56:30 +0800880 blkcnt_t count = 1;
Chao Yu0abd6752017-07-09 00:13:07 +0800881 int err;
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900882
Jaegeuk Kim91942322016-05-20 10:13:22 -0700883 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900884 return -EPERM;
Chao Yudf6136e2015-03-23 10:33:37 +0800885
Chao Yu77357302018-07-17 00:02:17 +0800886 err = f2fs_get_node_info(sbi, dn->nid, &ni);
887 if (err)
888 return err;
889
Chao Yu7a2af762017-07-19 00:19:06 +0800890 dn->data_blkaddr = datablock_addr(dn->inode,
891 dn->node_page, dn->ofs_in_node);
Chao Yudf6136e2015-03-23 10:33:37 +0800892 if (dn->data_blkaddr == NEW_ADDR)
893 goto alloc;
894
Chao Yu0abd6752017-07-09 00:13:07 +0800895 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
896 return err;
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900897
Chao Yudf6136e2015-03-23 10:33:37 +0800898alloc:
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900899 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
Chao Yu6aa58d82018-08-14 22:37:25 +0800900 old_blkaddr = dn->data_blkaddr;
901 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
Hyunchul Leed5097be2017-11-28 09:23:00 +0900902 &sum, seg_type, NULL, false);
Chao Yu6aa58d82018-08-14 22:37:25 +0800903 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
904 invalidate_mapping_pages(META_MAPPING(sbi),
905 old_blkaddr, old_blkaddr);
Chao Yu4d57b862018-05-30 00:20:41 +0800906 f2fs_set_data_blkaddr(dn);
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900907
Jaegeuk Kim976e4c52014-09-15 19:32:16 -0700908 /* update i_size */
Chao Yu4d57b862018-05-30 00:20:41 +0800909 fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
Jaegeuk Kim976e4c52014-09-15 19:32:16 -0700910 dn->ofs_in_node;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300911 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
Jaegeuk Kimfc9581c2016-05-20 09:22:03 -0700912 f2fs_i_size_write(dn->inode,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300913 ((loff_t)(fofs + 1) << PAGE_SHIFT));
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +0900914 return 0;
915}
916
Jaegeuk Kima7de6082016-11-11 16:31:56 -0800917int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
Jaegeuk Kim59b802e2015-02-09 12:09:53 -0800918{
Jaegeuk Kimb439b102016-02-03 13:09:09 -0800919 struct inode *inode = file_inode(iocb->ki_filp);
Chao Yu5b8db7f2016-01-26 15:38:29 +0800920 struct f2fs_map_blocks map;
Chao Yud6d478a12018-01-03 17:30:19 +0800921 int flag;
Jaegeuk Kima7de6082016-11-11 16:31:56 -0800922 int err = 0;
Chao Yud6d478a12018-01-03 17:30:19 +0800923 bool direct_io = iocb->ki_flags & IOCB_DIRECT;
Jaegeuk Kim59b802e2015-02-09 12:09:53 -0800924
Weichao Guo71ad6822017-09-29 22:43:23 +0800925 /* convert inline data for Direct I/O*/
Chao Yud6d478a12018-01-03 17:30:19 +0800926 if (direct_io) {
Weichao Guo71ad6822017-09-29 22:43:23 +0800927 err = f2fs_convert_inline_inode(inode);
928 if (err)
929 return err;
930 }
931
Jaegeuk Kimdc91de72017-01-13 13:12:29 -0800932 if (is_inode_flag_set(inode, FI_NO_PREALLOC))
933 return 0;
934
Jaegeuk Kim0080c502016-05-07 08:52:57 -0700935 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
Chao Yudfd02e42016-08-20 15:12:01 +0800936 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
937 if (map.m_len > map.m_lblk)
938 map.m_len -= map.m_lblk;
939 else
940 map.m_len = 0;
941
Chao Yuda859852016-01-26 15:42:58 +0800942 map.m_next_pgofs = NULL;
Chao Yuc4020b22018-01-11 14:42:30 +0800943 map.m_next_extent = NULL;
Hyunchul Leed5097be2017-11-28 09:23:00 +0900944 map.m_seg_type = NO_CHECK_TYPE;
Jaegeuk Kim59b802e2015-02-09 12:09:53 -0800945
Chao Yud6d478a12018-01-03 17:30:19 +0800946 if (direct_io) {
Chao Yu4d57b862018-05-30 00:20:41 +0800947 map.m_seg_type = f2fs_rw_hint_to_seg_type(iocb->ki_hint);
Hyunchul Leeb91050a2018-03-08 19:34:38 +0900948 flag = f2fs_force_buffered_io(inode, WRITE) ?
Chao Yud6d478a12018-01-03 17:30:19 +0800949 F2FS_GET_BLOCK_PRE_AIO :
950 F2FS_GET_BLOCK_PRE_DIO;
951 goto map_blocks;
Hyunchul Leed5097be2017-11-28 09:23:00 +0900952 }
Chao Yuf2470372017-07-19 00:19:05 +0800953 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
Jaegeuk Kima7de6082016-11-11 16:31:56 -0800954 err = f2fs_convert_inline_inode(inode);
955 if (err)
956 return err;
Jaegeuk Kim24b84912016-02-03 13:49:44 -0800957 }
Chao Yud6d478a12018-01-03 17:30:19 +0800958 if (f2fs_has_inline_data(inode))
Sheng Yong250066452017-11-22 18:23:39 +0800959 return err;
Chao Yud6d478a12018-01-03 17:30:19 +0800960
961 flag = F2FS_GET_BLOCK_PRE_AIO;
962
963map_blocks:
964 err = f2fs_map_blocks(inode, &map, 1, flag);
965 if (map.m_len > 0 && err == -ENOSPC) {
966 if (!direct_io)
967 set_inode_flag(inode, FI_NO_PREALLOC);
968 err = 0;
Sheng Yong250066452017-11-22 18:23:39 +0800969 }
Jaegeuk Kima7de6082016-11-11 16:31:56 -0800970 return err;
Jaegeuk Kim59b802e2015-02-09 12:09:53 -0800971}
972
Yunlei He59c90812017-03-13 20:22:18 +0800973static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
974{
975 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
976 if (lock)
977 down_read(&sbi->node_change);
978 else
979 up_read(&sbi->node_change);
980 } else {
981 if (lock)
982 f2fs_lock_op(sbi);
983 else
984 f2fs_unlock_op(sbi);
985 }
986}
987
Jaegeuk Kim0a8165d2012-11-29 13:28:09 +0900988/*
Jaegeuk Kim003a3e12015-04-06 19:55:34 -0700989 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
990 * f2fs_map_blocks structure.
Chao Yu4f4124d2013-12-21 18:02:14 +0800991 * If original data blocks are allocated, then give them to blockdev.
992 * Otherwise,
993 * a. preallocate requested block addresses
994 * b. do not use extent cache for better performance
995 * c. give the block addresses to blockdev
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900996 */
Chao Yud323d002015-10-27 09:53:45 +0800997int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
Chao Yue2b4e2b2015-08-19 19:11:19 +0800998 int create, int flag)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +0900999{
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001000 unsigned int maxblocks = map->m_len;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001001 struct dnode_of_data dn;
Chao Yuf9811702015-09-21 20:17:52 +08001002 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kimac6f1992016-06-16 17:03:23 -07001003 int mode = create ? ALLOC_NODE : LOOKUP_NODE;
Chao Yu46008c62016-05-09 19:56:30 +08001004 pgoff_t pgofs, end_offset, end;
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001005 int err = 0, ofs = 1;
Chao Yu46008c62016-05-09 19:56:30 +08001006 unsigned int ofs_in_node, last_ofs_in_node;
1007 blkcnt_t prealloc;
Hou Pengyange15882b2017-02-23 09:18:05 +00001008 struct extent_info ei = {0,0,0};
Fan Li7df3a432015-12-17 13:20:59 +08001009 block_t blkaddr;
Chao Yuc4020b22018-01-11 14:42:30 +08001010 unsigned int start_pgofs;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001011
Chao Yudfd02e42016-08-20 15:12:01 +08001012 if (!maxblocks)
1013 return 0;
1014
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001015 map->m_len = 0;
1016 map->m_flags = 0;
1017
1018 /* it only supports block size == page size */
1019 pgofs = (pgoff_t)map->m_lblk;
Chao Yu46008c62016-05-09 19:56:30 +08001020 end = pgofs + maxblocks;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001021
Jaegeuk Kim24b84912016-02-03 13:49:44 -08001022 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001023 map->m_pblk = ei.blk + pgofs - ei.fofs;
1024 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
1025 map->m_flags = F2FS_MAP_MAPPED;
Chao Yuc4020b22018-01-11 14:42:30 +08001026 if (map->m_next_extent)
1027 *map->m_next_extent = pgofs + map->m_len;
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001028 goto out;
Chao Yua2e7d1b2015-02-05 17:50:30 +08001029 }
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001030
Chao Yu4fe71e82016-01-26 15:37:38 +08001031next_dnode:
Jaegeuk Kim59b802e2015-02-09 12:09:53 -08001032 if (create)
Yunlei He59c90812017-03-13 20:22:18 +08001033 __do_map_lock(sbi, flag, true);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001034
1035 /* When reading holes, we need its node page */
1036 set_new_dnode(&dn, inode, NULL, NULL, 0);
Chao Yu4d57b862018-05-30 00:20:41 +08001037 err = f2fs_get_dnode_of_data(&dn, pgofs, mode);
Jaegeuk Kim1ec79082013-12-26 16:55:22 +09001038 if (err) {
Chao Yu43473f92016-05-05 19:13:02 +08001039 if (flag == F2FS_GET_BLOCK_BMAP)
1040 map->m_pblk = 0;
Chao Yuda859852016-01-26 15:42:58 +08001041 if (err == -ENOENT) {
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001042 err = 0;
Chao Yuda859852016-01-26 15:42:58 +08001043 if (map->m_next_pgofs)
1044 *map->m_next_pgofs =
Chao Yu4d57b862018-05-30 00:20:41 +08001045 f2fs_get_next_page_offset(&dn, pgofs);
Chao Yuc4020b22018-01-11 14:42:30 +08001046 if (map->m_next_extent)
1047 *map->m_next_extent =
Chao Yu4d57b862018-05-30 00:20:41 +08001048 f2fs_get_next_page_offset(&dn, pgofs);
Chao Yuda859852016-01-26 15:42:58 +08001049 }
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001050 goto unlock_out;
Namjae Jeon848753a2013-04-23 16:38:02 +09001051 }
Chao Yu973163f2015-09-18 16:51:51 +08001052
Chao Yuc4020b22018-01-11 14:42:30 +08001053 start_pgofs = pgofs;
Chao Yu46008c62016-05-09 19:56:30 +08001054 prealloc = 0;
Arnd Bergmann230436b32016-11-02 14:52:15 +01001055 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
Chao Yu81ca7352016-01-26 15:39:35 +08001056 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001057
Chao Yu4fe71e82016-01-26 15:37:38 +08001058next_block:
Chao Yu7a2af762017-07-19 00:19:06 +08001059 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
Chao Yu973163f2015-09-18 16:51:51 +08001060
Chao Yuc9b60782018-08-01 19:13:44 +08001061 if (__is_valid_data_blkaddr(blkaddr) &&
1062 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
1063 err = -EFAULT;
1064 goto sync_out;
1065 }
1066
Chao Yue1da7872018-06-05 17:44:11 +08001067 if (!is_valid_data_blkaddr(sbi, blkaddr)) {
Fan Li7df3a432015-12-17 13:20:59 +08001068 if (create) {
1069 if (unlikely(f2fs_cp_error(sbi))) {
1070 err = -EIO;
1071 goto sync_out;
Chao Yu973163f2015-09-18 16:51:51 +08001072 }
Jaegeuk Kim24b84912016-02-03 13:49:44 -08001073 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
Chao Yu46008c62016-05-09 19:56:30 +08001074 if (blkaddr == NULL_ADDR) {
1075 prealloc++;
1076 last_ofs_in_node = dn.ofs_in_node;
1077 }
Jaegeuk Kim24b84912016-02-03 13:49:44 -08001078 } else {
Hyunchul Leed5097be2017-11-28 09:23:00 +09001079 err = __allocate_data_block(&dn,
1080 map->m_seg_type);
Chao Yu6f2d8ed2016-10-11 22:57:03 +08001081 if (!err)
Jaegeuk Kim91942322016-05-20 10:13:22 -07001082 set_inode_flag(inode, FI_APPEND_WRITE);
Jaegeuk Kim24b84912016-02-03 13:49:44 -08001083 }
Fan Li7df3a432015-12-17 13:20:59 +08001084 if (err)
1085 goto sync_out;
Kinglong Mee3f2be042017-02-23 19:55:05 +08001086 map->m_flags |= F2FS_MAP_NEW;
Fan Li7df3a432015-12-17 13:20:59 +08001087 blkaddr = dn.data_blkaddr;
1088 } else {
Chao Yu43473f92016-05-05 19:13:02 +08001089 if (flag == F2FS_GET_BLOCK_BMAP) {
1090 map->m_pblk = 0;
1091 goto sync_out;
1092 }
Chao Yuc4020b22018-01-11 14:42:30 +08001093 if (flag == F2FS_GET_BLOCK_PRECACHE)
1094 goto sync_out;
Chao Yuda859852016-01-26 15:42:58 +08001095 if (flag == F2FS_GET_BLOCK_FIEMAP &&
1096 blkaddr == NULL_ADDR) {
1097 if (map->m_next_pgofs)
1098 *map->m_next_pgofs = pgofs + 1;
Fan Li7df3a432015-12-17 13:20:59 +08001099 goto sync_out;
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001100 }
Chao Yuf3d98e72018-01-10 18:18:52 +08001101 if (flag != F2FS_GET_BLOCK_FIEMAP) {
1102 /* for defragment case */
1103 if (map->m_next_pgofs)
1104 *map->m_next_pgofs = pgofs + 1;
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001105 goto sync_out;
Chao Yuf3d98e72018-01-10 18:18:52 +08001106 }
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001107 }
1108 }
Fan Li7df3a432015-12-17 13:20:59 +08001109
Chao Yu46008c62016-05-09 19:56:30 +08001110 if (flag == F2FS_GET_BLOCK_PRE_AIO)
1111 goto skip;
1112
Chao Yu4fe71e82016-01-26 15:37:38 +08001113 if (map->m_len == 0) {
1114 /* preallocated unwritten block should be mapped for fiemap. */
1115 if (blkaddr == NEW_ADDR)
1116 map->m_flags |= F2FS_MAP_UNWRITTEN;
1117 map->m_flags |= F2FS_MAP_MAPPED;
1118
1119 map->m_pblk = blkaddr;
1120 map->m_len = 1;
1121 } else if ((map->m_pblk != NEW_ADDR &&
Fan Li7df3a432015-12-17 13:20:59 +08001122 blkaddr == (map->m_pblk + ofs)) ||
Jaegeuk Kimb439b102016-02-03 13:09:09 -08001123 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
Chao Yu46008c62016-05-09 19:56:30 +08001124 flag == F2FS_GET_BLOCK_PRE_DIO) {
Fan Li7df3a432015-12-17 13:20:59 +08001125 ofs++;
Fan Li7df3a432015-12-17 13:20:59 +08001126 map->m_len++;
Chao Yu4fe71e82016-01-26 15:37:38 +08001127 } else {
1128 goto sync_out;
1129 }
1130
Chao Yu46008c62016-05-09 19:56:30 +08001131skip:
Chao Yu4fe71e82016-01-26 15:37:38 +08001132 dn.ofs_in_node++;
1133 pgofs++;
1134
Chao Yu46008c62016-05-09 19:56:30 +08001135 /* preallocate blocks in batch for one dnode page */
1136 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1137 (pgofs == end || dn.ofs_in_node == end_offset)) {
Chao Yu4fe71e82016-01-26 15:37:38 +08001138
Chao Yu46008c62016-05-09 19:56:30 +08001139 dn.ofs_in_node = ofs_in_node;
Chao Yu4d57b862018-05-30 00:20:41 +08001140 err = f2fs_reserve_new_blocks(&dn, prealloc);
Chao Yu46008c62016-05-09 19:56:30 +08001141 if (err)
1142 goto sync_out;
Chao Yu4fe71e82016-01-26 15:37:38 +08001143
Chao Yu46008c62016-05-09 19:56:30 +08001144 map->m_len += dn.ofs_in_node - ofs_in_node;
1145 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1146 err = -ENOSPC;
1147 goto sync_out;
Chao Yu4fe71e82016-01-26 15:37:38 +08001148 }
Chao Yu46008c62016-05-09 19:56:30 +08001149 dn.ofs_in_node = end_offset;
Fan Li7df3a432015-12-17 13:20:59 +08001150 }
1151
Chao Yu46008c62016-05-09 19:56:30 +08001152 if (pgofs >= end)
1153 goto sync_out;
1154 else if (dn.ofs_in_node < end_offset)
1155 goto next_block;
1156
Chao Yuc4020b22018-01-11 14:42:30 +08001157 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1158 if (map->m_flags & F2FS_MAP_MAPPED) {
1159 unsigned int ofs = start_pgofs - map->m_lblk;
1160
1161 f2fs_update_extent_cache_range(&dn,
1162 start_pgofs, map->m_pblk + ofs,
1163 map->m_len - ofs);
1164 }
1165 }
1166
Chao Yu46008c62016-05-09 19:56:30 +08001167 f2fs_put_dnode(&dn);
1168
1169 if (create) {
Yunlei He59c90812017-03-13 20:22:18 +08001170 __do_map_lock(sbi, flag, false);
Chao Yu6f2d8ed2016-10-11 22:57:03 +08001171 f2fs_balance_fs(sbi, dn.node_changed);
Chao Yu46008c62016-05-09 19:56:30 +08001172 }
Chao Yu46008c62016-05-09 19:56:30 +08001173 goto next_dnode;
1174
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001175sync_out:
Chao Yuc4020b22018-01-11 14:42:30 +08001176 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1177 if (map->m_flags & F2FS_MAP_MAPPED) {
1178 unsigned int ofs = start_pgofs - map->m_lblk;
1179
1180 f2fs_update_extent_cache_range(&dn,
1181 start_pgofs, map->m_pblk + ofs,
1182 map->m_len - ofs);
1183 }
1184 if (map->m_next_extent)
1185 *map->m_next_extent = pgofs + 1;
1186 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001187 f2fs_put_dnode(&dn);
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001188unlock_out:
Jaegeuk Kim2a340762015-12-22 13:23:35 -08001189 if (create) {
Yunlei He59c90812017-03-13 20:22:18 +08001190 __do_map_lock(sbi, flag, false);
Chao Yu6f2d8ed2016-10-11 22:57:03 +08001191 f2fs_balance_fs(sbi, dn.node_changed);
Jaegeuk Kim2a340762015-12-22 13:23:35 -08001192 }
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001193out:
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001194 trace_f2fs_map_blocks(inode, map, err);
Jaegeuk Kimbfad7c22013-12-16 19:04:05 +09001195 return err;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001196}
1197
Hyunchul Leeb91050a2018-03-08 19:34:38 +09001198bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1199{
1200 struct f2fs_map_blocks map;
1201 block_t last_lblk;
1202 int err;
1203
1204 if (pos + len > i_size_read(inode))
1205 return false;
1206
1207 map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1208 map.m_next_pgofs = NULL;
1209 map.m_next_extent = NULL;
1210 map.m_seg_type = NO_CHECK_TYPE;
1211 last_lblk = F2FS_BLK_ALIGN(pos + len);
1212
1213 while (map.m_lblk < last_lblk) {
1214 map.m_len = last_lblk - map.m_lblk;
1215 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1216 if (err || map.m_len == 0)
1217 return false;
1218 map.m_lblk += map.m_len;
1219 }
1220 return true;
1221}
1222
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001223static int __get_data_block(struct inode *inode, sector_t iblock,
Chao Yuda859852016-01-26 15:42:58 +08001224 struct buffer_head *bh, int create, int flag,
Hyunchul Leed5097be2017-11-28 09:23:00 +09001225 pgoff_t *next_pgofs, int seg_type)
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001226{
1227 struct f2fs_map_blocks map;
Jaegeuk Kima7de6082016-11-11 16:31:56 -08001228 int err;
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001229
1230 map.m_lblk = iblock;
1231 map.m_len = bh->b_size >> inode->i_blkbits;
Chao Yuda859852016-01-26 15:42:58 +08001232 map.m_next_pgofs = next_pgofs;
Chao Yuc4020b22018-01-11 14:42:30 +08001233 map.m_next_extent = NULL;
Hyunchul Leed5097be2017-11-28 09:23:00 +09001234 map.m_seg_type = seg_type;
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001235
Jaegeuk Kima7de6082016-11-11 16:31:56 -08001236 err = f2fs_map_blocks(inode, &map, create, flag);
1237 if (!err) {
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001238 map_bh(bh, inode->i_sb, map.m_pblk);
1239 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
Wei Fangb86e3302017-01-22 12:21:02 +08001240 bh->b_size = (u64)map.m_len << inode->i_blkbits;
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001241 }
Jaegeuk Kima7de6082016-11-11 16:31:56 -08001242 return err;
Jaegeuk Kim003a3e12015-04-06 19:55:34 -07001243}
1244
Jaegeuk Kimccfb3002014-06-13 13:02:11 +09001245static int get_data_block(struct inode *inode, sector_t iblock,
Chao Yuda859852016-01-26 15:42:58 +08001246 struct buffer_head *bh_result, int create, int flag,
1247 pgoff_t *next_pgofs)
Jaegeuk Kimccfb3002014-06-13 13:02:11 +09001248{
Chao Yuda859852016-01-26 15:42:58 +08001249 return __get_data_block(inode, iblock, bh_result, create,
Hyunchul Leed5097be2017-11-28 09:23:00 +09001250 flag, next_pgofs,
1251 NO_CHECK_TYPE);
Jaegeuk Kimccfb3002014-06-13 13:02:11 +09001252}
1253
Chao Yue2b4e2b2015-08-19 19:11:19 +08001254static int get_data_block_dio(struct inode *inode, sector_t iblock,
Jaegeuk Kimccfb3002014-06-13 13:02:11 +09001255 struct buffer_head *bh_result, int create)
1256{
Chao Yue2b4e2b2015-08-19 19:11:19 +08001257 return __get_data_block(inode, iblock, bh_result, create,
Hyunchul Leed5097be2017-11-28 09:23:00 +09001258 F2FS_GET_BLOCK_DEFAULT, NULL,
Chao Yu4d57b862018-05-30 00:20:41 +08001259 f2fs_rw_hint_to_seg_type(
Hyunchul Leed5097be2017-11-28 09:23:00 +09001260 inode->i_write_hint));
Chao Yue2b4e2b2015-08-19 19:11:19 +08001261}
1262
1263static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1264 struct buffer_head *bh_result, int create)
1265{
Yunlei He179448b2015-12-28 21:48:32 +08001266 /* Block number less than F2FS MAX BLOCKS */
Chao Yue0afc4d2015-12-31 14:35:37 +08001267 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
Yunlei He179448b2015-12-28 21:48:32 +08001268 return -EFBIG;
1269
Chao Yue2b4e2b2015-08-19 19:11:19 +08001270 return __get_data_block(inode, iblock, bh_result, create,
Hyunchul Leed5097be2017-11-28 09:23:00 +09001271 F2FS_GET_BLOCK_BMAP, NULL,
1272 NO_CHECK_TYPE);
Jaegeuk Kimccfb3002014-06-13 13:02:11 +09001273}
1274
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001275static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1276{
1277 return (offset >> inode->i_blkbits);
1278}
1279
1280static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1281{
1282 return (blk << inode->i_blkbits);
1283}
1284
Chao Yu442a9db2018-01-11 14:39:57 +08001285static int f2fs_xattr_fiemap(struct inode *inode,
1286 struct fiemap_extent_info *fieinfo)
1287{
1288 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1289 struct page *page;
1290 struct node_info ni;
1291 __u64 phys = 0, len;
1292 __u32 flags;
1293 nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1294 int err = 0;
1295
1296 if (f2fs_has_inline_xattr(inode)) {
1297 int offset;
1298
1299 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1300 inode->i_ino, false);
1301 if (!page)
1302 return -ENOMEM;
1303
Chao Yu77357302018-07-17 00:02:17 +08001304 err = f2fs_get_node_info(sbi, inode->i_ino, &ni);
1305 if (err) {
1306 f2fs_put_page(page, 1);
1307 return err;
1308 }
Chao Yu442a9db2018-01-11 14:39:57 +08001309
1310 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1311 offset = offsetof(struct f2fs_inode, i_addr) +
1312 sizeof(__le32) * (DEF_ADDRS_PER_INODE -
Chao Yub323fd22018-01-17 16:31:36 +08001313 get_inline_xattr_addrs(inode));
Chao Yu442a9db2018-01-11 14:39:57 +08001314
1315 phys += offset;
1316 len = inline_xattr_size(inode);
1317
1318 f2fs_put_page(page, 1);
1319
1320 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1321
1322 if (!xnid)
1323 flags |= FIEMAP_EXTENT_LAST;
1324
1325 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1326 if (err || err == 1)
1327 return err;
1328 }
1329
1330 if (xnid) {
1331 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1332 if (!page)
1333 return -ENOMEM;
1334
Chao Yu77357302018-07-17 00:02:17 +08001335 err = f2fs_get_node_info(sbi, xnid, &ni);
1336 if (err) {
1337 f2fs_put_page(page, 1);
1338 return err;
1339 }
Chao Yu442a9db2018-01-11 14:39:57 +08001340
1341 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1342 len = inode->i_sb->s_blocksize;
1343
1344 f2fs_put_page(page, 1);
1345
1346 flags = FIEMAP_EXTENT_LAST;
1347 }
1348
1349 if (phys)
1350 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1351
1352 return (err < 0 ? err : 0);
1353}
1354
Jaegeuk Kim9ab701342014-06-08 04:30:14 +09001355int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1356 u64 start, u64 len)
1357{
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001358 struct buffer_head map_bh;
1359 sector_t start_blk, last_blk;
Chao Yuda859852016-01-26 15:42:58 +08001360 pgoff_t next_pgofs;
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001361 u64 logical = 0, phys = 0, size = 0;
1362 u32 flags = 0;
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001363 int ret = 0;
1364
Chao Yuc4020b22018-01-11 14:42:30 +08001365 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1366 ret = f2fs_precache_extents(inode);
1367 if (ret)
1368 return ret;
1369 }
1370
Chao Yu442a9db2018-01-11 14:39:57 +08001371 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001372 if (ret)
1373 return ret;
1374
Chao Yuf1b43d42018-01-11 14:37:35 +08001375 inode_lock(inode);
1376
Chao Yu442a9db2018-01-11 14:39:57 +08001377 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1378 ret = f2fs_xattr_fiemap(inode, fieinfo);
1379 goto out;
1380 }
1381
Jaegeuk Kim67f8cf32015-10-15 11:34:49 -07001382 if (f2fs_has_inline_data(inode)) {
1383 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1384 if (ret != -EAGAIN)
Chao Yuf1b43d42018-01-11 14:37:35 +08001385 goto out;
Jaegeuk Kim67f8cf32015-10-15 11:34:49 -07001386 }
1387
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001388 if (logical_to_blk(inode, len) == 0)
1389 len = blk_to_logical(inode, 1);
1390
1391 start_blk = logical_to_blk(inode, start);
1392 last_blk = logical_to_blk(inode, start + len - 1);
Fan Li9a950d52015-12-26 18:07:41 +08001393
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001394next:
1395 memset(&map_bh, 0, sizeof(struct buffer_head));
1396 map_bh.b_size = len;
1397
Chao Yue2b4e2b2015-08-19 19:11:19 +08001398 ret = get_data_block(inode, start_blk, &map_bh, 0,
Chao Yuda859852016-01-26 15:42:58 +08001399 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001400 if (ret)
1401 goto out;
1402
1403 /* HOLE */
1404 if (!buffer_mapped(&map_bh)) {
Chao Yuda859852016-01-26 15:42:58 +08001405 start_blk = next_pgofs;
Chao Yu58736fa2016-10-11 22:57:04 +08001406
1407 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1408 F2FS_I_SB(inode)->max_file_blocks))
Fan Li9a950d52015-12-26 18:07:41 +08001409 goto prep_next;
Chao Yu58736fa2016-10-11 22:57:04 +08001410
Fan Li9a950d52015-12-26 18:07:41 +08001411 flags |= FIEMAP_EXTENT_LAST;
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001412 }
Fan Li9a950d52015-12-26 18:07:41 +08001413
Chao Yuda5af122016-01-08 20:19:27 +08001414 if (size) {
1415 if (f2fs_encrypted_inode(inode))
1416 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1417
Fan Li9a950d52015-12-26 18:07:41 +08001418 ret = fiemap_fill_next_extent(fieinfo, logical,
1419 phys, size, flags);
Chao Yuda5af122016-01-08 20:19:27 +08001420 }
Fan Li9a950d52015-12-26 18:07:41 +08001421
1422 if (start_blk > last_blk || ret)
1423 goto out;
1424
1425 logical = blk_to_logical(inode, start_blk);
1426 phys = blk_to_logical(inode, map_bh.b_blocknr);
1427 size = map_bh.b_size;
1428 flags = 0;
1429 if (buffer_unwritten(&map_bh))
1430 flags = FIEMAP_EXTENT_UNWRITTEN;
1431
1432 start_blk += logical_to_blk(inode, size);
1433
1434prep_next:
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001435 cond_resched();
1436 if (fatal_signal_pending(current))
1437 ret = -EINTR;
1438 else
1439 goto next;
1440out:
1441 if (ret == 1)
1442 ret = 0;
1443
Al Viro59551022016-01-22 15:40:57 -05001444 inode_unlock(inode);
Jaegeuk Kim7f63eb72015-05-08 19:30:32 -07001445 return ret;
Jaegeuk Kim9ab701342014-06-08 04:30:14 +09001446}
1447
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001448/*
1449 * This function was originally taken from fs/mpage.c, and customized for f2fs.
1450 * Major change was from block_size == page_size in f2fs by default.
Jaegeuk Kime2e59412018-06-21 11:29:43 -07001451 *
1452 * Note that the aops->readpages() function is ONLY used for read-ahead. If
1453 * this function ever deviates from doing just read-ahead, it should either
1454 * use ->readpage() or do the necessary surgery to decouple ->readpages()
1455 * from read-ahead.
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001456 */
1457static int f2fs_mpage_readpages(struct address_space *mapping,
1458 struct list_head *pages, struct page *page,
Jaegeuk Kime2e59412018-06-21 11:29:43 -07001459 unsigned nr_pages, bool is_readahead)
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001460{
1461 struct bio *bio = NULL;
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001462 sector_t last_block_in_bio = 0;
1463 struct inode *inode = mapping->host;
1464 const unsigned blkbits = inode->i_blkbits;
1465 const unsigned blocksize = 1 << blkbits;
1466 sector_t block_in_file;
1467 sector_t last_block;
1468 sector_t last_block_in_file;
1469 sector_t block_nr;
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001470 struct f2fs_map_blocks map;
1471
1472 map.m_pblk = 0;
1473 map.m_lblk = 0;
1474 map.m_len = 0;
1475 map.m_flags = 0;
Chao Yuda859852016-01-26 15:42:58 +08001476 map.m_next_pgofs = NULL;
Chao Yuc4020b22018-01-11 14:42:30 +08001477 map.m_next_extent = NULL;
Hyunchul Leed5097be2017-11-28 09:23:00 +09001478 map.m_seg_type = NO_CHECK_TYPE;
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001479
LiFan736c0a72017-11-25 11:46:18 +08001480 for (; nr_pages; nr_pages--) {
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001481 if (pages) {
Chao Yu939afa92017-01-07 18:49:42 +08001482 page = list_last_entry(pages, struct page, lru);
Kinglong Meea83d50b2017-03-13 16:35:13 +08001483
1484 prefetchw(&page->flags);
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001485 list_del(&page->lru);
1486 if (add_to_page_cache_lru(page, mapping,
Michal Hocko8a5c7432016-07-26 15:24:53 -07001487 page->index,
1488 readahead_gfp_mask(mapping)))
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001489 goto next_page;
1490 }
1491
1492 block_in_file = (sector_t)page->index;
1493 last_block = block_in_file + nr_pages;
1494 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1495 blkbits;
1496 if (last_block > last_block_in_file)
1497 last_block = last_block_in_file;
1498
1499 /*
1500 * Map blocks using the previous result first.
1501 */
1502 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1503 block_in_file > map.m_lblk &&
1504 block_in_file < (map.m_lblk + map.m_len))
1505 goto got_it;
1506
1507 /*
1508 * Then do more f2fs_map_blocks() calls until we are
1509 * done with this page.
1510 */
1511 map.m_flags = 0;
1512
1513 if (block_in_file < last_block) {
1514 map.m_lblk = block_in_file;
1515 map.m_len = last_block - block_in_file;
1516
Chao Yu46c9e142015-09-18 16:54:16 +08001517 if (f2fs_map_blocks(inode, &map, 0,
Qiuyang Sunf2220c72017-08-09 17:27:30 +08001518 F2FS_GET_BLOCK_DEFAULT))
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001519 goto set_error_page;
1520 }
1521got_it:
1522 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1523 block_nr = map.m_pblk + block_in_file - map.m_lblk;
1524 SetPageMappedToDisk(page);
1525
1526 if (!PageUptodate(page) && !cleancache_get_page(page)) {
1527 SetPageUptodate(page);
1528 goto confused;
1529 }
Chao Yuc9b60782018-08-01 19:13:44 +08001530
1531 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
1532 DATA_GENERIC))
1533 goto set_error_page;
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001534 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001535 zero_user_segment(page, 0, PAGE_SIZE);
Jaegeuk Kim237c0792016-06-30 18:49:15 -07001536 if (!PageUptodate(page))
1537 SetPageUptodate(page);
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001538 unlock_page(page);
1539 goto next_page;
1540 }
1541
1542 /*
1543 * This page will go to BIO. Do we need to send this
1544 * BIO off first?
1545 */
Jaegeuk Kim3c62be12016-10-06 19:02:05 -07001546 if (bio && (last_block_in_bio != block_nr - 1 ||
1547 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001548submit_and_realloc:
Linus Torvalds4fc29c12016-07-27 10:36:31 -07001549 __submit_bio(F2FS_I_SB(inode), bio, DATA);
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001550 bio = NULL;
1551 }
1552 if (bio == NULL) {
Jaegeuk Kime2e59412018-06-21 11:29:43 -07001553 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
1554 is_readahead ? REQ_RAHEAD : 0);
Jaegeuk Kim1d353eb2016-07-12 09:38:48 -07001555 if (IS_ERR(bio)) {
1556 bio = NULL;
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001557 goto set_error_page;
Jaegeuk Kim4375a332015-04-23 12:04:33 -07001558 }
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001559 }
1560
1561 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1562 goto submit_and_realloc;
1563
1564 last_block_in_bio = block_nr;
1565 goto next_page;
1566set_error_page:
1567 SetPageError(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001568 zero_user_segment(page, 0, PAGE_SIZE);
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001569 unlock_page(page);
1570 goto next_page;
1571confused:
1572 if (bio) {
Linus Torvalds4fc29c12016-07-27 10:36:31 -07001573 __submit_bio(F2FS_I_SB(inode), bio, DATA);
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001574 bio = NULL;
1575 }
1576 unlock_page(page);
1577next_page:
1578 if (pages)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001579 put_page(page);
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001580 }
1581 BUG_ON(pages && !list_empty(pages));
1582 if (bio)
Linus Torvalds4fc29c12016-07-27 10:36:31 -07001583 __submit_bio(F2FS_I_SB(inode), bio, DATA);
Jaegeuk Kimf1e88662015-04-09 11:20:42 -07001584 return 0;
1585}
1586
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001587static int f2fs_read_data_page(struct file *file, struct page *page)
1588{
Huajun Li9ffe0fb2013-11-10 23:13:20 +08001589 struct inode *inode = page->mapping->host;
Jaegeuk Kimb3d208f2014-10-23 19:48:09 -07001590 int ret = -EAGAIN;
Huajun Li9ffe0fb2013-11-10 23:13:20 +08001591
Chao Yuc20e89cd2014-05-06 16:53:08 +08001592 trace_f2fs_readpage(page, DATA);
1593
arter97e1c42042014-08-06 23:22:50 +09001594 /* If the file has inline data, try to read it directly */
Huajun Li9ffe0fb2013-11-10 23:13:20 +08001595 if (f2fs_has_inline_data(inode))
1596 ret = f2fs_read_inline_data(inode, page);
Jaegeuk Kimb3d208f2014-10-23 19:48:09 -07001597 if (ret == -EAGAIN)
Jaegeuk Kime2e59412018-06-21 11:29:43 -07001598 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1, false);
Huajun Li9ffe0fb2013-11-10 23:13:20 +08001599 return ret;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001600}
1601
1602static int f2fs_read_data_pages(struct file *file,
1603 struct address_space *mapping,
1604 struct list_head *pages, unsigned nr_pages)
1605{
Hsiang Kao71cb4af2017-09-24 02:45:42 +08001606 struct inode *inode = mapping->host;
Chao Yu939afa92017-01-07 18:49:42 +08001607 struct page *page = list_last_entry(pages, struct page, lru);
Chao Yub8c29402015-10-12 17:02:26 +08001608
1609 trace_f2fs_readpages(inode, page, nr_pages);
Huajun Li9ffe0fb2013-11-10 23:13:20 +08001610
1611 /* If the file has inline data, skip readpages */
1612 if (f2fs_has_inline_data(inode))
1613 return 0;
1614
Jaegeuk Kime2e59412018-06-21 11:29:43 -07001615 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages, true);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001616}
1617
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001618static int encrypt_one_page(struct f2fs_io_info *fio)
1619{
1620 struct inode *inode = fio->page->mapping->host;
Chao Yu6aa58d82018-08-14 22:37:25 +08001621 struct page *mpage;
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001622 gfp_t gfp_flags = GFP_NOFS;
1623
Jaegeuk Kim19585932017-09-05 16:54:24 -07001624 if (!f2fs_encrypted_file(inode))
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001625 return 0;
1626
Eric Biggers6dbb1792018-04-18 11:09:48 -07001627 /* wait for GCed page writeback via META_MAPPING */
Jaegeuk Kimd4c759e2017-09-05 17:04:35 -07001628 f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr);
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001629
1630retry_encrypt:
1631 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1632 PAGE_SIZE, 0, fio->page->index, gfp_flags);
Chao Yu6aa58d82018-08-14 22:37:25 +08001633 if (IS_ERR(fio->encrypted_page)) {
1634 /* flush pending IOs and wait for a while in the ENOMEM case */
1635 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1636 f2fs_flush_merged_writes(fio->sbi);
1637 congestion_wait(BLK_RW_ASYNC, HZ/50);
1638 gfp_flags |= __GFP_NOFAIL;
1639 goto retry_encrypt;
1640 }
1641 return PTR_ERR(fio->encrypted_page);
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001642 }
Chao Yu6aa58d82018-08-14 22:37:25 +08001643
1644 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);
1645 if (mpage) {
1646 if (PageUptodate(mpage))
1647 memcpy(page_address(mpage),
1648 page_address(fio->encrypted_page), PAGE_SIZE);
1649 f2fs_put_page(mpage, 1);
1650 }
1651 return 0;
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001652}
1653
Chao Yubb9e3bb8d2018-01-17 16:31:38 +08001654static inline bool check_inplace_update_policy(struct inode *inode,
1655 struct f2fs_io_info *fio)
1656{
1657 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1658 unsigned int policy = SM_I(sbi)->ipu_policy;
1659
1660 if (policy & (0x1 << F2FS_IPU_FORCE))
1661 return true;
Chao Yu4d57b862018-05-30 00:20:41 +08001662 if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi))
Chao Yubb9e3bb8d2018-01-17 16:31:38 +08001663 return true;
1664 if (policy & (0x1 << F2FS_IPU_UTIL) &&
1665 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1666 return true;
Chao Yu4d57b862018-05-30 00:20:41 +08001667 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) &&
Chao Yubb9e3bb8d2018-01-17 16:31:38 +08001668 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1669 return true;
1670
1671 /*
1672 * IPU for rewrite async pages
1673 */
1674 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
1675 fio && fio->op == REQ_OP_WRITE &&
1676 !(fio->op_flags & REQ_SYNC) &&
1677 !f2fs_encrypted_inode(inode))
1678 return true;
1679
1680 /* this is only set during fdatasync */
1681 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
1682 is_inode_flag_set(inode, FI_NEED_IPU))
1683 return true;
1684
1685 return false;
1686}
1687
Chao Yu4d57b862018-05-30 00:20:41 +08001688bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
Chao Yubb9e3bb8d2018-01-17 16:31:38 +08001689{
1690 if (f2fs_is_pinned_file(inode))
1691 return true;
1692
1693 /* if this is cold file, we should overwrite to avoid fragmentation */
1694 if (file_is_cold(inode))
1695 return true;
1696
1697 return check_inplace_update_policy(inode, fio);
1698}
1699
Chao Yu4d57b862018-05-30 00:20:41 +08001700bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
Chao Yubb9e3bb8d2018-01-17 16:31:38 +08001701{
1702 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1703
1704 if (test_opt(sbi, LFS))
1705 return true;
1706 if (S_ISDIR(inode->i_mode))
1707 return true;
1708 if (f2fs_is_atomic_file(inode))
1709 return true;
1710 if (fio) {
1711 if (is_cold_data(fio->page))
1712 return true;
1713 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1714 return true;
1715 }
1716 return false;
1717}
1718
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001719static inline bool need_inplace_update(struct f2fs_io_info *fio)
1720{
1721 struct inode *inode = fio->page->mapping->host;
1722
Chao Yu4d57b862018-05-30 00:20:41 +08001723 if (f2fs_should_update_outplace(inode, fio))
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001724 return false;
1725
Chao Yu4d57b862018-05-30 00:20:41 +08001726 return f2fs_should_update_inplace(inode, fio);
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001727}
1728
Chao Yu4d57b862018-05-30 00:20:41 +08001729int f2fs_do_write_data_page(struct f2fs_io_info *fio)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001730{
Jaegeuk Kim05ca3632015-04-23 14:38:15 -07001731 struct page *page = fio->page;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001732 struct inode *inode = page->mapping->host;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001733 struct dnode_of_data dn;
Hou Pengyange959c8f2017-04-25 12:45:13 +00001734 struct extent_info ei = {0,0,0};
Chao Yu77357302018-07-17 00:02:17 +08001735 struct node_info ni;
Hou Pengyange959c8f2017-04-25 12:45:13 +00001736 bool ipu_force = false;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001737 int err = 0;
1738
1739 set_new_dnode(&dn, inode, NULL, NULL, 0);
Hou Pengyange959c8f2017-04-25 12:45:13 +00001740 if (need_inplace_update(fio) &&
1741 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1742 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
Jaegeuk Kima8177372017-04-24 15:20:16 -07001743
Chao Yuc9b60782018-08-01 19:13:44 +08001744 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1745 DATA_GENERIC))
1746 return -EFAULT;
1747
1748 ipu_force = true;
1749 fio->need_lock = LOCK_DONE;
1750 goto got_it;
Hou Pengyange959c8f2017-04-25 12:45:13 +00001751 }
Hou Pengyang279d6df2017-04-27 00:17:21 +08001752
Jaegeuk Kimd29460e2017-06-21 17:52:39 -07001753 /* Deadlock due to between page->lock and f2fs_lock_op */
1754 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1755 return -EAGAIN;
Hou Pengyang279d6df2017-04-27 00:17:21 +08001756
Chao Yu4d57b862018-05-30 00:20:41 +08001757 err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001758 if (err)
Hou Pengyang279d6df2017-04-27 00:17:21 +08001759 goto out;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001760
Chao Yu28bc1062016-02-06 14:40:34 +08001761 fio->old_blkaddr = dn.data_blkaddr;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001762
1763 /* This page is already truncated */
Chao Yu7a9d7542016-02-22 18:36:38 +08001764 if (fio->old_blkaddr == NULL_ADDR) {
Jaegeuk Kim2bca1e22015-02-25 19:25:01 -08001765 ClearPageUptodate(page);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001766 goto out_writepage;
Jaegeuk Kim2bca1e22015-02-25 19:25:01 -08001767 }
Hou Pengyange959c8f2017-04-25 12:45:13 +00001768got_it:
Chao Yuc9b60782018-08-01 19:13:44 +08001769 if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
1770 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1771 DATA_GENERIC)) {
1772 err = -EFAULT;
1773 goto out_writepage;
1774 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001775 /*
1776 * If current allocation needs SSR,
1777 * it had better in-place writes for updated data.
1778 */
Chao Yue1da7872018-06-05 17:44:11 +08001779 if (ipu_force || (is_valid_data_blkaddr(fio->sbi, fio->old_blkaddr) &&
Chao Yu7b525dd2018-05-23 22:25:08 +08001780 need_inplace_update(fio))) {
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001781 err = encrypt_one_page(fio);
1782 if (err)
1783 goto out_writepage;
1784
1785 set_page_writeback(page);
Jaegeuk Kim17c50032018-04-11 23:09:04 -07001786 ClearPageError(page);
Hou Pengyang279d6df2017-04-27 00:17:21 +08001787 f2fs_put_dnode(&dn);
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001788 if (fio->need_lock == LOCK_REQ)
Hou Pengyang279d6df2017-04-27 00:17:21 +08001789 f2fs_unlock_op(fio->sbi);
Chao Yu4d57b862018-05-30 00:20:41 +08001790 err = f2fs_inplace_write_data(fio);
Hou Pengyang7eab0c02017-04-25 12:45:12 +00001791 trace_f2fs_do_write_data_page(fio->page, IPU);
Jaegeuk Kim91942322016-05-20 10:13:22 -07001792 set_inode_flag(inode, FI_UPDATE_WRITE);
Hou Pengyang279d6df2017-04-27 00:17:21 +08001793 return err;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001794 }
Hou Pengyang279d6df2017-04-27 00:17:21 +08001795
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001796 if (fio->need_lock == LOCK_RETRY) {
1797 if (!f2fs_trylock_op(fio->sbi)) {
1798 err = -EAGAIN;
1799 goto out_writepage;
1800 }
1801 fio->need_lock = LOCK_REQ;
1802 }
1803
Chao Yu77357302018-07-17 00:02:17 +08001804 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);
1805 if (err)
1806 goto out_writepage;
1807
1808 fio->version = ni.version;
1809
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001810 err = encrypt_one_page(fio);
1811 if (err)
1812 goto out_writepage;
1813
1814 set_page_writeback(page);
Jaegeuk Kim17c50032018-04-11 23:09:04 -07001815 ClearPageError(page);
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001816
Hou Pengyang279d6df2017-04-27 00:17:21 +08001817 /* LFS mode write path */
Chao Yu4d57b862018-05-30 00:20:41 +08001818 f2fs_outplace_write_data(&dn, fio);
Hou Pengyang279d6df2017-04-27 00:17:21 +08001819 trace_f2fs_do_write_data_page(page, OPU);
1820 set_inode_flag(inode, FI_APPEND_WRITE);
1821 if (page->index == 0)
1822 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001823out_writepage:
1824 f2fs_put_dnode(&dn);
Hou Pengyang279d6df2017-04-27 00:17:21 +08001825out:
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001826 if (fio->need_lock == LOCK_REQ)
Hou Pengyang279d6df2017-04-27 00:17:21 +08001827 f2fs_unlock_op(fio->sbi);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001828 return err;
1829}
1830
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001831static int __write_data_page(struct page *page, bool *submitted,
Chao Yub0af6d42017-08-02 23:21:48 +08001832 struct writeback_control *wbc,
1833 enum iostat_type io_type)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001834{
1835 struct inode *inode = page->mapping->host;
Jaegeuk Kim40813632014-09-02 15:31:18 -07001836 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001837 loff_t i_size = i_size_read(inode);
1838 const pgoff_t end_index = ((unsigned long long) i_size)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001839 >> PAGE_SHIFT;
Jaegeuk Kim26de9b12016-05-20 20:42:37 -07001840 loff_t psize = (page->index + 1) << PAGE_SHIFT;
Huajun Li9ffe0fb2013-11-10 23:13:20 +08001841 unsigned offset = 0;
Jaegeuk Kim39936832012-11-22 16:21:29 +09001842 bool need_balance_fs = false;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001843 int err = 0;
Jaegeuk Kim458e6192013-12-11 13:54:01 +09001844 struct f2fs_io_info fio = {
Jaegeuk Kim05ca3632015-04-23 14:38:15 -07001845 .sbi = sbi,
Chao Yu39d787b2017-09-29 13:59:38 +08001846 .ino = inode->i_ino,
Jaegeuk Kim458e6192013-12-11 13:54:01 +09001847 .type = DATA,
Mike Christie04d328d2016-06-05 14:31:55 -05001848 .op = REQ_OP_WRITE,
Jens Axboe76372412016-11-01 10:00:38 -06001849 .op_flags = wbc_to_write_flags(wbc),
Hou Pengyange959c8f2017-04-25 12:45:13 +00001850 .old_blkaddr = NULL_ADDR,
Jaegeuk Kim05ca3632015-04-23 14:38:15 -07001851 .page = page,
Jaegeuk Kim4375a332015-04-23 12:04:33 -07001852 .encrypted_page = NULL,
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001853 .submitted = false,
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001854 .need_lock = LOCK_RETRY,
Chao Yub0af6d42017-08-02 23:21:48 +08001855 .io_type = io_type,
Yufen Yu578c6472018-01-09 19:33:39 +08001856 .io_wbc = wbc,
Jaegeuk Kim458e6192013-12-11 13:54:01 +09001857 };
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001858
Chao Yuecda0de2014-05-06 16:48:26 +08001859 trace_f2fs_writepage(page, DATA);
1860
Chao Yudb198ae2018-01-18 17:29:10 +08001861 /* we should bypass data pages to proceed the kworkder jobs */
1862 if (unlikely(f2fs_cp_error(sbi))) {
1863 mapping_set_error(page->mapping, -EIO);
Chao Yu1174abf2018-05-28 16:59:26 +08001864 /*
1865 * don't drop any dirty dentry pages for keeping lastest
1866 * directory structure.
1867 */
1868 if (S_ISDIR(inode->i_mode))
1869 goto redirty_out;
Chao Yudb198ae2018-01-18 17:29:10 +08001870 goto out;
1871 }
1872
Chao Yu0771fcc2017-06-29 23:20:45 +08001873 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1874 goto redirty_out;
1875
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001876 if (page->index < end_index)
Jaegeuk Kim39936832012-11-22 16:21:29 +09001877 goto write;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001878
1879 /*
1880 * If the offset is out-of-range of file size,
1881 * this page does not have to be written to disk.
1882 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001883 offset = i_size & (PAGE_SIZE - 1);
Jaegeuk Kim76f60262014-04-15 16:04:15 +09001884 if ((page->index >= end_index + 1) || !offset)
Jaegeuk Kim39936832012-11-22 16:21:29 +09001885 goto out;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001886
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001887 zero_user_segment(page, offset, PAGE_SIZE);
Jaegeuk Kim39936832012-11-22 16:21:29 +09001888write:
Jaegeuk Kim1e843712014-12-09 06:08:59 -08001889 if (f2fs_is_drop_cache(inode))
1890 goto out;
Jaegeuk Kime6e5f562016-04-14 16:48:52 -07001891 /* we should not write 0'th page having journal header */
1892 if (f2fs_is_volatile_file(inode) && (!page->index ||
1893 (!wbc->for_reclaim &&
Chao Yu4d57b862018-05-30 00:20:41 +08001894 f2fs_available_free_memory(sbi, BASE_CHECK))))
Jaegeuk Kim1e843712014-12-09 06:08:59 -08001895 goto redirty_out;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001896
Jaegeuk Kimb230e6c2016-05-29 21:18:23 -07001897 /* Dentry blocks are controlled by checkpoint */
1898 if (S_ISDIR(inode->i_mode)) {
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001899 fio.need_lock = LOCK_DONE;
Chao Yu4d57b862018-05-30 00:20:41 +08001900 err = f2fs_do_write_data_page(&fio);
Jaegeuk Kimb230e6c2016-05-29 21:18:23 -07001901 goto done;
1902 }
1903
Jaegeuk Kim8618b882014-02-17 19:29:27 +09001904 if (!wbc->for_reclaim)
1905 need_balance_fs = true;
Jaegeuk Kim7f3037a2016-09-01 12:02:51 -07001906 else if (has_not_enough_free_secs(sbi, 0, 0))
Jaegeuk Kim39936832012-11-22 16:21:29 +09001907 goto redirty_out;
Jaegeuk Kimef095d12017-03-24 20:05:13 -04001908 else
1909 set_inode_flag(inode, FI_HOT_DATA);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001910
Jaegeuk Kimb3d208f2014-10-23 19:48:09 -07001911 err = -EAGAIN;
Yunlei Hedd7b2332017-02-23 20:31:20 +08001912 if (f2fs_has_inline_data(inode)) {
Jaegeuk Kimb3d208f2014-10-23 19:48:09 -07001913 err = f2fs_write_inline_data(inode, page);
Yunlei Hedd7b2332017-02-23 20:31:20 +08001914 if (!err)
1915 goto out;
1916 }
Hou Pengyang279d6df2017-04-27 00:17:21 +08001917
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001918 if (err == -EAGAIN) {
Chao Yu4d57b862018-05-30 00:20:41 +08001919 err = f2fs_do_write_data_page(&fio);
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001920 if (err == -EAGAIN) {
1921 fio.need_lock = LOCK_REQ;
Chao Yu4d57b862018-05-30 00:20:41 +08001922 err = f2fs_do_write_data_page(&fio);
Jaegeuk Kimcc156202017-05-12 13:51:34 -07001923 }
1924 }
Chao Yua0d00fa2017-10-09 17:55:19 +08001925
Chao Yueb449792018-01-17 16:31:37 +08001926 if (err) {
1927 file_set_keep_isize(inode);
1928 } else {
1929 down_write(&F2FS_I(inode)->i_sem);
1930 if (F2FS_I(inode)->last_disk_size < psize)
1931 F2FS_I(inode)->last_disk_size = psize;
1932 up_write(&F2FS_I(inode)->i_sem);
1933 }
Hou Pengyang279d6df2017-04-27 00:17:21 +08001934
Jaegeuk Kim8618b882014-02-17 19:29:27 +09001935done:
1936 if (err && err != -ENOENT)
1937 goto redirty_out;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001938
Jaegeuk Kim39936832012-11-22 16:21:29 +09001939out:
Jaegeuk Kima7ffdbe2014-09-12 15:53:45 -07001940 inode_dec_dirty_pages(inode);
Jaegeuk Kim2bca1e22015-02-25 19:25:01 -08001941 if (err)
1942 ClearPageUptodate(page);
Chao Yu0c3a5792016-01-18 18:28:11 +08001943
1944 if (wbc->for_reclaim) {
Jaegeuk Kimb9109b02017-05-10 11:28:38 -07001945 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
Jaegeuk Kimef095d12017-03-24 20:05:13 -04001946 clear_inode_flag(inode, FI_HOT_DATA);
Chao Yu4d57b862018-05-30 00:20:41 +08001947 f2fs_remove_dirty_inode(inode);
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001948 submitted = NULL;
Chao Yueb7e8132015-11-10 18:45:07 +08001949 }
Chao Yu0c3a5792016-01-18 18:28:11 +08001950
1951 unlock_page(page);
Jaegeuk Kima7881892017-04-20 13:51:57 -07001952 if (!S_ISDIR(inode->i_mode))
1953 f2fs_balance_fs(sbi, need_balance_fs);
Chao Yu0c3a5792016-01-18 18:28:11 +08001954
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001955 if (unlikely(f2fs_cp_error(sbi))) {
Jaegeuk Kimb9109b02017-05-10 11:28:38 -07001956 f2fs_submit_merged_write(sbi, DATA);
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08001957 submitted = NULL;
1958 }
1959
1960 if (submitted)
1961 *submitted = fio.submitted;
Chao Yu0c3a5792016-01-18 18:28:11 +08001962
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001963 return 0;
1964
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09001965redirty_out:
Jaegeuk Kim76f60262014-04-15 16:04:15 +09001966 redirty_page_for_writepage(wbc, page);
Jaegeuk Kim5b19d282018-05-03 23:26:02 -07001967 /*
1968 * pageout() in MM traslates EAGAIN, so calls handle_write_error()
1969 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
1970 * file_write_and_wait_range() will see EIO error, which is critical
1971 * to return value of fsync() followed by atomic_write failure to user.
1972 */
1973 if (!err || wbc->for_reclaim)
Chao Yu0002b612016-11-28 19:13:43 -08001974 return AOP_WRITEPAGE_ACTIVATE;
Jaegeuk Kimb230e6c2016-05-29 21:18:23 -07001975 unlock_page(page);
1976 return err;
Namjae Jeonfa9150a2013-01-15 16:45:24 +09001977}
1978
Jaegeuk Kimf566bae2017-02-03 17:18:00 -08001979static int f2fs_write_data_page(struct page *page,
1980 struct writeback_control *wbc)
1981{
Chao Yub0af6d42017-08-02 23:21:48 +08001982 return __write_data_page(page, NULL, wbc, FS_DATA_IO);
Jaegeuk Kimf566bae2017-02-03 17:18:00 -08001983}
1984
Chao Yu8f46dca2015-07-14 18:56:10 +08001985/*
1986 * This function was copied from write_cche_pages from mm/page-writeback.c.
1987 * The major change is making write step of cold data page separately from
1988 * warm/hot data page.
1989 */
1990static int f2fs_write_cache_pages(struct address_space *mapping,
Chao Yub0af6d42017-08-02 23:21:48 +08001991 struct writeback_control *wbc,
1992 enum iostat_type io_type)
Chao Yu8f46dca2015-07-14 18:56:10 +08001993{
1994 int ret = 0;
1995 int done = 0;
1996 struct pagevec pvec;
Chao Yuc29fd0c2018-06-04 23:20:36 +08001997 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
Chao Yu8f46dca2015-07-14 18:56:10 +08001998 int nr_pages;
1999 pgoff_t uninitialized_var(writeback_index);
2000 pgoff_t index;
2001 pgoff_t end; /* Inclusive */
2002 pgoff_t done_index;
Jaegeuk Kim942fd312017-02-01 16:51:22 -08002003 pgoff_t last_idx = ULONG_MAX;
Chao Yu8f46dca2015-07-14 18:56:10 +08002004 int cycled;
2005 int range_whole = 0;
2006 int tag;
Chao Yu8f46dca2015-07-14 18:56:10 +08002007
Mel Gorman86679822017-11-15 17:37:52 -08002008 pagevec_init(&pvec);
Jaegeuk Kim46ae9572016-05-25 20:57:16 -07002009
Jaegeuk Kimef095d12017-03-24 20:05:13 -04002010 if (get_dirty_pages(mapping->host) <=
2011 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
2012 set_inode_flag(mapping->host, FI_HOT_DATA);
2013 else
2014 clear_inode_flag(mapping->host, FI_HOT_DATA);
2015
Chao Yu8f46dca2015-07-14 18:56:10 +08002016 if (wbc->range_cyclic) {
2017 writeback_index = mapping->writeback_index; /* prev offset */
2018 index = writeback_index;
2019 if (index == 0)
2020 cycled = 1;
2021 else
2022 cycled = 0;
2023 end = -1;
2024 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002025 index = wbc->range_start >> PAGE_SHIFT;
2026 end = wbc->range_end >> PAGE_SHIFT;
Chao Yu8f46dca2015-07-14 18:56:10 +08002027 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2028 range_whole = 1;
2029 cycled = 1; /* ignore range_cyclic tests */
2030 }
2031 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2032 tag = PAGECACHE_TAG_TOWRITE;
2033 else
2034 tag = PAGECACHE_TAG_DIRTY;
2035retry:
2036 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2037 tag_pages_for_writeback(mapping, index, end);
2038 done_index = index;
2039 while (!done && (index <= end)) {
2040 int i;
2041
Jan Kara69c4f352017-11-15 17:34:48 -08002042 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
Jan Kara67fd7072017-11-15 17:35:19 -08002043 tag);
Chao Yu8f46dca2015-07-14 18:56:10 +08002044 if (nr_pages == 0)
2045 break;
2046
2047 for (i = 0; i < nr_pages; i++) {
2048 struct page *page = pvec.pages[i];
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08002049 bool submitted = false;
Chao Yu8f46dca2015-07-14 18:56:10 +08002050
Chao Yuf8de4332018-05-23 22:25:09 +08002051 /* give a priority to WB_SYNC threads */
Chao Yuc29fd0c2018-06-04 23:20:36 +08002052 if (atomic_read(&sbi->wb_sync_req[DATA]) &&
Chao Yuf8de4332018-05-23 22:25:09 +08002053 wbc->sync_mode == WB_SYNC_NONE) {
2054 done = 1;
2055 break;
2056 }
2057
Chao Yu8f46dca2015-07-14 18:56:10 +08002058 done_index = page->index;
Jaegeuk Kimd29460e2017-06-21 17:52:39 -07002059retry_write:
Chao Yu8f46dca2015-07-14 18:56:10 +08002060 lock_page(page);
2061
2062 if (unlikely(page->mapping != mapping)) {
2063continue_unlock:
2064 unlock_page(page);
2065 continue;
2066 }
2067
2068 if (!PageDirty(page)) {
2069 /* someone wrote it for us */
2070 goto continue_unlock;
2071 }
2072
Chao Yu8f46dca2015-07-14 18:56:10 +08002073 if (PageWriteback(page)) {
2074 if (wbc->sync_mode != WB_SYNC_NONE)
Jaegeuk Kimfec1d652016-01-20 23:43:51 +08002075 f2fs_wait_on_page_writeback(page,
2076 DATA, true);
Chao Yu8f46dca2015-07-14 18:56:10 +08002077 else
2078 goto continue_unlock;
2079 }
2080
2081 BUG_ON(PageWriteback(page));
2082 if (!clear_page_dirty_for_io(page))
2083 goto continue_unlock;
2084
Chao Yub0af6d42017-08-02 23:21:48 +08002085 ret = __write_data_page(page, &submitted, wbc, io_type);
Chao Yu8f46dca2015-07-14 18:56:10 +08002086 if (unlikely(ret)) {
Chao Yu0002b612016-11-28 19:13:43 -08002087 /*
2088 * keep nr_to_write, since vfs uses this to
2089 * get # of written pages.
2090 */
2091 if (ret == AOP_WRITEPAGE_ACTIVATE) {
2092 unlock_page(page);
2093 ret = 0;
2094 continue;
Jaegeuk Kimd29460e2017-06-21 17:52:39 -07002095 } else if (ret == -EAGAIN) {
2096 ret = 0;
2097 if (wbc->sync_mode == WB_SYNC_ALL) {
2098 cond_resched();
2099 congestion_wait(BLK_RW_ASYNC,
2100 HZ/50);
2101 goto retry_write;
2102 }
2103 continue;
Chao Yu0002b612016-11-28 19:13:43 -08002104 }
Jaegeuk Kimb230e6c2016-05-29 21:18:23 -07002105 done_index = page->index + 1;
2106 done = 1;
2107 break;
Jaegeuk Kimd68f7352017-02-03 17:44:04 -08002108 } else if (submitted) {
Jaegeuk Kim942fd312017-02-01 16:51:22 -08002109 last_idx = page->index;
Chao Yu8f46dca2015-07-14 18:56:10 +08002110 }
2111
Chao Yuf8de4332018-05-23 22:25:09 +08002112 if (--wbc->nr_to_write <= 0 &&
Jaegeuk Kim687de7f2017-03-28 18:07:38 -07002113 wbc->sync_mode == WB_SYNC_NONE) {
Chao Yu8f46dca2015-07-14 18:56:10 +08002114 done = 1;
2115 break;
2116 }
2117 }
2118 pagevec_release(&pvec);
2119 cond_resched();
2120 }
2121
Chao Yu8f46dca2015-07-14 18:56:10 +08002122 if (!cycled && !done) {
2123 cycled = 1;
2124 index = 0;
2125 end = writeback_index - 1;
2126 goto retry;
2127 }
2128 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2129 mapping->writeback_index = done_index;
2130
Jaegeuk Kim942fd312017-02-01 16:51:22 -08002131 if (last_idx != ULONG_MAX)
Jaegeuk Kimb9109b02017-05-10 11:28:38 -07002132 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
2133 0, last_idx, DATA);
Chao Yu6ca56ca2016-09-29 18:50:11 +08002134
Chao Yu8f46dca2015-07-14 18:56:10 +08002135 return ret;
2136}
2137
Jaegeuk Kim853137c2018-08-09 17:53:34 -07002138static inline bool __should_serialize_io(struct inode *inode,
2139 struct writeback_control *wbc)
2140{
2141 if (!S_ISREG(inode->i_mode))
2142 return false;
2143 if (wbc->sync_mode != WB_SYNC_ALL)
2144 return true;
2145 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks)
2146 return true;
2147 return false;
2148}
2149
Chao Yufc99fe22018-05-30 00:20:39 +08002150static int __f2fs_write_data_pages(struct address_space *mapping,
Chao Yub0af6d42017-08-02 23:21:48 +08002151 struct writeback_control *wbc,
2152 enum iostat_type io_type)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002153{
2154 struct inode *inode = mapping->host;
Jaegeuk Kim40813632014-09-02 15:31:18 -07002155 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kim9dfa1ba2016-07-13 19:33:19 -07002156 struct blk_plug plug;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002157 int ret;
Jaegeuk Kim853137c2018-08-09 17:53:34 -07002158 bool locked = false;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002159
P J Pcfb185a2013-04-03 11:38:00 +09002160 /* deal with chardevs and other special file */
2161 if (!mapping->a_ops->writepage)
2162 return 0;
2163
Chao Yu6a290542015-07-17 18:02:39 +08002164 /* skip writing if there is no dirty page in this inode */
2165 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
2166 return 0;
2167
Chao Yu0771fcc2017-06-29 23:20:45 +08002168 /* during POR, we don't need to trigger writepage at all. */
2169 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2170 goto skip_write;
2171
Jaegeuk Kima1257022015-10-08 10:40:07 -07002172 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
2173 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
Chao Yu4d57b862018-05-30 00:20:41 +08002174 f2fs_available_free_memory(sbi, DIRTY_DENTS))
Jaegeuk Kima1257022015-10-08 10:40:07 -07002175 goto skip_write;
2176
Chao Yud323d002015-10-27 09:53:45 +08002177 /* skip writing during file defragment */
Jaegeuk Kim91942322016-05-20 10:13:22 -07002178 if (is_inode_flag_set(inode, FI_DO_DEFRAG))
Chao Yud323d002015-10-27 09:53:45 +08002179 goto skip_write;
2180
Yunlei Hed31c7c32016-02-04 16:14:00 +08002181 trace_f2fs_writepages(mapping->host, wbc, DATA);
2182
Jaegeuk Kim687de7f2017-03-28 18:07:38 -07002183 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
2184 if (wbc->sync_mode == WB_SYNC_ALL)
Chao Yuc29fd0c2018-06-04 23:20:36 +08002185 atomic_inc(&sbi->wb_sync_req[DATA]);
2186 else if (atomic_read(&sbi->wb_sync_req[DATA]))
Jaegeuk Kim687de7f2017-03-28 18:07:38 -07002187 goto skip_write;
2188
Jaegeuk Kim853137c2018-08-09 17:53:34 -07002189 if (__should_serialize_io(inode, wbc)) {
2190 mutex_lock(&sbi->writepages);
2191 locked = true;
2192 }
2193
Jaegeuk Kim9dfa1ba2016-07-13 19:33:19 -07002194 blk_start_plug(&plug);
Chao Yub0af6d42017-08-02 23:21:48 +08002195 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
Jaegeuk Kim9dfa1ba2016-07-13 19:33:19 -07002196 blk_finish_plug(&plug);
Jaegeuk Kim687de7f2017-03-28 18:07:38 -07002197
Jaegeuk Kim853137c2018-08-09 17:53:34 -07002198 if (locked)
2199 mutex_unlock(&sbi->writepages);
2200
Jaegeuk Kim687de7f2017-03-28 18:07:38 -07002201 if (wbc->sync_mode == WB_SYNC_ALL)
Chao Yuc29fd0c2018-06-04 23:20:36 +08002202 atomic_dec(&sbi->wb_sync_req[DATA]);
Jaegeuk Kim28ea6162016-05-25 17:17:56 -07002203 /*
2204 * if some pages were truncated, we cannot guarantee its mapping->host
2205 * to detect pending bios.
2206 */
Jaegeuk Kim458e6192013-12-11 13:54:01 +09002207
Chao Yu4d57b862018-05-30 00:20:41 +08002208 f2fs_remove_dirty_inode(inode);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002209 return ret;
Jaegeuk Kimd3baf952014-03-18 13:43:05 +09002210
2211skip_write:
Jaegeuk Kima7ffdbe2014-09-12 15:53:45 -07002212 wbc->pages_skipped += get_dirty_pages(inode);
Yunlei Hed31c7c32016-02-04 16:14:00 +08002213 trace_f2fs_writepages(mapping->host, wbc, DATA);
Jaegeuk Kimd3baf952014-03-18 13:43:05 +09002214 return 0;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002215}
2216
Chao Yub0af6d42017-08-02 23:21:48 +08002217static int f2fs_write_data_pages(struct address_space *mapping,
2218 struct writeback_control *wbc)
2219{
2220 struct inode *inode = mapping->host;
2221
2222 return __f2fs_write_data_pages(mapping, wbc,
2223 F2FS_I(inode)->cp_task == current ?
2224 FS_CP_DATA_IO : FS_DATA_IO);
2225}
2226
Chao Yu3aab8f82014-07-02 13:25:04 +08002227static void f2fs_write_failed(struct address_space *mapping, loff_t to)
2228{
2229 struct inode *inode = mapping->host;
Jaegeuk Kim819d9152015-12-28 13:48:11 -08002230 loff_t i_size = i_size_read(inode);
Chao Yu3aab8f82014-07-02 13:25:04 +08002231
Jaegeuk Kim819d9152015-12-28 13:48:11 -08002232 if (to > i_size) {
Chao Yua33c1502018-08-05 23:04:25 +08002233 down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
Jaegeuk Kim6f8d4452018-07-25 12:11:56 +09002234 down_write(&F2FS_I(inode)->i_mmap_sem);
Chao Yua33c1502018-08-05 23:04:25 +08002235
Jaegeuk Kim819d9152015-12-28 13:48:11 -08002236 truncate_pagecache(inode, i_size);
Chao Yu4d57b862018-05-30 00:20:41 +08002237 f2fs_truncate_blocks(inode, i_size, true);
Chao Yua33c1502018-08-05 23:04:25 +08002238
Qiuyang Sun5a3a2d82017-05-18 11:06:45 +08002239 up_write(&F2FS_I(inode)->i_mmap_sem);
Jaegeuk Kim6f8d4452018-07-25 12:11:56 +09002240 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
Chao Yu3aab8f82014-07-02 13:25:04 +08002241 }
2242}
2243
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002244static int prepare_write_begin(struct f2fs_sb_info *sbi,
2245 struct page *page, loff_t pos, unsigned len,
2246 block_t *blk_addr, bool *node_changed)
2247{
2248 struct inode *inode = page->mapping->host;
2249 pgoff_t index = page->index;
2250 struct dnode_of_data dn;
2251 struct page *ipage;
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002252 bool locked = false;
Hou Pengyange15882b2017-02-23 09:18:05 +00002253 struct extent_info ei = {0,0,0};
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002254 int err = 0;
2255
Jaegeuk Kim24b84912016-02-03 13:49:44 -08002256 /*
2257 * we already allocated all the blocks, so we don't need to get
2258 * the block addresses when there is no need to fill the page.
2259 */
Jaegeuk Kimdc91de72017-01-13 13:12:29 -08002260 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
2261 !is_inode_flag_set(inode, FI_NO_PREALLOC))
Jaegeuk Kim24b84912016-02-03 13:49:44 -08002262 return 0;
2263
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002264 if (f2fs_has_inline_data(inode) ||
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002265 (pos & PAGE_MASK) >= i_size_read(inode)) {
Yunlei He59c90812017-03-13 20:22:18 +08002266 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002267 locked = true;
2268 }
2269restart:
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002270 /* check inline_data */
Chao Yu4d57b862018-05-30 00:20:41 +08002271 ipage = f2fs_get_node_page(sbi, inode->i_ino);
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002272 if (IS_ERR(ipage)) {
2273 err = PTR_ERR(ipage);
2274 goto unlock_out;
2275 }
2276
2277 set_new_dnode(&dn, inode, ipage, ipage, 0);
2278
2279 if (f2fs_has_inline_data(inode)) {
Chao Yuf2470372017-07-19 00:19:05 +08002280 if (pos + len <= MAX_INLINE_DATA(inode)) {
Chao Yu4d57b862018-05-30 00:20:41 +08002281 f2fs_do_read_inline_data(page, ipage);
Jaegeuk Kim91942322016-05-20 10:13:22 -07002282 set_inode_flag(inode, FI_DATA_EXIST);
Chao Yuab470362016-05-11 19:48:44 +08002283 if (inode->i_nlink)
2284 set_inline_node(ipage);
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002285 } else {
2286 err = f2fs_convert_inline_page(&dn, page);
2287 if (err)
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002288 goto out;
2289 if (dn.data_blkaddr == NULL_ADDR)
2290 err = f2fs_get_block(&dn, index);
2291 }
2292 } else if (locked) {
2293 err = f2fs_get_block(&dn, index);
2294 } else {
2295 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
2296 dn.data_blkaddr = ei.blk + index - ei.fofs;
2297 } else {
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002298 /* hole case */
Chao Yu4d57b862018-05-30 00:20:41 +08002299 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
Jaegeuk Kim4da7bf52016-04-06 11:27:03 -07002300 if (err || dn.data_blkaddr == NULL_ADDR) {
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002301 f2fs_put_dnode(&dn);
Yunlei He59c90812017-03-13 20:22:18 +08002302 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
2303 true);
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002304 locked = true;
2305 goto restart;
2306 }
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002307 }
2308 }
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002309
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002310 /* convert_inline_page can make node_changed */
2311 *blk_addr = dn.data_blkaddr;
2312 *node_changed = dn.node_changed;
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002313out:
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002314 f2fs_put_dnode(&dn);
2315unlock_out:
Jaegeuk Kimb4d07a32015-12-23 13:48:58 -08002316 if (locked)
Yunlei He59c90812017-03-13 20:22:18 +08002317 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002318 return err;
2319}
2320
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002321static int f2fs_write_begin(struct file *file, struct address_space *mapping,
2322 loff_t pos, unsigned len, unsigned flags,
2323 struct page **pagep, void **fsdata)
2324{
2325 struct inode *inode = mapping->host;
Jaegeuk Kim40813632014-09-02 15:31:18 -07002326 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kim86531d62015-07-15 13:08:21 -07002327 struct page *page = NULL;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002328 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
Chao Yua2e2e762018-01-15 17:16:46 +08002329 bool need_balance = false, drop_atomic = false;
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002330 block_t blkaddr = NULL_ADDR;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002331 int err = 0;
2332
Chao Yu62aed042014-05-06 16:46:04 +08002333 trace_f2fs_write_begin(inode, pos, len, flags);
2334
Jaegeuk Kim455e3a52018-07-27 18:15:11 +09002335 if ((f2fs_is_atomic_file(inode) &&
2336 !f2fs_available_free_memory(sbi, INMEM_PAGES)) ||
2337 is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST)) {
Jaegeuk Kim57864ae2017-10-18 19:05:57 -07002338 err = -ENOMEM;
Chao Yua2e2e762018-01-15 17:16:46 +08002339 drop_atomic = true;
Jaegeuk Kim57864ae2017-10-18 19:05:57 -07002340 goto fail;
2341 }
2342
Jaegeuk Kim5f727392014-11-25 10:59:45 -08002343 /*
2344 * We should check this at this moment to avoid deadlock on inode page
2345 * and #0 page. The locking rule for inline_data conversion should be:
2346 * lock_page(page #0) -> lock_page(inode_page)
2347 */
2348 if (index != 0) {
2349 err = f2fs_convert_inline_inode(inode);
2350 if (err)
2351 goto fail;
2352 }
Jaegeuk Kimafcb7ca02013-04-26 11:55:17 +09002353repeat:
Jaegeuk Kim86d54792017-02-17 09:55:55 -08002354 /*
2355 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
2356 * wait_for_stable_page. Will wait that below with our IO control.
2357 */
Chao Yu01eccef2017-10-28 16:52:30 +08002358 page = f2fs_pagecache_get_page(mapping, index,
Jaegeuk Kim86d54792017-02-17 09:55:55 -08002359 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
Chao Yu3aab8f82014-07-02 13:25:04 +08002360 if (!page) {
2361 err = -ENOMEM;
2362 goto fail;
2363 }
Jaegeuk Kimd5f66992014-04-30 09:22:45 +09002364
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002365 *pagep = page;
2366
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002367 err = prepare_write_begin(sbi, page, pos, len,
2368 &blkaddr, &need_balance);
Jaegeuk Kim9ba69cf2014-10-17 20:33:55 -07002369 if (err)
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002370 goto fail;
Jaegeuk Kim9ba69cf2014-10-17 20:33:55 -07002371
Jaegeuk Kim7f3037a2016-09-01 12:02:51 -07002372 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
Jaegeuk Kim2a340762015-12-22 13:23:35 -08002373 unlock_page(page);
Jaegeuk Kim2c4db1a2016-01-07 14:15:04 -08002374 f2fs_balance_fs(sbi, true);
Jaegeuk Kim2a340762015-12-22 13:23:35 -08002375 lock_page(page);
2376 if (page->mapping != mapping) {
2377 /* The page got truncated from under us */
2378 f2fs_put_page(page, 1);
2379 goto repeat;
2380 }
2381 }
2382
Jaegeuk Kimfec1d652016-01-20 23:43:51 +08002383 f2fs_wait_on_page_writeback(page, DATA, false);
Jaegeuk Kimb3d208f2014-10-23 19:48:09 -07002384
Eric Biggers6dbb1792018-04-18 11:09:48 -07002385 /* wait for GCed page writeback via META_MAPPING */
2386 if (f2fs_post_read_required(inode))
Jaegeuk Kimd4c759e2017-09-05 17:04:35 -07002387 f2fs_wait_on_block_writeback(sbi, blkaddr);
Chao Yu08b39fb2015-10-08 13:27:34 +08002388
Jaegeuk Kim649d7df2016-09-06 11:02:03 -07002389 if (len == PAGE_SIZE || PageUptodate(page))
2390 return 0;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002391
Yunlei He746e2402016-12-20 11:11:35 +08002392 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2393 zero_user_segment(page, len, PAGE_SIZE);
2394 return 0;
2395 }
2396
Jaegeuk Kim2aadac02015-12-23 11:55:18 -08002397 if (blkaddr == NEW_ADDR) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002398 zero_user_segment(page, 0, PAGE_SIZE);
Jaegeuk Kim649d7df2016-09-06 11:02:03 -07002399 SetPageUptodate(page);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002400 } else {
Jaegeuk Kim13ba41e2017-09-06 21:04:44 -07002401 err = f2fs_submit_page_read(inode, page, blkaddr);
2402 if (err)
Chao Yu3aab8f82014-07-02 13:25:04 +08002403 goto fail;
Chao Yud54c7952014-03-29 15:30:40 +08002404
Jaegeuk Kim393ff912013-03-08 21:29:23 +09002405 lock_page(page);
Jaegeuk Kim6bacf522013-12-06 15:00:58 +09002406 if (unlikely(page->mapping != mapping)) {
Jaegeuk Kimafcb7ca02013-04-26 11:55:17 +09002407 f2fs_put_page(page, 1);
2408 goto repeat;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002409 }
Chao Yu1563ac72016-07-03 22:05:12 +08002410 if (unlikely(!PageUptodate(page))) {
2411 err = -EIO;
2412 goto fail;
Jaegeuk Kim4375a332015-04-23 12:04:33 -07002413 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002414 }
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002415 return 0;
Jaegeuk Kim9ba69cf2014-10-17 20:33:55 -07002416
Chao Yu3aab8f82014-07-02 13:25:04 +08002417fail:
Jaegeuk Kim86531d62015-07-15 13:08:21 -07002418 f2fs_put_page(page, 1);
Chao Yu3aab8f82014-07-02 13:25:04 +08002419 f2fs_write_failed(mapping, pos + len);
Chao Yua2e2e762018-01-15 17:16:46 +08002420 if (drop_atomic)
Chao Yu4d57b862018-05-30 00:20:41 +08002421 f2fs_drop_inmem_pages_all(sbi, false);
Chao Yu3aab8f82014-07-02 13:25:04 +08002422 return err;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002423}
2424
Jaegeuk Kima1dd3c12013-06-27 13:04:08 +09002425static int f2fs_write_end(struct file *file,
2426 struct address_space *mapping,
2427 loff_t pos, unsigned len, unsigned copied,
2428 struct page *page, void *fsdata)
2429{
2430 struct inode *inode = page->mapping->host;
2431
Chao Yudfb2bf32014-05-06 16:47:23 +08002432 trace_f2fs_write_end(inode, pos, len, copied);
2433
Jaegeuk Kim649d7df2016-09-06 11:02:03 -07002434 /*
2435 * This should be come from len == PAGE_SIZE, and we expect copied
2436 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2437 * let generic_perform_write() try to copy data again through copied=0.
2438 */
2439 if (!PageUptodate(page)) {
Yunlei He746e2402016-12-20 11:11:35 +08002440 if (unlikely(copied != len))
Jaegeuk Kim649d7df2016-09-06 11:02:03 -07002441 copied = 0;
2442 else
2443 SetPageUptodate(page);
2444 }
2445 if (!copied)
2446 goto unlock_out;
2447
Jaegeuk Kim34ba94b2014-10-09 13:19:53 -07002448 set_page_dirty(page);
Jaegeuk Kima1dd3c12013-06-27 13:04:08 +09002449
Jaegeuk Kimfc9581c2016-05-20 09:22:03 -07002450 if (pos + copied > i_size_read(inode))
2451 f2fs_i_size_write(inode, pos + copied);
Jaegeuk Kim649d7df2016-09-06 11:02:03 -07002452unlock_out:
Chao Yu3024c9a2016-08-06 21:09:41 +08002453 f2fs_put_page(page, 1);
Jaegeuk Kimd0239e12016-01-08 16:57:48 -08002454 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
Jaegeuk Kima1dd3c12013-06-27 13:04:08 +09002455 return copied;
2456}
2457
Omar Sandoval6f673762015-03-16 04:33:52 -07002458static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2459 loff_t offset)
Jaegeuk Kim944fcfc2013-12-26 20:15:09 +09002460{
Jaegeuk Kim8a56dd92018-06-29 18:55:12 -07002461 unsigned i_blkbits = READ_ONCE(inode->i_blkbits);
2462 unsigned blkbits = i_blkbits;
2463 unsigned blocksize_mask = (1 << blkbits) - 1;
2464 unsigned long align = offset | iov_iter_alignment(iter);
2465 struct block_device *bdev = inode->i_sb->s_bdev;
Jaegeuk Kim944fcfc2013-12-26 20:15:09 +09002466
Jaegeuk Kim8a56dd92018-06-29 18:55:12 -07002467 if (align & blocksize_mask) {
2468 if (bdev)
2469 blkbits = blksize_bits(bdev_logical_block_size(bdev));
2470 blocksize_mask = (1 << blkbits) - 1;
2471 if (align & blocksize_mask)
2472 return -EINVAL;
2473 return 1;
2474 }
Jaegeuk Kim944fcfc2013-12-26 20:15:09 +09002475 return 0;
2476}
2477
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07002478static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002479{
Jaegeuk Kimb439b102016-02-03 13:09:09 -08002480 struct address_space *mapping = iocb->ki_filp->f_mapping;
Chao Yu3aab8f82014-07-02 13:25:04 +08002481 struct inode *inode = mapping->host;
Hyunchul Lee0cdd3192018-01-31 11:36:57 +09002482 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Chao Yu3aab8f82014-07-02 13:25:04 +08002483 size_t count = iov_iter_count(iter);
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07002484 loff_t offset = iocb->ki_pos;
Chao Yu82e0a5a2016-07-13 09:18:29 +08002485 int rw = iov_iter_rw(iter);
Chao Yu3aab8f82014-07-02 13:25:04 +08002486 int err;
Hyunchul Lee0cdd3192018-01-31 11:36:57 +09002487 enum rw_hint hint = iocb->ki_hint;
Chao Yu63189b72018-03-08 14:22:56 +08002488 int whint_mode = F2FS_OPTION(sbi).whint_mode;
Jaegeuk Kim944fcfc2013-12-26 20:15:09 +09002489
Jaegeuk Kimb439b102016-02-03 13:09:09 -08002490 err = check_direct_IO(inode, iter, offset);
Jaegeuk Kimb9d777b2015-12-22 11:09:35 -08002491 if (err)
Jaegeuk Kim8a56dd92018-06-29 18:55:12 -07002492 return err < 0 ? err : 0;
Huajun Li9ffe0fb2013-11-10 23:13:20 +08002493
Hyunchul Leeb91050a2018-03-08 19:34:38 +09002494 if (f2fs_force_buffered_io(inode, rw))
Jaegeuk Kim36abef42016-06-03 19:29:38 -07002495 return 0;
Jaegeuk Kimfcc85a42015-04-21 20:39:58 -07002496
Jaegeuk Kim5302fb02016-07-22 15:25:47 -07002497 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
Chao Yu70407fa2014-07-31 21:11:22 +08002498
Hyunchul Lee0cdd3192018-01-31 11:36:57 +09002499 if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
2500 iocb->ki_hint = WRITE_LIFE_NOT_SET;
2501
Chao Yub2532c62018-04-24 10:55:28 +08002502 if (!down_read_trylock(&F2FS_I(inode)->i_gc_rwsem[rw])) {
Hyunchul Leeb91050a2018-03-08 19:34:38 +09002503 if (iocb->ki_flags & IOCB_NOWAIT) {
2504 iocb->ki_hint = hint;
2505 err = -EAGAIN;
2506 goto out;
2507 }
Chao Yub2532c62018-04-24 10:55:28 +08002508 down_read(&F2FS_I(inode)->i_gc_rwsem[rw]);
Hyunchul Leeb91050a2018-03-08 19:34:38 +09002509 }
2510
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07002511 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
Chao Yub2532c62018-04-24 10:55:28 +08002512 up_read(&F2FS_I(inode)->i_gc_rwsem[rw]);
Chao Yu82e0a5a2016-07-13 09:18:29 +08002513
2514 if (rw == WRITE) {
Hyunchul Lee0cdd3192018-01-31 11:36:57 +09002515 if (whint_mode == WHINT_MODE_OFF)
2516 iocb->ki_hint = hint;
Chao Yub0af6d42017-08-02 23:21:48 +08002517 if (err > 0) {
2518 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2519 err);
Jaegeuk Kim91942322016-05-20 10:13:22 -07002520 set_inode_flag(inode, FI_UPDATE_WRITE);
Chao Yub0af6d42017-08-02 23:21:48 +08002521 } else if (err < 0) {
Jaegeuk Kim6bfc4912016-04-18 17:07:44 -04002522 f2fs_write_failed(mapping, offset + count);
Chao Yub0af6d42017-08-02 23:21:48 +08002523 }
Jaegeuk Kim6bfc4912016-04-18 17:07:44 -04002524 }
Chao Yu70407fa2014-07-31 21:11:22 +08002525
Hyunchul Leeb91050a2018-03-08 19:34:38 +09002526out:
Jaegeuk Kim5302fb02016-07-22 15:25:47 -07002527 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
Chao Yu70407fa2014-07-31 21:11:22 +08002528
Chao Yu3aab8f82014-07-02 13:25:04 +08002529 return err;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002530}
2531
Chao Yu487261f2015-02-05 17:44:29 +08002532void f2fs_invalidate_page(struct page *page, unsigned int offset,
2533 unsigned int length)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002534{
2535 struct inode *inode = page->mapping->host;
Chao Yu487261f2015-02-05 17:44:29 +08002536 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
Jaegeuk Kima7ffdbe2014-09-12 15:53:45 -07002537
Chao Yu487261f2015-02-05 17:44:29 +08002538 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002539 (offset % PAGE_SIZE || length != PAGE_SIZE))
Jaegeuk Kima7ffdbe2014-09-12 15:53:45 -07002540 return;
2541
Chao Yu487261f2015-02-05 17:44:29 +08002542 if (PageDirty(page)) {
Chao Yu933439c2016-10-11 22:57:01 +08002543 if (inode->i_ino == F2FS_META_INO(sbi)) {
Chao Yu487261f2015-02-05 17:44:29 +08002544 dec_page_count(sbi, F2FS_DIRTY_META);
Chao Yu933439c2016-10-11 22:57:01 +08002545 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
Chao Yu487261f2015-02-05 17:44:29 +08002546 dec_page_count(sbi, F2FS_DIRTY_NODES);
Chao Yu933439c2016-10-11 22:57:01 +08002547 } else {
Chao Yu487261f2015-02-05 17:44:29 +08002548 inode_dec_dirty_pages(inode);
Chao Yu4d57b862018-05-30 00:20:41 +08002549 f2fs_remove_dirty_inode(inode);
Chao Yu933439c2016-10-11 22:57:01 +08002550 }
Chao Yu487261f2015-02-05 17:44:29 +08002551 }
Chao Yudecd36b2015-08-07 18:42:09 +08002552
2553 /* This is atomic written page, keep Private */
2554 if (IS_ATOMIC_WRITTEN_PAGE(page))
Chao Yu4d57b862018-05-30 00:20:41 +08002555 return f2fs_drop_inmem_page(inode, page);
Chao Yudecd36b2015-08-07 18:42:09 +08002556
Chao Yu23dc9742016-04-29 20:09:15 +08002557 set_page_private(page, 0);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002558 ClearPagePrivate(page);
2559}
2560
Chao Yu487261f2015-02-05 17:44:29 +08002561int f2fs_release_page(struct page *page, gfp_t wait)
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002562{
Jaegeuk Kimf68daee2015-01-30 11:39:08 -08002563 /* If this is dirty page, keep PagePrivate */
2564 if (PageDirty(page))
2565 return 0;
2566
Chao Yudecd36b2015-08-07 18:42:09 +08002567 /* This is atomic written page, keep Private */
2568 if (IS_ATOMIC_WRITTEN_PAGE(page))
2569 return 0;
2570
Chao Yu23dc9742016-04-29 20:09:15 +08002571 set_page_private(page, 0);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002572 ClearPagePrivate(page);
Jaegeuk Kimc3850aa2013-03-14 09:24:32 +09002573 return 1;
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002574}
2575
2576static int f2fs_set_data_page_dirty(struct page *page)
2577{
2578 struct address_space *mapping = page->mapping;
2579 struct inode *inode = mapping->host;
2580
Jaegeuk Kim26c6b882013-10-24 17:53:29 +09002581 trace_f2fs_set_page_dirty(page, DATA);
2582
Jaegeuk Kim237c0792016-06-30 18:49:15 -07002583 if (!PageUptodate(page))
2584 SetPageUptodate(page);
Jaegeuk Kim34ba94b2014-10-09 13:19:53 -07002585
Chao Yu66110ab2018-07-29 12:16:59 +08002586 /* don't remain PG_checked flag which was set during GC */
2587 if (is_cold_data(page))
2588 clear_cold_data(page);
2589
Chao Yu5fe45742017-01-07 18:50:26 +08002590 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
Chao Yudecd36b2015-08-07 18:42:09 +08002591 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
Chao Yu4d57b862018-05-30 00:20:41 +08002592 f2fs_register_inmem_page(inode, page);
Chao Yudecd36b2015-08-07 18:42:09 +08002593 return 1;
2594 }
2595 /*
2596 * Previously, this page has been registered, we just
2597 * return here.
2598 */
2599 return 0;
Jaegeuk Kim34ba94b2014-10-09 13:19:53 -07002600 }
2601
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002602 if (!PageDirty(page)) {
Jaegeuk Kimb87078a2018-04-20 19:29:52 -07002603 __set_page_dirty_nobuffers(page);
Chao Yu4d57b862018-05-30 00:20:41 +08002604 f2fs_update_dirty_page(inode, page);
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002605 return 1;
2606 }
2607 return 0;
2608}
2609
Jaegeuk Kimc01e54b2013-01-17 20:30:23 +09002610static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2611{
Chao Yu454ae7e2014-04-22 13:34:01 +08002612 struct inode *inode = mapping->host;
2613
Jaegeuk Kim1d373a02015-10-19 10:29:51 -07002614 if (f2fs_has_inline_data(inode))
2615 return 0;
2616
2617 /* make sure allocating whole blocks */
2618 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2619 filemap_write_and_wait(mapping);
2620
Chao Yue2b4e2b2015-08-19 19:11:19 +08002621 return generic_block_bmap(mapping, block, get_data_block_bmap);
Chao Yu429511c2015-02-05 17:54:31 +08002622}
2623
Weichao Guo5b7a4872016-09-20 05:03:27 +08002624#ifdef CONFIG_MIGRATION
2625#include <linux/migrate.h>
2626
2627int f2fs_migrate_page(struct address_space *mapping,
2628 struct page *newpage, struct page *page, enum migrate_mode mode)
2629{
2630 int rc, extra_count;
2631 struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2632 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2633
2634 BUG_ON(PageWriteback(page));
2635
2636 /* migrating an atomic written page is safe with the inmem_lock hold */
Jaegeuk Kimff1048e2017-07-06 14:46:01 -07002637 if (atomic_written) {
2638 if (mode != MIGRATE_SYNC)
2639 return -EBUSY;
2640 if (!mutex_trylock(&fi->inmem_lock))
2641 return -EAGAIN;
2642 }
Weichao Guo5b7a4872016-09-20 05:03:27 +08002643
2644 /*
2645 * A reference is expected if PagePrivate set when move mapping,
2646 * however F2FS breaks this for maintaining dirty page counts when
2647 * truncating pages. So here adjusting the 'extra_count' make it work.
2648 */
2649 extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2650 rc = migrate_page_move_mapping(mapping, newpage,
2651 page, NULL, mode, extra_count);
2652 if (rc != MIGRATEPAGE_SUCCESS) {
2653 if (atomic_written)
2654 mutex_unlock(&fi->inmem_lock);
2655 return rc;
2656 }
2657
2658 if (atomic_written) {
2659 struct inmem_pages *cur;
2660 list_for_each_entry(cur, &fi->inmem_pages, list)
2661 if (cur->page == page) {
2662 cur->page = newpage;
2663 break;
2664 }
2665 mutex_unlock(&fi->inmem_lock);
2666 put_page(page);
2667 get_page(newpage);
2668 }
2669
2670 if (PagePrivate(page))
2671 SetPagePrivate(newpage);
2672 set_page_private(newpage, page_private(page));
2673
Jérôme Glisse2916ecc2017-09-08 16:12:06 -07002674 if (mode != MIGRATE_SYNC_NO_COPY)
2675 migrate_page_copy(newpage, page);
2676 else
2677 migrate_page_states(newpage, page);
Weichao Guo5b7a4872016-09-20 05:03:27 +08002678
2679 return MIGRATEPAGE_SUCCESS;
2680}
2681#endif
2682
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002683const struct address_space_operations f2fs_dblock_aops = {
2684 .readpage = f2fs_read_data_page,
2685 .readpages = f2fs_read_data_pages,
2686 .writepage = f2fs_write_data_page,
2687 .writepages = f2fs_write_data_pages,
2688 .write_begin = f2fs_write_begin,
Jaegeuk Kima1dd3c12013-06-27 13:04:08 +09002689 .write_end = f2fs_write_end,
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002690 .set_page_dirty = f2fs_set_data_page_dirty,
Chao Yu487261f2015-02-05 17:44:29 +08002691 .invalidatepage = f2fs_invalidate_page,
2692 .releasepage = f2fs_release_page,
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002693 .direct_IO = f2fs_direct_IO,
Jaegeuk Kimc01e54b2013-01-17 20:30:23 +09002694 .bmap = f2fs_bmap,
Weichao Guo5b7a4872016-09-20 05:03:27 +08002695#ifdef CONFIG_MIGRATION
2696 .migratepage = f2fs_migrate_page,
2697#endif
Jaegeuk Kimeb47b802012-11-02 17:10:12 +09002698};
Eric Biggers6dbb1792018-04-18 11:09:48 -07002699
Chao Yu4d57b862018-05-30 00:20:41 +08002700void f2fs_clear_radix_tree_dirty_tag(struct page *page)
Chao Yuaec2f722018-05-26 18:03:35 +08002701{
2702 struct address_space *mapping = page_mapping(page);
2703 unsigned long flags;
2704
2705 xa_lock_irqsave(&mapping->i_pages, flags);
2706 radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2707 PAGECACHE_TAG_DIRTY);
2708 xa_unlock_irqrestore(&mapping->i_pages, flags);
2709}
2710
Eric Biggers6dbb1792018-04-18 11:09:48 -07002711int __init f2fs_init_post_read_processing(void)
2712{
2713 bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, 0);
2714 if (!bio_post_read_ctx_cache)
2715 goto fail;
2716 bio_post_read_ctx_pool =
2717 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
2718 bio_post_read_ctx_cache);
2719 if (!bio_post_read_ctx_pool)
2720 goto fail_free_cache;
2721 return 0;
2722
2723fail_free_cache:
2724 kmem_cache_destroy(bio_post_read_ctx_cache);
2725fail:
2726 return -ENOMEM;
2727}
2728
2729void __exit f2fs_destroy_post_read_processing(void)
2730{
2731 mempool_destroy(bio_post_read_ctx_pool);
2732 kmem_cache_destroy(bio_post_read_ctx_cache);
2733}