blob: d2f53ddd826065bb151b9ca8204cd8ba0d024fa0 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/nfs/read.c
3 *
4 * Block I/O for NFS
5 *
6 * Partial copy of Linus' read cache modifications to fs/nfs/file.c
7 * modified for async RPC by okir@monad.swb.de
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 */
9
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/time.h>
11#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/fcntl.h>
14#include <linux/stat.h>
15#include <linux/mm.h>
16#include <linux/slab.h>
17#include <linux/pagemap.h>
18#include <linux/sunrpc/clnt.h>
19#include <linux/nfs_fs.h>
20#include <linux/nfs_page.h>
Andy Adamson64419a92011-03-01 01:34:16 +000021#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23#include <asm/system.h>
Fred Isamanbae724e2011-03-01 01:34:15 +000024#include "pnfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Andy Adamsonf11c88a2009-04-01 09:22:25 -040026#include "nfs4_fs.h"
Trond Myklebust49a70f2782006-12-05 00:35:38 -050027#include "internal.h"
Chuck Lever91d5b472006-03-20 13:44:14 -050028#include "iostat.h"
David Howells9a9fc1c2009-04-03 16:42:44 +010029#include "fscache.h"
Chuck Lever91d5b472006-03-20 13:44:14 -050030
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#define NFSDBG_FACILITY NFSDBG_PAGECACHE
32
Trond Myklebust1751c362011-06-10 13:30:23 -040033static const struct nfs_pageio_ops nfs_pageio_read_ops;
Trond Myklebustec06c092006-03-20 13:44:27 -050034static const struct rpc_call_ops nfs_read_partial_ops;
35static const struct rpc_call_ops nfs_read_full_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
Christoph Lametere18b8902006-12-06 20:33:20 -080037static struct kmem_cache *nfs_rdata_cachep;
Trond Myklebust3feb2d42006-03-20 13:44:37 -050038static mempool_t *nfs_rdata_mempool;
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
40#define MIN_POOL_READ (32)
41
Trond Myklebust8d5658c2007-04-10 09:26:35 -040042struct nfs_read_data *nfs_readdata_alloc(unsigned int pagecount)
Trond Myklebust3feb2d42006-03-20 13:44:37 -050043{
Trond Myklebust93870d72010-05-13 12:51:03 -040044 struct nfs_read_data *p = mempool_alloc(nfs_rdata_mempool, GFP_KERNEL);
Trond Myklebust3feb2d42006-03-20 13:44:37 -050045
46 if (p) {
47 memset(p, 0, sizeof(*p));
48 INIT_LIST_HEAD(&p->pages);
Trond Myklebuste9f7bee2006-09-08 09:48:54 -070049 p->npages = pagecount;
Chuck Lever0d0b5cb2006-05-25 01:40:53 -040050 if (pagecount <= ARRAY_SIZE(p->page_array))
51 p->pagevec = p->page_array;
Trond Myklebust3feb2d42006-03-20 13:44:37 -050052 else {
Trond Myklebust93870d72010-05-13 12:51:03 -040053 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
Chuck Lever0d0b5cb2006-05-25 01:40:53 -040054 if (!p->pagevec) {
Trond Myklebust3feb2d42006-03-20 13:44:37 -050055 mempool_free(p, nfs_rdata_mempool);
56 p = NULL;
57 }
58 }
59 }
60 return p;
61}
62
Trond Myklebust1ae88b22009-08-12 09:12:30 -040063void nfs_readdata_free(struct nfs_read_data *p)
Trond Myklebust3feb2d42006-03-20 13:44:37 -050064{
65 if (p && (p->pagevec != &p->page_array[0]))
66 kfree(p->pagevec);
67 mempool_free(p, nfs_rdata_mempool);
68}
69
Trond Myklebust493292d2011-07-13 15:58:28 -040070void nfs_readdata_release(struct nfs_read_data *rdata)
Linus Torvalds1da177e2005-04-16 15:20:36 -070071{
Fred Isamanbae724e2011-03-01 01:34:15 +000072 put_lseg(rdata->lseg);
Trond Myklebust383ba7192008-02-19 20:04:20 -050073 put_nfs_open_context(rdata->args.context);
74 nfs_readdata_free(rdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -070075}
76
77static
Linus Torvalds1da177e2005-04-16 15:20:36 -070078int nfs_return_empty_page(struct page *page)
79{
Christoph Lametereebd2aa2008-02-04 22:28:29 -080080 zero_user(page, 0, PAGE_CACHE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 SetPageUptodate(page);
82 unlock_page(page);
83 return 0;
84}
85
Trond Myklebust1de3fc12006-05-25 01:40:44 -040086static void nfs_readpage_truncate_uninitialised_page(struct nfs_read_data *data)
87{
88 unsigned int remainder = data->args.count - data->res.count;
89 unsigned int base = data->args.pgbase + data->res.count;
90 unsigned int pglen;
91 struct page **pages;
92
93 if (data->res.eof == 0 || remainder == 0)
94 return;
95 /*
96 * Note: "remainder" can never be negative, since we check for
97 * this in the XDR code.
98 */
99 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT];
100 base &= ~PAGE_CACHE_MASK;
101 pglen = PAGE_CACHE_SIZE - base;
Trond Myklebust79558f32006-08-22 13:44:32 -0400102 for (;;) {
103 if (remainder <= pglen) {
Christoph Lametereebd2aa2008-02-04 22:28:29 -0800104 zero_user(*pages, base, remainder);
Trond Myklebust79558f32006-08-22 13:44:32 -0400105 break;
106 }
Christoph Lametereebd2aa2008-02-04 22:28:29 -0800107 zero_user(*pages, base, pglen);
Trond Myklebust79558f32006-08-22 13:44:32 -0400108 pages++;
109 remainder -= pglen;
110 pglen = PAGE_CACHE_SIZE;
111 base = 0;
112 }
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400113}
114
Trond Myklebuste885de12011-06-10 13:30:23 -0400115void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
Trond Myklebust1751c362011-06-10 13:30:23 -0400116 struct inode *inode)
117{
118 nfs_pageio_init(pgio, inode, &nfs_pageio_read_ops,
119 NFS_SERVER(inode)->rsize, 0);
120}
Andy Adamson7c24d942011-06-13 18:22:38 -0400121EXPORT_SYMBOL_GPL(nfs_pageio_init_read_mds);
Trond Myklebust1751c362011-06-10 13:30:23 -0400122
Trond Myklebust493292d2011-07-13 15:58:28 -0400123void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
124{
125 pgio->pg_ops = &nfs_pageio_read_ops;
126 pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize;
127}
128
Trond Myklebust1751c362011-06-10 13:30:23 -0400129static void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio,
130 struct inode *inode)
131{
132 if (!pnfs_pageio_init_read(pgio, inode))
133 nfs_pageio_init_read_mds(pgio, inode);
134}
135
David Howellsf42b2932009-04-03 16:42:44 +0100136int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode,
137 struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 struct nfs_page *new;
140 unsigned int len;
Fred Isamanc76069b2011-03-03 15:13:48 +0000141 struct nfs_pageio_descriptor pgio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
Trond Myklebust49a70f2782006-12-05 00:35:38 -0500143 len = nfs_page_length(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 if (len == 0)
145 return nfs_return_empty_page(page);
146 new = nfs_create_request(ctx, inode, page, 0, len);
147 if (IS_ERR(new)) {
148 unlock_page(page);
149 return PTR_ERR(new);
150 }
151 if (len < PAGE_CACHE_SIZE)
Christoph Lametereebd2aa2008-02-04 22:28:29 -0800152 zero_user_segment(page, len, PAGE_CACHE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153
Trond Myklebust1751c362011-06-10 13:30:23 -0400154 nfs_pageio_init_read(&pgio, inode);
Trond Myklebustd8007d42011-06-10 13:30:23 -0400155 nfs_pageio_add_request(&pgio, new);
Trond Myklebust1751c362011-06-10 13:30:23 -0400156 nfs_pageio_complete(&pgio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 return 0;
158}
159
160static void nfs_readpage_release(struct nfs_page *req)
161{
David Howells7f8e05f2009-04-03 16:42:45 +0100162 struct inode *d_inode = req->wb_context->path.dentry->d_inode;
163
164 if (PageUptodate(req->wb_page))
165 nfs_readpage_to_fscache(d_inode, req->wb_page, 0);
166
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 unlock_page(req->wb_page);
168
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 dprintk("NFS: read done (%s/%Ld %d@%Ld)\n",
Trond Myklebust88be9f92007-06-05 10:42:27 -0400170 req->wb_context->path.dentry->d_inode->i_sb->s_id,
171 (long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 req->wb_bytes,
173 (long long)req_offset(req));
Nick Wilson10d2c462005-09-22 21:44:28 -0700174 nfs_release_request(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175}
176
Andy Adamsondc70d7b2011-03-01 01:34:19 +0000177int nfs_initiate_read(struct nfs_read_data *data, struct rpc_clnt *clnt,
Andy Adamson64419a92011-03-01 01:34:16 +0000178 const struct rpc_call_ops *call_ops)
179{
180 struct inode *inode = data->inode;
181 int swap_flags = IS_SWAPFILE(inode) ? NFS_RPC_SWAPFLAGS : 0;
182 struct rpc_task *task;
183 struct rpc_message msg = {
184 .rpc_argp = &data->args,
185 .rpc_resp = &data->res,
186 .rpc_cred = data->cred,
187 };
188 struct rpc_task_setup task_setup_data = {
189 .task = &data->task,
190 .rpc_client = clnt,
191 .rpc_message = &msg,
192 .callback_ops = call_ops,
193 .callback_data = data,
194 .workqueue = nfsiod_workqueue,
195 .flags = RPC_TASK_ASYNC | swap_flags,
196 };
197
198 /* Set up the initial task struct. */
199 NFS_PROTO(inode)->read_setup(data, &msg);
200
201 dprintk("NFS: %5u initiated read call (req %s/%lld, %u bytes @ "
202 "offset %llu)\n",
203 data->task.tk_pid,
204 inode->i_sb->s_id,
205 (long long)NFS_FILEID(inode),
206 data->args.count,
207 (unsigned long long)data->args.offset);
208
209 task = rpc_run_task(&task_setup_data);
210 if (IS_ERR(task))
211 return PTR_ERR(task);
212 rpc_put_task(task);
213 return 0;
214}
Andy Adamsondc70d7b2011-03-01 01:34:19 +0000215EXPORT_SYMBOL_GPL(nfs_initiate_read);
Andy Adamson64419a92011-03-01 01:34:16 +0000216
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217/*
218 * Set up the NFS read request struct
219 */
Trond Myklebust6e4efd52011-07-12 13:42:02 -0400220static void nfs_read_rpcsetup(struct nfs_page *req, struct nfs_read_data *data,
221 unsigned int count, unsigned int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222{
Trond Myklebust84115e12007-07-14 15:39:59 -0400223 struct inode *inode = req->wb_context->path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224
225 data->req = req;
Trond Myklebust84115e12007-07-14 15:39:59 -0400226 data->inode = inode;
Andy Adamson64419a92011-03-01 01:34:16 +0000227 data->cred = req->wb_context->cred;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228
229 data->args.fh = NFS_FH(inode);
230 data->args.offset = req_offset(req) + offset;
231 data->args.pgbase = req->wb_pgbase + offset;
232 data->args.pages = data->pagevec;
233 data->args.count = count;
Trond Myklebust383ba7192008-02-19 20:04:20 -0500234 data->args.context = get_nfs_open_context(req->wb_context);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400235 data->args.lock_context = req->wb_lock_context;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236
237 data->res.fattr = &data->fattr;
238 data->res.count = count;
239 data->res.eof = 0;
Trond Myklebust0e574af2005-10-27 22:12:38 -0400240 nfs_fattr_init(&data->fattr);
Trond Myklebust6e4efd52011-07-12 13:42:02 -0400241}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
Trond Myklebust6e4efd52011-07-12 13:42:02 -0400243static int nfs_do_read(struct nfs_read_data *data,
Trond Myklebust493292d2011-07-13 15:58:28 -0400244 const struct rpc_call_ops *call_ops)
Trond Myklebust6e4efd52011-07-12 13:42:02 -0400245{
246 struct inode *inode = data->args.context->path.dentry->d_inode;
247
Andy Adamson64419a92011-03-01 01:34:16 +0000248 return nfs_initiate_read(data, NFS_CLIENT(inode), call_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249}
250
Trond Myklebust275acaa2011-07-12 13:42:02 -0400251static int
252nfs_do_multiple_reads(struct list_head *head,
Trond Myklebust493292d2011-07-13 15:58:28 -0400253 const struct rpc_call_ops *call_ops)
Trond Myklebust275acaa2011-07-12 13:42:02 -0400254{
255 struct nfs_read_data *data;
256 int ret = 0;
257
258 while (!list_empty(head)) {
259 int ret2;
260
261 data = list_entry(head->next, struct nfs_read_data, list);
262 list_del_init(&data->list);
263
Trond Myklebust493292d2011-07-13 15:58:28 -0400264 ret2 = nfs_do_read(data, call_ops);
Trond Myklebust275acaa2011-07-12 13:42:02 -0400265 if (ret == 0)
266 ret = ret2;
267 }
268 return ret;
269}
270
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271static void
272nfs_async_read_error(struct list_head *head)
273{
274 struct nfs_page *req;
275
276 while (!list_empty(head)) {
277 req = nfs_list_entry(head->next);
278 nfs_list_remove_request(req);
279 SetPageError(req->wb_page);
280 nfs_readpage_release(req);
281 }
282}
283
284/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 * Generate multiple requests to fill a single page.
286 *
287 * We optimize to reduce the number of read operations on the wire. If we
288 * detect that we're reading a page, or an area of a page, that is past the
289 * end of file, we do not generate NFS read operations but just clear the
290 * parts of the page that would have come back zero from the server anyway.
291 *
292 * We rely on the cached value of i_size to make this determination; another
293 * client can fill pages on the server past our cached end-of-file, but we
294 * won't see the new data until our attribute cache is updated. This is more
295 * or less conventional NFS client behavior.
296 */
Trond Myklebust275acaa2011-07-12 13:42:02 -0400297static int nfs_pagein_multi(struct nfs_pageio_descriptor *desc, struct list_head *res)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298{
Fred Isamanc76069b2011-03-03 15:13:48 +0000299 struct nfs_page *req = nfs_list_entry(desc->pg_list.next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 struct page *page = req->wb_page;
301 struct nfs_read_data *data;
Trond Myklebustd0979712011-07-12 13:42:02 -0400302 size_t rsize = desc->pg_bsize, nbytes;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700303 unsigned int offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 int requests = 0;
Trond Myklebustdbae4c72008-04-14 14:54:53 -0400305 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
307 nfs_list_remove_request(req);
308
Trond Myklebust275acaa2011-07-12 13:42:02 -0400309 offset = 0;
Fred Isamanc76069b2011-03-03 15:13:48 +0000310 nbytes = desc->pg_count;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700311 do {
312 size_t len = min(nbytes,rsize);
313
Trond Myklebust8d5658c2007-04-10 09:26:35 -0400314 data = nfs_readdata_alloc(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 if (!data)
316 goto out_bad;
Trond Myklebust275acaa2011-07-12 13:42:02 -0400317 data->pagevec[0] = page;
318 nfs_read_rpcsetup(req, data, len, offset);
319 list_add(&data->list, res);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320 requests++;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700321 nbytes -= len;
Trond Myklebust275acaa2011-07-12 13:42:02 -0400322 offset += len;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700323 } while(nbytes != 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 atomic_set(&req->wb_complete, requests);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 ClearPageError(page);
Trond Myklebust50828d72011-07-12 13:42:02 -0400326 desc->pg_rpc_callops = &nfs_read_partial_ops;
Trond Myklebustdbae4c72008-04-14 14:54:53 -0400327 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328out_bad:
Trond Myklebust275acaa2011-07-12 13:42:02 -0400329 while (!list_empty(res)) {
330 data = list_entry(res->next, struct nfs_read_data, list);
Trond Myklebust6e4efd52011-07-12 13:42:02 -0400331 list_del(&data->list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 nfs_readdata_free(data);
333 }
334 SetPageError(page);
335 nfs_readpage_release(req);
336 return -ENOMEM;
337}
338
Trond Myklebust275acaa2011-07-12 13:42:02 -0400339static int nfs_pagein_one(struct nfs_pageio_descriptor *desc, struct list_head *res)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340{
341 struct nfs_page *req;
342 struct page **pages;
343 struct nfs_read_data *data;
Fred Isamanc76069b2011-03-03 15:13:48 +0000344 struct list_head *head = &desc->pg_list;
Peng Tao3b609182011-07-15 03:33:42 -0400345 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
Fred Isamanc76069b2011-03-03 15:13:48 +0000347 data = nfs_readdata_alloc(nfs_page_array_len(desc->pg_base,
348 desc->pg_count));
Fred Isamanbae724e2011-03-01 01:34:15 +0000349 if (!data) {
350 nfs_async_read_error(head);
Peng Tao3b609182011-07-15 03:33:42 -0400351 ret = -ENOMEM;
Fred Isamanbae724e2011-03-01 01:34:15 +0000352 goto out;
353 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 pages = data->pagevec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 while (!list_empty(head)) {
357 req = nfs_list_entry(head->next);
358 nfs_list_remove_request(req);
359 nfs_list_add_request(req, &data->pages);
360 ClearPageError(req->wb_page);
361 *pages++ = req->wb_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 }
363 req = nfs_list_entry(data->pages.next);
364
Trond Myklebust6e4efd52011-07-12 13:42:02 -0400365 nfs_read_rpcsetup(req, data, desc->pg_count, 0);
Trond Myklebust275acaa2011-07-12 13:42:02 -0400366 list_add(&data->list, res);
Trond Myklebust50828d72011-07-12 13:42:02 -0400367 desc->pg_rpc_callops = &nfs_read_full_ops;
Fred Isamanbae724e2011-03-01 01:34:15 +0000368out:
Trond Myklebustdbae4c72008-04-14 14:54:53 -0400369 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370}
371
Trond Myklebust493292d2011-07-13 15:58:28 -0400372int nfs_generic_pagein(struct nfs_pageio_descriptor *desc, struct list_head *head)
373{
374 if (desc->pg_bsize < PAGE_CACHE_SIZE)
375 return nfs_pagein_multi(desc, head);
376 return nfs_pagein_one(desc, head);
377}
378
379static int nfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
Trond Myklebust1751c362011-06-10 13:30:23 -0400380{
Trond Myklebust275acaa2011-07-12 13:42:02 -0400381 LIST_HEAD(head);
382 int ret;
383
Trond Myklebust493292d2011-07-13 15:58:28 -0400384 ret = nfs_generic_pagein(desc, &head);
Trond Myklebust50828d72011-07-12 13:42:02 -0400385 if (ret == 0)
Trond Myklebust493292d2011-07-13 15:58:28 -0400386 ret = nfs_do_multiple_reads(&head, desc->pg_rpc_callops);
Trond Myklebust275acaa2011-07-12 13:42:02 -0400387 return ret;
Trond Myklebust1751c362011-06-10 13:30:23 -0400388}
Trond Myklebust1751c362011-06-10 13:30:23 -0400389
390static const struct nfs_pageio_ops nfs_pageio_read_ops = {
391 .pg_test = nfs_generic_pg_test,
392 .pg_doio = nfs_generic_pg_readpages,
393};
394
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395/*
Trond Myklebust0b671302006-11-14 16:12:23 -0500396 * This is the callback from RPC telling us whether a reply was
397 * received or some error occurred (timeout or socket shutdown).
398 */
399int nfs_readpage_result(struct rpc_task *task, struct nfs_read_data *data)
400{
401 int status;
402
Harvey Harrison3110ff82008-05-02 13:42:44 -0700403 dprintk("NFS: %s: %5u, (status %d)\n", __func__, task->tk_pid,
Trond Myklebust0b671302006-11-14 16:12:23 -0500404 task->tk_status);
405
406 status = NFS_PROTO(data->inode)->read_done(task, data);
407 if (status != 0)
408 return status;
409
410 nfs_add_stats(data->inode, NFSIOS_SERVERREADBYTES, data->res.count);
411
412 if (task->tk_status == -ESTALE) {
Benny Halevy3a10c302008-01-23 08:58:59 +0200413 set_bit(NFS_INO_STALE, &NFS_I(data->inode)->flags);
Trond Myklebust0b671302006-11-14 16:12:23 -0500414 nfs_mark_for_revalidate(data->inode);
415 }
Trond Myklebust0b671302006-11-14 16:12:23 -0500416 return 0;
417}
418
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400419static void nfs_readpage_retry(struct rpc_task *task, struct nfs_read_data *data)
Trond Myklebust0b671302006-11-14 16:12:23 -0500420{
421 struct nfs_readargs *argp = &data->args;
422 struct nfs_readres *resp = &data->res;
423
424 if (resp->eof || resp->count == argp->count)
Trond Myklebustd61e6122009-12-05 19:32:19 -0500425 return;
Trond Myklebust0b671302006-11-14 16:12:23 -0500426
427 /* This is a short read! */
428 nfs_inc_stats(data->inode, NFSIOS_SHORTREAD);
429 /* Has the server at least made some progress? */
430 if (resp->count == 0)
Trond Myklebustd61e6122009-12-05 19:32:19 -0500431 return;
Trond Myklebust0b671302006-11-14 16:12:23 -0500432
433 /* Yes, so retry the read at the end of the data */
Andy Adamsoncbdabc72011-03-01 01:34:20 +0000434 data->mds_offset += resp->count;
Trond Myklebust0b671302006-11-14 16:12:23 -0500435 argp->offset += resp->count;
436 argp->pgbase += resp->count;
437 argp->count -= resp->count;
Trond Myklebust0110ee12009-12-07 09:00:24 -0500438 nfs_restart_rpc(task, NFS_SERVER(data->inode)->nfs_client);
Trond Myklebust0b671302006-11-14 16:12:23 -0500439}
440
441/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 * Handle a read reply that fills part of a page.
443 */
Trond Myklebustec06c092006-03-20 13:44:27 -0500444static void nfs_readpage_result_partial(struct rpc_task *task, void *calldata)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445{
Trond Myklebustec06c092006-03-20 13:44:27 -0500446 struct nfs_read_data *data = calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447
Trond Myklebustec06c092006-03-20 13:44:27 -0500448 if (nfs_readpage_result(task, data) != 0)
449 return;
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400450 if (task->tk_status < 0)
451 return;
Trond Myklebust0b671302006-11-14 16:12:23 -0500452
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400453 nfs_readpage_truncate_uninitialised_page(data);
454 nfs_readpage_retry(task, data);
455}
456
457static void nfs_readpage_release_partial(void *calldata)
458{
459 struct nfs_read_data *data = calldata;
460 struct nfs_page *req = data->req;
461 struct page *page = req->wb_page;
462 int status = data->task.tk_status;
463
464 if (status < 0)
Trond Myklebust0b671302006-11-14 16:12:23 -0500465 SetPageError(page);
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400466
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 if (atomic_dec_and_test(&req->wb_complete)) {
468 if (!PageError(page))
469 SetPageUptodate(page);
470 nfs_readpage_release(req);
471 }
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400472 nfs_readdata_release(calldata);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473}
474
Andy Adamsonf11c88a2009-04-01 09:22:25 -0400475#if defined(CONFIG_NFS_V4_1)
476void nfs_read_prepare(struct rpc_task *task, void *calldata)
477{
478 struct nfs_read_data *data = calldata;
479
Trond Myklebust035168ab2010-06-16 09:52:26 -0400480 if (nfs4_setup_sequence(NFS_SERVER(data->inode),
Andy Adamsonf11c88a2009-04-01 09:22:25 -0400481 &data->args.seq_args, &data->res.seq_res,
482 0, task))
483 return;
484 rpc_call_start(task);
485}
486#endif /* CONFIG_NFS_V4_1 */
487
Trond Myklebustec06c092006-03-20 13:44:27 -0500488static const struct rpc_call_ops nfs_read_partial_ops = {
Andy Adamsonf11c88a2009-04-01 09:22:25 -0400489#if defined(CONFIG_NFS_V4_1)
490 .rpc_call_prepare = nfs_read_prepare,
491#endif /* CONFIG_NFS_V4_1 */
Trond Myklebustec06c092006-03-20 13:44:27 -0500492 .rpc_call_done = nfs_readpage_result_partial,
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400493 .rpc_release = nfs_readpage_release_partial,
Trond Myklebustec06c092006-03-20 13:44:27 -0500494};
495
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400496static void nfs_readpage_set_pages_uptodate(struct nfs_read_data *data)
497{
498 unsigned int count = data->res.count;
499 unsigned int base = data->args.pgbase;
500 struct page **pages;
501
Trond Myklebust79558f32006-08-22 13:44:32 -0400502 if (data->res.eof)
503 count = data->args.count;
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400504 if (unlikely(count == 0))
505 return;
506 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT];
507 base &= ~PAGE_CACHE_MASK;
508 count += base;
509 for (;count >= PAGE_CACHE_SIZE; count -= PAGE_CACHE_SIZE, pages++)
510 SetPageUptodate(*pages);
Trond Myklebust0b671302006-11-14 16:12:23 -0500511 if (count == 0)
512 return;
513 /* Was this a short read? */
514 if (data->res.eof || data->res.count == data->args.count)
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400515 SetPageUptodate(*pages);
516}
517
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518/*
519 * This is the callback from RPC telling us whether a reply was
520 * received or some error occurred (timeout or socket shutdown).
521 */
Trond Myklebustec06c092006-03-20 13:44:27 -0500522static void nfs_readpage_result_full(struct rpc_task *task, void *calldata)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523{
Trond Myklebustec06c092006-03-20 13:44:27 -0500524 struct nfs_read_data *data = calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525
Trond Myklebust0b671302006-11-14 16:12:23 -0500526 if (nfs_readpage_result(task, data) != 0)
527 return;
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400528 if (task->tk_status < 0)
529 return;
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400530 /*
Trond Myklebust0b671302006-11-14 16:12:23 -0500531 * Note: nfs_readpage_retry may change the values of
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400532 * data->args. In the multi-page case, we therefore need
Trond Myklebust0b671302006-11-14 16:12:23 -0500533 * to ensure that we call nfs_readpage_set_pages_uptodate()
534 * first.
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400535 */
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400536 nfs_readpage_truncate_uninitialised_page(data);
537 nfs_readpage_set_pages_uptodate(data);
538 nfs_readpage_retry(task, data);
539}
540
541static void nfs_readpage_release_full(void *calldata)
542{
543 struct nfs_read_data *data = calldata;
544
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 while (!list_empty(&data->pages)) {
546 struct nfs_page *req = nfs_list_entry(data->pages.next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
Trond Myklebust1de3fc12006-05-25 01:40:44 -0400548 nfs_list_remove_request(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 nfs_readpage_release(req);
550 }
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400551 nfs_readdata_release(calldata);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552}
553
Trond Myklebustec06c092006-03-20 13:44:27 -0500554static const struct rpc_call_ops nfs_read_full_ops = {
Andy Adamsonf11c88a2009-04-01 09:22:25 -0400555#if defined(CONFIG_NFS_V4_1)
556 .rpc_call_prepare = nfs_read_prepare,
557#endif /* CONFIG_NFS_V4_1 */
Trond Myklebustec06c092006-03-20 13:44:27 -0500558 .rpc_call_done = nfs_readpage_result_full,
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400559 .rpc_release = nfs_readpage_release_full,
Trond Myklebustec06c092006-03-20 13:44:27 -0500560};
561
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 * Read a page over NFS.
564 * We read the page synchronously in the following case:
565 * - The error flag is set for this page. This happens only when a
566 * previous async read operation failed.
567 */
568int nfs_readpage(struct file *file, struct page *page)
569{
570 struct nfs_open_context *ctx;
571 struct inode *inode = page->mapping->host;
572 int error;
573
574 dprintk("NFS: nfs_readpage (%p %ld@%lu)\n",
575 page, PAGE_CACHE_SIZE, page->index);
Chuck Lever91d5b472006-03-20 13:44:14 -0500576 nfs_inc_stats(inode, NFSIOS_VFSREADPAGE);
577 nfs_add_stats(inode, NFSIOS_READPAGES, 1);
578
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 /*
580 * Try to flush any pending writes to the file..
581 *
582 * NOTE! Because we own the page lock, there cannot
583 * be any new pending writes generated at this point
584 * for this page (other pages can be written to).
585 */
586 error = nfs_wb_page(inode, page);
587 if (error)
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400588 goto out_unlock;
589 if (PageUptodate(page))
590 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591
Trond Myklebust5f004cf2006-09-14 14:03:14 -0400592 error = -ESTALE;
593 if (NFS_STALE(inode))
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400594 goto out_unlock;
Trond Myklebust5f004cf2006-09-14 14:03:14 -0400595
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 if (file == NULL) {
Trond Myklebustcf1308f2006-11-19 16:44:52 -0500597 error = -EBADF;
Trond Myklebustd5308382005-11-04 15:33:38 -0500598 ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 if (ctx == NULL)
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400600 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 } else
Trond Myklebustcd3758e2007-08-10 17:44:32 -0400602 ctx = get_nfs_open_context(nfs_file_open_context(file));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603
David Howells9a9fc1c2009-04-03 16:42:44 +0100604 if (!IS_SYNC(inode)) {
605 error = nfs_readpage_from_fscache(ctx, inode, page);
606 if (error == 0)
607 goto out;
608 }
609
Trond Myklebust8e0969f2006-12-13 15:23:44 -0500610 error = nfs_readpage_async(ctx, inode, page);
611
David Howells9a9fc1c2009-04-03 16:42:44 +0100612out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 put_nfs_open_context(ctx);
614 return error;
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400615out_unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 unlock_page(page);
617 return error;
618}
619
620struct nfs_readdesc {
Trond Myklebust8b09bee2007-04-02 18:48:28 -0400621 struct nfs_pageio_descriptor *pgio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 struct nfs_open_context *ctx;
623};
624
625static int
626readpage_async_filler(void *data, struct page *page)
627{
628 struct nfs_readdesc *desc = (struct nfs_readdesc *)data;
629 struct inode *inode = page->mapping->host;
630 struct nfs_page *new;
631 unsigned int len;
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400632 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633
Trond Myklebust49a70f2782006-12-05 00:35:38 -0500634 len = nfs_page_length(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 if (len == 0)
636 return nfs_return_empty_page(page);
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400637
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 new = nfs_create_request(desc->ctx, inode, page, 0, len);
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400639 if (IS_ERR(new))
640 goto out_error;
641
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 if (len < PAGE_CACHE_SIZE)
Christoph Lametereebd2aa2008-02-04 22:28:29 -0800643 zero_user_segment(page, len, PAGE_CACHE_SIZE);
Fred Isamanf8512ad2008-03-19 11:24:39 -0400644 if (!nfs_pageio_add_request(desc->pgio, new)) {
645 error = desc->pgio->pg_error;
646 goto out_unlock;
647 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 return 0;
Trond Myklebustde05a0c2007-05-20 13:05:05 -0400649out_error:
650 error = PTR_ERR(new);
651 SetPageError(page);
652out_unlock:
653 unlock_page(page);
654 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655}
656
657int nfs_readpages(struct file *filp, struct address_space *mapping,
658 struct list_head *pages, unsigned nr_pages)
659{
Trond Myklebust8b09bee2007-04-02 18:48:28 -0400660 struct nfs_pageio_descriptor pgio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 struct nfs_readdesc desc = {
Trond Myklebust8b09bee2007-04-02 18:48:28 -0400662 .pgio = &pgio,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 };
664 struct inode *inode = mapping->host;
Trond Myklebust8b09bee2007-04-02 18:48:28 -0400665 unsigned long npages;
Trond Myklebust5f004cf2006-09-14 14:03:14 -0400666 int ret = -ESTALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667
668 dprintk("NFS: nfs_readpages (%s/%Ld %d)\n",
669 inode->i_sb->s_id,
670 (long long)NFS_FILEID(inode),
671 nr_pages);
Chuck Lever91d5b472006-03-20 13:44:14 -0500672 nfs_inc_stats(inode, NFSIOS_VFSREADPAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673
Trond Myklebust5f004cf2006-09-14 14:03:14 -0400674 if (NFS_STALE(inode))
675 goto out;
676
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 if (filp == NULL) {
Trond Myklebustd5308382005-11-04 15:33:38 -0500678 desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 if (desc.ctx == NULL)
680 return -EBADF;
681 } else
Trond Myklebustcd3758e2007-08-10 17:44:32 -0400682 desc.ctx = get_nfs_open_context(nfs_file_open_context(filp));
David Howells9a9fc1c2009-04-03 16:42:44 +0100683
684 /* attempt to read as many of the pages as possible from the cache
685 * - this returns -ENOBUFS immediately if the cookie is negative
686 */
687 ret = nfs_readpages_from_fscache(desc.ctx, inode, mapping,
688 pages, &nr_pages);
689 if (ret == 0)
690 goto read_complete; /* all pages were read */
691
Trond Myklebust1751c362011-06-10 13:30:23 -0400692 nfs_pageio_init_read(&pgio, inode);
Trond Myklebust8b09bee2007-04-02 18:48:28 -0400693
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc);
Trond Myklebust8b09bee2007-04-02 18:48:28 -0400695
696 nfs_pageio_complete(&pgio);
697 npages = (pgio.pg_bytes_written + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
698 nfs_add_stats(inode, NFSIOS_READPAGES, npages);
David Howells9a9fc1c2009-04-03 16:42:44 +0100699read_complete:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 put_nfs_open_context(desc.ctx);
Trond Myklebust5f004cf2006-09-14 14:03:14 -0400701out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 return ret;
703}
704
David Howellsf7b422b2006-06-09 09:34:33 -0400705int __init nfs_init_readpagecache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706{
707 nfs_rdata_cachep = kmem_cache_create("nfs_read_data",
708 sizeof(struct nfs_read_data),
709 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +0900710 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711 if (nfs_rdata_cachep == NULL)
712 return -ENOMEM;
713
Matthew Dobson93d23412006-03-26 01:37:50 -0800714 nfs_rdata_mempool = mempool_create_slab_pool(MIN_POOL_READ,
715 nfs_rdata_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 if (nfs_rdata_mempool == NULL)
717 return -ENOMEM;
718
719 return 0;
720}
721
David Brownell266bee82006-06-27 12:59:15 -0700722void nfs_destroy_readpagecache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723{
724 mempool_destroy(nfs_rdata_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -0700725 kmem_cache_destroy(nfs_rdata_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726}