blob: 345aa5c0f3824cb78255b50a48d84be61eb17517 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/nfs/direct.c
3 *
4 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
5 *
6 * High-performance uncached I/O for the Linux NFS client
7 *
8 * There are important applications whose performance or correctness
9 * depends on uncached access to file data. Database clusters
Chuck Lever88467052006-03-20 13:44:34 -050010 * (multiple copies of the same instance running on separate hosts)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * implement their own cache coherency protocol that subsumes file
Chuck Lever88467052006-03-20 13:44:34 -050012 * system cache protocols. Applications that process datasets
13 * considerably larger than the client's memory do not always benefit
14 * from a local cache. A streaming video server, for instance, has no
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 * need to cache the contents of a file.
16 *
17 * When an application requests uncached I/O, all read and write requests
18 * are made directly to the server; data stored or fetched via these
19 * requests is not cached in the Linux page cache. The client does not
20 * correct unaligned requests from applications. All requested bytes are
21 * held on permanent storage before a direct write system call returns to
22 * an application.
23 *
24 * Solaris implements an uncached I/O facility called directio() that
25 * is used for backups and sequential I/O to very large files. Solaris
26 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
27 * an undocumented mount option.
28 *
29 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
30 * help from Andrew Morton.
31 *
32 * 18 Dec 2001 Initial implementation for 2.4 --cel
33 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
34 * 08 Jun 2003 Port to 2.5 APIs --cel
35 * 31 Mar 2004 Handle direct I/O without VFS support --cel
36 * 15 Sep 2004 Parallel async reads --cel
Chuck Lever88467052006-03-20 13:44:34 -050037 * 04 May 2005 support O_DIRECT with aio --cel
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 *
39 */
40
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/errno.h>
42#include <linux/sched.h>
43#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <linux/file.h>
45#include <linux/pagemap.h>
46#include <linux/kref.h>
47
48#include <linux/nfs_fs.h>
49#include <linux/nfs_page.h>
50#include <linux/sunrpc/clnt.h>
51
52#include <asm/system.h>
53#include <asm/uaccess.h>
54#include <asm/atomic.h>
55
Trond Myklebust8d5658c2007-04-10 09:26:35 -040056#include "internal.h"
Chuck Lever91d5b472006-03-20 13:44:14 -050057#include "iostat.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070058
Linus Torvalds1da177e2005-04-16 15:20:36 -070059#define NFSDBG_FACILITY NFSDBG_VFS
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
Christoph Lametere18b8902006-12-06 20:33:20 -080061static struct kmem_cache *nfs_direct_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
63/*
64 * This represents a set of asynchronous requests that we're waiting on
65 */
66struct nfs_direct_req {
67 struct kref kref; /* release manager */
Chuck Lever15ce4a02006-03-20 13:44:34 -050068
69 /* I/O parameters */
Trond Myklebusta8881f52006-03-20 13:44:36 -050070 struct nfs_open_context *ctx; /* file open context info */
Chuck Lever99514f82006-03-20 13:44:30 -050071 struct kiocb * iocb; /* controlling i/o request */
Chuck Lever88467052006-03-20 13:44:34 -050072 struct inode * inode; /* target file of i/o */
Chuck Lever15ce4a02006-03-20 13:44:34 -050073
74 /* completion state */
Trond Myklebust607f31e2006-06-28 16:52:45 -040075 atomic_t io_count; /* i/os we're waiting for */
Chuck Lever15ce4a02006-03-20 13:44:34 -050076 spinlock_t lock; /* protect completion state */
Chuck Lever15ce4a02006-03-20 13:44:34 -050077 ssize_t count, /* bytes actually processed */
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 error; /* any reported error */
Trond Myklebustd72b7a62006-03-20 13:44:43 -050079 struct completion completion; /* wait for i/o completion */
Trond Myklebustfad61492006-03-20 13:44:36 -050080
81 /* commit state */
Trond Myklebust607f31e2006-06-28 16:52:45 -040082 struct list_head rewrite_list; /* saved nfs_write_data structs */
Trond Myklebustfad61492006-03-20 13:44:36 -050083 struct nfs_write_data * commit_data; /* special write_data for commits */
84 int flags;
85#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
86#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
87 struct nfs_writeverf verf; /* unstable write verifier */
Linus Torvalds1da177e2005-04-16 15:20:36 -070088};
89
Trond Myklebustfad61492006-03-20 13:44:36 -050090static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
Trond Myklebust607f31e2006-06-28 16:52:45 -040091static const struct rpc_call_ops nfs_write_direct_ops;
92
93static inline void get_dreq(struct nfs_direct_req *dreq)
94{
95 atomic_inc(&dreq->io_count);
96}
97
98static inline int put_dreq(struct nfs_direct_req *dreq)
99{
100 return atomic_dec_and_test(&dreq->io_count);
101}
102
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103/**
Chuck Leverb8a32e22006-03-20 13:44:28 -0500104 * nfs_direct_IO - NFS address space operation for direct I/O
105 * @rw: direction (read or write)
106 * @iocb: target I/O control block
107 * @iov: array of vectors that define I/O buffer
108 * @pos: offset in file to begin the operation
109 * @nr_segs: size of iovec array
110 *
111 * The presence of this routine in the address space ops vector means
112 * the NFS client supports direct I/O. However, we shunt off direct
113 * read and write requests before the VFS gets them, so this method
114 * should never be called.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 */
Chuck Leverb8a32e22006-03-20 13:44:28 -0500116ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
117{
Chuck Leverb8a32e22006-03-20 13:44:28 -0500118 dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
Josef "Jeff" Sipek01cce932006-12-08 02:36:40 -0800119 iocb->ki_filp->f_path.dentry->d_name.name,
Trond Myklebuste99170f2006-04-18 13:21:42 -0400120 (long long) pos, nr_segs);
Chuck Leverb8a32e22006-03-20 13:44:28 -0500121
122 return -EINVAL;
123}
124
Trond Myklebust607f31e2006-06-28 16:52:45 -0400125static void nfs_direct_dirty_pages(struct page **pages, int npages)
Trond Myklebust6b45d852006-03-20 13:44:43 -0500126{
127 int i;
128 for (i = 0; i < npages; i++) {
129 struct page *page = pages[i];
Trond Myklebust607f31e2006-06-28 16:52:45 -0400130 if (!PageCompound(page))
Trond Myklebust6b45d852006-03-20 13:44:43 -0500131 set_page_dirty_lock(page);
Trond Myklebust6b45d852006-03-20 13:44:43 -0500132 }
Chuck Lever9c93ab72006-06-20 12:56:31 -0400133}
134
Trond Myklebust607f31e2006-06-28 16:52:45 -0400135static void nfs_direct_release_pages(struct page **pages, int npages)
Chuck Lever9c93ab72006-06-20 12:56:31 -0400136{
Trond Myklebust607f31e2006-06-28 16:52:45 -0400137 int i;
138 for (i = 0; i < npages; i++)
139 page_cache_release(pages[i]);
Trond Myklebust6b45d852006-03-20 13:44:43 -0500140}
141
Chuck Lever93619e52006-03-20 13:44:31 -0500142static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 struct nfs_direct_req *dreq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145
Christoph Lametere94b1762006-12-06 20:33:17 -0800146 dreq = kmem_cache_alloc(nfs_direct_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 if (!dreq)
148 return NULL;
149
150 kref_init(&dreq->kref);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400151 kref_get(&dreq->kref);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500152 init_completion(&dreq->completion);
Trond Myklebustfad61492006-03-20 13:44:36 -0500153 INIT_LIST_HEAD(&dreq->rewrite_list);
Chuck Lever93619e52006-03-20 13:44:31 -0500154 dreq->iocb = NULL;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500155 dreq->ctx = NULL;
Chuck Lever15ce4a02006-03-20 13:44:34 -0500156 spin_lock_init(&dreq->lock);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400157 atomic_set(&dreq->io_count, 0);
Chuck Lever15ce4a02006-03-20 13:44:34 -0500158 dreq->count = 0;
159 dreq->error = 0;
Trond Myklebustfad61492006-03-20 13:44:36 -0500160 dreq->flags = 0;
Chuck Lever93619e52006-03-20 13:44:31 -0500161
162 return dreq;
163}
164
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165static void nfs_direct_req_release(struct kref *kref)
166{
167 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500168
169 if (dreq->ctx != NULL)
170 put_nfs_open_context(dreq->ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 kmem_cache_free(nfs_direct_cachep, dreq);
172}
173
Chuck Leverd4cc9482006-03-20 13:44:28 -0500174/*
Chuck Leverbc0fb202006-03-20 13:44:31 -0500175 * Collects and returns the final error value/byte-count.
176 */
177static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
178{
Chuck Lever15ce4a02006-03-20 13:44:34 -0500179 ssize_t result = -EIOCBQUEUED;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500180
181 /* Async requests don't wait here */
182 if (dreq->iocb)
183 goto out;
184
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500185 result = wait_for_completion_interruptible(&dreq->completion);
Chuck Leverbc0fb202006-03-20 13:44:31 -0500186
187 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500188 result = dreq->error;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500189 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500190 result = dreq->count;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500191
192out:
193 kref_put(&dreq->kref, nfs_direct_req_release);
194 return (ssize_t) result;
195}
196
197/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400198 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
199 * the iocb is still valid here if this is a synchronous request.
Chuck Lever63ab46a2006-03-20 13:44:31 -0500200 */
201static void nfs_direct_complete(struct nfs_direct_req *dreq)
202{
Chuck Lever63ab46a2006-03-20 13:44:31 -0500203 if (dreq->iocb) {
Chuck Lever15ce4a02006-03-20 13:44:34 -0500204 long res = (long) dreq->error;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500205 if (!res)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500206 res = (long) dreq->count;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500207 aio_complete(dreq->iocb, res, 0);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500208 }
209 complete_all(&dreq->completion);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500210
211 kref_put(&dreq->kref, nfs_direct_req_release);
212}
213
214/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400215 * We must hold a reference to all the pages in this direct read request
216 * until the RPCs complete. This could be long *after* we are woken up in
217 * nfs_direct_wait (for instance, if someone hits ^C on a slow server).
Chuck Lever06cf6f22006-06-20 12:56:49 -0400218 */
Trond Myklebustec06c092006-03-20 13:44:27 -0500219static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220{
Trond Myklebustec06c092006-03-20 13:44:27 -0500221 struct nfs_read_data *data = calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
223
Trond Myklebustec06c092006-03-20 13:44:27 -0500224 if (nfs_readpage_result(task, data) != 0)
225 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226
Trond Myklebust607f31e2006-06-28 16:52:45 -0400227 nfs_direct_dirty_pages(data->pagevec, data->npages);
228 nfs_direct_release_pages(data->pagevec, data->npages);
229
Chuck Lever15ce4a02006-03-20 13:44:34 -0500230 spin_lock(&dreq->lock);
231
232 if (likely(task->tk_status >= 0))
233 dreq->count += data->res.count;
234 else
235 dreq->error = task->tk_status;
236
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400237 spin_unlock(&dreq->lock);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400238
239 if (put_dreq(dreq))
240 nfs_direct_complete(dreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241}
242
Trond Myklebustec06c092006-03-20 13:44:27 -0500243static const struct rpc_call_ops nfs_read_direct_ops = {
244 .rpc_call_done = nfs_direct_read_result,
245 .rpc_release = nfs_readdata_release,
246};
247
Chuck Leverd4cc9482006-03-20 13:44:28 -0500248/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400249 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
250 * operation. If nfs_readdata_alloc() or get_user_pages() fails,
251 * bail and stop sending more reads. Read length accounting is
252 * handled automatically by nfs_direct_read_result(). Otherwise, if
253 * no requests have been sent, just return an error.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 */
Trond Myklebust607f31e2006-06-28 16:52:45 -0400255static ssize_t nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256{
Trond Myklebusta8881f52006-03-20 13:44:36 -0500257 struct nfs_open_context *ctx = dreq->ctx;
258 struct inode *inode = ctx->dentry->d_inode;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500259 size_t rsize = NFS_SERVER(inode)->rsize;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400260 unsigned int pgbase;
261 int result;
262 ssize_t started = 0;
Chuck Lever82b145c2006-06-20 12:57:03 -0400263
Trond Myklebust607f31e2006-06-28 16:52:45 -0400264 get_dreq(dreq);
265
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 do {
Chuck Lever82b145c2006-06-20 12:57:03 -0400267 struct nfs_read_data *data;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500268 size_t bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700270 pgbase = user_addr & ~PAGE_MASK;
271 bytes = min(rsize,count);
272
Trond Myklebust607f31e2006-06-28 16:52:45 -0400273 result = -ENOMEM;
Trond Myklebust8d5658c2007-04-10 09:26:35 -0400274 data = nfs_readdata_alloc(nfs_page_array_len(pgbase, bytes));
Trond Myklebust607f31e2006-06-28 16:52:45 -0400275 if (unlikely(!data))
276 break;
277
Trond Myklebust607f31e2006-06-28 16:52:45 -0400278 down_read(&current->mm->mmap_sem);
279 result = get_user_pages(current, current->mm, user_addr,
280 data->npages, 1, 0, data->pagevec, NULL);
281 up_read(&current->mm->mmap_sem);
282 if (unlikely(result < data->npages)) {
283 if (result > 0)
284 nfs_direct_release_pages(data->pagevec, result);
285 nfs_readdata_release(data);
286 break;
287 }
Chuck Lever06cf6f22006-06-20 12:56:49 -0400288
Trond Myklebust607f31e2006-06-28 16:52:45 -0400289 get_dreq(dreq);
290
291 data->req = (struct nfs_page *) dreq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 data->inode = inode;
293 data->cred = ctx->cred;
294 data->args.fh = NFS_FH(inode);
295 data->args.context = ctx;
Chuck Lever88467052006-03-20 13:44:34 -0500296 data->args.offset = pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 data->args.pgbase = pgbase;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400298 data->args.pages = data->pagevec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 data->args.count = bytes;
300 data->res.fattr = &data->fattr;
301 data->res.eof = 0;
302 data->res.count = bytes;
303
Trond Myklebustec06c092006-03-20 13:44:27 -0500304 rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
305 &nfs_read_direct_ops, data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 NFS_PROTO(inode)->read_setup(data);
307
308 data->task.tk_cookie = (unsigned long) inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 rpc_execute(&data->task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311
Chuck Levera3f565b2007-01-31 12:14:01 -0500312 dprintk("NFS: %5u initiated direct read call "
313 "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 data->task.tk_pid,
315 inode->i_sb->s_id,
316 (long long)NFS_FILEID(inode),
317 bytes,
318 (unsigned long long)data->args.offset);
319
Trond Myklebust607f31e2006-06-28 16:52:45 -0400320 started += bytes;
321 user_addr += bytes;
Chuck Lever88467052006-03-20 13:44:34 -0500322 pos += bytes;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700323 /* FIXME: Remove this unnecessary math from final patch */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 pgbase += bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 pgbase &= ~PAGE_MASK;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700326 BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327
328 count -= bytes;
329 } while (count != 0);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400330
331 if (put_dreq(dreq))
332 nfs_direct_complete(dreq);
333
334 if (started)
335 return 0;
336 return result < 0 ? (ssize_t) result : -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337}
338
Trond Myklebust607f31e2006-06-28 16:52:45 -0400339static ssize_t nfs_direct_read(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340{
Trond Myklebust607f31e2006-06-28 16:52:45 -0400341 ssize_t result = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 sigset_t oldset;
Chuck Lever99514f82006-03-20 13:44:30 -0500343 struct inode *inode = iocb->ki_filp->f_mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 struct rpc_clnt *clnt = NFS_CLIENT(inode);
345 struct nfs_direct_req *dreq;
346
Trond Myklebust607f31e2006-06-28 16:52:45 -0400347 dreq = nfs_direct_req_alloc();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 if (!dreq)
349 return -ENOMEM;
350
Chuck Lever91d5b472006-03-20 13:44:14 -0500351 dreq->inode = inode;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500352 dreq->ctx = get_nfs_open_context((struct nfs_open_context *)iocb->ki_filp->private_data);
Chuck Lever487b8372006-03-20 13:44:30 -0500353 if (!is_sync_kiocb(iocb))
354 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355
Chuck Lever91d5b472006-03-20 13:44:14 -0500356 nfs_add_stats(inode, NFSIOS_DIRECTREADBYTES, count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 rpc_clnt_sigmask(clnt, &oldset);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400358 result = nfs_direct_read_schedule(dreq, user_addr, count, pos);
359 if (!result)
360 result = nfs_direct_wait(dreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 rpc_clnt_sigunmask(clnt, &oldset);
362
363 return result;
364}
365
Trond Myklebustfad61492006-03-20 13:44:36 -0500366static void nfs_direct_free_writedata(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367{
Trond Myklebust607f31e2006-06-28 16:52:45 -0400368 while (!list_empty(&dreq->rewrite_list)) {
369 struct nfs_write_data *data = list_entry(dreq->rewrite_list.next, struct nfs_write_data, pages);
Trond Myklebustfad61492006-03-20 13:44:36 -0500370 list_del(&data->pages);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400371 nfs_direct_release_pages(data->pagevec, data->npages);
Trond Myklebustfad61492006-03-20 13:44:36 -0500372 nfs_writedata_release(data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374}
375
Trond Myklebustfad61492006-03-20 13:44:36 -0500376#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
377static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378{
Trond Myklebust607f31e2006-06-28 16:52:45 -0400379 struct inode *inode = dreq->inode;
380 struct list_head *p;
381 struct nfs_write_data *data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382
Trond Myklebustfad61492006-03-20 13:44:36 -0500383 dreq->count = 0;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400384 get_dreq(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500385
Trond Myklebust607f31e2006-06-28 16:52:45 -0400386 list_for_each(p, &dreq->rewrite_list) {
387 data = list_entry(p, struct nfs_write_data, pages);
388
389 get_dreq(dreq);
390
391 /*
392 * Reset data->res.
393 */
394 nfs_fattr_init(&data->fattr);
395 data->res.count = data->args.count;
396 memset(&data->verf, 0, sizeof(data->verf));
397
398 /*
399 * Reuse data->task; data->args should not have changed
400 * since the original request was sent.
401 */
402 rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
403 &nfs_write_direct_ops, data);
404 NFS_PROTO(inode)->write_setup(data, FLUSH_STABLE);
405
406 data->task.tk_priority = RPC_PRIORITY_NORMAL;
407 data->task.tk_cookie = (unsigned long) inode;
408
409 /*
410 * We're called via an RPC callback, so BKL is already held.
411 */
412 rpc_execute(&data->task);
413
414 dprintk("NFS: %5u rescheduled direct write call (req %s/%Ld, %u bytes @ offset %Lu)\n",
415 data->task.tk_pid,
416 inode->i_sb->s_id,
417 (long long)NFS_FILEID(inode),
418 data->args.count,
419 (unsigned long long)data->args.offset);
420 }
421
422 if (put_dreq(dreq))
423 nfs_direct_write_complete(dreq, inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500424}
425
426static void nfs_direct_commit_result(struct rpc_task *task, void *calldata)
427{
428 struct nfs_write_data *data = calldata;
429 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
430
431 /* Call the NFS version-specific code */
432 if (NFS_PROTO(data->inode)->commit_done(task, data) != 0)
433 return;
434 if (unlikely(task->tk_status < 0)) {
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400435 dprintk("NFS: %5u commit failed with error %d.\n",
436 task->tk_pid, task->tk_status);
Trond Myklebustfad61492006-03-20 13:44:36 -0500437 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400438 } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) {
Trond Myklebustfad61492006-03-20 13:44:36 -0500439 dprintk("NFS: %5u commit verify failed\n", task->tk_pid);
440 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
441 }
442
443 dprintk("NFS: %5u commit returned %d\n", task->tk_pid, task->tk_status);
444 nfs_direct_write_complete(dreq, data->inode);
445}
446
447static const struct rpc_call_ops nfs_commit_direct_ops = {
448 .rpc_call_done = nfs_direct_commit_result,
449 .rpc_release = nfs_commit_release,
450};
451
452static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
453{
Trond Myklebustfad61492006-03-20 13:44:36 -0500454 struct nfs_write_data *data = dreq->commit_data;
Trond Myklebustfad61492006-03-20 13:44:36 -0500455
456 data->inode = dreq->inode;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500457 data->cred = dreq->ctx->cred;
Trond Myklebustfad61492006-03-20 13:44:36 -0500458
459 data->args.fh = NFS_FH(data->inode);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400460 data->args.offset = 0;
461 data->args.count = 0;
Trond Myklebustfad61492006-03-20 13:44:36 -0500462 data->res.count = 0;
463 data->res.fattr = &data->fattr;
464 data->res.verf = &data->verf;
465
466 rpc_init_task(&data->task, NFS_CLIENT(dreq->inode), RPC_TASK_ASYNC,
467 &nfs_commit_direct_ops, data);
468 NFS_PROTO(data->inode)->commit_setup(data, 0);
469
470 data->task.tk_priority = RPC_PRIORITY_NORMAL;
471 data->task.tk_cookie = (unsigned long)data->inode;
472 /* Note: task.tk_ops->rpc_release will free dreq->commit_data */
473 dreq->commit_data = NULL;
474
Trond Myklebuste99170f2006-04-18 13:21:42 -0400475 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
Trond Myklebustfad61492006-03-20 13:44:36 -0500476
Trond Myklebustfad61492006-03-20 13:44:36 -0500477 rpc_execute(&data->task);
Trond Myklebustfad61492006-03-20 13:44:36 -0500478}
479
480static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
481{
482 int flags = dreq->flags;
483
484 dreq->flags = 0;
485 switch (flags) {
486 case NFS_ODIRECT_DO_COMMIT:
487 nfs_direct_commit_schedule(dreq);
488 break;
489 case NFS_ODIRECT_RESCHED_WRITES:
490 nfs_direct_write_reschedule(dreq);
491 break;
492 default:
493 nfs_end_data_update(inode);
494 if (dreq->commit_data != NULL)
495 nfs_commit_free(dreq->commit_data);
496 nfs_direct_free_writedata(dreq);
Trond Myklebustcd9ae2b2006-10-19 23:28:40 -0700497 nfs_zap_mapping(inode, inode->i_mapping);
Trond Myklebustfad61492006-03-20 13:44:36 -0500498 nfs_direct_complete(dreq);
499 }
500}
501
502static void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
503{
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700504 dreq->commit_data = nfs_commit_alloc();
Trond Myklebustfad61492006-03-20 13:44:36 -0500505 if (dreq->commit_data != NULL)
506 dreq->commit_data->req = (struct nfs_page *) dreq;
507}
508#else
509static inline void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
510{
511 dreq->commit_data = NULL;
512}
513
514static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
515{
516 nfs_end_data_update(inode);
517 nfs_direct_free_writedata(dreq);
Trond Myklebustcd9ae2b2006-10-19 23:28:40 -0700518 nfs_zap_mapping(inode, inode->i_mapping);
Trond Myklebustfad61492006-03-20 13:44:36 -0500519 nfs_direct_complete(dreq);
520}
521#endif
522
Chuck Lever462d5b32006-03-20 13:44:32 -0500523static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
524{
525 struct nfs_write_data *data = calldata;
526 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
527 int status = task->tk_status;
528
529 if (nfs_writeback_done(task, data) != 0)
530 return;
Chuck Lever462d5b32006-03-20 13:44:32 -0500531
Chuck Lever15ce4a02006-03-20 13:44:34 -0500532 spin_lock(&dreq->lock);
Chuck Lever462d5b32006-03-20 13:44:32 -0500533
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400534 if (unlikely(dreq->error != 0))
Trond Myklebusteda3cef2006-10-19 23:28:38 -0700535 goto out_unlock;
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400536 if (unlikely(status < 0)) {
537 /* An error has occured, so we should not commit */
538 dreq->flags = 0;
539 dreq->error = status;
Trond Myklebusteda3cef2006-10-19 23:28:38 -0700540 }
541
542 dreq->count += data->res.count;
Chuck Lever15ce4a02006-03-20 13:44:34 -0500543
Trond Myklebustfad61492006-03-20 13:44:36 -0500544 if (data->res.verf->committed != NFS_FILE_SYNC) {
545 switch (dreq->flags) {
546 case 0:
547 memcpy(&dreq->verf, &data->verf, sizeof(dreq->verf));
548 dreq->flags = NFS_ODIRECT_DO_COMMIT;
549 break;
550 case NFS_ODIRECT_DO_COMMIT:
551 if (memcmp(&dreq->verf, &data->verf, sizeof(dreq->verf))) {
552 dprintk("NFS: %5u write verify failed\n", task->tk_pid);
553 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
554 }
555 }
556 }
Trond Myklebusteda3cef2006-10-19 23:28:38 -0700557out_unlock:
Trond Myklebustfad61492006-03-20 13:44:36 -0500558 spin_unlock(&dreq->lock);
559}
560
561/*
562 * NB: Return the value of the first error return code. Subsequent
563 * errors after the first one are ignored.
564 */
565static void nfs_direct_write_release(void *calldata)
566{
567 struct nfs_write_data *data = calldata;
568 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
569
Trond Myklebust607f31e2006-06-28 16:52:45 -0400570 if (put_dreq(dreq))
571 nfs_direct_write_complete(dreq, data->inode);
Chuck Lever462d5b32006-03-20 13:44:32 -0500572}
573
574static const struct rpc_call_ops nfs_write_direct_ops = {
575 .rpc_call_done = nfs_direct_write_result,
Trond Myklebustfad61492006-03-20 13:44:36 -0500576 .rpc_release = nfs_direct_write_release,
Chuck Lever462d5b32006-03-20 13:44:32 -0500577};
578
579/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400580 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
581 * operation. If nfs_writedata_alloc() or get_user_pages() fails,
582 * bail and stop sending more writes. Write length accounting is
583 * handled automatically by nfs_direct_write_result(). Otherwise, if
584 * no requests have been sent, just return an error.
Chuck Lever462d5b32006-03-20 13:44:32 -0500585 */
Trond Myklebust607f31e2006-06-28 16:52:45 -0400586static ssize_t nfs_direct_write_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t pos, int sync)
Chuck Lever462d5b32006-03-20 13:44:32 -0500587{
Trond Myklebusta8881f52006-03-20 13:44:36 -0500588 struct nfs_open_context *ctx = dreq->ctx;
589 struct inode *inode = ctx->dentry->d_inode;
Chuck Lever462d5b32006-03-20 13:44:32 -0500590 size_t wsize = NFS_SERVER(inode)->wsize;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400591 unsigned int pgbase;
592 int result;
593 ssize_t started = 0;
Chuck Lever82b145c2006-06-20 12:57:03 -0400594
Trond Myklebust607f31e2006-06-28 16:52:45 -0400595 get_dreq(dreq);
596
Chuck Lever462d5b32006-03-20 13:44:32 -0500597 do {
Chuck Lever82b145c2006-06-20 12:57:03 -0400598 struct nfs_write_data *data;
Chuck Lever462d5b32006-03-20 13:44:32 -0500599 size_t bytes;
600
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700601 pgbase = user_addr & ~PAGE_MASK;
602 bytes = min(wsize,count);
603
Trond Myklebust607f31e2006-06-28 16:52:45 -0400604 result = -ENOMEM;
Trond Myklebust8d5658c2007-04-10 09:26:35 -0400605 data = nfs_writedata_alloc(nfs_page_array_len(pgbase, bytes));
Trond Myklebust607f31e2006-06-28 16:52:45 -0400606 if (unlikely(!data))
607 break;
608
Trond Myklebust607f31e2006-06-28 16:52:45 -0400609 down_read(&current->mm->mmap_sem);
610 result = get_user_pages(current, current->mm, user_addr,
611 data->npages, 0, 0, data->pagevec, NULL);
612 up_read(&current->mm->mmap_sem);
613 if (unlikely(result < data->npages)) {
614 if (result > 0)
615 nfs_direct_release_pages(data->pagevec, result);
616 nfs_writedata_release(data);
617 break;
618 }
619
620 get_dreq(dreq);
621
Trond Myklebustfad61492006-03-20 13:44:36 -0500622 list_move_tail(&data->pages, &dreq->rewrite_list);
Chuck Lever462d5b32006-03-20 13:44:32 -0500623
Trond Myklebust607f31e2006-06-28 16:52:45 -0400624 data->req = (struct nfs_page *) dreq;
Chuck Lever462d5b32006-03-20 13:44:32 -0500625 data->inode = inode;
626 data->cred = ctx->cred;
627 data->args.fh = NFS_FH(inode);
628 data->args.context = ctx;
Chuck Lever88467052006-03-20 13:44:34 -0500629 data->args.offset = pos;
Chuck Lever462d5b32006-03-20 13:44:32 -0500630 data->args.pgbase = pgbase;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400631 data->args.pages = data->pagevec;
Chuck Lever462d5b32006-03-20 13:44:32 -0500632 data->args.count = bytes;
633 data->res.fattr = &data->fattr;
634 data->res.count = bytes;
Chuck Lever47989d72006-03-20 13:44:32 -0500635 data->res.verf = &data->verf;
Chuck Lever462d5b32006-03-20 13:44:32 -0500636
637 rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
638 &nfs_write_direct_ops, data);
Trond Myklebustfad61492006-03-20 13:44:36 -0500639 NFS_PROTO(inode)->write_setup(data, sync);
Chuck Lever462d5b32006-03-20 13:44:32 -0500640
641 data->task.tk_priority = RPC_PRIORITY_NORMAL;
642 data->task.tk_cookie = (unsigned long) inode;
643
Chuck Lever462d5b32006-03-20 13:44:32 -0500644 rpc_execute(&data->task);
Chuck Lever462d5b32006-03-20 13:44:32 -0500645
Chuck Levera3f565b2007-01-31 12:14:01 -0500646 dprintk("NFS: %5u initiated direct write call "
647 "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
Chuck Lever462d5b32006-03-20 13:44:32 -0500648 data->task.tk_pid,
649 inode->i_sb->s_id,
650 (long long)NFS_FILEID(inode),
651 bytes,
652 (unsigned long long)data->args.offset);
653
Trond Myklebust607f31e2006-06-28 16:52:45 -0400654 started += bytes;
655 user_addr += bytes;
Chuck Lever88467052006-03-20 13:44:34 -0500656 pos += bytes;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700657
658 /* FIXME: Remove this useless math from the final patch */
Chuck Lever462d5b32006-03-20 13:44:32 -0500659 pgbase += bytes;
Chuck Lever462d5b32006-03-20 13:44:32 -0500660 pgbase &= ~PAGE_MASK;
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700661 BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
Chuck Lever462d5b32006-03-20 13:44:32 -0500662
663 count -= bytes;
664 } while (count != 0);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400665
666 if (put_dreq(dreq))
667 nfs_direct_write_complete(dreq, inode);
668
669 if (started)
670 return 0;
671 return result < 0 ? (ssize_t) result : -EFAULT;
Chuck Lever462d5b32006-03-20 13:44:32 -0500672}
673
Trond Myklebust607f31e2006-06-28 16:52:45 -0400674static ssize_t nfs_direct_write(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675{
Trond Myklebust607f31e2006-06-28 16:52:45 -0400676 ssize_t result = 0;
Chuck Lever462d5b32006-03-20 13:44:32 -0500677 sigset_t oldset;
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500678 struct inode *inode = iocb->ki_filp->f_mapping->host;
Chuck Lever462d5b32006-03-20 13:44:32 -0500679 struct rpc_clnt *clnt = NFS_CLIENT(inode);
680 struct nfs_direct_req *dreq;
Trond Myklebustfad61492006-03-20 13:44:36 -0500681 size_t wsize = NFS_SERVER(inode)->wsize;
682 int sync = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
Trond Myklebust607f31e2006-06-28 16:52:45 -0400684 dreq = nfs_direct_req_alloc();
Chuck Lever462d5b32006-03-20 13:44:32 -0500685 if (!dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 return -ENOMEM;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400687 nfs_alloc_commit_data(dreq);
688
Trond Myklebustfad61492006-03-20 13:44:36 -0500689 if (dreq->commit_data == NULL || count < wsize)
690 sync = FLUSH_STABLE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500692 dreq->inode = inode;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500693 dreq->ctx = get_nfs_open_context((struct nfs_open_context *)iocb->ki_filp->private_data);
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500694 if (!is_sync_kiocb(iocb))
695 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696
Chuck Lever47989d72006-03-20 13:44:32 -0500697 nfs_add_stats(inode, NFSIOS_DIRECTWRITTENBYTES, count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698
699 nfs_begin_data_update(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700
Chuck Lever462d5b32006-03-20 13:44:32 -0500701 rpc_clnt_sigmask(clnt, &oldset);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400702 result = nfs_direct_write_schedule(dreq, user_addr, count, pos, sync);
703 if (!result)
704 result = nfs_direct_wait(dreq);
Chuck Lever462d5b32006-03-20 13:44:32 -0500705 rpc_clnt_sigunmask(clnt, &oldset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 return result;
708}
709
710/**
711 * nfs_file_direct_read - file direct read operation for NFS files
712 * @iocb: target I/O control block
Badari Pulavarty027445c2006-09-30 23:28:46 -0700713 * @iov: vector of user buffers into which to read data
714 * @nr_segs: size of iov vector
Chuck Lever88467052006-03-20 13:44:34 -0500715 * @pos: byte offset in file where reading starts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 *
717 * We use this function for direct reads instead of calling
718 * generic_file_aio_read() in order to avoid gfar's check to see if
719 * the request starts before the end of the file. For that check
720 * to work, we must generate a GETATTR before each direct read, and
721 * even then there is a window between the GETATTR and the subsequent
Chuck Lever88467052006-03-20 13:44:34 -0500722 * READ where the file size could change. Our preference is simply
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 * to do all reads the application wants, and the server will take
724 * care of managing the end of file boundary.
Chuck Lever88467052006-03-20 13:44:34 -0500725 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 * This function also eliminates unnecessarily updating the file's
727 * atime locally, as the NFS server sets the file's atime, and this
728 * client must read the updated atime from the server back into its
729 * cache.
730 */
Badari Pulavarty027445c2006-09-30 23:28:46 -0700731ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
732 unsigned long nr_segs, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733{
734 ssize_t retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 struct address_space *mapping = file->f_mapping;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700737 /* XXX: temporary */
738 const char __user *buf = iov[0].iov_base;
739 size_t count = iov[0].iov_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740
Chuck Leverce1a8e62005-11-30 18:08:17 -0500741 dprintk("nfs: direct read(%s/%s, %lu@%Ld)\n",
Josef "Jeff" Sipek01cce932006-12-08 02:36:40 -0800742 file->f_path.dentry->d_parent->d_name.name,
743 file->f_path.dentry->d_name.name,
Chuck Leverce1a8e62005-11-30 18:08:17 -0500744 (unsigned long) count, (long long) pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745
Badari Pulavarty027445c2006-09-30 23:28:46 -0700746 if (nr_segs != 1)
747 return -EINVAL;
748
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 if (count < 0)
750 goto out;
751 retval = -EFAULT;
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500752 if (!access_ok(VERIFY_WRITE, buf, count))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 goto out;
754 retval = 0;
755 if (!count)
756 goto out;
757
Trond Myklebust29884df2005-12-13 16:13:54 -0500758 retval = nfs_sync_mapping(mapping);
759 if (retval)
760 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
Trond Myklebust607f31e2006-06-28 16:52:45 -0400762 retval = nfs_direct_read(iocb, (unsigned long) buf, count, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 if (retval > 0)
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500764 iocb->ki_pos = pos + retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765
766out:
767 return retval;
768}
769
770/**
771 * nfs_file_direct_write - file direct write operation for NFS files
772 * @iocb: target I/O control block
Badari Pulavarty027445c2006-09-30 23:28:46 -0700773 * @iov: vector of user buffers from which to write data
774 * @nr_segs: size of iov vector
Chuck Lever88467052006-03-20 13:44:34 -0500775 * @pos: byte offset in file where writing starts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 *
777 * We use this function for direct writes instead of calling
778 * generic_file_aio_write() in order to avoid taking the inode
779 * semaphore and updating the i_size. The NFS server will set
780 * the new i_size and this client must read the updated size
781 * back into its cache. We let the server do generic write
782 * parameter checking and report problems.
783 *
784 * We also avoid an unnecessary invocation of generic_osync_inode(),
785 * as it is fairly meaningless to sync the metadata of an NFS file.
786 *
787 * We eliminate local atime updates, see direct read above.
788 *
789 * We avoid unnecessary page cache invalidations for normal cached
790 * readers of this file.
791 *
792 * Note that O_APPEND is not supported for NFS direct writes, as there
793 * is no atomic O_APPEND write facility in the NFS protocol.
794 */
Badari Pulavarty027445c2006-09-30 23:28:46 -0700795ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
796 unsigned long nr_segs, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797{
Chuck Leverce1a8e62005-11-30 18:08:17 -0500798 ssize_t retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800 struct address_space *mapping = file->f_mapping;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700801 /* XXX: temporary */
802 const char __user *buf = iov[0].iov_base;
803 size_t count = iov[0].iov_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804
Chuck Levera3f565b2007-01-31 12:14:01 -0500805 dprintk("nfs: direct write(%s/%s, %lu@%Ld)\n",
Josef "Jeff" Sipek01cce932006-12-08 02:36:40 -0800806 file->f_path.dentry->d_parent->d_name.name,
807 file->f_path.dentry->d_name.name,
Chuck Leverce1a8e62005-11-30 18:08:17 -0500808 (unsigned long) count, (long long) pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809
Badari Pulavarty027445c2006-09-30 23:28:46 -0700810 if (nr_segs != 1)
811 return -EINVAL;
812
Chuck Leverce1a8e62005-11-30 18:08:17 -0500813 retval = generic_write_checks(file, &pos, &count, 0);
814 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500816
817 retval = -EINVAL;
818 if ((ssize_t) count < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 retval = 0;
821 if (!count)
822 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500823
824 retval = -EFAULT;
Chuck Lever47989d72006-03-20 13:44:32 -0500825 if (!access_ok(VERIFY_READ, buf, count))
Chuck Leverce1a8e62005-11-30 18:08:17 -0500826 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827
Trond Myklebust29884df2005-12-13 16:13:54 -0500828 retval = nfs_sync_mapping(mapping);
829 if (retval)
830 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831
Trond Myklebust607f31e2006-06-28 16:52:45 -0400832 retval = nfs_direct_write(iocb, (unsigned long) buf, count, pos);
Chuck Lever9eafa8c2006-03-20 13:44:33 -0500833
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 if (retval > 0)
Chuck Leverce1a8e62005-11-30 18:08:17 -0500835 iocb->ki_pos = pos + retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836
837out:
838 return retval;
839}
840
Chuck Lever88467052006-03-20 13:44:34 -0500841/**
842 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
843 *
844 */
David Howellsf7b422b2006-06-09 09:34:33 -0400845int __init nfs_init_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846{
847 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
848 sizeof(struct nfs_direct_req),
Paul Jacksonfffb60f2006-03-24 03:16:06 -0800849 0, (SLAB_RECLAIM_ACCOUNT|
850 SLAB_MEM_SPREAD),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 NULL, NULL);
852 if (nfs_direct_cachep == NULL)
853 return -ENOMEM;
854
855 return 0;
856}
857
Chuck Lever88467052006-03-20 13:44:34 -0500858/**
David Howellsf7b422b2006-06-09 09:34:33 -0400859 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
Chuck Lever88467052006-03-20 13:44:34 -0500860 *
861 */
David Brownell266bee82006-06-27 12:59:15 -0700862void nfs_destroy_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863{
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -0700864 kmem_cache_destroy(nfs_direct_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865}