blob: eb4f88e3dc97fe101fcfe340837496c54cf614b8 [file] [log] [blame]
Miklos Szeredi334f4852005-09-09 13:10:27 -07001/*
2 FUSE: Filesystem in Userspace
Miklos Szeredi1729a162008-11-26 12:03:54 +01003 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
Miklos Szeredi334f4852005-09-09 13:10:27 -07004
5 This program can be distributed under the terms of the GNU GPL.
6 See the file COPYING.
7*/
8
9#include "fuse_i.h"
10
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/poll.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010014#include <linux/sched/signal.h>
Miklos Szeredi334f4852005-09-09 13:10:27 -070015#include <linux/uio.h>
16#include <linux/miscdevice.h>
17#include <linux/pagemap.h>
18#include <linux/file.h>
19#include <linux/slab.h>
Miklos Szeredidd3bb142010-05-25 15:06:06 +020020#include <linux/pipe_fs_i.h>
Miklos Szeredice534fb2010-05-25 15:06:07 +020021#include <linux/swap.h>
22#include <linux/splice.h>
Seth Forshee0b6e9ea2014-07-02 16:29:19 -050023#include <linux/sched.h>
Miklos Szeredi334f4852005-09-09 13:10:27 -070024
25MODULE_ALIAS_MISCDEV(FUSE_MINOR);
Kay Sievers578454f2010-05-20 18:07:20 +020026MODULE_ALIAS("devname:fuse");
Miklos Szeredi334f4852005-09-09 13:10:27 -070027
Kirill Tkhaic59fd852018-09-11 13:11:56 +030028/* Ordinary requests have even IDs, while interrupts IDs are odd */
29#define FUSE_INT_REQ_BIT (1ULL << 0)
30#define FUSE_REQ_ID_STEP (1ULL << 1)
31
Christoph Lametere18b8902006-12-06 20:33:20 -080032static struct kmem_cache *fuse_req_cachep;
Miklos Szeredi334f4852005-09-09 13:10:27 -070033
Miklos Szeredicc080e92015-07-01 16:26:08 +020034static struct fuse_dev *fuse_get_dev(struct file *file)
Miklos Szeredi334f4852005-09-09 13:10:27 -070035{
Miklos Szeredi0720b312006-04-10 22:54:55 -070036 /*
37 * Lockless access is OK, because file->private data is set
38 * once during mount and is valid until the file is released.
39 */
Mark Rutland6aa7de02017-10-23 14:07:29 -070040 return READ_ONCE(file->private_data);
Miklos Szeredi334f4852005-09-09 13:10:27 -070041}
42
Max Reitzfcee2162020-05-06 17:44:12 +020043static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -070044{
Miklos Szeredi334f4852005-09-09 13:10:27 -070045 INIT_LIST_HEAD(&req->list);
Miklos Szeredia4d27e72006-06-25 05:48:54 -070046 INIT_LIST_HEAD(&req->intr_entry);
Miklos Szeredi334f4852005-09-09 13:10:27 -070047 init_waitqueue_head(&req->waitq);
Elena Reshetovaec99f6d2017-03-03 11:04:04 +020048 refcount_set(&req->count, 1);
Miklos Szeredi33e14b42015-07-01 16:26:01 +020049 __set_bit(FR_PENDING, &req->flags);
Max Reitzfcee2162020-05-06 17:44:12 +020050 req->fm = fm;
Miklos Szeredi334f4852005-09-09 13:10:27 -070051}
52
Max Reitzfcee2162020-05-06 17:44:12 +020053static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags)
Miklos Szeredi334f4852005-09-09 13:10:27 -070054{
Miklos Szeredi8a7aa282018-10-01 10:07:05 +020055 struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags);
Miklos Szeredi72133942019-09-10 15:04:11 +020056 if (req)
Max Reitzfcee2162020-05-06 17:44:12 +020057 fuse_request_init(fm, req);
Maxim Patlasov4250c062012-10-26 19:48:07 +040058
Miklos Szeredi334f4852005-09-09 13:10:27 -070059 return req;
60}
Maxim Patlasov4250c062012-10-26 19:48:07 +040061
Miklos Szeredi66abc352019-09-10 15:04:11 +020062static void fuse_request_free(struct fuse_req *req)
Miklos Szeredie52a82502018-10-01 10:07:06 +020063{
Miklos Szeredi334f4852005-09-09 13:10:27 -070064 kmem_cache_free(fuse_req_cachep, req);
65}
66
Miklos Szeredi66abc352019-09-10 15:04:11 +020067static void __fuse_get_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -070068{
Elena Reshetovaec99f6d2017-03-03 11:04:04 +020069 refcount_inc(&req->count);
Miklos Szeredi334f4852005-09-09 13:10:27 -070070}
71
72/* Must be called with > 1 refcount */
73static void __fuse_put_request(struct fuse_req *req)
74{
Elena Reshetovaec99f6d2017-03-03 11:04:04 +020075 refcount_dec(&req->count);
Miklos Szeredi334f4852005-09-09 13:10:27 -070076}
77
Miklos Szeredi9759bd512015-01-06 10:45:35 +010078void fuse_set_initialized(struct fuse_conn *fc)
79{
80 /* Make sure stores before this are seen on another CPU */
81 smp_wmb();
82 fc->initialized = 1;
83}
84
Maxim Patlasov0aada882013-03-21 18:02:28 +040085static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
86{
87 return !fc->initialized || (for_background && fc->blocked);
88}
89
Miklos Szeredib8f95e52018-07-26 16:13:11 +020090static void fuse_drop_waiting(struct fuse_conn *fc)
91{
Miklos Szeredi2d84a2d2018-11-09 15:52:16 +010092 /*
93 * lockess check of fc->connected is okay, because atomic_dec_and_test()
Zheng Yongjunc4e0cd42021-06-04 09:46:17 +080094 * provides a memory barrier matched with the one in fuse_wait_aborted()
Miklos Szeredi2d84a2d2018-11-09 15:52:16 +010095 * to ensure no wake-up is missed.
96 */
97 if (atomic_dec_and_test(&fc->num_waiting) &&
98 !READ_ONCE(fc->connected)) {
Miklos Szeredib8f95e52018-07-26 16:13:11 +020099 /* wake up aborters */
100 wake_up_all(&fc->blocked_waitq);
101 }
102}
103
Max Reitz8f622e92020-04-20 17:59:34 +0200104static void fuse_put_request(struct fuse_req *req);
Miklos Szeredi66abc352019-09-10 15:04:11 +0200105
Max Reitzfcee2162020-05-06 17:44:12 +0200106static struct fuse_req *fuse_get_req(struct fuse_mount *fm, bool for_background)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700107{
Max Reitzfcee2162020-05-06 17:44:12 +0200108 struct fuse_conn *fc = fm->fc;
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700109 struct fuse_req *req;
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700110 int err;
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200111 atomic_inc(&fc->num_waiting);
Maxim Patlasov0aada882013-03-21 18:02:28 +0400112
113 if (fuse_block_alloc(fc, for_background)) {
Maxim Patlasov0aada882013-03-21 18:02:28 +0400114 err = -EINTR;
Al Viro7d3a07f2016-07-19 03:08:27 -0400115 if (wait_event_killable_exclusive(fc->blocked_waitq,
116 !fuse_block_alloc(fc, for_background)))
Maxim Patlasov0aada882013-03-21 18:02:28 +0400117 goto out;
118 }
Miklos Szeredi9759bd512015-01-06 10:45:35 +0100119 /* Matches smp_wmb() in fuse_set_initialized() */
120 smp_rmb();
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700121
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700122 err = -ENOTCONN;
123 if (!fc->connected)
124 goto out;
125
Miklos Szeredide155222015-07-01 16:25:57 +0200126 err = -ECONNREFUSED;
127 if (fc->conn_error)
128 goto out;
129
Max Reitzfcee2162020-05-06 17:44:12 +0200130 req = fuse_request_alloc(fm, GFP_KERNEL);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200131 err = -ENOMEM;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400132 if (!req) {
133 if (for_background)
134 wake_up(&fc->blocked_waitq);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200135 goto out;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400136 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700137
Eric W. Biederman8cb08322018-02-21 11:18:07 -0600138 req->in.h.uid = from_kuid(fc->user_ns, current_fsuid());
139 req->in.h.gid = from_kgid(fc->user_ns, current_fsgid());
Eric W. Biedermanc9582eb2018-02-21 10:52:06 -0600140 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
141
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200142 __set_bit(FR_WAITING, &req->flags);
143 if (for_background)
144 __set_bit(FR_BACKGROUND, &req->flags);
145
Eric W. Biedermanc9582eb2018-02-21 10:52:06 -0600146 if (unlikely(req->in.h.uid == ((uid_t)-1) ||
147 req->in.h.gid == ((gid_t)-1))) {
Max Reitz8f622e92020-04-20 17:59:34 +0200148 fuse_put_request(req);
Eric W. Biedermanc9582eb2018-02-21 10:52:06 -0600149 return ERR_PTR(-EOVERFLOW);
150 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700151 return req;
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200152
153 out:
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200154 fuse_drop_waiting(fc);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200155 return ERR_PTR(err);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700156}
Maxim Patlasov8b41e672013-03-21 18:02:04 +0400157
Max Reitz8f622e92020-04-20 17:59:34 +0200158static void fuse_put_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700159{
Max Reitzfcee2162020-05-06 17:44:12 +0200160 struct fuse_conn *fc = req->fm->fc;
Max Reitz8f622e92020-04-20 17:59:34 +0200161
Elena Reshetovaec99f6d2017-03-03 11:04:04 +0200162 if (refcount_dec_and_test(&req->count)) {
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200163 if (test_bit(FR_BACKGROUND, &req->flags)) {
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400164 /*
165 * We get here in the unlikely case that a background
166 * request was allocated but not sent
167 */
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +0300168 spin_lock(&fc->bg_lock);
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400169 if (!fc->blocked)
170 wake_up(&fc->blocked_waitq);
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +0300171 spin_unlock(&fc->bg_lock);
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400172 }
173
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200174 if (test_bit(FR_WAITING, &req->flags)) {
175 __clear_bit(FR_WAITING, &req->flags);
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200176 fuse_drop_waiting(fc);
Miklos Szeredi73e0e732015-07-01 16:25:56 +0200177 }
Miklos Szeredi33649c92006-06-25 05:48:52 -0700178
Miklos Szeredi40ac7ab2019-09-10 15:04:08 +0200179 fuse_request_free(req);
Miklos Szeredi7128ec22006-02-04 23:27:40 -0800180 }
181}
182
Stefan Hajnoczi14d46d72018-06-21 09:34:25 +0100183unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args)
Miklos Szeredid12def12008-02-06 01:38:39 -0800184{
185 unsigned nbytes = 0;
186 unsigned i;
187
188 for (i = 0; i < numargs; i++)
189 nbytes += args[i].size;
190
191 return nbytes;
192}
Stefan Hajnoczi14d46d72018-06-21 09:34:25 +0100193EXPORT_SYMBOL_GPL(fuse_len_args);
Miklos Szeredid12def12008-02-06 01:38:39 -0800194
Stefan Hajnoczi79d96ef2018-06-22 13:48:30 +0100195u64 fuse_get_unique(struct fuse_iqueue *fiq)
Miklos Szeredid12def12008-02-06 01:38:39 -0800196{
Kirill Tkhaic59fd852018-09-11 13:11:56 +0300197 fiq->reqctr += FUSE_REQ_ID_STEP;
198 return fiq->reqctr;
Miklos Szeredid12def12008-02-06 01:38:39 -0800199}
Stefan Hajnoczi79d96ef2018-06-22 13:48:30 +0100200EXPORT_SYMBOL_GPL(fuse_get_unique);
Miklos Szeredid12def12008-02-06 01:38:39 -0800201
Kirill Tkhaibe2ff422018-09-11 13:12:14 +0300202static unsigned int fuse_req_hash(u64 unique)
203{
204 return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS);
205}
206
Randy Dunlap06bbb762023-01-08 17:00:23 -0800207/*
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100208 * A new request is available, wake fiq->waitq
209 */
210static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)
211__releases(fiq->lock)
212{
213 wake_up(&fiq->waitq);
214 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
215 spin_unlock(&fiq->lock);
216}
217
218const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
219 .wake_forget_and_unlock = fuse_dev_wake_and_unlock,
220 .wake_interrupt_and_unlock = fuse_dev_wake_and_unlock,
221 .wake_pending_and_unlock = fuse_dev_wake_and_unlock,
222};
223EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops);
224
225static void queue_request_and_unlock(struct fuse_iqueue *fiq,
226 struct fuse_req *req)
227__releases(fiq->lock)
Miklos Szeredid12def12008-02-06 01:38:39 -0800228{
Miklos Szeredid12def12008-02-06 01:38:39 -0800229 req->in.h.len = sizeof(struct fuse_in_header) +
Stefan Hajnoczi14d46d72018-06-21 09:34:25 +0100230 fuse_len_args(req->args->in_numargs,
231 (struct fuse_arg *) req->args->in_args);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200232 list_add_tail(&req->list, &fiq->pending);
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100233 fiq->ops->wake_pending_and_unlock(fiq);
Miklos Szeredid12def12008-02-06 01:38:39 -0800234}
235
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100236void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
237 u64 nodeid, u64 nlookup)
238{
Miklos Szeredif88996a2015-07-01 16:26:01 +0200239 struct fuse_iqueue *fiq = &fc->iq;
240
Miklos Szeredi02c048b2010-12-07 20:16:56 +0100241 forget->forget_one.nodeid = nodeid;
242 forget->forget_one.nlookup = nlookup;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100243
Eric Biggers76e43c82019-09-08 20:15:18 -0700244 spin_lock(&fiq->lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200245 if (fiq->connected) {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200246 fiq->forget_list_tail->next = forget;
247 fiq->forget_list_tail = forget;
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100248 fiq->ops->wake_forget_and_unlock(fiq);
Miklos Szeredi5dfcc872011-09-12 09:38:03 +0200249 } else {
250 kfree(forget);
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100251 spin_unlock(&fiq->lock);
Miklos Szeredi5dfcc872011-09-12 09:38:03 +0200252 }
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100253}
254
Miklos Szeredid12def12008-02-06 01:38:39 -0800255static void flush_bg_queue(struct fuse_conn *fc)
256{
Kirill Tkhaie2871792018-07-31 13:25:25 +0300257 struct fuse_iqueue *fiq = &fc->iq;
258
Csaba Henk7a6d3c82009-07-01 17:28:41 -0700259 while (fc->active_background < fc->max_background &&
Miklos Szeredid12def12008-02-06 01:38:39 -0800260 !list_empty(&fc->bg_queue)) {
261 struct fuse_req *req;
262
Kirill Tkhaie2871792018-07-31 13:25:25 +0300263 req = list_first_entry(&fc->bg_queue, struct fuse_req, list);
Miklos Szeredid12def12008-02-06 01:38:39 -0800264 list_del(&req->list);
265 fc->active_background++;
Eric Biggers76e43c82019-09-08 20:15:18 -0700266 spin_lock(&fiq->lock);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200267 req->in.h.unique = fuse_get_unique(fiq);
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100268 queue_request_and_unlock(fiq, req);
Miklos Szeredid12def12008-02-06 01:38:39 -0800269 }
270}
271
Miklos Szeredi6dbbcb12006-04-26 10:49:06 +0200272/*
Miklos Szeredi334f4852005-09-09 13:10:27 -0700273 * This function is called when a request is finished. Either a reply
Miklos Szeredif9a28422006-06-25 05:48:53 -0700274 * has arrived or it was aborted (and not yet sent) or some error
Miklos Szeredif43b1552006-01-16 22:14:26 -0800275 * occurred during communication with userspace, or the device file
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700276 * was closed. The requester thread is woken up (if still waiting),
277 * the 'end' callback is called if given, else the reference to the
278 * request is released
Miklos Szeredi334f4852005-09-09 13:10:27 -0700279 */
Max Reitz8f622e92020-04-20 17:59:34 +0200280void fuse_request_end(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700281{
Max Reitzfcee2162020-05-06 17:44:12 +0200282 struct fuse_mount *fm = req->fm;
283 struct fuse_conn *fc = fm->fc;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200284 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi365ae712015-07-01 16:26:06 +0200285
Miklos Szerediefe28002015-07-01 16:26:07 +0200286 if (test_and_set_bit(FR_FINISHED, &req->flags))
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200287 goto put_request;
Miklos Szeredi2b319d12019-10-21 09:11:40 +0200288
Kirill Tkhai217316a2018-11-08 12:05:25 +0300289 /*
290 * test_and_set_bit() implies smp_mb() between bit
Miklos Szeredie1e71c12021-08-04 13:22:58 +0200291 * changing and below FR_INTERRUPTED check. Pairs with
Kirill Tkhai217316a2018-11-08 12:05:25 +0300292 * smp_mb() from queue_interrupt().
293 */
Miklos Szeredie1e71c12021-08-04 13:22:58 +0200294 if (test_bit(FR_INTERRUPTED, &req->flags)) {
Eric Biggers76e43c82019-09-08 20:15:18 -0700295 spin_lock(&fiq->lock);
Kirill Tkhai217316a2018-11-08 12:05:25 +0300296 list_del_init(&req->intr_entry);
Eric Biggers76e43c82019-09-08 20:15:18 -0700297 spin_unlock(&fiq->lock);
Kirill Tkhai217316a2018-11-08 12:05:25 +0300298 }
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200299 WARN_ON(test_bit(FR_PENDING, &req->flags));
300 WARN_ON(test_bit(FR_SENT, &req->flags));
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200301 if (test_bit(FR_BACKGROUND, &req->flags)) {
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +0300302 spin_lock(&fc->bg_lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200303 clear_bit(FR_BACKGROUND, &req->flags);
Miklos Szeredi908a5722018-09-28 16:43:22 +0200304 if (fc->num_background == fc->max_background) {
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700305 fc->blocked = 0;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400306 wake_up(&fc->blocked_waitq);
Miklos Szeredi908a5722018-09-28 16:43:22 +0200307 } else if (!fc->blocked) {
308 /*
309 * Wake up next waiter, if any. It's okay to use
310 * waitqueue_active(), as we've already synced up
311 * fc->blocked with waiters with the wake_up() call
312 * above.
313 */
314 if (waitqueue_active(&fc->blocked_waitq))
315 wake_up(&fc->blocked_waitq);
316 }
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400317
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700318 fc->num_background--;
Miklos Szeredid12def12008-02-06 01:38:39 -0800319 fc->active_background--;
320 flush_bg_queue(fc);
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +0300321 spin_unlock(&fc->bg_lock);
Kirill Tkhai5e0fed72018-11-08 12:05:31 +0300322 } else {
323 /* Wake up waiter sleeping in request_wait_answer() */
324 wake_up(&req->waitq);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700325 }
Kirill Tkhai5e0fed72018-11-08 12:05:31 +0300326
Miklos Szeredi3e8cb8b2020-02-13 09:16:07 +0100327 if (test_bit(FR_ASYNC, &req->flags))
Max Reitzfcee2162020-05-06 17:44:12 +0200328 req->args->end(fm, req->args, req->out.h.error);
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200329put_request:
Max Reitz8f622e92020-04-20 17:59:34 +0200330 fuse_put_request(req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700331}
Stefan Hajnoczi04ec5af2018-06-21 09:33:40 +0100332EXPORT_SYMBOL_GPL(fuse_request_end);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700333
Max Reitz8f622e92020-04-20 17:59:34 +0200334static int queue_interrupt(struct fuse_req *req)
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700335{
Max Reitzfcee2162020-05-06 17:44:12 +0200336 struct fuse_iqueue *fiq = &req->fm->fc->iq;
Max Reitz8f622e92020-04-20 17:59:34 +0200337
Eric Biggers76e43c82019-09-08 20:15:18 -0700338 spin_lock(&fiq->lock);
Kirill Tkhaib7829112018-11-08 12:05:42 +0300339 /* Check for we've sent request to interrupt this req */
340 if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) {
Eric Biggers76e43c82019-09-08 20:15:18 -0700341 spin_unlock(&fiq->lock);
Kirill Tkhaib7829112018-11-08 12:05:42 +0300342 return -EINVAL;
343 }
344
Miklos Szeredi8f7bb362015-07-01 16:26:03 +0200345 if (list_empty(&req->intr_entry)) {
346 list_add_tail(&req->intr_entry, &fiq->interrupts);
Kirill Tkhai217316a2018-11-08 12:05:25 +0300347 /*
348 * Pairs with smp_mb() implied by test_and_set_bit()
Kirill Tkhai75d89252020-02-28 15:15:24 +0300349 * from fuse_request_end().
Kirill Tkhai217316a2018-11-08 12:05:25 +0300350 */
351 smp_mb();
352 if (test_bit(FR_FINISHED, &req->flags)) {
353 list_del_init(&req->intr_entry);
Eric Biggers76e43c82019-09-08 20:15:18 -0700354 spin_unlock(&fiq->lock);
Kirill Tkhaib7829112018-11-08 12:05:42 +0300355 return 0;
Kirill Tkhai217316a2018-11-08 12:05:25 +0300356 }
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100357 fiq->ops->wake_interrupt_and_unlock(fiq);
358 } else {
359 spin_unlock(&fiq->lock);
Miklos Szeredi8f7bb362015-07-01 16:26:03 +0200360 }
Kirill Tkhaib7829112018-11-08 12:05:42 +0300361 return 0;
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700362}
363
Max Reitz8f622e92020-04-20 17:59:34 +0200364static void request_wait_answer(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700365{
Max Reitzfcee2162020-05-06 17:44:12 +0200366 struct fuse_conn *fc = req->fm->fc;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200367 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredic4775262015-07-01 16:26:00 +0200368 int err;
369
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700370 if (!fc->no_interrupt) {
371 /* Any signal may interrupt this */
Miklos Szeredic4775262015-07-01 16:26:00 +0200372 err = wait_event_interruptible(req->waitq,
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200373 test_bit(FR_FINISHED, &req->flags));
Miklos Szeredic4775262015-07-01 16:26:00 +0200374 if (!err)
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700375 return;
376
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200377 set_bit(FR_INTERRUPTED, &req->flags);
Miklos Szeredi8f7bb362015-07-01 16:26:03 +0200378 /* matches barrier in fuse_dev_do_read() */
379 smp_mb__after_atomic();
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200380 if (test_bit(FR_SENT, &req->flags))
Max Reitz8f622e92020-04-20 17:59:34 +0200381 queue_interrupt(req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700382 }
383
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200384 if (!test_bit(FR_FORCE, &req->flags)) {
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700385 /* Only fatal signals may interrupt this */
Al Viro7d3a07f2016-07-19 03:08:27 -0400386 err = wait_event_killable(req->waitq,
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200387 test_bit(FR_FINISHED, &req->flags));
Miklos Szeredic4775262015-07-01 16:26:00 +0200388 if (!err)
Miklos Szeredia131de02007-10-16 23:31:04 -0700389 return;
390
Eric Biggers76e43c82019-09-08 20:15:18 -0700391 spin_lock(&fiq->lock);
Miklos Szeredia131de02007-10-16 23:31:04 -0700392 /* Request is not yet in userspace, bail out */
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200393 if (test_bit(FR_PENDING, &req->flags)) {
Miklos Szeredia131de02007-10-16 23:31:04 -0700394 list_del(&req->list);
Eric Biggers76e43c82019-09-08 20:15:18 -0700395 spin_unlock(&fiq->lock);
Miklos Szeredia131de02007-10-16 23:31:04 -0700396 __fuse_put_request(req);
397 req->out.h.error = -EINTR;
398 return;
399 }
Eric Biggers76e43c82019-09-08 20:15:18 -0700400 spin_unlock(&fiq->lock);
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700401 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700402
Miklos Szeredia131de02007-10-16 23:31:04 -0700403 /*
404 * Either request is already in userspace, or it was forced.
405 * Wait it out.
406 */
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200407 wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
Miklos Szeredi334f4852005-09-09 13:10:27 -0700408}
409
Max Reitz8f622e92020-04-20 17:59:34 +0200410static void __fuse_request_send(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700411{
Max Reitzfcee2162020-05-06 17:44:12 +0200412 struct fuse_iqueue *fiq = &req->fm->fc->iq;
Miklos Szeredie16714d2015-07-01 16:26:01 +0200413
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200414 BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
Eric Biggers76e43c82019-09-08 20:15:18 -0700415 spin_lock(&fiq->lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200416 if (!fiq->connected) {
Eric Biggers76e43c82019-09-08 20:15:18 -0700417 spin_unlock(&fiq->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700418 req->out.h.error = -ENOTCONN;
Miklos Szeredic4775262015-07-01 16:26:00 +0200419 } else {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200420 req->in.h.unique = fuse_get_unique(fiq);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700421 /* acquire extra reference, since request is still needed
Stefan Hajnoczi04ec5af2018-06-21 09:33:40 +0100422 after fuse_request_end() */
Miklos Szeredi334f4852005-09-09 13:10:27 -0700423 __fuse_get_request(req);
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100424 queue_request_and_unlock(fiq, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700425
Max Reitz8f622e92020-04-20 17:59:34 +0200426 request_wait_answer(req);
Stefan Hajnoczi04ec5af2018-06-21 09:33:40 +0100427 /* Pairs with smp_wmb() in fuse_request_end() */
Miklos Szeredic4775262015-07-01 16:26:00 +0200428 smp_rmb();
Miklos Szeredi334f4852005-09-09 13:10:27 -0700429 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700430}
Eric Wong6a4e9222013-02-04 13:04:44 +0000431
Miklos Szeredi21f62172015-01-06 10:45:35 +0100432static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
433{
Miklos Szeredid5b48542019-09-10 15:04:08 +0200434 if (fc->minor < 4 && args->opcode == FUSE_STATFS)
435 args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE;
Miklos Szeredi21f62172015-01-06 10:45:35 +0100436
437 if (fc->minor < 9) {
Miklos Szeredid5b48542019-09-10 15:04:08 +0200438 switch (args->opcode) {
Miklos Szeredi21f62172015-01-06 10:45:35 +0100439 case FUSE_LOOKUP:
440 case FUSE_CREATE:
441 case FUSE_MKNOD:
442 case FUSE_MKDIR:
443 case FUSE_SYMLINK:
444 case FUSE_LINK:
Miklos Szeredid5b48542019-09-10 15:04:08 +0200445 args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
Miklos Szeredi21f62172015-01-06 10:45:35 +0100446 break;
447 case FUSE_GETATTR:
448 case FUSE_SETATTR:
Miklos Szeredid5b48542019-09-10 15:04:08 +0200449 args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
Miklos Szeredi21f62172015-01-06 10:45:35 +0100450 break;
451 }
452 }
453 if (fc->minor < 12) {
Miklos Szeredid5b48542019-09-10 15:04:08 +0200454 switch (args->opcode) {
Miklos Szeredi21f62172015-01-06 10:45:35 +0100455 case FUSE_CREATE:
Miklos Szeredid5b48542019-09-10 15:04:08 +0200456 args->in_args[0].size = sizeof(struct fuse_open_in);
Miklos Szeredi21f62172015-01-06 10:45:35 +0100457 break;
458 case FUSE_MKNOD:
Miklos Szeredid5b48542019-09-10 15:04:08 +0200459 args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
Miklos Szeredi21f62172015-01-06 10:45:35 +0100460 break;
461 }
462 }
463}
464
Max Reitz8f622e92020-04-20 17:59:34 +0200465static void fuse_force_creds(struct fuse_req *req)
Miklos Szeredie4137542019-09-10 15:04:08 +0200466{
Max Reitzfcee2162020-05-06 17:44:12 +0200467 struct fuse_conn *fc = req->fm->fc;
Max Reitz8f622e92020-04-20 17:59:34 +0200468
Miklos Szeredie4137542019-09-10 15:04:08 +0200469 req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
470 req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
471 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
472}
473
YueHaibing5addcd52019-09-23 13:52:31 +0800474static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
Miklos Szeredi68583162019-09-10 15:04:09 +0200475{
Miklos Szeredi68583162019-09-10 15:04:09 +0200476 req->in.h.opcode = args->opcode;
477 req->in.h.nodeid = args->nodeid;
Miklos Szeredid4993772019-09-10 15:04:11 +0200478 req->args = args;
Miklos Szeredi15d937d2022-11-10 15:46:33 +0100479 if (args->is_ext)
480 req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8;
Miklos Szeredi3e8cb8b2020-02-13 09:16:07 +0100481 if (args->end)
482 __set_bit(FR_ASYNC, &req->flags);
Miklos Szeredi68583162019-09-10 15:04:09 +0200483}
484
Max Reitzfcee2162020-05-06 17:44:12 +0200485ssize_t fuse_simple_request(struct fuse_mount *fm, struct fuse_args *args)
Miklos Szeredi70781872014-12-12 09:49:05 +0100486{
Max Reitzfcee2162020-05-06 17:44:12 +0200487 struct fuse_conn *fc = fm->fc;
Miklos Szeredi70781872014-12-12 09:49:05 +0100488 struct fuse_req *req;
489 ssize_t ret;
490
Miklos Szeredic500eba2019-09-10 15:04:08 +0200491 if (args->force) {
Miklos Szeredie4137542019-09-10 15:04:08 +0200492 atomic_inc(&fc->num_waiting);
Max Reitzfcee2162020-05-06 17:44:12 +0200493 req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL);
Miklos Szeredie4137542019-09-10 15:04:08 +0200494
495 if (!args->nocreds)
Max Reitz8f622e92020-04-20 17:59:34 +0200496 fuse_force_creds(req);
Miklos Szeredie4137542019-09-10 15:04:08 +0200497
498 __set_bit(FR_WAITING, &req->flags);
Miklos Szeredic500eba2019-09-10 15:04:08 +0200499 __set_bit(FR_FORCE, &req->flags);
500 } else {
Miklos Szeredie4137542019-09-10 15:04:08 +0200501 WARN_ON(args->nocreds);
Max Reitzfcee2162020-05-06 17:44:12 +0200502 req = fuse_get_req(fm, false);
Miklos Szeredic500eba2019-09-10 15:04:08 +0200503 if (IS_ERR(req))
504 return PTR_ERR(req);
505 }
Miklos Szeredi70781872014-12-12 09:49:05 +0100506
Miklos Szeredi21f62172015-01-06 10:45:35 +0100507 /* Needs to be done after fuse_get_req() so that fc->minor is valid */
508 fuse_adjust_compat(fc, args);
Miklos Szeredi68583162019-09-10 15:04:09 +0200509 fuse_args_to_req(req, args);
Miklos Szeredi21f62172015-01-06 10:45:35 +0100510
Miklos Szeredi454a7612019-09-10 15:04:08 +0200511 if (!args->noreply)
512 __set_bit(FR_ISREPLY, &req->flags);
Max Reitz8f622e92020-04-20 17:59:34 +0200513 __fuse_request_send(req);
Miklos Szeredi70781872014-12-12 09:49:05 +0100514 ret = req->out.h.error;
Miklos Szeredid5b48542019-09-10 15:04:08 +0200515 if (!ret && args->out_argvar) {
Miklos Szeredi093f38a2019-09-10 15:04:09 +0200516 BUG_ON(args->out_numargs == 0);
Miklos Szeredid4993772019-09-10 15:04:11 +0200517 ret = args->out_args[args->out_numargs - 1].size;
Miklos Szeredi70781872014-12-12 09:49:05 +0100518 }
Max Reitz8f622e92020-04-20 17:59:34 +0200519 fuse_put_request(req);
Miklos Szeredi70781872014-12-12 09:49:05 +0100520
521 return ret;
522}
523
Max Reitz8f622e92020-04-20 17:59:34 +0200524static bool fuse_request_queue_background(struct fuse_req *req)
Miklos Szeredid12def12008-02-06 01:38:39 -0800525{
Max Reitzfcee2162020-05-06 17:44:12 +0200526 struct fuse_mount *fm = req->fm;
527 struct fuse_conn *fc = fm->fc;
Kirill Tkhai63825b42018-08-27 18:29:56 +0300528 bool queued = false;
529
530 WARN_ON(!test_bit(FR_BACKGROUND, &req->flags));
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200531 if (!test_bit(FR_WAITING, &req->flags)) {
532 __set_bit(FR_WAITING, &req->flags);
Miklos Szeredi5437f242015-07-01 16:25:56 +0200533 atomic_inc(&fc->num_waiting);
534 }
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200535 __set_bit(FR_ISREPLY, &req->flags);
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +0300536 spin_lock(&fc->bg_lock);
Kirill Tkhai63825b42018-08-27 18:29:56 +0300537 if (likely(fc->connected)) {
538 fc->num_background++;
539 if (fc->num_background == fc->max_background)
540 fc->blocked = 1;
Kirill Tkhai63825b42018-08-27 18:29:56 +0300541 list_add_tail(&req->list, &fc->bg_queue);
542 flush_bg_queue(fc);
543 queued = true;
Miklos Szeredid12def12008-02-06 01:38:39 -0800544 }
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +0300545 spin_unlock(&fc->bg_lock);
Kirill Tkhai63825b42018-08-27 18:29:56 +0300546
547 return queued;
Miklos Szeredid12def12008-02-06 01:38:39 -0800548}
549
Max Reitzfcee2162020-05-06 17:44:12 +0200550int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
Miklos Szeredi12597282019-09-10 15:04:10 +0200551 gfp_t gfp_flags)
552{
553 struct fuse_req *req;
554
555 if (args->force) {
556 WARN_ON(!args->nocreds);
Max Reitzfcee2162020-05-06 17:44:12 +0200557 req = fuse_request_alloc(fm, gfp_flags);
Miklos Szeredi12597282019-09-10 15:04:10 +0200558 if (!req)
559 return -ENOMEM;
560 __set_bit(FR_BACKGROUND, &req->flags);
561 } else {
562 WARN_ON(args->nocreds);
Max Reitzfcee2162020-05-06 17:44:12 +0200563 req = fuse_get_req(fm, true);
Miklos Szeredi12597282019-09-10 15:04:10 +0200564 if (IS_ERR(req))
565 return PTR_ERR(req);
566 }
567
568 fuse_args_to_req(req, args);
569
Max Reitz8f622e92020-04-20 17:59:34 +0200570 if (!fuse_request_queue_background(req)) {
571 fuse_put_request(req);
Miklos Szeredi12597282019-09-10 15:04:10 +0200572 return -ENOTCONN;
573 }
574
575 return 0;
576}
577EXPORT_SYMBOL_GPL(fuse_simple_background);
578
Max Reitzfcee2162020-05-06 17:44:12 +0200579static int fuse_simple_notify_reply(struct fuse_mount *fm,
Miklos Szeredi75b399d2019-09-10 15:04:11 +0200580 struct fuse_args *args, u64 unique)
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200581{
Miklos Szeredi75b399d2019-09-10 15:04:11 +0200582 struct fuse_req *req;
Max Reitzfcee2162020-05-06 17:44:12 +0200583 struct fuse_iqueue *fiq = &fm->fc->iq;
Miklos Szeredi75b399d2019-09-10 15:04:11 +0200584 int err = 0;
585
Max Reitzfcee2162020-05-06 17:44:12 +0200586 req = fuse_get_req(fm, false);
Miklos Szeredi75b399d2019-09-10 15:04:11 +0200587 if (IS_ERR(req))
588 return PTR_ERR(req);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200589
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200590 __clear_bit(FR_ISREPLY, &req->flags);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200591 req->in.h.unique = unique;
Miklos Szeredi75b399d2019-09-10 15:04:11 +0200592
593 fuse_args_to_req(req, args);
Miklos Szeredi75b399d2019-09-10 15:04:11 +0200594
Eric Biggers76e43c82019-09-08 20:15:18 -0700595 spin_lock(&fiq->lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200596 if (fiq->connected) {
Stefan Hajnocziae3aad72018-06-18 15:53:19 +0100597 queue_request_and_unlock(fiq, req);
Miklos Szeredi75b399d2019-09-10 15:04:11 +0200598 } else {
599 err = -ENODEV;
600 spin_unlock(&fiq->lock);
Max Reitz8f622e92020-04-20 17:59:34 +0200601 fuse_put_request(req);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200602 }
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200603
604 return err;
605}
606
Miklos Szeredi3be5a522008-04-30 00:54:41 -0700607/*
Miklos Szeredi334f4852005-09-09 13:10:27 -0700608 * Lock the request. Up to the next unlock_request() there mustn't be
609 * anything that could cause a page-fault. If the request was already
Miklos Szeredif9a28422006-06-25 05:48:53 -0700610 * aborted bail out.
Miklos Szeredi334f4852005-09-09 13:10:27 -0700611 */
Miklos Szeredidc008092015-07-01 16:25:58 +0200612static int lock_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700613{
614 int err = 0;
615 if (req) {
Miklos Szeredidc008092015-07-01 16:25:58 +0200616 spin_lock(&req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200617 if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi334f4852005-09-09 13:10:27 -0700618 err = -ENOENT;
619 else
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200620 set_bit(FR_LOCKED, &req->flags);
Miklos Szeredidc008092015-07-01 16:25:58 +0200621 spin_unlock(&req->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700622 }
623 return err;
624}
625
626/*
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200627 * Unlock request. If it was aborted while locked, caller is responsible
628 * for unlocking and ending the request.
Miklos Szeredi334f4852005-09-09 13:10:27 -0700629 */
Miklos Szeredidc008092015-07-01 16:25:58 +0200630static int unlock_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700631{
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200632 int err = 0;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700633 if (req) {
Miklos Szeredidc008092015-07-01 16:25:58 +0200634 spin_lock(&req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200635 if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200636 err = -ENOENT;
637 else
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200638 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredidc008092015-07-01 16:25:58 +0200639 spin_unlock(&req->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700640 }
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200641 return err;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700642}
643
644struct fuse_copy_state {
645 int write;
646 struct fuse_req *req;
Al Viro6c09e942015-04-03 22:06:08 -0400647 struct iov_iter *iter;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200648 struct pipe_buffer *pipebufs;
649 struct pipe_buffer *currbuf;
650 struct pipe_inode_info *pipe;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700651 unsigned long nr_segs;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700652 struct page *pg;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700653 unsigned len;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200654 unsigned offset;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200655 unsigned move_pages:1;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700656};
657
Miklos Szeredidc008092015-07-01 16:25:58 +0200658static void fuse_copy_init(struct fuse_copy_state *cs, int write,
Al Viro6c09e942015-04-03 22:06:08 -0400659 struct iov_iter *iter)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700660{
661 memset(cs, 0, sizeof(*cs));
662 cs->write = write;
Al Viro6c09e942015-04-03 22:06:08 -0400663 cs->iter = iter;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700664}
665
666/* Unmap and put previous page of userspace buffer */
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800667static void fuse_copy_finish(struct fuse_copy_state *cs)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700668{
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200669 if (cs->currbuf) {
670 struct pipe_buffer *buf = cs->currbuf;
671
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200672 if (cs->write)
Miklos Szeredic3021622010-05-25 15:06:07 +0200673 buf->len = PAGE_SIZE - cs->len;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200674 cs->currbuf = NULL;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200675 } else if (cs->pg) {
Miklos Szeredi334f4852005-09-09 13:10:27 -0700676 if (cs->write) {
677 flush_dcache_page(cs->pg);
678 set_page_dirty_lock(cs->pg);
679 }
680 put_page(cs->pg);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700681 }
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200682 cs->pg = NULL;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700683}
684
685/*
686 * Get another pagefull of userspace buffer, and map it to kernel
687 * address space, and lock request
688 */
689static int fuse_copy_fill(struct fuse_copy_state *cs)
690{
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200691 struct page *page;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700692 int err;
693
Miklos Szeredidc008092015-07-01 16:25:58 +0200694 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200695 if (err)
696 return err;
697
Miklos Szeredi334f4852005-09-09 13:10:27 -0700698 fuse_copy_finish(cs);
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200699 if (cs->pipebufs) {
700 struct pipe_buffer *buf = cs->pipebufs;
701
Miklos Szeredic3021622010-05-25 15:06:07 +0200702 if (!cs->write) {
Miklos Szeredifba597d2016-09-27 10:45:12 +0200703 err = pipe_buf_confirm(cs->pipe, buf);
Miklos Szeredic3021622010-05-25 15:06:07 +0200704 if (err)
705 return err;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200706
Miklos Szeredic3021622010-05-25 15:06:07 +0200707 BUG_ON(!cs->nr_segs);
708 cs->currbuf = buf;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200709 cs->pg = buf->page;
710 cs->offset = buf->offset;
Miklos Szeredic3021622010-05-25 15:06:07 +0200711 cs->len = buf->len;
Miklos Szeredic3021622010-05-25 15:06:07 +0200712 cs->pipebufs++;
713 cs->nr_segs--;
714 } else {
David Howells6718b6f2019-10-16 16:47:32 +0100715 if (cs->nr_segs >= cs->pipe->max_usage)
Miklos Szeredic3021622010-05-25 15:06:07 +0200716 return -EIO;
717
718 page = alloc_page(GFP_HIGHUSER);
719 if (!page)
720 return -ENOMEM;
721
722 buf->page = page;
723 buf->offset = 0;
724 buf->len = 0;
725
726 cs->currbuf = buf;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200727 cs->pg = page;
728 cs->offset = 0;
Miklos Szeredic3021622010-05-25 15:06:07 +0200729 cs->len = PAGE_SIZE;
730 cs->pipebufs++;
731 cs->nr_segs++;
732 }
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200733 } else {
Al Viro6c09e942015-04-03 22:06:08 -0400734 size_t off;
Al Viro1ef255e2022-06-09 10:28:36 -0400735 err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off);
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200736 if (err < 0)
737 return err;
Al Viro6c09e942015-04-03 22:06:08 -0400738 BUG_ON(!err);
739 cs->len = err;
740 cs->offset = off;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200741 cs->pg = page;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700742 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700743
Miklos Szeredidc008092015-07-01 16:25:58 +0200744 return lock_request(cs->req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700745}
746
747/* Do as much copy to/from userspace buffer as we can */
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800748static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700749{
750 unsigned ncpy = min(*size, cs->len);
751 if (val) {
Peng Hao5fe0fc92021-09-08 16:38:28 +0800752 void *pgaddr = kmap_local_page(cs->pg);
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200753 void *buf = pgaddr + cs->offset;
754
Miklos Szeredi334f4852005-09-09 13:10:27 -0700755 if (cs->write)
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200756 memcpy(buf, *val, ncpy);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700757 else
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200758 memcpy(*val, buf, ncpy);
759
Peng Hao5fe0fc92021-09-08 16:38:28 +0800760 kunmap_local(pgaddr);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700761 *val += ncpy;
762 }
763 *size -= ncpy;
764 cs->len -= ncpy;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200765 cs->offset += ncpy;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700766 return ncpy;
767}
768
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700769static int fuse_check_folio(struct folio *folio)
Miklos Szeredice534fb2010-05-25 15:06:07 +0200770{
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700771 if (folio_mapped(folio) ||
772 folio->mapping != NULL ||
773 (folio->flags & PAGE_FLAGS_CHECK_AT_PREP &
Miklos Szeredice534fb2010-05-25 15:06:07 +0200774 ~(1 << PG_locked |
775 1 << PG_referenced |
776 1 << PG_uptodate |
777 1 << PG_lru |
778 1 << PG_active |
Miklos Szeredib89ecd602021-06-18 21:16:42 +0200779 1 << PG_workingset |
Miklos Szeredia5005c32020-05-19 14:50:37 +0200780 1 << PG_reclaim |
Yu Zhaoec1c86b2022-09-18 02:00:02 -0600781 1 << PG_waiters |
782 LRU_GEN_MASK | LRU_REFS_MASK))) {
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700783 dump_page(&folio->page, "fuse: trying to steal weird page");
Miklos Szeredice534fb2010-05-25 15:06:07 +0200784 return 1;
785 }
786 return 0;
787}
788
789static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
790{
791 int err;
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700792 struct folio *oldfolio = page_folio(*pagep);
793 struct folio *newfolio;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200794 struct pipe_buffer *buf = cs->pipebufs;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200795
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700796 folio_get(oldfolio);
Miklos Szeredidc008092015-07-01 16:25:58 +0200797 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200798 if (err)
Miklos Szeredid78092e2020-09-18 10:36:50 +0200799 goto out_put_old;
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200800
Miklos Szeredice534fb2010-05-25 15:06:07 +0200801 fuse_copy_finish(cs);
802
Miklos Szeredifba597d2016-09-27 10:45:12 +0200803 err = pipe_buf_confirm(cs->pipe, buf);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200804 if (err)
Miklos Szeredid78092e2020-09-18 10:36:50 +0200805 goto out_put_old;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200806
807 BUG_ON(!cs->nr_segs);
808 cs->currbuf = buf;
809 cs->len = buf->len;
810 cs->pipebufs++;
811 cs->nr_segs--;
812
813 if (cs->len != PAGE_SIZE)
814 goto out_fallback;
815
Christoph Hellwigc928f642020-05-20 17:58:16 +0200816 if (!pipe_buf_try_steal(cs->pipe, buf))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200817 goto out_fallback;
818
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700819 newfolio = page_folio(buf->page);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200820
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700821 if (!folio_test_uptodate(newfolio))
822 folio_mark_uptodate(newfolio);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200823
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700824 folio_clear_mappedtodisk(newfolio);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200825
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700826 if (fuse_check_folio(newfolio) != 0)
Miklos Szeredice534fb2010-05-25 15:06:07 +0200827 goto out_fallback_unlock;
828
Miklos Szeredice534fb2010-05-25 15:06:07 +0200829 /*
830 * This is a new and locked page, it shouldn't be mapped or
831 * have any special flags on it
832 */
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700833 if (WARN_ON(folio_mapped(oldfolio)))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200834 goto out_fallback_unlock;
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700835 if (WARN_ON(folio_has_private(oldfolio)))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200836 goto out_fallback_unlock;
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700837 if (WARN_ON(folio_test_dirty(oldfolio) ||
838 folio_test_writeback(oldfolio)))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200839 goto out_fallback_unlock;
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700840 if (WARN_ON(folio_test_mlocked(oldfolio)))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200841 goto out_fallback_unlock;
842
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700843 replace_page_cache_folio(oldfolio, newfolio);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700844
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700845 folio_get(newfolio);
Miklos Szeredi47344172021-11-25 14:05:18 +0100846
847 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700848 folio_add_lru(newfolio);
Miklos Szeredi47344172021-11-25 14:05:18 +0100849
Miklos Szeredi712a9512021-11-02 11:10:37 +0100850 /*
851 * Release while we have extra ref on stolen page. Otherwise
852 * anon_pipe_buf_release() might think the page can be reused.
853 */
854 pipe_buf_release(cs->pipe, buf);
855
Miklos Szeredice534fb2010-05-25 15:06:07 +0200856 err = 0;
Miklos Szeredidc008092015-07-01 16:25:58 +0200857 spin_lock(&cs->req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200858 if (test_bit(FR_ABORTED, &cs->req->flags))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200859 err = -ENOENT;
860 else
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700861 *pagep = &newfolio->page;
Miklos Szeredidc008092015-07-01 16:25:58 +0200862 spin_unlock(&cs->req->waitq.lock);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200863
864 if (err) {
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700865 folio_unlock(newfolio);
866 folio_put(newfolio);
Miklos Szeredid78092e2020-09-18 10:36:50 +0200867 goto out_put_old;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200868 }
869
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700870 folio_unlock(oldfolio);
Miklos Szeredid78092e2020-09-18 10:36:50 +0200871 /* Drop ref for ap->pages[] array */
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700872 folio_put(oldfolio);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200873 cs->len = 0;
874
Miklos Szeredid78092e2020-09-18 10:36:50 +0200875 err = 0;
876out_put_old:
877 /* Drop ref obtained in this function */
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700878 folio_put(oldfolio);
Miklos Szeredid78092e2020-09-18 10:36:50 +0200879 return err;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200880
881out_fallback_unlock:
Vishal Moola (Oracle)063aaad2022-11-01 10:53:23 -0700882 folio_unlock(newfolio);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200883out_fallback:
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200884 cs->pg = buf->page;
885 cs->offset = buf->offset;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200886
Miklos Szeredidc008092015-07-01 16:25:58 +0200887 err = lock_request(cs->req);
Miklos Szeredid78092e2020-09-18 10:36:50 +0200888 if (!err)
889 err = 1;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200890
Miklos Szeredid78092e2020-09-18 10:36:50 +0200891 goto out_put_old;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200892}
893
Miklos Szeredic3021622010-05-25 15:06:07 +0200894static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
895 unsigned offset, unsigned count)
896{
897 struct pipe_buffer *buf;
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200898 int err;
Miklos Szeredic3021622010-05-25 15:06:07 +0200899
David Howells6718b6f2019-10-16 16:47:32 +0100900 if (cs->nr_segs >= cs->pipe->max_usage)
Miklos Szeredic3021622010-05-25 15:06:07 +0200901 return -EIO;
902
Miklos Szeredid78092e2020-09-18 10:36:50 +0200903 get_page(page);
Miklos Szeredidc008092015-07-01 16:25:58 +0200904 err = unlock_request(cs->req);
Miklos Szeredid78092e2020-09-18 10:36:50 +0200905 if (err) {
906 put_page(page);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200907 return err;
Miklos Szeredid78092e2020-09-18 10:36:50 +0200908 }
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200909
Miklos Szeredic3021622010-05-25 15:06:07 +0200910 fuse_copy_finish(cs);
911
912 buf = cs->pipebufs;
Miklos Szeredic3021622010-05-25 15:06:07 +0200913 buf->page = page;
914 buf->offset = offset;
915 buf->len = count;
916
917 cs->pipebufs++;
918 cs->nr_segs++;
919 cs->len = 0;
920
921 return 0;
922}
923
Miklos Szeredi334f4852005-09-09 13:10:27 -0700924/*
925 * Copy a page in the request to/from the userspace buffer. Must be
926 * done atomically
927 */
Miklos Szeredice534fb2010-05-25 15:06:07 +0200928static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800929 unsigned offset, unsigned count, int zeroing)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700930{
Miklos Szeredice534fb2010-05-25 15:06:07 +0200931 int err;
932 struct page *page = *pagep;
933
Miklos Szeredib6777c42010-10-26 14:22:27 -0700934 if (page && zeroing && count < PAGE_SIZE)
935 clear_highpage(page);
936
Miklos Szeredi334f4852005-09-09 13:10:27 -0700937 while (count) {
Miklos Szeredic3021622010-05-25 15:06:07 +0200938 if (cs->write && cs->pipebufs && page) {
Miklos Szeredi0c4bcfd2022-03-07 16:30:44 +0100939 /*
940 * Can't control lifetime of pipe buffers, so always
941 * copy user pages.
942 */
943 if (cs->req->args->user_pages) {
944 err = fuse_copy_fill(cs);
945 if (err)
946 return err;
947 } else {
948 return fuse_ref_page(cs, page, offset, count);
949 }
Miklos Szeredic3021622010-05-25 15:06:07 +0200950 } else if (!cs->len) {
Miklos Szeredice534fb2010-05-25 15:06:07 +0200951 if (cs->move_pages && page &&
952 offset == 0 && count == PAGE_SIZE) {
953 err = fuse_try_move_page(cs, pagep);
954 if (err <= 0)
955 return err;
956 } else {
957 err = fuse_copy_fill(cs);
958 if (err)
959 return err;
960 }
Miklos Szeredi1729a162008-11-26 12:03:54 +0100961 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700962 if (page) {
Peng Hao5fe0fc92021-09-08 16:38:28 +0800963 void *mapaddr = kmap_local_page(page);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700964 void *buf = mapaddr + offset;
965 offset += fuse_copy_do(cs, &buf, &count);
Peng Hao5fe0fc92021-09-08 16:38:28 +0800966 kunmap_local(mapaddr);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700967 } else
968 offset += fuse_copy_do(cs, NULL, &count);
969 }
970 if (page && !cs->write)
971 flush_dcache_page(page);
972 return 0;
973}
974
975/* Copy pages in the request to/from userspace buffer */
976static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
977 int zeroing)
978{
979 unsigned i;
980 struct fuse_req *req = cs->req;
Miklos Szeredi05ea48c2019-09-10 15:04:11 +0200981 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700982
Miklos Szeredi05ea48c2019-09-10 15:04:11 +0200983
984 for (i = 0; i < ap->num_pages && (nbytes || zeroing); i++) {
Miklos Szeredice534fb2010-05-25 15:06:07 +0200985 int err;
Miklos Szeredi05ea48c2019-09-10 15:04:11 +0200986 unsigned int offset = ap->descs[i].offset;
987 unsigned int count = min(nbytes, ap->descs[i].length);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200988
Miklos Szeredi05ea48c2019-09-10 15:04:11 +0200989 err = fuse_copy_page(cs, &ap->pages[i], offset, count, zeroing);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700990 if (err)
991 return err;
992
993 nbytes -= count;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700994 }
995 return 0;
996}
997
998/* Copy a single argument in the request to/from userspace buffer */
999static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1000{
1001 while (size) {
Miklos Szeredi1729a162008-11-26 12:03:54 +01001002 if (!cs->len) {
1003 int err = fuse_copy_fill(cs);
1004 if (err)
1005 return err;
1006 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001007 fuse_copy_do(cs, &val, &size);
1008 }
1009 return 0;
1010}
1011
1012/* Copy request arguments to/from userspace buffer */
1013static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1014 unsigned argpages, struct fuse_arg *args,
1015 int zeroing)
1016{
1017 int err = 0;
1018 unsigned i;
1019
1020 for (i = 0; !err && i < numargs; i++) {
1021 struct fuse_arg *arg = &args[i];
1022 if (i == numargs - 1 && argpages)
1023 err = fuse_copy_pages(cs, arg->size, zeroing);
1024 else
1025 err = fuse_copy_one(cs, arg->value, arg->size);
1026 }
1027 return err;
1028}
1029
Miklos Szeredif88996a2015-07-01 16:26:01 +02001030static int forget_pending(struct fuse_iqueue *fiq)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001031{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001032 return fiq->forget_list_head.next != NULL;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001033}
1034
Miklos Szeredif88996a2015-07-01 16:26:01 +02001035static int request_pending(struct fuse_iqueue *fiq)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001036{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001037 return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1038 forget_pending(fiq);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001039}
1040
Miklos Szeredi334f4852005-09-09 13:10:27 -07001041/*
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001042 * Transfer an interrupt request to userspace
1043 *
1044 * Unlike other requests this is assembled on demand, without a need
1045 * to allocate a separate fuse_req structure.
1046 *
Eric Biggers76e43c82019-09-08 20:15:18 -07001047 * Called with fiq->lock held, releases it
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001048 */
Miklos Szeredifd22d622015-07-01 16:26:03 +02001049static int fuse_read_interrupt(struct fuse_iqueue *fiq,
1050 struct fuse_copy_state *cs,
Miklos Szeredic3021622010-05-25 15:06:07 +02001051 size_t nbytes, struct fuse_req *req)
Eric Biggers76e43c82019-09-08 20:15:18 -07001052__releases(fiq->lock)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001053{
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001054 struct fuse_in_header ih;
1055 struct fuse_interrupt_in arg;
1056 unsigned reqsize = sizeof(ih) + sizeof(arg);
1057 int err;
1058
1059 list_del_init(&req->intr_entry);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001060 memset(&ih, 0, sizeof(ih));
1061 memset(&arg, 0, sizeof(arg));
1062 ih.len = reqsize;
1063 ih.opcode = FUSE_INTERRUPT;
Kirill Tkhai3a5358d2018-09-11 13:12:05 +03001064 ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001065 arg.unique = req->in.h.unique;
1066
Eric Biggers76e43c82019-09-08 20:15:18 -07001067 spin_unlock(&fiq->lock);
Miklos Szeredic3021622010-05-25 15:06:07 +02001068 if (nbytes < reqsize)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001069 return -EINVAL;
1070
Miklos Szeredic3021622010-05-25 15:06:07 +02001071 err = fuse_copy_one(cs, &ih, sizeof(ih));
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001072 if (!err)
Miklos Szeredic3021622010-05-25 15:06:07 +02001073 err = fuse_copy_one(cs, &arg, sizeof(arg));
1074 fuse_copy_finish(cs);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001075
1076 return err ? err : reqsize;
1077}
1078
Vivek Goyal4388c5a2019-06-05 15:50:43 -04001079struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
1080 unsigned int max,
1081 unsigned int *countp)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001082{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001083 struct fuse_forget_link *head = fiq->forget_list_head.next;
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001084 struct fuse_forget_link **newhead = &head;
1085 unsigned count;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001086
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001087 for (count = 0; *newhead != NULL && count < max; count++)
1088 newhead = &(*newhead)->next;
1089
Miklos Szeredif88996a2015-07-01 16:26:01 +02001090 fiq->forget_list_head.next = *newhead;
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001091 *newhead = NULL;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001092 if (fiq->forget_list_head.next == NULL)
1093 fiq->forget_list_tail = &fiq->forget_list_head;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001094
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001095 if (countp != NULL)
1096 *countp = count;
1097
1098 return head;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001099}
Vivek Goyal4388c5a2019-06-05 15:50:43 -04001100EXPORT_SYMBOL(fuse_dequeue_forget);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001101
Miklos Szeredifd22d622015-07-01 16:26:03 +02001102static int fuse_read_single_forget(struct fuse_iqueue *fiq,
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001103 struct fuse_copy_state *cs,
1104 size_t nbytes)
Eric Biggers76e43c82019-09-08 20:15:18 -07001105__releases(fiq->lock)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001106{
1107 int err;
Vivek Goyal4388c5a2019-06-05 15:50:43 -04001108 struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001109 struct fuse_forget_in arg = {
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001110 .nlookup = forget->forget_one.nlookup,
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001111 };
1112 struct fuse_in_header ih = {
1113 .opcode = FUSE_FORGET,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001114 .nodeid = forget->forget_one.nodeid,
Miklos Szeredif88996a2015-07-01 16:26:01 +02001115 .unique = fuse_get_unique(fiq),
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001116 .len = sizeof(ih) + sizeof(arg),
1117 };
1118
Eric Biggers76e43c82019-09-08 20:15:18 -07001119 spin_unlock(&fiq->lock);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001120 kfree(forget);
1121 if (nbytes < ih.len)
1122 return -EINVAL;
1123
1124 err = fuse_copy_one(cs, &ih, sizeof(ih));
1125 if (!err)
1126 err = fuse_copy_one(cs, &arg, sizeof(arg));
1127 fuse_copy_finish(cs);
1128
1129 if (err)
1130 return err;
1131
1132 return ih.len;
1133}
1134
Miklos Szeredifd22d622015-07-01 16:26:03 +02001135static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001136 struct fuse_copy_state *cs, size_t nbytes)
Eric Biggers76e43c82019-09-08 20:15:18 -07001137__releases(fiq->lock)
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001138{
1139 int err;
1140 unsigned max_forgets;
1141 unsigned count;
1142 struct fuse_forget_link *head;
1143 struct fuse_batch_forget_in arg = { .count = 0 };
1144 struct fuse_in_header ih = {
1145 .opcode = FUSE_BATCH_FORGET,
Miklos Szeredif88996a2015-07-01 16:26:01 +02001146 .unique = fuse_get_unique(fiq),
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001147 .len = sizeof(ih) + sizeof(arg),
1148 };
1149
1150 if (nbytes < ih.len) {
Eric Biggers76e43c82019-09-08 20:15:18 -07001151 spin_unlock(&fiq->lock);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001152 return -EINVAL;
1153 }
1154
1155 max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
Vivek Goyal4388c5a2019-06-05 15:50:43 -04001156 head = fuse_dequeue_forget(fiq, max_forgets, &count);
Eric Biggers76e43c82019-09-08 20:15:18 -07001157 spin_unlock(&fiq->lock);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001158
1159 arg.count = count;
1160 ih.len += count * sizeof(struct fuse_forget_one);
1161 err = fuse_copy_one(cs, &ih, sizeof(ih));
1162 if (!err)
1163 err = fuse_copy_one(cs, &arg, sizeof(arg));
1164
1165 while (head) {
1166 struct fuse_forget_link *forget = head;
1167
1168 if (!err) {
1169 err = fuse_copy_one(cs, &forget->forget_one,
1170 sizeof(forget->forget_one));
1171 }
1172 head = forget->next;
1173 kfree(forget);
1174 }
1175
1176 fuse_copy_finish(cs);
1177
1178 if (err)
1179 return err;
1180
1181 return ih.len;
1182}
1183
Miklos Szeredifd22d622015-07-01 16:26:03 +02001184static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
1185 struct fuse_copy_state *cs,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001186 size_t nbytes)
Eric Biggers76e43c82019-09-08 20:15:18 -07001187__releases(fiq->lock)
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001188{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001189 if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001190 return fuse_read_single_forget(fiq, cs, nbytes);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001191 else
Miklos Szeredifd22d622015-07-01 16:26:03 +02001192 return fuse_read_batch_forget(fiq, cs, nbytes);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001193}
1194
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001195/*
Miklos Szeredi334f4852005-09-09 13:10:27 -07001196 * Read a single request into the userspace filesystem's buffer. This
1197 * function waits until a request is available, then removes it from
1198 * the pending list and copies request data to userspace buffer. If
Miklos Szeredif9a28422006-06-25 05:48:53 -07001199 * no reply is needed (FORGET) or request has been aborted or there
1200 * was an error during the copying then it's finished by calling
Stefan Hajnoczi04ec5af2018-06-21 09:33:40 +01001201 * fuse_request_end(). Otherwise add it to the processing list, and set
Miklos Szeredi334f4852005-09-09 13:10:27 -07001202 * the 'sent' flag.
1203 */
Miklos Szeredic36960462015-07-01 16:26:09 +02001204static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
Miklos Szeredic3021622010-05-25 15:06:07 +02001205 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001206{
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001207 ssize_t err;
Miklos Szeredic36960462015-07-01 16:26:09 +02001208 struct fuse_conn *fc = fud->fc;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001209 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredic36960462015-07-01 16:26:09 +02001210 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001211 struct fuse_req *req;
Miklos Szeredid4993772019-09-10 15:04:11 +02001212 struct fuse_args *args;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001213 unsigned reqsize;
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03001214 unsigned int hash;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001215
Kirill Smelkov1fb027d2019-07-08 17:03:31 +00001216 /*
1217 * Require sane minimum read buffer - that has capacity for fixed part
1218 * of any request header + negotiated max_write room for data.
1219 *
1220 * Historically libfuse reserves 4K for fixed header room, but e.g.
1221 * GlusterFS reserves only 80 bytes
1222 *
1223 * = `sizeof(fuse_in_header) + sizeof(fuse_write_in)`
1224 *
1225 * which is the absolute minimum any sane filesystem should be using
1226 * for header room.
1227 */
1228 if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER,
1229 sizeof(struct fuse_in_header) +
1230 sizeof(struct fuse_write_in) +
1231 fc->max_write))
1232 return -EINVAL;
1233
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001234 restart:
Eric Biggers76e43c82019-09-08 20:15:18 -07001235 for (;;) {
1236 spin_lock(&fiq->lock);
1237 if (!fiq->connected || request_pending(fiq))
1238 break;
1239 spin_unlock(&fiq->lock);
Jeff Dikee5ac1d12006-04-10 22:54:53 -07001240
Eric Biggers76e43c82019-09-08 20:15:18 -07001241 if (file->f_flags & O_NONBLOCK)
1242 return -EAGAIN;
1243 err = wait_event_interruptible_exclusive(fiq->waitq,
Miklos Szeredi52509212015-07-01 16:26:03 +02001244 !fiq->connected || request_pending(fiq));
Eric Biggers76e43c82019-09-08 20:15:18 -07001245 if (err)
1246 return err;
1247 }
Miklos Szeredi52509212015-07-01 16:26:03 +02001248
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01001249 if (!fiq->connected) {
Miklos Szeredieb98e3bd2019-01-24 10:40:16 +01001250 err = fc->aborted ? -ECONNABORTED : -ENODEV;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001251 goto err_unlock;
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01001252 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001253
Miklos Szeredif88996a2015-07-01 16:26:01 +02001254 if (!list_empty(&fiq->interrupts)) {
1255 req = list_entry(fiq->interrupts.next, struct fuse_req,
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001256 intr_entry);
Miklos Szeredifd22d622015-07-01 16:26:03 +02001257 return fuse_read_interrupt(fiq, cs, nbytes, req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001258 }
1259
Miklos Szeredif88996a2015-07-01 16:26:01 +02001260 if (forget_pending(fiq)) {
1261 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001262 return fuse_read_forget(fc, fiq, cs, nbytes);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001263
Miklos Szeredif88996a2015-07-01 16:26:01 +02001264 if (fiq->forget_batch <= -8)
1265 fiq->forget_batch = 16;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001266 }
1267
Miklos Szeredif88996a2015-07-01 16:26:01 +02001268 req = list_entry(fiq->pending.next, struct fuse_req, list);
Miklos Szeredi33e14b42015-07-01 16:26:01 +02001269 clear_bit(FR_PENDING, &req->flags);
Miklos Szeredief759252015-07-01 16:26:02 +02001270 list_del_init(&req->list);
Eric Biggers76e43c82019-09-08 20:15:18 -07001271 spin_unlock(&fiq->lock);
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001272
Miklos Szeredid4993772019-09-10 15:04:11 +02001273 args = req->args;
1274 reqsize = req->in.h.len;
Miklos Szeredi5d6d3a32017-09-12 16:57:53 +02001275
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001276 /* If request is too large, reply with an error and restart the read */
Miklos Szeredic3021622010-05-25 15:06:07 +02001277 if (nbytes < reqsize) {
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001278 req->out.h.error = -EIO;
1279 /* SETXATTR is special, since it may contain too large data */
Miklos Szeredid4993772019-09-10 15:04:11 +02001280 if (args->opcode == FUSE_SETXATTR)
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001281 req->out.h.error = -E2BIG;
Max Reitz8f622e92020-04-20 17:59:34 +02001282 fuse_request_end(req);
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001283 goto restart;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001284 }
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001285 spin_lock(&fpq->lock);
Miklos Szeredi80ef0862021-06-22 09:15:35 +02001286 /*
1287 * Must not put request on fpq->io queue after having been shut down by
1288 * fuse_abort_conn()
1289 */
1290 if (!fpq->connected) {
1291 req->out.h.error = err = -ECONNABORTED;
1292 goto out_end;
1293
1294 }
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001295 list_add(&req->list, &fpq->io);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001296 spin_unlock(&fpq->lock);
Miklos Szeredic3021622010-05-25 15:06:07 +02001297 cs->req = req;
Miklos Szeredid4993772019-09-10 15:04:11 +02001298 err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h));
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001299 if (!err)
Miklos Szeredid4993772019-09-10 15:04:11 +02001300 err = fuse_copy_args(cs, args->in_numargs, args->in_pages,
1301 (struct fuse_arg *) args->in_args, 0);
Miklos Szeredic3021622010-05-25 15:06:07 +02001302 fuse_copy_finish(cs);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001303 spin_lock(&fpq->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001304 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredie96edd92015-07-01 16:26:04 +02001305 if (!fpq->connected) {
Miklos Szeredieb98e3bd2019-01-24 10:40:16 +01001306 err = fc->aborted ? -ECONNABORTED : -ENODEV;
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001307 goto out_end;
Miklos Szeredic9c9d7d2007-10-16 23:31:05 -07001308 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001309 if (err) {
Miklos Szeredic9c9d7d2007-10-16 23:31:05 -07001310 req->out.h.error = -EIO;
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001311 goto out_end;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001312 }
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001313 if (!test_bit(FR_ISREPLY, &req->flags)) {
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001314 err = reqsize;
1315 goto out_end;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001316 }
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03001317 hash = fuse_req_hash(req->in.h.unique);
1318 list_move_tail(&req->list, &fpq->processing[hash]);
Kirill Tkhaibc78abb2018-09-25 12:28:55 +03001319 __fuse_get_request(req);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001320 set_bit(FR_SENT, &req->flags);
Miklos Szeredi4c316f22018-09-28 16:43:22 +02001321 spin_unlock(&fpq->lock);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001322 /* matches barrier in request_wait_answer() */
1323 smp_mb__after_atomic();
1324 if (test_bit(FR_INTERRUPTED, &req->flags))
Max Reitz8f622e92020-04-20 17:59:34 +02001325 queue_interrupt(req);
1326 fuse_put_request(req);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001327
Miklos Szeredi334f4852005-09-09 13:10:27 -07001328 return reqsize;
1329
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001330out_end:
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02001331 if (!test_bit(FR_PRIVATE, &req->flags))
1332 list_del_init(&req->list);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001333 spin_unlock(&fpq->lock);
Max Reitz8f622e92020-04-20 17:59:34 +02001334 fuse_request_end(req);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001335 return err;
1336
Miklos Szeredi334f4852005-09-09 13:10:27 -07001337 err_unlock:
Eric Biggers76e43c82019-09-08 20:15:18 -07001338 spin_unlock(&fiq->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001339 return err;
1340}
1341
Tom Van Braeckel94e4fe22015-01-12 05:22:16 +01001342static int fuse_dev_open(struct inode *inode, struct file *file)
1343{
1344 /*
1345 * The fuse device's file's private_data is used to hold
1346 * the fuse_conn(ection) when it is mounted, and is used to
1347 * keep track of whether the file has been mounted already.
1348 */
1349 file->private_data = NULL;
1350 return 0;
1351}
1352
Al Virofbdbacc2015-04-03 21:53:39 -04001353static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
Miklos Szeredic3021622010-05-25 15:06:07 +02001354{
1355 struct fuse_copy_state cs;
1356 struct file *file = iocb->ki_filp;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001357 struct fuse_dev *fud = fuse_get_dev(file);
1358
1359 if (!fud)
Miklos Szeredic3021622010-05-25 15:06:07 +02001360 return -EPERM;
1361
Al Virofcb14cb2022-05-22 14:59:25 -04001362 if (!user_backed_iter(to))
Al Virofbdbacc2015-04-03 21:53:39 -04001363 return -EINVAL;
Miklos Szeredic3021622010-05-25 15:06:07 +02001364
Miklos Szeredidc008092015-07-01 16:25:58 +02001365 fuse_copy_init(&cs, 1, to);
Al Virofbdbacc2015-04-03 21:53:39 -04001366
Miklos Szeredic36960462015-07-01 16:26:09 +02001367 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
Miklos Szeredic3021622010-05-25 15:06:07 +02001368}
1369
Miklos Szeredic3021622010-05-25 15:06:07 +02001370static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1371 struct pipe_inode_info *pipe,
1372 size_t len, unsigned int flags)
1373{
Al Virod82718e2016-09-17 22:56:25 -04001374 int total, ret;
Miklos Szeredic3021622010-05-25 15:06:07 +02001375 int page_nr = 0;
Miklos Szeredic3021622010-05-25 15:06:07 +02001376 struct pipe_buffer *bufs;
1377 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001378 struct fuse_dev *fud = fuse_get_dev(in);
1379
1380 if (!fud)
Miklos Szeredic3021622010-05-25 15:06:07 +02001381 return -EPERM;
1382
David Howells6718b6f2019-10-16 16:47:32 +01001383 bufs = kvmalloc_array(pipe->max_usage, sizeof(struct pipe_buffer),
Andrey Ryabinind6d931a2018-07-17 19:00:34 +03001384 GFP_KERNEL);
Miklos Szeredic3021622010-05-25 15:06:07 +02001385 if (!bufs)
1386 return -ENOMEM;
1387
Miklos Szeredidc008092015-07-01 16:25:58 +02001388 fuse_copy_init(&cs, 1, NULL);
Miklos Szeredic3021622010-05-25 15:06:07 +02001389 cs.pipebufs = bufs;
1390 cs.pipe = pipe;
Miklos Szeredic36960462015-07-01 16:26:09 +02001391 ret = fuse_dev_do_read(fud, in, &cs, len);
Miklos Szeredic3021622010-05-25 15:06:07 +02001392 if (ret < 0)
1393 goto out;
1394
David Howells6718b6f2019-10-16 16:47:32 +01001395 if (pipe_occupancy(pipe->head, pipe->tail) + cs.nr_segs > pipe->max_usage) {
Miklos Szeredic3021622010-05-25 15:06:07 +02001396 ret = -EIO;
Al Virod82718e2016-09-17 22:56:25 -04001397 goto out;
Miklos Szeredic3021622010-05-25 15:06:07 +02001398 }
1399
Al Virod82718e2016-09-17 22:56:25 -04001400 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
Miklos Szeredi28a625c2014-01-22 19:36:57 +01001401 /*
1402 * Need to be careful about this. Having buf->ops in module
1403 * code can Oops if the buffer persists after module unload.
1404 */
Al Virod82718e2016-09-17 22:56:25 -04001405 bufs[page_nr].ops = &nosteal_pipe_buf_ops;
Miklos Szeredi84588a92017-02-16 15:08:20 +01001406 bufs[page_nr].flags = 0;
Al Virod82718e2016-09-17 22:56:25 -04001407 ret = add_to_pipe(pipe, &bufs[page_nr++]);
1408 if (unlikely(ret < 0))
1409 break;
Miklos Szeredic3021622010-05-25 15:06:07 +02001410 }
Al Virod82718e2016-09-17 22:56:25 -04001411 if (total)
1412 ret = total;
Miklos Szeredic3021622010-05-25 15:06:07 +02001413out:
1414 for (; page_nr < cs.nr_segs; page_nr++)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001415 put_page(bufs[page_nr].page);
Miklos Szeredic3021622010-05-25 15:06:07 +02001416
Andrey Ryabinind6d931a2018-07-17 19:00:34 +03001417 kvfree(bufs);
Miklos Szeredic3021622010-05-25 15:06:07 +02001418 return ret;
1419}
1420
Tejun Heo95668a62008-11-26 12:03:55 +01001421static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1422 struct fuse_copy_state *cs)
1423{
1424 struct fuse_notify_poll_wakeup_out outarg;
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001425 int err = -EINVAL;
Tejun Heo95668a62008-11-26 12:03:55 +01001426
1427 if (size != sizeof(outarg))
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001428 goto err;
Tejun Heo95668a62008-11-26 12:03:55 +01001429
1430 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1431 if (err)
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001432 goto err;
Tejun Heo95668a62008-11-26 12:03:55 +01001433
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001434 fuse_copy_finish(cs);
Tejun Heo95668a62008-11-26 12:03:55 +01001435 return fuse_notify_poll_wakeup(fc, &outarg);
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001436
1437err:
1438 fuse_copy_finish(cs);
1439 return err;
Tejun Heo95668a62008-11-26 12:03:55 +01001440}
1441
John Muir3b463ae2009-05-31 11:13:57 -04001442static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1443 struct fuse_copy_state *cs)
1444{
1445 struct fuse_notify_inval_inode_out outarg;
1446 int err = -EINVAL;
1447
1448 if (size != sizeof(outarg))
1449 goto err;
1450
1451 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1452 if (err)
1453 goto err;
1454 fuse_copy_finish(cs);
1455
1456 down_read(&fc->killsb);
Max Reitzfcee2162020-05-06 17:44:12 +02001457 err = fuse_reverse_inval_inode(fc, outarg.ino,
1458 outarg.off, outarg.len);
John Muir3b463ae2009-05-31 11:13:57 -04001459 up_read(&fc->killsb);
1460 return err;
1461
1462err:
1463 fuse_copy_finish(cs);
1464 return err;
1465}
1466
1467static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1468 struct fuse_copy_state *cs)
1469{
1470 struct fuse_notify_inval_entry_out outarg;
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001471 int err = -ENOMEM;
1472 char *buf;
John Muir3b463ae2009-05-31 11:13:57 -04001473 struct qstr name;
1474
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001475 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1476 if (!buf)
1477 goto err;
1478
1479 err = -EINVAL;
John Muir3b463ae2009-05-31 11:13:57 -04001480 if (size < sizeof(outarg))
1481 goto err;
1482
1483 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1484 if (err)
1485 goto err;
1486
1487 err = -ENAMETOOLONG;
1488 if (outarg.namelen > FUSE_NAME_MAX)
1489 goto err;
1490
Miklos Szeredic2183d12011-08-24 10:20:17 +02001491 err = -EINVAL;
1492 if (size != sizeof(outarg) + outarg.namelen + 1)
1493 goto err;
1494
John Muir3b463ae2009-05-31 11:13:57 -04001495 name.name = buf;
1496 name.len = outarg.namelen;
1497 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1498 if (err)
1499 goto err;
1500 fuse_copy_finish(cs);
1501 buf[outarg.namelen] = 0;
John Muir3b463ae2009-05-31 11:13:57 -04001502
1503 down_read(&fc->killsb);
Miklos Szeredi4f8d3702022-10-28 14:25:21 +02001504 err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags);
John Muir451d0f52011-12-06 21:50:06 +01001505 up_read(&fc->killsb);
1506 kfree(buf);
1507 return err;
1508
1509err:
1510 kfree(buf);
1511 fuse_copy_finish(cs);
1512 return err;
1513}
1514
1515static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1516 struct fuse_copy_state *cs)
1517{
1518 struct fuse_notify_delete_out outarg;
1519 int err = -ENOMEM;
1520 char *buf;
1521 struct qstr name;
1522
1523 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1524 if (!buf)
1525 goto err;
1526
1527 err = -EINVAL;
1528 if (size < sizeof(outarg))
1529 goto err;
1530
1531 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1532 if (err)
1533 goto err;
1534
1535 err = -ENAMETOOLONG;
1536 if (outarg.namelen > FUSE_NAME_MAX)
1537 goto err;
1538
1539 err = -EINVAL;
1540 if (size != sizeof(outarg) + outarg.namelen + 1)
1541 goto err;
1542
1543 name.name = buf;
1544 name.len = outarg.namelen;
1545 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1546 if (err)
1547 goto err;
1548 fuse_copy_finish(cs);
1549 buf[outarg.namelen] = 0;
John Muir451d0f52011-12-06 21:50:06 +01001550
1551 down_read(&fc->killsb);
Miklos Szeredi4f8d3702022-10-28 14:25:21 +02001552 err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0);
John Muir3b463ae2009-05-31 11:13:57 -04001553 up_read(&fc->killsb);
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001554 kfree(buf);
John Muir3b463ae2009-05-31 11:13:57 -04001555 return err;
1556
1557err:
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001558 kfree(buf);
John Muir3b463ae2009-05-31 11:13:57 -04001559 fuse_copy_finish(cs);
1560 return err;
1561}
1562
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001563static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1564 struct fuse_copy_state *cs)
1565{
1566 struct fuse_notify_store_out outarg;
1567 struct inode *inode;
1568 struct address_space *mapping;
1569 u64 nodeid;
1570 int err;
1571 pgoff_t index;
1572 unsigned int offset;
1573 unsigned int num;
1574 loff_t file_size;
1575 loff_t end;
1576
1577 err = -EINVAL;
1578 if (size < sizeof(outarg))
1579 goto out_finish;
1580
1581 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1582 if (err)
1583 goto out_finish;
1584
1585 err = -EINVAL;
1586 if (size - sizeof(outarg) != outarg.size)
1587 goto out_finish;
1588
1589 nodeid = outarg.nodeid;
1590
1591 down_read(&fc->killsb);
1592
1593 err = -ENOENT;
Max Reitzfcee2162020-05-06 17:44:12 +02001594 inode = fuse_ilookup(fc, nodeid, NULL);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001595 if (!inode)
1596 goto out_up_killsb;
1597
1598 mapping = inode->i_mapping;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001599 index = outarg.offset >> PAGE_SHIFT;
1600 offset = outarg.offset & ~PAGE_MASK;
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001601 file_size = i_size_read(inode);
1602 end = outarg.offset + outarg.size;
1603 if (end > file_size) {
1604 file_size = end;
Miklos Szeredid3477392021-10-22 17:03:02 +02001605 fuse_write_update_attr(inode, file_size, outarg.size);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001606 }
1607
1608 num = outarg.size;
1609 while (num) {
1610 struct page *page;
1611 unsigned int this_num;
1612
1613 err = -ENOMEM;
1614 page = find_or_create_page(mapping, index,
1615 mapping_gfp_mask(mapping));
1616 if (!page)
1617 goto out_iput;
1618
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001619 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001620 err = fuse_copy_page(cs, &page, offset, this_num, 0);
Miklos Szeredi063ec1e2014-01-22 19:36:58 +01001621 if (!err && offset == 0 &&
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001622 (this_num == PAGE_SIZE || file_size == end))
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001623 SetPageUptodate(page);
1624 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001625 put_page(page);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001626
1627 if (err)
1628 goto out_iput;
1629
1630 num -= this_num;
1631 offset = 0;
1632 index++;
1633 }
1634
1635 err = 0;
1636
1637out_iput:
1638 iput(inode);
1639out_up_killsb:
1640 up_read(&fc->killsb);
1641out_finish:
1642 fuse_copy_finish(cs);
1643 return err;
1644}
1645
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001646struct fuse_retrieve_args {
1647 struct fuse_args_pages ap;
1648 struct fuse_notify_retrieve_in inarg;
1649};
1650
Max Reitzfcee2162020-05-06 17:44:12 +02001651static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args,
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001652 int error)
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001653{
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001654 struct fuse_retrieve_args *ra =
1655 container_of(args, typeof(*ra), ap.args);
1656
1657 release_pages(ra->ap.pages, ra->ap.num_pages);
1658 kfree(ra);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001659}
1660
Max Reitzfcee2162020-05-06 17:44:12 +02001661static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001662 struct fuse_notify_retrieve_out *outarg)
1663{
1664 int err;
1665 struct address_space *mapping = inode->i_mapping;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001666 pgoff_t index;
1667 loff_t file_size;
1668 unsigned int num;
1669 unsigned int offset;
Geert Uytterhoeven01574432010-09-30 22:06:21 +02001670 size_t total_len = 0;
Constantine Shulyupin5da784c2018-09-06 15:37:06 +03001671 unsigned int num_pages;
Max Reitzfcee2162020-05-06 17:44:12 +02001672 struct fuse_conn *fc = fm->fc;
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001673 struct fuse_retrieve_args *ra;
1674 size_t args_size = sizeof(*ra);
1675 struct fuse_args_pages *ap;
1676 struct fuse_args *args;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001677
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001678 offset = outarg->offset & ~PAGE_MASK;
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001679 file_size = i_size_read(inode);
1680
Kirill Smelkov76406822019-03-27 10:15:19 +00001681 num = min(outarg->size, fc->max_write);
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001682 if (outarg->offset > file_size)
1683 num = 0;
1684 else if (outarg->offset + num > file_size)
1685 num = file_size - outarg->offset;
1686
1687 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
Constantine Shulyupin5da784c2018-09-06 15:37:06 +03001688 num_pages = min(num_pages, fc->max_pages);
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001689
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001690 args_size += num_pages * (sizeof(ap->pages[0]) + sizeof(ap->descs[0]));
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001691
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001692 ra = kzalloc(args_size, GFP_KERNEL);
1693 if (!ra)
1694 return -ENOMEM;
1695
1696 ap = &ra->ap;
1697 ap->pages = (void *) (ra + 1);
1698 ap->descs = (void *) (ap->pages + num_pages);
1699
1700 args = &ap->args;
1701 args->nodeid = outarg->nodeid;
1702 args->opcode = FUSE_NOTIFY_REPLY;
1703 args->in_numargs = 2;
1704 args->in_pages = true;
1705 args->end = fuse_retrieve_end;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001706
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001707 index = outarg->offset >> PAGE_SHIFT;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001708
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001709 while (num && ap->num_pages < num_pages) {
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001710 struct page *page;
1711 unsigned int this_num;
1712
1713 page = find_get_page(mapping, index);
1714 if (!page)
1715 break;
1716
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001717 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001718 ap->pages[ap->num_pages] = page;
1719 ap->descs[ap->num_pages].offset = offset;
1720 ap->descs[ap->num_pages].length = this_num;
1721 ap->num_pages++;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001722
Miklos Szeredic9e67d42012-09-04 18:45:54 +02001723 offset = 0;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001724 num -= this_num;
1725 total_len += this_num;
Miklos Szeredi48706d02011-12-13 10:36:59 +01001726 index++;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001727 }
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001728 ra->inarg.offset = outarg->offset;
1729 ra->inarg.size = total_len;
1730 args->in_args[0].size = sizeof(ra->inarg);
1731 args->in_args[0].value = &ra->inarg;
1732 args->in_args[1].size = total_len;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001733
Max Reitzfcee2162020-05-06 17:44:12 +02001734 err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
Miklos Szeredi75b399d2019-09-10 15:04:11 +02001735 if (err)
Max Reitzfcee2162020-05-06 17:44:12 +02001736 fuse_retrieve_end(fm, args, err);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001737
1738 return err;
1739}
1740
1741static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1742 struct fuse_copy_state *cs)
1743{
1744 struct fuse_notify_retrieve_out outarg;
Max Reitzfcee2162020-05-06 17:44:12 +02001745 struct fuse_mount *fm;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001746 struct inode *inode;
Max Reitzfcee2162020-05-06 17:44:12 +02001747 u64 nodeid;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001748 int err;
1749
1750 err = -EINVAL;
1751 if (size != sizeof(outarg))
1752 goto copy_finish;
1753
1754 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1755 if (err)
1756 goto copy_finish;
1757
1758 fuse_copy_finish(cs);
1759
1760 down_read(&fc->killsb);
1761 err = -ENOENT;
Max Reitzfcee2162020-05-06 17:44:12 +02001762 nodeid = outarg.nodeid;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001763
Max Reitzfcee2162020-05-06 17:44:12 +02001764 inode = fuse_ilookup(fc, nodeid, &fm);
1765 if (inode) {
1766 err = fuse_retrieve(fm, inode, &outarg);
1767 iput(inode);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001768 }
1769 up_read(&fc->killsb);
1770
1771 return err;
1772
1773copy_finish:
1774 fuse_copy_finish(cs);
1775 return err;
1776}
1777
Tejun Heo85993962008-11-26 12:03:55 +01001778static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1779 unsigned int size, struct fuse_copy_state *cs)
1780{
Miklos Szeredi0d278362015-02-26 11:45:47 +01001781 /* Don't try to move pages (yet) */
1782 cs->move_pages = 0;
1783
Tejun Heo85993962008-11-26 12:03:55 +01001784 switch (code) {
Tejun Heo95668a62008-11-26 12:03:55 +01001785 case FUSE_NOTIFY_POLL:
1786 return fuse_notify_poll(fc, size, cs);
1787
John Muir3b463ae2009-05-31 11:13:57 -04001788 case FUSE_NOTIFY_INVAL_INODE:
1789 return fuse_notify_inval_inode(fc, size, cs);
1790
1791 case FUSE_NOTIFY_INVAL_ENTRY:
1792 return fuse_notify_inval_entry(fc, size, cs);
1793
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001794 case FUSE_NOTIFY_STORE:
1795 return fuse_notify_store(fc, size, cs);
1796
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001797 case FUSE_NOTIFY_RETRIEVE:
1798 return fuse_notify_retrieve(fc, size, cs);
1799
John Muir451d0f52011-12-06 21:50:06 +01001800 case FUSE_NOTIFY_DELETE:
1801 return fuse_notify_delete(fc, size, cs);
1802
Tejun Heo85993962008-11-26 12:03:55 +01001803 default:
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001804 fuse_copy_finish(cs);
Tejun Heo85993962008-11-26 12:03:55 +01001805 return -EINVAL;
1806 }
1807}
1808
Miklos Szeredi334f4852005-09-09 13:10:27 -07001809/* Look up request on processing list by unique ID */
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001810static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001811{
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03001812 unsigned int hash = fuse_req_hash(unique);
Dong Fang05726ac2013-07-30 22:50:01 -04001813 struct fuse_req *req;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001814
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03001815 list_for_each_entry(req, &fpq->processing[hash], list) {
Kirill Tkhai3a5358d2018-09-11 13:12:05 +03001816 if (req->in.h.unique == unique)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001817 return req;
1818 }
1819 return NULL;
1820}
1821
Miklos Szeredid4993772019-09-10 15:04:11 +02001822static int copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
Miklos Szeredi334f4852005-09-09 13:10:27 -07001823 unsigned nbytes)
1824{
1825 unsigned reqsize = sizeof(struct fuse_out_header);
1826
Stefan Hajnoczi14d46d72018-06-21 09:34:25 +01001827 reqsize += fuse_len_args(args->out_numargs, args->out_args);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001828
Miklos Szeredid4993772019-09-10 15:04:11 +02001829 if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar))
Miklos Szeredi334f4852005-09-09 13:10:27 -07001830 return -EINVAL;
1831 else if (reqsize > nbytes) {
Miklos Szeredid4993772019-09-10 15:04:11 +02001832 struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1];
Miklos Szeredi334f4852005-09-09 13:10:27 -07001833 unsigned diffsize = reqsize - nbytes;
Miklos Szeredid4993772019-09-10 15:04:11 +02001834
Miklos Szeredi334f4852005-09-09 13:10:27 -07001835 if (diffsize > lastarg->size)
1836 return -EINVAL;
1837 lastarg->size -= diffsize;
1838 }
Miklos Szeredid4993772019-09-10 15:04:11 +02001839 return fuse_copy_args(cs, args->out_numargs, args->out_pages,
1840 args->out_args, args->page_zeroing);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001841}
1842
1843/*
1844 * Write a single reply to a request. First the header is copied from
1845 * the write buffer. The request is then searched on the processing
1846 * list by the unique ID found in the header. If found, then remove
1847 * it from the list and copy the rest of the buffer to the request.
Stefan Hajnoczi04ec5af2018-06-21 09:33:40 +01001848 * The request is finished by calling fuse_request_end().
Miklos Szeredi334f4852005-09-09 13:10:27 -07001849 */
Miklos Szeredic36960462015-07-01 16:26:09 +02001850static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001851 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001852{
1853 int err;
Miklos Szeredic36960462015-07-01 16:26:09 +02001854 struct fuse_conn *fc = fud->fc;
1855 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001856 struct fuse_req *req;
1857 struct fuse_out_header oh;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001858
Kirill Tkhai7407a102018-11-08 12:05:36 +03001859 err = -EINVAL;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001860 if (nbytes < sizeof(struct fuse_out_header))
Kirill Tkhai7407a102018-11-08 12:05:36 +03001861 goto out;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001862
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001863 err = fuse_copy_one(cs, &oh, sizeof(oh));
Miklos Szeredi334f4852005-09-09 13:10:27 -07001864 if (err)
Kirill Tkhai7407a102018-11-08 12:05:36 +03001865 goto copy_finish;
Tejun Heo85993962008-11-26 12:03:55 +01001866
Miklos Szeredi334f4852005-09-09 13:10:27 -07001867 err = -EINVAL;
Tejun Heo85993962008-11-26 12:03:55 +01001868 if (oh.len != nbytes)
Kirill Tkhai7407a102018-11-08 12:05:36 +03001869 goto copy_finish;
Tejun Heo85993962008-11-26 12:03:55 +01001870
1871 /*
1872 * Zero oh.unique indicates unsolicited notification message
1873 * and error contains notification code.
1874 */
1875 if (!oh.unique) {
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001876 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
Kirill Tkhai7407a102018-11-08 12:05:36 +03001877 goto out;
Tejun Heo85993962008-11-26 12:03:55 +01001878 }
1879
1880 err = -EINVAL;
Miklos Szeredi49221cf2021-06-22 09:15:35 +02001881 if (oh.error <= -512 || oh.error > 0)
Kirill Tkhai7407a102018-11-08 12:05:36 +03001882 goto copy_finish;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001883
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001884 spin_lock(&fpq->lock);
Kirill Tkhai7407a102018-11-08 12:05:36 +03001885 req = NULL;
1886 if (fpq->connected)
1887 req = request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08001888
Kirill Tkhai7407a102018-11-08 12:05:36 +03001889 err = -ENOENT;
1890 if (!req) {
1891 spin_unlock(&fpq->lock);
1892 goto copy_finish;
1893 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001894
Kirill Tkhai3a5358d2018-09-11 13:12:05 +03001895 /* Is it an interrupt reply ID? */
1896 if (oh.unique & FUSE_INT_REQ_BIT) {
Kirill Tkhaid2d2d4f2018-09-25 12:52:42 +03001897 __fuse_get_request(req);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001898 spin_unlock(&fpq->lock);
1899
Kirill Tkhai7407a102018-11-08 12:05:36 +03001900 err = 0;
1901 if (nbytes != sizeof(struct fuse_out_header))
1902 err = -EINVAL;
1903 else if (oh.error == -ENOSYS)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001904 fc->no_interrupt = 1;
1905 else if (oh.error == -EAGAIN)
Max Reitz8f622e92020-04-20 17:59:34 +02001906 err = queue_interrupt(req);
Kirill Tkhai7407a102018-11-08 12:05:36 +03001907
Max Reitz8f622e92020-04-20 17:59:34 +02001908 fuse_put_request(req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001909
Kirill Tkhai7407a102018-11-08 12:05:36 +03001910 goto copy_finish;
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001911 }
1912
Miklos Szeredi33e14b42015-07-01 16:26:01 +02001913 clear_bit(FR_SENT, &req->flags);
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001914 list_move(&req->list, &fpq->io);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001915 req->out.h = oh;
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001916 set_bit(FR_LOCKED, &req->flags);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001917 spin_unlock(&fpq->lock);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001918 cs->req = req;
Miklos Szeredid4993772019-09-10 15:04:11 +02001919 if (!req->args->page_replace)
Miklos Szeredice534fb2010-05-25 15:06:07 +02001920 cs->move_pages = 0;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001921
Miklos Szeredid4993772019-09-10 15:04:11 +02001922 if (oh.error)
1923 err = nbytes != sizeof(oh) ? -EINVAL : 0;
1924 else
1925 err = copy_out_args(cs, req->args, nbytes);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001926 fuse_copy_finish(cs);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001927
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001928 spin_lock(&fpq->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001929 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredie96edd92015-07-01 16:26:04 +02001930 if (!fpq->connected)
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +02001931 err = -ENOENT;
1932 else if (err)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001933 req->out.h.error = -EIO;
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02001934 if (!test_bit(FR_PRIVATE, &req->flags))
1935 list_del_init(&req->list);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001936 spin_unlock(&fpq->lock);
Miklos Szeredi46c34a32015-07-01 16:26:07 +02001937
Max Reitz8f622e92020-04-20 17:59:34 +02001938 fuse_request_end(req);
Kirill Tkhai7407a102018-11-08 12:05:36 +03001939out:
Miklos Szeredi334f4852005-09-09 13:10:27 -07001940 return err ? err : nbytes;
1941
Kirill Tkhai7407a102018-11-08 12:05:36 +03001942copy_finish:
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001943 fuse_copy_finish(cs);
Kirill Tkhai7407a102018-11-08 12:05:36 +03001944 goto out;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001945}
1946
Al Virofbdbacc2015-04-03 21:53:39 -04001947static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001948{
1949 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001950 struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);
1951
1952 if (!fud)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001953 return -EPERM;
1954
Al Virofcb14cb2022-05-22 14:59:25 -04001955 if (!user_backed_iter(from))
Al Virofbdbacc2015-04-03 21:53:39 -04001956 return -EINVAL;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001957
Miklos Szeredidc008092015-07-01 16:25:58 +02001958 fuse_copy_init(&cs, 0, from);
Al Virofbdbacc2015-04-03 21:53:39 -04001959
Miklos Szeredic36960462015-07-01 16:26:09 +02001960 return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001961}
1962
1963static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1964 struct file *out, loff_t *ppos,
1965 size_t len, unsigned int flags)
1966{
David Howells8cefc102019-11-15 13:30:32 +00001967 unsigned int head, tail, mask, count;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001968 unsigned nbuf;
1969 unsigned idx;
1970 struct pipe_buffer *bufs;
1971 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001972 struct fuse_dev *fud;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001973 size_t rem;
1974 ssize_t ret;
1975
Miklos Szeredicc080e92015-07-01 16:26:08 +02001976 fud = fuse_get_dev(out);
1977 if (!fud)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001978 return -EPERM;
1979
Andrey Ryabinina2477b02018-07-17 19:00:33 +03001980 pipe_lock(pipe);
1981
David Howells8cefc102019-11-15 13:30:32 +00001982 head = pipe->head;
1983 tail = pipe->tail;
1984 mask = pipe->ring_size - 1;
1985 count = head - tail;
1986
1987 bufs = kvmalloc_array(count, sizeof(struct pipe_buffer), GFP_KERNEL);
Andrey Ryabinina2477b02018-07-17 19:00:33 +03001988 if (!bufs) {
1989 pipe_unlock(pipe);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001990 return -ENOMEM;
Andrey Ryabinina2477b02018-07-17 19:00:33 +03001991 }
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001992
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001993 nbuf = 0;
1994 rem = 0;
David Howells76f67772019-12-06 21:34:51 +00001995 for (idx = tail; idx != head && rem < len; idx++)
David Howells8cefc102019-11-15 13:30:32 +00001996 rem += pipe->bufs[idx & mask].len;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001997
1998 ret = -EINVAL;
Matthew Wilcox15fab632019-04-05 14:02:10 -07001999 if (rem < len)
2000 goto out_free;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002001
2002 rem = len;
2003 while (rem) {
2004 struct pipe_buffer *ibuf;
2005 struct pipe_buffer *obuf;
2006
Vasily Averin0e9fb6f2019-08-19 09:53:50 +03002007 if (WARN_ON(nbuf >= count || tail == head))
2008 goto out_free;
2009
David Howells8cefc102019-11-15 13:30:32 +00002010 ibuf = &pipe->bufs[tail & mask];
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002011 obuf = &bufs[nbuf];
2012
2013 if (rem >= ibuf->len) {
2014 *obuf = *ibuf;
2015 ibuf->ops = NULL;
David Howells8cefc102019-11-15 13:30:32 +00002016 tail++;
2017 pipe->tail = tail;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002018 } else {
Matthew Wilcox15fab632019-04-05 14:02:10 -07002019 if (!pipe_buf_get(pipe, ibuf))
2020 goto out_free;
2021
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002022 *obuf = *ibuf;
2023 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2024 obuf->len = rem;
2025 ibuf->offset += obuf->len;
2026 ibuf->len -= obuf->len;
2027 }
2028 nbuf++;
2029 rem -= obuf->len;
2030 }
2031 pipe_unlock(pipe);
2032
Miklos Szeredidc008092015-07-01 16:25:58 +02002033 fuse_copy_init(&cs, 0, NULL);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002034 cs.pipebufs = bufs;
Al Viro6c09e942015-04-03 22:06:08 -04002035 cs.nr_segs = nbuf;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002036 cs.pipe = pipe;
2037
Miklos Szeredice534fb2010-05-25 15:06:07 +02002038 if (flags & SPLICE_F_MOVE)
2039 cs.move_pages = 1;
2040
Miklos Szeredic36960462015-07-01 16:26:09 +02002041 ret = fuse_dev_do_write(fud, &cs, len);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002042
Jann Horn95099412019-01-12 02:39:05 +01002043 pipe_lock(pipe);
Matthew Wilcox15fab632019-04-05 14:02:10 -07002044out_free:
Miklos Szeredi712a9512021-11-02 11:10:37 +01002045 for (idx = 0; idx < nbuf; idx++) {
2046 struct pipe_buffer *buf = &bufs[idx];
2047
2048 if (buf->ops)
2049 pipe_buf_release(pipe, buf);
2050 }
Jann Horn95099412019-01-12 02:39:05 +01002051 pipe_unlock(pipe);
Miklos Szeredia7796382016-09-27 10:45:12 +02002052
Andrey Ryabinind6d931a2018-07-17 19:00:34 +03002053 kvfree(bufs);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002054 return ret;
2055}
2056
Al Viro076ccb72017-07-03 01:02:18 -04002057static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
Miklos Szeredi334f4852005-09-09 13:10:27 -07002058{
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002059 __poll_t mask = EPOLLOUT | EPOLLWRNORM;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002060 struct fuse_iqueue *fiq;
Miklos Szeredicc080e92015-07-01 16:26:08 +02002061 struct fuse_dev *fud = fuse_get_dev(file);
2062
2063 if (!fud)
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002064 return EPOLLERR;
Miklos Szeredi334f4852005-09-09 13:10:27 -07002065
Miklos Szeredicc080e92015-07-01 16:26:08 +02002066 fiq = &fud->fc->iq;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002067 poll_wait(file, &fiq->waitq, wait);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002068
Eric Biggers76e43c82019-09-08 20:15:18 -07002069 spin_lock(&fiq->lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +02002070 if (!fiq->connected)
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002071 mask = EPOLLERR;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002072 else if (request_pending(fiq))
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002073 mask |= EPOLLIN | EPOLLRDNORM;
Eric Biggers76e43c82019-09-08 20:15:18 -07002074 spin_unlock(&fiq->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002075
2076 return mask;
2077}
2078
Kirill Tkhai34061752018-11-06 12:15:20 +03002079/* Abort all requests on the given list (pending or processing) */
Max Reitz8f622e92020-04-20 17:59:34 +02002080static void end_requests(struct list_head *head)
Miklos Szeredi334f4852005-09-09 13:10:27 -07002081{
2082 while (!list_empty(head)) {
2083 struct fuse_req *req;
2084 req = list_entry(head->next, struct fuse_req, list);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002085 req->out.h.error = -ECONNABORTED;
Miklos Szeredi33e14b42015-07-01 16:26:01 +02002086 clear_bit(FR_SENT, &req->flags);
Miklos Szeredif377cb72015-07-01 16:26:04 +02002087 list_del_init(&req->list);
Max Reitz8f622e92020-04-20 17:59:34 +02002088 fuse_request_end(req);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002089 }
2090}
2091
Bryan Green357ccf22011-03-01 16:43:52 -08002092static void end_polls(struct fuse_conn *fc)
2093{
2094 struct rb_node *p;
2095
2096 p = rb_first(&fc->polled_files);
2097
2098 while (p) {
2099 struct fuse_file *ff;
2100 ff = rb_entry(p, struct fuse_file, polled_node);
2101 wake_up_interruptible_all(&ff->poll_wait);
2102
2103 p = rb_next(p);
2104 }
2105}
2106
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002107/*
2108 * Abort all requests.
2109 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002110 * Emergency exit in case of a malicious or accidental deadlock, or just a hung
2111 * filesystem.
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002112 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002113 * The same effect is usually achievable through killing the filesystem daemon
2114 * and all users of the filesystem. The exception is the combination of an
2115 * asynchronous request and the tricky deadlock (see
Mauro Carvalho Chehab72ef5e52020-04-14 18:48:35 +02002116 * Documentation/filesystems/fuse.rst).
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002117 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002118 * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2119 * requests, they should be finished off immediately. Locked requests will be
2120 * finished after unlock; see unlock_request(). 2: Finish off the unlocked
2121 * requests. It is possible that some request will finish before we can. This
2122 * is OK, the request will in that case be removed from the list before we touch
2123 * it.
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002124 */
Miklos Szeredieb98e3bd2019-01-24 10:40:16 +01002125void fuse_abort_conn(struct fuse_conn *fc)
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002126{
Miklos Szeredif88996a2015-07-01 16:26:01 +02002127 struct fuse_iqueue *fiq = &fc->iq;
2128
Miklos Szeredid7133112006-04-10 22:54:55 -07002129 spin_lock(&fc->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002130 if (fc->connected) {
Miklos Szeredic36960462015-07-01 16:26:09 +02002131 struct fuse_dev *fud;
Miklos Szeredib716d422015-07-01 16:25:59 +02002132 struct fuse_req *req, *next;
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002133 LIST_HEAD(to_end);
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03002134 unsigned int i;
Miklos Szeredib716d422015-07-01 16:25:59 +02002135
Kirill Tkhai63825b42018-08-27 18:29:56 +03002136 /* Background queuing checks fc->connected under bg_lock */
2137 spin_lock(&fc->bg_lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002138 fc->connected = 0;
Kirill Tkhai63825b42018-08-27 18:29:56 +03002139 spin_unlock(&fc->bg_lock);
2140
Miklos Szeredi9759bd512015-01-06 10:45:35 +01002141 fuse_set_initialized(fc);
Miklos Szeredic36960462015-07-01 16:26:09 +02002142 list_for_each_entry(fud, &fc->devices, entry) {
2143 struct fuse_pqueue *fpq = &fud->pq;
2144
2145 spin_lock(&fpq->lock);
2146 fpq->connected = 0;
2147 list_for_each_entry_safe(req, next, &fpq->io, list) {
2148 req->out.h.error = -ECONNABORTED;
2149 spin_lock(&req->waitq.lock);
2150 set_bit(FR_ABORTED, &req->flags);
2151 if (!test_bit(FR_LOCKED, &req->flags)) {
2152 set_bit(FR_PRIVATE, &req->flags);
Miklos Szeredi87114372018-07-26 16:13:11 +02002153 __fuse_get_request(req);
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002154 list_move(&req->list, &to_end);
Miklos Szeredic36960462015-07-01 16:26:09 +02002155 }
2156 spin_unlock(&req->waitq.lock);
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02002157 }
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03002158 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2159 list_splice_tail_init(&fpq->processing[i],
2160 &to_end);
Miklos Szeredic36960462015-07-01 16:26:09 +02002161 spin_unlock(&fpq->lock);
Miklos Szeredib716d422015-07-01 16:25:59 +02002162 }
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +03002163 spin_lock(&fc->bg_lock);
2164 fc->blocked = 0;
Miklos Szeredi41f98272015-07-01 16:25:59 +02002165 fc->max_background = UINT_MAX;
2166 flush_bg_queue(fc);
Kirill Tkhaiae2dffa2018-08-27 18:29:46 +03002167 spin_unlock(&fc->bg_lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002168
Eric Biggers76e43c82019-09-08 20:15:18 -07002169 spin_lock(&fiq->lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002170 fiq->connected = 0;
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002171 list_for_each_entry(req, &fiq->pending, list)
Tahsin Erdogana8a86d72017-01-12 12:04:04 -08002172 clear_bit(FR_PENDING, &req->flags);
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002173 list_splice_tail_init(&fiq->pending, &to_end);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002174 while (forget_pending(fiq))
Vivek Goyal4388c5a2019-06-05 15:50:43 -04002175 kfree(fuse_dequeue_forget(fiq, 1, NULL));
Eric Biggers76e43c82019-09-08 20:15:18 -07002176 wake_up_all(&fiq->waitq);
2177 spin_unlock(&fiq->lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002178 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szerediee314a82015-07-01 16:26:08 +02002179 end_polls(fc);
2180 wake_up_all(&fc->blocked_waitq);
2181 spin_unlock(&fc->lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002182
Max Reitz8f622e92020-04-20 17:59:34 +02002183 end_requests(&to_end);
Miklos Szerediee314a82015-07-01 16:26:08 +02002184 } else {
2185 spin_unlock(&fc->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002186 }
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002187}
Tejun Heo08cbf5422009-04-14 10:54:53 +09002188EXPORT_SYMBOL_GPL(fuse_abort_conn);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002189
Miklos Szeredib8f95e52018-07-26 16:13:11 +02002190void fuse_wait_aborted(struct fuse_conn *fc)
2191{
Miklos Szeredi2d84a2d2018-11-09 15:52:16 +01002192 /* matches implicit memory barrier in fuse_drop_waiting() */
2193 smp_mb();
Miklos Szeredib8f95e52018-07-26 16:13:11 +02002194 wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
2195}
2196
Tejun Heo08cbf5422009-04-14 10:54:53 +09002197int fuse_dev_release(struct inode *inode, struct file *file)
Miklos Szeredi334f4852005-09-09 13:10:27 -07002198{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002199 struct fuse_dev *fud = fuse_get_dev(file);
2200
2201 if (fud) {
2202 struct fuse_conn *fc = fud->fc;
Miklos Szeredic36960462015-07-01 16:26:09 +02002203 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi45ff3502018-07-26 16:13:11 +02002204 LIST_HEAD(to_end);
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03002205 unsigned int i;
Miklos Szeredicc080e92015-07-01 16:26:08 +02002206
Miklos Szeredi45ff3502018-07-26 16:13:11 +02002207 spin_lock(&fpq->lock);
Miklos Szeredic36960462015-07-01 16:26:09 +02002208 WARN_ON(!list_empty(&fpq->io));
Kirill Tkhaibe2ff422018-09-11 13:12:14 +03002209 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2210 list_splice_init(&fpq->processing[i], &to_end);
Miklos Szeredi45ff3502018-07-26 16:13:11 +02002211 spin_unlock(&fpq->lock);
2212
Max Reitz8f622e92020-04-20 17:59:34 +02002213 end_requests(&to_end);
Miklos Szeredi45ff3502018-07-26 16:13:11 +02002214
Miklos Szeredic36960462015-07-01 16:26:09 +02002215 /* Are we the last open device? */
2216 if (atomic_dec_and_test(&fc->dev_count)) {
2217 WARN_ON(fc->iq.fasync != NULL);
Miklos Szeredieb98e3bd2019-01-24 10:40:16 +01002218 fuse_abort_conn(fc);
Miklos Szeredic36960462015-07-01 16:26:09 +02002219 }
Miklos Szeredicc080e92015-07-01 16:26:08 +02002220 fuse_dev_free(fud);
Jeff Dike385a17b2006-04-10 22:54:52 -07002221 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07002222 return 0;
2223}
Tejun Heo08cbf5422009-04-14 10:54:53 +09002224EXPORT_SYMBOL_GPL(fuse_dev_release);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002225
Jeff Dike385a17b2006-04-10 22:54:52 -07002226static int fuse_dev_fasync(int fd, struct file *file, int on)
2227{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002228 struct fuse_dev *fud = fuse_get_dev(file);
2229
2230 if (!fud)
Miklos Szeredia87046d2006-04-10 22:54:56 -07002231 return -EPERM;
Jeff Dike385a17b2006-04-10 22:54:52 -07002232
2233 /* No locking - fasync_helper does its own locking */
Miklos Szeredicc080e92015-07-01 16:26:08 +02002234 return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
Jeff Dike385a17b2006-04-10 22:54:52 -07002235}
2236
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002237static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
2238{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002239 struct fuse_dev *fud;
2240
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002241 if (new->private_data)
2242 return -EINVAL;
2243
Vivek Goyal0cd1eb9a2019-03-06 16:51:40 -05002244 fud = fuse_dev_alloc_install(fc);
Miklos Szeredicc080e92015-07-01 16:26:08 +02002245 if (!fud)
2246 return -ENOMEM;
2247
2248 new->private_data = fud;
Miklos Szeredic36960462015-07-01 16:26:09 +02002249 atomic_inc(&fc->dev_count);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002250
2251 return 0;
2252}
2253
2254static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
2255 unsigned long arg)
2256{
Alessio Balsinif8425c92021-01-25 15:30:51 +00002257 int res;
2258 int oldfd;
2259 struct fuse_dev *fud = NULL;
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002260
Alessio Balsini6076f5f2021-03-19 15:05:14 +00002261 switch (cmd) {
2262 case FUSE_DEV_IOC_CLONE:
Alessio Balsinif8425c92021-01-25 15:30:51 +00002263 res = -EFAULT;
2264 if (!get_user(oldfd, (__u32 __user *)arg)) {
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002265 struct file *old = fget(oldfd);
2266
Alessio Balsinif8425c92021-01-25 15:30:51 +00002267 res = -EINVAL;
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002268 if (old) {
Jann Horn8ed1f0e2015-08-16 20:27:01 +02002269 /*
2270 * Check against file->f_op because CUSE
2271 * uses the same ioctl handler.
2272 */
Jann Horn06180212022-09-14 16:26:32 +02002273 if (old->f_op == file->f_op)
Jann Horn8ed1f0e2015-08-16 20:27:01 +02002274 fud = fuse_get_dev(old);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002275
Miklos Szeredicc080e92015-07-01 16:26:08 +02002276 if (fud) {
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002277 mutex_lock(&fuse_mutex);
Alessio Balsinif8425c92021-01-25 15:30:51 +00002278 res = fuse_device_clone(fud->fc, file);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002279 mutex_unlock(&fuse_mutex);
2280 }
2281 fput(old);
2282 }
2283 }
Alessio Balsinif8425c92021-01-25 15:30:51 +00002284 break;
2285 default:
2286 res = -ENOTTY;
2287 break;
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002288 }
Alessio Balsinif8425c92021-01-25 15:30:51 +00002289 return res;
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002290}
2291
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -08002292const struct file_operations fuse_dev_operations = {
Miklos Szeredi334f4852005-09-09 13:10:27 -07002293 .owner = THIS_MODULE,
Tom Van Braeckel94e4fe22015-01-12 05:22:16 +01002294 .open = fuse_dev_open,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002295 .llseek = no_llseek,
Al Virofbdbacc2015-04-03 21:53:39 -04002296 .read_iter = fuse_dev_read,
Miklos Szeredic3021622010-05-25 15:06:07 +02002297 .splice_read = fuse_dev_splice_read,
Al Virofbdbacc2015-04-03 21:53:39 -04002298 .write_iter = fuse_dev_write,
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002299 .splice_write = fuse_dev_splice_write,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002300 .poll = fuse_dev_poll,
2301 .release = fuse_dev_release,
Jeff Dike385a17b2006-04-10 22:54:52 -07002302 .fasync = fuse_dev_fasync,
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002303 .unlocked_ioctl = fuse_dev_ioctl,
Arnd Bergmann1832f2d2018-09-11 21:59:08 +02002304 .compat_ioctl = compat_ptr_ioctl,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002305};
Tejun Heo08cbf5422009-04-14 10:54:53 +09002306EXPORT_SYMBOL_GPL(fuse_dev_operations);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002307
2308static struct miscdevice fuse_miscdevice = {
2309 .minor = FUSE_MINOR,
2310 .name = "fuse",
2311 .fops = &fuse_dev_operations,
2312};
2313
2314int __init fuse_dev_init(void)
2315{
2316 int err = -ENOMEM;
2317 fuse_req_cachep = kmem_cache_create("fuse_request",
2318 sizeof(struct fuse_req),
Paul Mundt20c2df82007-07-20 10:11:58 +09002319 0, 0, NULL);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002320 if (!fuse_req_cachep)
2321 goto out;
2322
2323 err = misc_register(&fuse_miscdevice);
2324 if (err)
2325 goto out_cache_clean;
2326
2327 return 0;
2328
2329 out_cache_clean:
2330 kmem_cache_destroy(fuse_req_cachep);
2331 out:
2332 return err;
2333}
2334
2335void fuse_dev_cleanup(void)
2336{
2337 misc_deregister(&fuse_miscdevice);
2338 kmem_cache_destroy(fuse_req_cachep);
2339}