blob: 6af3f131e468ad3ee7bedffe41f4bb0b6b03e91c [file] [log] [blame]
Stefan Hajnoczia62a8ef2018-06-12 09:41:17 +01001// SPDX-License-Identifier: GPL-2.0
2/*
3 * virtio-fs: Virtio Filesystem
4 * Copyright (C) 2018 Red Hat, Inc.
5 */
6
7#include <linux/fs.h>
8#include <linux/module.h>
9#include <linux/virtio.h>
10#include <linux/virtio_fs.h>
11#include <linux/delay.h>
12#include <linux/fs_context.h>
13#include <linux/highmem.h>
14#include "fuse_i.h"
15
16/* List of virtio-fs device instances and a lock for the list. Also provides
17 * mutual exclusion in device removal and mounting path
18 */
19static DEFINE_MUTEX(virtio_fs_mutex);
20static LIST_HEAD(virtio_fs_instances);
21
22enum {
23 VQ_HIPRIO,
24 VQ_REQUEST
25};
26
27/* Per-virtqueue state */
28struct virtio_fs_vq {
29 spinlock_t lock;
30 struct virtqueue *vq; /* protected by ->lock */
31 struct work_struct done_work;
32 struct list_head queued_reqs;
33 struct delayed_work dispatch_work;
34 struct fuse_dev *fud;
35 bool connected;
36 long in_flight;
37 char name[24];
38} ____cacheline_aligned_in_smp;
39
40/* A virtio-fs device instance */
41struct virtio_fs {
42 struct kref refcount;
43 struct list_head list; /* on virtio_fs_instances */
44 char *tag;
45 struct virtio_fs_vq *vqs;
46 unsigned int nvqs; /* number of virtqueues */
47 unsigned int num_request_queues; /* number of request queues */
48};
49
50struct virtio_fs_forget {
51 struct fuse_in_header ih;
52 struct fuse_forget_in arg;
53 /* This request can be temporarily queued on virt queue */
54 struct list_head list;
55};
56
57static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
58{
59 struct virtio_fs *fs = vq->vdev->priv;
60
61 return &fs->vqs[vq->index];
62}
63
64static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)
65{
66 return &vq_to_fsvq(vq)->fud->pq;
67}
68
69static void release_virtio_fs_obj(struct kref *ref)
70{
71 struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
72
73 kfree(vfs->vqs);
74 kfree(vfs);
75}
76
77/* Make sure virtiofs_mutex is held */
78static void virtio_fs_put(struct virtio_fs *fs)
79{
80 kref_put(&fs->refcount, release_virtio_fs_obj);
81}
82
83static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
84{
85 struct virtio_fs *vfs = fiq->priv;
86
87 mutex_lock(&virtio_fs_mutex);
88 virtio_fs_put(vfs);
89 mutex_unlock(&virtio_fs_mutex);
90}
91
92static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
93{
94 WARN_ON(fsvq->in_flight < 0);
95
96 /* Wait for in flight requests to finish.*/
97 while (1) {
98 spin_lock(&fsvq->lock);
99 if (!fsvq->in_flight) {
100 spin_unlock(&fsvq->lock);
101 break;
102 }
103 spin_unlock(&fsvq->lock);
104 /* TODO use completion instead of timeout */
105 usleep_range(1000, 2000);
106 }
107
108 flush_work(&fsvq->done_work);
109 flush_delayed_work(&fsvq->dispatch_work);
110}
111
112static inline void drain_hiprio_queued_reqs(struct virtio_fs_vq *fsvq)
113{
114 struct virtio_fs_forget *forget;
115
116 spin_lock(&fsvq->lock);
117 while (1) {
118 forget = list_first_entry_or_null(&fsvq->queued_reqs,
119 struct virtio_fs_forget, list);
120 if (!forget)
121 break;
122 list_del(&forget->list);
123 kfree(forget);
124 }
125 spin_unlock(&fsvq->lock);
126}
127
128static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
129{
130 struct virtio_fs_vq *fsvq;
131 int i;
132
133 for (i = 0; i < fs->nvqs; i++) {
134 fsvq = &fs->vqs[i];
135 if (i == VQ_HIPRIO)
136 drain_hiprio_queued_reqs(fsvq);
137
138 virtio_fs_drain_queue(fsvq);
139 }
140}
141
142static void virtio_fs_start_all_queues(struct virtio_fs *fs)
143{
144 struct virtio_fs_vq *fsvq;
145 int i;
146
147 for (i = 0; i < fs->nvqs; i++) {
148 fsvq = &fs->vqs[i];
149 spin_lock(&fsvq->lock);
150 fsvq->connected = true;
151 spin_unlock(&fsvq->lock);
152 }
153}
154
155/* Add a new instance to the list or return -EEXIST if tag name exists*/
156static int virtio_fs_add_instance(struct virtio_fs *fs)
157{
158 struct virtio_fs *fs2;
159 bool duplicate = false;
160
161 mutex_lock(&virtio_fs_mutex);
162
163 list_for_each_entry(fs2, &virtio_fs_instances, list) {
164 if (strcmp(fs->tag, fs2->tag) == 0)
165 duplicate = true;
166 }
167
168 if (!duplicate)
169 list_add_tail(&fs->list, &virtio_fs_instances);
170
171 mutex_unlock(&virtio_fs_mutex);
172
173 if (duplicate)
174 return -EEXIST;
175 return 0;
176}
177
178/* Return the virtio_fs with a given tag, or NULL */
179static struct virtio_fs *virtio_fs_find_instance(const char *tag)
180{
181 struct virtio_fs *fs;
182
183 mutex_lock(&virtio_fs_mutex);
184
185 list_for_each_entry(fs, &virtio_fs_instances, list) {
186 if (strcmp(fs->tag, tag) == 0) {
187 kref_get(&fs->refcount);
188 goto found;
189 }
190 }
191
192 fs = NULL; /* not found */
193
194found:
195 mutex_unlock(&virtio_fs_mutex);
196
197 return fs;
198}
199
200static void virtio_fs_free_devs(struct virtio_fs *fs)
201{
202 unsigned int i;
203
204 for (i = 0; i < fs->nvqs; i++) {
205 struct virtio_fs_vq *fsvq = &fs->vqs[i];
206
207 if (!fsvq->fud)
208 continue;
209
210 fuse_dev_free(fsvq->fud);
211 fsvq->fud = NULL;
212 }
213}
214
215/* Read filesystem name from virtio config into fs->tag (must kfree()). */
216static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs)
217{
218 char tag_buf[sizeof_field(struct virtio_fs_config, tag)];
219 char *end;
220 size_t len;
221
222 virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag),
223 &tag_buf, sizeof(tag_buf));
224 end = memchr(tag_buf, '\0', sizeof(tag_buf));
225 if (end == tag_buf)
226 return -EINVAL; /* empty tag */
227 if (!end)
228 end = &tag_buf[sizeof(tag_buf)];
229
230 len = end - tag_buf;
231 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL);
232 if (!fs->tag)
233 return -ENOMEM;
234 memcpy(fs->tag, tag_buf, len);
235 fs->tag[len] = '\0';
236 return 0;
237}
238
239/* Work function for hiprio completion */
240static void virtio_fs_hiprio_done_work(struct work_struct *work)
241{
242 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
243 done_work);
244 struct virtqueue *vq = fsvq->vq;
245
246 /* Free completed FUSE_FORGET requests */
247 spin_lock(&fsvq->lock);
248 do {
249 unsigned int len;
250 void *req;
251
252 virtqueue_disable_cb(vq);
253
254 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
255 kfree(req);
256 fsvq->in_flight--;
257 }
258 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
259 spin_unlock(&fsvq->lock);
260}
261
262static void virtio_fs_dummy_dispatch_work(struct work_struct *work)
263{
264}
265
266static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
267{
268 struct virtio_fs_forget *forget;
269 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
270 dispatch_work.work);
271 struct virtqueue *vq = fsvq->vq;
272 struct scatterlist sg;
273 struct scatterlist *sgs[] = {&sg};
274 bool notify;
275 int ret;
276
277 pr_debug("virtio-fs: worker %s called.\n", __func__);
278 while (1) {
279 spin_lock(&fsvq->lock);
280 forget = list_first_entry_or_null(&fsvq->queued_reqs,
281 struct virtio_fs_forget, list);
282 if (!forget) {
283 spin_unlock(&fsvq->lock);
284 return;
285 }
286
287 list_del(&forget->list);
288 if (!fsvq->connected) {
289 spin_unlock(&fsvq->lock);
290 kfree(forget);
291 continue;
292 }
293
294 sg_init_one(&sg, forget, sizeof(*forget));
295
296 /* Enqueue the request */
297 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
298 ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
299 if (ret < 0) {
300 if (ret == -ENOMEM || ret == -ENOSPC) {
301 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
302 ret);
303 list_add_tail(&forget->list,
304 &fsvq->queued_reqs);
305 schedule_delayed_work(&fsvq->dispatch_work,
306 msecs_to_jiffies(1));
307 } else {
308 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
309 ret);
310 kfree(forget);
311 }
312 spin_unlock(&fsvq->lock);
313 return;
314 }
315
316 fsvq->in_flight++;
317 notify = virtqueue_kick_prepare(vq);
318 spin_unlock(&fsvq->lock);
319
320 if (notify)
321 virtqueue_notify(vq);
322 pr_debug("virtio-fs: worker %s dispatched one forget request.\n",
323 __func__);
324 }
325}
326
327/* Allocate and copy args into req->argbuf */
328static int copy_args_to_argbuf(struct fuse_req *req)
329{
330 struct fuse_args *args = req->args;
331 unsigned int offset = 0;
332 unsigned int num_in;
333 unsigned int num_out;
334 unsigned int len;
335 unsigned int i;
336
337 num_in = args->in_numargs - args->in_pages;
338 num_out = args->out_numargs - args->out_pages;
339 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
340 fuse_len_args(num_out, args->out_args);
341
342 req->argbuf = kmalloc(len, GFP_ATOMIC);
343 if (!req->argbuf)
344 return -ENOMEM;
345
346 for (i = 0; i < num_in; i++) {
347 memcpy(req->argbuf + offset,
348 args->in_args[i].value,
349 args->in_args[i].size);
350 offset += args->in_args[i].size;
351 }
352
353 return 0;
354}
355
356/* Copy args out of and free req->argbuf */
357static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
358{
359 unsigned int remaining;
360 unsigned int offset;
361 unsigned int num_in;
362 unsigned int num_out;
363 unsigned int i;
364
365 remaining = req->out.h.len - sizeof(req->out.h);
366 num_in = args->in_numargs - args->in_pages;
367 num_out = args->out_numargs - args->out_pages;
368 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
369
370 for (i = 0; i < num_out; i++) {
371 unsigned int argsize = args->out_args[i].size;
372
373 if (args->out_argvar &&
374 i == args->out_numargs - 1 &&
375 argsize > remaining) {
376 argsize = remaining;
377 }
378
379 memcpy(args->out_args[i].value, req->argbuf + offset, argsize);
380 offset += argsize;
381
382 if (i != args->out_numargs - 1)
383 remaining -= argsize;
384 }
385
386 /* Store the actual size of the variable-length arg */
387 if (args->out_argvar)
388 args->out_args[args->out_numargs - 1].size = remaining;
389
390 kfree(req->argbuf);
391 req->argbuf = NULL;
392}
393
394/* Work function for request completion */
395static void virtio_fs_requests_done_work(struct work_struct *work)
396{
397 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
398 done_work);
399 struct fuse_pqueue *fpq = &fsvq->fud->pq;
400 struct fuse_conn *fc = fsvq->fud->fc;
401 struct virtqueue *vq = fsvq->vq;
402 struct fuse_req *req;
403 struct fuse_args_pages *ap;
404 struct fuse_req *next;
405 struct fuse_args *args;
406 unsigned int len, i, thislen;
407 struct page *page;
408 LIST_HEAD(reqs);
409
410 /* Collect completed requests off the virtqueue */
411 spin_lock(&fsvq->lock);
412 do {
413 virtqueue_disable_cb(vq);
414
415 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
416 spin_lock(&fpq->lock);
417 list_move_tail(&req->list, &reqs);
418 spin_unlock(&fpq->lock);
419 }
420 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
421 spin_unlock(&fsvq->lock);
422
423 /* End requests */
424 list_for_each_entry_safe(req, next, &reqs, list) {
425 /*
426 * TODO verify that server properly follows FUSE protocol
427 * (oh.uniq, oh.len)
428 */
429 args = req->args;
430 copy_args_from_argbuf(args, req);
431
432 if (args->out_pages && args->page_zeroing) {
433 len = args->out_args[args->out_numargs - 1].size;
434 ap = container_of(args, typeof(*ap), args);
435 for (i = 0; i < ap->num_pages; i++) {
436 thislen = ap->descs[i].length;
437 if (len < thislen) {
438 WARN_ON(ap->descs[i].offset);
439 page = ap->pages[i];
440 zero_user_segment(page, len, thislen);
441 len = 0;
442 } else {
443 len -= thislen;
444 }
445 }
446 }
447
448 spin_lock(&fpq->lock);
449 clear_bit(FR_SENT, &req->flags);
450 list_del_init(&req->list);
451 spin_unlock(&fpq->lock);
452
453 fuse_request_end(fc, req);
454 spin_lock(&fsvq->lock);
455 fsvq->in_flight--;
456 spin_unlock(&fsvq->lock);
457 }
458}
459
460/* Virtqueue interrupt handler */
461static void virtio_fs_vq_done(struct virtqueue *vq)
462{
463 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
464
465 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
466
467 schedule_work(&fsvq->done_work);
468}
469
470/* Initialize virtqueues */
471static int virtio_fs_setup_vqs(struct virtio_device *vdev,
472 struct virtio_fs *fs)
473{
474 struct virtqueue **vqs;
475 vq_callback_t **callbacks;
476 const char **names;
477 unsigned int i;
478 int ret = 0;
479
480 virtio_cread(vdev, struct virtio_fs_config, num_request_queues,
481 &fs->num_request_queues);
482 if (fs->num_request_queues == 0)
483 return -EINVAL;
484
485 fs->nvqs = 1 + fs->num_request_queues;
486 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
487 if (!fs->vqs)
488 return -ENOMEM;
489
490 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
491 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
492 GFP_KERNEL);
493 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
494 if (!vqs || !callbacks || !names) {
495 ret = -ENOMEM;
496 goto out;
497 }
498
499 callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
500 snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name),
501 "hiprio");
502 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
503 INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work);
504 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs);
505 INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
506 virtio_fs_hiprio_dispatch_work);
507 spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
508
509 /* Initialize the requests virtqueues */
510 for (i = VQ_REQUEST; i < fs->nvqs; i++) {
511 spin_lock_init(&fs->vqs[i].lock);
512 INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work);
513 INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work,
514 virtio_fs_dummy_dispatch_work);
515 INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
516 snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
517 "requests.%u", i - VQ_REQUEST);
518 callbacks[i] = virtio_fs_vq_done;
519 names[i] = fs->vqs[i].name;
520 }
521
522 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
523 if (ret < 0)
524 goto out;
525
526 for (i = 0; i < fs->nvqs; i++)
527 fs->vqs[i].vq = vqs[i];
528
529 virtio_fs_start_all_queues(fs);
530out:
531 kfree(names);
532 kfree(callbacks);
533 kfree(vqs);
534 if (ret)
535 kfree(fs->vqs);
536 return ret;
537}
538
539/* Free virtqueues (device must already be reset) */
540static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
541 struct virtio_fs *fs)
542{
543 vdev->config->del_vqs(vdev);
544}
545
546static int virtio_fs_probe(struct virtio_device *vdev)
547{
548 struct virtio_fs *fs;
549 int ret;
550
551 fs = kzalloc(sizeof(*fs), GFP_KERNEL);
552 if (!fs)
553 return -ENOMEM;
554 kref_init(&fs->refcount);
555 vdev->priv = fs;
556
557 ret = virtio_fs_read_tag(vdev, fs);
558 if (ret < 0)
559 goto out;
560
561 ret = virtio_fs_setup_vqs(vdev, fs);
562 if (ret < 0)
563 goto out;
564
565 /* TODO vq affinity */
566
567 /* Bring the device online in case the filesystem is mounted and
568 * requests need to be sent before we return.
569 */
570 virtio_device_ready(vdev);
571
572 ret = virtio_fs_add_instance(fs);
573 if (ret < 0)
574 goto out_vqs;
575
576 return 0;
577
578out_vqs:
579 vdev->config->reset(vdev);
580 virtio_fs_cleanup_vqs(vdev, fs);
581
582out:
583 vdev->priv = NULL;
584 kfree(fs);
585 return ret;
586}
587
588static void virtio_fs_stop_all_queues(struct virtio_fs *fs)
589{
590 struct virtio_fs_vq *fsvq;
591 int i;
592
593 for (i = 0; i < fs->nvqs; i++) {
594 fsvq = &fs->vqs[i];
595 spin_lock(&fsvq->lock);
596 fsvq->connected = false;
597 spin_unlock(&fsvq->lock);
598 }
599}
600
601static void virtio_fs_remove(struct virtio_device *vdev)
602{
603 struct virtio_fs *fs = vdev->priv;
604
605 mutex_lock(&virtio_fs_mutex);
606 /* This device is going away. No one should get new reference */
607 list_del_init(&fs->list);
608 virtio_fs_stop_all_queues(fs);
609 virtio_fs_drain_all_queues(fs);
610 vdev->config->reset(vdev);
611 virtio_fs_cleanup_vqs(vdev, fs);
612
613 vdev->priv = NULL;
614 /* Put device reference on virtio_fs object */
615 virtio_fs_put(fs);
616 mutex_unlock(&virtio_fs_mutex);
617}
618
619#ifdef CONFIG_PM_SLEEP
620static int virtio_fs_freeze(struct virtio_device *vdev)
621{
622 /* TODO need to save state here */
623 pr_warn("virtio-fs: suspend/resume not yet supported\n");
624 return -EOPNOTSUPP;
625}
626
627static int virtio_fs_restore(struct virtio_device *vdev)
628{
629 /* TODO need to restore state here */
630 return 0;
631}
632#endif /* CONFIG_PM_SLEEP */
633
634const static struct virtio_device_id id_table[] = {
635 { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
636 {},
637};
638
639const static unsigned int feature_table[] = {};
640
641static struct virtio_driver virtio_fs_driver = {
642 .driver.name = KBUILD_MODNAME,
643 .driver.owner = THIS_MODULE,
644 .id_table = id_table,
645 .feature_table = feature_table,
646 .feature_table_size = ARRAY_SIZE(feature_table),
647 .probe = virtio_fs_probe,
648 .remove = virtio_fs_remove,
649#ifdef CONFIG_PM_SLEEP
650 .freeze = virtio_fs_freeze,
651 .restore = virtio_fs_restore,
652#endif
653};
654
655static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
656__releases(fiq->lock)
657{
658 struct fuse_forget_link *link;
659 struct virtio_fs_forget *forget;
660 struct scatterlist sg;
661 struct scatterlist *sgs[] = {&sg};
662 struct virtio_fs *fs;
663 struct virtqueue *vq;
664 struct virtio_fs_vq *fsvq;
665 bool notify;
666 u64 unique;
667 int ret;
668
669 link = fuse_dequeue_forget(fiq, 1, NULL);
670 unique = fuse_get_unique(fiq);
671
672 fs = fiq->priv;
673 fsvq = &fs->vqs[VQ_HIPRIO];
674 spin_unlock(&fiq->lock);
675
676 /* Allocate a buffer for the request */
677 forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
678
679 forget->ih = (struct fuse_in_header){
680 .opcode = FUSE_FORGET,
681 .nodeid = link->forget_one.nodeid,
682 .unique = unique,
683 .len = sizeof(*forget),
684 };
685 forget->arg = (struct fuse_forget_in){
686 .nlookup = link->forget_one.nlookup,
687 };
688
689 sg_init_one(&sg, forget, sizeof(*forget));
690
691 /* Enqueue the request */
692 spin_lock(&fsvq->lock);
693
694 if (!fsvq->connected) {
695 kfree(forget);
696 spin_unlock(&fsvq->lock);
697 goto out;
698 }
699
700 vq = fsvq->vq;
701 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
702
703 ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
704 if (ret < 0) {
705 if (ret == -ENOMEM || ret == -ENOSPC) {
706 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later.\n",
707 ret);
708 list_add_tail(&forget->list, &fsvq->queued_reqs);
709 schedule_delayed_work(&fsvq->dispatch_work,
710 msecs_to_jiffies(1));
711 } else {
712 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
713 ret);
714 kfree(forget);
715 }
716 spin_unlock(&fsvq->lock);
717 goto out;
718 }
719
720 fsvq->in_flight++;
721 notify = virtqueue_kick_prepare(vq);
722
723 spin_unlock(&fsvq->lock);
724
725 if (notify)
726 virtqueue_notify(vq);
727out:
728 kfree(link);
729}
730
731static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
732__releases(fiq->lock)
733{
734 /*
735 * TODO interrupts.
736 *
737 * Normal fs operations on a local filesystems aren't interruptible.
738 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
739 * with shared lock between host and guest.
740 */
741 spin_unlock(&fiq->lock);
742}
743
744/* Return the number of scatter-gather list elements required */
745static unsigned int sg_count_fuse_req(struct fuse_req *req)
746{
747 struct fuse_args *args = req->args;
748 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
749 unsigned int total_sgs = 1 /* fuse_in_header */;
750
751 if (args->in_numargs - args->in_pages)
752 total_sgs += 1;
753
754 if (args->in_pages)
755 total_sgs += ap->num_pages;
756
757 if (!test_bit(FR_ISREPLY, &req->flags))
758 return total_sgs;
759
760 total_sgs += 1 /* fuse_out_header */;
761
762 if (args->out_numargs - args->out_pages)
763 total_sgs += 1;
764
765 if (args->out_pages)
766 total_sgs += ap->num_pages;
767
768 return total_sgs;
769}
770
771/* Add pages to scatter-gather list and return number of elements used */
772static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
773 struct page **pages,
774 struct fuse_page_desc *page_descs,
775 unsigned int num_pages,
776 unsigned int total_len)
777{
778 unsigned int i;
779 unsigned int this_len;
780
781 for (i = 0; i < num_pages && total_len; i++) {
782 sg_init_table(&sg[i], 1);
783 this_len = min(page_descs[i].length, total_len);
784 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);
785 total_len -= this_len;
786 }
787
788 return i;
789}
790
791/* Add args to scatter-gather list and return number of elements used */
792static unsigned int sg_init_fuse_args(struct scatterlist *sg,
793 struct fuse_req *req,
794 struct fuse_arg *args,
795 unsigned int numargs,
796 bool argpages,
797 void *argbuf,
798 unsigned int *len_used)
799{
800 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
801 unsigned int total_sgs = 0;
802 unsigned int len;
803
804 len = fuse_len_args(numargs - argpages, args);
805 if (len)
806 sg_init_one(&sg[total_sgs++], argbuf, len);
807
808 if (argpages)
809 total_sgs += sg_init_fuse_pages(&sg[total_sgs],
810 ap->pages, ap->descs,
811 ap->num_pages,
812 args[numargs - 1].size);
813
814 if (len_used)
815 *len_used = len;
816
817 return total_sgs;
818}
819
820/* Add a request to a virtqueue and kick the device */
821static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
822 struct fuse_req *req)
823{
824 /* requests need at least 4 elements */
825 struct scatterlist *stack_sgs[6];
826 struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)];
827 struct scatterlist **sgs = stack_sgs;
828 struct scatterlist *sg = stack_sg;
829 struct virtqueue *vq;
830 struct fuse_args *args = req->args;
831 unsigned int argbuf_used = 0;
832 unsigned int out_sgs = 0;
833 unsigned int in_sgs = 0;
834 unsigned int total_sgs;
835 unsigned int i;
836 int ret;
837 bool notify;
838
839 /* Does the sglist fit on the stack? */
840 total_sgs = sg_count_fuse_req(req);
841 if (total_sgs > ARRAY_SIZE(stack_sgs)) {
842 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);
843 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);
844 if (!sgs || !sg) {
845 ret = -ENOMEM;
846 goto out;
847 }
848 }
849
850 /* Use a bounce buffer since stack args cannot be mapped */
851 ret = copy_args_to_argbuf(req);
852 if (ret < 0)
853 goto out;
854
855 /* Request elements */
856 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h));
857 out_sgs += sg_init_fuse_args(&sg[out_sgs], req,
858 (struct fuse_arg *)args->in_args,
859 args->in_numargs, args->in_pages,
860 req->argbuf, &argbuf_used);
861
862 /* Reply elements */
863 if (test_bit(FR_ISREPLY, &req->flags)) {
864 sg_init_one(&sg[out_sgs + in_sgs++],
865 &req->out.h, sizeof(req->out.h));
866 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req,
867 args->out_args, args->out_numargs,
868 args->out_pages,
869 req->argbuf + argbuf_used, NULL);
870 }
871
872 WARN_ON(out_sgs + in_sgs != total_sgs);
873
874 for (i = 0; i < total_sgs; i++)
875 sgs[i] = &sg[i];
876
877 spin_lock(&fsvq->lock);
878
879 if (!fsvq->connected) {
880 spin_unlock(&fsvq->lock);
881 ret = -ENOTCONN;
882 goto out;
883 }
884
885 vq = fsvq->vq;
886 ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC);
887 if (ret < 0) {
888 spin_unlock(&fsvq->lock);
889 goto out;
890 }
891
892 fsvq->in_flight++;
893 notify = virtqueue_kick_prepare(vq);
894
895 spin_unlock(&fsvq->lock);
896
897 if (notify)
898 virtqueue_notify(vq);
899
900out:
901 if (ret < 0 && req->argbuf) {
902 kfree(req->argbuf);
903 req->argbuf = NULL;
904 }
905 if (sgs != stack_sgs) {
906 kfree(sgs);
907 kfree(sg);
908 }
909
910 return ret;
911}
912
913static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
914__releases(fiq->lock)
915{
916 unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
917 struct virtio_fs *fs;
918 struct fuse_conn *fc;
919 struct fuse_req *req;
920 struct fuse_pqueue *fpq;
921 int ret;
922
923 WARN_ON(list_empty(&fiq->pending));
924 req = list_last_entry(&fiq->pending, struct fuse_req, list);
925 clear_bit(FR_PENDING, &req->flags);
926 list_del_init(&req->list);
927 WARN_ON(!list_empty(&fiq->pending));
928 spin_unlock(&fiq->lock);
929
930 fs = fiq->priv;
931 fc = fs->vqs[queue_id].fud->fc;
932
933 pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n",
934 __func__, req->in.h.opcode, req->in.h.unique,
935 req->in.h.nodeid, req->in.h.len,
936 fuse_len_args(req->args->out_numargs, req->args->out_args));
937
938 fpq = &fs->vqs[queue_id].fud->pq;
939 spin_lock(&fpq->lock);
940 if (!fpq->connected) {
941 spin_unlock(&fpq->lock);
942 req->out.h.error = -ENODEV;
943 pr_err("virtio-fs: %s disconnected\n", __func__);
944 fuse_request_end(fc, req);
945 return;
946 }
947 list_add_tail(&req->list, fpq->processing);
948 spin_unlock(&fpq->lock);
949 set_bit(FR_SENT, &req->flags);
950 /* matches barrier in request_wait_answer() */
951 smp_mb__after_atomic();
952
953retry:
954 ret = virtio_fs_enqueue_req(&fs->vqs[queue_id], req);
955 if (ret < 0) {
956 if (ret == -ENOMEM || ret == -ENOSPC) {
957 /* Virtqueue full. Retry submission */
958 /* TODO use completion instead of timeout */
959 usleep_range(20, 30);
960 goto retry;
961 }
962 req->out.h.error = ret;
963 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret);
964 spin_lock(&fpq->lock);
965 clear_bit(FR_SENT, &req->flags);
966 list_del_init(&req->list);
967 spin_unlock(&fpq->lock);
968 fuse_request_end(fc, req);
969 return;
970 }
971}
972
973const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
974 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
975 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
976 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
977 .release = virtio_fs_fiq_release,
978};
979
980static int virtio_fs_fill_super(struct super_block *sb)
981{
982 struct fuse_conn *fc = get_fuse_conn_super(sb);
983 struct virtio_fs *fs = fc->iq.priv;
984 unsigned int i;
985 int err;
986 struct fuse_fs_context ctx = {
987 .rootmode = S_IFDIR,
988 .default_permissions = 1,
989 .allow_other = 1,
990 .max_read = UINT_MAX,
991 .blksize = 512,
992 .destroy = true,
993 .no_control = true,
994 .no_force_umount = true,
995 };
996
997 mutex_lock(&virtio_fs_mutex);
998
999 /* After holding mutex, make sure virtiofs device is still there.
1000 * Though we are holding a reference to it, drive ->remove might
1001 * still have cleaned up virtual queues. In that case bail out.
1002 */
1003 err = -EINVAL;
1004 if (list_empty(&fs->list)) {
1005 pr_info("virtio-fs: tag <%s> not found\n", fs->tag);
1006 goto err;
1007 }
1008
1009 err = -ENOMEM;
1010 /* Allocate fuse_dev for hiprio and notification queues */
1011 for (i = 0; i < VQ_REQUEST; i++) {
1012 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1013
1014 fsvq->fud = fuse_dev_alloc();
1015 if (!fsvq->fud)
1016 goto err_free_fuse_devs;
1017 }
1018
1019 ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud;
1020 err = fuse_fill_super_common(sb, &ctx);
1021 if (err < 0)
1022 goto err_free_fuse_devs;
1023
1024 fc = fs->vqs[VQ_REQUEST].fud->fc;
1025
1026 for (i = 0; i < fs->nvqs; i++) {
1027 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1028
1029 if (i == VQ_REQUEST)
1030 continue; /* already initialized */
1031 fuse_dev_install(fsvq->fud, fc);
1032 }
1033
1034 /* Previous unmount will stop all queues. Start these again */
1035 virtio_fs_start_all_queues(fs);
1036 fuse_send_init(fc);
1037 mutex_unlock(&virtio_fs_mutex);
1038 return 0;
1039
1040err_free_fuse_devs:
1041 virtio_fs_free_devs(fs);
1042err:
1043 mutex_unlock(&virtio_fs_mutex);
1044 return err;
1045}
1046
1047static void virtio_kill_sb(struct super_block *sb)
1048{
1049 struct fuse_conn *fc = get_fuse_conn_super(sb);
1050 struct virtio_fs *vfs;
1051 struct virtio_fs_vq *fsvq;
1052
1053 /* If mount failed, we can still be called without any fc */
1054 if (!fc)
1055 return fuse_kill_sb_anon(sb);
1056
1057 vfs = fc->iq.priv;
1058 fsvq = &vfs->vqs[VQ_HIPRIO];
1059
1060 /* Stop forget queue. Soon destroy will be sent */
1061 spin_lock(&fsvq->lock);
1062 fsvq->connected = false;
1063 spin_unlock(&fsvq->lock);
1064 virtio_fs_drain_all_queues(vfs);
1065
1066 fuse_kill_sb_anon(sb);
1067
1068 /* fuse_kill_sb_anon() must have sent destroy. Stop all queues
1069 * and drain one more time and free fuse devices. Freeing fuse
1070 * devices will drop their reference on fuse_conn and that in
1071 * turn will drop its reference on virtio_fs object.
1072 */
1073 virtio_fs_stop_all_queues(vfs);
1074 virtio_fs_drain_all_queues(vfs);
1075 virtio_fs_free_devs(vfs);
1076}
1077
1078static int virtio_fs_test_super(struct super_block *sb,
1079 struct fs_context *fsc)
1080{
1081 struct fuse_conn *fc = fsc->s_fs_info;
1082
1083 return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv;
1084}
1085
1086static int virtio_fs_set_super(struct super_block *sb,
1087 struct fs_context *fsc)
1088{
1089 int err;
1090
1091 err = get_anon_bdev(&sb->s_dev);
1092 if (!err)
1093 fuse_conn_get(fsc->s_fs_info);
1094
1095 return err;
1096}
1097
1098static int virtio_fs_get_tree(struct fs_context *fsc)
1099{
1100 struct virtio_fs *fs;
1101 struct super_block *sb;
1102 struct fuse_conn *fc;
1103 int err;
1104
1105 /* This gets a reference on virtio_fs object. This ptr gets installed
1106 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1107 * to drop the reference to this object.
1108 */
1109 fs = virtio_fs_find_instance(fsc->source);
1110 if (!fs) {
1111 pr_info("virtio-fs: tag <%s> not found\n", fsc->source);
1112 return -EINVAL;
1113 }
1114
1115 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1116 if (!fc) {
1117 mutex_lock(&virtio_fs_mutex);
1118 virtio_fs_put(fs);
1119 mutex_unlock(&virtio_fs_mutex);
1120 return -ENOMEM;
1121 }
1122
1123 fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops,
1124 fs);
1125 fc->release = fuse_free_conn;
1126 fc->delete_stale = true;
1127
1128 fsc->s_fs_info = fc;
1129 sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
1130 fuse_conn_put(fc);
1131 if (IS_ERR(sb))
1132 return PTR_ERR(sb);
1133
1134 if (!sb->s_root) {
1135 err = virtio_fs_fill_super(sb);
1136 if (err) {
1137 deactivate_locked_super(sb);
1138 return err;
1139 }
1140
1141 sb->s_flags |= SB_ACTIVE;
1142 }
1143
1144 WARN_ON(fsc->root);
1145 fsc->root = dget(sb->s_root);
1146 return 0;
1147}
1148
1149static const struct fs_context_operations virtio_fs_context_ops = {
1150 .get_tree = virtio_fs_get_tree,
1151};
1152
1153static int virtio_fs_init_fs_context(struct fs_context *fsc)
1154{
1155 fsc->ops = &virtio_fs_context_ops;
1156 return 0;
1157}
1158
1159static struct file_system_type virtio_fs_type = {
1160 .owner = THIS_MODULE,
1161 .name = "virtiofs",
1162 .init_fs_context = virtio_fs_init_fs_context,
1163 .kill_sb = virtio_kill_sb,
1164};
1165
1166static int __init virtio_fs_init(void)
1167{
1168 int ret;
1169
1170 ret = register_virtio_driver(&virtio_fs_driver);
1171 if (ret < 0)
1172 return ret;
1173
1174 ret = register_filesystem(&virtio_fs_type);
1175 if (ret < 0) {
1176 unregister_virtio_driver(&virtio_fs_driver);
1177 return ret;
1178 }
1179
1180 return 0;
1181}
1182module_init(virtio_fs_init);
1183
1184static void __exit virtio_fs_exit(void)
1185{
1186 unregister_filesystem(&virtio_fs_type);
1187 unregister_virtio_driver(&virtio_fs_driver);
1188}
1189module_exit(virtio_fs_exit);
1190
1191MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>");
1192MODULE_DESCRIPTION("Virtio Filesystem");
1193MODULE_LICENSE("GPL");
1194MODULE_ALIAS_FS(KBUILD_MODNAME);
1195MODULE_DEVICE_TABLE(virtio, id_table);