Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (c) 2016 Trond Myklebust |
| 4 | * |
| 5 | * I/O and data path helper functionality. |
| 6 | */ |
| 7 | |
| 8 | #include <linux/types.h> |
| 9 | #include <linux/kernel.h> |
| 10 | #include <linux/bitops.h> |
| 11 | #include <linux/rwsem.h> |
| 12 | #include <linux/fs.h> |
| 13 | #include <linux/nfs_fs.h> |
| 14 | |
| 15 | #include "internal.h" |
| 16 | |
| 17 | /* Call with exclusively locked inode->i_rwsem */ |
| 18 | static void nfs_block_o_direct(struct nfs_inode *nfsi, struct inode *inode) |
| 19 | { |
| 20 | if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { |
| 21 | clear_bit(NFS_INO_ODIRECT, &nfsi->flags); |
| 22 | inode_dio_wait(inode); |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | /** |
| 27 | * nfs_start_io_read - declare the file is being used for buffered reads |
Trond Myklebust | 302fad7 | 2019-02-18 13:32:38 -0500 | [diff] [blame] | 28 | * @inode: file inode |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 29 | * |
| 30 | * Declare that a buffered read operation is about to start, and ensure |
| 31 | * that we block all direct I/O. |
| 32 | * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset, |
| 33 | * and holds a shared lock on inode->i_rwsem to ensure that the flag |
| 34 | * cannot be changed. |
| 35 | * In practice, this means that buffered read operations are allowed to |
| 36 | * execute in parallel, thanks to the shared lock, whereas direct I/O |
| 37 | * operations need to wait to grab an exclusive lock in order to set |
| 38 | * NFS_INO_ODIRECT. |
| 39 | * Note that buffered writes and truncates both take a write lock on |
| 40 | * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. |
| 41 | */ |
| 42 | void |
| 43 | nfs_start_io_read(struct inode *inode) |
| 44 | { |
| 45 | struct nfs_inode *nfsi = NFS_I(inode); |
| 46 | /* Be an optimist! */ |
| 47 | down_read(&inode->i_rwsem); |
| 48 | if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0) |
| 49 | return; |
| 50 | up_read(&inode->i_rwsem); |
| 51 | /* Slow path.... */ |
| 52 | down_write(&inode->i_rwsem); |
| 53 | nfs_block_o_direct(nfsi, inode); |
| 54 | downgrade_write(&inode->i_rwsem); |
| 55 | } |
| 56 | |
| 57 | /** |
| 58 | * nfs_end_io_read - declare that the buffered read operation is done |
Trond Myklebust | 302fad7 | 2019-02-18 13:32:38 -0500 | [diff] [blame] | 59 | * @inode: file inode |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 60 | * |
| 61 | * Declare that a buffered read operation is done, and release the shared |
| 62 | * lock on inode->i_rwsem. |
| 63 | */ |
| 64 | void |
| 65 | nfs_end_io_read(struct inode *inode) |
| 66 | { |
| 67 | up_read(&inode->i_rwsem); |
| 68 | } |
| 69 | |
| 70 | /** |
| 71 | * nfs_start_io_write - declare the file is being used for buffered writes |
Trond Myklebust | 302fad7 | 2019-02-18 13:32:38 -0500 | [diff] [blame] | 72 | * @inode: file inode |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 73 | * |
| 74 | * Declare that a buffered read operation is about to start, and ensure |
| 75 | * that we block all direct I/O. |
| 76 | */ |
| 77 | void |
| 78 | nfs_start_io_write(struct inode *inode) |
| 79 | { |
| 80 | down_write(&inode->i_rwsem); |
| 81 | nfs_block_o_direct(NFS_I(inode), inode); |
| 82 | } |
| 83 | |
| 84 | /** |
| 85 | * nfs_end_io_write - declare that the buffered write operation is done |
Trond Myklebust | 302fad7 | 2019-02-18 13:32:38 -0500 | [diff] [blame] | 86 | * @inode: file inode |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 87 | * |
| 88 | * Declare that a buffered write operation is done, and release the |
| 89 | * lock on inode->i_rwsem. |
| 90 | */ |
| 91 | void |
| 92 | nfs_end_io_write(struct inode *inode) |
| 93 | { |
| 94 | up_write(&inode->i_rwsem); |
| 95 | } |
| 96 | |
| 97 | /* Call with exclusively locked inode->i_rwsem */ |
| 98 | static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode) |
| 99 | { |
| 100 | if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { |
| 101 | set_bit(NFS_INO_ODIRECT, &nfsi->flags); |
Trond Myklebust | e231c68 | 2018-01-28 09:29:41 -0500 | [diff] [blame] | 102 | nfs_sync_mapping(inode->i_mapping); |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 103 | } |
| 104 | } |
| 105 | |
| 106 | /** |
| 107 | * nfs_end_io_direct - declare the file is being used for direct i/o |
Trond Myklebust | 302fad7 | 2019-02-18 13:32:38 -0500 | [diff] [blame] | 108 | * @inode: file inode |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 109 | * |
| 110 | * Declare that a direct I/O operation is about to start, and ensure |
| 111 | * that we block all buffered I/O. |
| 112 | * On exit, the function ensures that the NFS_INO_ODIRECT flag is set, |
| 113 | * and holds a shared lock on inode->i_rwsem to ensure that the flag |
| 114 | * cannot be changed. |
| 115 | * In practice, this means that direct I/O operations are allowed to |
| 116 | * execute in parallel, thanks to the shared lock, whereas buffered I/O |
| 117 | * operations need to wait to grab an exclusive lock in order to clear |
| 118 | * NFS_INO_ODIRECT. |
| 119 | * Note that buffered writes and truncates both take a write lock on |
| 120 | * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. |
| 121 | */ |
| 122 | void |
| 123 | nfs_start_io_direct(struct inode *inode) |
| 124 | { |
| 125 | struct nfs_inode *nfsi = NFS_I(inode); |
| 126 | /* Be an optimist! */ |
| 127 | down_read(&inode->i_rwsem); |
| 128 | if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0) |
| 129 | return; |
| 130 | up_read(&inode->i_rwsem); |
| 131 | /* Slow path.... */ |
| 132 | down_write(&inode->i_rwsem); |
| 133 | nfs_block_buffered(nfsi, inode); |
| 134 | downgrade_write(&inode->i_rwsem); |
| 135 | } |
| 136 | |
| 137 | /** |
| 138 | * nfs_end_io_direct - declare that the direct i/o operation is done |
Trond Myklebust | 302fad7 | 2019-02-18 13:32:38 -0500 | [diff] [blame] | 139 | * @inode: file inode |
Trond Myklebust | a5864c9 | 2016-06-03 17:07:19 -0400 | [diff] [blame] | 140 | * |
| 141 | * Declare that a direct I/O operation is done, and release the shared |
| 142 | * lock on inode->i_rwsem. |
| 143 | */ |
| 144 | void |
| 145 | nfs_end_io_direct(struct inode *inode) |
| 146 | { |
| 147 | up_read(&inode->i_rwsem); |
| 148 | } |