blob: c3ca28ca68a65d70865d19accea5f07a9ff39b4b [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Herbert Xu79990962020-06-12 16:57:37 +10002#include <crypto/hash.h>
Al Viro4f18cd32014-02-05 19:11:33 -05003#include <linux/export.h>
Christoph Hellwig2f8b5442016-11-01 07:40:13 -06004#include <linux/bvec.h>
Albert van der Linde4d0e9df2020-10-15 20:13:50 -07005#include <linux/fault-inject-usercopy.h>
Al Viro4f18cd32014-02-05 19:11:33 -05006#include <linux/uio.h>
7#include <linux/pagemap.h>
Ira Weiny28961992021-05-04 18:40:03 -07008#include <linux/highmem.h>
Al Viro91f79c42014-03-21 04:58:33 -04009#include <linux/slab.h>
10#include <linux/vmalloc.h>
Al Viro241699c2016-09-22 16:33:12 -040011#include <linux/splice.h>
Christoph Hellwigbfdc5972020-09-25 06:51:40 +020012#include <linux/compat.h>
Al Viroa604ec72014-11-24 01:08:00 -050013#include <net/checksum.h>
Sagi Grimbergd05f4432018-12-03 17:52:09 -080014#include <linux/scatterlist.h>
Marco Elverd0ef4c32020-01-21 17:05:11 +010015#include <linux/instrumented.h>
Al Viro4f18cd32014-02-05 19:11:33 -050016
Al Viro241699c2016-09-22 16:33:12 -040017#define PIPE_PARANOIA /* for now */
18
Al Virofcb14cb2022-05-22 14:59:25 -040019/* covers ubuf and kbuf alike */
20#define iterate_buf(i, n, base, len, off, __p, STEP) { \
21 size_t __maybe_unused off = 0; \
22 len = n; \
23 base = __p + i->iov_offset; \
24 len -= (STEP); \
25 i->iov_offset += len; \
26 n = len; \
27}
28
Al Viro5c67aa92021-04-25 23:57:42 -040029/* covers iovec and kvec alike */
Al Viroa6e4ec72021-05-02 13:03:41 -040030#define iterate_iovec(i, n, base, len, off, __p, STEP) { \
Al Viro7baa5092021-05-02 11:35:03 -040031 size_t off = 0; \
Al Viroa6e4ec72021-05-02 13:03:41 -040032 size_t skip = i->iov_offset; \
Al Viro7a1bcb52021-04-25 23:46:09 -040033 do { \
Al Viro7baa5092021-05-02 11:35:03 -040034 len = min(n, __p->iov_len - skip); \
35 if (likely(len)) { \
36 base = __p->iov_base + skip; \
37 len -= (STEP); \
38 off += len; \
39 skip += len; \
40 n -= len; \
Al Viro7a1bcb52021-04-25 23:46:09 -040041 if (skip < __p->iov_len) \
42 break; \
43 } \
44 __p++; \
45 skip = 0; \
46 } while (n); \
Al Viroa6e4ec72021-05-02 13:03:41 -040047 i->iov_offset = skip; \
Al Viro7baa5092021-05-02 11:35:03 -040048 n = off; \
Al Viro04a31162014-11-27 13:51:41 -050049}
50
Al Viroa6e4ec72021-05-02 13:03:41 -040051#define iterate_bvec(i, n, base, len, off, p, STEP) { \
Al Viro7baa5092021-05-02 11:35:03 -040052 size_t off = 0; \
Al Viroa6e4ec72021-05-02 13:03:41 -040053 unsigned skip = i->iov_offset; \
Al Viro7491a2b2021-04-26 20:19:14 -040054 while (n) { \
55 unsigned offset = p->bv_offset + skip; \
Al Viro1b4fb5f2021-04-26 20:33:42 -040056 unsigned left; \
Al Viro21b56c82021-04-26 20:50:05 -040057 void *kaddr = kmap_local_page(p->bv_page + \
58 offset / PAGE_SIZE); \
Al Viro7baa5092021-05-02 11:35:03 -040059 base = kaddr + offset % PAGE_SIZE; \
Al Viroa6e4ec72021-05-02 13:03:41 -040060 len = min(min(n, (size_t)(p->bv_len - skip)), \
Al Viro7491a2b2021-04-26 20:19:14 -040061 (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \
Al Viro1b4fb5f2021-04-26 20:33:42 -040062 left = (STEP); \
Al Viro21b56c82021-04-26 20:50:05 -040063 kunmap_local(kaddr); \
Al Viro7baa5092021-05-02 11:35:03 -040064 len -= left; \
65 off += len; \
66 skip += len; \
Al Viro7491a2b2021-04-26 20:19:14 -040067 if (skip == p->bv_len) { \
68 skip = 0; \
69 p++; \
70 } \
Al Viro7baa5092021-05-02 11:35:03 -040071 n -= len; \
Al Viro1b4fb5f2021-04-26 20:33:42 -040072 if (left) \
73 break; \
Al Viro7491a2b2021-04-26 20:19:14 -040074 } \
Al Viroa6e4ec72021-05-02 13:03:41 -040075 i->iov_offset = skip; \
Al Viro7baa5092021-05-02 11:35:03 -040076 n = off; \
Al Viro04a31162014-11-27 13:51:41 -050077}
78
Al Viroa6e4ec72021-05-02 13:03:41 -040079#define iterate_xarray(i, n, base, len, __off, STEP) { \
Al Viro1b4fb5f2021-04-26 20:33:42 -040080 __label__ __out; \
Al Viro622838f2021-05-02 11:13:09 -040081 size_t __off = 0; \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -050082 struct folio *folio; \
Al Viroa6e4ec72021-05-02 13:03:41 -040083 loff_t start = i->xarray_start + i->iov_offset; \
Al Viro4b179e92021-05-04 17:50:07 -040084 pgoff_t index = start / PAGE_SIZE; \
David Howells7ff5062072020-02-10 10:00:21 +000085 XA_STATE(xas, i->xarray, index); \
86 \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -050087 len = PAGE_SIZE - offset_in_page(start); \
Al Viro7baa5092021-05-02 11:35:03 -040088 rcu_read_lock(); \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -050089 xas_for_each(&xas, folio, ULONG_MAX) { \
Al Viro7baa5092021-05-02 11:35:03 -040090 unsigned left; \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -050091 size_t offset; \
92 if (xas_retry(&xas, folio)) \
Al Viro7baa5092021-05-02 11:35:03 -040093 continue; \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -050094 if (WARN_ON(xa_is_value(folio))) \
Al Viro7baa5092021-05-02 11:35:03 -040095 break; \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -050096 if (WARN_ON(folio_test_hugetlb(folio))) \
Al Viro7baa5092021-05-02 11:35:03 -040097 break; \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -050098 offset = offset_in_folio(folio, start + __off); \
99 while (offset < folio_size(folio)) { \
100 base = kmap_local_folio(folio, offset); \
Al Viro7baa5092021-05-02 11:35:03 -0400101 len = min(n, len); \
102 left = (STEP); \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -0500103 kunmap_local(base); \
Al Viro7baa5092021-05-02 11:35:03 -0400104 len -= left; \
105 __off += len; \
106 n -= len; \
107 if (left || n == 0) \
108 goto __out; \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -0500109 offset += len; \
110 len = PAGE_SIZE; \
Al Viro7baa5092021-05-02 11:35:03 -0400111 } \
David Howells7ff5062072020-02-10 10:00:21 +0000112 } \
Al Viro1b4fb5f2021-04-26 20:33:42 -0400113__out: \
David Howells7ff5062072020-02-10 10:00:21 +0000114 rcu_read_unlock(); \
Matthew Wilcox (Oracle)821979f2021-11-28 19:18:27 -0500115 i->iov_offset += __off; \
Al Viro622838f2021-05-02 11:13:09 -0400116 n = __off; \
David Howells7ff5062072020-02-10 10:00:21 +0000117}
118
Al Viro7baa5092021-05-02 11:35:03 -0400119#define __iterate_and_advance(i, n, base, len, off, I, K) { \
Al Virodd254f52016-05-09 11:54:48 -0400120 if (unlikely(i->count < n)) \
121 n = i->count; \
Al Virof5da8352021-04-28 20:59:08 -0400122 if (likely(n)) { \
Al Virofcb14cb2022-05-22 14:59:25 -0400123 if (likely(iter_is_ubuf(i))) { \
124 void __user *base; \
125 size_t len; \
126 iterate_buf(i, n, base, len, off, \
127 i->ubuf, (I)) \
128 } else if (likely(iter_is_iovec(i))) { \
Al Viro5c67aa92021-04-25 23:57:42 -0400129 const struct iovec *iov = i->iov; \
Al Viro7baa5092021-05-02 11:35:03 -0400130 void __user *base; \
131 size_t len; \
132 iterate_iovec(i, n, base, len, off, \
Al Viroa6e4ec72021-05-02 13:03:41 -0400133 iov, (I)) \
Al Virodd254f52016-05-09 11:54:48 -0400134 i->nr_segs -= iov - i->iov; \
135 i->iov = iov; \
Al Viro28f38db2021-06-02 17:25:59 -0400136 } else if (iov_iter_is_bvec(i)) { \
137 const struct bio_vec *bvec = i->bvec; \
Al Viro7baa5092021-05-02 11:35:03 -0400138 void *base; \
139 size_t len; \
140 iterate_bvec(i, n, base, len, off, \
Al Viroa6e4ec72021-05-02 13:03:41 -0400141 bvec, (K)) \
Al Viro7491a2b2021-04-26 20:19:14 -0400142 i->nr_segs -= bvec - i->bvec; \
143 i->bvec = bvec; \
Al Viro28f38db2021-06-02 17:25:59 -0400144 } else if (iov_iter_is_kvec(i)) { \
Al Viro5c67aa92021-04-25 23:57:42 -0400145 const struct kvec *kvec = i->kvec; \
Al Viro7baa5092021-05-02 11:35:03 -0400146 void *base; \
147 size_t len; \
148 iterate_iovec(i, n, base, len, off, \
Al Viroa6e4ec72021-05-02 13:03:41 -0400149 kvec, (K)) \
Al Viro28f38db2021-06-02 17:25:59 -0400150 i->nr_segs -= kvec - i->kvec; \
151 i->kvec = kvec; \
152 } else if (iov_iter_is_xarray(i)) { \
Al Viro7baa5092021-05-02 11:35:03 -0400153 void *base; \
154 size_t len; \
155 iterate_xarray(i, n, base, len, off, \
Al Viroa6e4ec72021-05-02 13:03:41 -0400156 (K)) \
Al Viro7ce2a912014-11-27 13:59:45 -0500157 } \
Al Virodd254f52016-05-09 11:54:48 -0400158 i->count -= n; \
Al Viro7ce2a912014-11-27 13:59:45 -0500159 } \
Al Viro7ce2a912014-11-27 13:59:45 -0500160}
Al Viro7baa5092021-05-02 11:35:03 -0400161#define iterate_and_advance(i, n, base, len, off, I, K) \
162 __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0))
Al Viro7ce2a912014-11-27 13:59:45 -0500163
Al Viro09fc68dc2017-06-29 22:25:14 -0400164static int copyout(void __user *to, const void *from, size_t n)
165{
Albert van der Linde4d0e9df2020-10-15 20:13:50 -0700166 if (should_fail_usercopy())
167 return n;
Linus Torvalds96d4f262019-01-03 18:57:57 -0800168 if (access_ok(to, n)) {
Marco Elverd0ef4c32020-01-21 17:05:11 +0100169 instrument_copy_to_user(to, from, n);
Al Viro09fc68dc2017-06-29 22:25:14 -0400170 n = raw_copy_to_user(to, from, n);
171 }
172 return n;
173}
174
175static int copyin(void *to, const void __user *from, size_t n)
176{
Alexander Potapenko33b75c12022-09-15 17:03:37 +0200177 size_t res = n;
178
Albert van der Linde4d0e9df2020-10-15 20:13:50 -0700179 if (should_fail_usercopy())
180 return n;
Linus Torvalds96d4f262019-01-03 18:57:57 -0800181 if (access_ok(from, n)) {
Alexander Potapenko33b75c12022-09-15 17:03:37 +0200182 instrument_copy_from_user_before(to, from, n);
183 res = raw_copy_from_user(to, from, n);
184 instrument_copy_from_user_after(to, from, n, res);
Al Viro09fc68dc2017-06-29 22:25:14 -0400185 }
Alexander Potapenko33b75c12022-09-15 17:03:37 +0200186 return res;
Al Viro09fc68dc2017-06-29 22:25:14 -0400187}
188
Al Viro2dcedb22022-06-14 10:24:37 -0400189static inline struct pipe_buffer *pipe_buf(const struct pipe_inode_info *pipe,
190 unsigned int slot)
191{
192 return &pipe->bufs[slot & (pipe->ring_size - 1)];
193}
194
Al Viro241699c2016-09-22 16:33:12 -0400195#ifdef PIPE_PARANOIA
196static bool sanity(const struct iov_iter *i)
197{
198 struct pipe_inode_info *pipe = i->pipe;
David Howells8cefc102019-11-15 13:30:32 +0000199 unsigned int p_head = pipe->head;
200 unsigned int p_tail = pipe->tail;
David Howells8cefc102019-11-15 13:30:32 +0000201 unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
202 unsigned int i_head = i->head;
203 unsigned int idx;
204
Al Viro10f525a2022-06-15 02:02:51 -0400205 if (i->last_offset) {
Al Viro241699c2016-09-22 16:33:12 -0400206 struct pipe_buffer *p;
David Howells8cefc102019-11-15 13:30:32 +0000207 if (unlikely(p_occupancy == 0))
Al Viro241699c2016-09-22 16:33:12 -0400208 goto Bad; // pipe must be non-empty
David Howells8cefc102019-11-15 13:30:32 +0000209 if (unlikely(i_head != p_head - 1))
Al Viro241699c2016-09-22 16:33:12 -0400210 goto Bad; // must be at the last buffer...
211
Al Viro2dcedb22022-06-14 10:24:37 -0400212 p = pipe_buf(pipe, i_head);
Al Viro10f525a2022-06-15 02:02:51 -0400213 if (unlikely(p->offset + p->len != abs(i->last_offset)))
Al Viro241699c2016-09-22 16:33:12 -0400214 goto Bad; // ... at the end of segment
215 } else {
David Howells8cefc102019-11-15 13:30:32 +0000216 if (i_head != p_head)
Al Viro241699c2016-09-22 16:33:12 -0400217 goto Bad; // must be right after the last buffer
218 }
219 return true;
220Bad:
Al Viro10f525a2022-06-15 02:02:51 -0400221 printk(KERN_ERR "idx = %d, offset = %d\n", i_head, i->last_offset);
David Howells8cefc102019-11-15 13:30:32 +0000222 printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
223 p_head, p_tail, pipe->ring_size);
224 for (idx = 0; idx < pipe->ring_size; idx++)
Al Viro241699c2016-09-22 16:33:12 -0400225 printk(KERN_ERR "[%p %p %d %d]\n",
226 pipe->bufs[idx].ops,
227 pipe->bufs[idx].page,
228 pipe->bufs[idx].offset,
229 pipe->bufs[idx].len);
230 WARN_ON(1);
231 return false;
232}
233#else
234#define sanity(i) true
235#endif
236
Al Viro47b7fca2022-06-13 14:30:15 -0400237static struct page *push_anon(struct pipe_inode_info *pipe, unsigned size)
238{
239 struct page *page = alloc_page(GFP_USER);
240 if (page) {
241 struct pipe_buffer *buf = pipe_buf(pipe, pipe->head++);
242 *buf = (struct pipe_buffer) {
243 .ops = &default_pipe_buf_ops,
244 .page = page,
245 .offset = 0,
246 .len = size
247 };
248 }
249 return page;
250}
251
252static void push_page(struct pipe_inode_info *pipe, struct page *page,
253 unsigned int offset, unsigned int size)
254{
255 struct pipe_buffer *buf = pipe_buf(pipe, pipe->head++);
256 *buf = (struct pipe_buffer) {
257 .ops = &page_cache_pipe_buf_ops,
258 .page = page,
259 .offset = offset,
260 .len = size
261 };
262 get_page(page);
263}
264
Al Viro10f525a2022-06-15 02:02:51 -0400265static inline int last_offset(const struct pipe_buffer *buf)
Al Viro8fad7762022-06-14 13:53:53 -0400266{
Al Viro10f525a2022-06-15 02:02:51 -0400267 if (buf->ops == &default_pipe_buf_ops)
268 return buf->len; // buf->offset is 0 for those
269 else
270 return -(buf->offset + buf->len);
Al Viro8fad7762022-06-14 13:53:53 -0400271}
272
273static struct page *append_pipe(struct iov_iter *i, size_t size,
274 unsigned int *off)
275{
276 struct pipe_inode_info *pipe = i->pipe;
Al Viro10f525a2022-06-15 02:02:51 -0400277 int offset = i->last_offset;
Al Viro8fad7762022-06-14 13:53:53 -0400278 struct pipe_buffer *buf;
279 struct page *page;
280
Al Viro10f525a2022-06-15 02:02:51 -0400281 if (offset > 0 && offset < PAGE_SIZE) {
282 // some space in the last buffer; add to it
Al Viro8fad7762022-06-14 13:53:53 -0400283 buf = pipe_buf(pipe, pipe->head - 1);
Al Viro10f525a2022-06-15 02:02:51 -0400284 size = min_t(size_t, size, PAGE_SIZE - offset);
285 buf->len += size;
286 i->last_offset += size;
287 i->count -= size;
288 *off = offset;
289 return buf->page;
Al Viro8fad7762022-06-14 13:53:53 -0400290 }
291 // OK, we need a new buffer
292 *off = 0;
293 size = min_t(size_t, size, PAGE_SIZE);
294 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
295 return NULL;
296 page = push_anon(pipe, size);
297 if (!page)
298 return NULL;
299 i->head = pipe->head - 1;
Al Viro10f525a2022-06-15 02:02:51 -0400300 i->last_offset = size;
Al Viro8fad7762022-06-14 13:53:53 -0400301 i->count -= size;
302 return page;
303}
304
Al Viro241699c2016-09-22 16:33:12 -0400305static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
306 struct iov_iter *i)
307{
308 struct pipe_inode_info *pipe = i->pipe;
Al Viro47b7fca2022-06-13 14:30:15 -0400309 unsigned int head = pipe->head;
Al Viro241699c2016-09-22 16:33:12 -0400310
311 if (unlikely(bytes > i->count))
312 bytes = i->count;
313
314 if (unlikely(!bytes))
315 return 0;
316
317 if (!sanity(i))
318 return 0;
319
Al Viro10f525a2022-06-15 02:02:51 -0400320 if (offset && i->last_offset == -offset) { // could we merge it?
Al Viro47b7fca2022-06-13 14:30:15 -0400321 struct pipe_buffer *buf = pipe_buf(pipe, head - 1);
322 if (buf->page == page) {
Al Viro241699c2016-09-22 16:33:12 -0400323 buf->len += bytes;
Al Viro10f525a2022-06-15 02:02:51 -0400324 i->last_offset -= bytes;
Al Viro47b7fca2022-06-13 14:30:15 -0400325 i->count -= bytes;
326 return bytes;
Al Viro241699c2016-09-22 16:33:12 -0400327 }
Al Viro241699c2016-09-22 16:33:12 -0400328 }
Al Viro47b7fca2022-06-13 14:30:15 -0400329 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
Al Viro241699c2016-09-22 16:33:12 -0400330 return 0;
David Howells8cefc102019-11-15 13:30:32 +0000331
Al Viro47b7fca2022-06-13 14:30:15 -0400332 push_page(pipe, page, offset, bytes);
Al Viro10f525a2022-06-15 02:02:51 -0400333 i->last_offset = -(offset + bytes);
Al Viro47b7fca2022-06-13 14:30:15 -0400334 i->head = head;
Al Viro241699c2016-09-22 16:33:12 -0400335 i->count -= bytes;
336 return bytes;
337}
338
Al Viro4f18cd32014-02-05 19:11:33 -0500339/*
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200340 * fault_in_iov_iter_readable - fault in iov iterator for reading
341 * @i: iterator
342 * @size: maximum length
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400343 *
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200344 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
345 * @size. For each iovec, fault in each page that constitutes the iovec.
346 *
347 * Returns the number of bytes not faulted in (like copy_to_user() and
348 * copy_from_user()).
349 *
350 * Always returns 0 for non-userspace iterators.
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400351 */
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200352size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400353{
Al Virofcb14cb2022-05-22 14:59:25 -0400354 if (iter_is_ubuf(i)) {
355 size_t n = min(size, iov_iter_count(i));
356 n -= fault_in_readable(i->ubuf + i->iov_offset, n);
357 return size - n;
358 } else if (iter_is_iovec(i)) {
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200359 size_t count = min(size, iov_iter_count(i));
Al Viro8409a0d2021-05-02 11:57:37 -0400360 const struct iovec *p;
361 size_t skip;
362
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200363 size -= count;
364 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
365 size_t len = min(count, p->iov_len - skip);
366 size_t ret;
Al Viro8409a0d2021-05-02 11:57:37 -0400367
368 if (unlikely(!len))
369 continue;
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200370 ret = fault_in_readable(p->iov_base + skip, len);
371 count -= len - ret;
372 if (ret)
373 break;
Al Viro8409a0d2021-05-02 11:57:37 -0400374 }
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200375 return count + size;
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400376 }
377 return 0;
378}
Andreas Gruenbachera6294592021-08-02 14:54:16 +0200379EXPORT_SYMBOL(fault_in_iov_iter_readable);
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400380
Andreas Gruenbachercdd591f2021-07-05 17:26:28 +0200381/*
382 * fault_in_iov_iter_writeable - fault in iov iterator for writing
383 * @i: iterator
384 * @size: maximum length
385 *
386 * Faults in the iterator using get_user_pages(), i.e., without triggering
387 * hardware page faults. This is primarily useful when we already know that
388 * some or all of the pages in @i aren't in memory.
389 *
390 * Returns the number of bytes not faulted in, like copy_to_user() and
391 * copy_from_user().
392 *
393 * Always returns 0 for non-user-space iterators.
394 */
395size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)
396{
Al Virofcb14cb2022-05-22 14:59:25 -0400397 if (iter_is_ubuf(i)) {
398 size_t n = min(size, iov_iter_count(i));
399 n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n);
400 return size - n;
401 } else if (iter_is_iovec(i)) {
Andreas Gruenbachercdd591f2021-07-05 17:26:28 +0200402 size_t count = min(size, iov_iter_count(i));
403 const struct iovec *p;
404 size_t skip;
405
406 size -= count;
407 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
408 size_t len = min(count, p->iov_len - skip);
409 size_t ret;
410
411 if (unlikely(!len))
412 continue;
413 ret = fault_in_safe_writeable(p->iov_base + skip, len);
414 count -= len - ret;
415 if (ret)
416 break;
417 }
418 return count + size;
419 }
420 return 0;
421}
422EXPORT_SYMBOL(fault_in_iov_iter_writeable);
423
David Howellsaa563d72018-10-20 00:57:56 +0100424void iov_iter_init(struct iov_iter *i, unsigned int direction,
Al Viro71d8e532014-03-05 19:28:09 -0500425 const struct iovec *iov, unsigned long nr_segs,
426 size_t count)
427{
David Howellsaa563d72018-10-20 00:57:56 +0100428 WARN_ON(direction & ~(READ | WRITE));
Al Viro8cd54c12021-04-22 14:50:39 -0400429 *i = (struct iov_iter) {
430 .iter_type = ITER_IOVEC,
Andreas Gruenbacher3337ab02021-07-12 12:06:14 +0200431 .nofault = false,
Al Virofcb14cb2022-05-22 14:59:25 -0400432 .user_backed = true,
Al Viro8cd54c12021-04-22 14:50:39 -0400433 .data_source = direction,
434 .iov = iov,
435 .nr_segs = nr_segs,
436 .iov_offset = 0,
437 .count = count
438 };
Al Viro71d8e532014-03-05 19:28:09 -0500439}
440EXPORT_SYMBOL(iov_iter_init);
Al Viro7b2c99d2014-03-15 04:05:57 -0400441
Al Viro12d426a2022-06-15 09:44:38 -0400442// returns the offset in partial buffer (if any)
443static inline unsigned int pipe_npages(const struct iov_iter *i, int *npages)
Al Viro241699c2016-09-22 16:33:12 -0400444{
Al Viro12d426a2022-06-15 09:44:38 -0400445 struct pipe_inode_info *pipe = i->pipe;
446 int used = pipe->head - pipe->tail;
Al Viro10f525a2022-06-15 02:02:51 -0400447 int off = i->last_offset;
David Howells8cefc102019-11-15 13:30:32 +0000448
Al Viro12d426a2022-06-15 09:44:38 -0400449 *npages = max((int)pipe->max_usage - used, 0);
450
Al Viro10f525a2022-06-15 02:02:51 -0400451 if (off > 0 && off < PAGE_SIZE) { // anon and not full
Al Viro12d426a2022-06-15 09:44:38 -0400452 (*npages)++;
453 return off;
Al Viro241699c2016-09-22 16:33:12 -0400454 }
Al Viro12d426a2022-06-15 09:44:38 -0400455 return 0;
Al Viro241699c2016-09-22 16:33:12 -0400456}
457
Al Viro241699c2016-09-22 16:33:12 -0400458static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
459 struct iov_iter *i)
460{
Al Viro8fad7762022-06-14 13:53:53 -0400461 unsigned int off, chunk;
462
463 if (unlikely(bytes > i->count))
464 bytes = i->count;
465 if (unlikely(!bytes))
466 return 0;
Al Viro241699c2016-09-22 16:33:12 -0400467
468 if (!sanity(i))
469 return 0;
470
Al Viro8fad7762022-06-14 13:53:53 -0400471 for (size_t n = bytes; n; n -= chunk) {
472 struct page *page = append_pipe(i, n, &off);
473 chunk = min_t(size_t, n, PAGE_SIZE - off);
474 if (!page)
475 return bytes - n;
476 memcpy_to_page(page, off, addr, chunk);
Al Viro241699c2016-09-22 16:33:12 -0400477 addr += chunk;
Al Viro8fad7762022-06-14 13:53:53 -0400478 }
Al Viro241699c2016-09-22 16:33:12 -0400479 return bytes;
480}
481
Al Virof9152892018-11-27 22:32:59 -0500482static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
483 __wsum sum, size_t off)
484{
Al Virocc44c172020-07-11 00:12:07 -0400485 __wsum next = csum_partial_copy_nocheck(from, to, len);
Al Virof9152892018-11-27 22:32:59 -0500486 return csum_block_add(sum, next, off);
487}
488
Al Viro78e1f382018-11-25 16:24:16 -0500489static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
Al Viro6852df12021-05-02 17:24:40 -0400490 struct iov_iter *i, __wsum *sump)
Al Viro78e1f382018-11-25 16:24:16 -0500491{
Al Viro6852df12021-05-02 17:24:40 -0400492 __wsum sum = *sump;
493 size_t off = 0;
Al Viro8fad7762022-06-14 13:53:53 -0400494 unsigned int chunk, r;
495
496 if (unlikely(bytes > i->count))
497 bytes = i->count;
498 if (unlikely(!bytes))
499 return 0;
Al Viro78e1f382018-11-25 16:24:16 -0500500
501 if (!sanity(i))
502 return 0;
503
Al Viro6852df12021-05-02 17:24:40 -0400504 while (bytes) {
Al Viro8fad7762022-06-14 13:53:53 -0400505 struct page *page = append_pipe(i, bytes, &r);
506 char *p;
507
508 if (!page)
509 break;
510 chunk = min_t(size_t, bytes, PAGE_SIZE - r);
511 p = kmap_local_page(page);
Al Viro6852df12021-05-02 17:24:40 -0400512 sum = csum_and_memcpy(p + r, addr + off, chunk, sum, off);
Al Viro2495bdcc2021-04-30 13:40:48 -0400513 kunmap_local(p);
Al Viro78e1f382018-11-25 16:24:16 -0500514 off += chunk;
Al Viro8fad7762022-06-14 13:53:53 -0400515 bytes -= chunk;
Al Viro6852df12021-05-02 17:24:40 -0400516 }
517 *sump = sum;
Al Viro6852df12021-05-02 17:24:40 -0400518 return off;
Al Viro78e1f382018-11-25 16:24:16 -0500519}
520
Al Viroaa28de22017-06-29 21:45:10 -0400521size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
Al Viro62a80672014-04-04 23:12:29 -0400522{
David Howells00e23702018-10-22 13:07:28 +0100523 if (unlikely(iov_iter_is_pipe(i)))
Al Viro241699c2016-09-22 16:33:12 -0400524 return copy_pipe_to_iter(addr, bytes, i);
Al Virofcb14cb2022-05-22 14:59:25 -0400525 if (user_backed_iter(i))
Al Viro09fc68dc2017-06-29 22:25:14 -0400526 might_fault();
Al Viro7baa5092021-05-02 11:35:03 -0400527 iterate_and_advance(i, bytes, base, len, off,
528 copyout(base, addr + off, len),
529 memcpy(base, addr + off, len)
Al Viro3d4d3e42014-11-27 14:28:06 -0500530 )
Al Viro62a80672014-04-04 23:12:29 -0400531
Al Viro3d4d3e42014-11-27 14:28:06 -0500532 return bytes;
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400533}
Al Viroaa28de22017-06-29 21:45:10 -0400534EXPORT_SYMBOL(_copy_to_iter);
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400535
Dan Williamsec6347b2020-10-05 20:40:16 -0700536#ifdef CONFIG_ARCH_HAS_COPY_MC
537static int copyout_mc(void __user *to, const void *from, size_t n)
Dan Williams87803562018-05-03 17:06:31 -0700538{
Linus Torvalds96d4f262019-01-03 18:57:57 -0800539 if (access_ok(to, n)) {
Marco Elverd0ef4c32020-01-21 17:05:11 +0100540 instrument_copy_to_user(to, from, n);
Dan Williamsec6347b2020-10-05 20:40:16 -0700541 n = copy_mc_to_user((__force void *) to, from, n);
Dan Williams87803562018-05-03 17:06:31 -0700542 }
543 return n;
544}
545
Dan Williamsec6347b2020-10-05 20:40:16 -0700546static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
Dan Williamsca146f62018-07-08 13:46:12 -0700547 struct iov_iter *i)
548{
Al Viro8fad7762022-06-14 13:53:53 -0400549 size_t xfer = 0;
550 unsigned int off, chunk;
551
552 if (unlikely(bytes > i->count))
553 bytes = i->count;
554 if (unlikely(!bytes))
555 return 0;
Dan Williamsca146f62018-07-08 13:46:12 -0700556
557 if (!sanity(i))
558 return 0;
559
Al Viro8fad7762022-06-14 13:53:53 -0400560 while (bytes) {
561 struct page *page = append_pipe(i, bytes, &off);
Dan Williamsca146f62018-07-08 13:46:12 -0700562 unsigned long rem;
Al Viro8fad7762022-06-14 13:53:53 -0400563 char *p;
564
565 if (!page)
566 break;
567 chunk = min_t(size_t, bytes, PAGE_SIZE - off);
568 p = kmap_local_page(page);
Al Viro2a510a72021-05-02 17:16:34 -0400569 rem = copy_mc_to_kernel(p + off, addr + xfer, chunk);
570 chunk -= rem;
571 kunmap_local(p);
Al Viro8fad7762022-06-14 13:53:53 -0400572 xfer += chunk;
573 bytes -= chunk;
Al Viroc3497fd2022-06-12 19:50:29 -0400574 if (rem) {
Al Viro8fad7762022-06-14 13:53:53 -0400575 iov_iter_revert(i, rem);
Dan Williamsca146f62018-07-08 13:46:12 -0700576 break;
Al Viroc3497fd2022-06-12 19:50:29 -0400577 }
Al Viro2a510a72021-05-02 17:16:34 -0400578 }
Dan Williamsca146f62018-07-08 13:46:12 -0700579 return xfer;
580}
581
Dan Williamsbf3eeb92018-07-08 13:46:02 -0700582/**
Dan Williamsec6347b2020-10-05 20:40:16 -0700583 * _copy_mc_to_iter - copy to iter with source memory error exception handling
Dan Williamsbf3eeb92018-07-08 13:46:02 -0700584 * @addr: source kernel address
585 * @bytes: total transfer length
Randy Dunlap44e55992021-09-07 19:58:54 -0700586 * @i: destination iterator
Dan Williamsbf3eeb92018-07-08 13:46:02 -0700587 *
Dan Williamsec6347b2020-10-05 20:40:16 -0700588 * The pmem driver deploys this for the dax operation
589 * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
590 * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
591 * successfully copied.
Dan Williamsbf3eeb92018-07-08 13:46:02 -0700592 *
Dan Williamsec6347b2020-10-05 20:40:16 -0700593 * The main differences between this and typical _copy_to_iter().
Dan Williamsbf3eeb92018-07-08 13:46:02 -0700594 *
595 * * Typical tail/residue handling after a fault retries the copy
596 * byte-by-byte until the fault happens again. Re-triggering machine
597 * checks is potentially fatal so the implementation uses source
598 * alignment and poison alignment assumptions to avoid re-triggering
599 * hardware exceptions.
600 *
601 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
602 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
603 * a short copy.
Randy Dunlap44e55992021-09-07 19:58:54 -0700604 *
605 * Return: number of bytes copied (may be %0)
Dan Williamsbf3eeb92018-07-08 13:46:02 -0700606 */
Dan Williamsec6347b2020-10-05 20:40:16 -0700607size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
Dan Williams87803562018-05-03 17:06:31 -0700608{
David Howells00e23702018-10-22 13:07:28 +0100609 if (unlikely(iov_iter_is_pipe(i)))
Dan Williamsec6347b2020-10-05 20:40:16 -0700610 return copy_mc_pipe_to_iter(addr, bytes, i);
Al Virofcb14cb2022-05-22 14:59:25 -0400611 if (user_backed_iter(i))
Dan Williams87803562018-05-03 17:06:31 -0700612 might_fault();
Al Viro7baa5092021-05-02 11:35:03 -0400613 __iterate_and_advance(i, bytes, base, len, off,
614 copyout_mc(base, addr + off, len),
615 copy_mc_to_kernel(base, addr + off, len)
Dan Williams87803562018-05-03 17:06:31 -0700616 )
617
618 return bytes;
619}
Dan Williamsec6347b2020-10-05 20:40:16 -0700620EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
621#endif /* CONFIG_ARCH_HAS_COPY_MC */
Dan Williams87803562018-05-03 17:06:31 -0700622
Al Viroaa28de22017-06-29 21:45:10 -0400623size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400624{
David Howells00e23702018-10-22 13:07:28 +0100625 if (unlikely(iov_iter_is_pipe(i))) {
Al Viro241699c2016-09-22 16:33:12 -0400626 WARN_ON(1);
627 return 0;
628 }
Al Virofcb14cb2022-05-22 14:59:25 -0400629 if (user_backed_iter(i))
Al Viro09fc68dc2017-06-29 22:25:14 -0400630 might_fault();
Al Viro7baa5092021-05-02 11:35:03 -0400631 iterate_and_advance(i, bytes, base, len, off,
632 copyin(addr + off, base, len),
633 memcpy(addr + off, base, len)
Al Viro0dbca9a2014-11-27 14:26:43 -0500634 )
635
636 return bytes;
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400637}
Al Viroaa28de22017-06-29 21:45:10 -0400638EXPORT_SYMBOL(_copy_from_iter);
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400639
Al Viroaa28de22017-06-29 21:45:10 -0400640size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
Al Viroaa583092014-11-27 20:27:08 -0500641{
David Howells00e23702018-10-22 13:07:28 +0100642 if (unlikely(iov_iter_is_pipe(i))) {
Al Viro241699c2016-09-22 16:33:12 -0400643 WARN_ON(1);
644 return 0;
645 }
Al Viro7baa5092021-05-02 11:35:03 -0400646 iterate_and_advance(i, bytes, base, len, off,
647 __copy_from_user_inatomic_nocache(addr + off, base, len),
648 memcpy(addr + off, base, len)
Al Viroaa583092014-11-27 20:27:08 -0500649 )
650
651 return bytes;
652}
Al Viroaa28de22017-06-29 21:45:10 -0400653EXPORT_SYMBOL(_copy_from_iter_nocache);
Al Viroaa583092014-11-27 20:27:08 -0500654
Dan Williams0aed55a2017-05-29 12:22:50 -0700655#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
Dan Williamsabd08d72018-07-08 13:46:07 -0700656/**
657 * _copy_from_iter_flushcache - write destination through cpu cache
658 * @addr: destination kernel address
659 * @bytes: total transfer length
Randy Dunlap44e55992021-09-07 19:58:54 -0700660 * @i: source iterator
Dan Williamsabd08d72018-07-08 13:46:07 -0700661 *
662 * The pmem driver arranges for filesystem-dax to use this facility via
663 * dax_copy_from_iter() for ensuring that writes to persistent memory
664 * are flushed through the CPU cache. It is differentiated from
665 * _copy_from_iter_nocache() in that guarantees all data is flushed for
666 * all iterator types. The _copy_from_iter_nocache() only attempts to
667 * bypass the cache for the ITER_IOVEC case, and on some archs may use
668 * instructions that strand dirty-data in the cache.
Randy Dunlap44e55992021-09-07 19:58:54 -0700669 *
670 * Return: number of bytes copied (may be %0)
Dan Williamsabd08d72018-07-08 13:46:07 -0700671 */
Linus Torvalds6a37e942017-07-07 20:39:20 -0700672size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
Dan Williams0aed55a2017-05-29 12:22:50 -0700673{
David Howells00e23702018-10-22 13:07:28 +0100674 if (unlikely(iov_iter_is_pipe(i))) {
Dan Williams0aed55a2017-05-29 12:22:50 -0700675 WARN_ON(1);
676 return 0;
677 }
Al Viro7baa5092021-05-02 11:35:03 -0400678 iterate_and_advance(i, bytes, base, len, off,
679 __copy_from_user_flushcache(addr + off, base, len),
680 memcpy_flushcache(addr + off, base, len)
Dan Williams0aed55a2017-05-29 12:22:50 -0700681 )
682
683 return bytes;
684}
Linus Torvalds6a37e942017-07-07 20:39:20 -0700685EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
Dan Williams0aed55a2017-05-29 12:22:50 -0700686#endif
687
Al Viro72e809e2017-06-29 21:52:57 -0400688static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
689{
Eric Dumazet6daef952019-02-26 10:42:39 -0800690 struct page *head;
691 size_t v = n + offset;
692
693 /*
694 * The general case needs to access the page order in order
695 * to compute the page size.
696 * However, we mostly deal with order-0 pages and thus can
697 * avoid a possible cache line miss for requests that fit all
698 * page orders.
699 */
700 if (n <= v && v <= PAGE_SIZE)
701 return true;
702
703 head = compound_head(page);
704 v += (page - head) << PAGE_SHIFT;
Petar Penkova90bcb82017-08-29 11:20:32 -0700705
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -0700706 if (likely(n <= v && v <= (page_size(head))))
Al Viro72e809e2017-06-29 21:52:57 -0400707 return true;
708 WARN_ON(1);
709 return false;
710}
Al Virocbbd26b2016-11-01 22:09:04 -0400711
Al Viro08aa6472021-04-29 20:42:25 -0400712size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
713 struct iov_iter *i)
714{
715 size_t res = 0;
716 if (unlikely(!page_copy_sane(page, offset, bytes)))
717 return 0;
Al Virof0f6b612022-06-23 17:21:37 -0400718 if (unlikely(iov_iter_is_pipe(i)))
719 return copy_page_to_iter_pipe(page, offset, bytes, i);
Al Viro08aa6472021-04-29 20:42:25 -0400720 page += offset / PAGE_SIZE; // first subpage
721 offset %= PAGE_SIZE;
722 while (1) {
Al Virof0f6b612022-06-23 17:21:37 -0400723 void *kaddr = kmap_local_page(page);
724 size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
725 n = _copy_to_iter(kaddr + offset, n, i);
726 kunmap_local(kaddr);
Al Viro08aa6472021-04-29 20:42:25 -0400727 res += n;
728 bytes -= n;
729 if (!bytes || !n)
730 break;
731 offset += n;
732 if (offset == PAGE_SIZE) {
733 page++;
734 offset = 0;
735 }
736 }
737 return res;
738}
Al Virod2715242014-11-27 14:22:37 -0500739EXPORT_SYMBOL(copy_page_to_iter);
740
741size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
742 struct iov_iter *i)
743{
Al Viroc03f05f2022-07-29 12:54:53 -0400744 size_t res = 0;
745 if (!page_copy_sane(page, offset, bytes))
746 return 0;
747 page += offset / PAGE_SIZE; // first subpage
748 offset %= PAGE_SIZE;
749 while (1) {
Al Viro55ca3752021-04-27 12:33:24 -0400750 void *kaddr = kmap_local_page(page);
Al Viroc03f05f2022-07-29 12:54:53 -0400751 size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
752 n = _copy_from_iter(kaddr + offset, n, i);
Al Viro55ca3752021-04-27 12:33:24 -0400753 kunmap_local(kaddr);
Al Viroc03f05f2022-07-29 12:54:53 -0400754 res += n;
755 bytes -= n;
756 if (!bytes || !n)
757 break;
758 offset += n;
759 if (offset == PAGE_SIZE) {
760 page++;
761 offset = 0;
762 }
Al Viro28f38db2021-06-02 17:25:59 -0400763 }
Al Viroc03f05f2022-07-29 12:54:53 -0400764 return res;
Al Virod2715242014-11-27 14:22:37 -0500765}
766EXPORT_SYMBOL(copy_page_from_iter);
767
Al Viro241699c2016-09-22 16:33:12 -0400768static size_t pipe_zero(size_t bytes, struct iov_iter *i)
769{
Al Viro8fad7762022-06-14 13:53:53 -0400770 unsigned int chunk, off;
771
772 if (unlikely(bytes > i->count))
773 bytes = i->count;
774 if (unlikely(!bytes))
775 return 0;
Al Viro241699c2016-09-22 16:33:12 -0400776
777 if (!sanity(i))
778 return 0;
779
Al Viro8fad7762022-06-14 13:53:53 -0400780 for (size_t n = bytes; n; n -= chunk) {
781 struct page *page = append_pipe(i, n, &off);
782 char *p;
Al Viro241699c2016-09-22 16:33:12 -0400783
Al Viro8fad7762022-06-14 13:53:53 -0400784 if (!page)
785 return bytes - n;
786 chunk = min_t(size_t, n, PAGE_SIZE - off);
787 p = kmap_local_page(page);
Al Viro893839f2021-04-30 18:39:25 -0400788 memset(p + off, 0, chunk);
789 kunmap_local(p);
Al Viro8fad7762022-06-14 13:53:53 -0400790 }
Al Viro241699c2016-09-22 16:33:12 -0400791 return bytes;
792}
793
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400794size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
795{
David Howells00e23702018-10-22 13:07:28 +0100796 if (unlikely(iov_iter_is_pipe(i)))
Al Viro241699c2016-09-22 16:33:12 -0400797 return pipe_zero(bytes, i);
Al Viro7baa5092021-05-02 11:35:03 -0400798 iterate_and_advance(i, bytes, base, len, count,
799 clear_user(base, len),
800 memset(base, 0, len)
Al Viro8442fa42014-11-27 14:18:54 -0500801 )
802
803 return bytes;
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400804}
805EXPORT_SYMBOL(iov_iter_zero);
806
Al Virof0b65f32021-04-30 10:26:41 -0400807size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes,
808 struct iov_iter *i)
Al Viro62a80672014-04-04 23:12:29 -0400809{
Al Viro04a31162014-11-27 13:51:41 -0500810 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
Al Viro72e809e2017-06-29 21:52:57 -0400811 if (unlikely(!page_copy_sane(page, offset, bytes))) {
812 kunmap_atomic(kaddr);
813 return 0;
814 }
David Howells9ea9ce02018-10-20 00:57:56 +0100815 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
Al Viro241699c2016-09-22 16:33:12 -0400816 kunmap_atomic(kaddr);
817 WARN_ON(1);
818 return 0;
819 }
Al Viro7baa5092021-05-02 11:35:03 -0400820 iterate_and_advance(i, bytes, base, len, off,
821 copyin(p + off, base, len),
822 memcpy(p + off, base, len)
Al Viro04a31162014-11-27 13:51:41 -0500823 )
824 kunmap_atomic(kaddr);
825 return bytes;
Al Viro62a80672014-04-04 23:12:29 -0400826}
Al Virof0b65f32021-04-30 10:26:41 -0400827EXPORT_SYMBOL(copy_page_from_iter_atomic);
Al Viro62a80672014-04-04 23:12:29 -0400828
Al Virob9dc6f62017-01-14 19:33:08 -0500829static void pipe_advance(struct iov_iter *i, size_t size)
830{
831 struct pipe_inode_info *pipe = i->pipe;
Al Viro10f525a2022-06-15 02:02:51 -0400832 int off = i->last_offset;
David Howells8cefc102019-11-15 13:30:32 +0000833
Al Viro2c855de2022-06-15 16:03:25 -0400834 if (!off && !size) {
835 pipe_discard_from(pipe, i->start_head); // discard everything
836 return;
Al Virob9dc6f62017-01-14 19:33:08 -0500837 }
838 i->count -= size;
Al Viro2c855de2022-06-15 16:03:25 -0400839 while (1) {
840 struct pipe_buffer *buf = pipe_buf(pipe, i->head);
841 if (off) /* make it relative to the beginning of buffer */
Al Viro10f525a2022-06-15 02:02:51 -0400842 size += abs(off) - buf->offset;
Al Viro2c855de2022-06-15 16:03:25 -0400843 if (size <= buf->len) {
844 buf->len = size;
Al Viro10f525a2022-06-15 02:02:51 -0400845 i->last_offset = last_offset(buf);
Al Viro2c855de2022-06-15 16:03:25 -0400846 break;
847 }
848 size -= buf->len;
849 i->head++;
850 off = 0;
851 }
852 pipe_discard_from(pipe, i->head + 1); // discard everything past this one
Al Viro241699c2016-09-22 16:33:12 -0400853}
854
Pavel Begunkov54c81952021-01-09 16:03:01 +0000855static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
856{
Al Viro18fa9af2022-06-06 23:44:33 -0400857 const struct bio_vec *bvec, *end;
Pavel Begunkov54c81952021-01-09 16:03:01 +0000858
Al Viro18fa9af2022-06-06 23:44:33 -0400859 if (!i->count)
860 return;
861 i->count -= size;
Pavel Begunkov54c81952021-01-09 16:03:01 +0000862
Al Viro18fa9af2022-06-06 23:44:33 -0400863 size += i->iov_offset;
864
865 for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) {
866 if (likely(size < bvec->bv_len))
867 break;
868 size -= bvec->bv_len;
869 }
870 i->iov_offset = size;
871 i->nr_segs -= bvec - i->bvec;
872 i->bvec = bvec;
Pavel Begunkov54c81952021-01-09 16:03:01 +0000873}
874
Al Viro185ac4d2021-04-23 12:58:53 -0400875static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)
876{
877 const struct iovec *iov, *end;
878
879 if (!i->count)
880 return;
881 i->count -= size;
882
883 size += i->iov_offset; // from beginning of current segment
884 for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) {
885 if (likely(size < iov->iov_len))
886 break;
887 size -= iov->iov_len;
888 }
889 i->iov_offset = size;
890 i->nr_segs -= iov - i->iov;
891 i->iov = iov;
892}
893
Al Viro62a80672014-04-04 23:12:29 -0400894void iov_iter_advance(struct iov_iter *i, size_t size)
895{
Al Viro3b3fc052021-04-23 22:24:08 -0400896 if (unlikely(i->count < size))
897 size = i->count;
Al Virofcb14cb2022-05-22 14:59:25 -0400898 if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) {
899 i->iov_offset += size;
900 i->count -= size;
901 } else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) {
Al Viro185ac4d2021-04-23 12:58:53 -0400902 /* iovec and kvec have identical layouts */
903 iov_iter_iovec_advance(i, size);
904 } else if (iov_iter_is_bvec(i)) {
905 iov_iter_bvec_advance(i, size);
906 } else if (iov_iter_is_pipe(i)) {
Al Viro241699c2016-09-22 16:33:12 -0400907 pipe_advance(i, size);
Al Viro185ac4d2021-04-23 12:58:53 -0400908 } else if (iov_iter_is_discard(i)) {
909 i->count -= size;
David Howells7ff5062072020-02-10 10:00:21 +0000910 }
Al Viro62a80672014-04-04 23:12:29 -0400911}
912EXPORT_SYMBOL(iov_iter_advance);
913
Al Viro27c0e372017-02-17 18:42:24 -0500914void iov_iter_revert(struct iov_iter *i, size_t unroll)
915{
916 if (!unroll)
917 return;
Al Viro5b47d592017-05-08 13:54:47 -0400918 if (WARN_ON(unroll > MAX_RW_COUNT))
919 return;
Al Viro27c0e372017-02-17 18:42:24 -0500920 i->count += unroll;
David Howells00e23702018-10-22 13:07:28 +0100921 if (unlikely(iov_iter_is_pipe(i))) {
Al Viro27c0e372017-02-17 18:42:24 -0500922 struct pipe_inode_info *pipe = i->pipe;
Al Viro92acdc42022-06-12 17:54:35 -0400923 unsigned int head = pipe->head;
924
925 while (head > i->start_head) {
926 struct pipe_buffer *b = pipe_buf(pipe, --head);
927 if (unroll < b->len) {
928 b->len -= unroll;
Al Viro10f525a2022-06-15 02:02:51 -0400929 i->last_offset = last_offset(b);
Al Viro92acdc42022-06-12 17:54:35 -0400930 i->head = head;
931 return;
Al Viro27c0e372017-02-17 18:42:24 -0500932 }
Al Viro92acdc42022-06-12 17:54:35 -0400933 unroll -= b->len;
934 pipe_buf_release(pipe, b);
935 pipe->head--;
Al Viro27c0e372017-02-17 18:42:24 -0500936 }
Al Viro10f525a2022-06-15 02:02:51 -0400937 i->last_offset = 0;
Al Viro92acdc42022-06-12 17:54:35 -0400938 i->head = head;
Al Viro27c0e372017-02-17 18:42:24 -0500939 return;
940 }
David Howells9ea9ce02018-10-20 00:57:56 +0100941 if (unlikely(iov_iter_is_discard(i)))
942 return;
Al Viro27c0e372017-02-17 18:42:24 -0500943 if (unroll <= i->iov_offset) {
944 i->iov_offset -= unroll;
945 return;
946 }
947 unroll -= i->iov_offset;
Al Virofcb14cb2022-05-22 14:59:25 -0400948 if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) {
David Howells7ff5062072020-02-10 10:00:21 +0000949 BUG(); /* We should never go beyond the start of the specified
950 * range since we might then be straying into pages that
951 * aren't pinned.
952 */
953 } else if (iov_iter_is_bvec(i)) {
Al Viro27c0e372017-02-17 18:42:24 -0500954 const struct bio_vec *bvec = i->bvec;
955 while (1) {
956 size_t n = (--bvec)->bv_len;
957 i->nr_segs++;
958 if (unroll <= n) {
959 i->bvec = bvec;
960 i->iov_offset = n - unroll;
961 return;
962 }
963 unroll -= n;
964 }
965 } else { /* same logics for iovec and kvec */
966 const struct iovec *iov = i->iov;
967 while (1) {
968 size_t n = (--iov)->iov_len;
969 i->nr_segs++;
970 if (unroll <= n) {
971 i->iov = iov;
972 i->iov_offset = n - unroll;
973 return;
974 }
975 unroll -= n;
976 }
977 }
978}
979EXPORT_SYMBOL(iov_iter_revert);
980
Al Viro62a80672014-04-04 23:12:29 -0400981/*
982 * Return the count of just the current iov_iter segment.
983 */
984size_t iov_iter_single_seg_count(const struct iov_iter *i)
985{
Al Viro28f38db2021-06-02 17:25:59 -0400986 if (i->nr_segs > 1) {
987 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
988 return min(i->count, i->iov->iov_len - i->iov_offset);
989 if (iov_iter_is_bvec(i))
990 return min(i->count, i->bvec->bv_len - i->iov_offset);
991 }
992 return i->count;
Al Viro62a80672014-04-04 23:12:29 -0400993}
994EXPORT_SYMBOL(iov_iter_single_seg_count);
995
David Howellsaa563d72018-10-20 00:57:56 +0100996void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
Al Viro05afcb72015-01-23 01:08:07 -0500997 const struct kvec *kvec, unsigned long nr_segs,
Al Viroabb78f82014-11-24 14:46:11 -0500998 size_t count)
999{
David Howellsaa563d72018-10-20 00:57:56 +01001000 WARN_ON(direction & ~(READ | WRITE));
Al Viro8cd54c12021-04-22 14:50:39 -04001001 *i = (struct iov_iter){
1002 .iter_type = ITER_KVEC,
1003 .data_source = direction,
1004 .kvec = kvec,
1005 .nr_segs = nr_segs,
1006 .iov_offset = 0,
1007 .count = count
1008 };
Al Viroabb78f82014-11-24 14:46:11 -05001009}
1010EXPORT_SYMBOL(iov_iter_kvec);
1011
David Howellsaa563d72018-10-20 00:57:56 +01001012void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
Al Viro05afcb72015-01-23 01:08:07 -05001013 const struct bio_vec *bvec, unsigned long nr_segs,
1014 size_t count)
1015{
David Howellsaa563d72018-10-20 00:57:56 +01001016 WARN_ON(direction & ~(READ | WRITE));
Al Viro8cd54c12021-04-22 14:50:39 -04001017 *i = (struct iov_iter){
1018 .iter_type = ITER_BVEC,
1019 .data_source = direction,
1020 .bvec = bvec,
1021 .nr_segs = nr_segs,
1022 .iov_offset = 0,
1023 .count = count
1024 };
Al Viro05afcb72015-01-23 01:08:07 -05001025}
1026EXPORT_SYMBOL(iov_iter_bvec);
1027
David Howellsaa563d72018-10-20 00:57:56 +01001028void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
Al Viro241699c2016-09-22 16:33:12 -04001029 struct pipe_inode_info *pipe,
1030 size_t count)
1031{
David Howellsaa563d72018-10-20 00:57:56 +01001032 BUG_ON(direction != READ);
David Howells8cefc102019-11-15 13:30:32 +00001033 WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
Al Viro8cd54c12021-04-22 14:50:39 -04001034 *i = (struct iov_iter){
1035 .iter_type = ITER_PIPE,
1036 .data_source = false,
1037 .pipe = pipe,
1038 .head = pipe->head,
1039 .start_head = pipe->head,
Al Viro10f525a2022-06-15 02:02:51 -04001040 .last_offset = 0,
Al Viro8cd54c12021-04-22 14:50:39 -04001041 .count = count
1042 };
Al Viro241699c2016-09-22 16:33:12 -04001043}
1044EXPORT_SYMBOL(iov_iter_pipe);
1045
David Howells9ea9ce02018-10-20 00:57:56 +01001046/**
David Howells7ff5062072020-02-10 10:00:21 +00001047 * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
1048 * @i: The iterator to initialise.
1049 * @direction: The direction of the transfer.
1050 * @xarray: The xarray to access.
1051 * @start: The start file position.
1052 * @count: The size of the I/O buffer in bytes.
1053 *
1054 * Set up an I/O iterator to either draw data out of the pages attached to an
1055 * inode or to inject data into those pages. The pages *must* be prevented
1056 * from evaporation, either by taking a ref on them or locking them by the
1057 * caller.
1058 */
1059void iov_iter_xarray(struct iov_iter *i, unsigned int direction,
1060 struct xarray *xarray, loff_t start, size_t count)
1061{
1062 BUG_ON(direction & ~1);
Al Viro8cd54c12021-04-22 14:50:39 -04001063 *i = (struct iov_iter) {
1064 .iter_type = ITER_XARRAY,
1065 .data_source = direction,
1066 .xarray = xarray,
1067 .xarray_start = start,
1068 .count = count,
1069 .iov_offset = 0
1070 };
David Howells7ff5062072020-02-10 10:00:21 +00001071}
1072EXPORT_SYMBOL(iov_iter_xarray);
1073
1074/**
David Howells9ea9ce02018-10-20 00:57:56 +01001075 * iov_iter_discard - Initialise an I/O iterator that discards data
1076 * @i: The iterator to initialise.
1077 * @direction: The direction of the transfer.
1078 * @count: The size of the I/O buffer in bytes.
1079 *
1080 * Set up an I/O iterator that just discards everything that's written to it.
1081 * It's only available as a READ iterator.
1082 */
1083void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1084{
1085 BUG_ON(direction != READ);
Al Viro8cd54c12021-04-22 14:50:39 -04001086 *i = (struct iov_iter){
1087 .iter_type = ITER_DISCARD,
1088 .data_source = false,
1089 .count = count,
1090 .iov_offset = 0
1091 };
David Howells9ea9ce02018-10-20 00:57:56 +01001092}
1093EXPORT_SYMBOL(iov_iter_discard);
1094
Keith Buschcfa320f2022-06-10 12:58:27 -07001095static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
1096 unsigned len_mask)
1097{
1098 size_t size = i->count;
1099 size_t skip = i->iov_offset;
1100 unsigned k;
1101
1102 for (k = 0; k < i->nr_segs; k++, skip = 0) {
1103 size_t len = i->iov[k].iov_len - skip;
1104
1105 if (len > size)
1106 len = size;
1107 if (len & len_mask)
1108 return false;
1109 if ((unsigned long)(i->iov[k].iov_base + skip) & addr_mask)
1110 return false;
1111
1112 size -= len;
1113 if (!size)
1114 break;
1115 }
1116 return true;
1117}
1118
1119static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
1120 unsigned len_mask)
1121{
1122 size_t size = i->count;
1123 unsigned skip = i->iov_offset;
1124 unsigned k;
1125
1126 for (k = 0; k < i->nr_segs; k++, skip = 0) {
1127 size_t len = i->bvec[k].bv_len - skip;
1128
1129 if (len > size)
1130 len = size;
1131 if (len & len_mask)
1132 return false;
1133 if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)
1134 return false;
1135
1136 size -= len;
1137 if (!size)
1138 break;
1139 }
1140 return true;
1141}
1142
1143/**
1144 * iov_iter_is_aligned() - Check if the addresses and lengths of each segments
1145 * are aligned to the parameters.
1146 *
1147 * @i: &struct iov_iter to restore
1148 * @addr_mask: bit mask to check against the iov element's addresses
1149 * @len_mask: bit mask to check against the iov element's lengths
1150 *
1151 * Return: false if any addresses or lengths intersect with the provided masks
1152 */
1153bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask,
1154 unsigned len_mask)
1155{
Al Virofcb14cb2022-05-22 14:59:25 -04001156 if (likely(iter_is_ubuf(i))) {
1157 if (i->count & len_mask)
1158 return false;
1159 if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask)
1160 return false;
1161 return true;
1162 }
1163
Keith Buschcfa320f2022-06-10 12:58:27 -07001164 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
1165 return iov_iter_aligned_iovec(i, addr_mask, len_mask);
1166
1167 if (iov_iter_is_bvec(i))
1168 return iov_iter_aligned_bvec(i, addr_mask, len_mask);
1169
1170 if (iov_iter_is_pipe(i)) {
Keith Buschcfa320f2022-06-10 12:58:27 -07001171 size_t size = i->count;
1172
1173 if (size & len_mask)
1174 return false;
Al Viro10f525a2022-06-15 02:02:51 -04001175 if (size && i->last_offset > 0) {
1176 if (i->last_offset & addr_mask)
Keith Buschcfa320f2022-06-10 12:58:27 -07001177 return false;
1178 }
1179
1180 return true;
1181 }
1182
1183 if (iov_iter_is_xarray(i)) {
1184 if (i->count & len_mask)
1185 return false;
1186 if ((i->xarray_start + i->iov_offset) & addr_mask)
1187 return false;
1188 }
1189
1190 return true;
1191}
1192EXPORT_SYMBOL_GPL(iov_iter_is_aligned);
1193
Al Viro9221d2e2021-04-25 00:44:35 -04001194static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
Al Viro62a80672014-04-04 23:12:29 -04001195{
Al Viro04a31162014-11-27 13:51:41 -05001196 unsigned long res = 0;
1197 size_t size = i->count;
Al Viro9221d2e2021-04-25 00:44:35 -04001198 size_t skip = i->iov_offset;
1199 unsigned k;
Al Viro04a31162014-11-27 13:51:41 -05001200
Al Viro9221d2e2021-04-25 00:44:35 -04001201 for (k = 0; k < i->nr_segs; k++, skip = 0) {
1202 size_t len = i->iov[k].iov_len - skip;
1203 if (len) {
1204 res |= (unsigned long)i->iov[k].iov_base + skip;
1205 if (len > size)
1206 len = size;
1207 res |= len;
1208 size -= len;
1209 if (!size)
1210 break;
1211 }
1212 }
1213 return res;
1214}
1215
1216static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i)
1217{
1218 unsigned res = 0;
1219 size_t size = i->count;
1220 unsigned skip = i->iov_offset;
1221 unsigned k;
1222
1223 for (k = 0; k < i->nr_segs; k++, skip = 0) {
1224 size_t len = i->bvec[k].bv_len - skip;
1225 res |= (unsigned long)i->bvec[k].bv_offset + skip;
1226 if (len > size)
1227 len = size;
1228 res |= len;
1229 size -= len;
1230 if (!size)
1231 break;
1232 }
1233 return res;
1234}
1235
1236unsigned long iov_iter_alignment(const struct iov_iter *i)
1237{
Al Virofcb14cb2022-05-22 14:59:25 -04001238 if (likely(iter_is_ubuf(i))) {
1239 size_t size = i->count;
1240 if (size)
1241 return ((unsigned long)i->ubuf + i->iov_offset) | size;
1242 return 0;
1243 }
1244
Al Viro9221d2e2021-04-25 00:44:35 -04001245 /* iovec and kvec have identical layouts */
1246 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
1247 return iov_iter_alignment_iovec(i);
1248
1249 if (iov_iter_is_bvec(i))
1250 return iov_iter_alignment_bvec(i);
1251
1252 if (iov_iter_is_pipe(i)) {
Al Viro9221d2e2021-04-25 00:44:35 -04001253 size_t size = i->count;
Jan Karae0ff1262019-12-16 11:54:32 +01001254
Al Viro10f525a2022-06-15 02:02:51 -04001255 if (size && i->last_offset > 0)
1256 return size | i->last_offset;
Al Viro241699c2016-09-22 16:33:12 -04001257 return size;
1258 }
Al Viro9221d2e2021-04-25 00:44:35 -04001259
1260 if (iov_iter_is_xarray(i))
David Howells3d14ec12021-04-25 22:02:38 +01001261 return (i->xarray_start + i->iov_offset) | i->count;
Al Viro9221d2e2021-04-25 00:44:35 -04001262
1263 return 0;
Al Viro62a80672014-04-04 23:12:29 -04001264}
1265EXPORT_SYMBOL(iov_iter_alignment);
1266
Al Viro357f4352016-04-08 19:05:19 -04001267unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1268{
Al Viro33844e62016-12-21 21:55:02 -05001269 unsigned long res = 0;
Al Viro610c7a72021-04-25 01:03:16 -04001270 unsigned long v = 0;
Al Viro357f4352016-04-08 19:05:19 -04001271 size_t size = i->count;
Al Viro610c7a72021-04-25 01:03:16 -04001272 unsigned k;
Al Viro357f4352016-04-08 19:05:19 -04001273
Al Virofcb14cb2022-05-22 14:59:25 -04001274 if (iter_is_ubuf(i))
1275 return 0;
1276
Al Viro610c7a72021-04-25 01:03:16 -04001277 if (WARN_ON(!iter_is_iovec(i)))
Al Viro241699c2016-09-22 16:33:12 -04001278 return ~0U;
Al Viro241699c2016-09-22 16:33:12 -04001279
Al Viro610c7a72021-04-25 01:03:16 -04001280 for (k = 0; k < i->nr_segs; k++) {
1281 if (i->iov[k].iov_len) {
1282 unsigned long base = (unsigned long)i->iov[k].iov_base;
1283 if (v) // if not the first one
1284 res |= base | v; // this start | previous end
1285 v = base + i->iov[k].iov_len;
1286 if (size <= i->iov[k].iov_len)
1287 break;
1288 size -= i->iov[k].iov_len;
1289 }
1290 }
Al Viro33844e62016-12-21 21:55:02 -05001291 return res;
Al Viro357f4352016-04-08 19:05:19 -04001292}
1293EXPORT_SYMBOL(iov_iter_gap_alignment);
1294
Al Viro3cf42da2022-06-17 14:45:41 -04001295static int want_pages_array(struct page ***res, size_t size,
1296 size_t start, unsigned int maxpages)
Al Viroacbdeb82022-06-17 13:35:35 -04001297{
Al Viro3cf42da2022-06-17 14:45:41 -04001298 unsigned int count = DIV_ROUND_UP(size + start, PAGE_SIZE);
1299
1300 if (count > maxpages)
1301 count = maxpages;
1302 WARN_ON(!count); // caller should've prevented that
1303 if (!*res) {
1304 *res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
1305 if (!*res)
1306 return 0;
1307 }
1308 return count;
Al Viroacbdeb82022-06-17 13:35:35 -04001309}
1310
Al Viro85200082022-06-17 14:30:39 -04001311static ssize_t pipe_get_pages(struct iov_iter *i,
1312 struct page ***pages, size_t maxsize, unsigned maxpages,
1313 size_t *start)
Al Viro241699c2016-09-22 16:33:12 -04001314{
Al Viro746de1f2022-06-14 16:38:53 -04001315 unsigned int npages, count, off, chunk;
Al Viro85200082022-06-17 14:30:39 -04001316 struct page **p;
Al Viro746de1f2022-06-14 16:38:53 -04001317 size_t left;
Al Viro241699c2016-09-22 16:33:12 -04001318
Al Viro85200082022-06-17 14:30:39 -04001319 if (!sanity(i))
1320 return -EFAULT;
1321
1322 *start = off = pipe_npages(i, &npages);
Al Viro3cf42da2022-06-17 14:45:41 -04001323 if (!npages)
1324 return -EFAULT;
1325 count = want_pages_array(pages, maxsize, off, min(npages, maxpages));
1326 if (!count)
1327 return -ENOMEM;
Al Viro85200082022-06-17 14:30:39 -04001328 p = *pages;
Al Viro746de1f2022-06-14 16:38:53 -04001329 for (npages = 0, left = maxsize ; npages < count; npages++, left -= chunk) {
1330 struct page *page = append_pipe(i, left, &off);
Al Viroe3b42962022-06-11 02:52:03 -04001331 if (!page)
1332 break;
Al Viro746de1f2022-06-14 16:38:53 -04001333 chunk = min_t(size_t, left, PAGE_SIZE - off);
Al Viro85200082022-06-17 14:30:39 -04001334 get_page(*p++ = page);
Al Viroe3b42962022-06-11 02:52:03 -04001335 }
Al Viro85200082022-06-17 14:30:39 -04001336 if (!npages)
Al Viro241699c2016-09-22 16:33:12 -04001337 return -EFAULT;
Al Viro746de1f2022-06-14 16:38:53 -04001338 return maxsize - left;
Al Viro241699c2016-09-22 16:33:12 -04001339}
1340
David Howells7ff5062072020-02-10 10:00:21 +00001341static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa,
1342 pgoff_t index, unsigned int nr_pages)
1343{
1344 XA_STATE(xas, xa, index);
1345 struct page *page;
1346 unsigned int ret = 0;
1347
1348 rcu_read_lock();
1349 for (page = xas_load(&xas); page; page = xas_next(&xas)) {
1350 if (xas_retry(&xas, page))
1351 continue;
1352
1353 /* Has the page moved or been split? */
1354 if (unlikely(page != xas_reload(&xas))) {
1355 xas_reset(&xas);
1356 continue;
1357 }
1358
1359 pages[ret] = find_subpage(page, xas.xa_index);
1360 get_page(pages[ret]);
1361 if (++ret == nr_pages)
1362 break;
1363 }
1364 rcu_read_unlock();
1365 return ret;
1366}
1367
1368static ssize_t iter_xarray_get_pages(struct iov_iter *i,
Al Viro68fe5062022-06-17 13:48:03 -04001369 struct page ***pages, size_t maxsize,
David Howells7ff5062072020-02-10 10:00:21 +00001370 unsigned maxpages, size_t *_start_offset)
1371{
Al Viro3cf42da2022-06-17 14:45:41 -04001372 unsigned nr, offset, count;
1373 pgoff_t index;
David Howells7ff5062072020-02-10 10:00:21 +00001374 loff_t pos;
1375
David Howells7ff5062072020-02-10 10:00:21 +00001376 pos = i->xarray_start + i->iov_offset;
1377 index = pos >> PAGE_SHIFT;
1378 offset = pos & ~PAGE_MASK;
1379 *_start_offset = offset;
1380
Al Viro3cf42da2022-06-17 14:45:41 -04001381 count = want_pages_array(pages, maxsize, offset, maxpages);
1382 if (!count)
1383 return -ENOMEM;
Al Viro68fe5062022-06-17 13:48:03 -04001384 nr = iter_xarray_populate_pages(*pages, i->xarray, index, count);
David Howells7ff5062072020-02-10 10:00:21 +00001385 if (nr == 0)
1386 return 0;
1387
Al Viroeba2d3d2022-06-10 13:05:12 -04001388 maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);
Al Viro310d9d52022-06-11 04:04:33 -04001389 i->iov_offset += maxsize;
1390 i->count -= maxsize;
Al Viroeba2d3d2022-06-10 13:05:12 -04001391 return maxsize;
David Howells7ff5062072020-02-10 10:00:21 +00001392}
1393
Al Virofcb14cb2022-05-22 14:59:25 -04001394/* must be done on non-empty ITER_UBUF or ITER_IOVEC one */
Al Virodd45ab92022-06-17 16:07:49 -04001395static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)
Al Viro3d671ca2021-04-25 09:14:44 -04001396{
1397 size_t skip;
1398 long k;
1399
Al Virofcb14cb2022-05-22 14:59:25 -04001400 if (iter_is_ubuf(i))
1401 return (unsigned long)i->ubuf + i->iov_offset;
1402
Al Viro3d671ca2021-04-25 09:14:44 -04001403 for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {
Al Viro3d671ca2021-04-25 09:14:44 -04001404 size_t len = i->iov[k].iov_len - skip;
1405
1406 if (unlikely(!len))
1407 continue;
Al Viro59dbd7d2022-06-21 16:10:37 -04001408 if (*size > len)
1409 *size = len;
Al Virodd45ab92022-06-17 16:07:49 -04001410 return (unsigned long)i->iov[k].iov_base + skip;
Al Viro3d671ca2021-04-25 09:14:44 -04001411 }
1412 BUG(); // if it had been empty, we wouldn't get called
1413}
1414
1415/* must be done on non-empty ITER_BVEC one */
1416static struct page *first_bvec_segment(const struct iov_iter *i,
Al Viro59dbd7d2022-06-21 16:10:37 -04001417 size_t *size, size_t *start)
Al Viro3d671ca2021-04-25 09:14:44 -04001418{
1419 struct page *page;
1420 size_t skip = i->iov_offset, len;
1421
1422 len = i->bvec->bv_len - skip;
Al Viro59dbd7d2022-06-21 16:10:37 -04001423 if (*size > len)
1424 *size = len;
Al Viro3d671ca2021-04-25 09:14:44 -04001425 skip += i->bvec->bv_offset;
1426 page = i->bvec->bv_page + skip / PAGE_SIZE;
Al Virodda8e5d2022-06-21 15:55:19 -04001427 *start = skip % PAGE_SIZE;
Al Viro3d671ca2021-04-25 09:14:44 -04001428 return page;
1429}
1430
Al Viro91329552022-06-10 20:38:20 -04001431static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
Al Viro62a80672014-04-04 23:12:29 -04001432 struct page ***pages, size_t maxsize,
Al Viro451c0ba2022-06-17 13:54:15 -04001433 unsigned int maxpages, size_t *start)
Al Viro62a80672014-04-04 23:12:29 -04001434{
Al Viro3cf42da2022-06-17 14:45:41 -04001435 unsigned int n;
Al Viro1b17f1f2014-11-27 14:14:31 -05001436
1437 if (maxsize > i->count)
1438 maxsize = i->count;
Al Viro3d671ca2021-04-25 09:14:44 -04001439 if (!maxsize)
1440 return 0;
Al Viro7392ed12022-06-11 16:44:21 -04001441 if (maxsize > MAX_RW_COUNT)
1442 maxsize = MAX_RW_COUNT;
Al Viro1b17f1f2014-11-27 14:14:31 -05001443
Al Virofcb14cb2022-05-22 14:59:25 -04001444 if (likely(user_backed_iter(i))) {
Andreas Gruenbacher3337ab02021-07-12 12:06:14 +02001445 unsigned int gup_flags = 0;
Al Viro3d671ca2021-04-25 09:14:44 -04001446 unsigned long addr;
Al Viro3cf42da2022-06-17 14:45:41 -04001447 int res;
David Howells9ea9ce02018-10-20 00:57:56 +01001448
Andreas Gruenbacher3337ab02021-07-12 12:06:14 +02001449 if (iov_iter_rw(i) != WRITE)
1450 gup_flags |= FOLL_WRITE;
1451 if (i->nofault)
1452 gup_flags |= FOLL_NOFAULT;
1453
Al Virodd45ab92022-06-17 16:07:49 -04001454 addr = first_iovec_segment(i, &maxsize);
1455 *start = addr % PAGE_SIZE;
1456 addr &= PAGE_MASK;
Al Viro3cf42da2022-06-17 14:45:41 -04001457 n = want_pages_array(pages, maxsize, *start, maxpages);
1458 if (!n)
1459 return -ENOMEM;
Al Viro451c0ba2022-06-17 13:54:15 -04001460 res = get_user_pages_fast(addr, n, gup_flags, *pages);
Al Viro91329552022-06-10 20:38:20 -04001461 if (unlikely(res <= 0))
Al Viro1b17f1f2014-11-27 14:14:31 -05001462 return res;
Al Viroeba2d3d2022-06-10 13:05:12 -04001463 maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start);
1464 iov_iter_advance(i, maxsize);
1465 return maxsize;
Al Viro3d671ca2021-04-25 09:14:44 -04001466 }
1467 if (iov_iter_is_bvec(i)) {
Al Viro451c0ba2022-06-17 13:54:15 -04001468 struct page **p;
Al Viro3d671ca2021-04-25 09:14:44 -04001469 struct page *page;
1470
Al Viro59dbd7d2022-06-21 16:10:37 -04001471 page = first_bvec_segment(i, &maxsize, start);
Al Viro3cf42da2022-06-17 14:45:41 -04001472 n = want_pages_array(pages, maxsize, *start, maxpages);
1473 if (!n)
1474 return -ENOMEM;
Al Viro451c0ba2022-06-17 13:54:15 -04001475 p = *pages;
Al Virodda8e5d2022-06-21 15:55:19 -04001476 for (int k = 0; k < n; k++)
Al Viroeba2d3d2022-06-10 13:05:12 -04001477 get_page(p[k] = page + k);
1478 maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start);
Al Viro310d9d52022-06-11 04:04:33 -04001479 i->count -= maxsize;
1480 i->iov_offset += maxsize;
1481 if (i->iov_offset == i->bvec->bv_len) {
1482 i->iov_offset = 0;
1483 i->bvec++;
1484 i->nr_segs--;
1485 }
Al Viroeba2d3d2022-06-10 13:05:12 -04001486 return maxsize;
Al Viro3d671ca2021-04-25 09:14:44 -04001487 }
1488 if (iov_iter_is_pipe(i))
Al Viro451c0ba2022-06-17 13:54:15 -04001489 return pipe_get_pages(i, pages, maxsize, maxpages, start);
Al Viro3d671ca2021-04-25 09:14:44 -04001490 if (iov_iter_is_xarray(i))
Al Viro451c0ba2022-06-17 13:54:15 -04001491 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);
Al Viro3d671ca2021-04-25 09:14:44 -04001492 return -EFAULT;
Al Viro62a80672014-04-04 23:12:29 -04001493}
Al Viro91329552022-06-10 20:38:20 -04001494
Al Viroeba2d3d2022-06-10 13:05:12 -04001495ssize_t iov_iter_get_pages2(struct iov_iter *i,
Al Viro451c0ba2022-06-17 13:54:15 -04001496 struct page **pages, size_t maxsize, unsigned maxpages,
1497 size_t *start)
1498{
1499 if (!maxpages)
1500 return 0;
1501 BUG_ON(!pages);
1502
1503 return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start);
1504}
Al Viroeba2d3d2022-06-10 13:05:12 -04001505EXPORT_SYMBOL(iov_iter_get_pages2);
Al Viro451c0ba2022-06-17 13:54:15 -04001506
Al Viroeba2d3d2022-06-10 13:05:12 -04001507ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i,
Al Viro91329552022-06-10 20:38:20 -04001508 struct page ***pages, size_t maxsize,
1509 size_t *start)
1510{
1511 ssize_t len;
1512
1513 *pages = NULL;
1514
Al Viro451c0ba2022-06-17 13:54:15 -04001515 len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start);
Al Viro91329552022-06-10 20:38:20 -04001516 if (len <= 0) {
1517 kvfree(*pages);
1518 *pages = NULL;
1519 }
1520 return len;
1521}
Al Viroeba2d3d2022-06-10 13:05:12 -04001522EXPORT_SYMBOL(iov_iter_get_pages_alloc2);
Al Viro62a80672014-04-04 23:12:29 -04001523
Al Viroa604ec72014-11-24 01:08:00 -05001524size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1525 struct iov_iter *i)
1526{
Al Viroa604ec72014-11-24 01:08:00 -05001527 __wsum sum, next;
Al Viroa604ec72014-11-24 01:08:00 -05001528 sum = *csum;
David Howells9ea9ce02018-10-20 00:57:56 +01001529 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
Al Viro241699c2016-09-22 16:33:12 -04001530 WARN_ON(1);
1531 return 0;
1532 }
Al Viro7baa5092021-05-02 11:35:03 -04001533 iterate_and_advance(i, bytes, base, len, off, ({
1534 next = csum_and_copy_from_user(base, addr + off, len);
Al Viro2495bdcc2021-04-30 13:40:48 -04001535 sum = csum_block_add(sum, next, off);
Al Viro7baa5092021-05-02 11:35:03 -04001536 next ? 0 : len;
Al Viroa604ec72014-11-24 01:08:00 -05001537 }), ({
Al Viro7baa5092021-05-02 11:35:03 -04001538 sum = csum_and_memcpy(addr + off, base, len, sum, off);
Al Viroa604ec72014-11-24 01:08:00 -05001539 })
1540 )
1541 *csum = sum;
1542 return bytes;
1543}
1544EXPORT_SYMBOL(csum_and_copy_from_iter);
1545
Willem de Bruijn52cbd232021-02-03 14:29:52 -05001546size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
Al Viroa604ec72014-11-24 01:08:00 -05001547 struct iov_iter *i)
1548{
Willem de Bruijn52cbd232021-02-03 14:29:52 -05001549 struct csum_state *csstate = _csstate;
Al Viroa604ec72014-11-24 01:08:00 -05001550 __wsum sum, next;
Al Viro78e1f382018-11-25 16:24:16 -05001551
Al Viro78e1f382018-11-25 16:24:16 -05001552 if (unlikely(iov_iter_is_discard(i))) {
Al Viro241699c2016-09-22 16:33:12 -04001553 WARN_ON(1); /* for now */
1554 return 0;
1555 }
Al Viro6852df12021-05-02 17:24:40 -04001556
1557 sum = csum_shift(csstate->csum, csstate->off);
1558 if (unlikely(iov_iter_is_pipe(i)))
1559 bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum);
1560 else iterate_and_advance(i, bytes, base, len, off, ({
Al Viro7baa5092021-05-02 11:35:03 -04001561 next = csum_and_copy_to_user(addr + off, base, len);
Al Viro2495bdcc2021-04-30 13:40:48 -04001562 sum = csum_block_add(sum, next, off);
Al Viro7baa5092021-05-02 11:35:03 -04001563 next ? 0 : len;
Al Viroa604ec72014-11-24 01:08:00 -05001564 }), ({
Al Viro7baa5092021-05-02 11:35:03 -04001565 sum = csum_and_memcpy(base, addr + off, len, sum, off);
Al Viroa604ec72014-11-24 01:08:00 -05001566 })
1567 )
Al Viro594e4502021-06-05 10:19:30 -04001568 csstate->csum = csum_shift(sum, csstate->off);
1569 csstate->off += bytes;
Al Viroa604ec72014-11-24 01:08:00 -05001570 return bytes;
1571}
1572EXPORT_SYMBOL(csum_and_copy_to_iter);
1573
Sagi Grimbergd05f4432018-12-03 17:52:09 -08001574size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1575 struct iov_iter *i)
1576{
Herbert Xu79990962020-06-12 16:57:37 +10001577#ifdef CONFIG_CRYPTO_HASH
Sagi Grimbergd05f4432018-12-03 17:52:09 -08001578 struct ahash_request *hash = hashp;
1579 struct scatterlist sg;
1580 size_t copied;
1581
1582 copied = copy_to_iter(addr, bytes, i);
1583 sg_init_one(&sg, addr, copied);
1584 ahash_request_set_crypt(hash, &sg, NULL, copied);
1585 crypto_ahash_update(hash);
1586 return copied;
YueHaibing27fad742019-04-04 10:31:14 +08001587#else
1588 return 0;
1589#endif
Sagi Grimbergd05f4432018-12-03 17:52:09 -08001590}
1591EXPORT_SYMBOL(hash_and_copy_to_iter);
1592
Al Viro66531c62021-04-25 16:00:48 -04001593static int iov_npages(const struct iov_iter *i, int maxpages)
Al Viro62a80672014-04-04 23:12:29 -04001594{
Al Viro66531c62021-04-25 16:00:48 -04001595 size_t skip = i->iov_offset, size = i->count;
1596 const struct iovec *p;
Al Viroe0f2dc42014-11-27 14:09:46 -05001597 int npages = 0;
1598
Al Viro66531c62021-04-25 16:00:48 -04001599 for (p = i->iov; size; skip = 0, p++) {
1600 unsigned offs = offset_in_page(p->iov_base + skip);
1601 size_t len = min(p->iov_len - skip, size);
Al Viroe0f2dc42014-11-27 14:09:46 -05001602
Al Viro66531c62021-04-25 16:00:48 -04001603 if (len) {
1604 size -= len;
1605 npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
1606 if (unlikely(npages > maxpages))
1607 return maxpages;
1608 }
1609 }
1610 return npages;
1611}
1612
1613static int bvec_npages(const struct iov_iter *i, int maxpages)
1614{
1615 size_t skip = i->iov_offset, size = i->count;
1616 const struct bio_vec *p;
1617 int npages = 0;
1618
1619 for (p = i->bvec; size; skip = 0, p++) {
1620 unsigned offs = (p->bv_offset + skip) % PAGE_SIZE;
1621 size_t len = min(p->bv_len - skip, size);
1622
1623 size -= len;
1624 npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
1625 if (unlikely(npages > maxpages))
1626 return maxpages;
1627 }
1628 return npages;
1629}
1630
1631int iov_iter_npages(const struct iov_iter *i, int maxpages)
1632{
1633 if (unlikely(!i->count))
1634 return 0;
Al Virofcb14cb2022-05-22 14:59:25 -04001635 if (likely(iter_is_ubuf(i))) {
1636 unsigned offs = offset_in_page(i->ubuf + i->iov_offset);
1637 int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE);
1638 return min(npages, maxpages);
1639 }
Al Viro66531c62021-04-25 16:00:48 -04001640 /* iovec and kvec have identical layouts */
1641 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
1642 return iov_npages(i, maxpages);
1643 if (iov_iter_is_bvec(i))
1644 return bvec_npages(i, maxpages);
1645 if (iov_iter_is_pipe(i)) {
Al Viro66531c62021-04-25 16:00:48 -04001646 int npages;
Al Viro241699c2016-09-22 16:33:12 -04001647
1648 if (!sanity(i))
1649 return 0;
1650
Al Viro12d426a2022-06-15 09:44:38 -04001651 pipe_npages(i, &npages);
Al Viro66531c62021-04-25 16:00:48 -04001652 return min(npages, maxpages);
1653 }
1654 if (iov_iter_is_xarray(i)) {
Al Viroe4f8df82021-05-03 11:05:29 -04001655 unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE;
1656 int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);
Al Viro66531c62021-04-25 16:00:48 -04001657 return min(npages, maxpages);
1658 }
1659 return 0;
Al Viro62a80672014-04-04 23:12:29 -04001660}
Al Virof67da302014-03-19 01:16:16 -04001661EXPORT_SYMBOL(iov_iter_npages);
Al Viro4b8164b2015-01-31 20:08:47 -05001662
1663const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1664{
1665 *new = *old;
David Howells00e23702018-10-22 13:07:28 +01001666 if (unlikely(iov_iter_is_pipe(new))) {
Al Viro241699c2016-09-22 16:33:12 -04001667 WARN_ON(1);
1668 return NULL;
1669 }
David Howells00e23702018-10-22 13:07:28 +01001670 if (iov_iter_is_bvec(new))
Al Viro4b8164b2015-01-31 20:08:47 -05001671 return new->bvec = kmemdup(new->bvec,
1672 new->nr_segs * sizeof(struct bio_vec),
1673 flags);
Al Virofcb14cb2022-05-22 14:59:25 -04001674 else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
Al Viro4b8164b2015-01-31 20:08:47 -05001675 /* iovec and kvec have identical layout */
1676 return new->iov = kmemdup(new->iov,
1677 new->nr_segs * sizeof(struct iovec),
1678 flags);
Al Virofcb14cb2022-05-22 14:59:25 -04001679 return NULL;
Al Viro4b8164b2015-01-31 20:08:47 -05001680}
1681EXPORT_SYMBOL(dup_iter);
Al Virobc917be2015-03-21 17:45:43 -04001682
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001683static int copy_compat_iovec_from_user(struct iovec *iov,
1684 const struct iovec __user *uvec, unsigned long nr_segs)
1685{
1686 const struct compat_iovec __user *uiov =
1687 (const struct compat_iovec __user *)uvec;
1688 int ret = -EFAULT, i;
1689
Christoph Hellwiga959a972021-01-11 18:19:26 +01001690 if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001691 return -EFAULT;
1692
1693 for (i = 0; i < nr_segs; i++) {
1694 compat_uptr_t buf;
1695 compat_ssize_t len;
1696
1697 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
1698 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
1699
1700 /* check for compat_size_t not fitting in compat_ssize_t .. */
1701 if (len < 0) {
1702 ret = -EINVAL;
1703 goto uaccess_end;
1704 }
1705 iov[i].iov_base = compat_ptr(buf);
1706 iov[i].iov_len = len;
1707 }
1708
1709 ret = 0;
1710uaccess_end:
1711 user_access_end();
1712 return ret;
1713}
1714
1715static int copy_iovec_from_user(struct iovec *iov,
1716 const struct iovec __user *uvec, unsigned long nr_segs)
David Laightfb041b52020-09-25 06:51:39 +02001717{
1718 unsigned long seg;
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001719
1720 if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))
1721 return -EFAULT;
1722 for (seg = 0; seg < nr_segs; seg++) {
1723 if ((ssize_t)iov[seg].iov_len < 0)
1724 return -EINVAL;
1725 }
1726
1727 return 0;
1728}
1729
1730struct iovec *iovec_from_user(const struct iovec __user *uvec,
1731 unsigned long nr_segs, unsigned long fast_segs,
1732 struct iovec *fast_iov, bool compat)
1733{
1734 struct iovec *iov = fast_iov;
1735 int ret;
David Laightfb041b52020-09-25 06:51:39 +02001736
1737 /*
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001738 * SuS says "The readv() function *may* fail if the iovcnt argument was
1739 * less than or equal to 0, or greater than {IOV_MAX}. Linux has
David Laightfb041b52020-09-25 06:51:39 +02001740 * traditionally returned zero for zero segments, so...
1741 */
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001742 if (nr_segs == 0)
1743 return iov;
1744 if (nr_segs > UIO_MAXIOV)
1745 return ERR_PTR(-EINVAL);
David Laightfb041b52020-09-25 06:51:39 +02001746 if (nr_segs > fast_segs) {
1747 iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001748 if (!iov)
1749 return ERR_PTR(-ENOMEM);
David Laightfb041b52020-09-25 06:51:39 +02001750 }
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001751
1752 if (compat)
1753 ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
1754 else
1755 ret = copy_iovec_from_user(iov, uvec, nr_segs);
1756 if (ret) {
1757 if (iov != fast_iov)
1758 kfree(iov);
1759 return ERR_PTR(ret);
1760 }
1761
1762 return iov;
1763}
1764
1765ssize_t __import_iovec(int type, const struct iovec __user *uvec,
1766 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
1767 struct iov_iter *i, bool compat)
1768{
1769 ssize_t total_len = 0;
1770 unsigned long seg;
1771 struct iovec *iov;
1772
1773 iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
1774 if (IS_ERR(iov)) {
1775 *iovp = NULL;
1776 return PTR_ERR(iov);
David Laightfb041b52020-09-25 06:51:39 +02001777 }
1778
1779 /*
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001780 * According to the Single Unix Specification we should return EINVAL if
1781 * an element length is < 0 when cast to ssize_t or if the total length
1782 * would overflow the ssize_t return value of the system call.
David Laightfb041b52020-09-25 06:51:39 +02001783 *
1784 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
1785 * overflow case.
1786 */
David Laightfb041b52020-09-25 06:51:39 +02001787 for (seg = 0; seg < nr_segs; seg++) {
David Laightfb041b52020-09-25 06:51:39 +02001788 ssize_t len = (ssize_t)iov[seg].iov_len;
1789
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001790 if (!access_ok(iov[seg].iov_base, len)) {
1791 if (iov != *iovp)
1792 kfree(iov);
1793 *iovp = NULL;
1794 return -EFAULT;
David Laightfb041b52020-09-25 06:51:39 +02001795 }
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001796
1797 if (len > MAX_RW_COUNT - total_len) {
1798 len = MAX_RW_COUNT - total_len;
David Laightfb041b52020-09-25 06:51:39 +02001799 iov[seg].iov_len = len;
1800 }
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001801 total_len += len;
David Laightfb041b52020-09-25 06:51:39 +02001802 }
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001803
1804 iov_iter_init(i, type, iov, nr_segs, total_len);
1805 if (iov == *iovp)
1806 *iovp = NULL;
1807 else
1808 *iovp = iov;
1809 return total_len;
David Laightfb041b52020-09-25 06:51:39 +02001810}
1811
Vegard Nossumffecee42016-10-08 11:18:07 +02001812/**
1813 * import_iovec() - Copy an array of &struct iovec from userspace
1814 * into the kernel, check that it is valid, and initialize a new
1815 * &struct iov_iter iterator to access it.
1816 *
1817 * @type: One of %READ or %WRITE.
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001818 * @uvec: Pointer to the userspace array.
Vegard Nossumffecee42016-10-08 11:18:07 +02001819 * @nr_segs: Number of elements in userspace array.
1820 * @fast_segs: Number of elements in @iov.
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001821 * @iovp: (input and output parameter) Pointer to pointer to (usually small
Vegard Nossumffecee42016-10-08 11:18:07 +02001822 * on-stack) kernel array.
1823 * @i: Pointer to iterator that will be initialized on success.
1824 *
1825 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1826 * then this function places %NULL in *@iov on return. Otherwise, a new
1827 * array will be allocated and the result placed in *@iov. This means that
1828 * the caller may call kfree() on *@iov regardless of whether the small
1829 * on-stack array was used or not (and regardless of whether this function
1830 * returns an error or not).
1831 *
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001832 * Return: Negative error code on error, bytes imported on success
Vegard Nossumffecee42016-10-08 11:18:07 +02001833 */
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001834ssize_t import_iovec(int type, const struct iovec __user *uvec,
Al Virobc917be2015-03-21 17:45:43 -04001835 unsigned nr_segs, unsigned fast_segs,
Christoph Hellwigbfdc5972020-09-25 06:51:40 +02001836 struct iovec **iovp, struct iov_iter *i)
Al Virobc917be2015-03-21 17:45:43 -04001837{
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02001838 return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
1839 in_compat_syscall());
Al Virobc917be2015-03-21 17:45:43 -04001840}
1841EXPORT_SYMBOL(import_iovec);
1842
Al Virobc917be2015-03-21 17:45:43 -04001843int import_single_range(int rw, void __user *buf, size_t len,
1844 struct iovec *iov, struct iov_iter *i)
1845{
1846 if (len > MAX_RW_COUNT)
1847 len = MAX_RW_COUNT;
Linus Torvalds96d4f262019-01-03 18:57:57 -08001848 if (unlikely(!access_ok(buf, len)))
Al Virobc917be2015-03-21 17:45:43 -04001849 return -EFAULT;
1850
1851 iov->iov_base = buf;
1852 iov->iov_len = len;
1853 iov_iter_init(i, rw, iov, 1, len);
1854 return 0;
1855}
Al Viroe1267582015-12-06 20:38:56 -05001856EXPORT_SYMBOL(import_single_range);
Jens Axboe8fb0f472021-09-10 11:18:36 -06001857
1858/**
1859 * iov_iter_restore() - Restore a &struct iov_iter to the same state as when
1860 * iov_iter_save_state() was called.
1861 *
1862 * @i: &struct iov_iter to restore
1863 * @state: state to restore from
1864 *
1865 * Used after iov_iter_save_state() to bring restore @i, if operations may
1866 * have advanced it.
1867 *
1868 * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC
1869 */
1870void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
1871{
1872 if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) &&
Al Virofcb14cb2022-05-22 14:59:25 -04001873 !iov_iter_is_kvec(i) && !iter_is_ubuf(i))
Jens Axboe8fb0f472021-09-10 11:18:36 -06001874 return;
1875 i->iov_offset = state->iov_offset;
1876 i->count = state->count;
Al Virofcb14cb2022-05-22 14:59:25 -04001877 if (iter_is_ubuf(i))
1878 return;
Jens Axboe8fb0f472021-09-10 11:18:36 -06001879 /*
1880 * For the *vec iters, nr_segs + iov is constant - if we increment
1881 * the vec, then we also decrement the nr_segs count. Hence we don't
1882 * need to track both of these, just one is enough and we can deduct
1883 * the other from that. ITER_KVEC and ITER_IOVEC are the same struct
1884 * size, so we can just increment the iov pointer as they are unionzed.
1885 * ITER_BVEC _may_ be the same size on some archs, but on others it is
1886 * not. Be safe and handle it separately.
1887 */
1888 BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
1889 if (iov_iter_is_bvec(i))
1890 i->bvec -= state->nr_segs - i->nr_segs;
1891 else
1892 i->iov -= state->nr_segs - i->nr_segs;
1893 i->nr_segs = state->nr_segs;
1894}