blob: 42c7ff41c2dba2904cf6c8f2d10ea6984a402823 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/fs/pipe.c
4 *
5 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
6 */
7
8#include <linux/mm.h>
9#include <linux/file.h>
10#include <linux/poll.h>
11#include <linux/slab.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/fs.h>
Jens Axboe35f3d142010-05-20 10:43:18 +020015#include <linux/log2.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/mount.h>
David Howells4fa7ec52019-03-25 16:38:23 +000017#include <linux/pseudo_fs.h>
Muthu Kumarb502bd12012-03-23 15:01:50 -070018#include <linux/magic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/pipe_fs_i.h>
20#include <linux/uio.h>
21#include <linux/highmem.h>
Jens Axboe5274f052006-03-30 15:15:30 +020022#include <linux/pagemap.h>
Al Virodb349502007-02-07 01:48:00 -050023#include <linux/audit.h>
Ulrich Drepperba719ba2008-05-06 20:42:38 -070024#include <linux/syscalls.h>
Jens Axboeb492e952010-05-19 21:03:16 +020025#include <linux/fcntl.h>
Vladimir Davydovd86133b2016-07-26 15:24:33 -070026#include <linux/memcontrol.h>
David Howellsc73be612020-01-14 17:07:11 +000027#include <linux/watch_queue.h>
Luis Chamberlain1998f192022-01-21 22:13:20 -080028#include <linux/sysctl.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080030#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#include <asm/ioctls.h>
32
Al Viro599a0ac2013-03-12 09:58:10 -040033#include "internal.h"
34
Linus Torvalds1da177e2005-04-16 15:20:36 -070035/*
Alex Xu (Hello71)46c4c9d2021-08-05 10:40:47 -040036 * New pipe buffers will be restricted to this size while the user is exceeding
37 * their pipe buffer quota. The general pipe use case needs at least two
38 * buffers: one for data yet to be read, and one for new data. If this is less
39 * than two, then a write to a non-empty pipe may block even if the pipe is not
40 * full. This can occur with GNU make jobserver or similar uses of pipes as
41 * semaphores: multiple processes may be waiting to write tokens back to the
42 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
43 *
44 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
45 * own risk, namely: pipe writes to non-full pipes may block until the pipe is
46 * emptied.
47 */
48#define PIPE_MIN_DEF_BUFFERS 2
49
50/*
Jens Axboeb492e952010-05-19 21:03:16 +020051 * The max size that a non-root user is allowed to grow the pipe. Can
Jens Axboeff9da692010-06-03 14:54:39 +020052 * be set by root in /proc/sys/fs/pipe-max-size
Jens Axboeb492e952010-05-19 21:03:16 +020053 */
Luis Chamberlain1998f192022-01-21 22:13:20 -080054static unsigned int pipe_max_size = 1048576;
Jens Axboeff9da692010-06-03 14:54:39 +020055
Willy Tarreau759c0112016-01-18 16:36:09 +010056/* Maximum allocatable pages per user. Hard limit is unset by default, soft
57 * matches default values.
58 */
Luis Chamberlain1998f192022-01-21 22:13:20 -080059static unsigned long pipe_user_pages_hard;
60static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
Willy Tarreau759c0112016-01-18 16:36:09 +010061
Jens Axboeb492e952010-05-19 21:03:16 +020062/*
David Howells8cefc102019-11-15 13:30:32 +000063 * We use head and tail indices that aren't masked off, except at the point of
64 * dereference, but rather they're allowed to wrap naturally. This means there
65 * isn't a dead spot in the buffer, but the ring has to be a power of two and
66 * <= 2^31.
67 * -- David Howells 2019-09-23.
68 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 * Reads with count = 0 should always return 0.
70 * -- Julian Bradfield 1999-06-07.
71 *
72 * FIFOs and Pipes now generate SIGIO for both readers and writers.
73 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
74 *
75 * pipe_read & write cleanup
76 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
77 */
78
Miklos Szeredi61e0d472009-04-14 19:48:41 +020079static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
80{
Al Viro6447a3c2013-03-21 11:01:38 -040081 if (pipe->files)
Al Viro72b0d9a2013-03-21 02:32:24 -040082 mutex_lock_nested(&pipe->mutex, subclass);
Miklos Szeredi61e0d472009-04-14 19:48:41 +020083}
84
85void pipe_lock(struct pipe_inode_info *pipe)
86{
87 /*
88 * pipe_lock() nests non-pipe inode locks (for writing to a file)
89 */
90 pipe_lock_nested(pipe, I_MUTEX_PARENT);
91}
92EXPORT_SYMBOL(pipe_lock);
93
94void pipe_unlock(struct pipe_inode_info *pipe)
95{
Al Viro6447a3c2013-03-21 11:01:38 -040096 if (pipe->files)
Al Viro72b0d9a2013-03-21 02:32:24 -040097 mutex_unlock(&pipe->mutex);
Miklos Szeredi61e0d472009-04-14 19:48:41 +020098}
99EXPORT_SYMBOL(pipe_unlock);
100
Al Viroebec73f2013-03-21 12:24:01 -0400101static inline void __pipe_lock(struct pipe_inode_info *pipe)
102{
103 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
104}
105
106static inline void __pipe_unlock(struct pipe_inode_info *pipe)
107{
108 mutex_unlock(&pipe->mutex);
109}
110
Miklos Szeredi61e0d472009-04-14 19:48:41 +0200111void pipe_double_lock(struct pipe_inode_info *pipe1,
112 struct pipe_inode_info *pipe2)
113{
114 BUG_ON(pipe1 == pipe2);
115
116 if (pipe1 < pipe2) {
117 pipe_lock_nested(pipe1, I_MUTEX_PARENT);
118 pipe_lock_nested(pipe2, I_MUTEX_CHILD);
119 } else {
Peter Zijlstra023d43c2009-07-21 10:09:23 +0200120 pipe_lock_nested(pipe2, I_MUTEX_PARENT);
121 pipe_lock_nested(pipe1, I_MUTEX_CHILD);
Miklos Szeredi61e0d472009-04-14 19:48:41 +0200122 }
123}
124
Ingo Molnar341b4462006-04-11 13:57:45 +0200125static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
126 struct pipe_buffer *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127{
128 struct page *page = buf->page;
129
Jens Axboe5274f052006-03-30 15:15:30 +0200130 /*
131 * If nobody else uses this page, and we don't already have a
132 * temporary page, let's keep track of it as a one-deep
Ingo Molnar341b4462006-04-11 13:57:45 +0200133 * allocation cache. (Otherwise just release our reference to it)
Jens Axboe5274f052006-03-30 15:15:30 +0200134 */
Ingo Molnar341b4462006-04-11 13:57:45 +0200135 if (page_count(page) == 1 && !pipe->tmp_page)
Ingo Molnar923f4f22006-04-11 13:53:33 +0200136 pipe->tmp_page = page;
Ingo Molnar341b4462006-04-11 13:57:45 +0200137 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300138 put_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139}
140
Christoph Hellwigc928f642020-05-20 17:58:16 +0200141static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
142 struct pipe_buffer *buf)
Vladimir Davydovd86133b2016-07-26 15:24:33 -0700143{
144 struct page *page = buf->page;
145
Christoph Hellwigc928f642020-05-20 17:58:16 +0200146 if (page_count(page) != 1)
147 return false;
148 memcg_kmem_uncharge_page(page, 0);
149 __SetPageLocked(page);
150 return true;
Vladimir Davydovd86133b2016-07-26 15:24:33 -0700151}
152
Jens Axboe08457182007-06-12 20:51:32 +0200153/**
Christoph Hellwigc928f642020-05-20 17:58:16 +0200154 * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
Jens Axboe08457182007-06-12 20:51:32 +0200155 * @pipe: the pipe that the buffer belongs to
156 * @buf: the buffer to attempt to steal
157 *
158 * Description:
Randy Dunlapb51d63c2008-02-13 15:03:22 -0800159 * This function attempts to steal the &struct page attached to
Jens Axboe08457182007-06-12 20:51:32 +0200160 * @buf. If successful, this function returns 0 and returns with
161 * the page locked. The caller may then reuse the page for whatever
Randy Dunlapb51d63c2008-02-13 15:03:22 -0800162 * he wishes; the typical use is insertion into a different file
Jens Axboe08457182007-06-12 20:51:32 +0200163 * page cache.
164 */
Christoph Hellwigc928f642020-05-20 17:58:16 +0200165bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
166 struct pipe_buffer *buf)
Jens Axboe5abc97a2006-03-30 15:16:46 +0200167{
Jens Axboe46e678c2006-04-30 16:36:32 +0200168 struct page *page = buf->page;
169
Jens Axboe08457182007-06-12 20:51:32 +0200170 /*
171 * A reference of one is golden, that means that the owner of this
172 * page is the only one holding a reference to it. lock the page
173 * and return OK.
174 */
Jens Axboe46e678c2006-04-30 16:36:32 +0200175 if (page_count(page) == 1) {
Jens Axboe46e678c2006-04-30 16:36:32 +0200176 lock_page(page);
Christoph Hellwigc928f642020-05-20 17:58:16 +0200177 return true;
Jens Axboe46e678c2006-04-30 16:36:32 +0200178 }
Christoph Hellwigc928f642020-05-20 17:58:16 +0200179 return false;
Jens Axboe5abc97a2006-03-30 15:16:46 +0200180}
Christoph Hellwigc928f642020-05-20 17:58:16 +0200181EXPORT_SYMBOL(generic_pipe_buf_try_steal);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200182
Jens Axboe08457182007-06-12 20:51:32 +0200183/**
Randy Dunlapb51d63c2008-02-13 15:03:22 -0800184 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
Jens Axboe08457182007-06-12 20:51:32 +0200185 * @pipe: the pipe that the buffer belongs to
186 * @buf: the buffer to get a reference to
187 *
188 * Description:
189 * This function grabs an extra reference to @buf. It's used in
Randy Dunlap3d742d42021-02-24 12:00:48 -0800190 * the tee() system call, when we duplicate the buffers in one
Jens Axboe08457182007-06-12 20:51:32 +0200191 * pipe into another.
192 */
Matthew Wilcox15fab632019-04-05 14:02:10 -0700193bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
Jens Axboe70524492006-04-11 15:51:17 +0200194{
Linus Torvaldscd1adf12021-09-07 11:03:45 -0700195 return try_get_page(buf->page);
Jens Axboe70524492006-04-11 15:51:17 +0200196}
Miklos Szeredi51921cb2010-05-26 08:44:22 +0200197EXPORT_SYMBOL(generic_pipe_buf_get);
Jens Axboe70524492006-04-11 15:51:17 +0200198
Jens Axboe08457182007-06-12 20:51:32 +0200199/**
Miklos Szeredi68181732009-05-07 15:37:36 +0200200 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
201 * @pipe: the pipe that the buffer belongs to
202 * @buf: the buffer to put a reference to
203 *
204 * Description:
205 * This function releases a reference to @buf.
206 */
207void generic_pipe_buf_release(struct pipe_inode_info *pipe,
208 struct pipe_buffer *buf)
209{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300210 put_page(buf->page);
Miklos Szeredi68181732009-05-07 15:37:36 +0200211}
Miklos Szeredi51921cb2010-05-26 08:44:22 +0200212EXPORT_SYMBOL(generic_pipe_buf_release);
Miklos Szeredi68181732009-05-07 15:37:36 +0200213
Eric Dumazetd4c3cca2006-12-13 00:34:04 -0800214static const struct pipe_buf_operations anon_pipe_buf_ops = {
Christoph Hellwigc928f642020-05-20 17:58:16 +0200215 .release = anon_pipe_buf_release,
216 .try_steal = anon_pipe_buf_try_steal,
217 .get = generic_pipe_buf_get,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218};
219
Linus Torvalds85190d12019-12-07 13:53:09 -0800220/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
221static inline bool pipe_readable(const struct pipe_inode_info *pipe)
222{
223 unsigned int head = READ_ONCE(pipe->head);
224 unsigned int tail = READ_ONCE(pipe->tail);
225 unsigned int writers = READ_ONCE(pipe->writers);
226
227 return !pipe_empty(head, tail) || !writers;
228}
229
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230static ssize_t
Al Virofb9096a2014-04-02 19:56:54 -0400231pipe_read(struct kiocb *iocb, struct iov_iter *to)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232{
Al Virofb9096a2014-04-02 19:56:54 -0400233 size_t total_len = iov_iter_count(to);
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700234 struct file *filp = iocb->ki_filp;
Al Virode32ec42013-03-21 11:16:56 -0400235 struct pipe_inode_info *pipe = filp->private_data;
Linus Torvalds0ddad212019-12-09 09:48:27 -0800236 bool was_full, wake_next_reader = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 /* Null read succeeds. */
240 if (unlikely(total_len == 0))
241 return 0;
242
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 ret = 0;
Al Viroebec73f2013-03-21 12:24:01 -0400244 __pipe_lock(pipe);
Linus Torvaldsf467a6a2019-12-07 12:54:26 -0800245
246 /*
247 * We only wake up writers if the pipe was full when we started
248 * reading in order to avoid unnecessary wakeups.
249 *
250 * But when we do wake up writers, we do so using a sync wakeup
251 * (WF_SYNC), because we want them to get going and generate more
252 * data for us.
253 */
254 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 for (;;) {
David Howells2ed147f2022-03-11 13:24:36 +0000256 /* Read ->head with a barrier vs post_one_notification() */
257 unsigned int head = smp_load_acquire(&pipe->head);
David Howells8cefc102019-11-15 13:30:32 +0000258 unsigned int tail = pipe->tail;
259 unsigned int mask = pipe->ring_size - 1;
260
David Howellse7d553d2020-01-14 17:07:12 +0000261#ifdef CONFIG_WATCH_QUEUE
262 if (pipe->note_loss) {
263 struct watch_notification n;
264
265 if (total_len < 8) {
266 if (ret == 0)
267 ret = -ENOBUFS;
268 break;
269 }
270
271 n.type = WATCH_TYPE_META;
272 n.subtype = WATCH_META_LOSS_NOTIFICATION;
273 n.info = watch_sizeof(n);
274 if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
275 if (ret == 0)
276 ret = -EFAULT;
277 break;
278 }
279 ret += sizeof(n);
280 total_len -= sizeof(n);
281 pipe->note_loss = false;
282 }
283#endif
284
David Howells8cefc102019-11-15 13:30:32 +0000285 if (!pipe_empty(head, tail)) {
286 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 size_t chars = buf->len;
Al Viro637b58c2014-02-03 19:11:42 -0500288 size_t written;
289 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290
David Howells8cfba762020-01-14 17:07:11 +0000291 if (chars > total_len) {
292 if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
293 if (ret == 0)
294 ret = -ENOBUFS;
295 break;
296 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 chars = total_len;
David Howells8cfba762020-01-14 17:07:11 +0000298 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
Miklos Szeredifba597d2016-09-27 10:45:12 +0200300 error = pipe_buf_confirm(pipe, buf);
Jens Axboef84d7512006-05-01 19:59:03 +0200301 if (error) {
Jens Axboe5274f052006-03-30 15:15:30 +0200302 if (!ret)
Nicolas Kaisere5953cb2010-10-21 14:56:00 +0200303 ret = error;
Jens Axboe5274f052006-03-30 15:15:30 +0200304 break;
305 }
Jens Axboef84d7512006-05-01 19:59:03 +0200306
Al Virofb9096a2014-04-02 19:56:54 -0400307 written = copy_page_to_iter(buf->page, buf->offset, chars, to);
Al Viro637b58c2014-02-03 19:11:42 -0500308 if (unlikely(written < chars)) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200309 if (!ret)
Al Viro637b58c2014-02-03 19:11:42 -0500310 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 break;
312 }
313 ret += chars;
314 buf->offset += chars;
315 buf->len -= chars;
Linus Torvalds9883035a2012-04-29 13:12:42 -0700316
317 /* Was it a packet buffer? Clean up and exit */
318 if (buf->flags & PIPE_BUF_FLAG_PACKET) {
319 total_len = chars;
320 buf->len = 0;
321 }
322
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 if (!buf->len) {
Miklos Szeredia7796382016-09-27 10:45:12 +0200324 pipe_buf_release(pipe, buf);
Linus Torvalds0ddad212019-12-09 09:48:27 -0800325 spin_lock_irq(&pipe->rd_wait.lock);
David Howellse7d553d2020-01-14 17:07:12 +0000326#ifdef CONFIG_WATCH_QUEUE
327 if (buf->flags & PIPE_BUF_FLAG_LOSS)
328 pipe->note_loss = true;
329#endif
David Howells8cefc102019-11-15 13:30:32 +0000330 tail++;
331 pipe->tail = tail;
Linus Torvalds0ddad212019-12-09 09:48:27 -0800332 spin_unlock_irq(&pipe->rd_wait.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 }
334 total_len -= chars;
335 if (!total_len)
336 break; /* common path: read succeeded */
David Howells8cefc102019-11-15 13:30:32 +0000337 if (!pipe_empty(head, tail)) /* More to do? */
338 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 }
David Howells8cefc102019-11-15 13:30:32 +0000340
Ingo Molnar923f4f22006-04-11 13:53:33 +0200341 if (!pipe->writers)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 break;
Linus Torvaldsa28c8b92019-12-07 13:21:01 -0800343 if (ret)
344 break;
345 if (filp->f_flags & O_NONBLOCK) {
346 ret = -EAGAIN;
347 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 }
Linus Torvalds85190d12019-12-07 13:53:09 -0800349 __pipe_unlock(pipe);
Linus Torvaldsd1c6a2a2019-12-11 11:46:19 -0800350
351 /*
352 * We only get here if we didn't actually read anything.
353 *
354 * However, we could have seen (and removed) a zero-sized
355 * pipe buffer, and might have made space in the buffers
356 * that way.
357 *
358 * You can't make zero-sized pipe buffers by doing an empty
359 * write (not even in packet mode), but they can happen if
360 * the writer gets an EFAULT when trying to fill a buffer
361 * that already got allocated and inserted in the buffer
362 * array.
363 *
364 * So we still need to wake up any pending writers in the
365 * _very_ unlikely case that the pipe was full, but we got
366 * no data.
367 */
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700368 if (unlikely(was_full))
Linus Torvalds0ddad212019-12-09 09:48:27 -0800369 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700370 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvaldsd1c6a2a2019-12-11 11:46:19 -0800371
372 /*
373 * But because we didn't read anything, at this point we can
374 * just return directly with -ERESTARTSYS if we're interrupted,
375 * since we've done any required wakeups and there's no need
376 * to mark anything accessed. And we've dropped the lock.
377 */
Linus Torvalds0ddad212019-12-09 09:48:27 -0800378 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
Linus Torvaldsd1c6a2a2019-12-11 11:46:19 -0800379 return -ERESTARTSYS;
380
Linus Torvalds85190d12019-12-07 13:53:09 -0800381 __pipe_lock(pipe);
Linus Torvaldsf467a6a2019-12-07 12:54:26 -0800382 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
Linus Torvalds0ddad212019-12-09 09:48:27 -0800383 wake_next_reader = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 }
Linus Torvalds0ddad212019-12-09 09:48:27 -0800385 if (pipe_empty(pipe->head, pipe->tail))
386 wake_next_reader = false;
Al Viroebec73f2013-03-21 12:24:01 -0400387 __pipe_unlock(pipe);
Ingo Molnar341b4462006-04-11 13:57:45 +0200388
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700389 if (was_full)
Linus Torvalds0ddad212019-12-09 09:48:27 -0800390 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
Linus Torvalds0ddad212019-12-09 09:48:27 -0800391 if (wake_next_reader)
392 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700393 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 if (ret > 0)
395 file_accessed(filp);
396 return ret;
397}
398
Linus Torvalds9883035a2012-04-29 13:12:42 -0700399static inline int is_packetized(struct file *file)
400{
401 return (file->f_flags & O_DIRECT) != 0;
402}
403
Linus Torvalds85190d12019-12-07 13:53:09 -0800404/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
405static inline bool pipe_writable(const struct pipe_inode_info *pipe)
406{
407 unsigned int head = READ_ONCE(pipe->head);
408 unsigned int tail = READ_ONCE(pipe->tail);
409 unsigned int max_usage = READ_ONCE(pipe->max_usage);
410
411 return !pipe_full(head, tail, max_usage) ||
412 !READ_ONCE(pipe->readers);
413}
414
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415static ssize_t
Al Virof0d1bec2014-04-03 15:05:18 -0400416pipe_write(struct kiocb *iocb, struct iov_iter *from)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417{
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700418 struct file *filp = iocb->ki_filp;
Al Virode32ec42013-03-21 11:16:56 -0400419 struct pipe_inode_info *pipe = filp->private_data;
David Howells8f868d62019-12-05 22:30:37 +0000420 unsigned int head;
Al Virof0d1bec2014-04-03 15:05:18 -0400421 ssize_t ret = 0;
Al Virof0d1bec2014-04-03 15:05:18 -0400422 size_t total_len = iov_iter_count(from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 ssize_t chars;
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800424 bool was_empty = false;
Linus Torvalds0ddad212019-12-09 09:48:27 -0800425 bool wake_next_writer = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 /* Null write succeeds. */
428 if (unlikely(total_len == 0))
429 return 0;
430
Al Viroebec73f2013-03-21 12:24:01 -0400431 __pipe_lock(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432
Ingo Molnar923f4f22006-04-11 13:53:33 +0200433 if (!pipe->readers) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 send_sig(SIGPIPE, current, 0);
435 ret = -EPIPE;
436 goto out;
437 }
438
David Howellsc73be612020-01-14 17:07:11 +0000439#ifdef CONFIG_WATCH_QUEUE
440 if (pipe->watch_queue) {
441 ret = -EXDEV;
442 goto out;
443 }
444#endif
445
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800446 /*
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800447 * If it wasn't empty we try to merge new data into
448 * the last buffer.
449 *
450 * That naturally merges small writes, but it also
Linus Torvalds3a34b132021-07-30 15:42:34 -0700451 * page-aligns the rest of the writes for large writes
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800452 * spanning multiple pages.
453 */
David Howells8cefc102019-11-15 13:30:32 +0000454 head = pipe->head;
Linus Torvalds3b844822021-08-05 10:04:43 -0700455 was_empty = pipe_empty(head, pipe->tail);
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800456 chars = total_len & (PAGE_SIZE-1);
Linus Torvalds3b844822021-08-05 10:04:43 -0700457 if (chars && !was_empty) {
David Howells8f868d62019-12-05 22:30:37 +0000458 unsigned int mask = pipe->ring_size - 1;
David Howells8cefc102019-11-15 13:30:32 +0000459 struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 int offset = buf->offset + buf->len;
Ingo Molnar341b4462006-04-11 13:57:45 +0200461
Christoph Hellwigf6dd9752020-05-20 17:58:12 +0200462 if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
463 offset + chars <= PAGE_SIZE) {
Miklos Szeredifba597d2016-09-27 10:45:12 +0200464 ret = pipe_buf_confirm(pipe, buf);
Eric Biggers6ae08062015-10-17 16:26:09 -0500465 if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200466 goto out;
Jens Axboef84d7512006-05-01 19:59:03 +0200467
Al Virof0d1bec2014-04-03 15:05:18 -0400468 ret = copy_page_from_iter(buf->page, offset, chars, from);
469 if (unlikely(ret < chars)) {
Eric Biggers6ae08062015-10-17 16:26:09 -0500470 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 goto out;
Jens Axboef6762b72006-05-01 20:02:05 +0200472 }
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800473
Eric Biggers6ae08062015-10-17 16:26:09 -0500474 buf->len += ret;
Al Virof0d1bec2014-04-03 15:05:18 -0400475 if (!iov_iter_count(from))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 goto out;
477 }
478 }
479
480 for (;;) {
Ingo Molnar923f4f22006-04-11 13:53:33 +0200481 if (!pipe->readers) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482 send_sig(SIGPIPE, current, 0);
Ingo Molnar341b4462006-04-11 13:57:45 +0200483 if (!ret)
484 ret = -EPIPE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 break;
486 }
David Howells8cefc102019-11-15 13:30:32 +0000487
David Howellsa194dfe2019-09-20 16:32:19 +0100488 head = pipe->head;
David Howells8f868d62019-12-05 22:30:37 +0000489 if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
490 unsigned int mask = pipe->ring_size - 1;
David Howells8cefc102019-11-15 13:30:32 +0000491 struct pipe_buffer *buf = &pipe->bufs[head & mask];
Ingo Molnar923f4f22006-04-11 13:53:33 +0200492 struct page *page = pipe->tmp_page;
Al Virof0d1bec2014-04-03 15:05:18 -0400493 int copied;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494
495 if (!page) {
Vladimir Davydovd86133b2016-07-26 15:24:33 -0700496 page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 if (unlikely(!page)) {
498 ret = ret ? : -ENOMEM;
499 break;
500 }
Ingo Molnar923f4f22006-04-11 13:53:33 +0200501 pipe->tmp_page = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 }
David Howellsa194dfe2019-09-20 16:32:19 +0100503
504 /* Allocate a slot in the ring in advance and attach an
505 * empty buffer. If we fault or otherwise fail to use
506 * it, either the reader will consume it or it'll still
507 * be there for the next write.
508 */
Linus Torvalds0ddad212019-12-09 09:48:27 -0800509 spin_lock_irq(&pipe->rd_wait.lock);
David Howellsa194dfe2019-09-20 16:32:19 +0100510
511 head = pipe->head;
David Howells8f868d62019-12-05 22:30:37 +0000512 if (pipe_full(head, pipe->tail, pipe->max_usage)) {
Linus Torvalds0ddad212019-12-09 09:48:27 -0800513 spin_unlock_irq(&pipe->rd_wait.lock);
David Howells8df44122019-10-07 16:30:51 +0100514 continue;
515 }
516
David Howellsa194dfe2019-09-20 16:32:19 +0100517 pipe->head = head + 1;
Linus Torvalds0ddad212019-12-09 09:48:27 -0800518 spin_unlock_irq(&pipe->rd_wait.lock);
David Howellsa194dfe2019-09-20 16:32:19 +0100519
520 /* Insert it into the buffer array */
521 buf = &pipe->bufs[head & mask];
522 buf->page = page;
523 buf->ops = &anon_pipe_buf_ops;
524 buf->offset = 0;
525 buf->len = 0;
Christoph Hellwigf6dd9752020-05-20 17:58:12 +0200526 if (is_packetized(filp))
David Howellsa194dfe2019-09-20 16:32:19 +0100527 buf->flags = PIPE_BUF_FLAG_PACKET;
Christoph Hellwigf6dd9752020-05-20 17:58:12 +0200528 else
529 buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
David Howellsa194dfe2019-09-20 16:32:19 +0100530 pipe->tmp_page = NULL;
531
Al Virof0d1bec2014-04-03 15:05:18 -0400532 copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
533 if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200534 if (!ret)
Al Virof0d1bec2014-04-03 15:05:18 -0400535 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 break;
537 }
Al Virof0d1bec2014-04-03 15:05:18 -0400538 ret += copied;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 buf->offset = 0;
Al Virof0d1bec2014-04-03 15:05:18 -0400540 buf->len = copied;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541
Al Virof0d1bec2014-04-03 15:05:18 -0400542 if (!iov_iter_count(from))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 break;
544 }
David Howells8cefc102019-11-15 13:30:32 +0000545
David Howells8f868d62019-12-05 22:30:37 +0000546 if (!pipe_full(head, pipe->tail, pipe->max_usage))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 continue;
David Howells8cefc102019-11-15 13:30:32 +0000548
549 /* Wait for buffer space to become available. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 if (filp->f_flags & O_NONBLOCK) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200551 if (!ret)
552 ret = -EAGAIN;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 break;
554 }
555 if (signal_pending(current)) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200556 if (!ret)
557 ret = -ERESTARTSYS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558 break;
559 }
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800560
561 /*
562 * We're going to release the pipe lock and wait for more
563 * space. We wake up any readers if necessary, and then
564 * after waiting we need to re-check whether the pipe
565 * become empty while we dropped the lock.
566 */
Linus Torvalds85190d12019-12-07 13:53:09 -0800567 __pipe_unlock(pipe);
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700568 if (was_empty)
Linus Torvalds0ddad212019-12-09 09:48:27 -0800569 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700570 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Linus Torvalds0ddad212019-12-09 09:48:27 -0800571 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
Linus Torvalds85190d12019-12-07 13:53:09 -0800572 __pipe_lock(pipe);
Jan Stancek0dd1e372019-12-22 13:33:24 +0100573 was_empty = pipe_empty(pipe->head, pipe->tail);
Linus Torvalds0ddad212019-12-09 09:48:27 -0800574 wake_next_writer = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 }
576out:
Linus Torvalds0ddad212019-12-09 09:48:27 -0800577 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
578 wake_next_writer = false;
Al Viroebec73f2013-03-21 12:24:01 -0400579 __pipe_unlock(pipe);
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800580
581 /*
582 * If we do do a wakeup event, we do a 'sync' wakeup, because we
583 * want the reader to start processing things asap, rather than
584 * leave the data pending.
585 *
586 * This is particularly important for small writes, because of
587 * how (for example) the GNU make jobserver uses small writes to
588 * wake up pending jobs
Linus Torvalds3b844822021-08-05 10:04:43 -0700589 *
590 * Epoll nonsensically wants a wakeup whether the pipe
591 * was already empty or not.
Linus Torvalds1b6b26a2019-12-07 12:14:28 -0800592 */
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700593 if (was_empty || pipe->poll_usage)
Linus Torvalds0ddad212019-12-09 09:48:27 -0800594 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
Linus Torvaldsfe67f4d2021-08-24 10:39:25 -0700595 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Linus Torvalds0ddad212019-12-09 09:48:27 -0800596 if (wake_next_writer)
597 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
Dmitry Monakhov7e775f42014-01-23 15:55:21 -0800598 if (ret > 0 && sb_start_write_trylock(file_inode(filp)->i_sb)) {
Josef Bacikc3b2da32012-03-26 09:59:21 -0400599 int err = file_update_time(filp);
600 if (err)
601 ret = err;
Dmitry Monakhov7e775f42014-01-23 15:55:21 -0800602 sb_end_write(file_inode(filp)->i_sb);
Josef Bacikc3b2da32012-03-26 09:59:21 -0400603 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 return ret;
605}
606
Andi Kleend59d0b12008-02-08 04:21:23 -0800607static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608{
Al Virode32ec42013-03-21 11:16:56 -0400609 struct pipe_inode_info *pipe = filp->private_data;
Andrei Vaginaeb213c2022-03-23 16:06:20 -0700610 unsigned int count, head, tail, mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611
612 switch (cmd) {
David Howellsc73be612020-01-14 17:07:11 +0000613 case FIONREAD:
614 __pipe_lock(pipe);
615 count = 0;
616 head = pipe->head;
617 tail = pipe->tail;
618 mask = pipe->ring_size - 1;
David Howells8cefc102019-11-15 13:30:32 +0000619
David Howellsc73be612020-01-14 17:07:11 +0000620 while (tail != head) {
621 count += pipe->bufs[tail & mask].len;
622 tail++;
623 }
624 __pipe_unlock(pipe);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200625
David Howellsc73be612020-01-14 17:07:11 +0000626 return put_user(count, (int __user *)arg);
627
628#ifdef CONFIG_WATCH_QUEUE
629 case IOC_WATCH_QUEUE_SET_SIZE: {
630 int ret;
631 __pipe_lock(pipe);
632 ret = watch_queue_set_size(pipe, arg);
633 __pipe_unlock(pipe);
634 return ret;
635 }
636
637 case IOC_WATCH_QUEUE_SET_FILTER:
638 return watch_queue_set_filter(
639 pipe, (struct watch_notification_filter __user *)arg);
640#endif
641
642 default:
643 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 }
645}
646
Christoph Hellwigdd670812017-12-31 16:42:12 +0100647/* No kernel lock held - fine */
Linus Torvaldsa11e1d42018-06-28 09:43:44 -0700648static __poll_t
649pipe_poll(struct file *filp, poll_table *wait)
Christoph Hellwigdd670812017-12-31 16:42:12 +0100650{
Linus Torvaldsa11e1d42018-06-28 09:43:44 -0700651 __poll_t mask;
Christoph Hellwigdd670812017-12-31 16:42:12 +0100652 struct pipe_inode_info *pipe = filp->private_data;
Linus Torvaldsad910e32019-12-07 10:41:17 -0800653 unsigned int head, tail;
Linus Torvaldsa11e1d42018-06-28 09:43:44 -0700654
Linus Torvalds3b844822021-08-05 10:04:43 -0700655 /* Epoll has some historical nasty semantics, this enables them */
Kuniyuki Iwashimaf4859222022-04-29 14:38:01 -0700656 WRITE_ONCE(pipe->poll_usage, true);
Linus Torvalds3b844822021-08-05 10:04:43 -0700657
Linus Torvaldsad910e32019-12-07 10:41:17 -0800658 /*
Linus Torvalds0ddad212019-12-09 09:48:27 -0800659 * Reading pipe state only -- no need for acquiring the semaphore.
Linus Torvaldsad910e32019-12-07 10:41:17 -0800660 *
661 * But because this is racy, the code has to add the
662 * entry to the poll table _first_ ..
663 */
Linus Torvalds0ddad212019-12-09 09:48:27 -0800664 if (filp->f_mode & FMODE_READ)
665 poll_wait(filp, &pipe->rd_wait, wait);
666 if (filp->f_mode & FMODE_WRITE)
667 poll_wait(filp, &pipe->wr_wait, wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668
Linus Torvaldsad910e32019-12-07 10:41:17 -0800669 /*
670 * .. and only then can you do the racy tests. That way,
671 * if something changes and you got it wrong, the poll
672 * table entry will wake you up and fix it.
673 */
674 head = READ_ONCE(pipe->head);
675 tail = READ_ONCE(pipe->tail);
676
Linus Torvaldsa11e1d42018-06-28 09:43:44 -0700677 mask = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678 if (filp->f_mode & FMODE_READ) {
David Howells8cefc102019-11-15 13:30:32 +0000679 if (!pipe_empty(head, tail))
680 mask |= EPOLLIN | EPOLLRDNORM;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200681 if (!pipe->writers && filp->f_version != pipe->w_counter)
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800682 mask |= EPOLLHUP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 }
684
685 if (filp->f_mode & FMODE_WRITE) {
David Howells6718b6f2019-10-16 16:47:32 +0100686 if (!pipe_full(head, tail, pipe->max_usage))
David Howells8cefc102019-11-15 13:30:32 +0000687 mask |= EPOLLOUT | EPOLLWRNORM;
Pekka Enberg5e5d7a22005-09-06 15:17:48 -0700688 /*
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800689 * Most Unices do not set EPOLLERR for FIFOs but on Linux they
Pekka Enberg5e5d7a22005-09-06 15:17:48 -0700690 * behave exactly like pipes for poll().
691 */
Ingo Molnar923f4f22006-04-11 13:53:33 +0200692 if (!pipe->readers)
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800693 mask |= EPOLLERR;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 }
695
696 return mask;
697}
698
Linus Torvaldsb0d8d222013-12-02 09:44:51 -0800699static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
700{
701 int kill = 0;
702
703 spin_lock(&inode->i_lock);
704 if (!--pipe->files) {
705 inode->i_pipe = NULL;
706 kill = 1;
707 }
708 spin_unlock(&inode->i_lock);
709
710 if (kill)
711 free_pipe_info(pipe);
712}
713
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714static int
Al Viro599a0ac2013-03-12 09:58:10 -0400715pipe_release(struct inode *inode, struct file *file)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716{
Linus Torvaldsb0d8d222013-12-02 09:44:51 -0800717 struct pipe_inode_info *pipe = file->private_data;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200718
Al Viroebec73f2013-03-21 12:24:01 -0400719 __pipe_lock(pipe);
Al Viro599a0ac2013-03-12 09:58:10 -0400720 if (file->f_mode & FMODE_READ)
721 pipe->readers--;
722 if (file->f_mode & FMODE_WRITE)
723 pipe->writers--;
Ingo Molnar341b4462006-04-11 13:57:45 +0200724
Linus Torvalds6551d5c2020-02-18 10:12:58 -0800725 /* Was that the last reader or writer, but not the other side? */
726 if (!pipe->readers != !pipe->writers) {
727 wake_up_interruptible_all(&pipe->rd_wait);
728 wake_up_interruptible_all(&pipe->wr_wait);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200729 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
730 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 }
Al Viroebec73f2013-03-21 12:24:01 -0400732 __pipe_unlock(pipe);
Al Viroba5bb142013-03-21 02:21:19 -0400733
Linus Torvaldsb0d8d222013-12-02 09:44:51 -0800734 put_pipe_info(inode, pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 return 0;
736}
737
738static int
Al Viro599a0ac2013-03-12 09:58:10 -0400739pipe_fasync(int fd, struct file *filp, int on)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740{
Al Virode32ec42013-03-21 11:16:56 -0400741 struct pipe_inode_info *pipe = filp->private_data;
Al Viro599a0ac2013-03-12 09:58:10 -0400742 int retval = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743
Al Viroebec73f2013-03-21 12:24:01 -0400744 __pipe_lock(pipe);
Al Viro599a0ac2013-03-12 09:58:10 -0400745 if (filp->f_mode & FMODE_READ)
746 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
747 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200748 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
Al Viro599a0ac2013-03-12 09:58:10 -0400749 if (retval < 0 && (filp->f_mode & FMODE_READ))
750 /* this can happen only if on == T */
Oleg Nesterove5bc49b2009-03-12 14:31:28 -0700751 fasync_helper(-1, filp, 0, &pipe->fasync_readers);
752 }
Al Viroebec73f2013-03-21 12:24:01 -0400753 __pipe_unlock(pipe);
Jonathan Corbet60aa4922009-02-01 14:52:56 -0700754 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755}
756
David Howellsc73be612020-01-14 17:07:11 +0000757unsigned long account_pipe_buffers(struct user_struct *user,
758 unsigned long old, unsigned long new)
Willy Tarreau759c0112016-01-18 16:36:09 +0100759{
Michael Kerrisk (man-pages)9c87bcf2016-10-11 13:53:40 -0700760 return atomic_long_add_return(new - old, &user->pipe_bufs);
Willy Tarreau759c0112016-01-18 16:36:09 +0100761}
762
David Howellsc73be612020-01-14 17:07:11 +0000763bool too_many_pipe_buffers_soft(unsigned long user_bufs)
Willy Tarreau759c0112016-01-18 16:36:09 +0100764{
Eric Biggersf7340762018-02-06 15:42:08 -0800765 unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
766
767 return soft_limit && user_bufs > soft_limit;
Willy Tarreau759c0112016-01-18 16:36:09 +0100768}
769
David Howellsc73be612020-01-14 17:07:11 +0000770bool too_many_pipe_buffers_hard(unsigned long user_bufs)
Willy Tarreau759c0112016-01-18 16:36:09 +0100771{
Eric Biggersf7340762018-02-06 15:42:08 -0800772 unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
773
774 return hard_limit && user_bufs > hard_limit;
Willy Tarreau759c0112016-01-18 16:36:09 +0100775}
776
David Howellsc73be612020-01-14 17:07:11 +0000777bool pipe_is_unprivileged_user(void)
Eric Biggers85c2dd52018-02-06 15:41:53 -0800778{
779 return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
780}
781
Al Viro7bee1302013-03-21 11:04:15 -0400782struct pipe_inode_info *alloc_pipe_info(void)
Ingo Molnar3a326a22006-04-10 15:18:35 +0200783{
Ingo Molnar923f4f22006-04-11 13:53:33 +0200784 struct pipe_inode_info *pipe;
Michael Kerrisk (man-pages)09b4d192016-10-11 13:53:34 -0700785 unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
786 struct user_struct *user = get_current_user();
Michael Kerrisk (man-pages)9c87bcf2016-10-11 13:53:40 -0700787 unsigned long user_bufs;
Eric Biggersf7340762018-02-06 15:42:08 -0800788 unsigned int max_size = READ_ONCE(pipe_max_size);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200789
Vladimir Davydovd86133b2016-07-26 15:24:33 -0700790 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
Michael Kerrisk (man-pages)09b4d192016-10-11 13:53:34 -0700791 if (pipe == NULL)
792 goto out_free_uid;
Willy Tarreau759c0112016-01-18 16:36:09 +0100793
Eric Biggersf7340762018-02-06 15:42:08 -0800794 if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
795 pipe_bufs = max_size >> PAGE_SHIFT;
Michael Kerrisk (man-pages)086e7742016-10-11 13:53:43 -0700796
Michael Kerrisk (man-pages)9c87bcf2016-10-11 13:53:40 -0700797 user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
Michael Kerrisk (man-pages)a005ca02016-10-11 13:53:37 -0700798
David Howellsc73be612020-01-14 17:07:11 +0000799 if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
Alex Xu (Hello71)46c4c9d2021-08-05 10:40:47 -0400800 user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
801 pipe_bufs = PIPE_MIN_DEF_BUFFERS;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200802 }
803
David Howellsc73be612020-01-14 17:07:11 +0000804 if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
Michael Kerrisk (man-pages)a005ca02016-10-11 13:53:37 -0700805 goto out_revert_acct;
806
Linus Torvalds906f9042022-04-20 12:07:53 -0700807 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
Michael Kerrisk (man-pages)a005ca02016-10-11 13:53:37 -0700808 GFP_KERNEL_ACCOUNT);
809
Michael Kerrisk (man-pages)09b4d192016-10-11 13:53:34 -0700810 if (pipe->bufs) {
Linus Torvalds0ddad212019-12-09 09:48:27 -0800811 init_waitqueue_head(&pipe->rd_wait);
812 init_waitqueue_head(&pipe->wr_wait);
Michael Kerrisk (man-pages)09b4d192016-10-11 13:53:34 -0700813 pipe->r_counter = pipe->w_counter = 1;
David Howells6718b6f2019-10-16 16:47:32 +0100814 pipe->max_usage = pipe_bufs;
David Howells8cefc102019-11-15 13:30:32 +0000815 pipe->ring_size = pipe_bufs;
David Howellsc73be612020-01-14 17:07:11 +0000816 pipe->nr_accounted = pipe_bufs;
Michael Kerrisk (man-pages)09b4d192016-10-11 13:53:34 -0700817 pipe->user = user;
Michael Kerrisk (man-pages)09b4d192016-10-11 13:53:34 -0700818 mutex_init(&pipe->mutex);
819 return pipe;
820 }
821
Michael Kerrisk (man-pages)a005ca02016-10-11 13:53:37 -0700822out_revert_acct:
Michael Kerrisk (man-pages)9c87bcf2016-10-11 13:53:40 -0700823 (void) account_pipe_buffers(user, pipe_bufs, 0);
Michael Kerrisk (man-pages)09b4d192016-10-11 13:53:34 -0700824 kfree(pipe);
825out_free_uid:
826 free_uid(user);
Jens Axboe35f3d142010-05-20 10:43:18 +0200827 return NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200828}
829
Al Viro4b8a8f12013-03-21 11:06:46 -0400830void free_pipe_info(struct pipe_inode_info *pipe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831{
Andrei Vaginaeb213c2022-03-23 16:06:20 -0700832 unsigned int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833
David Howellsc73be612020-01-14 17:07:11 +0000834#ifdef CONFIG_WATCH_QUEUE
David Howellsdb8facf2022-03-11 13:23:38 +0000835 if (pipe->watch_queue)
David Howellsc73be612020-01-14 17:07:11 +0000836 watch_queue_clear(pipe->watch_queue);
David Howellsc73be612020-01-14 17:07:11 +0000837#endif
838
839 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
Willy Tarreau759c0112016-01-18 16:36:09 +0100840 free_uid(pipe->user);
David Howells8cefc102019-11-15 13:30:32 +0000841 for (i = 0; i < pipe->ring_size; i++) {
Ingo Molnar923f4f22006-04-11 13:53:33 +0200842 struct pipe_buffer *buf = pipe->bufs + i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843 if (buf->ops)
Miklos Szeredia7796382016-09-27 10:45:12 +0200844 pipe_buf_release(pipe, buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845 }
David Howellsdb8facf2022-03-11 13:23:38 +0000846#ifdef CONFIG_WATCH_QUEUE
847 if (pipe->watch_queue)
848 put_watch_queue(pipe->watch_queue);
849#endif
Ingo Molnar923f4f22006-04-11 13:53:33 +0200850 if (pipe->tmp_page)
851 __free_page(pipe->tmp_page);
Linus Torvalds906f9042022-04-20 12:07:53 -0700852 kfree(pipe->bufs);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200853 kfree(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854}
855
Eric Dumazetfa3536c2006-03-26 01:37:24 -0800856static struct vfsmount *pipe_mnt __read_mostly;
Ingo Molnar341b4462006-04-11 13:57:45 +0200857
Eric Dumazetc23fbb62007-05-08 00:26:18 -0700858/*
859 * pipefs_dname() is called from d_path().
860 */
861static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
862{
Al Viro0f60d282022-01-30 15:03:49 -0500863 return dynamic_dname(buffer, buflen, "pipe:[%lu]",
David Howells75c3cfa2015-03-17 22:26:12 +0000864 d_inode(dentry)->i_ino);
Eric Dumazetc23fbb62007-05-08 00:26:18 -0700865}
866
Al Viro3ba13d12009-02-20 06:02:22 +0000867static const struct dentry_operations pipefs_dentry_operations = {
Eric Dumazetc23fbb62007-05-08 00:26:18 -0700868 .d_dname = pipefs_dname,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869};
870
871static struct inode * get_pipe_inode(void)
872{
Eric Dumazeta209dfc2011-07-26 11:36:34 +0200873 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200874 struct pipe_inode_info *pipe;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875
876 if (!inode)
877 goto fail_inode;
878
Christoph Hellwig85fe4022010-10-23 11:19:54 -0400879 inode->i_ino = get_next_ino();
880
Al Viro7bee1302013-03-21 11:04:15 -0400881 pipe = alloc_pipe_info();
Ingo Molnar923f4f22006-04-11 13:53:33 +0200882 if (!pipe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 goto fail_iput;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200884
Al Viroba5bb142013-03-21 02:21:19 -0400885 inode->i_pipe = pipe;
886 pipe->files = 2;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200887 pipe->readers = pipe->writers = 1;
Al Viro599a0ac2013-03-12 09:58:10 -0400888 inode->i_fop = &pipefifo_fops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889
890 /*
891 * Mark the inode dirty from the very beginning,
892 * that way it will never be moved to the dirty
893 * list because "mark_inode_dirty()" will think
894 * that it already _is_ on the dirty list.
895 */
896 inode->i_state = I_DIRTY;
897 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
David Howellsda9592e2008-11-14 10:39:05 +1100898 inode->i_uid = current_fsuid();
899 inode->i_gid = current_fsgid();
Deepa Dinamani078cd822016-09-14 07:48:04 -0700900 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200901
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 return inode;
903
904fail_iput:
905 iput(inode);
Ingo Molnar341b4462006-04-11 13:57:45 +0200906
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907fail_inode:
908 return NULL;
909}
910
Al Viroe4fad8e2012-07-21 15:33:25 +0400911int create_pipe_files(struct file **res, int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912{
Al Viroe4fad8e2012-07-21 15:33:25 +0400913 struct inode *inode = get_pipe_inode();
Andi Kleend6cbd282006-09-30 23:29:26 -0700914 struct file *f;
Qian Cai8a018eb2020-10-01 08:50:55 -0400915 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 if (!inode)
Al Viroe4fad8e2012-07-21 15:33:25 +0400918 return -ENFILE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919
David Howellsc73be612020-01-14 17:07:11 +0000920 if (flags & O_NOTIFICATION_PIPE) {
Qian Cai8a018eb2020-10-01 08:50:55 -0400921 error = watch_queue_init(inode->i_pipe);
922 if (error) {
923 free_pipe_info(inode->i_pipe);
David Howellsc73be612020-01-14 17:07:11 +0000924 iput(inode);
Qian Cai8a018eb2020-10-01 08:50:55 -0400925 return error;
David Howellsc73be612020-01-14 17:07:11 +0000926 }
David Howellsc73be612020-01-14 17:07:11 +0000927 }
928
Al Viro152b6372018-06-09 10:05:18 -0400929 f = alloc_file_pseudo(inode, pipe_mnt, "",
930 O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
931 &pipefifo_fops);
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500932 if (IS_ERR(f)) {
Al Viro152b6372018-06-09 10:05:18 -0400933 free_pipe_info(inode->i_pipe);
934 iput(inode);
935 return PTR_ERR(f);
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500936 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937
Al Virode32ec42013-03-21 11:16:56 -0400938 f->private_data = inode->i_pipe;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939
Al Viro183266f2018-06-17 14:15:10 -0400940 res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),
941 &pipefifo_fops);
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500942 if (IS_ERR(res[0])) {
Al Virob10a4a92018-07-09 02:29:58 -0400943 put_pipe_info(inode, inode->i_pipe);
944 fput(f);
945 return PTR_ERR(res[0]);
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500946 }
Al Virode32ec42013-03-21 11:16:56 -0400947 res[0]->private_data = inode->i_pipe;
Al Viroe4fad8e2012-07-21 15:33:25 +0400948 res[1] = f;
Linus Torvaldsd8e464e2019-11-17 11:20:48 -0800949 stream_open(inode, res[0]);
950 stream_open(inode, res[1]);
Al Viroe4fad8e2012-07-21 15:33:25 +0400951 return 0;
Andi Kleend6cbd282006-09-30 23:29:26 -0700952}
953
Al Viro5b249b12012-08-19 12:17:29 -0400954static int __do_pipe_flags(int *fd, struct file **files, int flags)
Andi Kleend6cbd282006-09-30 23:29:26 -0700955{
Andi Kleend6cbd282006-09-30 23:29:26 -0700956 int error;
957 int fdw, fdr;
958
David Howellsc73be612020-01-14 17:07:11 +0000959 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
Ulrich Dreppered8cae82008-07-23 21:29:30 -0700960 return -EINVAL;
961
Al Viroe4fad8e2012-07-21 15:33:25 +0400962 error = create_pipe_files(files, flags);
963 if (error)
964 return error;
Andi Kleend6cbd282006-09-30 23:29:26 -0700965
Ulrich Dreppered8cae82008-07-23 21:29:30 -0700966 error = get_unused_fd_flags(flags);
Andi Kleend6cbd282006-09-30 23:29:26 -0700967 if (error < 0)
968 goto err_read_pipe;
969 fdr = error;
970
Ulrich Dreppered8cae82008-07-23 21:29:30 -0700971 error = get_unused_fd_flags(flags);
Andi Kleend6cbd282006-09-30 23:29:26 -0700972 if (error < 0)
973 goto err_fdr;
974 fdw = error;
975
Al Viro157cf642008-12-14 04:57:47 -0500976 audit_fd_pair(fdr, fdw);
Andi Kleend6cbd282006-09-30 23:29:26 -0700977 fd[0] = fdr;
978 fd[1] = fdw;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 return 0;
980
Andi Kleend6cbd282006-09-30 23:29:26 -0700981 err_fdr:
982 put_unused_fd(fdr);
983 err_read_pipe:
Al Viroe4fad8e2012-07-21 15:33:25 +0400984 fput(files[0]);
985 fput(files[1]);
Andi Kleend6cbd282006-09-30 23:29:26 -0700986 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987}
988
Al Viro5b249b12012-08-19 12:17:29 -0400989int do_pipe_flags(int *fd, int flags)
990{
991 struct file *files[2];
992 int error = __do_pipe_flags(fd, files, flags);
993 if (!error) {
994 fd_install(fd[0], files[0]);
995 fd_install(fd[1], files[1]);
996 }
997 return error;
998}
999
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000/*
Ulrich Drepperd35c7b02008-05-03 15:10:37 -04001001 * sys_pipe() is the normal C calling standard for creating
1002 * a pipe. It's not the way Unix traditionally does this, though.
1003 */
Dominik Brodowski0a216dd2018-03-11 11:34:28 +01001004static int do_pipe2(int __user *fildes, int flags)
Ulrich Drepperd35c7b02008-05-03 15:10:37 -04001005{
Al Viro5b249b12012-08-19 12:17:29 -04001006 struct file *files[2];
Ulrich Drepperd35c7b02008-05-03 15:10:37 -04001007 int fd[2];
1008 int error;
1009
Al Viro5b249b12012-08-19 12:17:29 -04001010 error = __do_pipe_flags(fd, files, flags);
Ulrich Drepperd35c7b02008-05-03 15:10:37 -04001011 if (!error) {
Al Viro5b249b12012-08-19 12:17:29 -04001012 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
1013 fput(files[0]);
1014 fput(files[1]);
1015 put_unused_fd(fd[0]);
1016 put_unused_fd(fd[1]);
Ulrich Drepperd35c7b02008-05-03 15:10:37 -04001017 error = -EFAULT;
Al Viro5b249b12012-08-19 12:17:29 -04001018 } else {
1019 fd_install(fd[0], files[0]);
1020 fd_install(fd[1], files[1]);
Ulrich Drepperba719ba2008-05-06 20:42:38 -07001021 }
Ulrich Drepperd35c7b02008-05-03 15:10:37 -04001022 }
1023 return error;
1024}
1025
Dominik Brodowski0a216dd2018-03-11 11:34:28 +01001026SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
1027{
1028 return do_pipe2(fildes, flags);
1029}
1030
Heiko Carstens2b664212009-01-14 14:14:35 +01001031SYSCALL_DEFINE1(pipe, int __user *, fildes)
Ulrich Dreppered8cae82008-07-23 21:29:30 -07001032{
Dominik Brodowski0a216dd2018-03-11 11:34:28 +01001033 return do_pipe2(fildes, 0);
Ulrich Dreppered8cae82008-07-23 21:29:30 -07001034}
1035
Linus Torvalds472e5b02020-10-01 19:14:36 -07001036/*
1037 * This is the stupid "wait for pipe to be readable or writable"
1038 * model.
1039 *
1040 * See pipe_read/write() for the proper kind of exclusive wait,
1041 * but that requires that we wake up any other readers/writers
1042 * if we then do not end up reading everything (ie the whole
1043 * "wake_next_reader/writer" logic in pipe_read/write()).
1044 */
1045void pipe_wait_readable(struct pipe_inode_info *pipe)
1046{
1047 pipe_unlock(pipe);
1048 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
1049 pipe_lock(pipe);
1050}
1051
1052void pipe_wait_writable(struct pipe_inode_info *pipe)
1053{
1054 pipe_unlock(pipe);
1055 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
1056 pipe_lock(pipe);
1057}
1058
1059/*
1060 * This depends on both the wait (here) and the wakeup (wake_up_partner)
1061 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1062 * race with the count check and waitqueue prep.
1063 *
1064 * Normally in order to avoid races, you'd do the prepare_to_wait() first,
1065 * then check the condition you're waiting for, and only then sleep. But
1066 * because of the pipe lock, we can check the condition before being on
1067 * the wait queue.
1068 *
1069 * We use the 'rd_wait' waitqueue for pipe partner waiting.
1070 */
Al Virofc7478a2013-03-21 02:07:59 -04001071static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
Al Virof776c732013-03-12 09:46:27 -04001072{
Linus Torvalds472e5b02020-10-01 19:14:36 -07001073 DEFINE_WAIT(rdwait);
David Howells8cefc102019-11-15 13:30:32 +00001074 int cur = *cnt;
Al Virof776c732013-03-12 09:46:27 -04001075
1076 while (cur == *cnt) {
Linus Torvalds472e5b02020-10-01 19:14:36 -07001077 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
1078 pipe_unlock(pipe);
1079 schedule();
1080 finish_wait(&pipe->rd_wait, &rdwait);
1081 pipe_lock(pipe);
Al Virof776c732013-03-12 09:46:27 -04001082 if (signal_pending(current))
1083 break;
1084 }
1085 return cur == *cnt ? -ERESTARTSYS : 0;
1086}
1087
Al Virofc7478a2013-03-21 02:07:59 -04001088static void wake_up_partner(struct pipe_inode_info *pipe)
Al Virof776c732013-03-12 09:46:27 -04001089{
Linus Torvalds6551d5c2020-02-18 10:12:58 -08001090 wake_up_interruptible_all(&pipe->rd_wait);
Al Virof776c732013-03-12 09:46:27 -04001091}
1092
1093static int fifo_open(struct inode *inode, struct file *filp)
1094{
1095 struct pipe_inode_info *pipe;
Al Viro599a0ac2013-03-12 09:58:10 -04001096 bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC;
Al Virof776c732013-03-12 09:46:27 -04001097 int ret;
1098
Al Viroba5bb142013-03-21 02:21:19 -04001099 filp->f_version = 0;
1100
1101 spin_lock(&inode->i_lock);
1102 if (inode->i_pipe) {
1103 pipe = inode->i_pipe;
1104 pipe->files++;
1105 spin_unlock(&inode->i_lock);
1106 } else {
1107 spin_unlock(&inode->i_lock);
Al Viro7bee1302013-03-21 11:04:15 -04001108 pipe = alloc_pipe_info();
Al Virof776c732013-03-12 09:46:27 -04001109 if (!pipe)
Al Viroba5bb142013-03-21 02:21:19 -04001110 return -ENOMEM;
1111 pipe->files = 1;
1112 spin_lock(&inode->i_lock);
1113 if (unlikely(inode->i_pipe)) {
1114 inode->i_pipe->files++;
1115 spin_unlock(&inode->i_lock);
Al Viro4b8a8f12013-03-21 11:06:46 -04001116 free_pipe_info(pipe);
Al Viroba5bb142013-03-21 02:21:19 -04001117 pipe = inode->i_pipe;
1118 } else {
1119 inode->i_pipe = pipe;
1120 spin_unlock(&inode->i_lock);
1121 }
Al Virof776c732013-03-12 09:46:27 -04001122 }
Al Virode32ec42013-03-21 11:16:56 -04001123 filp->private_data = pipe;
Al Viroba5bb142013-03-21 02:21:19 -04001124 /* OK, we have a pipe and it's pinned down */
1125
Al Viroebec73f2013-03-21 12:24:01 -04001126 __pipe_lock(pipe);
Al Virof776c732013-03-12 09:46:27 -04001127
1128 /* We can only do regular read/write on fifos */
Linus Torvaldsd8e464e2019-11-17 11:20:48 -08001129 stream_open(inode, filp);
Al Virof776c732013-03-12 09:46:27 -04001130
Linus Torvaldsd8e464e2019-11-17 11:20:48 -08001131 switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {
Al Virof776c732013-03-12 09:46:27 -04001132 case FMODE_READ:
1133 /*
1134 * O_RDONLY
1135 * POSIX.1 says that O_NONBLOCK means return with the FIFO
1136 * opened, even when there is no process writing the FIFO.
1137 */
Al Virof776c732013-03-12 09:46:27 -04001138 pipe->r_counter++;
1139 if (pipe->readers++ == 0)
Al Virofc7478a2013-03-21 02:07:59 -04001140 wake_up_partner(pipe);
Al Virof776c732013-03-12 09:46:27 -04001141
Al Viro599a0ac2013-03-12 09:58:10 -04001142 if (!is_pipe && !pipe->writers) {
Al Virof776c732013-03-12 09:46:27 -04001143 if ((filp->f_flags & O_NONBLOCK)) {
Linus Torvaldsa9a08842018-02-11 14:34:03 -08001144 /* suppress EPOLLHUP until we have
Al Virof776c732013-03-12 09:46:27 -04001145 * seen a writer */
1146 filp->f_version = pipe->w_counter;
1147 } else {
Al Virofc7478a2013-03-21 02:07:59 -04001148 if (wait_for_partner(pipe, &pipe->w_counter))
Al Virof776c732013-03-12 09:46:27 -04001149 goto err_rd;
1150 }
1151 }
1152 break;
David Howells8cefc102019-11-15 13:30:32 +00001153
Al Virof776c732013-03-12 09:46:27 -04001154 case FMODE_WRITE:
1155 /*
1156 * O_WRONLY
1157 * POSIX.1 says that O_NONBLOCK means return -1 with
1158 * errno=ENXIO when there is no process reading the FIFO.
1159 */
1160 ret = -ENXIO;
Al Viro599a0ac2013-03-12 09:58:10 -04001161 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
Al Virof776c732013-03-12 09:46:27 -04001162 goto err;
1163
Al Virof776c732013-03-12 09:46:27 -04001164 pipe->w_counter++;
1165 if (!pipe->writers++)
Al Virofc7478a2013-03-21 02:07:59 -04001166 wake_up_partner(pipe);
Al Virof776c732013-03-12 09:46:27 -04001167
Al Viro599a0ac2013-03-12 09:58:10 -04001168 if (!is_pipe && !pipe->readers) {
Al Virofc7478a2013-03-21 02:07:59 -04001169 if (wait_for_partner(pipe, &pipe->r_counter))
Al Virof776c732013-03-12 09:46:27 -04001170 goto err_wr;
1171 }
1172 break;
David Howells8cefc102019-11-15 13:30:32 +00001173
Al Virof776c732013-03-12 09:46:27 -04001174 case FMODE_READ | FMODE_WRITE:
1175 /*
1176 * O_RDWR
1177 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
1178 * This implementation will NEVER block on a O_RDWR open, since
1179 * the process can at least talk to itself.
1180 */
Al Virof776c732013-03-12 09:46:27 -04001181
1182 pipe->readers++;
1183 pipe->writers++;
1184 pipe->r_counter++;
1185 pipe->w_counter++;
1186 if (pipe->readers == 1 || pipe->writers == 1)
Al Virofc7478a2013-03-21 02:07:59 -04001187 wake_up_partner(pipe);
Al Virof776c732013-03-12 09:46:27 -04001188 break;
1189
1190 default:
1191 ret = -EINVAL;
1192 goto err;
1193 }
1194
1195 /* Ok! */
Al Viroebec73f2013-03-21 12:24:01 -04001196 __pipe_unlock(pipe);
Al Virof776c732013-03-12 09:46:27 -04001197 return 0;
1198
1199err_rd:
1200 if (!--pipe->readers)
Linus Torvalds0ddad212019-12-09 09:48:27 -08001201 wake_up_interruptible(&pipe->wr_wait);
Al Virof776c732013-03-12 09:46:27 -04001202 ret = -ERESTARTSYS;
1203 goto err;
1204
1205err_wr:
1206 if (!--pipe->writers)
Linus Torvalds6551d5c2020-02-18 10:12:58 -08001207 wake_up_interruptible_all(&pipe->rd_wait);
Al Virof776c732013-03-12 09:46:27 -04001208 ret = -ERESTARTSYS;
1209 goto err;
1210
1211err:
Al Viroebec73f2013-03-21 12:24:01 -04001212 __pipe_unlock(pipe);
Linus Torvaldsb0d8d222013-12-02 09:44:51 -08001213
1214 put_pipe_info(inode, pipe);
Al Virof776c732013-03-12 09:46:27 -04001215 return ret;
1216}
1217
Al Viro599a0ac2013-03-12 09:58:10 -04001218const struct file_operations pipefifo_fops = {
1219 .open = fifo_open,
1220 .llseek = no_llseek,
Al Virofb9096a2014-04-02 19:56:54 -04001221 .read_iter = pipe_read,
Al Virof0d1bec2014-04-03 15:05:18 -04001222 .write_iter = pipe_write,
Linus Torvaldsa11e1d42018-06-28 09:43:44 -07001223 .poll = pipe_poll,
Al Viro599a0ac2013-03-12 09:58:10 -04001224 .unlocked_ioctl = pipe_ioctl,
1225 .release = pipe_release,
1226 .fasync = pipe_fasync,
Johannes Bergf8ad8182021-01-25 10:16:15 +01001227 .splice_write = iter_file_splice_write,
Al Virof776c732013-03-12 09:46:27 -04001228};
1229
Ulrich Drepperd35c7b02008-05-03 15:10:37 -04001230/*
Michael Kerrisk (man-pages)f491bd72016-10-11 13:53:22 -07001231 * Currently we rely on the pipe array holding a power-of-2 number
Joe Lawrenced3f14c42017-11-17 15:29:21 -08001232 * of pages. Returns 0 on error.
Michael Kerrisk (man-pages)f491bd72016-10-11 13:53:22 -07001233 */
Eric Biggers96e99be402018-02-06 15:42:00 -08001234unsigned int round_pipe_size(unsigned long size)
Michael Kerrisk (man-pages)f491bd72016-10-11 13:53:22 -07001235{
Eric Biggersc4fed5a2018-02-06 15:42:05 -08001236 if (size > (1U << 31))
Eric Biggers96e99be402018-02-06 15:42:00 -08001237 return 0;
1238
Eric Biggers4c2e4be2018-02-06 15:41:45 -08001239 /* Minimum pipe size, as required by POSIX */
1240 if (size < PAGE_SIZE)
Eric Biggersc4fed5a2018-02-06 15:42:05 -08001241 return PAGE_SIZE;
Joe Lawrenced3f14c42017-11-17 15:29:21 -08001242
Eric Biggersc4fed5a2018-02-06 15:42:05 -08001243 return roundup_pow_of_two(size);
Michael Kerrisk (man-pages)f491bd72016-10-11 13:53:22 -07001244}
1245
1246/*
David Howellsc73be612020-01-14 17:07:11 +00001247 * Resize the pipe ring to a number of slots.
David Howells189b0dd2022-05-26 07:34:52 +01001248 *
1249 * Note the pipe can be reduced in capacity, but only if the current
1250 * occupancy doesn't exceed nr_slots; if it does, EBUSY will be
1251 * returned instead.
Jens Axboe35f3d142010-05-20 10:43:18 +02001252 */
David Howellsc73be612020-01-14 17:07:11 +00001253int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
Jens Axboe35f3d142010-05-20 10:43:18 +02001254{
1255 struct pipe_buffer *bufs;
David Howellsc73be612020-01-14 17:07:11 +00001256 unsigned int head, tail, mask, n;
Jens Axboe35f3d142010-05-20 10:43:18 +02001257
Linus Torvalds906f9042022-04-20 12:07:53 -07001258 bufs = kcalloc(nr_slots, sizeof(*bufs),
1259 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
David Howellsc73be612020-01-14 17:07:11 +00001260 if (unlikely(!bufs))
1261 return -ENOMEM;
Jens Axboe35f3d142010-05-20 10:43:18 +02001262
David Howells189b0dd2022-05-26 07:34:52 +01001263 spin_lock_irq(&pipe->rd_wait.lock);
1264 mask = pipe->ring_size - 1;
1265 head = pipe->head;
1266 tail = pipe->tail;
1267
1268 n = pipe_occupancy(head, tail);
1269 if (nr_slots < n) {
1270 spin_unlock_irq(&pipe->rd_wait.lock);
1271 kfree(bufs);
1272 return -EBUSY;
1273 }
1274
Jens Axboe35f3d142010-05-20 10:43:18 +02001275 /*
1276 * The pipe array wraps around, so just start the new one at zero
David Howells8cefc102019-11-15 13:30:32 +00001277 * and adjust the indices.
Jens Axboe35f3d142010-05-20 10:43:18 +02001278 */
David Howells8cefc102019-11-15 13:30:32 +00001279 if (n > 0) {
1280 unsigned int h = head & mask;
1281 unsigned int t = tail & mask;
1282 if (h > t) {
1283 memcpy(bufs, pipe->bufs + t,
1284 n * sizeof(struct pipe_buffer));
1285 } else {
1286 unsigned int tsize = pipe->ring_size - t;
1287 if (h > 0)
1288 memcpy(bufs + tsize, pipe->bufs,
1289 h * sizeof(struct pipe_buffer));
1290 memcpy(bufs, pipe->bufs + t,
1291 tsize * sizeof(struct pipe_buffer));
1292 }
Jens Axboe35f3d142010-05-20 10:43:18 +02001293 }
1294
David Howells8cefc102019-11-15 13:30:32 +00001295 head = n;
1296 tail = 0;
1297
Linus Torvalds906f9042022-04-20 12:07:53 -07001298 kfree(pipe->bufs);
Jens Axboe35f3d142010-05-20 10:43:18 +02001299 pipe->bufs = bufs;
David Howells8cefc102019-11-15 13:30:32 +00001300 pipe->ring_size = nr_slots;
David Howellsc73be612020-01-14 17:07:11 +00001301 if (pipe->max_usage > nr_slots)
1302 pipe->max_usage = nr_slots;
David Howells8cefc102019-11-15 13:30:32 +00001303 pipe->tail = tail;
1304 pipe->head = head;
Linus Torvalds6551d5c2020-02-18 10:12:58 -08001305
David Howells189b0dd2022-05-26 07:34:52 +01001306 spin_unlock_irq(&pipe->rd_wait.lock);
1307
Linus Torvalds6551d5c2020-02-18 10:12:58 -08001308 /* This might have made more room for writers */
1309 wake_up_interruptible(&pipe->wr_wait);
David Howellsc73be612020-01-14 17:07:11 +00001310 return 0;
1311}
1312
1313/*
1314 * Allocate a new array of pipe buffers and copy the info over. Returns the
1315 * pipe size if successful, or return -ERROR on error.
1316 */
1317static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
1318{
1319 unsigned long user_bufs;
1320 unsigned int nr_slots, size;
1321 long ret = 0;
1322
1323#ifdef CONFIG_WATCH_QUEUE
1324 if (pipe->watch_queue)
1325 return -EBUSY;
1326#endif
1327
1328 size = round_pipe_size(arg);
1329 nr_slots = size >> PAGE_SHIFT;
1330
1331 if (!nr_slots)
1332 return -EINVAL;
1333
1334 /*
1335 * If trying to increase the pipe capacity, check that an
1336 * unprivileged user is not trying to exceed various limits
1337 * (soft limit check here, hard limit check just below).
1338 * Decreasing the pipe capacity is always permitted, even
1339 * if the user is currently over a limit.
1340 */
1341 if (nr_slots > pipe->max_usage &&
1342 size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
1343 return -EPERM;
1344
1345 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
1346
1347 if (nr_slots > pipe->max_usage &&
1348 (too_many_pipe_buffers_hard(user_bufs) ||
1349 too_many_pipe_buffers_soft(user_bufs)) &&
1350 pipe_is_unprivileged_user()) {
1351 ret = -EPERM;
1352 goto out_revert_acct;
1353 }
1354
1355 ret = pipe_resize_ring(pipe, nr_slots);
1356 if (ret < 0)
1357 goto out_revert_acct;
1358
1359 pipe->max_usage = nr_slots;
1360 pipe->nr_accounted = nr_slots;
David Howells6718b6f2019-10-16 16:47:32 +01001361 return pipe->max_usage * PAGE_SIZE;
Michael Kerrisk (man-pages)b0b91d12016-10-11 13:53:31 -07001362
1363out_revert_acct:
David Howellsc73be612020-01-14 17:07:11 +00001364 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
Michael Kerrisk (man-pages)b0b91d12016-10-11 13:53:31 -07001365 return ret;
Jens Axboe35f3d142010-05-20 10:43:18 +02001366}
1367
Jens Axboeff9da692010-06-03 14:54:39 +02001368/*
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01001369 * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is
1370 * not enough to verify that this is a pipe.
Linus Torvalds72083642010-11-28 16:27:19 -08001371 */
David Howellsc73be612020-01-14 17:07:11 +00001372struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
Linus Torvalds72083642010-11-28 16:27:19 -08001373{
David Howellsc73be612020-01-14 17:07:11 +00001374 struct pipe_inode_info *pipe = file->private_data;
1375
1376 if (file->f_op != &pipefifo_fops || !pipe)
1377 return NULL;
1378#ifdef CONFIG_WATCH_QUEUE
1379 if (for_splice && pipe->watch_queue)
1380 return NULL;
1381#endif
1382 return pipe;
Linus Torvalds72083642010-11-28 16:27:19 -08001383}
1384
Jens Axboe35f3d142010-05-20 10:43:18 +02001385long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
1386{
1387 struct pipe_inode_info *pipe;
1388 long ret;
1389
David Howellsc73be612020-01-14 17:07:11 +00001390 pipe = get_pipe_info(file, false);
Jens Axboe35f3d142010-05-20 10:43:18 +02001391 if (!pipe)
1392 return -EBADF;
1393
Al Viroebec73f2013-03-21 12:24:01 -04001394 __pipe_lock(pipe);
Jens Axboe35f3d142010-05-20 10:43:18 +02001395
1396 switch (cmd) {
Michael Kerrisk (man-pages)d37d4162016-10-11 13:53:25 -07001397 case F_SETPIPE_SZ:
1398 ret = pipe_set_size(pipe, arg);
Jens Axboe35f3d142010-05-20 10:43:18 +02001399 break;
1400 case F_GETPIPE_SZ:
David Howells6718b6f2019-10-16 16:47:32 +01001401 ret = pipe->max_usage * PAGE_SIZE;
Jens Axboe35f3d142010-05-20 10:43:18 +02001402 break;
1403 default:
1404 ret = -EINVAL;
1405 break;
1406 }
1407
Al Viroebec73f2013-03-21 12:24:01 -04001408 __pipe_unlock(pipe);
Jens Axboe35f3d142010-05-20 10:43:18 +02001409 return ret;
1410}
1411
Nick Pigginff0c7d12011-01-07 17:49:50 +11001412static const struct super_operations pipefs_ops = {
1413 .destroy_inode = free_inode_nonrcu,
Pavel Emelyanovd70ef972011-10-31 17:10:04 -07001414 .statfs = simple_statfs,
Nick Pigginff0c7d12011-01-07 17:49:50 +11001415};
1416
Jens Axboe35f3d142010-05-20 10:43:18 +02001417/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 * pipefs should _never_ be mounted by userland - too much of security hassle,
1419 * no real gain from having the whole whorehouse mounted. So we don't need
1420 * any operations on the root directory. However, we need a non-trivial
1421 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1422 */
David Howells4fa7ec52019-03-25 16:38:23 +00001423
1424static int pipefs_init_fs_context(struct fs_context *fc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425{
David Howells4fa7ec52019-03-25 16:38:23 +00001426 struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
1427 if (!ctx)
1428 return -ENOMEM;
1429 ctx->ops = &pipefs_ops;
1430 ctx->dops = &pipefs_dentry_operations;
1431 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432}
1433
1434static struct file_system_type pipe_fs_type = {
1435 .name = "pipefs",
David Howells4fa7ec52019-03-25 16:38:23 +00001436 .init_fs_context = pipefs_init_fs_context,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001437 .kill_sb = kill_anon_super,
1438};
1439
Luis Chamberlain1998f192022-01-21 22:13:20 -08001440#ifdef CONFIG_SYSCTL
1441static int do_proc_dopipe_max_size_conv(unsigned long *lvalp,
1442 unsigned int *valp,
1443 int write, void *data)
1444{
1445 if (write) {
1446 unsigned int val;
1447
1448 val = round_pipe_size(*lvalp);
1449 if (val == 0)
1450 return -EINVAL;
1451
1452 *valp = val;
1453 } else {
1454 unsigned int val = *valp;
1455 *lvalp = (unsigned long) val;
1456 }
1457
1458 return 0;
1459}
1460
1461static int proc_dopipe_max_size(struct ctl_table *table, int write,
1462 void *buffer, size_t *lenp, loff_t *ppos)
1463{
1464 return do_proc_douintvec(table, write, buffer, lenp, ppos,
1465 do_proc_dopipe_max_size_conv, NULL);
1466}
1467
1468static struct ctl_table fs_pipe_sysctls[] = {
1469 {
1470 .procname = "pipe-max-size",
1471 .data = &pipe_max_size,
1472 .maxlen = sizeof(pipe_max_size),
1473 .mode = 0644,
1474 .proc_handler = proc_dopipe_max_size,
1475 },
1476 {
1477 .procname = "pipe-user-pages-hard",
1478 .data = &pipe_user_pages_hard,
1479 .maxlen = sizeof(pipe_user_pages_hard),
1480 .mode = 0644,
1481 .proc_handler = proc_doulongvec_minmax,
1482 },
1483 {
1484 .procname = "pipe-user-pages-soft",
1485 .data = &pipe_user_pages_soft,
1486 .maxlen = sizeof(pipe_user_pages_soft),
1487 .mode = 0644,
1488 .proc_handler = proc_doulongvec_minmax,
1489 },
1490 { }
1491};
1492#endif
1493
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494static int __init init_pipe_fs(void)
1495{
1496 int err = register_filesystem(&pipe_fs_type);
Ingo Molnar341b4462006-04-11 13:57:45 +02001497
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498 if (!err) {
1499 pipe_mnt = kern_mount(&pipe_fs_type);
1500 if (IS_ERR(pipe_mnt)) {
1501 err = PTR_ERR(pipe_mnt);
1502 unregister_filesystem(&pipe_fs_type);
1503 }
1504 }
Luis Chamberlain1998f192022-01-21 22:13:20 -08001505#ifdef CONFIG_SYSCTL
1506 register_sysctl_init("fs", fs_pipe_sysctls);
1507#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508 return err;
1509}
1510
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511fs_initcall(init_pipe_fs);