blob: 7893ea161d77075ca0ab524d7c8021ab668c0978 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/fs/file.c
4 *
5 * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
6 *
7 * Manage the dynamic fd arrays in the process files_struct.
8 */
9
Al Virofe17f222012-08-21 11:48:11 -040010#include <linux/syscalls.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050011#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/fs.h>
Christian Brauner278a5fb2019-05-24 11:30:34 +020013#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/mm.h>
Ingo Molnar3f07c012017-02-08 18:51:30 +010015#include <linux/sched/signal.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/file.h>
Al Viro9f3acc32008-04-24 07:44:08 -040018#include <linux/fdtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/bitops.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070020#include <linux/spinlock.h>
21#include <linux/rcupdate.h>
Christian Brauner60997c32020-06-03 21:48:55 +020022#include <linux/close_range.h>
Kees Cook66590612020-06-10 08:20:05 -070023#include <net/sock.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070024
Jens Axboe53dec2e2021-01-19 15:41:52 -070025#include "internal.h"
26
Alexey Dobriyan9b80a182016-09-02 00:38:52 +030027unsigned int sysctl_nr_open __read_mostly = 1024*1024;
28unsigned int sysctl_nr_open_min = BITS_PER_LONG;
Rasmus Villemoes752343b2015-10-29 12:01:41 +010029/* our min() is unusable in constant expressions ;-/ */
30#define __const_min(x, y) ((x) < (y) ? (x) : (y))
Alexey Dobriyan9b80a182016-09-02 00:38:52 +030031unsigned int sysctl_nr_open_max =
32 __const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG;
Eric Dumazet9cfe0152008-02-06 01:37:16 -080033
Changli Gaoa892e2d2010-08-10 18:01:35 -070034static void __free_fdtable(struct fdtable *fdt)
Vadim Lobanov5466b452006-12-10 02:21:22 -080035{
Al Virof6c0a192014-04-23 10:18:46 -040036 kvfree(fdt->fd);
37 kvfree(fdt->open_fds);
Changli Gaoa892e2d2010-08-10 18:01:35 -070038 kfree(fdt);
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070039}
40
Al Viro7cf4dc32012-08-15 19:56:12 -040041static void free_fdtable_rcu(struct rcu_head *rcu)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070042{
Al Viroac3e3c52013-04-28 21:42:33 -040043 __free_fdtable(container_of(rcu, struct fdtable, rcu));
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070044}
45
Linus Torvaldsf3f86e32015-10-30 16:53:57 -070046#define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr))
47#define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long))
48
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070049/*
Eric Biggersea5c58e2015-11-06 00:32:04 -060050 * Copy 'count' fd bits from the old table to the new table and clear the extra
51 * space if any. This does not copy the file pointers. Called with the files
52 * spinlock held for write.
53 */
54static void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt,
55 unsigned int count)
56{
57 unsigned int cpy, set;
58
59 cpy = count / BITS_PER_BYTE;
60 set = (nfdt->max_fds - count) / BITS_PER_BYTE;
61 memcpy(nfdt->open_fds, ofdt->open_fds, cpy);
62 memset((char *)nfdt->open_fds + cpy, 0, set);
63 memcpy(nfdt->close_on_exec, ofdt->close_on_exec, cpy);
64 memset((char *)nfdt->close_on_exec + cpy, 0, set);
65
66 cpy = BITBIT_SIZE(count);
67 set = BITBIT_SIZE(nfdt->max_fds) - cpy;
68 memcpy(nfdt->full_fds_bits, ofdt->full_fds_bits, cpy);
69 memset((char *)nfdt->full_fds_bits + cpy, 0, set);
70}
71
72/*
73 * Copy all file descriptors from the old table to the new, expanded table and
74 * clear the extra space. Called with the files spinlock held for write.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070075 */
Vadim Lobanov5466b452006-12-10 02:21:22 -080076static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070077{
Al Viro4e89b722020-05-19 17:48:52 -040078 size_t cpy, set;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070079
Vadim Lobanov5466b452006-12-10 02:21:22 -080080 BUG_ON(nfdt->max_fds < ofdt->max_fds);
Vadim Lobanov5466b452006-12-10 02:21:22 -080081
82 cpy = ofdt->max_fds * sizeof(struct file *);
83 set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
84 memcpy(nfdt->fd, ofdt->fd, cpy);
Eric Biggersea5c58e2015-11-06 00:32:04 -060085 memset((char *)nfdt->fd + cpy, 0, set);
Vadim Lobanov5466b452006-12-10 02:21:22 -080086
Eric Biggersea5c58e2015-11-06 00:32:04 -060087 copy_fd_bitmaps(nfdt, ofdt, ofdt->max_fds);
Linus Torvalds1da177e2005-04-16 15:20:36 -070088}
89
Linus Torvalds1c24a182022-03-29 15:06:39 -070090/*
91 * Note how the fdtable bitmap allocations very much have to be a multiple of
92 * BITS_PER_LONG. This is not only because we walk those things in chunks of
93 * 'unsigned long' in some places, but simply because that is how the Linux
94 * kernel bitmaps are defined to work: they are not "bits in an array of bytes",
95 * they are very much "bits in an array of unsigned long".
96 *
97 * The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied
98 * by that "1024/sizeof(ptr)" before, we already know there are sufficient
99 * clear low bits. Clang seems to realize that, gcc ends up being confused.
100 *
101 * On a 128-bit machine, the ALIGN() would actually matter. In the meantime,
102 * let's consider it documentation (and maybe a test-case for gcc to improve
103 * its code generation ;)
104 */
Vadim Lobanov5466b452006-12-10 02:21:22 -0800105static struct fdtable * alloc_fdtable(unsigned int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106{
Vadim Lobanov5466b452006-12-10 02:21:22 -0800107 struct fdtable *fdt;
David Howells1fd36ad2012-02-16 17:49:54 +0000108 void *data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700110 /*
Vadim Lobanov5466b452006-12-10 02:21:22 -0800111 * Figure out how many fds we actually want to support in this fdtable.
112 * Allocation steps are keyed to the size of the fdarray, since it
113 * grows far faster than any of the other dynamic data. We try to fit
114 * the fdarray into comfortable page-tuned chunks: starting at 1024B
115 * and growing in powers of two from there on.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700116 */
Vadim Lobanov5466b452006-12-10 02:21:22 -0800117 nr /= (1024 / sizeof(struct file *));
118 nr = roundup_pow_of_two(nr + 1);
119 nr *= (1024 / sizeof(struct file *));
Linus Torvalds1c24a182022-03-29 15:06:39 -0700120 nr = ALIGN(nr, BITS_PER_LONG);
Al Viro5c598b32008-04-27 20:04:15 -0400121 /*
122 * Note that this can drive nr *below* what we had passed if sysctl_nr_open
123 * had been set lower between the check in expand_files() and here. Deal
124 * with that in caller, it's cheaper that way.
125 *
126 * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
127 * bitmaps handling below becomes unpleasant, to put it mildly...
128 */
129 if (unlikely(nr > sysctl_nr_open))
130 nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1;
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800131
Vladimir Davydov5d097052016-01-14 15:18:21 -0800132 fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800133 if (!fdt)
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800134 goto out;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800135 fdt->max_fds = nr;
Michal Hockoc823bd92017-07-06 15:36:19 -0700136 data = kvmalloc_array(nr, sizeof(struct file *), GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800137 if (!data)
138 goto out_fdt;
David Howells1fd36ad2012-02-16 17:49:54 +0000139 fdt->fd = data;
140
Michal Hockoc823bd92017-07-06 15:36:19 -0700141 data = kvmalloc(max_t(size_t,
142 2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
143 GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800144 if (!data)
145 goto out_arr;
David Howells1fd36ad2012-02-16 17:49:54 +0000146 fdt->open_fds = data;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800147 data += nr / BITS_PER_BYTE;
David Howells1fd36ad2012-02-16 17:49:54 +0000148 fdt->close_on_exec = data;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700149 data += nr / BITS_PER_BYTE;
150 fdt->full_fds_bits = data;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800151
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700152 return fdt;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800153
154out_arr:
Al Virof6c0a192014-04-23 10:18:46 -0400155 kvfree(fdt->fd);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800156out_fdt:
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700157 kfree(fdt);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800158out:
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700159 return NULL;
160}
161
162/*
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700163 * Expand the file descriptor table.
164 * This function will allocate a new fdtable and both fd array and fdset, of
165 * the given size.
166 * Return <0 error code on error; 1 on successful completion.
167 * The files->file_lock should be held on entry, and will be held on exit.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700168 */
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300169static int expand_fdtable(struct files_struct *files, unsigned int nr)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700170 __releases(files->file_lock)
171 __acquires(files->file_lock)
172{
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700173 struct fdtable *new_fdt, *cur_fdt;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 spin_unlock(&files->file_lock);
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700176 new_fdt = alloc_fdtable(nr);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200177
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600178 /* make sure all fd_install() have seen resize_in_progress
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200179 * or have finished their rcu_read_lock_sched() section.
180 */
181 if (atomic_read(&files->count) > 1)
Paul E. McKenneyc93ffc12018-11-05 17:31:31 -0800182 synchronize_rcu();
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200183
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 spin_lock(&files->file_lock);
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700185 if (!new_fdt)
186 return -ENOMEM;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700187 /*
Al Viro5c598b32008-04-27 20:04:15 -0400188 * extremely unlikely race - sysctl_nr_open decreased between the check in
189 * caller and alloc_fdtable(). Cheaper to catch it here...
190 */
191 if (unlikely(new_fdt->max_fds <= nr)) {
Changli Gaoa892e2d2010-08-10 18:01:35 -0700192 __free_fdtable(new_fdt);
Al Viro5c598b32008-04-27 20:04:15 -0400193 return -EMFILE;
194 }
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700195 cur_fdt = files_fdtable(files);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200196 BUG_ON(nr < cur_fdt->max_fds);
197 copy_fdtable(new_fdt, cur_fdt);
198 rcu_assign_pointer(files->fdt, new_fdt);
199 if (cur_fdt != &files->fdtab)
200 call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600201 /* coupled with smp_rmb() in fd_install() */
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200202 smp_wmb();
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700203 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204}
205
206/*
207 * Expand files.
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700208 * This function will expand the file structures, if the requested size exceeds
209 * the current capacity and there is room for expansion.
210 * Return <0 error code on error; 0 when nothing done; 1 when files were
211 * expanded and execution may have blocked.
212 * The files->file_lock should be held on entry, and will be held on exit.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 */
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300214static int expand_files(struct files_struct *files, unsigned int nr)
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200215 __releases(files->file_lock)
216 __acquires(files->file_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217{
Dipankar Sarmabadf1662005-09-09 13:04:10 -0700218 struct fdtable *fdt;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200219 int expanded = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200221repeat:
Dipankar Sarmabadf1662005-09-09 13:04:10 -0700222 fdt = files_fdtable(files);
Al Viro4e1e0182008-07-26 16:01:20 -0400223
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700224 /* Do we need to expand? */
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800225 if (nr < fdt->max_fds)
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200226 return expanded;
Al Viro4e1e0182008-07-26 16:01:20 -0400227
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700228 /* Can we expand? */
Eric Dumazet9cfe0152008-02-06 01:37:16 -0800229 if (nr >= sysctl_nr_open)
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700230 return -EMFILE;
231
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200232 if (unlikely(files->resize_in_progress)) {
233 spin_unlock(&files->file_lock);
234 expanded = 1;
235 wait_event(files->resize_wait, !files->resize_in_progress);
236 spin_lock(&files->file_lock);
237 goto repeat;
238 }
239
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700240 /* All good, so we try */
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200241 files->resize_in_progress = true;
242 expanded = expand_fdtable(files, nr);
243 files->resize_in_progress = false;
244
245 wake_up_all(&files->resize_wait);
246 return expanded;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247}
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700248
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300249static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400250{
251 __set_bit(fd, fdt->close_on_exec);
252}
253
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300254static inline void __clear_close_on_exec(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400255{
Linus Torvaldsfc908882015-10-31 16:06:40 -0700256 if (test_bit(fd, fdt->close_on_exec))
257 __clear_bit(fd, fdt->close_on_exec);
Al Virob8318b02012-08-21 20:09:42 -0400258}
259
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700260static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400261{
262 __set_bit(fd, fdt->open_fds);
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700263 fd /= BITS_PER_LONG;
264 if (!~fdt->open_fds[fd])
265 __set_bit(fd, fdt->full_fds_bits);
Al Virob8318b02012-08-21 20:09:42 -0400266}
267
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700268static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400269{
270 __clear_bit(fd, fdt->open_fds);
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700271 __clear_bit(fd / BITS_PER_LONG, fdt->full_fds_bits);
Al Virob8318b02012-08-21 20:09:42 -0400272}
273
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300274static unsigned int count_open_files(struct fdtable *fdt)
Al Viro02afc6262008-05-08 19:42:56 -0400275{
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300276 unsigned int size = fdt->max_fds;
277 unsigned int i;
Al Viro02afc6262008-05-08 19:42:56 -0400278
279 /* Find the last open fd */
David Howells1fd36ad2012-02-16 17:49:54 +0000280 for (i = size / BITS_PER_LONG; i > 0; ) {
281 if (fdt->open_fds[--i])
Al Viro02afc6262008-05-08 19:42:56 -0400282 break;
283 }
David Howells1fd36ad2012-02-16 17:49:54 +0000284 i = (i + 1) * BITS_PER_LONG;
Al Viro02afc6262008-05-08 19:42:56 -0400285 return i;
286}
287
Linus Torvalds1c24a182022-03-29 15:06:39 -0700288/*
289 * Note that a sane fdtable size always has to be a multiple of
290 * BITS_PER_LONG, since we have bitmaps that are sized by this.
291 *
292 * 'max_fds' will normally already be properly aligned, but it
293 * turns out that in the close_range() -> __close_range() ->
294 * unshare_fd() -> dup_fd() -> sane_fdtable_size() we can end
295 * up having a 'max_fds' value that isn't already aligned.
296 *
297 * Rather than make close_range() have to worry about this,
298 * just make that BITS_PER_LONG alignment be part of a sane
299 * fdtable size. Becuase that's really what it is.
300 */
Christian Brauner60997c32020-06-03 21:48:55 +0200301static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
302{
303 unsigned int count;
304
305 count = count_open_files(fdt);
306 if (max_fds < NR_OPEN_DEFAULT)
307 max_fds = NR_OPEN_DEFAULT;
Linus Torvaldsd888c832022-03-29 23:29:18 -0700308 return ALIGN(min(count, max_fds), BITS_PER_LONG);
Christian Brauner60997c32020-06-03 21:48:55 +0200309}
310
Al Viro02afc6262008-05-08 19:42:56 -0400311/*
312 * Allocate a new files structure and copy contents from the
313 * passed in files structure.
314 * errorp will be valid only when the returned files_struct is NULL.
315 */
Christian Brauner60997c32020-06-03 21:48:55 +0200316struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, int *errorp)
Al Viro02afc6262008-05-08 19:42:56 -0400317{
318 struct files_struct *newf;
319 struct file **old_fds, **new_fds;
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300320 unsigned int open_files, i;
Al Viro02afc6262008-05-08 19:42:56 -0400321 struct fdtable *old_fdt, *new_fdt;
322
323 *errorp = -ENOMEM;
Al Viroafbec7f2008-05-08 21:11:17 -0400324 newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
Al Viro02afc6262008-05-08 19:42:56 -0400325 if (!newf)
326 goto out;
327
Al Viroafbec7f2008-05-08 21:11:17 -0400328 atomic_set(&newf->count, 1);
329
330 spin_lock_init(&newf->file_lock);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200331 newf->resize_in_progress = false;
332 init_waitqueue_head(&newf->resize_wait);
Al Viroafbec7f2008-05-08 21:11:17 -0400333 newf->next_fd = 0;
334 new_fdt = &newf->fdtab;
335 new_fdt->max_fds = NR_OPEN_DEFAULT;
David Howells1fd36ad2012-02-16 17:49:54 +0000336 new_fdt->close_on_exec = newf->close_on_exec_init;
337 new_fdt->open_fds = newf->open_fds_init;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700338 new_fdt->full_fds_bits = newf->full_fds_bits_init;
Al Viroafbec7f2008-05-08 21:11:17 -0400339 new_fdt->fd = &newf->fd_array[0];
Al Viroafbec7f2008-05-08 21:11:17 -0400340
Al Viro02afc6262008-05-08 19:42:56 -0400341 spin_lock(&oldf->file_lock);
342 old_fdt = files_fdtable(oldf);
Christian Brauner60997c32020-06-03 21:48:55 +0200343 open_files = sane_fdtable_size(old_fdt, max_fds);
Al Viro02afc6262008-05-08 19:42:56 -0400344
345 /*
346 * Check whether we need to allocate a larger fd array and fd set.
Al Viro02afc6262008-05-08 19:42:56 -0400347 */
Al Viroadbecb12008-05-08 21:19:42 -0400348 while (unlikely(open_files > new_fdt->max_fds)) {
Al Viro02afc6262008-05-08 19:42:56 -0400349 spin_unlock(&oldf->file_lock);
Al Viro9dec3c42008-05-08 21:02:45 -0400350
Changli Gaoa892e2d2010-08-10 18:01:35 -0700351 if (new_fdt != &newf->fdtab)
352 __free_fdtable(new_fdt);
Al Viroadbecb12008-05-08 21:19:42 -0400353
Al Viro9dec3c42008-05-08 21:02:45 -0400354 new_fdt = alloc_fdtable(open_files - 1);
355 if (!new_fdt) {
356 *errorp = -ENOMEM;
Al Viro02afc6262008-05-08 19:42:56 -0400357 goto out_release;
Al Viro9dec3c42008-05-08 21:02:45 -0400358 }
359
360 /* beyond sysctl_nr_open; nothing to do */
361 if (unlikely(new_fdt->max_fds < open_files)) {
Changli Gaoa892e2d2010-08-10 18:01:35 -0700362 __free_fdtable(new_fdt);
Al Viro9dec3c42008-05-08 21:02:45 -0400363 *errorp = -EMFILE;
364 goto out_release;
365 }
Al Viro9dec3c42008-05-08 21:02:45 -0400366
Al Viro02afc6262008-05-08 19:42:56 -0400367 /*
368 * Reacquire the oldf lock and a pointer to its fd table
369 * who knows it may have a new bigger fd table. We need
370 * the latest pointer.
371 */
372 spin_lock(&oldf->file_lock);
373 old_fdt = files_fdtable(oldf);
Christian Brauner60997c32020-06-03 21:48:55 +0200374 open_files = sane_fdtable_size(old_fdt, max_fds);
Al Viro02afc6262008-05-08 19:42:56 -0400375 }
376
Eric Biggersea5c58e2015-11-06 00:32:04 -0600377 copy_fd_bitmaps(new_fdt, old_fdt, open_files);
378
Al Viro02afc6262008-05-08 19:42:56 -0400379 old_fds = old_fdt->fd;
380 new_fds = new_fdt->fd;
381
Al Viro02afc6262008-05-08 19:42:56 -0400382 for (i = open_files; i != 0; i--) {
383 struct file *f = *old_fds++;
384 if (f) {
385 get_file(f);
386 } else {
387 /*
388 * The fd may be claimed in the fd bitmap but not yet
389 * instantiated in the files array if a sibling thread
390 * is partway through open(). So make sure that this
391 * fd is available to the new process.
392 */
David Howells1dce27c2012-02-16 17:49:42 +0000393 __clear_open_fd(open_files - i, new_fdt);
Al Viro02afc6262008-05-08 19:42:56 -0400394 }
395 rcu_assign_pointer(*new_fds++, f);
396 }
397 spin_unlock(&oldf->file_lock);
398
Eric Biggersea5c58e2015-11-06 00:32:04 -0600399 /* clear the remainder */
400 memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *));
Al Viro02afc6262008-05-08 19:42:56 -0400401
Al Viroafbec7f2008-05-08 21:11:17 -0400402 rcu_assign_pointer(newf->fdt, new_fdt);
403
Al Viro02afc6262008-05-08 19:42:56 -0400404 return newf;
405
406out_release:
407 kmem_cache_free(files_cachep, newf);
408out:
409 return NULL;
410}
411
Oleg Nesterovce08b622014-01-11 19:19:53 +0100412static struct fdtable *close_files(struct files_struct * files)
Al Viro7cf4dc32012-08-15 19:56:12 -0400413{
Al Viro7cf4dc32012-08-15 19:56:12 -0400414 /*
415 * It is safe to dereference the fd table without RCU or
416 * ->file_lock because this is the last reference to the
Oleg Nesterovce08b622014-01-11 19:19:53 +0100417 * files structure.
Al Viro7cf4dc32012-08-15 19:56:12 -0400418 */
Oleg Nesterovce08b622014-01-11 19:19:53 +0100419 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300420 unsigned int i, j = 0;
Oleg Nesterovce08b622014-01-11 19:19:53 +0100421
Al Viro7cf4dc32012-08-15 19:56:12 -0400422 for (;;) {
423 unsigned long set;
424 i = j * BITS_PER_LONG;
425 if (i >= fdt->max_fds)
426 break;
427 set = fdt->open_fds[j++];
428 while (set) {
429 if (set & 1) {
430 struct file * file = xchg(&fdt->fd[i], NULL);
431 if (file) {
432 filp_close(file, files);
Paul E. McKenney388a4c82017-10-24 08:39:34 -0700433 cond_resched();
Al Viro7cf4dc32012-08-15 19:56:12 -0400434 }
435 }
436 i++;
437 set >>= 1;
438 }
439 }
Oleg Nesterovce08b622014-01-11 19:19:53 +0100440
441 return fdt;
Al Viro7cf4dc32012-08-15 19:56:12 -0400442}
443
Al Viro7cf4dc32012-08-15 19:56:12 -0400444void put_files_struct(struct files_struct *files)
445{
Al Viro7cf4dc32012-08-15 19:56:12 -0400446 if (atomic_dec_and_test(&files->count)) {
Oleg Nesterovce08b622014-01-11 19:19:53 +0100447 struct fdtable *fdt = close_files(files);
448
Al Virob9e02af2012-08-15 20:00:58 -0400449 /* free the arrays if they are not embedded */
450 if (fdt != &files->fdtab)
451 __free_fdtable(fdt);
452 kmem_cache_free(files_cachep, files);
Al Viro7cf4dc32012-08-15 19:56:12 -0400453 }
454}
455
Al Viro7cf4dc32012-08-15 19:56:12 -0400456void exit_files(struct task_struct *tsk)
457{
458 struct files_struct * files = tsk->files;
459
460 if (files) {
461 task_lock(tsk);
462 tsk->files = NULL;
463 task_unlock(tsk);
464 put_files_struct(files);
465 }
466}
467
Al Virof52111b2008-05-08 18:19:16 -0400468struct files_struct init_files = {
469 .count = ATOMIC_INIT(1),
470 .fdt = &init_files.fdtab,
471 .fdtab = {
472 .max_fds = NR_OPEN_DEFAULT,
473 .fd = &init_files.fd_array[0],
David Howells1fd36ad2012-02-16 17:49:54 +0000474 .close_on_exec = init_files.close_on_exec_init,
475 .open_fds = init_files.open_fds_init,
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700476 .full_fds_bits = init_files.full_fds_bits_init,
Al Virof52111b2008-05-08 18:19:16 -0400477 },
Thomas Gleixnereece09e2011-07-17 21:25:03 +0200478 .file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock),
Shuriyc Chu5704a062019-03-05 15:41:56 -0800479 .resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait),
Al Virof52111b2008-05-08 18:19:16 -0400480};
Al Viro1027abe2008-07-30 04:13:04 -0400481
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300482static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700483{
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300484 unsigned int maxfd = fdt->max_fds;
485 unsigned int maxbit = maxfd / BITS_PER_LONG;
486 unsigned int bitbit = start / BITS_PER_LONG;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700487
488 bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;
489 if (bitbit > maxfd)
490 return maxfd;
491 if (bitbit > start)
492 start = bitbit;
493 return find_next_zero_bit(fdt->open_fds, maxfd, start);
494}
495
Al Viro1027abe2008-07-30 04:13:04 -0400496/*
497 * allocate a file descriptor, mark it busy.
498 */
Eric W. Biedermanaa384d12020-11-20 17:14:37 -0600499static int alloc_fd(unsigned start, unsigned end, unsigned flags)
Al Viro1027abe2008-07-30 04:13:04 -0400500{
Eric W. Biedermanaa384d12020-11-20 17:14:37 -0600501 struct files_struct *files = current->files;
Al Viro1027abe2008-07-30 04:13:04 -0400502 unsigned int fd;
503 int error;
504 struct fdtable *fdt;
505
506 spin_lock(&files->file_lock);
507repeat:
508 fdt = files_fdtable(files);
509 fd = start;
510 if (fd < files->next_fd)
511 fd = files->next_fd;
512
513 if (fd < fdt->max_fds)
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700514 fd = find_next_fd(fdt, fd);
Al Viro1027abe2008-07-30 04:13:04 -0400515
Al Virof33ff992012-08-12 16:17:59 -0400516 /*
517 * N.B. For clone tasks sharing a files structure, this test
518 * will limit the total number of files that can be opened.
519 */
520 error = -EMFILE;
521 if (fd >= end)
522 goto out;
523
Al Viro1027abe2008-07-30 04:13:04 -0400524 error = expand_files(files, fd);
525 if (error < 0)
526 goto out;
527
528 /*
529 * If we needed to expand the fs array we
530 * might have blocked - try again.
531 */
532 if (error)
533 goto repeat;
534
535 if (start <= files->next_fd)
536 files->next_fd = fd + 1;
537
David Howells1dce27c2012-02-16 17:49:42 +0000538 __set_open_fd(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400539 if (flags & O_CLOEXEC)
David Howells1dce27c2012-02-16 17:49:42 +0000540 __set_close_on_exec(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400541 else
David Howells1dce27c2012-02-16 17:49:42 +0000542 __clear_close_on_exec(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400543 error = fd;
544#if 1
545 /* Sanity check */
Paul E. McKenneyadd1f092014-02-12 12:51:09 -0800546 if (rcu_access_pointer(fdt->fd[fd]) != NULL) {
Al Viro1027abe2008-07-30 04:13:04 -0400547 printk(KERN_WARNING "alloc_fd: slot %d not NULL!\n", fd);
548 rcu_assign_pointer(fdt->fd[fd], NULL);
549 }
550#endif
551
552out:
553 spin_unlock(&files->file_lock);
554 return error;
555}
556
Jens Axboe4022e7a2020-03-19 19:23:18 -0600557int __get_unused_fd_flags(unsigned flags, unsigned long nofile)
558{
Eric W. Biedermanaa384d12020-11-20 17:14:37 -0600559 return alloc_fd(0, nofile, flags);
Jens Axboe4022e7a2020-03-19 19:23:18 -0600560}
561
Al Viro1a7bd222012-08-12 17:18:05 -0400562int get_unused_fd_flags(unsigned flags)
Al Viro1027abe2008-07-30 04:13:04 -0400563{
Jens Axboe4022e7a2020-03-19 19:23:18 -0600564 return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE));
Al Viro1027abe2008-07-30 04:13:04 -0400565}
Al Viro1a7bd222012-08-12 17:18:05 -0400566EXPORT_SYMBOL(get_unused_fd_flags);
Al Viro56007ca2012-08-15 21:03:26 -0400567
568static void __put_unused_fd(struct files_struct *files, unsigned int fd)
569{
570 struct fdtable *fdt = files_fdtable(files);
571 __clear_open_fd(fd, fdt);
572 if (fd < files->next_fd)
573 files->next_fd = fd;
574}
575
576void put_unused_fd(unsigned int fd)
577{
578 struct files_struct *files = current->files;
579 spin_lock(&files->file_lock);
580 __put_unused_fd(files, fd);
581 spin_unlock(&files->file_lock);
582}
583
584EXPORT_SYMBOL(put_unused_fd);
585
586/*
587 * Install a file pointer in the fd array.
588 *
589 * The VFS is full of places where we drop the files lock between
590 * setting the open_fds bitmap and installing the file in the file
591 * array. At any such point, we are vulnerable to a dup2() race
592 * installing a file in the array before us. We need to detect this and
593 * fput() the struct file we are about to overwrite in this case.
594 *
595 * It should never happen - if we allow dup2() do it, _really_ bad things
596 * will follow.
Al Virof869e8a2012-08-15 21:06:33 -0400597 *
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600598 * This consumes the "file" refcount, so callers should treat it
599 * as if they had called fput(file).
Al Viro56007ca2012-08-15 21:03:26 -0400600 */
601
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600602void fd_install(unsigned int fd, struct file *file)
Al Viro56007ca2012-08-15 21:03:26 -0400603{
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600604 struct files_struct *files = current->files;
Al Viro56007ca2012-08-15 21:03:26 -0400605 struct fdtable *fdt;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200606
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200607 rcu_read_lock_sched();
608
Mateusz Guzikc02b1a92017-10-03 12:58:15 +0200609 if (unlikely(files->resize_in_progress)) {
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200610 rcu_read_unlock_sched();
Mateusz Guzikc02b1a92017-10-03 12:58:15 +0200611 spin_lock(&files->file_lock);
612 fdt = files_fdtable(files);
613 BUG_ON(fdt->fd[fd] != NULL);
614 rcu_assign_pointer(fdt->fd[fd], file);
615 spin_unlock(&files->file_lock);
616 return;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200617 }
618 /* coupled with smp_wmb() in expand_fdtable() */
619 smp_rmb();
620 fdt = rcu_dereference_sched(files->fdt);
Al Viro56007ca2012-08-15 21:03:26 -0400621 BUG_ON(fdt->fd[fd] != NULL);
622 rcu_assign_pointer(fdt->fd[fd], file);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200623 rcu_read_unlock_sched();
Al Viro56007ca2012-08-15 21:03:26 -0400624}
625
626EXPORT_SYMBOL(fd_install);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400627
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200628/**
629 * pick_file - return file associatd with fd
630 * @files: file struct to retrieve file from
631 * @fd: file descriptor to retrieve file for
632 *
Al Viro6319194e2022-05-12 17:08:03 -0400633 * Context: files_lock must be held.
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200634 *
Al Viro6319194e2022-05-12 17:08:03 -0400635 * Returns: The file associated with @fd (NULL if @fd is not open)
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200636 */
Christian Brauner278a5fb2019-05-24 11:30:34 +0200637static struct file *pick_file(struct files_struct *files, unsigned fd)
Al Viro483ce1d2012-08-19 12:04:24 -0400638{
Al Viro6319194e2022-05-12 17:08:03 -0400639 struct fdtable *fdt = files_fdtable(files);
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200640 struct file *file;
Al Viro483ce1d2012-08-19 12:04:24 -0400641
Al Viro6319194e2022-05-12 17:08:03 -0400642 if (fd >= fdt->max_fds)
643 return NULL;
644
Theodore Ts'o609d5442023-03-06 13:54:50 -0500645 fd = array_index_nospec(fd, fdt->max_fds);
Al Viro483ce1d2012-08-19 12:04:24 -0400646 file = fdt->fd[fd];
Al Viro6319194e2022-05-12 17:08:03 -0400647 if (file) {
648 rcu_assign_pointer(fdt->fd[fd], NULL);
649 __put_unused_fd(files, fd);
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200650 }
Christian Brauner278a5fb2019-05-24 11:30:34 +0200651 return file;
652}
653
Eric W. Biederman8760c902020-11-20 17:14:38 -0600654int close_fd(unsigned fd)
Christian Brauner278a5fb2019-05-24 11:30:34 +0200655{
Eric W. Biederman8760c902020-11-20 17:14:38 -0600656 struct files_struct *files = current->files;
Christian Brauner278a5fb2019-05-24 11:30:34 +0200657 struct file *file;
658
Al Viro6319194e2022-05-12 17:08:03 -0400659 spin_lock(&files->file_lock);
Christian Brauner278a5fb2019-05-24 11:30:34 +0200660 file = pick_file(files, fd);
Al Viro6319194e2022-05-12 17:08:03 -0400661 spin_unlock(&files->file_lock);
662 if (!file)
Christian Brauner278a5fb2019-05-24 11:30:34 +0200663 return -EBADF;
664
665 return filp_close(file, files);
Al Viro483ce1d2012-08-19 12:04:24 -0400666}
Eric W. Biederman8760c902020-11-20 17:14:38 -0600667EXPORT_SYMBOL(close_fd); /* for ksys_close() */
Al Viro483ce1d2012-08-19 12:04:24 -0400668
Christian Brauner9b5b87222021-04-02 10:29:36 +0200669/**
670 * last_fd - return last valid index into fd table
671 * @cur_fds: files struct
672 *
673 * Context: Either rcu read lock or files_lock must be held.
674 *
675 * Returns: Last valid index into fdtable.
676 */
677static inline unsigned last_fd(struct fdtable *fdt)
678{
679 return fdt->max_fds - 1;
680}
681
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100682static inline void __range_cloexec(struct files_struct *cur_fds,
683 unsigned int fd, unsigned int max_fd)
684{
685 struct fdtable *fdt;
686
Christian Brauner9b5b87222021-04-02 10:29:36 +0200687 /* make sure we're using the correct maximum value */
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100688 spin_lock(&cur_fds->file_lock);
689 fdt = files_fdtable(cur_fds);
Christian Brauner9b5b87222021-04-02 10:29:36 +0200690 max_fd = min(last_fd(fdt), max_fd);
691 if (fd <= max_fd)
692 bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100693 spin_unlock(&cur_fds->file_lock);
694}
695
696static inline void __range_close(struct files_struct *cur_fds, unsigned int fd,
697 unsigned int max_fd)
698{
Al Viro6319194e2022-05-12 17:08:03 -0400699 unsigned n;
700
701 rcu_read_lock();
702 n = last_fd(files_fdtable(cur_fds));
703 rcu_read_unlock();
704 max_fd = min(max_fd, n);
705
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100706 while (fd <= max_fd) {
707 struct file *file;
708
Al Viro6319194e2022-05-12 17:08:03 -0400709 spin_lock(&cur_fds->file_lock);
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100710 file = pick_file(cur_fds, fd++);
Al Viro6319194e2022-05-12 17:08:03 -0400711 spin_unlock(&cur_fds->file_lock);
712
713 if (file) {
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200714 /* found a valid file to close */
715 filp_close(file, cur_fds);
716 cond_resched();
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200717 }
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100718 }
719}
720
Christian Brauner278a5fb2019-05-24 11:30:34 +0200721/**
722 * __close_range() - Close all file descriptors in a given range.
723 *
724 * @fd: starting file descriptor to close
725 * @max_fd: last file descriptor to close
726 *
727 * This closes a range of file descriptors. All file descriptors
728 * from @fd up to and including @max_fd are closed.
729 */
Christian Brauner60997c32020-06-03 21:48:55 +0200730int __close_range(unsigned fd, unsigned max_fd, unsigned int flags)
Christian Brauner278a5fb2019-05-24 11:30:34 +0200731{
Christian Brauner60997c32020-06-03 21:48:55 +0200732 struct task_struct *me = current;
733 struct files_struct *cur_fds = me->files, *fds = NULL;
734
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100735 if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC))
Christian Brauner60997c32020-06-03 21:48:55 +0200736 return -EINVAL;
Christian Brauner278a5fb2019-05-24 11:30:34 +0200737
738 if (fd > max_fd)
739 return -EINVAL;
740
Christian Brauner60997c32020-06-03 21:48:55 +0200741 if (flags & CLOSE_RANGE_UNSHARE) {
742 int ret;
743 unsigned int max_unshare_fds = NR_OPEN_MAX;
744
745 /*
Christian Brauner03ba0fe2021-04-02 10:38:09 +0200746 * If the caller requested all fds to be made cloexec we always
747 * copy all of the file descriptors since they still want to
748 * use them.
Christian Brauner60997c32020-06-03 21:48:55 +0200749 */
Christian Brauner03ba0fe2021-04-02 10:38:09 +0200750 if (!(flags & CLOSE_RANGE_CLOEXEC)) {
751 /*
752 * If the requested range is greater than the current
753 * maximum, we're closing everything so only copy all
754 * file descriptors beneath the lowest file descriptor.
755 */
756 rcu_read_lock();
757 if (max_fd >= last_fd(files_fdtable(cur_fds)))
758 max_unshare_fds = fd;
759 rcu_read_unlock();
760 }
Christian Brauner60997c32020-06-03 21:48:55 +0200761
762 ret = unshare_fd(CLONE_FILES, max_unshare_fds, &fds);
763 if (ret)
764 return ret;
765
766 /*
767 * We used to share our file descriptor table, and have now
768 * created a private one, make sure we're using it below.
769 */
770 if (fds)
771 swap(cur_fds, fds);
772 }
773
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100774 if (flags & CLOSE_RANGE_CLOEXEC)
775 __range_cloexec(cur_fds, fd, max_fd);
776 else
777 __range_close(cur_fds, fd, max_fd);
Christian Brauner278a5fb2019-05-24 11:30:34 +0200778
Christian Brauner60997c32020-06-03 21:48:55 +0200779 if (fds) {
780 /*
781 * We're done closing the files we were supposed to. Time to install
782 * the new file descriptor table and drop the old one.
783 */
784 task_lock(me);
785 me->files = cur_fds;
786 task_unlock(me);
787 put_files_struct(fds);
788 }
789
Christian Brauner278a5fb2019-05-24 11:30:34 +0200790 return 0;
791}
792
Todd Kjos80cd7952018-12-14 15:58:21 -0800793/*
Jens Axboe53dec2e2021-01-19 15:41:52 -0700794 * See close_fd_get_file() below, this variant assumes current->files->file_lock
795 * is held.
796 */
Al Viro6319194e2022-05-12 17:08:03 -0400797struct file *__close_fd_get_file(unsigned int fd)
Jens Axboe53dec2e2021-01-19 15:41:52 -0700798{
Al Viro6319194e2022-05-12 17:08:03 -0400799 return pick_file(current->files, fd);
Jens Axboe53dec2e2021-01-19 15:41:52 -0700800}
801
802/*
Eric W. Biederman9fe83c42020-11-20 17:14:40 -0600803 * variant of close_fd that gets a ref on the file for later fput.
Al Viro40a19262022-06-05 14:01:42 -0400804 * The caller must ensure that filp_close() called on the file.
Todd Kjos80cd7952018-12-14 15:58:21 -0800805 */
Al Viro6319194e2022-05-12 17:08:03 -0400806struct file *close_fd_get_file(unsigned int fd)
Todd Kjos80cd7952018-12-14 15:58:21 -0800807{
808 struct files_struct *files = current->files;
Al Viro6319194e2022-05-12 17:08:03 -0400809 struct file *file;
Todd Kjos80cd7952018-12-14 15:58:21 -0800810
811 spin_lock(&files->file_lock);
Al Viro6319194e2022-05-12 17:08:03 -0400812 file = pick_file(files, fd);
Todd Kjos80cd7952018-12-14 15:58:21 -0800813 spin_unlock(&files->file_lock);
Todd Kjos80cd7952018-12-14 15:58:21 -0800814
Al Viro6319194e2022-05-12 17:08:03 -0400815 return file;
Todd Kjos80cd7952018-12-14 15:58:21 -0800816}
817
Al Viro6a6d27d2012-08-21 09:56:33 -0400818void do_close_on_exec(struct files_struct *files)
819{
820 unsigned i;
821 struct fdtable *fdt;
822
823 /* exec unshares first */
Al Viro6a6d27d2012-08-21 09:56:33 -0400824 spin_lock(&files->file_lock);
825 for (i = 0; ; i++) {
826 unsigned long set;
827 unsigned fd = i * BITS_PER_LONG;
828 fdt = files_fdtable(files);
829 if (fd >= fdt->max_fds)
830 break;
831 set = fdt->close_on_exec[i];
832 if (!set)
833 continue;
834 fdt->close_on_exec[i] = 0;
835 for ( ; set ; fd++, set >>= 1) {
836 struct file *file;
837 if (!(set & 1))
838 continue;
839 file = fdt->fd[fd];
840 if (!file)
841 continue;
842 rcu_assign_pointer(fdt->fd[fd], NULL);
843 __put_unused_fd(files, fd);
844 spin_unlock(&files->file_lock);
845 filp_close(file, files);
846 cond_resched();
847 spin_lock(&files->file_lock);
848 }
849
850 }
851 spin_unlock(&files->file_lock);
852}
853
Linus Torvaldse386dfc2021-12-10 14:00:15 -0800854static inline struct file *__fget_files_rcu(struct files_struct *files,
Gou Hao81132a32021-11-02 10:46:48 +0800855 unsigned int fd, fmode_t mask)
Linus Torvaldse386dfc2021-12-10 14:00:15 -0800856{
857 for (;;) {
858 struct file *file;
859 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
860 struct file __rcu **fdentry;
861
862 if (unlikely(fd >= fdt->max_fds))
863 return NULL;
864
865 fdentry = fdt->fd + array_index_nospec(fd, fdt->max_fds);
866 file = rcu_dereference_raw(*fdentry);
867 if (unlikely(!file))
868 return NULL;
869
870 if (unlikely(file->f_mode & mask))
871 return NULL;
872
873 /*
874 * Ok, we have a file pointer. However, because we do
875 * this all locklessly under RCU, we may be racing with
876 * that file being closed.
877 *
878 * Such a race can take two forms:
879 *
880 * (a) the file ref already went down to zero,
Gou Hao81132a32021-11-02 10:46:48 +0800881 * and get_file_rcu() fails. Just try again:
Linus Torvaldse386dfc2021-12-10 14:00:15 -0800882 */
Gou Hao81132a32021-11-02 10:46:48 +0800883 if (unlikely(!get_file_rcu(file)))
Linus Torvaldse386dfc2021-12-10 14:00:15 -0800884 continue;
885
886 /*
887 * (b) the file table entry has changed under us.
888 * Note that we don't need to re-check the 'fdt->fd'
889 * pointer having changed, because it always goes
890 * hand-in-hand with 'fdt'.
891 *
Gou Hao81132a32021-11-02 10:46:48 +0800892 * If so, we need to put our ref and try again.
Linus Torvaldse386dfc2021-12-10 14:00:15 -0800893 */
894 if (unlikely(rcu_dereference_raw(files->fdt) != fdt) ||
895 unlikely(rcu_dereference_raw(*fdentry) != file)) {
Gou Hao81132a32021-11-02 10:46:48 +0800896 fput(file);
Linus Torvaldse386dfc2021-12-10 14:00:15 -0800897 continue;
898 }
899
900 /*
901 * Ok, we have a ref to the file, and checked that it
902 * still exists.
903 */
904 return file;
905 }
906}
907
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800908static struct file *__fget_files(struct files_struct *files, unsigned int fd,
Gou Hao81132a32021-11-02 10:46:48 +0800909 fmode_t mask)
Al Viro0ee8cdf2012-08-15 21:12:10 -0400910{
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100911 struct file *file;
Al Viro0ee8cdf2012-08-15 21:12:10 -0400912
913 rcu_read_lock();
Gou Hao81132a32021-11-02 10:46:48 +0800914 file = __fget_files_rcu(files, fd, mask);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400915 rcu_read_unlock();
916
917 return file;
918}
919
Gou Hao81132a32021-11-02 10:46:48 +0800920static inline struct file *__fget(unsigned int fd, fmode_t mask)
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800921{
Gou Hao81132a32021-11-02 10:46:48 +0800922 return __fget_files(current->files, fd, mask);
Jens Axboe091141a2018-11-21 10:32:39 -0700923}
924
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100925struct file *fget(unsigned int fd)
926{
Gou Hao81132a32021-11-02 10:46:48 +0800927 return __fget(fd, FMODE_PATH);
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100928}
Al Viro0ee8cdf2012-08-15 21:12:10 -0400929EXPORT_SYMBOL(fget);
930
931struct file *fget_raw(unsigned int fd)
932{
Gou Hao81132a32021-11-02 10:46:48 +0800933 return __fget(fd, 0);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400934}
Al Viro0ee8cdf2012-08-15 21:12:10 -0400935EXPORT_SYMBOL(fget_raw);
936
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800937struct file *fget_task(struct task_struct *task, unsigned int fd)
938{
939 struct file *file = NULL;
940
941 task_lock(task);
942 if (task->files)
Gou Hao81132a32021-11-02 10:46:48 +0800943 file = __fget_files(task->files, fd, 0);
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800944 task_unlock(task);
945
946 return file;
947}
948
Eric W. Biederman3a879fb2020-11-20 17:14:28 -0600949struct file *task_lookup_fd_rcu(struct task_struct *task, unsigned int fd)
950{
951 /* Must be called with rcu_read_lock held */
952 struct files_struct *files;
953 struct file *file = NULL;
954
955 task_lock(task);
956 files = task->files;
957 if (files)
958 file = files_lookup_fd_rcu(files, fd);
959 task_unlock(task);
960
961 return file;
962}
963
Eric W. Biedermane9a53ae2020-11-20 17:14:31 -0600964struct file *task_lookup_next_fd_rcu(struct task_struct *task, unsigned int *ret_fd)
965{
966 /* Must be called with rcu_read_lock held */
967 struct files_struct *files;
968 unsigned int fd = *ret_fd;
969 struct file *file = NULL;
970
971 task_lock(task);
972 files = task->files;
973 if (files) {
974 for (; fd < files_fdtable(files)->max_fds; fd++) {
975 file = files_lookup_fd_rcu(files, fd);
976 if (file)
977 break;
978 }
979 }
980 task_unlock(task);
981 *ret_fd = fd;
982 return file;
983}
Andreas Gruenbacher4480c272022-06-08 16:22:55 +0200984EXPORT_SYMBOL(task_lookup_next_fd_rcu);
Eric W. Biedermane9a53ae2020-11-20 17:14:31 -0600985
Al Viro0ee8cdf2012-08-15 21:12:10 -0400986/*
987 * Lightweight file lookup - no refcnt increment if fd table isn't shared.
988 *
989 * You can use this instead of fget if you satisfy all of the following
990 * conditions:
991 * 1) You must call fput_light before exiting the syscall and returning control
992 * to userspace (i.e. you cannot remember the returned struct file * after
993 * returning to userspace).
994 * 2) You must not call filp_close on the returned struct file * in between
995 * calls to fget_light and fput_light.
996 * 3) You must not clone the current task in between the calls to fget_light
997 * and fput_light.
998 *
999 * The fput_needed flag returned by fget_light should be passed to the
1000 * corresponding fput_light.
1001 */
Al Virobd2a31d2014-03-04 14:54:22 -05001002static unsigned long __fget_light(unsigned int fd, fmode_t mask)
Al Viro0ee8cdf2012-08-15 21:12:10 -04001003{
Al Viro0ee8cdf2012-08-15 21:12:10 -04001004 struct files_struct *files = current->files;
Oleg Nesterovad461832014-01-13 16:48:40 +01001005 struct file *file;
Al Viro0ee8cdf2012-08-15 21:12:10 -04001006
Jann Horn7ee47dc2022-10-31 18:52:56 +01001007 /*
1008 * If another thread is concurrently calling close_fd() followed
1009 * by put_files_struct(), we must not observe the old table
1010 * entry combined with the new refcount - otherwise we could
1011 * return a file that is concurrently being freed.
1012 *
1013 * atomic_read_acquire() pairs with atomic_dec_and_test() in
1014 * put_files_struct().
1015 */
1016 if (atomic_read_acquire(&files->count) == 1) {
Eric W. Biedermanbebf6842020-11-20 17:14:24 -06001017 file = files_lookup_fd_raw(files, fd);
Al Virobd2a31d2014-03-04 14:54:22 -05001018 if (!file || unlikely(file->f_mode & mask))
1019 return 0;
1020 return (unsigned long)file;
Al Viro0ee8cdf2012-08-15 21:12:10 -04001021 } else {
Gou Hao81132a32021-11-02 10:46:48 +08001022 file = __fget(fd, mask);
Al Virobd2a31d2014-03-04 14:54:22 -05001023 if (!file)
1024 return 0;
1025 return FDPUT_FPUT | (unsigned long)file;
Al Viro0ee8cdf2012-08-15 21:12:10 -04001026 }
Al Viro0ee8cdf2012-08-15 21:12:10 -04001027}
Al Virobd2a31d2014-03-04 14:54:22 -05001028unsigned long __fdget(unsigned int fd)
Oleg Nesterovad461832014-01-13 16:48:40 +01001029{
Al Virobd2a31d2014-03-04 14:54:22 -05001030 return __fget_light(fd, FMODE_PATH);
Oleg Nesterovad461832014-01-13 16:48:40 +01001031}
Al Virobd2a31d2014-03-04 14:54:22 -05001032EXPORT_SYMBOL(__fdget);
Al Viro0ee8cdf2012-08-15 21:12:10 -04001033
Al Virobd2a31d2014-03-04 14:54:22 -05001034unsigned long __fdget_raw(unsigned int fd)
Al Viro0ee8cdf2012-08-15 21:12:10 -04001035{
Al Virobd2a31d2014-03-04 14:54:22 -05001036 return __fget_light(fd, 0);
Al Viro0ee8cdf2012-08-15 21:12:10 -04001037}
Al Virofe17f222012-08-21 11:48:11 -04001038
Al Virobd2a31d2014-03-04 14:54:22 -05001039unsigned long __fdget_pos(unsigned int fd)
1040{
Eric Biggers99aea682014-03-16 15:47:48 -05001041 unsigned long v = __fdget(fd);
1042 struct file *file = (struct file *)(v & ~3);
Al Virobd2a31d2014-03-04 14:54:22 -05001043
Linus Torvalds2be7d342019-11-26 11:34:06 -08001044 if (file && (file->f_mode & FMODE_ATOMIC_POS)) {
Al Virobd2a31d2014-03-04 14:54:22 -05001045 if (file_count(file) > 1) {
1046 v |= FDPUT_POS_UNLOCK;
1047 mutex_lock(&file->f_pos_lock);
1048 }
1049 }
Eric Biggers99aea682014-03-16 15:47:48 -05001050 return v;
Al Virobd2a31d2014-03-04 14:54:22 -05001051}
1052
Al Viro63b6df12016-04-20 17:08:21 -04001053void __f_unlock_pos(struct file *f)
1054{
1055 mutex_unlock(&f->f_pos_lock);
1056}
1057
Al Virobd2a31d2014-03-04 14:54:22 -05001058/*
1059 * We only lock f_pos if we have threads or if the file might be
1060 * shared with another process. In both cases we'll have an elevated
1061 * file count (done either by fdget() or by fork()).
1062 */
1063
Al Virofe17f222012-08-21 11:48:11 -04001064void set_close_on_exec(unsigned int fd, int flag)
1065{
1066 struct files_struct *files = current->files;
1067 struct fdtable *fdt;
1068 spin_lock(&files->file_lock);
1069 fdt = files_fdtable(files);
1070 if (flag)
1071 __set_close_on_exec(fd, fdt);
1072 else
1073 __clear_close_on_exec(fd, fdt);
1074 spin_unlock(&files->file_lock);
1075}
1076
1077bool get_close_on_exec(unsigned int fd)
1078{
1079 struct files_struct *files = current->files;
1080 struct fdtable *fdt;
1081 bool res;
1082 rcu_read_lock();
1083 fdt = files_fdtable(files);
1084 res = close_on_exec(fd, fdt);
1085 rcu_read_unlock();
1086 return res;
1087}
1088
Al Viro8280d162012-08-21 12:11:46 -04001089static int do_dup2(struct files_struct *files,
1090 struct file *file, unsigned fd, unsigned flags)
Al Viroe9830942014-08-31 14:12:09 -04001091__releases(&files->file_lock)
Al Viro8280d162012-08-21 12:11:46 -04001092{
1093 struct file *tofree;
1094 struct fdtable *fdt;
1095
1096 /*
1097 * We need to detect attempts to do dup2() over allocated but still
1098 * not finished descriptor. NB: OpenBSD avoids that at the price of
1099 * extra work in their equivalent of fget() - they insert struct
1100 * file immediately after grabbing descriptor, mark it larval if
1101 * more work (e.g. actual opening) is needed and make sure that
1102 * fget() treats larval files as absent. Potentially interesting,
1103 * but while extra work in fget() is trivial, locking implications
1104 * and amount of surgery on open()-related paths in VFS are not.
1105 * FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
1106 * deadlocks in rather amusing ways, AFAICS. All of that is out of
1107 * scope of POSIX or SUS, since neither considers shared descriptor
1108 * tables and this condition does not arise without those.
1109 */
1110 fdt = files_fdtable(files);
1111 tofree = fdt->fd[fd];
1112 if (!tofree && fd_is_open(fd, fdt))
1113 goto Ebusy;
1114 get_file(file);
1115 rcu_assign_pointer(fdt->fd[fd], file);
1116 __set_open_fd(fd, fdt);
1117 if (flags & O_CLOEXEC)
1118 __set_close_on_exec(fd, fdt);
1119 else
1120 __clear_close_on_exec(fd, fdt);
1121 spin_unlock(&files->file_lock);
1122
1123 if (tofree)
1124 filp_close(tofree, files);
1125
1126 return fd;
1127
1128Ebusy:
1129 spin_unlock(&files->file_lock);
1130 return -EBUSY;
1131}
1132
1133int replace_fd(unsigned fd, struct file *file, unsigned flags)
1134{
1135 int err;
1136 struct files_struct *files = current->files;
1137
1138 if (!file)
Eric W. Biederman8760c902020-11-20 17:14:38 -06001139 return close_fd(fd);
Al Viro8280d162012-08-21 12:11:46 -04001140
1141 if (fd >= rlimit(RLIMIT_NOFILE))
Al Viro08f05c42012-10-31 03:37:48 +00001142 return -EBADF;
Al Viro8280d162012-08-21 12:11:46 -04001143
1144 spin_lock(&files->file_lock);
1145 err = expand_files(files, fd);
1146 if (unlikely(err < 0))
1147 goto out_unlock;
1148 return do_dup2(files, file, fd, flags);
1149
1150out_unlock:
1151 spin_unlock(&files->file_lock);
1152 return err;
1153}
1154
Kees Cook66590612020-06-10 08:20:05 -07001155/**
1156 * __receive_fd() - Install received file into file descriptor table
Kees Cook66590612020-06-10 08:20:05 -07001157 * @file: struct file that was received from another process
1158 * @ufd: __user pointer to write new fd number to
1159 * @o_flags: the O_* flags to apply to the new fd entry
1160 *
1161 * Installs a received file into the file descriptor table, with appropriate
Kees Cookdeefa7f2020-06-10 20:47:45 -07001162 * checks and count updates. Optionally writes the fd number to userspace, if
1163 * @ufd is non-NULL.
Kees Cook66590612020-06-10 08:20:05 -07001164 *
1165 * This helper handles its own reference counting of the incoming
1166 * struct file.
1167 *
Kees Cookdeefa7f2020-06-10 20:47:45 -07001168 * Returns newly install fd or -ve on error.
Kees Cook66590612020-06-10 08:20:05 -07001169 */
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001170int __receive_fd(struct file *file, int __user *ufd, unsigned int o_flags)
Kees Cook66590612020-06-10 08:20:05 -07001171{
1172 int new_fd;
1173 int error;
1174
1175 error = security_file_receive(file);
1176 if (error)
1177 return error;
1178
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001179 new_fd = get_unused_fd_flags(o_flags);
1180 if (new_fd < 0)
1181 return new_fd;
Kees Cook66590612020-06-10 08:20:05 -07001182
Kees Cookdeefa7f2020-06-10 20:47:45 -07001183 if (ufd) {
1184 error = put_user(new_fd, ufd);
1185 if (error) {
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001186 put_unused_fd(new_fd);
Kees Cookdeefa7f2020-06-10 20:47:45 -07001187 return error;
1188 }
Kees Cook66590612020-06-10 08:20:05 -07001189 }
1190
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001191 fd_install(new_fd, get_file(file));
1192 __receive_sock(file);
1193 return new_fd;
1194}
Kees Cook173817152020-06-10 08:46:58 -07001195
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001196int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags)
1197{
1198 int error;
1199
1200 error = security_file_receive(file);
1201 if (error)
1202 return error;
1203 error = replace_fd(new_fd, file, o_flags);
1204 if (error)
1205 return error;
Kees Cook66590612020-06-10 08:20:05 -07001206 __receive_sock(file);
Kees Cookdeefa7f2020-06-10 20:47:45 -07001207 return new_fd;
Kees Cook66590612020-06-10 08:20:05 -07001208}
1209
Xie Yongji9c930052021-08-31 18:36:24 +08001210int receive_fd(struct file *file, unsigned int o_flags)
1211{
1212 return __receive_fd(file, NULL, o_flags);
1213}
1214EXPORT_SYMBOL_GPL(receive_fd);
1215
Dominik Brodowskic7248322018-03-11 11:34:40 +01001216static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
Al Virofe17f222012-08-21 11:48:11 -04001217{
1218 int err = -EBADF;
Al Viro8280d162012-08-21 12:11:46 -04001219 struct file *file;
1220 struct files_struct *files = current->files;
Al Virofe17f222012-08-21 11:48:11 -04001221
1222 if ((flags & ~O_CLOEXEC) != 0)
1223 return -EINVAL;
1224
Richard W.M. Jonesaed97642012-10-09 15:27:43 +01001225 if (unlikely(oldfd == newfd))
1226 return -EINVAL;
1227
Al Virofe17f222012-08-21 11:48:11 -04001228 if (newfd >= rlimit(RLIMIT_NOFILE))
Al Viro08f05c42012-10-31 03:37:48 +00001229 return -EBADF;
Al Virofe17f222012-08-21 11:48:11 -04001230
1231 spin_lock(&files->file_lock);
1232 err = expand_files(files, newfd);
Eric W. Biederman120ce2b2020-11-20 17:14:25 -06001233 file = files_lookup_fd_locked(files, oldfd);
Al Virofe17f222012-08-21 11:48:11 -04001234 if (unlikely(!file))
1235 goto Ebadf;
1236 if (unlikely(err < 0)) {
1237 if (err == -EMFILE)
1238 goto Ebadf;
1239 goto out_unlock;
1240 }
Al Viro8280d162012-08-21 12:11:46 -04001241 return do_dup2(files, file, newfd, flags);
Al Virofe17f222012-08-21 11:48:11 -04001242
1243Ebadf:
1244 err = -EBADF;
1245out_unlock:
1246 spin_unlock(&files->file_lock);
1247 return err;
1248}
1249
Dominik Brodowskic7248322018-03-11 11:34:40 +01001250SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
1251{
1252 return ksys_dup3(oldfd, newfd, flags);
1253}
1254
Al Virofe17f222012-08-21 11:48:11 -04001255SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
1256{
1257 if (unlikely(newfd == oldfd)) { /* corner case */
1258 struct files_struct *files = current->files;
1259 int retval = oldfd;
1260
1261 rcu_read_lock();
Eric W. Biedermanf36c2942020-11-20 17:14:26 -06001262 if (!files_lookup_fd_rcu(files, oldfd))
Al Virofe17f222012-08-21 11:48:11 -04001263 retval = -EBADF;
1264 rcu_read_unlock();
1265 return retval;
1266 }
Dominik Brodowskic7248322018-03-11 11:34:40 +01001267 return ksys_dup3(oldfd, newfd, 0);
Al Virofe17f222012-08-21 11:48:11 -04001268}
1269
Christoph Hellwigbc1cd992020-07-14 08:58:49 +02001270SYSCALL_DEFINE1(dup, unsigned int, fildes)
Al Virofe17f222012-08-21 11:48:11 -04001271{
1272 int ret = -EBADF;
1273 struct file *file = fget_raw(fildes);
1274
1275 if (file) {
Yann Droneaud8d10a032014-12-10 15:45:44 -08001276 ret = get_unused_fd_flags(0);
Al Virofe17f222012-08-21 11:48:11 -04001277 if (ret >= 0)
1278 fd_install(ret, file);
1279 else
1280 fput(file);
1281 }
1282 return ret;
1283}
1284
1285int f_dupfd(unsigned int from, struct file *file, unsigned flags)
1286{
Eric W. Biedermane06b53c2020-11-20 17:14:36 -06001287 unsigned long nofile = rlimit(RLIMIT_NOFILE);
Al Virofe17f222012-08-21 11:48:11 -04001288 int err;
Eric W. Biedermane06b53c2020-11-20 17:14:36 -06001289 if (from >= nofile)
Al Virofe17f222012-08-21 11:48:11 -04001290 return -EINVAL;
Eric W. Biedermane06b53c2020-11-20 17:14:36 -06001291 err = alloc_fd(from, nofile, flags);
Al Virofe17f222012-08-21 11:48:11 -04001292 if (err >= 0) {
1293 get_file(file);
1294 fd_install(err, file);
1295 }
1296 return err;
1297}
Al Viroc3c073f2012-08-21 22:32:06 -04001298
1299int iterate_fd(struct files_struct *files, unsigned n,
1300 int (*f)(const void *, struct file *, unsigned),
1301 const void *p)
1302{
1303 struct fdtable *fdt;
Al Viroc3c073f2012-08-21 22:32:06 -04001304 int res = 0;
1305 if (!files)
1306 return 0;
1307 spin_lock(&files->file_lock);
Al Viroa77cfcb2012-11-29 22:57:33 -05001308 for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
1309 struct file *file;
1310 file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
1311 if (!file)
1312 continue;
1313 res = f(p, file, n);
1314 if (res)
1315 break;
Al Viroc3c073f2012-08-21 22:32:06 -04001316 }
1317 spin_unlock(&files->file_lock);
1318 return res;
1319}
1320EXPORT_SYMBOL(iterate_fd);