blob: e943370107d06afab21d37fc43e2aa3714fb816e [file] [log] [blame]
Thomas Gleixner20c8ccb2019-06-04 10:11:32 +02001// SPDX-License-Identifier: GPL-2.0-only
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002/*
3 * fs/userfaultfd.c
4 *
5 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
6 * Copyright (C) 2008-2009 Red Hat, Inc.
7 * Copyright (C) 2015 Red Hat, Inc.
8 *
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07009 * Some part derived from fs/eventfd.c (anon inode setup) and
10 * mm/ksm.c (mm hashing).
11 */
12
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -080013#include <linux/list.h>
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070014#include <linux/hashtable.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010015#include <linux/sched/signal.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010016#include <linux/sched/mm.h>
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070017#include <linux/mm.h>
Arnd Bergmann17fca132022-01-14 14:06:07 -080018#include <linux/mm_inline.h>
Peter Xu6dfeaff2021-05-04 18:33:13 -070019#include <linux/mmu_notifier.h>
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070020#include <linux/poll.h>
21#include <linux/slab.h>
22#include <linux/seq_file.h>
23#include <linux/file.h>
24#include <linux/bug.h>
25#include <linux/anon_inodes.h>
26#include <linux/syscalls.h>
27#include <linux/userfaultfd_k.h>
28#include <linux/mempolicy.h>
29#include <linux/ioctl.h>
30#include <linux/security.h>
Mike Kravetzcab350a2017-02-22 15:43:04 -080031#include <linux/hugetlb.h>
Peter Xu5c041f52022-05-12 20:22:52 -070032#include <linux/swapops.h>
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070033
Lokesh Gidrad0d47302020-12-14 19:13:54 -080034int sysctl_unprivileged_userfaultfd __read_mostly;
Peter Xucefdca02019-05-13 17:16:41 -070035
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070036static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
37
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070038/*
39 * Start with fault_pending_wqh and fault_wqh so they're more likely
40 * to be in the same cacheline.
Eric Biggerscbcfa132019-07-04 15:14:39 -070041 *
42 * Locking order:
43 * fd_wqh.lock
44 * fault_pending_wqh.lock
45 * fault_wqh.lock
46 * event_wqh.lock
47 *
48 * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
49 * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
50 * also taken in IRQ context.
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070051 */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070052struct userfaultfd_ctx {
Andrea Arcangeli15b726e2015-09-04 15:46:44 -070053 /* waitqueue head for the pending (i.e. not read) userfaults */
54 wait_queue_head_t fault_pending_wqh;
55 /* waitqueue head for the userfaults */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070056 wait_queue_head_t fault_wqh;
57 /* waitqueue head for the pseudo fd to wakeup poll/read */
58 wait_queue_head_t fd_wqh;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -080059 /* waitqueue head for events */
60 wait_queue_head_t event_wqh;
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -070061 /* a refile sequence protected by fault_pending_wqh lock */
Ahmed S. Darwish2ca97ac2020-07-20 17:55:28 +020062 seqcount_spinlock_t refile_seq;
Andrea Arcangeli3004ec92015-09-04 15:46:48 -070063 /* pseudo fd refcounting */
Eric Biggersca880422018-12-28 00:34:43 -080064 refcount_t refcount;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070065 /* userfaultfd syscall flags */
66 unsigned int flags;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -080067 /* features requested from the userspace */
68 unsigned int features;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070069 /* released */
70 bool released;
Mike Rapoportdf2cc962018-06-07 17:09:25 -070071 /* memory mappings are changing because of non-cooperative event */
Nadav Amita759a902021-09-02 14:58:56 -070072 atomic_t mmap_changing;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070073 /* mm with one ore more vmas attached to this userfaultfd_ctx */
74 struct mm_struct *mm;
75};
76
Pavel Emelyanov893e26e2017-02-22 15:42:27 -080077struct userfaultfd_fork_ctx {
78 struct userfaultfd_ctx *orig;
79 struct userfaultfd_ctx *new;
80 struct list_head list;
81};
82
Mike Rapoport897ab3e2017-02-24 14:58:22 -080083struct userfaultfd_unmap_ctx {
84 struct userfaultfd_ctx *ctx;
85 unsigned long start;
86 unsigned long end;
87 struct list_head list;
88};
89
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070090struct userfaultfd_wait_queue {
Andrea Arcangelia9b85f92015-09-04 15:46:37 -070091 struct uffd_msg msg;
Ingo Molnarac6424b2017-06-20 12:06:13 +020092 wait_queue_entry_t wq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070093 struct userfaultfd_ctx *ctx;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -080094 bool waken;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -070095};
96
97struct userfaultfd_wake_range {
98 unsigned long start;
99 unsigned long len;
100};
101
Nadav Amit22e5fe2a2021-09-02 14:58:59 -0700102/* internal indication that UFFD_API ioctl was successfully executed */
103#define UFFD_FEATURE_INITIALIZED (1u << 31)
104
105static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
106{
107 return ctx->features & UFFD_FEATURE_INITIALIZED;
108}
109
Ingo Molnarac6424b2017-06-20 12:06:13 +0200110static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700111 int wake_flags, void *key)
112{
113 struct userfaultfd_wake_range *range = key;
114 int ret;
115 struct userfaultfd_wait_queue *uwq;
116 unsigned long start, len;
117
118 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
119 ret = 0;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700120 /* len == 0 means wake all */
121 start = range->start;
122 len = range->len;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700123 if (len && (start > uwq->msg.arg.pagefault.address ||
124 start + len <= uwq->msg.arg.pagefault.address))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700125 goto out;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800126 WRITE_ONCE(uwq->waken, true);
127 /*
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200128 * The Program-Order guarantees provided by the scheduler
129 * ensure uwq->waken is visible before the task is woken.
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800130 */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700131 ret = wake_up_state(wq->private, mode);
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200132 if (ret) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700133 /*
134 * Wake only once, autoremove behavior.
135 *
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200136 * After the effect of list_del_init is visible to the other
137 * CPUs, the waitqueue may disappear from under us, see the
138 * !list_empty_careful() in handle_userfault().
139 *
140 * try_to_wake_up() has an implicit smp_mb(), and the
141 * wq->private is read before calling the extern function
142 * "wake_up_state" (which in turns calls try_to_wake_up).
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700143 */
Ingo Molnar2055da92017-06-20 12:06:46 +0200144 list_del_init(&wq->entry);
Peter Zijlstraa9668cd2017-06-07 17:51:27 +0200145 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700146out:
147 return ret;
148}
149
150/**
151 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
152 * context.
153 * @ctx: [in] Pointer to the userfaultfd context.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700154 */
155static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
156{
Eric Biggersca880422018-12-28 00:34:43 -0800157 refcount_inc(&ctx->refcount);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700158}
159
160/**
161 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
162 * context.
163 * @ctx: [in] Pointer to userfaultfd context.
164 *
165 * The userfaultfd context reference must have been previously acquired either
166 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
167 */
168static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
169{
Eric Biggersca880422018-12-28 00:34:43 -0800170 if (refcount_dec_and_test(&ctx->refcount)) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700171 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
172 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
173 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
174 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800175 VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
176 VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700177 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
178 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
Oleg Nesterovd2005e32016-05-20 16:58:36 -0700179 mmdrop(ctx->mm);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -0700180 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700181 }
182}
183
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700184static inline void msg_init(struct uffd_msg *msg)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700185{
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700186 BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
187 /*
188 * Must use memset to zero out the paddings or kernel data is
189 * leaked to userland.
190 */
191 memset(msg, 0, sizeof(struct uffd_msg));
192}
193
194static inline struct uffd_msg userfault_msg(unsigned long address,
195 unsigned int flags,
Alexey Perevalov9d4ac932017-09-06 16:23:56 -0700196 unsigned long reason,
197 unsigned int features)
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700198{
199 struct uffd_msg msg;
200 msg_init(&msg);
201 msg.event = UFFD_EVENT_PAGEFAULT;
Nadav Amit824ddc62022-03-22 14:45:32 -0700202
203 if (!(features & UFFD_FEATURE_EXACT_ADDRESS))
204 address &= PAGE_MASK;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700205 msg.arg.pagefault.address = address;
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700206 /*
207 * These flags indicate why the userfault occurred:
208 * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.
209 * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.
210 * - Neither of these flags being set indicates a MISSING fault.
211 *
212 * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write
213 * fault. Otherwise, it was a read fault.
214 */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700215 if (flags & FAULT_FLAG_WRITE)
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700216 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700217 if (reason & VM_UFFD_WP)
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700218 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700219 if (reason & VM_UFFD_MINOR)
220 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
Alexey Perevalov9d4ac932017-09-06 16:23:56 -0700221 if (features & UFFD_FEATURE_THREAD_ID)
Andrea Arcangelia36985d2017-09-06 16:23:59 -0700222 msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
Andrea Arcangelia9b85f92015-09-04 15:46:37 -0700223 return msg;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700224}
225
Mike Kravetz369cd2122017-02-22 15:43:10 -0800226#ifdef CONFIG_HUGETLB_PAGE
227/*
228 * Same functionality as userfaultfd_must_wait below with modifications for
229 * hugepmd ranges.
230 */
231static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
Punit Agrawal7868a202017-07-06 15:39:42 -0700232 struct vm_area_struct *vma,
Mike Kravetz369cd2122017-02-22 15:43:10 -0800233 unsigned long address,
234 unsigned long flags,
235 unsigned long reason)
236{
237 struct mm_struct *mm = ctx->mm;
Janosch Frank1e2c0432018-07-03 17:02:39 -0700238 pte_t *ptep, pte;
Mike Kravetz369cd2122017-02-22 15:43:10 -0800239 bool ret = true;
240
Michel Lespinasse42fc5412020-06-08 21:33:44 -0700241 mmap_assert_locked(mm);
Mike Kravetz369cd2122017-02-22 15:43:10 -0800242
Janosch Frank1e2c0432018-07-03 17:02:39 -0700243 ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
244
245 if (!ptep)
Mike Kravetz369cd2122017-02-22 15:43:10 -0800246 goto out;
247
248 ret = false;
Janosch Frank1e2c0432018-07-03 17:02:39 -0700249 pte = huge_ptep_get(ptep);
Mike Kravetz369cd2122017-02-22 15:43:10 -0800250
251 /*
252 * Lockless access: we're in a wait_event so it's ok if it
Peter Xu5c041f52022-05-12 20:22:52 -0700253 * changes under us. PTE markers should be handled the same as none
254 * ptes here.
Mike Kravetz369cd2122017-02-22 15:43:10 -0800255 */
Peter Xu5c041f52022-05-12 20:22:52 -0700256 if (huge_pte_none_mostly(pte))
Mike Kravetz369cd2122017-02-22 15:43:10 -0800257 ret = true;
Janosch Frank1e2c0432018-07-03 17:02:39 -0700258 if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
Mike Kravetz369cd2122017-02-22 15:43:10 -0800259 ret = true;
260out:
261 return ret;
262}
263#else
264static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
Punit Agrawal7868a202017-07-06 15:39:42 -0700265 struct vm_area_struct *vma,
Mike Kravetz369cd2122017-02-22 15:43:10 -0800266 unsigned long address,
267 unsigned long flags,
268 unsigned long reason)
269{
270 return false; /* should never get here */
271}
272#endif /* CONFIG_HUGETLB_PAGE */
273
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700274/*
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700275 * Verify the pagetables are still not ok after having reigstered into
276 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
277 * userfault that has already been resolved, if userfaultfd_read and
278 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
279 * threads.
280 */
281static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
282 unsigned long address,
283 unsigned long flags,
284 unsigned long reason)
285{
286 struct mm_struct *mm = ctx->mm;
287 pgd_t *pgd;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300288 p4d_t *p4d;
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700289 pud_t *pud;
290 pmd_t *pmd, _pmd;
291 pte_t *pte;
292 bool ret = true;
293
Michel Lespinasse42fc5412020-06-08 21:33:44 -0700294 mmap_assert_locked(mm);
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700295
296 pgd = pgd_offset(mm, address);
297 if (!pgd_present(*pgd))
298 goto out;
Kirill A. Shutemovc2febaf2017-03-09 17:24:07 +0300299 p4d = p4d_offset(pgd, address);
300 if (!p4d_present(*p4d))
301 goto out;
302 pud = pud_offset(p4d, address);
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700303 if (!pud_present(*pud))
304 goto out;
305 pmd = pmd_offset(pud, address);
306 /*
307 * READ_ONCE must function as a barrier with narrower scope
308 * and it must be equivalent to:
309 * _pmd = *pmd; barrier();
310 *
311 * This is to deal with the instability (as in
312 * pmd_trans_unstable) of the pmd.
313 */
314 _pmd = READ_ONCE(*pmd);
Huang Yinga365ac02018-01-31 16:17:32 -0800315 if (pmd_none(_pmd))
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700316 goto out;
317
318 ret = false;
Huang Yinga365ac02018-01-31 16:17:32 -0800319 if (!pmd_present(_pmd))
320 goto out;
321
Andrea Arcangeli63b2d412020-04-06 20:06:12 -0700322 if (pmd_trans_huge(_pmd)) {
323 if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
324 ret = true;
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700325 goto out;
Andrea Arcangeli63b2d412020-04-06 20:06:12 -0700326 }
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700327
328 /*
329 * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
330 * and use the standard pte_offset_map() instead of parsing _pmd.
331 */
332 pte = pte_offset_map(pmd, address);
333 /*
334 * Lockless access: we're in a wait_event so it's ok if it
Peter Xu5c041f52022-05-12 20:22:52 -0700335 * changes under us. PTE markers should be handled the same as none
336 * ptes here.
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700337 */
Peter Xu5c041f52022-05-12 20:22:52 -0700338 if (pte_none_mostly(*pte))
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700339 ret = true;
Andrea Arcangeli63b2d412020-04-06 20:06:12 -0700340 if (!pte_write(*pte) && (reason & VM_UFFD_WP))
341 ret = true;
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700342 pte_unmap(pte);
343
344out:
345 return ret;
346}
347
Peter Zijlstra2f064a52021-06-11 10:28:17 +0200348static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags)
Peter Xu3e69ad02020-04-01 21:09:00 -0700349{
350 if (flags & FAULT_FLAG_INTERRUPTIBLE)
351 return TASK_INTERRUPTIBLE;
352
353 if (flags & FAULT_FLAG_KILLABLE)
354 return TASK_KILLABLE;
355
356 return TASK_UNINTERRUPTIBLE;
357}
358
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700359/*
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700360 * The locking rules involved in returning VM_FAULT_RETRY depending on
361 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
362 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
363 * recommendation in __lock_page_or_retry is not an understatement.
364 *
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700365 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700366 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
367 * not set.
368 *
369 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
370 * set, VM_FAULT_RETRY can still be returned if and only if there are
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700371 * fatal_signal_pending()s, and the mmap_lock must be released before
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700372 * returning it.
373 */
Souptick Joarder2b740302018-08-23 17:01:36 -0700374vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700375{
Jan Kara82b0f8c2016-12-14 15:06:58 -0800376 struct mm_struct *mm = vmf->vma->vm_mm;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700377 struct userfaultfd_ctx *ctx;
378 struct userfaultfd_wait_queue uwq;
Souptick Joarder2b740302018-08-23 17:01:36 -0700379 vm_fault_t ret = VM_FAULT_SIGBUS;
Peter Xu3e69ad02020-04-01 21:09:00 -0700380 bool must_wait;
Peter Zijlstra2f064a52021-06-11 10:28:17 +0200381 unsigned int blocking_state;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700382
Andrea Arcangeli64c2b202017-06-16 14:02:37 -0700383 /*
384 * We don't do userfault handling for the final child pid update.
385 *
386 * We also don't do userfault handling during
387 * coredumping. hugetlbfs has the special
388 * follow_hugetlb_page() to skip missing pages in the
389 * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
390 * the no_page_table() helper in follow_page_mask(), but the
391 * shmem_vm_ops->fault method is invoked even during
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700392 * coredumping without mmap_lock and it ends up here.
Andrea Arcangeli64c2b202017-06-16 14:02:37 -0700393 */
394 if (current->flags & (PF_EXITING|PF_DUMPCORE))
395 goto out;
396
397 /*
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700398 * Coredumping runs without mmap_lock so we can only check that
399 * the mmap_lock is held, if PF_DUMPCORE was not set.
Andrea Arcangeli64c2b202017-06-16 14:02:37 -0700400 */
Michel Lespinasse42fc5412020-06-08 21:33:44 -0700401 mmap_assert_locked(mm);
Andrea Arcangeli64c2b202017-06-16 14:02:37 -0700402
Jan Kara82b0f8c2016-12-14 15:06:58 -0800403 ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700404 if (!ctx)
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700405 goto out;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700406
407 BUG_ON(ctx->mm != mm);
408
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700409 /* Any unrecognized flag is a bug. */
410 VM_BUG_ON(reason & ~__VM_UFFD_FLAGS);
411 /* 0 or > 1 flags set is a bug; we expect exactly 1. */
412 VM_BUG_ON(!reason || (reason & (reason - 1)));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700413
Prakash Sangappa2d6d6f52017-09-06 16:23:39 -0700414 if (ctx->features & UFFD_FEATURE_SIGBUS)
415 goto out;
Lokesh Gidra37cd0572020-12-14 19:13:49 -0800416 if ((vmf->flags & FAULT_FLAG_USER) == 0 &&
417 ctx->flags & UFFD_USER_MODE_ONLY) {
418 printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd "
419 "sysctl knob to 1 if kernel faults must be handled "
420 "without obtaining CAP_SYS_PTRACE capability\n");
421 goto out;
422 }
Prakash Sangappa2d6d6f52017-09-06 16:23:39 -0700423
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700424 /*
425 * If it's already released don't get it. This avoids to loop
426 * in __get_user_pages if userfaultfd_release waits on the
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700427 * caller of handle_userfault to release the mmap_lock.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700428 */
Mark Rutland6aa7de02017-10-23 14:07:29 -0700429 if (unlikely(READ_ONCE(ctx->released))) {
Andrea Arcangeli656710a2017-09-08 16:12:42 -0700430 /*
431 * Don't return VM_FAULT_SIGBUS in this case, so a non
432 * cooperative manager can close the uffd after the
433 * last UFFDIO_COPY, without risking to trigger an
434 * involuntary SIGBUS if the process was starting the
435 * userfaultfd while the userfaultfd was still armed
436 * (but after the last UFFDIO_COPY). If the uffd
437 * wasn't already closed when the userfault reached
438 * this point, that would normally be solved by
439 * userfaultfd_must_wait returning 'false'.
440 *
441 * If we were to return VM_FAULT_SIGBUS here, the non
442 * cooperative manager would be instead forced to
443 * always call UFFDIO_UNREGISTER before it can safely
444 * close the uffd.
445 */
446 ret = VM_FAULT_NOPAGE;
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700447 goto out;
Andrea Arcangeli656710a2017-09-08 16:12:42 -0700448 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700449
450 /*
451 * Check that we can return VM_FAULT_RETRY.
452 *
453 * NOTE: it should become possible to return VM_FAULT_RETRY
454 * even if FAULT_FLAG_TRIED is set without leading to gup()
455 * -EBUSY failures, if the userfaultfd is to be extended for
456 * VM_UFFD_WP tracking and we intend to arm the userfault
457 * without first stopping userland access to the memory. For
458 * VM_UFFD_MISSING userfaults this is enough for now.
459 */
Jan Kara82b0f8c2016-12-14 15:06:58 -0800460 if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700461 /*
462 * Validate the invariant that nowait must allow retry
463 * to be sure not to return SIGBUS erroneously on
464 * nowait invocations.
465 */
Jan Kara82b0f8c2016-12-14 15:06:58 -0800466 BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700467#ifdef CONFIG_DEBUG_VM
468 if (printk_ratelimit()) {
469 printk(KERN_WARNING
Jan Kara82b0f8c2016-12-14 15:06:58 -0800470 "FAULT_FLAG_ALLOW_RETRY missing %x\n",
471 vmf->flags);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700472 dump_stack();
473 }
474#endif
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700475 goto out;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700476 }
477
478 /*
479 * Handle nowait, not much to do other than tell it to retry
480 * and wait.
481 */
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700482 ret = VM_FAULT_RETRY;
Jan Kara82b0f8c2016-12-14 15:06:58 -0800483 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700484 goto out;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700485
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700486 /* take the reference before dropping the mmap_lock */
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700487 userfaultfd_ctx_get(ctx);
488
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700489 init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
490 uwq.wq.private = current;
Nadav Amit824ddc62022-03-22 14:45:32 -0700491 uwq.msg = userfault_msg(vmf->real_address, vmf->flags, reason,
Alexey Perevalov9d4ac932017-09-06 16:23:56 -0700492 ctx->features);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700493 uwq.ctx = ctx;
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800494 uwq.waken = false;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700495
Peter Xu3e69ad02020-04-01 21:09:00 -0700496 blocking_state = userfaultfd_get_blocking_state(vmf->flags);
Andrea Arcangelidfa37dc2015-09-04 15:47:18 -0700497
Eric Biggerscbcfa132019-07-04 15:14:39 -0700498 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700499 /*
500 * After the __add_wait_queue the uwq is visible to userland
501 * through poll/read().
502 */
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700503 __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
504 /*
505 * The smp_mb() after __set_current_state prevents the reads
506 * following the spin_unlock to happen before the list_add in
507 * __add_wait_queue.
508 */
Andrea Arcangeli15a77c62017-01-24 15:17:59 -0800509 set_current_state(blocking_state);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700510 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700511
Mike Kravetz369cd2122017-02-22 15:43:10 -0800512 if (!is_vm_hugetlb_page(vmf->vma))
513 must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
514 reason);
515 else
Punit Agrawal7868a202017-07-06 15:39:42 -0700516 must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma,
517 vmf->address,
Mike Kravetz369cd2122017-02-22 15:43:10 -0800518 vmf->flags, reason);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700519 mmap_read_unlock(mm);
Andrea Arcangeli8d2afd92015-09-04 15:46:51 -0700520
Linus Torvaldsf9bf3522020-08-02 10:42:31 -0700521 if (likely(must_wait && !READ_ONCE(ctx->released))) {
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800522 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700523 schedule();
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700524 }
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700525
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700526 __set_current_state(TASK_RUNNING);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700527
528 /*
529 * Here we race with the list_del; list_add in
530 * userfaultfd_ctx_read(), however because we don't ever run
531 * list_del_init() to refile across the two lists, the prev
532 * and next pointers will never point to self. list_add also
533 * would never let any of the two pointers to point to
534 * self. So list_empty_careful won't risk to see both pointers
535 * pointing to self at any time during the list refile. The
536 * only case where list_del_init() is called is the full
537 * removal in the wake function and there we don't re-list_add
538 * and it's fine not to block on the spinlock. The uwq on this
539 * kernel stack can be released after the list_del_init.
540 */
Ingo Molnar2055da92017-06-20 12:06:46 +0200541 if (!list_empty_careful(&uwq.wq.entry)) {
Eric Biggerscbcfa132019-07-04 15:14:39 -0700542 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700543 /*
544 * No need of list_del_init(), the uwq on the stack
545 * will be freed shortly anyway.
546 */
Ingo Molnar2055da92017-06-20 12:06:46 +0200547 list_del(&uwq.wq.entry);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700548 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700549 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700550
551 /*
552 * ctx may go away after this if the userfault pseudo fd is
553 * already released.
554 */
555 userfaultfd_ctx_put(ctx);
556
Andrea Arcangeliba85c702015-09-04 15:46:41 -0700557out:
558 return ret;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700559}
560
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800561static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
562 struct userfaultfd_wait_queue *ewq)
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800563{
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800564 struct userfaultfd_ctx *release_new_ctx;
565
Andrea Arcangeli9a69a822017-03-09 16:16:52 -0800566 if (WARN_ON_ONCE(current->flags & PF_EXITING))
567 goto out;
568
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800569 ewq->ctx = ctx;
570 init_waitqueue_entry(&ewq->wq, current);
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800571 release_new_ctx = NULL;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800572
Eric Biggerscbcfa132019-07-04 15:14:39 -0700573 spin_lock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800574 /*
575 * After the __add_wait_queue the uwq is visible to userland
576 * through poll/read().
577 */
578 __add_wait_queue(&ctx->event_wqh, &ewq->wq);
579 for (;;) {
580 set_current_state(TASK_KILLABLE);
581 if (ewq->msg.event == 0)
582 break;
Mark Rutland6aa7de02017-10-23 14:07:29 -0700583 if (READ_ONCE(ctx->released) ||
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800584 fatal_signal_pending(current)) {
Andrea Arcangeli384632e2017-10-03 16:15:38 -0700585 /*
586 * &ewq->wq may be queued in fork_event, but
587 * __remove_wait_queue ignores the head
588 * parameter. It would be a problem if it
589 * didn't.
590 */
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800591 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
Mike Rapoport7eb76d42017-03-09 16:17:09 -0800592 if (ewq->msg.event == UFFD_EVENT_FORK) {
593 struct userfaultfd_ctx *new;
594
595 new = (struct userfaultfd_ctx *)
596 (unsigned long)
597 ewq->msg.arg.reserved.reserved1;
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800598 release_new_ctx = new;
Mike Rapoport7eb76d42017-03-09 16:17:09 -0800599 }
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800600 break;
601 }
602
Eric Biggerscbcfa132019-07-04 15:14:39 -0700603 spin_unlock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800604
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800605 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800606 schedule();
607
Eric Biggerscbcfa132019-07-04 15:14:39 -0700608 spin_lock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800609 }
610 __set_current_state(TASK_RUNNING);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700611 spin_unlock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800612
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800613 if (release_new_ctx) {
614 struct vm_area_struct *vma;
615 struct mm_struct *mm = release_new_ctx->mm;
616
617 /* the various vma->vm_userfaultfd_ctx still points to it */
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700618 mmap_write_lock(mm);
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800619 for (vma = mm->mmap; vma; vma = vma->vm_next)
Mike Rapoport31e810a2018-08-02 15:36:09 -0700620 if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800621 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700622 vma->vm_flags &= ~__VM_UFFD_FLAGS;
Mike Rapoport31e810a2018-08-02 15:36:09 -0700623 }
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700624 mmap_write_unlock(mm);
Andrea Arcangeli0cbb4b42018-01-04 16:18:09 -0800625
626 userfaultfd_ctx_put(release_new_ctx);
627 }
628
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800629 /*
630 * ctx may go away after this if the userfault pseudo fd is
631 * already released.
632 */
Andrea Arcangeli9a69a822017-03-09 16:16:52 -0800633out:
Nadav Amita759a902021-09-02 14:58:56 -0700634 atomic_dec(&ctx->mmap_changing);
635 VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800636 userfaultfd_ctx_put(ctx);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800637}
638
639static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
640 struct userfaultfd_wait_queue *ewq)
641{
642 ewq->msg.event = 0;
643 wake_up_locked(&ctx->event_wqh);
644 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
645}
646
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800647int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
648{
649 struct userfaultfd_ctx *ctx = NULL, *octx;
650 struct userfaultfd_fork_ctx *fctx;
651
652 octx = vma->vm_userfaultfd_ctx.ctx;
653 if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
654 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700655 vma->vm_flags &= ~__VM_UFFD_FLAGS;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800656 return 0;
657 }
658
659 list_for_each_entry(fctx, fcs, list)
660 if (fctx->orig == octx) {
661 ctx = fctx->new;
662 break;
663 }
664
665 if (!ctx) {
666 fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
667 if (!fctx)
668 return -ENOMEM;
669
670 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
671 if (!ctx) {
672 kfree(fctx);
673 return -ENOMEM;
674 }
675
Eric Biggersca880422018-12-28 00:34:43 -0800676 refcount_set(&ctx->refcount, 1);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800677 ctx->flags = octx->flags;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800678 ctx->features = octx->features;
679 ctx->released = false;
Nadav Amita759a902021-09-02 14:58:56 -0700680 atomic_set(&ctx->mmap_changing, 0);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800681 ctx->mm = vma->vm_mm;
Mike Rapoport00bb31f2017-11-15 17:36:56 -0800682 mmgrab(ctx->mm);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800683
684 userfaultfd_ctx_get(octx);
Nadav Amita759a902021-09-02 14:58:56 -0700685 atomic_inc(&octx->mmap_changing);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800686 fctx->orig = octx;
687 fctx->new = ctx;
688 list_add_tail(&fctx->list, fcs);
689 }
690
691 vma->vm_userfaultfd_ctx.ctx = ctx;
692 return 0;
693}
694
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800695static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800696{
697 struct userfaultfd_ctx *ctx = fctx->orig;
698 struct userfaultfd_wait_queue ewq;
699
700 msg_init(&ewq.msg);
701
702 ewq.msg.event = UFFD_EVENT_FORK;
703 ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
704
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800705 userfaultfd_event_wait_completion(ctx, &ewq);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800706}
707
708void dup_userfaultfd_complete(struct list_head *fcs)
709{
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800710 struct userfaultfd_fork_ctx *fctx, *n;
711
712 list_for_each_entry_safe(fctx, n, fcs, list) {
Andrea Arcangeli8c9e7bb2017-03-09 16:16:54 -0800713 dup_fctx(fctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800714 list_del(&fctx->list);
715 kfree(fctx);
716 }
717}
718
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800719void mremap_userfaultfd_prep(struct vm_area_struct *vma,
720 struct vm_userfaultfd_ctx *vm_ctx)
721{
722 struct userfaultfd_ctx *ctx;
723
724 ctx = vma->vm_userfaultfd_ctx.ctx;
Peter Xu3cfd22b2018-12-28 00:38:47 -0800725
726 if (!ctx)
727 return;
728
729 if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800730 vm_ctx->ctx = ctx;
731 userfaultfd_ctx_get(ctx);
Nadav Amita759a902021-09-02 14:58:56 -0700732 atomic_inc(&ctx->mmap_changing);
Peter Xu3cfd22b2018-12-28 00:38:47 -0800733 } else {
734 /* Drop uffd context if remap feature not enabled */
735 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700736 vma->vm_flags &= ~__VM_UFFD_FLAGS;
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800737 }
738}
739
Andrea Arcangeli90794bf2017-02-22 15:42:37 -0800740void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800741 unsigned long from, unsigned long to,
742 unsigned long len)
743{
Andrea Arcangeli90794bf2017-02-22 15:42:37 -0800744 struct userfaultfd_ctx *ctx = vm_ctx->ctx;
Pavel Emelyanov72f87652017-02-22 15:42:34 -0800745 struct userfaultfd_wait_queue ewq;
746
747 if (!ctx)
748 return;
749
750 if (to & ~PAGE_MASK) {
751 userfaultfd_ctx_put(ctx);
752 return;
753 }
754
755 msg_init(&ewq.msg);
756
757 ewq.msg.event = UFFD_EVENT_REMAP;
758 ewq.msg.arg.remap.from = from;
759 ewq.msg.arg.remap.to = to;
760 ewq.msg.arg.remap.len = len;
761
762 userfaultfd_event_wait_completion(ctx, &ewq);
763}
764
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800765bool userfaultfd_remove(struct vm_area_struct *vma,
Mike Rapoportd8119142017-02-24 14:56:02 -0800766 unsigned long start, unsigned long end)
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800767{
768 struct mm_struct *mm = vma->vm_mm;
769 struct userfaultfd_ctx *ctx;
770 struct userfaultfd_wait_queue ewq;
771
772 ctx = vma->vm_userfaultfd_ctx.ctx;
Mike Rapoportd8119142017-02-24 14:56:02 -0800773 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800774 return true;
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800775
776 userfaultfd_ctx_get(ctx);
Nadav Amita759a902021-09-02 14:58:56 -0700777 atomic_inc(&ctx->mmap_changing);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700778 mmap_read_unlock(mm);
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800779
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800780 msg_init(&ewq.msg);
781
Mike Rapoportd8119142017-02-24 14:56:02 -0800782 ewq.msg.event = UFFD_EVENT_REMOVE;
783 ewq.msg.arg.remove.start = start;
784 ewq.msg.arg.remove.end = end;
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800785
786 userfaultfd_event_wait_completion(ctx, &ewq);
787
Andrea Arcangeli70ccb922017-03-09 16:17:11 -0800788 return false;
Pavel Emelyanov05ce7722017-02-22 15:42:40 -0800789}
790
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800791static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
792 unsigned long start, unsigned long end)
793{
794 struct userfaultfd_unmap_ctx *unmap_ctx;
795
796 list_for_each_entry(unmap_ctx, unmaps, list)
797 if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
798 unmap_ctx->end == end)
799 return true;
800
801 return false;
802}
803
804int userfaultfd_unmap_prep(struct vm_area_struct *vma,
805 unsigned long start, unsigned long end,
806 struct list_head *unmaps)
807{
808 for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
809 struct userfaultfd_unmap_ctx *unmap_ctx;
810 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
811
812 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
813 has_unmap_ctx(ctx, unmaps, start, end))
814 continue;
815
816 unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
817 if (!unmap_ctx)
818 return -ENOMEM;
819
820 userfaultfd_ctx_get(ctx);
Nadav Amita759a902021-09-02 14:58:56 -0700821 atomic_inc(&ctx->mmap_changing);
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800822 unmap_ctx->ctx = ctx;
823 unmap_ctx->start = start;
824 unmap_ctx->end = end;
825 list_add_tail(&unmap_ctx->list, unmaps);
826 }
827
828 return 0;
829}
830
831void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
832{
833 struct userfaultfd_unmap_ctx *ctx, *n;
834 struct userfaultfd_wait_queue ewq;
835
836 list_for_each_entry_safe(ctx, n, uf, list) {
837 msg_init(&ewq.msg);
838
839 ewq.msg.event = UFFD_EVENT_UNMAP;
840 ewq.msg.arg.remove.start = ctx->start;
841 ewq.msg.arg.remove.end = ctx->end;
842
843 userfaultfd_event_wait_completion(ctx->ctx, &ewq);
844
845 list_del(&ctx->list);
846 kfree(ctx);
847 }
848}
849
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700850static int userfaultfd_release(struct inode *inode, struct file *file)
851{
852 struct userfaultfd_ctx *ctx = file->private_data;
853 struct mm_struct *mm = ctx->mm;
854 struct vm_area_struct *vma, *prev;
855 /* len == 0 means wake all */
856 struct userfaultfd_wake_range range = { .len = 0, };
857 unsigned long new_flags;
858
Mark Rutland6aa7de02017-10-23 14:07:29 -0700859 WRITE_ONCE(ctx->released, true);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700860
Oleg Nesterovd2005e32016-05-20 16:58:36 -0700861 if (!mmget_not_zero(mm))
862 goto wakeup;
863
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700864 /*
865 * Flush page faults out of all CPUs. NOTE: all page faults
866 * must be retried without returning VM_FAULT_SIGBUS if
867 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700868 * changes while handle_userfault released the mmap_lock. So
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700869 * it's critical that released is set to true (above), before
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700870 * taking the mmap_lock for writing.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700871 */
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700872 mmap_write_lock(mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700873 prev = NULL;
874 for (vma = mm->mmap; vma; vma = vma->vm_next) {
875 cond_resched();
876 BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700877 !!(vma->vm_flags & __VM_UFFD_FLAGS));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700878 if (vma->vm_userfaultfd_ctx.ctx != ctx) {
879 prev = vma;
880 continue;
881 }
Axel Rasmussen7677f7f2021-05-04 18:35:36 -0700882 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
Jann Horn4d45e752020-10-15 20:13:00 -0700883 prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
884 new_flags, vma->anon_vma,
885 vma->vm_file, vma->vm_pgoff,
886 vma_policy(vma),
Suren Baghdasaryan5c26f6a2022-03-04 20:28:51 -0800887 NULL_VM_UFFD_CTX, anon_vma_name(vma));
Jann Horn4d45e752020-10-15 20:13:00 -0700888 if (prev)
889 vma = prev;
890 else
891 prev = vma;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700892 vma->vm_flags = new_flags;
893 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
894 }
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700895 mmap_write_unlock(mm);
Oleg Nesterovd2005e32016-05-20 16:58:36 -0700896 mmput(mm);
897wakeup:
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700898 /*
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700899 * After no new page faults can wait on this fault_*wqh, flush
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700900 * the last page faults that may have been already waiting on
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700901 * the fault_*wqh.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700902 */
Eric Biggerscbcfa132019-07-04 15:14:39 -0700903 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeliac5be6b2015-09-22 14:58:49 -0700904 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
Matthew Wilcoxc430d1e2018-08-21 21:56:30 -0700905 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
Eric Biggerscbcfa132019-07-04 15:14:39 -0700906 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700907
Mike Rapoport5a18b642017-08-02 13:32:24 -0700908 /* Flush pending events that may still wait on event_wqh */
909 wake_up_all(&ctx->event_wqh);
910
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800911 wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700912 userfaultfd_ctx_put(ctx);
913 return 0;
914}
915
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700916/* fault_pending_wqh.lock must be hold by the caller */
Pavel Emelyanov6dcc27f2017-02-22 15:42:18 -0800917static inline struct userfaultfd_wait_queue *find_userfault_in(
918 wait_queue_head_t *wqh)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700919{
Ingo Molnarac6424b2017-06-20 12:06:13 +0200920 wait_queue_entry_t *wq;
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700921 struct userfaultfd_wait_queue *uwq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700922
Lance Roy456a7372018-10-04 23:45:44 -0700923 lockdep_assert_held(&wqh->lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700924
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700925 uwq = NULL;
Pavel Emelyanov6dcc27f2017-02-22 15:42:18 -0800926 if (!waitqueue_active(wqh))
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700927 goto out;
928 /* walk in reverse to provide FIFO behavior to read userfaults */
Ingo Molnar2055da92017-06-20 12:06:46 +0200929 wq = list_last_entry(&wqh->head, typeof(*wq), entry);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -0700930 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
931out:
932 return uwq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700933}
934
Pavel Emelyanov6dcc27f2017-02-22 15:42:18 -0800935static inline struct userfaultfd_wait_queue *find_userfault(
936 struct userfaultfd_ctx *ctx)
937{
938 return find_userfault_in(&ctx->fault_pending_wqh);
939}
940
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800941static inline struct userfaultfd_wait_queue *find_userfault_evt(
942 struct userfaultfd_ctx *ctx)
943{
944 return find_userfault_in(&ctx->event_wqh);
945}
946
Al Viro076ccb72017-07-03 01:02:18 -0400947static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700948{
949 struct userfaultfd_ctx *ctx = file->private_data;
Al Viro076ccb72017-07-03 01:02:18 -0400950 __poll_t ret;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700951
952 poll_wait(file, &ctx->fd_wqh, wait);
953
Nadav Amit22e5fe2a2021-09-02 14:58:59 -0700954 if (!userfaultfd_is_initialized(ctx))
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800955 return EPOLLERR;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -0800956
Nadav Amit22e5fe2a2021-09-02 14:58:59 -0700957 /*
958 * poll() never guarantees that read won't block.
959 * userfaults can be waken before they're read().
960 */
961 if (unlikely(!(file->f_flags & O_NONBLOCK)))
Linus Torvaldsa9a08842018-02-11 14:34:03 -0800962 return EPOLLERR;
Nadav Amit22e5fe2a2021-09-02 14:58:59 -0700963 /*
964 * lockless access to see if there are pending faults
965 * __pollwait last action is the add_wait_queue but
966 * the spin_unlock would allow the waitqueue_active to
967 * pass above the actual list_add inside
968 * add_wait_queue critical section. So use a full
969 * memory barrier to serialize the list_add write of
970 * add_wait_queue() with the waitqueue_active read
971 * below.
972 */
973 ret = 0;
974 smp_mb();
975 if (waitqueue_active(&ctx->fault_pending_wqh))
976 ret = EPOLLIN;
977 else if (waitqueue_active(&ctx->event_wqh))
978 ret = EPOLLIN;
979
980 return ret;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -0700981}
982
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800983static const struct file_operations userfaultfd_fops;
984
Daniel Colascioneb5379002021-01-08 14:22:23 -0800985static int resolve_userfault_fork(struct userfaultfd_ctx *new,
986 struct inode *inode,
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800987 struct uffd_msg *msg)
988{
989 int fd;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800990
Daniel Colascioneb5379002021-01-08 14:22:23 -0800991 fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new,
992 O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800993 if (fd < 0)
994 return fd;
995
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800996 msg->arg.reserved.reserved1 = 0;
997 msg->arg.fork.ufd = fd;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800998 return 0;
999}
1000
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001001static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
Daniel Colascioneb5379002021-01-08 14:22:23 -08001002 struct uffd_msg *msg, struct inode *inode)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001003{
1004 ssize_t ret;
1005 DECLARE_WAITQUEUE(wait, current);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001006 struct userfaultfd_wait_queue *uwq;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001007 /*
1008 * Handling fork event requires sleeping operations, so
1009 * we drop the event_wqh lock, then do these ops, then
1010 * lock it back and wake up the waiter. While the lock is
1011 * dropped the ewq may go away so we keep track of it
1012 * carefully.
1013 */
1014 LIST_HEAD(fork_event);
1015 struct userfaultfd_ctx *fork_nctx = NULL;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001016
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001017 /* always take the fd_wqh lock before the fault_pending_wqh lock */
Christoph Hellwigae62c162018-10-26 15:02:19 -07001018 spin_lock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001019 __add_wait_queue(&ctx->fd_wqh, &wait);
1020 for (;;) {
1021 set_current_state(TASK_INTERRUPTIBLE);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001022 spin_lock(&ctx->fault_pending_wqh.lock);
1023 uwq = find_userfault(ctx);
1024 if (uwq) {
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001025 /*
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001026 * Use a seqcount to repeat the lockless check
1027 * in wake_userfault() to avoid missing
1028 * wakeups because during the refile both
1029 * waitqueue could become empty if this is the
1030 * only userfault.
1031 */
1032 write_seqcount_begin(&ctx->refile_seq);
1033
1034 /*
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001035 * The fault_pending_wqh.lock prevents the uwq
1036 * to disappear from under us.
1037 *
1038 * Refile this userfault from
1039 * fault_pending_wqh to fault_wqh, it's not
1040 * pending anymore after we read it.
1041 *
1042 * Use list_del() by hand (as
1043 * userfaultfd_wake_function also uses
1044 * list_del_init() by hand) to be sure nobody
1045 * changes __remove_wait_queue() to use
1046 * list_del_init() in turn breaking the
1047 * !list_empty_careful() check in
Ingo Molnar2055da92017-06-20 12:06:46 +02001048 * handle_userfault(). The uwq->wq.head list
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001049 * must never be empty at any time during the
1050 * refile, or the waitqueue could disappear
1051 * from under us. The "wait_queue_head_t"
1052 * parameter of __remove_wait_queue() is unused
1053 * anyway.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001054 */
Ingo Molnar2055da92017-06-20 12:06:46 +02001055 list_del(&uwq->wq.entry);
Matthew Wilcoxc430d1e2018-08-21 21:56:30 -07001056 add_wait_queue(&ctx->fault_wqh, &uwq->wq);
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001057
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001058 write_seqcount_end(&ctx->refile_seq);
1059
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001060 /* careful to always initialize msg if ret == 0 */
1061 *msg = uwq->msg;
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001062 spin_unlock(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001063 ret = 0;
1064 break;
1065 }
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001066 spin_unlock(&ctx->fault_pending_wqh.lock);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001067
1068 spin_lock(&ctx->event_wqh.lock);
1069 uwq = find_userfault_evt(ctx);
1070 if (uwq) {
1071 *msg = uwq->msg;
1072
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001073 if (uwq->msg.event == UFFD_EVENT_FORK) {
1074 fork_nctx = (struct userfaultfd_ctx *)
1075 (unsigned long)
1076 uwq->msg.arg.reserved.reserved1;
Ingo Molnar2055da92017-06-20 12:06:46 +02001077 list_move(&uwq->wq.entry, &fork_event);
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001078 /*
1079 * fork_nctx can be freed as soon as
1080 * we drop the lock, unless we take a
1081 * reference on it.
1082 */
1083 userfaultfd_ctx_get(fork_nctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001084 spin_unlock(&ctx->event_wqh.lock);
1085 ret = 0;
1086 break;
1087 }
1088
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001089 userfaultfd_event_complete(ctx, uwq);
1090 spin_unlock(&ctx->event_wqh.lock);
1091 ret = 0;
1092 break;
1093 }
1094 spin_unlock(&ctx->event_wqh.lock);
1095
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001096 if (signal_pending(current)) {
1097 ret = -ERESTARTSYS;
1098 break;
1099 }
1100 if (no_wait) {
1101 ret = -EAGAIN;
1102 break;
1103 }
Christoph Hellwigae62c162018-10-26 15:02:19 -07001104 spin_unlock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001105 schedule();
Christoph Hellwigae62c162018-10-26 15:02:19 -07001106 spin_lock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001107 }
1108 __remove_wait_queue(&ctx->fd_wqh, &wait);
1109 __set_current_state(TASK_RUNNING);
Christoph Hellwigae62c162018-10-26 15:02:19 -07001110 spin_unlock_irq(&ctx->fd_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001111
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001112 if (!ret && msg->event == UFFD_EVENT_FORK) {
Daniel Colascioneb5379002021-01-08 14:22:23 -08001113 ret = resolve_userfault_fork(fork_nctx, inode, msg);
Eric Biggerscbcfa132019-07-04 15:14:39 -07001114 spin_lock_irq(&ctx->event_wqh.lock);
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001115 if (!list_empty(&fork_event)) {
1116 /*
1117 * The fork thread didn't abort, so we can
1118 * drop the temporary refcount.
1119 */
1120 userfaultfd_ctx_put(fork_nctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001121
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001122 uwq = list_first_entry(&fork_event,
1123 typeof(*uwq),
1124 wq.entry);
1125 /*
1126 * If fork_event list wasn't empty and in turn
1127 * the event wasn't already released by fork
1128 * (the event is allocated on fork kernel
1129 * stack), put the event back to its place in
1130 * the event_wq. fork_event head will be freed
1131 * as soon as we return so the event cannot
1132 * stay queued there no matter the current
1133 * "ret" value.
1134 */
1135 list_del(&uwq->wq.entry);
1136 __add_wait_queue(&ctx->event_wqh, &uwq->wq);
1137
1138 /*
1139 * Leave the event in the waitqueue and report
1140 * error to userland if we failed to resolve
1141 * the userfault fork.
1142 */
1143 if (likely(!ret))
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001144 userfaultfd_event_complete(ctx, uwq);
Andrea Arcangeli384632e2017-10-03 16:15:38 -07001145 } else {
1146 /*
1147 * Here the fork thread aborted and the
1148 * refcount from the fork thread on fork_nctx
1149 * has already been released. We still hold
1150 * the reference we took before releasing the
1151 * lock above. If resolve_userfault_fork
1152 * failed we've to drop it because the
1153 * fork_nctx has to be freed in such case. If
1154 * it succeeded we'll hold it because the new
1155 * uffd references it.
1156 */
1157 if (ret)
1158 userfaultfd_ctx_put(fork_nctx);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001159 }
Eric Biggerscbcfa132019-07-04 15:14:39 -07001160 spin_unlock_irq(&ctx->event_wqh.lock);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -08001161 }
1162
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001163 return ret;
1164}
1165
1166static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1167 size_t count, loff_t *ppos)
1168{
1169 struct userfaultfd_ctx *ctx = file->private_data;
1170 ssize_t _ret, ret = 0;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001171 struct uffd_msg msg;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001172 int no_wait = file->f_flags & O_NONBLOCK;
Daniel Colascioneb5379002021-01-08 14:22:23 -08001173 struct inode *inode = file_inode(file);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001174
Nadav Amit22e5fe2a2021-09-02 14:58:59 -07001175 if (!userfaultfd_is_initialized(ctx))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001176 return -EINVAL;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001177
1178 for (;;) {
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001179 if (count < sizeof(msg))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001180 return ret ? ret : -EINVAL;
Daniel Colascioneb5379002021-01-08 14:22:23 -08001181 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001182 if (_ret < 0)
1183 return ret ? ret : _ret;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001184 if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001185 return ret ? ret : -EFAULT;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001186 ret += sizeof(msg);
1187 buf += sizeof(msg);
1188 count -= sizeof(msg);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001189 /*
1190 * Allow to read more than one fault at time but only
1191 * block if waiting for the very first one.
1192 */
1193 no_wait = O_NONBLOCK;
1194 }
1195}
1196
1197static void __wake_userfault(struct userfaultfd_ctx *ctx,
1198 struct userfaultfd_wake_range *range)
1199{
Eric Biggerscbcfa132019-07-04 15:14:39 -07001200 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001201 /* wake all in the range and autoremove */
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001202 if (waitqueue_active(&ctx->fault_pending_wqh))
Andrea Arcangeliac5be6b2015-09-22 14:58:49 -07001203 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07001204 range);
1205 if (waitqueue_active(&ctx->fault_wqh))
Matthew Wilcoxc430d1e2018-08-21 21:56:30 -07001206 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
Eric Biggerscbcfa132019-07-04 15:14:39 -07001207 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001208}
1209
1210static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1211 struct userfaultfd_wake_range *range)
1212{
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001213 unsigned seq;
1214 bool need_wakeup;
1215
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001216 /*
1217 * To be sure waitqueue_active() is not reordered by the CPU
1218 * before the pagetable update, use an explicit SMP memory
Michel Lespinasse3e4e28c2020-06-08 21:33:51 -07001219 * barrier here. PT lock release or mmap_read_unlock(mm) still
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001220 * have release semantics that can allow the
1221 * waitqueue_active() to be reordered before the pte update.
1222 */
1223 smp_mb();
1224
1225 /*
1226 * Use waitqueue_active because it's very frequent to
1227 * change the address space atomically even if there are no
1228 * userfaults yet. So we take the spinlock only when we're
1229 * sure we've userfaults to wake.
1230 */
Andrea Arcangeli2c5b7e12015-09-04 15:47:23 -07001231 do {
1232 seq = read_seqcount_begin(&ctx->refile_seq);
1233 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1234 waitqueue_active(&ctx->fault_wqh);
1235 cond_resched();
1236 } while (read_seqcount_retry(&ctx->refile_seq, seq));
1237 if (need_wakeup)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001238 __wake_userfault(ctx, range);
1239}
1240
1241static __always_inline int validate_range(struct mm_struct *mm,
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001242 __u64 start, __u64 len)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001243{
1244 __u64 task_size = mm->task_size;
1245
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001246 if (start & ~PAGE_MASK)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001247 return -EINVAL;
1248 if (len & ~PAGE_MASK)
1249 return -EINVAL;
1250 if (!len)
1251 return -EINVAL;
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001252 if (start < mmap_min_addr)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001253 return -EINVAL;
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001254 if (start >= task_size)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001255 return -EINVAL;
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001256 if (len > task_size - start)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001257 return -EINVAL;
1258 return 0;
1259}
1260
1261static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1262 unsigned long arg)
1263{
1264 struct mm_struct *mm = ctx->mm;
1265 struct vm_area_struct *vma, *prev, *cur;
1266 int ret;
1267 struct uffdio_register uffdio_register;
1268 struct uffdio_register __user *user_uffdio_register;
1269 unsigned long vm_flags, new_flags;
1270 bool found;
Mike Rapoportce53e8e2017-09-06 16:23:12 -07001271 bool basic_ioctls;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001272 unsigned long start, end, vma_end;
1273
1274 user_uffdio_register = (struct uffdio_register __user *) arg;
1275
1276 ret = -EFAULT;
1277 if (copy_from_user(&uffdio_register, user_uffdio_register,
1278 sizeof(uffdio_register)-sizeof(__u64)))
1279 goto out;
1280
1281 ret = -EINVAL;
1282 if (!uffdio_register.mode)
1283 goto out;
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001284 if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001285 goto out;
1286 vm_flags = 0;
1287 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1288 vm_flags |= VM_UFFD_MISSING;
Peter Xu00b151f22021-06-30 18:49:06 -07001289 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1290#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1291 goto out;
1292#endif
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001293 vm_flags |= VM_UFFD_WP;
Peter Xu00b151f22021-06-30 18:49:06 -07001294 }
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001295 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
1296#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1297 goto out;
1298#endif
1299 vm_flags |= VM_UFFD_MINOR;
1300 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001301
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001302 ret = validate_range(mm, uffdio_register.range.start,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001303 uffdio_register.range.len);
1304 if (ret)
1305 goto out;
1306
1307 start = uffdio_register.range.start;
1308 end = start + uffdio_register.range.len;
1309
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001310 ret = -ENOMEM;
1311 if (!mmget_not_zero(mm))
1312 goto out;
1313
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001314 mmap_write_lock(mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001315 vma = find_vma_prev(mm, start, &prev);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001316 if (!vma)
1317 goto out_unlock;
1318
1319 /* check that there's at least one vma in the range */
1320 ret = -EINVAL;
1321 if (vma->vm_start >= end)
1322 goto out_unlock;
1323
1324 /*
Mike Kravetzcab350a2017-02-22 15:43:04 -08001325 * If the first vma contains huge pages, make sure start address
1326 * is aligned to huge page size.
1327 */
1328 if (is_vm_hugetlb_page(vma)) {
1329 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1330
1331 if (start & (vma_hpagesize - 1))
1332 goto out_unlock;
1333 }
1334
1335 /*
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001336 * Search for not compatible vmas.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001337 */
1338 found = false;
Mike Rapoportce53e8e2017-09-06 16:23:12 -07001339 basic_ioctls = false;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001340 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1341 cond_resched();
1342
1343 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001344 !!(cur->vm_flags & __VM_UFFD_FLAGS));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001345
1346 /* check not compatible vmas */
1347 ret = -EINVAL;
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001348 if (!vma_can_userfault(cur, vm_flags))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001349 goto out_unlock;
Andrea Arcangeli29ec90662018-11-30 14:09:32 -08001350
1351 /*
1352 * UFFDIO_COPY will fill file holes even without
1353 * PROT_WRITE. This check enforces that if this is a
1354 * MAP_SHARED, the process has write permission to the backing
1355 * file. If VM_MAYWRITE is set it also enforces that on a
1356 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1357 * F_WRITE_SEAL can be taken until the vma is destroyed.
1358 */
1359 ret = -EPERM;
1360 if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1361 goto out_unlock;
1362
Mike Kravetzcab350a2017-02-22 15:43:04 -08001363 /*
1364 * If this vma contains ending address, and huge pages
1365 * check alignment.
1366 */
1367 if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1368 end > cur->vm_start) {
1369 unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1370
1371 ret = -EINVAL;
1372
1373 if (end & (vma_hpagesize - 1))
1374 goto out_unlock;
1375 }
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001376 if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))
1377 goto out_unlock;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001378
1379 /*
1380 * Check that this vma isn't already owned by a
1381 * different userfaultfd. We can't allow more than one
1382 * userfaultfd to own a single vma simultaneously or we
1383 * wouldn't know which one to deliver the userfaults to.
1384 */
1385 ret = -EBUSY;
1386 if (cur->vm_userfaultfd_ctx.ctx &&
1387 cur->vm_userfaultfd_ctx.ctx != ctx)
1388 goto out_unlock;
1389
Mike Kravetzcab350a2017-02-22 15:43:04 -08001390 /*
1391 * Note vmas containing huge pages
1392 */
Mike Rapoportce53e8e2017-09-06 16:23:12 -07001393 if (is_vm_hugetlb_page(cur))
1394 basic_ioctls = true;
Mike Kravetzcab350a2017-02-22 15:43:04 -08001395
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001396 found = true;
1397 }
1398 BUG_ON(!found);
1399
1400 if (vma->vm_start < start)
1401 prev = vma;
1402
1403 ret = 0;
1404 do {
1405 cond_resched();
1406
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001407 BUG_ON(!vma_can_userfault(vma, vm_flags));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001408 BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1409 vma->vm_userfaultfd_ctx.ctx != ctx);
Andrea Arcangeli29ec90662018-11-30 14:09:32 -08001410 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001411
1412 /*
1413 * Nothing to do: this vma is already registered into this
1414 * userfaultfd and with the right tracking mode too.
1415 */
1416 if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1417 (vma->vm_flags & vm_flags) == vm_flags)
1418 goto skip;
1419
1420 if (vma->vm_start > start)
1421 start = vma->vm_start;
1422 vma_end = min(end, vma->vm_end);
1423
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001424 new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001425 prev = vma_merge(mm, prev, start, vma_end, new_flags,
1426 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1427 vma_policy(vma),
Colin Cross9a100642022-01-14 14:05:59 -08001428 ((struct vm_userfaultfd_ctx){ ctx }),
Suren Baghdasaryan5c26f6a2022-03-04 20:28:51 -08001429 anon_vma_name(vma));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001430 if (prev) {
1431 vma = prev;
1432 goto next;
1433 }
1434 if (vma->vm_start < start) {
1435 ret = split_vma(mm, vma, start, 1);
1436 if (ret)
1437 break;
1438 }
1439 if (vma->vm_end > end) {
1440 ret = split_vma(mm, vma, end, 0);
1441 if (ret)
1442 break;
1443 }
1444 next:
1445 /*
1446 * In the vma_merge() successful mprotect-like case 8:
1447 * the next vma was merged into the current one and
1448 * the current one has not been updated yet.
1449 */
1450 vma->vm_flags = new_flags;
1451 vma->vm_userfaultfd_ctx.ctx = ctx;
1452
Peter Xu6dfeaff2021-05-04 18:33:13 -07001453 if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
1454 hugetlb_unshare_all_pmds(vma);
1455
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001456 skip:
1457 prev = vma;
1458 start = vma->vm_end;
1459 vma = vma->vm_next;
1460 } while (vma && vma->vm_start < end);
1461out_unlock:
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001462 mmap_write_unlock(mm);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001463 mmput(mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001464 if (!ret) {
Peter Xu14819302020-04-06 20:06:29 -07001465 __u64 ioctls_out;
1466
1467 ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1468 UFFD_API_RANGE_IOCTLS;
1469
1470 /*
1471 * Declare the WP ioctl only if the WP mode is
1472 * specified and all checks passed with the range
1473 */
1474 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))
1475 ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);
1476
Axel Rasmussenf6191472021-05-04 18:35:49 -07001477 /* CONTINUE ioctl is only supported for MINOR ranges. */
1478 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
1479 ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
1480
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001481 /*
1482 * Now that we scanned all vmas we can already tell
1483 * userland which ioctls methods are guaranteed to
1484 * succeed on this range.
1485 */
Peter Xu14819302020-04-06 20:06:29 -07001486 if (put_user(ioctls_out, &user_uffdio_register->ioctls))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001487 ret = -EFAULT;
1488 }
1489out:
1490 return ret;
1491}
1492
1493static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1494 unsigned long arg)
1495{
1496 struct mm_struct *mm = ctx->mm;
1497 struct vm_area_struct *vma, *prev, *cur;
1498 int ret;
1499 struct uffdio_range uffdio_unregister;
1500 unsigned long new_flags;
1501 bool found;
1502 unsigned long start, end, vma_end;
1503 const void __user *buf = (void __user *)arg;
1504
1505 ret = -EFAULT;
1506 if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1507 goto out;
1508
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001509 ret = validate_range(mm, uffdio_unregister.start,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001510 uffdio_unregister.len);
1511 if (ret)
1512 goto out;
1513
1514 start = uffdio_unregister.start;
1515 end = start + uffdio_unregister.len;
1516
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001517 ret = -ENOMEM;
1518 if (!mmget_not_zero(mm))
1519 goto out;
1520
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001521 mmap_write_lock(mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001522 vma = find_vma_prev(mm, start, &prev);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001523 if (!vma)
1524 goto out_unlock;
1525
1526 /* check that there's at least one vma in the range */
1527 ret = -EINVAL;
1528 if (vma->vm_start >= end)
1529 goto out_unlock;
1530
1531 /*
Mike Kravetzcab350a2017-02-22 15:43:04 -08001532 * If the first vma contains huge pages, make sure start address
1533 * is aligned to huge page size.
1534 */
1535 if (is_vm_hugetlb_page(vma)) {
1536 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1537
1538 if (start & (vma_hpagesize - 1))
1539 goto out_unlock;
1540 }
1541
1542 /*
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001543 * Search for not compatible vmas.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001544 */
1545 found = false;
1546 ret = -EINVAL;
1547 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1548 cond_resched();
1549
1550 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001551 !!(cur->vm_flags & __VM_UFFD_FLAGS));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001552
1553 /*
1554 * Check not compatible vmas, not strictly required
1555 * here as not compatible vmas cannot have an
1556 * userfaultfd_ctx registered on them, but this
1557 * provides for more strict behavior to notice
1558 * unregistration errors.
1559 */
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001560 if (!vma_can_userfault(cur, cur->vm_flags))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001561 goto out_unlock;
1562
1563 found = true;
1564 }
1565 BUG_ON(!found);
1566
1567 if (vma->vm_start < start)
1568 prev = vma;
1569
1570 ret = 0;
1571 do {
1572 cond_resched();
1573
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001574 BUG_ON(!vma_can_userfault(vma, vma->vm_flags));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001575
1576 /*
1577 * Nothing to do: this vma is already registered into this
1578 * userfaultfd and with the right tracking mode too.
1579 */
1580 if (!vma->vm_userfaultfd_ctx.ctx)
1581 goto skip;
1582
Andrea Arcangeli01e881f2018-12-14 14:17:17 -08001583 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1584
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001585 if (vma->vm_start > start)
1586 start = vma->vm_start;
1587 vma_end = min(end, vma->vm_end);
1588
Andrea Arcangeli09fa5292017-02-22 15:42:46 -08001589 if (userfaultfd_missing(vma)) {
1590 /*
1591 * Wake any concurrent pending userfault while
1592 * we unregister, so they will not hang
1593 * permanently and it avoids userland to call
1594 * UFFDIO_WAKE explicitly.
1595 */
1596 struct userfaultfd_wake_range range;
1597 range.start = start;
1598 range.len = vma_end - start;
1599 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1600 }
1601
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001602 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001603 prev = vma_merge(mm, prev, start, vma_end, new_flags,
1604 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1605 vma_policy(vma),
Suren Baghdasaryan5c26f6a2022-03-04 20:28:51 -08001606 NULL_VM_UFFD_CTX, anon_vma_name(vma));
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001607 if (prev) {
1608 vma = prev;
1609 goto next;
1610 }
1611 if (vma->vm_start < start) {
1612 ret = split_vma(mm, vma, start, 1);
1613 if (ret)
1614 break;
1615 }
1616 if (vma->vm_end > end) {
1617 ret = split_vma(mm, vma, end, 0);
1618 if (ret)
1619 break;
1620 }
1621 next:
1622 /*
1623 * In the vma_merge() successful mprotect-like case 8:
1624 * the next vma was merged into the current one and
1625 * the current one has not been updated yet.
1626 */
1627 vma->vm_flags = new_flags;
1628 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1629
1630 skip:
1631 prev = vma;
1632 start = vma->vm_end;
1633 vma = vma->vm_next;
1634 } while (vma && vma->vm_start < end);
1635out_unlock:
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001636 mmap_write_unlock(mm);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001637 mmput(mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001638out:
1639 return ret;
1640}
1641
1642/*
Andrea Arcangeliba85c702015-09-04 15:46:41 -07001643 * userfaultfd_wake may be used in combination with the
1644 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001645 */
1646static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1647 unsigned long arg)
1648{
1649 int ret;
1650 struct uffdio_range uffdio_wake;
1651 struct userfaultfd_wake_range range;
1652 const void __user *buf = (void __user *)arg;
1653
1654 ret = -EFAULT;
1655 if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1656 goto out;
1657
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001658 ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001659 if (ret)
1660 goto out;
1661
1662 range.start = uffdio_wake.start;
1663 range.len = uffdio_wake.len;
1664
1665 /*
1666 * len == 0 means wake all and we don't want to wake all here,
1667 * so check it again to be sure.
1668 */
1669 VM_BUG_ON(!range.len);
1670
1671 wake_userfault(ctx, &range);
1672 ret = 0;
1673
1674out:
1675 return ret;
1676}
1677
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001678static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1679 unsigned long arg)
1680{
1681 __s64 ret;
1682 struct uffdio_copy uffdio_copy;
1683 struct uffdio_copy __user *user_uffdio_copy;
1684 struct userfaultfd_wake_range range;
1685
1686 user_uffdio_copy = (struct uffdio_copy __user *) arg;
1687
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001688 ret = -EAGAIN;
Nadav Amita759a902021-09-02 14:58:56 -07001689 if (atomic_read(&ctx->mmap_changing))
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001690 goto out;
1691
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001692 ret = -EFAULT;
1693 if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1694 /* don't copy "copy" last field */
1695 sizeof(uffdio_copy)-sizeof(__s64)))
1696 goto out;
1697
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001698 ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001699 if (ret)
1700 goto out;
1701 /*
1702 * double check for wraparound just in case. copy_from_user()
1703 * will later check uffdio_copy.src + uffdio_copy.len to fit
1704 * in the userland range.
1705 */
1706 ret = -EINVAL;
1707 if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
1708 goto out;
Andrea Arcangeli72981e02020-04-06 20:05:41 -07001709 if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001710 goto out;
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001711 if (mmget_not_zero(ctx->mm)) {
1712 ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
Andrea Arcangeli72981e02020-04-06 20:05:41 -07001713 uffdio_copy.len, &ctx->mmap_changing,
1714 uffdio_copy.mode);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001715 mmput(ctx->mm);
Mike Rapoport96333182017-02-24 14:58:31 -08001716 } else {
Mike Rapoporte86b2982017-08-10 15:24:32 -07001717 return -ESRCH;
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001718 }
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001719 if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1720 return -EFAULT;
1721 if (ret < 0)
1722 goto out;
1723 BUG_ON(!ret);
1724 /* len == 0 would wake all */
1725 range.len = ret;
1726 if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1727 range.start = uffdio_copy.dst;
1728 wake_userfault(ctx, &range);
1729 }
1730 ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1731out:
1732 return ret;
1733}
1734
1735static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1736 unsigned long arg)
1737{
1738 __s64 ret;
1739 struct uffdio_zeropage uffdio_zeropage;
1740 struct uffdio_zeropage __user *user_uffdio_zeropage;
1741 struct userfaultfd_wake_range range;
1742
1743 user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1744
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001745 ret = -EAGAIN;
Nadav Amita759a902021-09-02 14:58:56 -07001746 if (atomic_read(&ctx->mmap_changing))
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001747 goto out;
1748
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001749 ret = -EFAULT;
1750 if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1751 /* don't copy "zeropage" last field */
1752 sizeof(uffdio_zeropage)-sizeof(__s64)))
1753 goto out;
1754
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001755 ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001756 uffdio_zeropage.range.len);
1757 if (ret)
1758 goto out;
1759 ret = -EINVAL;
1760 if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1761 goto out;
1762
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001763 if (mmget_not_zero(ctx->mm)) {
1764 ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
Mike Rapoportdf2cc962018-06-07 17:09:25 -07001765 uffdio_zeropage.range.len,
1766 &ctx->mmap_changing);
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001767 mmput(ctx->mm);
Mike Rapoport9d95aa42017-08-02 13:32:15 -07001768 } else {
Mike Rapoporte86b2982017-08-10 15:24:32 -07001769 return -ESRCH;
Oleg Nesterovd2005e32016-05-20 16:58:36 -07001770 }
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001771 if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1772 return -EFAULT;
1773 if (ret < 0)
1774 goto out;
1775 /* len == 0 would wake all */
1776 BUG_ON(!ret);
1777 range.len = ret;
1778 if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1779 range.start = uffdio_zeropage.range.start;
1780 wake_userfault(ctx, &range);
1781 }
1782 ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1783out:
1784 return ret;
1785}
1786
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001787static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
1788 unsigned long arg)
1789{
1790 int ret;
1791 struct uffdio_writeprotect uffdio_wp;
1792 struct uffdio_writeprotect __user *user_uffdio_wp;
1793 struct userfaultfd_wake_range range;
Peter Xu23080e22020-04-06 20:06:20 -07001794 bool mode_wp, mode_dontwake;
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001795
Nadav Amita759a902021-09-02 14:58:56 -07001796 if (atomic_read(&ctx->mmap_changing))
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001797 return -EAGAIN;
1798
1799 user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;
1800
1801 if (copy_from_user(&uffdio_wp, user_uffdio_wp,
1802 sizeof(struct uffdio_writeprotect)))
1803 return -EFAULT;
1804
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001805 ret = validate_range(ctx->mm, uffdio_wp.range.start,
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001806 uffdio_wp.range.len);
1807 if (ret)
1808 return ret;
1809
1810 if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |
1811 UFFDIO_WRITEPROTECT_MODE_WP))
1812 return -EINVAL;
Peter Xu23080e22020-04-06 20:06:20 -07001813
1814 mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;
1815 mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
1816
1817 if (mode_wp && mode_dontwake)
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001818 return -EINVAL;
1819
Nadav Amitcb185d52021-10-18 15:15:25 -07001820 if (mmget_not_zero(ctx->mm)) {
1821 ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start,
1822 uffdio_wp.range.len, mode_wp,
1823 &ctx->mmap_changing);
1824 mmput(ctx->mm);
1825 } else {
1826 return -ESRCH;
1827 }
1828
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001829 if (ret)
1830 return ret;
1831
Peter Xu23080e22020-04-06 20:06:20 -07001832 if (!mode_wp && !mode_dontwake) {
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001833 range.start = uffdio_wp.range.start;
1834 range.len = uffdio_wp.range.len;
1835 wake_userfault(ctx, &range);
1836 }
1837 return ret;
1838}
1839
Axel Rasmussenf6191472021-05-04 18:35:49 -07001840static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
1841{
1842 __s64 ret;
1843 struct uffdio_continue uffdio_continue;
1844 struct uffdio_continue __user *user_uffdio_continue;
1845 struct userfaultfd_wake_range range;
1846
1847 user_uffdio_continue = (struct uffdio_continue __user *)arg;
1848
1849 ret = -EAGAIN;
Nadav Amita759a902021-09-02 14:58:56 -07001850 if (atomic_read(&ctx->mmap_changing))
Axel Rasmussenf6191472021-05-04 18:35:49 -07001851 goto out;
1852
1853 ret = -EFAULT;
1854 if (copy_from_user(&uffdio_continue, user_uffdio_continue,
1855 /* don't copy the output fields */
1856 sizeof(uffdio_continue) - (sizeof(__s64))))
1857 goto out;
1858
Peter Collingbournee71e2ac2021-07-23 15:50:01 -07001859 ret = validate_range(ctx->mm, uffdio_continue.range.start,
Axel Rasmussenf6191472021-05-04 18:35:49 -07001860 uffdio_continue.range.len);
1861 if (ret)
1862 goto out;
1863
1864 ret = -EINVAL;
1865 /* double check for wraparound just in case. */
1866 if (uffdio_continue.range.start + uffdio_continue.range.len <=
1867 uffdio_continue.range.start) {
1868 goto out;
1869 }
1870 if (uffdio_continue.mode & ~UFFDIO_CONTINUE_MODE_DONTWAKE)
1871 goto out;
1872
1873 if (mmget_not_zero(ctx->mm)) {
1874 ret = mcopy_continue(ctx->mm, uffdio_continue.range.start,
1875 uffdio_continue.range.len,
1876 &ctx->mmap_changing);
1877 mmput(ctx->mm);
1878 } else {
1879 return -ESRCH;
1880 }
1881
1882 if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
1883 return -EFAULT;
1884 if (ret < 0)
1885 goto out;
1886
1887 /* len == 0 would wake all */
1888 BUG_ON(!ret);
1889 range.len = ret;
1890 if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {
1891 range.start = uffdio_continue.range.start;
1892 wake_userfault(ctx, &range);
1893 }
1894 ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;
1895
1896out:
1897 return ret;
1898}
1899
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001900static inline unsigned int uffd_ctx_features(__u64 user_features)
1901{
1902 /*
Nadav Amit22e5fe2a2021-09-02 14:58:59 -07001903 * For the current set of features the bits just coincide. Set
1904 * UFFD_FEATURE_INITIALIZED to mark the features as enabled.
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001905 */
Nadav Amit22e5fe2a2021-09-02 14:58:59 -07001906 return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08001907}
1908
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001909/*
1910 * userland asks for a certain API version and we return which bits
1911 * and ioctl commands are implemented in this kernel for such API
1912 * version or -EINVAL if unknown.
1913 */
1914static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1915 unsigned long arg)
1916{
1917 struct uffdio_api uffdio_api;
1918 void __user *buf = (void __user *)arg;
Nadav Amit22e5fe2a2021-09-02 14:58:59 -07001919 unsigned int ctx_features;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001920 int ret;
Andrea Arcangeli65603142017-02-22 15:42:24 -08001921 __u64 features;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001922
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001923 ret = -EFAULT;
Andrea Arcangelia9b85f92015-09-04 15:46:37 -07001924 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001925 goto out;
Andrea Arcangeli65603142017-02-22 15:42:24 -08001926 features = uffdio_api.features;
Mike Rapoport3c1c24d2019-11-30 17:58:01 -08001927 ret = -EINVAL;
1928 if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES))
1929 goto err_out;
1930 ret = -EPERM;
1931 if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
1932 goto err_out;
Andrea Arcangeli65603142017-02-22 15:42:24 -08001933 /* report all available features and ioctls to userland */
1934 uffdio_api.features = UFFD_API_FEATURES;
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001935#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
Axel Rasmussen964ab002021-06-30 18:49:27 -07001936 uffdio_api.features &=
1937 ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
Axel Rasmussen7677f7f2021-05-04 18:35:36 -07001938#endif
Peter Xu00b151f22021-06-30 18:49:06 -07001939#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1940 uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001941#endif
Peter Xub1f9e872022-05-12 20:22:56 -07001942#ifndef CONFIG_PTE_MARKER_UFFD_WP
1943 uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
1944#endif
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001945 uffdio_api.ioctls = UFFD_API_IOCTLS;
1946 ret = -EFAULT;
1947 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1948 goto out;
Nadav Amit22e5fe2a2021-09-02 14:58:59 -07001949
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001950 /* only enable the requested features for this uffd context */
Nadav Amit22e5fe2a2021-09-02 14:58:59 -07001951 ctx_features = uffd_ctx_features(features);
1952 ret = -EINVAL;
1953 if (cmpxchg(&ctx->features, 0, ctx_features) != 0)
1954 goto err_out;
1955
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001956 ret = 0;
1957out:
1958 return ret;
1959err_out:
1960 memset(&uffdio_api, 0, sizeof(uffdio_api));
1961 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1962 ret = -EFAULT;
1963 goto out;
1964}
1965
1966static long userfaultfd_ioctl(struct file *file, unsigned cmd,
Andrea Arcangelie6485a42015-09-04 15:47:15 -07001967 unsigned long arg)
1968{
1969 int ret = -EINVAL;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001970 struct userfaultfd_ctx *ctx = file->private_data;
1971
Nadav Amit22e5fe2a2021-09-02 14:58:59 -07001972 if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001973 return -EINVAL;
1974
1975 switch(cmd) {
1976 case UFFDIO_API:
1977 ret = userfaultfd_api(ctx, arg);
1978 break;
1979 case UFFDIO_REGISTER:
1980 ret = userfaultfd_register(ctx, arg);
1981 break;
1982 case UFFDIO_UNREGISTER:
1983 ret = userfaultfd_unregister(ctx, arg);
1984 break;
1985 case UFFDIO_WAKE:
Andrea Arcangeliad465cae2015-09-04 15:47:11 -07001986 ret = userfaultfd_wake(ctx, arg);
1987 break;
1988 case UFFDIO_COPY:
1989 ret = userfaultfd_copy(ctx, arg);
1990 break;
1991 case UFFDIO_ZEROPAGE:
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07001992 ret = userfaultfd_zeropage(ctx, arg);
1993 break;
Andrea Arcangeli63b2d412020-04-06 20:06:12 -07001994 case UFFDIO_WRITEPROTECT:
1995 ret = userfaultfd_writeprotect(ctx, arg);
1996 break;
Axel Rasmussenf6191472021-05-04 18:35:49 -07001997 case UFFDIO_CONTINUE:
1998 ret = userfaultfd_continue(ctx, arg);
1999 break;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002000 }
2001 return ret;
2002}
2003
2004#ifdef CONFIG_PROC_FS
2005static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
2006{
2007 struct userfaultfd_ctx *ctx = f->private_data;
Ingo Molnarac6424b2017-06-20 12:06:13 +02002008 wait_queue_entry_t *wq;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002009 unsigned long pending = 0, total = 0;
2010
Eric Biggerscbcfa132019-07-04 15:14:39 -07002011 spin_lock_irq(&ctx->fault_pending_wqh.lock);
Ingo Molnar2055da92017-06-20 12:06:46 +02002012 list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07002013 pending++;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002014 total++;
2015 }
Ingo Molnar2055da92017-06-20 12:06:46 +02002016 list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
Andrea Arcangeli15b726e2015-09-04 15:46:44 -07002017 total++;
2018 }
Eric Biggerscbcfa132019-07-04 15:14:39 -07002019 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002020
2021 /*
2022 * If more protocols will be added, there will be all shown
2023 * separated by a space. Like this:
2024 * protocols: aa:... bb:...
2025 */
2026 seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
Mike Rapoport045098e2017-04-07 16:04:42 -07002027 pending, total, UFFD_API, ctx->features,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002028 UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
2029}
2030#endif
2031
2032static const struct file_operations userfaultfd_fops = {
2033#ifdef CONFIG_PROC_FS
2034 .show_fdinfo = userfaultfd_show_fdinfo,
2035#endif
2036 .release = userfaultfd_release,
2037 .poll = userfaultfd_poll,
2038 .read = userfaultfd_read,
2039 .unlocked_ioctl = userfaultfd_ioctl,
Arnd Bergmann1832f2d2018-09-11 21:59:08 +02002040 .compat_ioctl = compat_ptr_ioctl,
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002041 .llseek = noop_llseek,
2042};
2043
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07002044static void init_once_userfaultfd_ctx(void *mem)
2045{
2046 struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
2047
2048 init_waitqueue_head(&ctx->fault_pending_wqh);
2049 init_waitqueue_head(&ctx->fault_wqh);
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08002050 init_waitqueue_head(&ctx->event_wqh);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07002051 init_waitqueue_head(&ctx->fd_wqh);
Ahmed S. Darwish2ca97ac2020-07-20 17:55:28 +02002052 seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07002053}
2054
Eric Biggers284cd242018-01-31 16:19:48 -08002055SYSCALL_DEFINE1(userfaultfd, int, flags)
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002056{
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002057 struct userfaultfd_ctx *ctx;
Eric Biggers284cd242018-01-31 16:19:48 -08002058 int fd;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002059
Lokesh Gidrad0d47302020-12-14 19:13:54 -08002060 if (!sysctl_unprivileged_userfaultfd &&
2061 (flags & UFFD_USER_MODE_ONLY) == 0 &&
2062 !capable(CAP_SYS_PTRACE)) {
2063 printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd "
2064 "sysctl knob to 1 if kernel faults must be handled "
2065 "without obtaining CAP_SYS_PTRACE capability\n");
Peter Xucefdca02019-05-13 17:16:41 -07002066 return -EPERM;
Lokesh Gidrad0d47302020-12-14 19:13:54 -08002067 }
Peter Xucefdca02019-05-13 17:16:41 -07002068
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002069 BUG_ON(!current->mm);
2070
2071 /* Check the UFFD_* constants for consistency. */
Lokesh Gidra37cd0572020-12-14 19:13:49 -08002072 BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002073 BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
2074 BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
2075
Lokesh Gidra37cd0572020-12-14 19:13:49 -08002076 if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))
Eric Biggers284cd242018-01-31 16:19:48 -08002077 return -EINVAL;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002078
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07002079 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002080 if (!ctx)
Eric Biggers284cd242018-01-31 16:19:48 -08002081 return -ENOMEM;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002082
Eric Biggersca880422018-12-28 00:34:43 -08002083 refcount_set(&ctx->refcount, 1);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002084 ctx->flags = flags;
Pavel Emelyanov9cd75c32017-02-22 15:42:21 -08002085 ctx->features = 0;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002086 ctx->released = false;
Nadav Amita759a902021-09-02 14:58:56 -07002087 atomic_set(&ctx->mmap_changing, 0);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002088 ctx->mm = current->mm;
2089 /* prevent the mm struct to be freed */
Vegard Nossumf1f10072017-02-27 14:30:07 -08002090 mmgrab(ctx->mm);
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002091
Daniel Colascioneb5379002021-01-08 14:22:23 -08002092 fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx,
2093 O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
Eric Biggers284cd242018-01-31 16:19:48 -08002094 if (fd < 0) {
Oleg Nesterovd2005e32016-05-20 16:58:36 -07002095 mmdrop(ctx->mm);
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07002096 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
Eric Biggersc03e946f2015-09-17 16:01:54 -07002097 }
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002098 return fd;
Andrea Arcangeli86039bd2015-09-04 15:46:31 -07002099}
Andrea Arcangeli3004ec92015-09-04 15:46:48 -07002100
2101static int __init userfaultfd_init(void)
2102{
2103 userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
2104 sizeof(struct userfaultfd_ctx),
2105 0,
2106 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2107 init_once_userfaultfd_ctx);
2108 return 0;
2109}
2110__initcall(userfaultfd_init);