Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/kernel/fork.c |
| 3 | * |
| 4 | * Copyright (C) 1991, 1992 Linus Torvalds |
| 5 | */ |
| 6 | |
| 7 | /* |
| 8 | * 'fork.c' contains the help-routines for the 'fork' system call |
| 9 | * (see also entry.S and others). |
| 10 | * Fork is rather simple, once you get the hang of it, but the memory |
| 11 | * management can be a bitch. See 'mm/memory.c': 'copy_page_range()' |
| 12 | */ |
| 13 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <linux/slab.h> |
| 15 | #include <linux/init.h> |
| 16 | #include <linux/unistd.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <linux/module.h> |
| 18 | #include <linux/vmalloc.h> |
| 19 | #include <linux/completion.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <linux/personality.h> |
| 21 | #include <linux/mempolicy.h> |
| 22 | #include <linux/sem.h> |
| 23 | #include <linux/file.h> |
Al Viro | 9f3acc3 | 2008-04-24 07:44:08 -0400 | [diff] [blame] | 24 | #include <linux/fdtable.h> |
Jens Axboe | da9cbc8 | 2008-06-30 20:42:08 +0200 | [diff] [blame] | 25 | #include <linux/iocontext.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #include <linux/key.h> |
| 27 | #include <linux/binfmts.h> |
| 28 | #include <linux/mman.h> |
Andrea Arcangeli | cddb8a5 | 2008-07-28 15:46:29 -0700 | [diff] [blame] | 29 | #include <linux/mmu_notifier.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | #include <linux/fs.h> |
Serge E. Hallyn | ab51601 | 2006-10-02 02:18:06 -0700 | [diff] [blame] | 31 | #include <linux/nsproxy.h> |
Randy.Dunlap | c59ede7 | 2006-01-11 12:17:46 -0800 | [diff] [blame] | 32 | #include <linux/capability.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 33 | #include <linux/cpu.h> |
Paul Menage | b4f48b6 | 2007-10-18 23:39:33 -0700 | [diff] [blame] | 34 | #include <linux/cgroup.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | #include <linux/security.h> |
Mel Gorman | a1e7877 | 2008-07-23 21:27:23 -0700 | [diff] [blame] | 36 | #include <linux/hugetlb.h> |
Will Drewry | e2cfabdf | 2012-04-12 16:47:57 -0500 | [diff] [blame] | 37 | #include <linux/seccomp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | #include <linux/swap.h> |
| 39 | #include <linux/syscalls.h> |
| 40 | #include <linux/jiffies.h> |
| 41 | #include <linux/futex.h> |
Linus Torvalds | 8141c7f | 2008-11-15 10:20:36 -0800 | [diff] [blame] | 42 | #include <linux/compat.h> |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 43 | #include <linux/kthread.h> |
Andrew Morton | 7c3ab738 | 2006-12-10 02:19:19 -0800 | [diff] [blame] | 44 | #include <linux/task_io_accounting_ops.h> |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 45 | #include <linux/rcupdate.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 46 | #include <linux/ptrace.h> |
| 47 | #include <linux/mount.h> |
| 48 | #include <linux/audit.h> |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 49 | #include <linux/memcontrol.h> |
Frederic Weisbecker | f201ae2 | 2008-11-23 06:22:56 +0100 | [diff] [blame] | 50 | #include <linux/ftrace.h> |
Mike Galbraith | 5e2bf01 | 2012-05-10 13:01:45 -0700 | [diff] [blame] | 51 | #include <linux/proc_fs.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 52 | #include <linux/profile.h> |
| 53 | #include <linux/rmap.h> |
Hugh Dickins | f8af4da | 2009-09-21 17:01:57 -0700 | [diff] [blame] | 54 | #include <linux/ksm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | #include <linux/acct.h> |
Jay Lan | 8f0ab51 | 2006-09-30 23:28:59 -0700 | [diff] [blame] | 56 | #include <linux/tsacct_kern.h> |
Matt Helsley | 9f46080 | 2005-11-07 00:59:16 -0800 | [diff] [blame] | 57 | #include <linux/cn_proc.h> |
Rafael J. Wysocki | ba96a0c | 2007-05-23 13:57:25 -0700 | [diff] [blame] | 58 | #include <linux/freezer.h> |
Shailabh Nagar | ca74e92 | 2006-07-14 00:24:36 -0700 | [diff] [blame] | 59 | #include <linux/delayacct.h> |
Shailabh Nagar | ad4ecbc | 2006-07-14 00:24:44 -0700 | [diff] [blame] | 60 | #include <linux/taskstats_kern.h> |
Arjan van de Ven | 0a425405 | 2006-09-26 10:52:38 +0200 | [diff] [blame] | 61 | #include <linux/random.h> |
Miloslav Trmac | 522ed77 | 2007-07-15 23:40:56 -0700 | [diff] [blame] | 62 | #include <linux/tty.h> |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 63 | #include <linux/blkdev.h> |
Al Viro | 5ad4e53 | 2009-03-29 19:50:06 -0400 | [diff] [blame] | 64 | #include <linux/fs_struct.h> |
Eric Sandeen | 7c9f886 | 2008-04-22 16:38:23 -0500 | [diff] [blame] | 65 | #include <linux/magic.h> |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 66 | #include <linux/perf_event.h> |
Stanislaw Gruszka | 42c4ab4 | 2009-07-29 12:15:26 +0200 | [diff] [blame] | 67 | #include <linux/posix-timers.h> |
Avi Kivity | 8e7cac7 | 2009-11-29 16:34:48 +0200 | [diff] [blame] | 68 | #include <linux/user-return-notifier.h> |
Ying Han | 3d5992d | 2010-10-26 14:21:23 -0700 | [diff] [blame] | 69 | #include <linux/oom.h> |
Andrea Arcangeli | ba76149 | 2011-01-13 15:46:58 -0800 | [diff] [blame] | 70 | #include <linux/khugepaged.h> |
Oleg Nesterov | d80e731 | 2012-02-24 20:07:11 +0100 | [diff] [blame] | 71 | #include <linux/signalfd.h> |
Srikar Dronamraju | 0326f5a9 | 2012-03-13 23:30:11 +0530 | [diff] [blame] | 72 | #include <linux/uprobes.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | |
| 74 | #include <asm/pgtable.h> |
| 75 | #include <asm/pgalloc.h> |
| 76 | #include <asm/uaccess.h> |
| 77 | #include <asm/mmu_context.h> |
| 78 | #include <asm/cacheflush.h> |
| 79 | #include <asm/tlbflush.h> |
| 80 | |
Steven Rostedt | ad8d75f | 2009-04-14 19:39:12 -0400 | [diff] [blame] | 81 | #include <trace/events/sched.h> |
| 82 | |
KAMEZAWA Hiroyuki | 43d2b11 | 2012-01-10 15:08:09 -0800 | [diff] [blame] | 83 | #define CREATE_TRACE_POINTS |
| 84 | #include <trace/events/task.h> |
| 85 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 86 | /* |
| 87 | * Protected counters by write_lock_irq(&tasklist_lock) |
| 88 | */ |
| 89 | unsigned long total_forks; /* Handle normal Linux uptimes. */ |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 90 | int nr_threads; /* The idle threads do not count.. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 91 | |
| 92 | int max_threads; /* tunable limit on nr_threads */ |
| 93 | |
| 94 | DEFINE_PER_CPU(unsigned long, process_counts) = 0; |
| 95 | |
Christoph Hellwig | c59923a | 2006-07-10 04:45:40 -0700 | [diff] [blame] | 96 | __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ |
Paul E. McKenney | db1466b | 2010-03-03 07:46:56 -0800 | [diff] [blame] | 97 | |
| 98 | #ifdef CONFIG_PROVE_RCU |
| 99 | int lockdep_tasklist_lock_is_held(void) |
| 100 | { |
| 101 | return lockdep_is_held(&tasklist_lock); |
| 102 | } |
| 103 | EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); |
| 104 | #endif /* #ifdef CONFIG_PROVE_RCU */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | |
| 106 | int nr_processes(void) |
| 107 | { |
| 108 | int cpu; |
| 109 | int total = 0; |
| 110 | |
Ian Campbell | 1d51075 | 2009-11-03 10:11:14 +0000 | [diff] [blame] | 111 | for_each_possible_cpu(cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | total += per_cpu(process_counts, cpu); |
| 113 | |
| 114 | return total; |
| 115 | } |
| 116 | |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 117 | void __weak arch_release_task_struct(struct task_struct *tsk) |
| 118 | { |
| 119 | } |
| 120 | |
Thomas Gleixner | f5e1028 | 2012-05-05 15:05:48 +0000 | [diff] [blame] | 121 | #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 122 | static struct kmem_cache *task_struct_cachep; |
Thomas Gleixner | 4110180 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 123 | |
| 124 | static inline struct task_struct *alloc_task_struct_node(int node) |
| 125 | { |
| 126 | return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node); |
| 127 | } |
| 128 | |
Thomas Gleixner | 4110180 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 129 | static inline void free_task_struct(struct task_struct *tsk) |
| 130 | { |
Thomas Gleixner | 4110180 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 131 | kmem_cache_free(task_struct_cachep, tsk); |
| 132 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | #endif |
| 134 | |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 135 | void __weak arch_release_thread_info(struct thread_info *ti) |
| 136 | { |
| 137 | } |
| 138 | |
Thomas Gleixner | f5e1028 | 2012-05-05 15:05:48 +0000 | [diff] [blame] | 139 | #ifndef CONFIG_ARCH_THREAD_INFO_ALLOCATOR |
Thomas Gleixner | 4110180 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 140 | |
Thomas Gleixner | 0d15d74 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 141 | /* |
| 142 | * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a |
| 143 | * kmemcache based allocator. |
| 144 | */ |
| 145 | # if THREAD_SIZE >= PAGE_SIZE |
Eric Dumazet | b6a8401 | 2011-03-22 16:30:42 -0700 | [diff] [blame] | 146 | static struct thread_info *alloc_thread_info_node(struct task_struct *tsk, |
| 147 | int node) |
FUJITA Tomonori | b69c49b | 2008-07-25 01:45:40 -0700 | [diff] [blame] | 148 | { |
Thomas Gleixner | 2889f60 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 149 | struct page *page = alloc_pages_node(node, THREADINFO_GFP, |
| 150 | THREAD_SIZE_ORDER); |
Eric Dumazet | b6a8401 | 2011-03-22 16:30:42 -0700 | [diff] [blame] | 151 | |
| 152 | return page ? page_address(page) : NULL; |
FUJITA Tomonori | b69c49b | 2008-07-25 01:45:40 -0700 | [diff] [blame] | 153 | } |
| 154 | |
| 155 | static inline void free_thread_info(struct thread_info *ti) |
| 156 | { |
| 157 | free_pages((unsigned long)ti, THREAD_SIZE_ORDER); |
| 158 | } |
Thomas Gleixner | 0d15d74 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 159 | # else |
| 160 | static struct kmem_cache *thread_info_cache; |
| 161 | |
| 162 | static struct thread_info *alloc_thread_info_node(struct task_struct *tsk, |
| 163 | int node) |
| 164 | { |
| 165 | return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node); |
| 166 | } |
| 167 | |
| 168 | static void free_thread_info(struct thread_info *ti) |
| 169 | { |
Thomas Gleixner | 0d15d74 | 2012-05-05 15:05:41 +0000 | [diff] [blame] | 170 | kmem_cache_free(thread_info_cache, ti); |
| 171 | } |
| 172 | |
| 173 | void thread_info_cache_init(void) |
| 174 | { |
| 175 | thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE, |
| 176 | THREAD_SIZE, 0, NULL); |
| 177 | BUG_ON(thread_info_cache == NULL); |
| 178 | } |
| 179 | # endif |
FUJITA Tomonori | b69c49b | 2008-07-25 01:45:40 -0700 | [diff] [blame] | 180 | #endif |
| 181 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 182 | /* SLAB cache for signal_struct structures (tsk->signal) */ |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 183 | static struct kmem_cache *signal_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | |
| 185 | /* SLAB cache for sighand_struct structures (tsk->sighand) */ |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 186 | struct kmem_cache *sighand_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 187 | |
| 188 | /* SLAB cache for files_struct structures (tsk->files) */ |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 189 | struct kmem_cache *files_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 190 | |
| 191 | /* SLAB cache for fs_struct structures (tsk->fs) */ |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 192 | struct kmem_cache *fs_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | |
| 194 | /* SLAB cache for vm_area_struct structures */ |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 195 | struct kmem_cache *vm_area_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | |
| 197 | /* SLAB cache for mm_struct structures (tsk->mm) */ |
Christoph Lameter | e18b890 | 2006-12-06 20:33:20 -0800 | [diff] [blame] | 198 | static struct kmem_cache *mm_cachep; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | |
KOSAKI Motohiro | c6a7f57 | 2009-09-21 17:01:32 -0700 | [diff] [blame] | 200 | static void account_kernel_stack(struct thread_info *ti, int account) |
| 201 | { |
| 202 | struct zone *zone = page_zone(virt_to_page(ti)); |
| 203 | |
| 204 | mod_zone_page_state(zone, NR_KERNEL_STACK, account); |
| 205 | } |
| 206 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 207 | void free_task(struct task_struct *tsk) |
| 208 | { |
KOSAKI Motohiro | c6a7f57 | 2009-09-21 17:01:32 -0700 | [diff] [blame] | 209 | account_kernel_stack(tsk->stack, -1); |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 210 | arch_release_thread_info(tsk->stack); |
Roman Zippel | f7e4217 | 2007-05-09 02:35:17 -0700 | [diff] [blame] | 211 | free_thread_info(tsk->stack); |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 212 | rt_mutex_debug_task_free(tsk); |
Frederic Weisbecker | fb52607 | 2008-11-25 21:07:04 +0100 | [diff] [blame] | 213 | ftrace_graph_exit_task(tsk); |
Will Drewry | e2cfabdf | 2012-04-12 16:47:57 -0500 | [diff] [blame] | 214 | put_seccomp_filter(tsk); |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 215 | arch_release_task_struct(tsk); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 216 | free_task_struct(tsk); |
| 217 | } |
| 218 | EXPORT_SYMBOL(free_task); |
| 219 | |
Oleg Nesterov | ea6d290 | 2010-05-26 14:43:16 -0700 | [diff] [blame] | 220 | static inline void free_signal_struct(struct signal_struct *sig) |
| 221 | { |
Oleg Nesterov | 97101eb | 2010-05-26 14:43:20 -0700 | [diff] [blame] | 222 | taskstats_tgid_free(sig); |
Mike Galbraith | 1c5354d | 2011-01-05 11:16:04 +0100 | [diff] [blame] | 223 | sched_autogroup_exit(sig); |
Oleg Nesterov | ea6d290 | 2010-05-26 14:43:16 -0700 | [diff] [blame] | 224 | kmem_cache_free(signal_cachep, sig); |
| 225 | } |
| 226 | |
| 227 | static inline void put_signal_struct(struct signal_struct *sig) |
| 228 | { |
Mike Galbraith | 1c5354d | 2011-01-05 11:16:04 +0100 | [diff] [blame] | 229 | if (atomic_dec_and_test(&sig->sigcnt)) |
Oleg Nesterov | ea6d290 | 2010-05-26 14:43:16 -0700 | [diff] [blame] | 230 | free_signal_struct(sig); |
| 231 | } |
| 232 | |
Andrew Morton | 158d9eb | 2006-03-31 02:31:34 -0800 | [diff] [blame] | 233 | void __put_task_struct(struct task_struct *tsk) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 234 | { |
Eugene Teo | 270f722 | 2007-10-18 23:40:38 -0700 | [diff] [blame] | 235 | WARN_ON(!tsk->exit_state); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 236 | WARN_ON(atomic_read(&tsk->usage)); |
| 237 | WARN_ON(tsk == current); |
| 238 | |
Kees Cook | 1a2a4d0 | 2011-12-21 12:17:03 -0800 | [diff] [blame] | 239 | security_task_free(tsk); |
David Howells | e0e8173 | 2009-09-02 09:13:40 +0100 | [diff] [blame] | 240 | exit_creds(tsk); |
Shailabh Nagar | 35df17c | 2006-08-31 21:27:38 -0700 | [diff] [blame] | 241 | delayacct_tsk_free(tsk); |
Oleg Nesterov | ea6d290 | 2010-05-26 14:43:16 -0700 | [diff] [blame] | 242 | put_signal_struct(tsk->signal); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 243 | |
| 244 | if (!profile_handoff_task(tsk)) |
| 245 | free_task(tsk); |
| 246 | } |
Rik van Riel | 77c100c | 2011-02-01 09:51:46 -0500 | [diff] [blame] | 247 | EXPORT_SYMBOL_GPL(__put_task_struct); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 248 | |
Thomas Gleixner | 6c0a9fa | 2012-05-05 15:05:40 +0000 | [diff] [blame] | 249 | void __init __weak arch_task_cache_init(void) { } |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 250 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 251 | void __init fork_init(unsigned long mempages) |
| 252 | { |
Thomas Gleixner | f5e1028 | 2012-05-05 15:05:48 +0000 | [diff] [blame] | 253 | #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 254 | #ifndef ARCH_MIN_TASKALIGN |
| 255 | #define ARCH_MIN_TASKALIGN L1_CACHE_BYTES |
| 256 | #endif |
| 257 | /* create a slab on which task_structs can be allocated */ |
| 258 | task_struct_cachep = |
| 259 | kmem_cache_create("task_struct", sizeof(struct task_struct), |
Vegard Nossum | 2dff440 | 2008-05-31 15:56:17 +0200 | [diff] [blame] | 260 | ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 261 | #endif |
| 262 | |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 263 | /* do the arch specific task caches init */ |
| 264 | arch_task_cache_init(); |
| 265 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | /* |
| 267 | * The default maximum number of threads is set to a safe |
| 268 | * value: the thread structures can take up at most half |
| 269 | * of memory. |
| 270 | */ |
| 271 | max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE); |
| 272 | |
| 273 | /* |
| 274 | * we need to allow at least 20 threads to boot a system |
| 275 | */ |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 276 | if (max_threads < 20) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 277 | max_threads = 20; |
| 278 | |
| 279 | init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; |
| 280 | init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; |
| 281 | init_task.signal->rlim[RLIMIT_SIGPENDING] = |
| 282 | init_task.signal->rlim[RLIMIT_NPROC]; |
| 283 | } |
| 284 | |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 285 | int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst, |
| 286 | struct task_struct *src) |
| 287 | { |
| 288 | *dst = *src; |
| 289 | return 0; |
| 290 | } |
| 291 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 292 | static struct task_struct *dup_task_struct(struct task_struct *orig) |
| 293 | { |
| 294 | struct task_struct *tsk; |
| 295 | struct thread_info *ti; |
Eric Sandeen | 7c9f886 | 2008-04-22 16:38:23 -0500 | [diff] [blame] | 296 | unsigned long *stackend; |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 297 | int node = tsk_fork_get_node(orig); |
Peter Zijlstra | 3e26c14 | 2007-10-16 23:25:50 -0700 | [diff] [blame] | 298 | int err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | |
Eric Dumazet | 504f52b | 2011-03-22 16:30:41 -0700 | [diff] [blame] | 300 | tsk = alloc_task_struct_node(node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 301 | if (!tsk) |
| 302 | return NULL; |
| 303 | |
Eric Dumazet | b6a8401 | 2011-03-22 16:30:42 -0700 | [diff] [blame] | 304 | ti = alloc_thread_info_node(tsk, node); |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 305 | if (!ti) |
| 306 | goto free_tsk; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 307 | |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 308 | err = arch_dup_task_struct(tsk, orig); |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 309 | if (err) |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 310 | goto free_ti; |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 311 | |
Andrew Morton | 87bec58 | 2012-07-30 14:42:31 -0700 | [diff] [blame] | 312 | tsk->stack = ti; |
| 313 | |
| 314 | setup_thread_stack(tsk, orig); |
Avi Kivity | 8e7cac7 | 2009-11-29 16:34:48 +0200 | [diff] [blame] | 315 | clear_user_return_notifier(tsk); |
Mike Galbraith | f26f9af | 2010-12-08 11:05:42 +0100 | [diff] [blame] | 316 | clear_tsk_need_resched(tsk); |
Eric Sandeen | 7c9f886 | 2008-04-22 16:38:23 -0500 | [diff] [blame] | 317 | stackend = end_of_stack(tsk); |
| 318 | *stackend = STACK_END_MAGIC; /* for overflow detection */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 319 | |
Arjan van de Ven | 0a425405 | 2006-09-26 10:52:38 +0200 | [diff] [blame] | 320 | #ifdef CONFIG_CC_STACKPROTECTOR |
| 321 | tsk->stack_canary = get_random_int(); |
| 322 | #endif |
| 323 | |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 324 | /* |
| 325 | * One for us, one for whoever does the "release_task()" (usually |
| 326 | * parent) |
| 327 | */ |
| 328 | atomic_set(&tsk->usage, 2); |
Alexey Dobriyan | 6c5c934 | 2006-09-29 01:59:40 -0700 | [diff] [blame] | 329 | #ifdef CONFIG_BLK_DEV_IO_TRACE |
Jens Axboe | 2056a78 | 2006-03-23 20:00:26 +0100 | [diff] [blame] | 330 | tsk->btrace_seq = 0; |
Alexey Dobriyan | 6c5c934 | 2006-09-29 01:59:40 -0700 | [diff] [blame] | 331 | #endif |
Jens Axboe | a0aa7f6 | 2006-04-20 13:05:33 +0200 | [diff] [blame] | 332 | tsk->splice_pipe = NULL; |
Eric Dumazet | 5640f76 | 2012-09-23 23:04:42 +0000 | [diff] [blame] | 333 | tsk->task_frag.page = NULL; |
KOSAKI Motohiro | c6a7f57 | 2009-09-21 17:01:32 -0700 | [diff] [blame] | 334 | |
| 335 | account_kernel_stack(ti, 1); |
| 336 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 337 | return tsk; |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 338 | |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 339 | free_ti: |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 340 | free_thread_info(ti); |
Akinobu Mita | f19b9f7 | 2012-07-30 14:42:33 -0700 | [diff] [blame] | 341 | free_tsk: |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 342 | free_task_struct(tsk); |
| 343 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 344 | } |
| 345 | |
| 346 | #ifdef CONFIG_MMU |
Alexey Dobriyan | a39bc51 | 2007-10-18 23:41:10 -0700 | [diff] [blame] | 347 | static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 348 | { |
Linus Torvalds | 297c5ee | 2010-08-20 16:24:55 -0700 | [diff] [blame] | 349 | struct vm_area_struct *mpnt, *tmp, *prev, **pprev; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 350 | struct rb_node **rb_link, *rb_parent; |
| 351 | int retval; |
| 352 | unsigned long charge; |
| 353 | struct mempolicy *pol; |
| 354 | |
| 355 | down_write(&oldmm->mmap_sem); |
Ralf Baechle | ec8c0446 | 2006-12-12 17:14:57 +0000 | [diff] [blame] | 356 | flush_cache_dup_mm(oldmm); |
Oleg Nesterov | f8ac4ec | 2012-08-08 17:11:42 +0200 | [diff] [blame] | 357 | uprobe_dup_mmap(oldmm, mm); |
Ingo Molnar | ad33945 | 2006-07-03 00:25:15 -0700 | [diff] [blame] | 358 | /* |
| 359 | * Not linked in yet - no deadlock potential: |
| 360 | */ |
| 361 | down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); |
Hugh Dickins | 7ee7823 | 2005-10-29 18:16:08 -0700 | [diff] [blame] | 362 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 363 | mm->locked_vm = 0; |
| 364 | mm->mmap = NULL; |
| 365 | mm->mmap_cache = NULL; |
| 366 | mm->free_area_cache = oldmm->mmap_base; |
Wolfgang Wander | 1363c3c | 2005-06-21 17:14:49 -0700 | [diff] [blame] | 367 | mm->cached_hole_size = ~0UL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 368 | mm->map_count = 0; |
Rusty Russell | 9489424 | 2009-03-30 22:05:12 -0600 | [diff] [blame] | 369 | cpumask_clear(mm_cpumask(mm)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | mm->mm_rb = RB_ROOT; |
| 371 | rb_link = &mm->mm_rb.rb_node; |
| 372 | rb_parent = NULL; |
| 373 | pprev = &mm->mmap; |
Hugh Dickins | f8af4da | 2009-09-21 17:01:57 -0700 | [diff] [blame] | 374 | retval = ksm_fork(mm, oldmm); |
| 375 | if (retval) |
| 376 | goto out; |
Andrea Arcangeli | ba76149 | 2011-01-13 15:46:58 -0800 | [diff] [blame] | 377 | retval = khugepaged_fork(mm, oldmm); |
| 378 | if (retval) |
| 379 | goto out; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 380 | |
Linus Torvalds | 297c5ee | 2010-08-20 16:24:55 -0700 | [diff] [blame] | 381 | prev = NULL; |
Hugh Dickins | fd3e42f | 2005-10-29 18:16:06 -0700 | [diff] [blame] | 382 | for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 383 | struct file *file; |
| 384 | |
| 385 | if (mpnt->vm_flags & VM_DONTCOPY) { |
Hugh Dickins | ab50b8e | 2005-10-29 18:15:56 -0700 | [diff] [blame] | 386 | vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file, |
Huang Shijie | 44de9d0 | 2012-07-31 16:41:49 -0700 | [diff] [blame] | 387 | -vma_pages(mpnt)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | continue; |
| 389 | } |
| 390 | charge = 0; |
| 391 | if (mpnt->vm_flags & VM_ACCOUNT) { |
Huang Shijie | b2412b7 | 2012-07-30 14:42:30 -0700 | [diff] [blame] | 392 | unsigned long len = vma_pages(mpnt); |
| 393 | |
Al Viro | 191c542 | 2012-02-13 03:58:52 +0000 | [diff] [blame] | 394 | if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 395 | goto fail_nomem; |
| 396 | charge = len; |
| 397 | } |
Christoph Lameter | e94b176 | 2006-12-06 20:33:17 -0800 | [diff] [blame] | 398 | tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 399 | if (!tmp) |
| 400 | goto fail_nomem; |
| 401 | *tmp = *mpnt; |
Rik van Riel | 5beb493 | 2010-03-05 13:42:07 -0800 | [diff] [blame] | 402 | INIT_LIST_HEAD(&tmp->anon_vma_chain); |
Lee Schermerhorn | 846a16b | 2008-04-28 02:13:09 -0700 | [diff] [blame] | 403 | pol = mpol_dup(vma_policy(mpnt)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 404 | retval = PTR_ERR(pol); |
| 405 | if (IS_ERR(pol)) |
| 406 | goto fail_nomem_policy; |
| 407 | vma_set_policy(tmp, pol); |
Andrea Arcangeli | a247c3a | 2010-09-22 13:05:12 -0700 | [diff] [blame] | 408 | tmp->vm_mm = mm; |
Rik van Riel | 5beb493 | 2010-03-05 13:42:07 -0800 | [diff] [blame] | 409 | if (anon_vma_fork(tmp, mpnt)) |
| 410 | goto fail_nomem_anon_vma_fork; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 411 | tmp->vm_flags &= ~VM_LOCKED; |
Linus Torvalds | 297c5ee | 2010-08-20 16:24:55 -0700 | [diff] [blame] | 412 | tmp->vm_next = tmp->vm_prev = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 413 | file = tmp->vm_file; |
| 414 | if (file) { |
Josef "Jeff" Sipek | f3a43f3 | 2006-12-08 02:36:43 -0800 | [diff] [blame] | 415 | struct inode *inode = file->f_path.dentry->d_inode; |
Hugh Dickins | b88ed20 | 2008-12-10 20:48:52 +0000 | [diff] [blame] | 416 | struct address_space *mapping = file->f_mapping; |
| 417 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | get_file(file); |
| 419 | if (tmp->vm_flags & VM_DENYWRITE) |
| 420 | atomic_dec(&inode->i_writecount); |
Peter Zijlstra | 3d48ae4 | 2011-05-24 17:12:06 -0700 | [diff] [blame] | 421 | mutex_lock(&mapping->i_mmap_mutex); |
Hugh Dickins | b88ed20 | 2008-12-10 20:48:52 +0000 | [diff] [blame] | 422 | if (tmp->vm_flags & VM_SHARED) |
| 423 | mapping->i_mmap_writable++; |
Hugh Dickins | b88ed20 | 2008-12-10 20:48:52 +0000 | [diff] [blame] | 424 | flush_dcache_mmap_lock(mapping); |
| 425 | /* insert tmp into the share list, just after mpnt */ |
Michel Lespinasse | 9826a51 | 2012-10-08 16:31:35 -0700 | [diff] [blame] | 426 | if (unlikely(tmp->vm_flags & VM_NONLINEAR)) |
| 427 | vma_nonlinear_insert(tmp, |
| 428 | &mapping->i_mmap_nonlinear); |
| 429 | else |
| 430 | vma_interval_tree_insert_after(tmp, mpnt, |
| 431 | &mapping->i_mmap); |
Hugh Dickins | b88ed20 | 2008-12-10 20:48:52 +0000 | [diff] [blame] | 432 | flush_dcache_mmap_unlock(mapping); |
Peter Zijlstra | 3d48ae4 | 2011-05-24 17:12:06 -0700 | [diff] [blame] | 433 | mutex_unlock(&mapping->i_mmap_mutex); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 434 | } |
| 435 | |
| 436 | /* |
Mel Gorman | a1e7877 | 2008-07-23 21:27:23 -0700 | [diff] [blame] | 437 | * Clear hugetlb-related page reserves for children. This only |
| 438 | * affects MAP_PRIVATE mappings. Faults generated by the child |
| 439 | * are not guaranteed to succeed, even if read-only |
| 440 | */ |
| 441 | if (is_vm_hugetlb_page(tmp)) |
| 442 | reset_vma_resv_huge_pages(tmp); |
| 443 | |
| 444 | /* |
Hugh Dickins | 7ee7823 | 2005-10-29 18:16:08 -0700 | [diff] [blame] | 445 | * Link in the new vma and copy the page table entries. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 446 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 447 | *pprev = tmp; |
| 448 | pprev = &tmp->vm_next; |
Linus Torvalds | 297c5ee | 2010-08-20 16:24:55 -0700 | [diff] [blame] | 449 | tmp->vm_prev = prev; |
| 450 | prev = tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 451 | |
| 452 | __vma_link_rb(mm, tmp, rb_link, rb_parent); |
| 453 | rb_link = &tmp->vm_rb.rb_right; |
| 454 | rb_parent = &tmp->vm_rb; |
| 455 | |
| 456 | mm->map_count++; |
Hugh Dickins | 0b0db14 | 2005-11-21 21:32:20 -0800 | [diff] [blame] | 457 | retval = copy_page_range(mm, oldmm, mpnt); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 458 | |
| 459 | if (tmp->vm_ops && tmp->vm_ops->open) |
| 460 | tmp->vm_ops->open(tmp); |
| 461 | |
| 462 | if (retval) |
| 463 | goto out; |
| 464 | } |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 465 | /* a new mm has just been created */ |
| 466 | arch_dup_mmap(oldmm, mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 467 | retval = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | out: |
Hugh Dickins | 7ee7823 | 2005-10-29 18:16:08 -0700 | [diff] [blame] | 469 | up_write(&mm->mmap_sem); |
Hugh Dickins | fd3e42f | 2005-10-29 18:16:06 -0700 | [diff] [blame] | 470 | flush_tlb_mm(oldmm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | up_write(&oldmm->mmap_sem); |
| 472 | return retval; |
Rik van Riel | 5beb493 | 2010-03-05 13:42:07 -0800 | [diff] [blame] | 473 | fail_nomem_anon_vma_fork: |
| 474 | mpol_put(pol); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 475 | fail_nomem_policy: |
| 476 | kmem_cache_free(vm_area_cachep, tmp); |
| 477 | fail_nomem: |
| 478 | retval = -ENOMEM; |
| 479 | vm_unacct_memory(charge); |
| 480 | goto out; |
| 481 | } |
| 482 | |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 483 | static inline int mm_alloc_pgd(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 484 | { |
| 485 | mm->pgd = pgd_alloc(mm); |
| 486 | if (unlikely(!mm->pgd)) |
| 487 | return -ENOMEM; |
| 488 | return 0; |
| 489 | } |
| 490 | |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 491 | static inline void mm_free_pgd(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 492 | { |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 493 | pgd_free(mm, mm->pgd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 494 | } |
| 495 | #else |
| 496 | #define dup_mmap(mm, oldmm) (0) |
| 497 | #define mm_alloc_pgd(mm) (0) |
| 498 | #define mm_free_pgd(mm) |
| 499 | #endif /* CONFIG_MMU */ |
| 500 | |
Daniel Walker | 23ff444 | 2007-10-18 03:06:07 -0700 | [diff] [blame] | 501 | __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 502 | |
Christoph Lameter | e94b176 | 2006-12-06 20:33:17 -0800 | [diff] [blame] | 503 | #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 504 | #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) |
| 505 | |
Hidehiro Kawai | 4cb0e11 | 2009-01-06 14:42:47 -0800 | [diff] [blame] | 506 | static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT; |
| 507 | |
| 508 | static int __init coredump_filter_setup(char *s) |
| 509 | { |
| 510 | default_dump_filter = |
| 511 | (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & |
| 512 | MMF_DUMP_FILTER_MASK; |
| 513 | return 1; |
| 514 | } |
| 515 | |
| 516 | __setup("coredump_filter=", coredump_filter_setup); |
| 517 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 518 | #include <linux/init_task.h> |
| 519 | |
Alexey Dobriyan | 858f099 | 2009-09-23 15:57:32 -0700 | [diff] [blame] | 520 | static void mm_init_aio(struct mm_struct *mm) |
| 521 | { |
| 522 | #ifdef CONFIG_AIO |
| 523 | spin_lock_init(&mm->ioctx_lock); |
| 524 | INIT_HLIST_HEAD(&mm->ioctx_list); |
| 525 | #endif |
| 526 | } |
| 527 | |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 528 | static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 529 | { |
| 530 | atomic_set(&mm->mm_users, 1); |
| 531 | atomic_set(&mm->mm_count, 1); |
| 532 | init_rwsem(&mm->mmap_sem); |
| 533 | INIT_LIST_HEAD(&mm->mmlist); |
Hugh Dickins | f8af4da | 2009-09-21 17:01:57 -0700 | [diff] [blame] | 534 | mm->flags = (current->mm) ? |
| 535 | (current->mm->flags & MMF_INIT_MASK) : default_dump_filter; |
Oleg Nesterov | 999d9fc | 2008-07-25 01:47:41 -0700 | [diff] [blame] | 536 | mm->core_state = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 537 | mm->nr_ptes = 0; |
KAMEZAWA Hiroyuki | d559db0 | 2010-03-05 13:41:39 -0800 | [diff] [blame] | 538 | memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 539 | spin_lock_init(&mm->page_table_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 540 | mm->free_area_cache = TASK_UNMAPPED_BASE; |
Wolfgang Wander | 1363c3c | 2005-06-21 17:14:49 -0700 | [diff] [blame] | 541 | mm->cached_hole_size = ~0UL; |
Alexey Dobriyan | 858f099 | 2009-09-23 15:57:32 -0700 | [diff] [blame] | 542 | mm_init_aio(mm); |
Balbir Singh | cf475ad | 2008-04-29 01:00:16 -0700 | [diff] [blame] | 543 | mm_init_owner(mm, p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 544 | |
| 545 | if (likely(!mm_alloc_pgd(mm))) { |
| 546 | mm->def_flags = 0; |
Andrea Arcangeli | cddb8a5 | 2008-07-28 15:46:29 -0700 | [diff] [blame] | 547 | mmu_notifier_mm_init(mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 548 | return mm; |
| 549 | } |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 550 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 551 | free_mm(mm); |
| 552 | return NULL; |
| 553 | } |
| 554 | |
Konstantin Khlebnikov | c3f0327 | 2012-03-21 16:33:48 -0700 | [diff] [blame] | 555 | static void check_mm(struct mm_struct *mm) |
| 556 | { |
| 557 | int i; |
| 558 | |
| 559 | for (i = 0; i < NR_MM_COUNTERS; i++) { |
| 560 | long x = atomic_long_read(&mm->rss_stat.count[i]); |
| 561 | |
| 562 | if (unlikely(x)) |
| 563 | printk(KERN_ALERT "BUG: Bad rss-counter state " |
| 564 | "mm:%p idx:%d val:%ld\n", mm, i, x); |
| 565 | } |
| 566 | |
| 567 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 568 | VM_BUG_ON(mm->pmd_huge_pte); |
| 569 | #endif |
| 570 | } |
| 571 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 572 | /* |
| 573 | * Allocate and initialize an mm_struct. |
| 574 | */ |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 575 | struct mm_struct *mm_alloc(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 576 | { |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 577 | struct mm_struct *mm; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 578 | |
| 579 | mm = allocate_mm(); |
KOSAKI Motohiro | de03c72 | 2011-05-24 17:12:15 -0700 | [diff] [blame] | 580 | if (!mm) |
| 581 | return NULL; |
| 582 | |
| 583 | memset(mm, 0, sizeof(*mm)); |
Linus Torvalds | 6345d24 | 2011-05-29 11:32:28 -0700 | [diff] [blame] | 584 | mm_init_cpumask(mm); |
| 585 | return mm_init(mm, current); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 586 | } |
| 587 | |
| 588 | /* |
| 589 | * Called when the last reference to the mm |
| 590 | * is dropped: either by a lazy thread or by |
| 591 | * mmput. Free the page directory and the mm. |
| 592 | */ |
Harvey Harrison | 7ad5b3a | 2008-02-08 04:19:53 -0800 | [diff] [blame] | 593 | void __mmdrop(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 594 | { |
| 595 | BUG_ON(mm == &init_mm); |
| 596 | mm_free_pgd(mm); |
| 597 | destroy_context(mm); |
Andrea Arcangeli | cddb8a5 | 2008-07-28 15:46:29 -0700 | [diff] [blame] | 598 | mmu_notifier_mm_destroy(mm); |
Konstantin Khlebnikov | c3f0327 | 2012-03-21 16:33:48 -0700 | [diff] [blame] | 599 | check_mm(mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 600 | free_mm(mm); |
| 601 | } |
Avi Kivity | 6d4e4c4 | 2007-11-21 16:41:05 +0200 | [diff] [blame] | 602 | EXPORT_SYMBOL_GPL(__mmdrop); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 603 | |
| 604 | /* |
| 605 | * Decrement the use count and release all resources for an mm. |
| 606 | */ |
| 607 | void mmput(struct mm_struct *mm) |
| 608 | { |
Andrew Morton | 0ae26f1 | 2006-06-23 02:05:15 -0700 | [diff] [blame] | 609 | might_sleep(); |
| 610 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 611 | if (atomic_dec_and_test(&mm->mm_users)) { |
Srikar Dronamraju | d4b3b638 | 2012-03-30 23:56:31 +0530 | [diff] [blame] | 612 | uprobe_clear_state(mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 613 | exit_aio(mm); |
Andrea Arcangeli | 1c2fb7a | 2009-09-21 17:02:22 -0700 | [diff] [blame] | 614 | ksm_exit(mm); |
Andrea Arcangeli | ba76149 | 2011-01-13 15:46:58 -0800 | [diff] [blame] | 615 | khugepaged_exit(mm); /* must run before exit_mmap */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 616 | exit_mmap(mm); |
Matt Helsley | 925d1c4 | 2008-04-29 01:01:36 -0700 | [diff] [blame] | 617 | set_mm_exe_file(mm, NULL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 618 | if (!list_empty(&mm->mmlist)) { |
| 619 | spin_lock(&mmlist_lock); |
| 620 | list_del(&mm->mmlist); |
| 621 | spin_unlock(&mmlist_lock); |
| 622 | } |
Hiroshi Shimamoto | 801460d | 2009-09-23 15:57:41 -0700 | [diff] [blame] | 623 | if (mm->binfmt) |
| 624 | module_put(mm->binfmt->module); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 625 | mmdrop(mm); |
| 626 | } |
| 627 | } |
| 628 | EXPORT_SYMBOL_GPL(mmput); |
| 629 | |
Jiri Slaby | 3864601 | 2011-05-26 16:25:46 -0700 | [diff] [blame] | 630 | void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) |
| 631 | { |
| 632 | if (new_exe_file) |
| 633 | get_file(new_exe_file); |
| 634 | if (mm->exe_file) |
| 635 | fput(mm->exe_file); |
| 636 | mm->exe_file = new_exe_file; |
Jiri Slaby | 3864601 | 2011-05-26 16:25:46 -0700 | [diff] [blame] | 637 | } |
| 638 | |
| 639 | struct file *get_mm_exe_file(struct mm_struct *mm) |
| 640 | { |
| 641 | struct file *exe_file; |
| 642 | |
Konstantin Khlebnikov | 2dd8ad8 | 2012-10-08 16:28:51 -0700 | [diff] [blame] | 643 | /* We need mmap_sem to protect against races with removal of exe_file */ |
Jiri Slaby | 3864601 | 2011-05-26 16:25:46 -0700 | [diff] [blame] | 644 | down_read(&mm->mmap_sem); |
| 645 | exe_file = mm->exe_file; |
| 646 | if (exe_file) |
| 647 | get_file(exe_file); |
| 648 | up_read(&mm->mmap_sem); |
| 649 | return exe_file; |
| 650 | } |
| 651 | |
| 652 | static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm) |
| 653 | { |
| 654 | /* It's safe to write the exe_file pointer without exe_file_lock because |
| 655 | * this is called during fork when the task is not yet in /proc */ |
| 656 | newmm->exe_file = get_mm_exe_file(oldmm); |
| 657 | } |
| 658 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 659 | /** |
| 660 | * get_task_mm - acquire a reference to the task's mm |
| 661 | * |
Oleg Nesterov | 246bb0b | 2008-07-25 01:47:38 -0700 | [diff] [blame] | 662 | * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 663 | * this kernel workthread has transiently adopted a user mm with use_mm, |
| 664 | * to do its AIO) is not set and if so returns a reference to it, after |
| 665 | * bumping up the use count. User must release the mm via mmput() |
| 666 | * after use. Typically used by /proc and ptrace. |
| 667 | */ |
| 668 | struct mm_struct *get_task_mm(struct task_struct *task) |
| 669 | { |
| 670 | struct mm_struct *mm; |
| 671 | |
| 672 | task_lock(task); |
| 673 | mm = task->mm; |
| 674 | if (mm) { |
Oleg Nesterov | 246bb0b | 2008-07-25 01:47:38 -0700 | [diff] [blame] | 675 | if (task->flags & PF_KTHREAD) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 676 | mm = NULL; |
| 677 | else |
| 678 | atomic_inc(&mm->mm_users); |
| 679 | } |
| 680 | task_unlock(task); |
| 681 | return mm; |
| 682 | } |
| 683 | EXPORT_SYMBOL_GPL(get_task_mm); |
| 684 | |
Christopher Yeoh | 8cdb878 | 2012-02-02 11:34:09 +1030 | [diff] [blame] | 685 | struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) |
| 686 | { |
| 687 | struct mm_struct *mm; |
| 688 | int err; |
| 689 | |
| 690 | err = mutex_lock_killable(&task->signal->cred_guard_mutex); |
| 691 | if (err) |
| 692 | return ERR_PTR(err); |
| 693 | |
| 694 | mm = get_task_mm(task); |
| 695 | if (mm && mm != current->mm && |
| 696 | !ptrace_may_access(task, mode)) { |
| 697 | mmput(mm); |
| 698 | mm = ERR_PTR(-EACCES); |
| 699 | } |
| 700 | mutex_unlock(&task->signal->cred_guard_mutex); |
| 701 | |
| 702 | return mm; |
| 703 | } |
| 704 | |
Oleg Nesterov | 57b59c4 | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 705 | static void complete_vfork_done(struct task_struct *tsk) |
Oleg Nesterov | c415c3b | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 706 | { |
Oleg Nesterov | d68b46f | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 707 | struct completion *vfork; |
Oleg Nesterov | c415c3b | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 708 | |
Oleg Nesterov | d68b46f | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 709 | task_lock(tsk); |
| 710 | vfork = tsk->vfork_done; |
| 711 | if (likely(vfork)) { |
| 712 | tsk->vfork_done = NULL; |
| 713 | complete(vfork); |
| 714 | } |
| 715 | task_unlock(tsk); |
| 716 | } |
| 717 | |
| 718 | static int wait_for_vfork_done(struct task_struct *child, |
| 719 | struct completion *vfork) |
| 720 | { |
| 721 | int killed; |
| 722 | |
| 723 | freezer_do_not_count(); |
| 724 | killed = wait_for_completion_killable(vfork); |
| 725 | freezer_count(); |
| 726 | |
| 727 | if (killed) { |
| 728 | task_lock(child); |
| 729 | child->vfork_done = NULL; |
| 730 | task_unlock(child); |
| 731 | } |
| 732 | |
| 733 | put_task_struct(child); |
| 734 | return killed; |
Oleg Nesterov | c415c3b | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 735 | } |
| 736 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 737 | /* Please note the differences between mmput and mm_release. |
| 738 | * mmput is called whenever we stop holding onto a mm_struct, |
| 739 | * error success whatever. |
| 740 | * |
| 741 | * mm_release is called after a mm_struct has been removed |
| 742 | * from the current process. |
| 743 | * |
| 744 | * This difference is important for error handling, when we |
| 745 | * only half set up a mm_struct for a new process and need to restore |
| 746 | * the old one. Because we mmput the new mm_struct before |
| 747 | * restoring the old one. . . |
| 748 | * Eric Biederman 10 January 1998 |
| 749 | */ |
| 750 | void mm_release(struct task_struct *tsk, struct mm_struct *mm) |
| 751 | { |
Linus Torvalds | 8141c7f | 2008-11-15 10:20:36 -0800 | [diff] [blame] | 752 | /* Get rid of any futexes when releasing the mm */ |
| 753 | #ifdef CONFIG_FUTEX |
Peter Zijlstra | fc6b177 | 2009-10-05 18:17:32 +0200 | [diff] [blame] | 754 | if (unlikely(tsk->robust_list)) { |
Linus Torvalds | 8141c7f | 2008-11-15 10:20:36 -0800 | [diff] [blame] | 755 | exit_robust_list(tsk); |
Peter Zijlstra | fc6b177 | 2009-10-05 18:17:32 +0200 | [diff] [blame] | 756 | tsk->robust_list = NULL; |
| 757 | } |
Linus Torvalds | 8141c7f | 2008-11-15 10:20:36 -0800 | [diff] [blame] | 758 | #ifdef CONFIG_COMPAT |
Peter Zijlstra | fc6b177 | 2009-10-05 18:17:32 +0200 | [diff] [blame] | 759 | if (unlikely(tsk->compat_robust_list)) { |
Linus Torvalds | 8141c7f | 2008-11-15 10:20:36 -0800 | [diff] [blame] | 760 | compat_exit_robust_list(tsk); |
Peter Zijlstra | fc6b177 | 2009-10-05 18:17:32 +0200 | [diff] [blame] | 761 | tsk->compat_robust_list = NULL; |
| 762 | } |
Linus Torvalds | 8141c7f | 2008-11-15 10:20:36 -0800 | [diff] [blame] | 763 | #endif |
Thomas Gleixner | 322a2c1 | 2009-10-05 18:18:03 +0200 | [diff] [blame] | 764 | if (unlikely(!list_empty(&tsk->pi_state_list))) |
| 765 | exit_pi_state_list(tsk); |
Linus Torvalds | 8141c7f | 2008-11-15 10:20:36 -0800 | [diff] [blame] | 766 | #endif |
| 767 | |
Srikar Dronamraju | 0326f5a9 | 2012-03-13 23:30:11 +0530 | [diff] [blame] | 768 | uprobe_free_utask(tsk); |
| 769 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 770 | /* Get rid of any cached register state */ |
| 771 | deactivate_mm(tsk, mm); |
| 772 | |
Roland McGrath | fec1d01 | 2006-12-06 20:36:34 -0800 | [diff] [blame] | 773 | /* |
| 774 | * If we're exiting normally, clear a user-space tid field if |
| 775 | * requested. We leave this alone when dying by signal, to leave |
| 776 | * the value intact in a core dump, and to save the unnecessary |
Oleg Nesterov | d68b46f | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 777 | * trouble, say, a killed vfork parent shouldn't touch this mm. |
| 778 | * Userland only wants this done for a sys_exit. |
Roland McGrath | fec1d01 | 2006-12-06 20:36:34 -0800 | [diff] [blame] | 779 | */ |
Eric Dumazet | 9c8a822 | 2009-08-06 15:09:28 -0700 | [diff] [blame] | 780 | if (tsk->clear_child_tid) { |
| 781 | if (!(tsk->flags & PF_SIGNALED) && |
| 782 | atomic_read(&mm->mm_users) > 1) { |
| 783 | /* |
| 784 | * We don't check the error code - if userspace has |
| 785 | * not set up a proper pointer then tough luck. |
| 786 | */ |
| 787 | put_user(0, tsk->clear_child_tid); |
| 788 | sys_futex(tsk->clear_child_tid, FUTEX_WAKE, |
| 789 | 1, NULL, NULL, 0); |
| 790 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 791 | tsk->clear_child_tid = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 792 | } |
Konstantin Khlebnikov | f7505d64 | 2012-05-31 16:26:21 -0700 | [diff] [blame] | 793 | |
| 794 | /* |
| 795 | * All done, finally we can wake up parent and return this mm to him. |
| 796 | * Also kthread_stop() uses this completion for synchronization. |
| 797 | */ |
| 798 | if (tsk->vfork_done) |
| 799 | complete_vfork_done(tsk); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 800 | } |
| 801 | |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 802 | /* |
| 803 | * Allocate a new mm structure and copy contents from the |
| 804 | * mm structure of the passed in task structure. |
| 805 | */ |
Carsten Otte | 402b086 | 2008-03-25 18:47:10 +0100 | [diff] [blame] | 806 | struct mm_struct *dup_mm(struct task_struct *tsk) |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 807 | { |
| 808 | struct mm_struct *mm, *oldmm = current->mm; |
| 809 | int err; |
| 810 | |
| 811 | if (!oldmm) |
| 812 | return NULL; |
| 813 | |
| 814 | mm = allocate_mm(); |
| 815 | if (!mm) |
| 816 | goto fail_nomem; |
| 817 | |
| 818 | memcpy(mm, oldmm, sizeof(*mm)); |
Linus Torvalds | 6345d24 | 2011-05-29 11:32:28 -0700 | [diff] [blame] | 819 | mm_init_cpumask(mm); |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 820 | |
Andrea Arcangeli | e7a00c4 | 2011-01-13 15:46:45 -0800 | [diff] [blame] | 821 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 822 | mm->pmd_huge_pte = NULL; |
| 823 | #endif |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 824 | if (!mm_init(mm, tsk)) |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 825 | goto fail_nomem; |
| 826 | |
| 827 | if (init_new_context(tsk, mm)) |
| 828 | goto fail_nocontext; |
| 829 | |
Matt Helsley | 925d1c4 | 2008-04-29 01:01:36 -0700 | [diff] [blame] | 830 | dup_mm_exe_file(oldmm, mm); |
| 831 | |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 832 | err = dup_mmap(mm, oldmm); |
| 833 | if (err) |
| 834 | goto free_pt; |
| 835 | |
| 836 | mm->hiwater_rss = get_mm_rss(mm); |
| 837 | mm->hiwater_vm = mm->total_vm; |
| 838 | |
Hiroshi Shimamoto | 801460d | 2009-09-23 15:57:41 -0700 | [diff] [blame] | 839 | if (mm->binfmt && !try_module_get(mm->binfmt->module)) |
| 840 | goto free_pt; |
| 841 | |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 842 | return mm; |
| 843 | |
| 844 | free_pt: |
Hiroshi Shimamoto | 801460d | 2009-09-23 15:57:41 -0700 | [diff] [blame] | 845 | /* don't put binfmt in mmput, we haven't got module yet */ |
| 846 | mm->binfmt = NULL; |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 847 | mmput(mm); |
| 848 | |
| 849 | fail_nomem: |
| 850 | return NULL; |
| 851 | |
| 852 | fail_nocontext: |
| 853 | /* |
| 854 | * If init_new_context() failed, we cannot use mmput() to free the mm |
| 855 | * because it calls destroy_context() |
| 856 | */ |
| 857 | mm_free_pgd(mm); |
| 858 | free_mm(mm); |
| 859 | return NULL; |
| 860 | } |
| 861 | |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 862 | static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 863 | { |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 864 | struct mm_struct *mm, *oldmm; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 865 | int retval; |
| 866 | |
| 867 | tsk->min_flt = tsk->maj_flt = 0; |
| 868 | tsk->nvcsw = tsk->nivcsw = 0; |
Mandeep Singh Baines | 17406b8 | 2009-02-06 15:37:47 -0800 | [diff] [blame] | 869 | #ifdef CONFIG_DETECT_HUNG_TASK |
| 870 | tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; |
| 871 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 872 | |
| 873 | tsk->mm = NULL; |
| 874 | tsk->active_mm = NULL; |
| 875 | |
| 876 | /* |
| 877 | * Are we cloning a kernel thread? |
| 878 | * |
| 879 | * We need to steal a active VM for that.. |
| 880 | */ |
| 881 | oldmm = current->mm; |
| 882 | if (!oldmm) |
| 883 | return 0; |
| 884 | |
| 885 | if (clone_flags & CLONE_VM) { |
| 886 | atomic_inc(&oldmm->mm_users); |
| 887 | mm = oldmm; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 888 | goto good_mm; |
| 889 | } |
| 890 | |
| 891 | retval = -ENOMEM; |
JANAK DESAI | a0a7ec3 | 2006-02-07 12:59:01 -0800 | [diff] [blame] | 892 | mm = dup_mm(tsk); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 893 | if (!mm) |
| 894 | goto fail_nomem; |
| 895 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 896 | good_mm: |
| 897 | tsk->mm = mm; |
| 898 | tsk->active_mm = mm; |
| 899 | return 0; |
| 900 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 901 | fail_nomem: |
| 902 | return retval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 903 | } |
| 904 | |
Alexey Dobriyan | a39bc51 | 2007-10-18 23:41:10 -0700 | [diff] [blame] | 905 | static int copy_fs(unsigned long clone_flags, struct task_struct *tsk) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 906 | { |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 907 | struct fs_struct *fs = current->fs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 908 | if (clone_flags & CLONE_FS) { |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 909 | /* tsk->fs is already what we want */ |
Nick Piggin | 2a4419b | 2010-08-18 04:37:33 +1000 | [diff] [blame] | 910 | spin_lock(&fs->lock); |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 911 | if (fs->in_exec) { |
Nick Piggin | 2a4419b | 2010-08-18 04:37:33 +1000 | [diff] [blame] | 912 | spin_unlock(&fs->lock); |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 913 | return -EAGAIN; |
| 914 | } |
| 915 | fs->users++; |
Nick Piggin | 2a4419b | 2010-08-18 04:37:33 +1000 | [diff] [blame] | 916 | spin_unlock(&fs->lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 917 | return 0; |
| 918 | } |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 919 | tsk->fs = copy_fs_struct(fs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 920 | if (!tsk->fs) |
| 921 | return -ENOMEM; |
| 922 | return 0; |
| 923 | } |
| 924 | |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 925 | static int copy_files(unsigned long clone_flags, struct task_struct *tsk) |
JANAK DESAI | a016f33 | 2006-02-07 12:59:02 -0800 | [diff] [blame] | 926 | { |
| 927 | struct files_struct *oldf, *newf; |
| 928 | int error = 0; |
| 929 | |
| 930 | /* |
| 931 | * A background process may not have any files ... |
| 932 | */ |
| 933 | oldf = current->files; |
| 934 | if (!oldf) |
| 935 | goto out; |
| 936 | |
| 937 | if (clone_flags & CLONE_FILES) { |
| 938 | atomic_inc(&oldf->count); |
| 939 | goto out; |
| 940 | } |
| 941 | |
JANAK DESAI | a016f33 | 2006-02-07 12:59:02 -0800 | [diff] [blame] | 942 | newf = dup_fd(oldf, &error); |
| 943 | if (!newf) |
| 944 | goto out; |
| 945 | |
| 946 | tsk->files = newf; |
| 947 | error = 0; |
| 948 | out: |
| 949 | return error; |
| 950 | } |
| 951 | |
Jens Axboe | fadad878 | 2008-01-24 08:54:47 +0100 | [diff] [blame] | 952 | static int copy_io(unsigned long clone_flags, struct task_struct *tsk) |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 953 | { |
| 954 | #ifdef CONFIG_BLOCK |
| 955 | struct io_context *ioc = current->io_context; |
Tejun Heo | 6e736be | 2011-12-14 00:33:38 +0100 | [diff] [blame] | 956 | struct io_context *new_ioc; |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 957 | |
| 958 | if (!ioc) |
| 959 | return 0; |
Jens Axboe | fadad878 | 2008-01-24 08:54:47 +0100 | [diff] [blame] | 960 | /* |
| 961 | * Share io context with parent, if CLONE_IO is set |
| 962 | */ |
| 963 | if (clone_flags & CLONE_IO) { |
Tejun Heo | 3d48749 | 2012-03-05 13:15:25 -0800 | [diff] [blame] | 964 | ioc_task_link(ioc); |
| 965 | tsk->io_context = ioc; |
Jens Axboe | fadad878 | 2008-01-24 08:54:47 +0100 | [diff] [blame] | 966 | } else if (ioprio_valid(ioc->ioprio)) { |
Tejun Heo | 6e736be | 2011-12-14 00:33:38 +0100 | [diff] [blame] | 967 | new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE); |
| 968 | if (unlikely(!new_ioc)) |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 969 | return -ENOMEM; |
| 970 | |
Tejun Heo | 6e736be | 2011-12-14 00:33:38 +0100 | [diff] [blame] | 971 | new_ioc->ioprio = ioc->ioprio; |
Tejun Heo | 11a3122 | 2012-02-07 07:51:30 +0100 | [diff] [blame] | 972 | put_io_context(new_ioc); |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 973 | } |
| 974 | #endif |
| 975 | return 0; |
| 976 | } |
| 977 | |
Alexey Dobriyan | a39bc51 | 2007-10-18 23:41:10 -0700 | [diff] [blame] | 978 | static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 979 | { |
| 980 | struct sighand_struct *sig; |
| 981 | |
Zhaolei | 6034880 | 2009-01-06 14:40:46 -0800 | [diff] [blame] | 982 | if (clone_flags & CLONE_SIGHAND) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 983 | atomic_inc(¤t->sighand->count); |
| 984 | return 0; |
| 985 | } |
| 986 | sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); |
Ingo Molnar | e56d090 | 2006-01-08 01:01:37 -0800 | [diff] [blame] | 987 | rcu_assign_pointer(tsk->sighand, sig); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 988 | if (!sig) |
| 989 | return -ENOMEM; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 990 | atomic_set(&sig->count, 1); |
| 991 | memcpy(sig->action, current->sighand->action, sizeof(sig->action)); |
| 992 | return 0; |
| 993 | } |
| 994 | |
Oleg Nesterov | a7e5328 | 2006-03-28 16:11:27 -0800 | [diff] [blame] | 995 | void __cleanup_sighand(struct sighand_struct *sighand) |
Oleg Nesterov | c81addc | 2006-03-28 16:11:17 -0800 | [diff] [blame] | 996 | { |
Oleg Nesterov | d80e731 | 2012-02-24 20:07:11 +0100 | [diff] [blame] | 997 | if (atomic_dec_and_test(&sighand->count)) { |
| 998 | signalfd_cleanup(sighand); |
Oleg Nesterov | c81addc | 2006-03-28 16:11:17 -0800 | [diff] [blame] | 999 | kmem_cache_free(sighand_cachep, sighand); |
Oleg Nesterov | d80e731 | 2012-02-24 20:07:11 +0100 | [diff] [blame] | 1000 | } |
Oleg Nesterov | c81addc | 2006-03-28 16:11:17 -0800 | [diff] [blame] | 1001 | } |
| 1002 | |
Frank Mayhar | f06febc | 2008-09-12 09:54:39 -0700 | [diff] [blame] | 1003 | |
| 1004 | /* |
| 1005 | * Initialize POSIX timer handling for a thread group. |
| 1006 | */ |
| 1007 | static void posix_cpu_timers_init_group(struct signal_struct *sig) |
| 1008 | { |
Jiri Slaby | 78d7d40 | 2010-03-05 13:42:54 -0800 | [diff] [blame] | 1009 | unsigned long cpu_limit; |
| 1010 | |
Frank Mayhar | f06febc | 2008-09-12 09:54:39 -0700 | [diff] [blame] | 1011 | /* Thread group counters. */ |
| 1012 | thread_group_cputime_init(sig); |
| 1013 | |
Jiri Slaby | 78d7d40 | 2010-03-05 13:42:54 -0800 | [diff] [blame] | 1014 | cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); |
| 1015 | if (cpu_limit != RLIM_INFINITY) { |
| 1016 | sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit); |
Oleg Nesterov | 6279a751 | 2009-03-27 01:06:07 +0100 | [diff] [blame] | 1017 | sig->cputimer.running = 1; |
| 1018 | } |
| 1019 | |
Frank Mayhar | f06febc | 2008-09-12 09:54:39 -0700 | [diff] [blame] | 1020 | /* The timer lists. */ |
| 1021 | INIT_LIST_HEAD(&sig->cpu_timers[0]); |
| 1022 | INIT_LIST_HEAD(&sig->cpu_timers[1]); |
| 1023 | INIT_LIST_HEAD(&sig->cpu_timers[2]); |
| 1024 | } |
| 1025 | |
Alexey Dobriyan | a39bc51 | 2007-10-18 23:41:10 -0700 | [diff] [blame] | 1026 | static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1027 | { |
| 1028 | struct signal_struct *sig; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1029 | |
Oleg Nesterov | 4ab6c08 | 2009-08-26 14:29:24 -0700 | [diff] [blame] | 1030 | if (clone_flags & CLONE_THREAD) |
Peter Zijlstra | 490dea4 | 2008-11-24 17:06:57 +0100 | [diff] [blame] | 1031 | return 0; |
Oleg Nesterov | 6279a751 | 2009-03-27 01:06:07 +0100 | [diff] [blame] | 1032 | |
Veaceslav Falico | a56704e | 2010-03-10 15:23:01 -0800 | [diff] [blame] | 1033 | sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1034 | tsk->signal = sig; |
| 1035 | if (!sig) |
| 1036 | return -ENOMEM; |
| 1037 | |
Oleg Nesterov | b3ac022 | 2010-05-26 14:43:24 -0700 | [diff] [blame] | 1038 | sig->nr_threads = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1039 | atomic_set(&sig->live, 1); |
Oleg Nesterov | b3ac022 | 2010-05-26 14:43:24 -0700 | [diff] [blame] | 1040 | atomic_set(&sig->sigcnt, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1041 | init_waitqueue_head(&sig->wait_chldexit); |
Sukadev Bhattiprolu | b3bfa0c | 2009-04-02 16:58:08 -0700 | [diff] [blame] | 1042 | if (clone_flags & CLONE_NEWPID) |
| 1043 | sig->flags |= SIGNAL_UNKILLABLE; |
Oleg Nesterov | db51aec | 2008-04-30 00:52:52 -0700 | [diff] [blame] | 1044 | sig->curr_target = tsk; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1045 | init_sigpending(&sig->shared_pending); |
| 1046 | INIT_LIST_HEAD(&sig->posix_timers); |
| 1047 | |
Thomas Gleixner | c9cb2e3 | 2007-02-16 01:27:49 -0800 | [diff] [blame] | 1048 | hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1049 | sig->real_timer.function = it_real_fn; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1050 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1051 | task_lock(current->group_leader); |
| 1052 | memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); |
| 1053 | task_unlock(current->group_leader); |
| 1054 | |
Oleg Nesterov | 6279a751 | 2009-03-27 01:06:07 +0100 | [diff] [blame] | 1055 | posix_cpu_timers_init_group(sig); |
| 1056 | |
Miloslav Trmac | 522ed77 | 2007-07-15 23:40:56 -0700 | [diff] [blame] | 1057 | tty_audit_fork(sig); |
Mike Galbraith | 5091faa | 2010-11-30 14:18:03 +0100 | [diff] [blame] | 1058 | sched_autogroup_fork(sig); |
Miloslav Trmac | 522ed77 | 2007-07-15 23:40:56 -0700 | [diff] [blame] | 1059 | |
Ben Blum | 4714d1d | 2011-05-26 16:25:18 -0700 | [diff] [blame] | 1060 | #ifdef CONFIG_CGROUPS |
Tejun Heo | 257058a | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 1061 | init_rwsem(&sig->group_rwsem); |
Ben Blum | 4714d1d | 2011-05-26 16:25:18 -0700 | [diff] [blame] | 1062 | #endif |
| 1063 | |
David Rientjes | a63d83f | 2010-08-09 17:19:46 -0700 | [diff] [blame] | 1064 | sig->oom_score_adj = current->signal->oom_score_adj; |
Mandeep Singh Baines | dabb16f63 | 2011-01-13 15:46:05 -0800 | [diff] [blame] | 1065 | sig->oom_score_adj_min = current->signal->oom_score_adj_min; |
KOSAKI Motohiro | 28b83c5 | 2009-09-21 17:03:13 -0700 | [diff] [blame] | 1066 | |
Lennart Poettering | ebec18a | 2012-03-23 15:01:54 -0700 | [diff] [blame] | 1067 | sig->has_child_subreaper = current->signal->has_child_subreaper || |
| 1068 | current->signal->is_child_subreaper; |
| 1069 | |
KOSAKI Motohiro | 9b1bf12 | 2010-10-27 15:34:08 -0700 | [diff] [blame] | 1070 | mutex_init(&sig->cred_guard_mutex); |
| 1071 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1072 | return 0; |
| 1073 | } |
| 1074 | |
Alexey Dobriyan | a39bc51 | 2007-10-18 23:41:10 -0700 | [diff] [blame] | 1075 | static void copy_flags(unsigned long clone_flags, struct task_struct *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1076 | { |
| 1077 | unsigned long new_flags = p->flags; |
| 1078 | |
Tejun Heo | 21aa9af | 2010-06-08 21:40:37 +0200 | [diff] [blame] | 1079 | new_flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1080 | new_flags |= PF_FORKNOEXEC; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1081 | p->flags = new_flags; |
| 1082 | } |
| 1083 | |
Heiko Carstens | 17da2bd | 2009-01-14 14:14:10 +0100 | [diff] [blame] | 1084 | SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1085 | { |
| 1086 | current->clear_child_tid = tidptr; |
| 1087 | |
Pavel Emelyanov | b488893 | 2007-10-18 23:40:14 -0700 | [diff] [blame] | 1088 | return task_pid_vnr(current); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1089 | } |
| 1090 | |
Alexey Dobriyan | a39bc51 | 2007-10-18 23:41:10 -0700 | [diff] [blame] | 1091 | static void rt_mutex_init_task(struct task_struct *p) |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 1092 | { |
Thomas Gleixner | 1d61548 | 2009-11-17 14:54:03 +0100 | [diff] [blame] | 1093 | raw_spin_lock_init(&p->pi_lock); |
Zilvinas Valinskas | e29e175 | 2007-03-16 13:38:34 -0800 | [diff] [blame] | 1094 | #ifdef CONFIG_RT_MUTEXES |
Dima Zavin | 732375c | 2011-07-07 17:27:59 -0700 | [diff] [blame] | 1095 | plist_head_init(&p->pi_waiters); |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 1096 | p->pi_blocked_on = NULL; |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 1097 | #endif |
| 1098 | } |
| 1099 | |
Balbir Singh | cf475ad | 2008-04-29 01:00:16 -0700 | [diff] [blame] | 1100 | #ifdef CONFIG_MM_OWNER |
| 1101 | void mm_init_owner(struct mm_struct *mm, struct task_struct *p) |
| 1102 | { |
| 1103 | mm->owner = p; |
| 1104 | } |
| 1105 | #endif /* CONFIG_MM_OWNER */ |
| 1106 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1107 | /* |
Frank Mayhar | f06febc | 2008-09-12 09:54:39 -0700 | [diff] [blame] | 1108 | * Initialize POSIX timer handling for a single task. |
| 1109 | */ |
| 1110 | static void posix_cpu_timers_init(struct task_struct *tsk) |
| 1111 | { |
Martin Schwidefsky | 6486163 | 2011-12-15 14:56:09 +0100 | [diff] [blame] | 1112 | tsk->cputime_expires.prof_exp = 0; |
| 1113 | tsk->cputime_expires.virt_exp = 0; |
Frank Mayhar | f06febc | 2008-09-12 09:54:39 -0700 | [diff] [blame] | 1114 | tsk->cputime_expires.sched_exp = 0; |
| 1115 | INIT_LIST_HEAD(&tsk->cpu_timers[0]); |
| 1116 | INIT_LIST_HEAD(&tsk->cpu_timers[1]); |
| 1117 | INIT_LIST_HEAD(&tsk->cpu_timers[2]); |
| 1118 | } |
| 1119 | |
| 1120 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1121 | * This creates a new process as a copy of the old one, |
| 1122 | * but does not actually start it yet. |
| 1123 | * |
| 1124 | * It copies the registers, and all the appropriate |
| 1125 | * parts of the process environment (as per the clone |
| 1126 | * flags). The actual kick-off is left to the caller. |
| 1127 | */ |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 1128 | static struct task_struct *copy_process(unsigned long clone_flags, |
| 1129 | unsigned long stack_start, |
| 1130 | struct pt_regs *regs, |
| 1131 | unsigned long stack_size, |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 1132 | int __user *child_tidptr, |
Roland McGrath | 09a0539 | 2008-07-25 19:45:47 -0700 | [diff] [blame] | 1133 | struct pid *pid, |
| 1134 | int trace) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1135 | { |
| 1136 | int retval; |
Mariusz Kozlowski | a24efe6 | 2007-10-18 23:41:09 -0700 | [diff] [blame] | 1137 | struct task_struct *p; |
Paul Menage | b4f48b6 | 2007-10-18 23:39:33 -0700 | [diff] [blame] | 1138 | int cgroup_callbacks_done = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1139 | |
| 1140 | if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) |
| 1141 | return ERR_PTR(-EINVAL); |
| 1142 | |
| 1143 | /* |
| 1144 | * Thread groups must share signals as well, and detached threads |
| 1145 | * can only be started up within the thread group. |
| 1146 | */ |
| 1147 | if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND)) |
| 1148 | return ERR_PTR(-EINVAL); |
| 1149 | |
| 1150 | /* |
| 1151 | * Shared signal handlers imply shared VM. By way of the above, |
| 1152 | * thread groups also imply shared VM. Blocking this case allows |
| 1153 | * for various simplifications in other code. |
| 1154 | */ |
| 1155 | if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM)) |
| 1156 | return ERR_PTR(-EINVAL); |
| 1157 | |
Sukadev Bhattiprolu | 123be07 | 2009-09-23 15:57:20 -0700 | [diff] [blame] | 1158 | /* |
| 1159 | * Siblings of global init remain as zombies on exit since they are |
| 1160 | * not reaped by their parent (swapper). To solve this and to avoid |
| 1161 | * multi-rooted process trees, prevent global and container-inits |
| 1162 | * from creating siblings. |
| 1163 | */ |
| 1164 | if ((clone_flags & CLONE_PARENT) && |
| 1165 | current->signal->flags & SIGNAL_UNKILLABLE) |
| 1166 | return ERR_PTR(-EINVAL); |
| 1167 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1168 | retval = security_task_create(clone_flags); |
| 1169 | if (retval) |
| 1170 | goto fork_out; |
| 1171 | |
| 1172 | retval = -ENOMEM; |
| 1173 | p = dup_task_struct(current); |
| 1174 | if (!p) |
| 1175 | goto fork_out; |
| 1176 | |
Steven Rostedt | f7e8b61 | 2009-06-02 16:39:48 -0400 | [diff] [blame] | 1177 | ftrace_graph_init_task(p); |
Will Drewry | e2cfabdf | 2012-04-12 16:47:57 -0500 | [diff] [blame] | 1178 | get_seccomp_filter(p); |
Steven Rostedt | f7e8b61 | 2009-06-02 16:39:48 -0400 | [diff] [blame] | 1179 | |
Peter Zijlstra | bea493a | 2006-10-17 00:10:33 -0700 | [diff] [blame] | 1180 | rt_mutex_init_task(p); |
| 1181 | |
Ingo Molnar | d12c1a3 | 2008-07-14 12:09:28 +0200 | [diff] [blame] | 1182 | #ifdef CONFIG_PROVE_LOCKING |
Ingo Molnar | de30a2b | 2006-07-03 00:24:42 -0700 | [diff] [blame] | 1183 | DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); |
| 1184 | DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); |
| 1185 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1186 | retval = -EAGAIN; |
David Howells | 3b11a1d | 2008-11-14 10:39:26 +1100 | [diff] [blame] | 1187 | if (atomic_read(&p->real_cred->user->processes) >= |
Jiri Slaby | 78d7d40 | 2010-03-05 13:42:54 -0800 | [diff] [blame] | 1188 | task_rlimit(p, RLIMIT_NPROC)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1189 | if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) && |
Serge Hallyn | 18b6e04 | 2008-10-15 16:38:45 -0500 | [diff] [blame] | 1190 | p->real_cred->user != INIT_USER) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1191 | goto bad_fork_free; |
| 1192 | } |
Vasiliy Kulikov | 72fa599 | 2011-08-08 19:02:04 +0400 | [diff] [blame] | 1193 | current->flags &= ~PF_NPROC_EXCEEDED; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1194 | |
David Howells | f1752ee | 2008-11-14 10:39:17 +1100 | [diff] [blame] | 1195 | retval = copy_creds(p, clone_flags); |
| 1196 | if (retval < 0) |
| 1197 | goto bad_fork_free; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1198 | |
| 1199 | /* |
| 1200 | * If multiple threads are within copy_process(), then this check |
| 1201 | * triggers too late. This doesn't hurt, the check is only there |
| 1202 | * to stop root fork bombs. |
| 1203 | */ |
Li Zefan | 04ec93f | 2009-02-06 08:17:19 +0000 | [diff] [blame] | 1204 | retval = -EAGAIN; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1205 | if (nr_threads >= max_threads) |
| 1206 | goto bad_fork_cleanup_count; |
| 1207 | |
Al Viro | a1261f54 | 2005-11-13 16:06:55 -0800 | [diff] [blame] | 1208 | if (!try_module_get(task_thread_info(p)->exec_domain->module)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1209 | goto bad_fork_cleanup_count; |
| 1210 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1211 | p->did_exec = 0; |
Shailabh Nagar | ca74e92 | 2006-07-14 00:24:36 -0700 | [diff] [blame] | 1212 | delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1213 | copy_flags(clone_flags, p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1214 | INIT_LIST_HEAD(&p->children); |
| 1215 | INIT_LIST_HEAD(&p->sibling); |
Paul E. McKenney | f41d911 | 2009-08-22 13:56:52 -0700 | [diff] [blame] | 1216 | rcu_copy_process(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1217 | p->vfork_done = NULL; |
| 1218 | spin_lock_init(&p->alloc_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1219 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1220 | init_sigpending(&p->pending); |
| 1221 | |
Martin Schwidefsky | 6486163 | 2011-12-15 14:56:09 +0100 | [diff] [blame] | 1222 | p->utime = p->stime = p->gtime = 0; |
| 1223 | p->utimescaled = p->stimescaled = 0; |
Hidetoshi Seto | d99ca3b | 2009-12-02 17:26:47 +0900 | [diff] [blame] | 1224 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING |
Martin Schwidefsky | 6486163 | 2011-12-15 14:56:09 +0100 | [diff] [blame] | 1225 | p->prev_utime = p->prev_stime = 0; |
Hidetoshi Seto | d99ca3b | 2009-12-02 17:26:47 +0900 | [diff] [blame] | 1226 | #endif |
KAMEZAWA Hiroyuki | a3a2e76 | 2010-04-06 14:34:42 -0700 | [diff] [blame] | 1227 | #if defined(SPLIT_RSS_COUNTING) |
| 1228 | memset(&p->rss_stat, 0, sizeof(p->rss_stat)); |
| 1229 | #endif |
Balbir Singh | 172ba84 | 2007-07-09 18:52:00 +0200 | [diff] [blame] | 1230 | |
Arjan van de Ven | 6976675 | 2008-09-01 15:52:40 -0700 | [diff] [blame] | 1231 | p->default_timer_slack_ns = current->timer_slack_ns; |
| 1232 | |
Andrea Righi | 5995477 | 2008-07-27 17:29:15 +0200 | [diff] [blame] | 1233 | task_io_accounting_init(&p->ioac); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1234 | acct_clear_integrals(p); |
| 1235 | |
Frank Mayhar | f06febc | 2008-09-12 09:54:39 -0700 | [diff] [blame] | 1236 | posix_cpu_timers_init(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1237 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1238 | do_posix_clock_monotonic_gettime(&p->start_time); |
Tomas Janousek | 924b42d | 2007-07-15 23:39:42 -0700 | [diff] [blame] | 1239 | p->real_start_time = p->start_time; |
| 1240 | monotonic_to_bootbased(&p->real_start_time); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1241 | p->io_context = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1242 | p->audit_context = NULL; |
Ben Blum | 4714d1d | 2011-05-26 16:25:18 -0700 | [diff] [blame] | 1243 | if (clone_flags & CLONE_THREAD) |
Tejun Heo | 257058a | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 1244 | threadgroup_change_begin(current); |
Paul Menage | b4f48b6 | 2007-10-18 23:39:33 -0700 | [diff] [blame] | 1245 | cgroup_fork(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1246 | #ifdef CONFIG_NUMA |
Lee Schermerhorn | 846a16b | 2008-04-28 02:13:09 -0700 | [diff] [blame] | 1247 | p->mempolicy = mpol_dup(p->mempolicy); |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1248 | if (IS_ERR(p->mempolicy)) { |
| 1249 | retval = PTR_ERR(p->mempolicy); |
| 1250 | p->mempolicy = NULL; |
| 1251 | goto bad_fork_cleanup_cgroup; |
| 1252 | } |
Paul Jackson | c61afb1 | 2006-03-24 03:16:08 -0800 | [diff] [blame] | 1253 | mpol_fix_fork_child_flag(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1254 | #endif |
Michal Hocko | 778d3b0 | 2011-07-26 16:08:30 -0700 | [diff] [blame] | 1255 | #ifdef CONFIG_CPUSETS |
| 1256 | p->cpuset_mem_spread_rotor = NUMA_NO_NODE; |
| 1257 | p->cpuset_slab_spread_rotor = NUMA_NO_NODE; |
Mel Gorman | cc9a6c8 | 2012-03-21 16:34:11 -0700 | [diff] [blame] | 1258 | seqcount_init(&p->mems_allowed_seq); |
Michal Hocko | 778d3b0 | 2011-07-26 16:08:30 -0700 | [diff] [blame] | 1259 | #endif |
Ingo Molnar | de30a2b | 2006-07-03 00:24:42 -0700 | [diff] [blame] | 1260 | #ifdef CONFIG_TRACE_IRQFLAGS |
| 1261 | p->irq_events = 0; |
| 1262 | p->hardirqs_enabled = 0; |
| 1263 | p->hardirq_enable_ip = 0; |
| 1264 | p->hardirq_enable_event = 0; |
| 1265 | p->hardirq_disable_ip = _THIS_IP_; |
| 1266 | p->hardirq_disable_event = 0; |
| 1267 | p->softirqs_enabled = 1; |
| 1268 | p->softirq_enable_ip = _THIS_IP_; |
| 1269 | p->softirq_enable_event = 0; |
| 1270 | p->softirq_disable_ip = 0; |
| 1271 | p->softirq_disable_event = 0; |
| 1272 | p->hardirq_context = 0; |
| 1273 | p->softirq_context = 0; |
| 1274 | #endif |
Ingo Molnar | fbb9ce95 | 2006-07-03 00:24:50 -0700 | [diff] [blame] | 1275 | #ifdef CONFIG_LOCKDEP |
| 1276 | p->lockdep_depth = 0; /* no locks held yet */ |
| 1277 | p->curr_chain_key = 0; |
| 1278 | p->lockdep_recursion = 0; |
| 1279 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1280 | |
Ingo Molnar | 408894e | 2006-01-09 15:59:20 -0800 | [diff] [blame] | 1281 | #ifdef CONFIG_DEBUG_MUTEXES |
| 1282 | p->blocked_on = NULL; /* not blocked yet */ |
| 1283 | #endif |
Andrew Morton | c255a45 | 2012-07-31 16:43:02 -0700 | [diff] [blame] | 1284 | #ifdef CONFIG_MEMCG |
KAMEZAWA Hiroyuki | 569b846 | 2009-12-15 16:47:03 -0800 | [diff] [blame] | 1285 | p->memcg_batch.do_batch = 0; |
| 1286 | p->memcg_batch.memcg = NULL; |
| 1287 | #endif |
Markus Metzger | 0f48140 | 2009-04-03 16:43:48 +0200 | [diff] [blame] | 1288 | |
Srivatsa Vaddagiri | 3c90e6e | 2007-11-09 22:39:39 +0100 | [diff] [blame] | 1289 | /* Perform scheduler related setup. Assign this task to a CPU. */ |
Samir Bellabes | 3e51e3e | 2011-05-11 18:18:05 +0200 | [diff] [blame] | 1290 | sched_fork(p); |
Peter Zijlstra | 6ab423e | 2009-05-25 14:45:27 +0200 | [diff] [blame] | 1291 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1292 | retval = perf_event_init_task(p); |
Peter Zijlstra | 6ab423e | 2009-05-25 14:45:27 +0200 | [diff] [blame] | 1293 | if (retval) |
| 1294 | goto bad_fork_cleanup_policy; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1295 | retval = audit_alloc(p); |
| 1296 | if (retval) |
David Howells | f1752ee | 2008-11-14 10:39:17 +1100 | [diff] [blame] | 1297 | goto bad_fork_cleanup_policy; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1298 | /* copy all the process information */ |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1299 | retval = copy_semundo(clone_flags, p); |
| 1300 | if (retval) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1301 | goto bad_fork_cleanup_audit; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1302 | retval = copy_files(clone_flags, p); |
| 1303 | if (retval) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1304 | goto bad_fork_cleanup_semundo; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1305 | retval = copy_fs(clone_flags, p); |
| 1306 | if (retval) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1307 | goto bad_fork_cleanup_files; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1308 | retval = copy_sighand(clone_flags, p); |
| 1309 | if (retval) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1310 | goto bad_fork_cleanup_fs; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1311 | retval = copy_signal(clone_flags, p); |
| 1312 | if (retval) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1313 | goto bad_fork_cleanup_sighand; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1314 | retval = copy_mm(clone_flags, p); |
| 1315 | if (retval) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1316 | goto bad_fork_cleanup_signal; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1317 | retval = copy_namespaces(clone_flags, p); |
| 1318 | if (retval) |
David Howells | d84f4f9 | 2008-11-14 10:39:23 +1100 | [diff] [blame] | 1319 | goto bad_fork_cleanup_mm; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1320 | retval = copy_io(clone_flags, p); |
| 1321 | if (retval) |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 1322 | goto bad_fork_cleanup_namespaces; |
Alexey Dobriyan | 6f2c55b | 2009-04-02 16:56:59 -0700 | [diff] [blame] | 1323 | retval = copy_thread(clone_flags, stack_start, stack_size, p, regs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1324 | if (retval) |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 1325 | goto bad_fork_cleanup_io; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1326 | |
Pavel Emelyanov | 425fb2b | 2007-10-18 23:40:07 -0700 | [diff] [blame] | 1327 | if (pid != &init_struct_pid) { |
| 1328 | retval = -ENOMEM; |
Eric W. Biederman | 61bce0f | 2009-01-07 18:08:49 -0800 | [diff] [blame] | 1329 | pid = alloc_pid(p->nsproxy->pid_ns); |
Pavel Emelyanov | 425fb2b | 2007-10-18 23:40:07 -0700 | [diff] [blame] | 1330 | if (!pid) |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 1331 | goto bad_fork_cleanup_io; |
Pavel Emelyanov | 425fb2b | 2007-10-18 23:40:07 -0700 | [diff] [blame] | 1332 | } |
| 1333 | |
| 1334 | p->pid = pid_nr(pid); |
| 1335 | p->tgid = p->pid; |
| 1336 | if (clone_flags & CLONE_THREAD) |
| 1337 | p->tgid = current->tgid; |
| 1338 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1339 | p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; |
| 1340 | /* |
| 1341 | * Clear TID on mm_release()? |
| 1342 | */ |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1343 | p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL; |
Jens Axboe | 73c1010 | 2011-03-08 13:19:51 +0100 | [diff] [blame] | 1344 | #ifdef CONFIG_BLOCK |
| 1345 | p->plug = NULL; |
| 1346 | #endif |
Alexey Dobriyan | 42b2dd0 | 2007-10-16 23:27:30 -0700 | [diff] [blame] | 1347 | #ifdef CONFIG_FUTEX |
Ingo Molnar | 8f17d3a | 2006-03-27 01:16:27 -0800 | [diff] [blame] | 1348 | p->robust_list = NULL; |
| 1349 | #ifdef CONFIG_COMPAT |
| 1350 | p->compat_robust_list = NULL; |
| 1351 | #endif |
Ingo Molnar | c87e283 | 2006-06-27 02:54:58 -0700 | [diff] [blame] | 1352 | INIT_LIST_HEAD(&p->pi_state_list); |
| 1353 | p->pi_state_cache = NULL; |
Alexey Dobriyan | 42b2dd0 | 2007-10-16 23:27:30 -0700 | [diff] [blame] | 1354 | #endif |
Srikar Dronamraju | 0326f5a9 | 2012-03-13 23:30:11 +0530 | [diff] [blame] | 1355 | uprobe_copy_process(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1356 | /* |
GOTO Masanori | f9a3879 | 2006-03-13 21:20:44 -0800 | [diff] [blame] | 1357 | * sigaltstack should be cleared when sharing the same VM |
| 1358 | */ |
| 1359 | if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) |
| 1360 | p->sas_ss_sp = p->sas_ss_size = 0; |
| 1361 | |
| 1362 | /* |
Oleg Nesterov | 6580807 | 2009-12-15 16:47:16 -0800 | [diff] [blame] | 1363 | * Syscall tracing and stepping should be turned off in the |
| 1364 | * child regardless of CLONE_PTRACE. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1365 | */ |
Oleg Nesterov | 6580807 | 2009-12-15 16:47:16 -0800 | [diff] [blame] | 1366 | user_disable_single_step(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1367 | clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE); |
Laurent Vivier | ed75e8d | 2005-09-03 15:57:18 -0700 | [diff] [blame] | 1368 | #ifdef TIF_SYSCALL_EMU |
| 1369 | clear_tsk_thread_flag(p, TIF_SYSCALL_EMU); |
| 1370 | #endif |
Arjan van de Ven | 9745512 | 2008-01-25 21:08:34 +0100 | [diff] [blame] | 1371 | clear_all_latency_tracing(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1372 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1373 | /* ok, now we should be set up.. */ |
Oleg Nesterov | 5f8aadd | 2012-03-14 19:55:38 +0100 | [diff] [blame] | 1374 | if (clone_flags & CLONE_THREAD) |
| 1375 | p->exit_signal = -1; |
| 1376 | else if (clone_flags & CLONE_PARENT) |
| 1377 | p->exit_signal = current->group_leader->exit_signal; |
| 1378 | else |
| 1379 | p->exit_signal = (clone_flags & CSIGNAL); |
| 1380 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1381 | p->pdeath_signal = 0; |
| 1382 | p->exit_state = 0; |
| 1383 | |
Wu Fengguang | 9d823e8 | 2011-06-11 18:10:12 -0600 | [diff] [blame] | 1384 | p->nr_dirtied = 0; |
| 1385 | p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); |
Wu Fengguang | 8371235 | 2011-06-11 19:25:42 -0600 | [diff] [blame] | 1386 | p->dirty_paused_when = 0; |
Wu Fengguang | 9d823e8 | 2011-06-11 18:10:12 -0600 | [diff] [blame] | 1387 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1388 | /* |
| 1389 | * Ok, make it visible to the rest of the system. |
| 1390 | * We dont wake it up yet. |
| 1391 | */ |
| 1392 | p->group_leader = p; |
Oleg Nesterov | 47e6532 | 2006-03-28 16:11:25 -0800 | [diff] [blame] | 1393 | INIT_LIST_HEAD(&p->thread_group); |
Al Viro | 158e164 | 2012-06-27 09:24:13 +0400 | [diff] [blame] | 1394 | p->task_works = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1395 | |
Paul Menage | b4f48b6 | 2007-10-18 23:39:33 -0700 | [diff] [blame] | 1396 | /* Now that the task is set up, run cgroup callbacks if |
| 1397 | * necessary. We need to run them before the task is visible |
| 1398 | * on the tasklist. */ |
| 1399 | cgroup_fork_callbacks(p); |
| 1400 | cgroup_callbacks_done = 1; |
| 1401 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1402 | /* Need tasklist lock for parent etc handling! */ |
| 1403 | write_lock_irq(&tasklist_lock); |
| 1404 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1405 | /* CLONE_PARENT re-uses the old parent */ |
Oleg Nesterov | 2d5516c | 2009-03-02 22:58:45 +0100 | [diff] [blame] | 1406 | if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1407 | p->real_parent = current->real_parent; |
Oleg Nesterov | 2d5516c | 2009-03-02 22:58:45 +0100 | [diff] [blame] | 1408 | p->parent_exec_id = current->parent_exec_id; |
| 1409 | } else { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1410 | p->real_parent = current; |
Oleg Nesterov | 2d5516c | 2009-03-02 22:58:45 +0100 | [diff] [blame] | 1411 | p->parent_exec_id = current->self_exec_id; |
| 1412 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1413 | |
Oleg Nesterov | 3f17da6 | 2006-02-15 22:13:24 +0300 | [diff] [blame] | 1414 | spin_lock(¤t->sighand->siglock); |
Oleg Nesterov | 4a2c7a7 | 2006-03-28 16:11:26 -0800 | [diff] [blame] | 1415 | |
| 1416 | /* |
| 1417 | * Process group and session signals need to be delivered to just the |
| 1418 | * parent before the fork or both the parent and the child after the |
| 1419 | * fork. Restart if a signal comes in before we add the new process to |
| 1420 | * it's process group. |
| 1421 | * A fatal signal pending means that current will exit, so the new |
| 1422 | * thread can't slip out of an OOM kill (or normal SIGKILL). |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1423 | */ |
Daniel Walker | 23ff444 | 2007-10-18 03:06:07 -0700 | [diff] [blame] | 1424 | recalc_sigpending(); |
Oleg Nesterov | 4a2c7a7 | 2006-03-28 16:11:26 -0800 | [diff] [blame] | 1425 | if (signal_pending(current)) { |
| 1426 | spin_unlock(¤t->sighand->siglock); |
| 1427 | write_unlock_irq(&tasklist_lock); |
| 1428 | retval = -ERESTARTNOINTR; |
Steven Rostedt | f7e8b61 | 2009-06-02 16:39:48 -0400 | [diff] [blame] | 1429 | goto bad_fork_free_pid; |
Oleg Nesterov | 4a2c7a7 | 2006-03-28 16:11:26 -0800 | [diff] [blame] | 1430 | } |
| 1431 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1432 | if (clone_flags & CLONE_THREAD) { |
Oleg Nesterov | b3ac022 | 2010-05-26 14:43:24 -0700 | [diff] [blame] | 1433 | current->signal->nr_threads++; |
Oleg Nesterov | 4ab6c08 | 2009-08-26 14:29:24 -0700 | [diff] [blame] | 1434 | atomic_inc(¤t->signal->live); |
Oleg Nesterov | b3ac022 | 2010-05-26 14:43:24 -0700 | [diff] [blame] | 1435 | atomic_inc(¤t->signal->sigcnt); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1436 | p->group_leader = current->group_leader; |
Oleg Nesterov | 47e6532 | 2006-03-28 16:11:25 -0800 | [diff] [blame] | 1437 | list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1438 | } |
| 1439 | |
Oleg Nesterov | 73b9ebf | 2006-03-28 16:11:07 -0800 | [diff] [blame] | 1440 | if (likely(p->pid)) { |
Tejun Heo | 4b9d33e | 2011-06-17 16:50:38 +0200 | [diff] [blame] | 1441 | ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1442 | |
Oleg Nesterov | 73b9ebf | 2006-03-28 16:11:07 -0800 | [diff] [blame] | 1443 | if (thread_group_leader(p)) { |
Eric W. Biederman | 45a6862 | 2011-03-23 16:43:12 -0700 | [diff] [blame] | 1444 | if (is_child_reaper(pid)) |
Pavel Emelyanov | 30e49c2 | 2007-10-18 23:40:10 -0700 | [diff] [blame] | 1445 | p->nsproxy->pid_ns->child_reaper = p; |
Oleg Nesterov | c97d989 | 2006-03-28 16:11:06 -0800 | [diff] [blame] | 1446 | |
Oleg Nesterov | fea9d17 | 2008-02-08 04:19:19 -0800 | [diff] [blame] | 1447 | p->signal->leader_pid = pid; |
Alan Cox | 9c9f4de | 2008-10-13 10:37:26 +0100 | [diff] [blame] | 1448 | p->signal->tty = tty_kref_get(current->signal->tty); |
Eric W. Biederman | 5cd1756 | 2007-12-04 23:45:04 -0800 | [diff] [blame] | 1449 | attach_pid(p, PIDTYPE_PGID, task_pgrp(current)); |
| 1450 | attach_pid(p, PIDTYPE_SID, task_session(current)); |
Oleg Nesterov | 9cd80bb | 2009-12-17 15:27:15 -0800 | [diff] [blame] | 1451 | list_add_tail(&p->sibling, &p->real_parent->children); |
Eric W. Biederman | 5e85d4a | 2006-04-18 22:20:16 -0700 | [diff] [blame] | 1452 | list_add_tail_rcu(&p->tasks, &init_task.tasks); |
Christoph Lameter | 909ea96 | 2010-12-08 16:22:55 +0100 | [diff] [blame] | 1453 | __this_cpu_inc(process_counts); |
Oleg Nesterov | 73b9ebf | 2006-03-28 16:11:07 -0800 | [diff] [blame] | 1454 | } |
Sukadev Bhattiprolu | 8586899 | 2007-05-10 22:23:03 -0700 | [diff] [blame] | 1455 | attach_pid(p, PIDTYPE_PID, pid); |
Oleg Nesterov | 73b9ebf | 2006-03-28 16:11:07 -0800 | [diff] [blame] | 1456 | nr_threads++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1457 | } |
| 1458 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1459 | total_forks++; |
Oleg Nesterov | 3f17da6 | 2006-02-15 22:13:24 +0300 | [diff] [blame] | 1460 | spin_unlock(¤t->sighand->siglock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1461 | write_unlock_irq(&tasklist_lock); |
Andrew Morton | c13cf85 | 2005-11-28 13:43:48 -0800 | [diff] [blame] | 1462 | proc_fork_connector(p); |
Paul Menage | 817929e | 2007-10-18 23:39:36 -0700 | [diff] [blame] | 1463 | cgroup_post_fork(p); |
Ben Blum | 4714d1d | 2011-05-26 16:25:18 -0700 | [diff] [blame] | 1464 | if (clone_flags & CLONE_THREAD) |
Tejun Heo | 257058a | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 1465 | threadgroup_change_end(current); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1466 | perf_event_fork(p); |
KAMEZAWA Hiroyuki | 43d2b11 | 2012-01-10 15:08:09 -0800 | [diff] [blame] | 1467 | |
| 1468 | trace_task_newtask(p, clone_flags); |
| 1469 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1470 | return p; |
| 1471 | |
Pavel Emelyanov | 425fb2b | 2007-10-18 23:40:07 -0700 | [diff] [blame] | 1472 | bad_fork_free_pid: |
| 1473 | if (pid != &init_struct_pid) |
| 1474 | free_pid(pid); |
Jens Axboe | fd0928d | 2008-01-24 08:52:45 +0100 | [diff] [blame] | 1475 | bad_fork_cleanup_io: |
Louis Rilling | b69f229 | 2009-12-04 14:52:42 +0100 | [diff] [blame] | 1476 | if (p->io_context) |
| 1477 | exit_io_context(p); |
Serge E. Hallyn | ab51601 | 2006-10-02 02:18:06 -0700 | [diff] [blame] | 1478 | bad_fork_cleanup_namespaces: |
Mike Galbraith | 5e2bf01 | 2012-05-10 13:01:45 -0700 | [diff] [blame] | 1479 | if (unlikely(clone_flags & CLONE_NEWPID)) |
| 1480 | pid_ns_release_proc(p->nsproxy->pid_ns); |
Linus Torvalds | 444f378 | 2007-01-30 13:35:18 -0800 | [diff] [blame] | 1481 | exit_task_namespaces(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1482 | bad_fork_cleanup_mm: |
David Rientjes | c9f0124 | 2011-10-31 17:07:15 -0700 | [diff] [blame] | 1483 | if (p->mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1484 | mmput(p->mm); |
| 1485 | bad_fork_cleanup_signal: |
Oleg Nesterov | 4ab6c08 | 2009-08-26 14:29:24 -0700 | [diff] [blame] | 1486 | if (!(clone_flags & CLONE_THREAD)) |
Mike Galbraith | 1c5354d | 2011-01-05 11:16:04 +0100 | [diff] [blame] | 1487 | free_signal_struct(p->signal); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1488 | bad_fork_cleanup_sighand: |
Oleg Nesterov | a7e5328 | 2006-03-28 16:11:27 -0800 | [diff] [blame] | 1489 | __cleanup_sighand(p->sighand); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1490 | bad_fork_cleanup_fs: |
| 1491 | exit_fs(p); /* blocking */ |
| 1492 | bad_fork_cleanup_files: |
| 1493 | exit_files(p); /* blocking */ |
| 1494 | bad_fork_cleanup_semundo: |
| 1495 | exit_sem(p); |
| 1496 | bad_fork_cleanup_audit: |
| 1497 | audit_free(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1498 | bad_fork_cleanup_policy: |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1499 | perf_event_free_task(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1500 | #ifdef CONFIG_NUMA |
Lee Schermerhorn | f0be3d3 | 2008-04-28 02:13:08 -0700 | [diff] [blame] | 1501 | mpol_put(p->mempolicy); |
Paul Menage | b4f48b6 | 2007-10-18 23:39:33 -0700 | [diff] [blame] | 1502 | bad_fork_cleanup_cgroup: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1503 | #endif |
Ben Blum | 4714d1d | 2011-05-26 16:25:18 -0700 | [diff] [blame] | 1504 | if (clone_flags & CLONE_THREAD) |
Tejun Heo | 257058a | 2011-12-12 18:12:21 -0800 | [diff] [blame] | 1505 | threadgroup_change_end(current); |
Paul Menage | b4f48b6 | 2007-10-18 23:39:33 -0700 | [diff] [blame] | 1506 | cgroup_exit(p, cgroup_callbacks_done); |
Shailabh Nagar | 35df17c | 2006-08-31 21:27:38 -0700 | [diff] [blame] | 1507 | delayacct_tsk_free(p); |
Al Viro | a1261f54 | 2005-11-13 16:06:55 -0800 | [diff] [blame] | 1508 | module_put(task_thread_info(p)->exec_domain->module); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1509 | bad_fork_cleanup_count: |
David Howells | d84f4f9 | 2008-11-14 10:39:23 +1100 | [diff] [blame] | 1510 | atomic_dec(&p->cred->user->processes); |
David Howells | e0e8173 | 2009-09-02 09:13:40 +0100 | [diff] [blame] | 1511 | exit_creds(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1512 | bad_fork_free: |
| 1513 | free_task(p); |
Oleg Nesterov | fe7d37d | 2006-01-08 01:04:02 -0800 | [diff] [blame] | 1514 | fork_out: |
| 1515 | return ERR_PTR(retval); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1516 | } |
| 1517 | |
Adrian Bunk | 6b2fb3c | 2008-02-06 01:37:55 -0800 | [diff] [blame] | 1518 | noinline struct pt_regs * __cpuinit __attribute__((weak)) idle_regs(struct pt_regs *regs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1519 | { |
| 1520 | memset(regs, 0, sizeof(struct pt_regs)); |
| 1521 | return regs; |
| 1522 | } |
| 1523 | |
Oleg Nesterov | f106eee | 2010-05-26 14:44:11 -0700 | [diff] [blame] | 1524 | static inline void init_idle_pids(struct pid_link *links) |
| 1525 | { |
| 1526 | enum pid_type type; |
| 1527 | |
| 1528 | for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) { |
| 1529 | INIT_HLIST_NODE(&links[type].node); /* not really needed */ |
| 1530 | links[type].pid = &init_struct_pid; |
| 1531 | } |
| 1532 | } |
| 1533 | |
Al Viro | 9abcf40 | 2007-02-01 13:52:48 +0000 | [diff] [blame] | 1534 | struct task_struct * __cpuinit fork_idle(int cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1535 | { |
Ingo Molnar | 36c8b58 | 2006-07-03 00:25:41 -0700 | [diff] [blame] | 1536 | struct task_struct *task; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1537 | struct pt_regs regs; |
| 1538 | |
Pavel Emelyanov | 30e49c2 | 2007-10-18 23:40:10 -0700 | [diff] [blame] | 1539 | task = copy_process(CLONE_VM, 0, idle_regs(®s), 0, NULL, |
Roland McGrath | 09a0539 | 2008-07-25 19:45:47 -0700 | [diff] [blame] | 1540 | &init_struct_pid, 0); |
Oleg Nesterov | f106eee | 2010-05-26 14:44:11 -0700 | [diff] [blame] | 1541 | if (!IS_ERR(task)) { |
| 1542 | init_idle_pids(task->pids); |
Akinobu Mita | 753ca4f | 2006-11-25 11:09:34 -0800 | [diff] [blame] | 1543 | init_idle(task, cpu); |
Oleg Nesterov | f106eee | 2010-05-26 14:44:11 -0700 | [diff] [blame] | 1544 | } |
Oleg Nesterov | 73b9ebf | 2006-03-28 16:11:07 -0800 | [diff] [blame] | 1545 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1546 | return task; |
| 1547 | } |
| 1548 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1549 | /* |
| 1550 | * Ok, this is the main fork-routine. |
| 1551 | * |
| 1552 | * It copies the process, and if successful kick-starts |
| 1553 | * it and waits for it to finish using the VM if required. |
| 1554 | */ |
| 1555 | long do_fork(unsigned long clone_flags, |
| 1556 | unsigned long stack_start, |
| 1557 | struct pt_regs *regs, |
| 1558 | unsigned long stack_size, |
| 1559 | int __user *parent_tidptr, |
| 1560 | int __user *child_tidptr) |
| 1561 | { |
| 1562 | struct task_struct *p; |
| 1563 | int trace = 0; |
Eric W. Biederman | 92476d7 | 2006-03-31 02:31:42 -0800 | [diff] [blame] | 1564 | long nr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1565 | |
Andrew Morton | bdff746 | 2008-02-04 22:27:22 -0800 | [diff] [blame] | 1566 | /* |
Serge Hallyn | 18b6e04 | 2008-10-15 16:38:45 -0500 | [diff] [blame] | 1567 | * Do some preliminary argument and permissions checking before we |
| 1568 | * actually start allocating stuff |
| 1569 | */ |
| 1570 | if (clone_flags & CLONE_NEWUSER) { |
| 1571 | if (clone_flags & CLONE_THREAD) |
| 1572 | return -EINVAL; |
| 1573 | /* hopefully this check will go away when userns support is |
| 1574 | * complete |
| 1575 | */ |
Serge E. Hallyn | 7657d90 | 2008-12-03 13:17:33 -0600 | [diff] [blame] | 1576 | if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SETUID) || |
| 1577 | !capable(CAP_SETGID)) |
Serge Hallyn | 18b6e04 | 2008-10-15 16:38:45 -0500 | [diff] [blame] | 1578 | return -EPERM; |
| 1579 | } |
| 1580 | |
| 1581 | /* |
Tejun Heo | 4b9d33e | 2011-06-17 16:50:38 +0200 | [diff] [blame] | 1582 | * Determine whether and which event to report to ptracer. When |
| 1583 | * called from kernel_thread or CLONE_UNTRACED is explicitly |
| 1584 | * requested, no event is reported; otherwise, report if the event |
| 1585 | * for the type of forking is enabled. |
Roland McGrath | 09a0539 | 2008-07-25 19:45:47 -0700 | [diff] [blame] | 1586 | */ |
Tejun Heo | 4b9d33e | 2011-06-17 16:50:38 +0200 | [diff] [blame] | 1587 | if (likely(user_mode(regs)) && !(clone_flags & CLONE_UNTRACED)) { |
| 1588 | if (clone_flags & CLONE_VFORK) |
| 1589 | trace = PTRACE_EVENT_VFORK; |
| 1590 | else if ((clone_flags & CSIGNAL) != SIGCHLD) |
| 1591 | trace = PTRACE_EVENT_CLONE; |
| 1592 | else |
| 1593 | trace = PTRACE_EVENT_FORK; |
| 1594 | |
| 1595 | if (likely(!ptrace_event_enabled(current, trace))) |
| 1596 | trace = 0; |
| 1597 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1598 | |
Sukadev Bhattiprolu | a6f5e06 | 2007-10-18 23:39:53 -0700 | [diff] [blame] | 1599 | p = copy_process(clone_flags, stack_start, regs, stack_size, |
Roland McGrath | 09a0539 | 2008-07-25 19:45:47 -0700 | [diff] [blame] | 1600 | child_tidptr, NULL, trace); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1601 | /* |
| 1602 | * Do this prior waking up the new thread - the thread pointer |
| 1603 | * might get invalid after that point, if the thread exits quickly. |
| 1604 | */ |
| 1605 | if (!IS_ERR(p)) { |
| 1606 | struct completion vfork; |
| 1607 | |
Mathieu Desnoyers | 0a16b60 | 2008-07-18 12:16:17 -0400 | [diff] [blame] | 1608 | trace_sched_process_fork(current, p); |
| 1609 | |
Pavel Emelyanov | 6c5f3e7 | 2008-02-08 04:19:20 -0800 | [diff] [blame] | 1610 | nr = task_pid_vnr(p); |
Pavel Emelyanov | 30e49c2 | 2007-10-18 23:40:10 -0700 | [diff] [blame] | 1611 | |
| 1612 | if (clone_flags & CLONE_PARENT_SETTID) |
| 1613 | put_user(nr, parent_tidptr); |
Sukadev Bhattiprolu | a6f5e06 | 2007-10-18 23:39:53 -0700 | [diff] [blame] | 1614 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1615 | if (clone_flags & CLONE_VFORK) { |
| 1616 | p->vfork_done = &vfork; |
| 1617 | init_completion(&vfork); |
Oleg Nesterov | d68b46f | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 1618 | get_task_struct(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1619 | } |
| 1620 | |
Samir Bellabes | 3e51e3e | 2011-05-11 18:18:05 +0200 | [diff] [blame] | 1621 | wake_up_new_task(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1622 | |
Tejun Heo | 4b9d33e | 2011-06-17 16:50:38 +0200 | [diff] [blame] | 1623 | /* forking complete and child started to run, tell ptracer */ |
| 1624 | if (unlikely(trace)) |
| 1625 | ptrace_event(trace, nr); |
Roland McGrath | 09a0539 | 2008-07-25 19:45:47 -0700 | [diff] [blame] | 1626 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1627 | if (clone_flags & CLONE_VFORK) { |
Oleg Nesterov | d68b46f | 2012-03-05 14:59:13 -0800 | [diff] [blame] | 1628 | if (!wait_for_vfork_done(p, &vfork)) |
| 1629 | ptrace_event(PTRACE_EVENT_VFORK_DONE, nr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1630 | } |
| 1631 | } else { |
Eric W. Biederman | 92476d7 | 2006-03-31 02:31:42 -0800 | [diff] [blame] | 1632 | nr = PTR_ERR(p); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1633 | } |
Eric W. Biederman | 92476d7 | 2006-03-31 02:31:42 -0800 | [diff] [blame] | 1634 | return nr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1635 | } |
| 1636 | |
Ravikiran G Thirumalai | 5fd63b3 | 2006-01-11 22:46:15 +0100 | [diff] [blame] | 1637 | #ifndef ARCH_MIN_MMSTRUCT_ALIGN |
| 1638 | #define ARCH_MIN_MMSTRUCT_ALIGN 0 |
| 1639 | #endif |
| 1640 | |
Alexey Dobriyan | 51cc506 | 2008-07-25 19:45:34 -0700 | [diff] [blame] | 1641 | static void sighand_ctor(void *data) |
Oleg Nesterov | aa1757f | 2006-03-28 16:11:12 -0800 | [diff] [blame] | 1642 | { |
| 1643 | struct sighand_struct *sighand = data; |
| 1644 | |
Christoph Lameter | a35afb8 | 2007-05-16 22:10:57 -0700 | [diff] [blame] | 1645 | spin_lock_init(&sighand->siglock); |
Davide Libenzi | b8fceee | 2007-09-20 12:40:16 -0700 | [diff] [blame] | 1646 | init_waitqueue_head(&sighand->signalfd_wqh); |
Oleg Nesterov | aa1757f | 2006-03-28 16:11:12 -0800 | [diff] [blame] | 1647 | } |
| 1648 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1649 | void __init proc_caches_init(void) |
| 1650 | { |
| 1651 | sighand_cachep = kmem_cache_create("sighand_cache", |
| 1652 | sizeof(struct sighand_struct), 0, |
Vegard Nossum | 2dff440 | 2008-05-31 15:56:17 +0200 | [diff] [blame] | 1653 | SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU| |
| 1654 | SLAB_NOTRACK, sighand_ctor); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1655 | signal_cachep = kmem_cache_create("signal_cache", |
| 1656 | sizeof(struct signal_struct), 0, |
Vegard Nossum | 2dff440 | 2008-05-31 15:56:17 +0200 | [diff] [blame] | 1657 | SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); |
Paul Mundt | 20c2df8 | 2007-07-20 10:11:58 +0900 | [diff] [blame] | 1658 | files_cachep = kmem_cache_create("files_cache", |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1659 | sizeof(struct files_struct), 0, |
Vegard Nossum | 2dff440 | 2008-05-31 15:56:17 +0200 | [diff] [blame] | 1660 | SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); |
Paul Mundt | 20c2df8 | 2007-07-20 10:11:58 +0900 | [diff] [blame] | 1661 | fs_cachep = kmem_cache_create("fs_cache", |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1662 | sizeof(struct fs_struct), 0, |
Vegard Nossum | 2dff440 | 2008-05-31 15:56:17 +0200 | [diff] [blame] | 1663 | SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); |
Linus Torvalds | 6345d24 | 2011-05-29 11:32:28 -0700 | [diff] [blame] | 1664 | /* |
| 1665 | * FIXME! The "sizeof(struct mm_struct)" currently includes the |
| 1666 | * whole struct cpumask for the OFFSTACK case. We could change |
| 1667 | * this to *only* allocate as much of it as required by the |
| 1668 | * maximum number of CPU's we can ever have. The cpumask_allocation |
| 1669 | * is at the end of the structure, exactly for that reason. |
| 1670 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1671 | mm_cachep = kmem_cache_create("mm_struct", |
Ravikiran G Thirumalai | 5fd63b3 | 2006-01-11 22:46:15 +0100 | [diff] [blame] | 1672 | sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN, |
Vegard Nossum | 2dff440 | 2008-05-31 15:56:17 +0200 | [diff] [blame] | 1673 | SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); |
David Howells | 33e5d769 | 2009-04-02 16:56:32 -0700 | [diff] [blame] | 1674 | vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC); |
David Howells | 8feae13 | 2009-01-08 12:04:47 +0000 | [diff] [blame] | 1675 | mmap_init(); |
Al Viro | 6657719 | 2011-06-28 15:41:10 -0400 | [diff] [blame] | 1676 | nsproxy_cache_init(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1677 | } |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1678 | |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1679 | /* |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1680 | * Check constraints on flags passed to the unshare system call. |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1681 | */ |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1682 | static int check_unshare_flags(unsigned long unshare_flags) |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1683 | { |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1684 | if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| |
| 1685 | CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| |
| 1686 | CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET)) |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1687 | return -EINVAL; |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1688 | /* |
| 1689 | * Not implemented, but pretend it works if there is nothing to |
| 1690 | * unshare. Note that unsharing CLONE_THREAD or CLONE_SIGHAND |
| 1691 | * needs to unshare vm. |
| 1692 | */ |
| 1693 | if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) { |
| 1694 | /* FIXME: get_task_mm() increments ->mm_users */ |
| 1695 | if (atomic_read(¤t->mm->mm_users) > 1) |
| 1696 | return -EINVAL; |
| 1697 | } |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1698 | |
| 1699 | return 0; |
| 1700 | } |
| 1701 | |
| 1702 | /* |
JANAK DESAI | 99d1419 | 2006-02-07 12:58:59 -0800 | [diff] [blame] | 1703 | * Unshare the filesystem structure if it is being shared |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1704 | */ |
| 1705 | static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) |
| 1706 | { |
| 1707 | struct fs_struct *fs = current->fs; |
| 1708 | |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 1709 | if (!(unshare_flags & CLONE_FS) || !fs) |
| 1710 | return 0; |
| 1711 | |
| 1712 | /* don't need lock here; in the worst case we'll do useless copy */ |
| 1713 | if (fs->users == 1) |
| 1714 | return 0; |
| 1715 | |
| 1716 | *new_fsp = copy_fs_struct(fs); |
| 1717 | if (!*new_fsp) |
| 1718 | return -ENOMEM; |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1719 | |
| 1720 | return 0; |
| 1721 | } |
| 1722 | |
| 1723 | /* |
JANAK DESAI | a016f33 | 2006-02-07 12:59:02 -0800 | [diff] [blame] | 1724 | * Unshare file descriptor table if it is being shared |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1725 | */ |
| 1726 | static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp) |
| 1727 | { |
| 1728 | struct files_struct *fd = current->files; |
JANAK DESAI | a016f33 | 2006-02-07 12:59:02 -0800 | [diff] [blame] | 1729 | int error = 0; |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1730 | |
| 1731 | if ((unshare_flags & CLONE_FILES) && |
JANAK DESAI | a016f33 | 2006-02-07 12:59:02 -0800 | [diff] [blame] | 1732 | (fd && atomic_read(&fd->count) > 1)) { |
| 1733 | *new_fdp = dup_fd(fd, &error); |
| 1734 | if (!*new_fdp) |
| 1735 | return error; |
| 1736 | } |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1737 | |
| 1738 | return 0; |
| 1739 | } |
| 1740 | |
| 1741 | /* |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1742 | * unshare allows a process to 'unshare' part of the process |
| 1743 | * context which was originally shared using clone. copy_* |
| 1744 | * functions used by do_fork() cannot be used here directly |
| 1745 | * because they modify an inactive task_struct that is being |
| 1746 | * constructed. Here we are modifying the current, active, |
| 1747 | * task_struct. |
| 1748 | */ |
Heiko Carstens | 6559eed8 | 2009-01-14 14:14:32 +0100 | [diff] [blame] | 1749 | SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1750 | { |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1751 | struct fs_struct *fs, *new_fs = NULL; |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1752 | struct files_struct *fd, *new_fd = NULL; |
Pavel Emelyanov | cf7b708 | 2007-10-18 23:39:54 -0700 | [diff] [blame] | 1753 | struct nsproxy *new_nsproxy = NULL; |
Manfred Spraul | 9edff4a | 2008-04-29 01:00:57 -0700 | [diff] [blame] | 1754 | int do_sysvsem = 0; |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1755 | int err; |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1756 | |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1757 | err = check_unshare_flags(unshare_flags); |
| 1758 | if (err) |
Eric W. Biederman | 06f9d4f | 2006-03-22 00:07:40 -0800 | [diff] [blame] | 1759 | goto bad_unshare_out; |
| 1760 | |
Manfred Spraul | 6013f67 | 2008-04-29 01:00:59 -0700 | [diff] [blame] | 1761 | /* |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1762 | * If unsharing namespace, must also unshare filesystem information. |
| 1763 | */ |
| 1764 | if (unshare_flags & CLONE_NEWNS) |
| 1765 | unshare_flags |= CLONE_FS; |
| 1766 | /* |
Manfred Spraul | 6013f67 | 2008-04-29 01:00:59 -0700 | [diff] [blame] | 1767 | * CLONE_NEWIPC must also detach from the undolist: after switching |
| 1768 | * to a new ipc namespace, the semaphore arrays from the old |
| 1769 | * namespace are unreachable. |
| 1770 | */ |
| 1771 | if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) |
Manfred Spraul | 9edff4a | 2008-04-29 01:00:57 -0700 | [diff] [blame] | 1772 | do_sysvsem = 1; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1773 | err = unshare_fs(unshare_flags, &new_fs); |
| 1774 | if (err) |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1775 | goto bad_unshare_out; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1776 | err = unshare_fd(unshare_flags, &new_fd); |
| 1777 | if (err) |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1778 | goto bad_unshare_cleanup_fs; |
Daniel Rebelo de Oliveira | fb0a685 | 2011-07-26 16:08:39 -0700 | [diff] [blame] | 1779 | err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, new_fs); |
| 1780 | if (err) |
Manfred Spraul | 9edff4a | 2008-04-29 01:00:57 -0700 | [diff] [blame] | 1781 | goto bad_unshare_cleanup_fd; |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1782 | |
Oleg Nesterov | 9bfb23f | 2011-03-22 16:34:09 -0700 | [diff] [blame] | 1783 | if (new_fs || new_fd || do_sysvsem || new_nsproxy) { |
Manfred Spraul | 9edff4a | 2008-04-29 01:00:57 -0700 | [diff] [blame] | 1784 | if (do_sysvsem) { |
| 1785 | /* |
| 1786 | * CLONE_SYSVSEM is equivalent to sys_exit(). |
| 1787 | */ |
| 1788 | exit_sem(current); |
| 1789 | } |
Serge E. Hallyn | ab51601 | 2006-10-02 02:18:06 -0700 | [diff] [blame] | 1790 | |
Serge Hallyn | c0b2fc3 | 2006-10-02 02:18:18 -0700 | [diff] [blame] | 1791 | if (new_nsproxy) { |
Pavel Emelyanov | cf7b708 | 2007-10-18 23:39:54 -0700 | [diff] [blame] | 1792 | switch_task_namespaces(current, new_nsproxy); |
| 1793 | new_nsproxy = NULL; |
Serge Hallyn | c0b2fc3 | 2006-10-02 02:18:18 -0700 | [diff] [blame] | 1794 | } |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1795 | |
Pavel Emelyanov | cf7b708 | 2007-10-18 23:39:54 -0700 | [diff] [blame] | 1796 | task_lock(current); |
| 1797 | |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1798 | if (new_fs) { |
| 1799 | fs = current->fs; |
Nick Piggin | 2a4419b | 2010-08-18 04:37:33 +1000 | [diff] [blame] | 1800 | spin_lock(&fs->lock); |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1801 | current->fs = new_fs; |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 1802 | if (--fs->users) |
| 1803 | new_fs = NULL; |
| 1804 | else |
| 1805 | new_fs = fs; |
Nick Piggin | 2a4419b | 2010-08-18 04:37:33 +1000 | [diff] [blame] | 1806 | spin_unlock(&fs->lock); |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1807 | } |
| 1808 | |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1809 | if (new_fd) { |
| 1810 | fd = current->files; |
| 1811 | current->files = new_fd; |
| 1812 | new_fd = fd; |
| 1813 | } |
| 1814 | |
| 1815 | task_unlock(current); |
| 1816 | } |
| 1817 | |
Serge Hallyn | c0b2fc3 | 2006-10-02 02:18:18 -0700 | [diff] [blame] | 1818 | if (new_nsproxy) |
Linus Torvalds | 444f378 | 2007-01-30 13:35:18 -0800 | [diff] [blame] | 1819 | put_nsproxy(new_nsproxy); |
Serge Hallyn | c0b2fc3 | 2006-10-02 02:18:18 -0700 | [diff] [blame] | 1820 | |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1821 | bad_unshare_cleanup_fd: |
| 1822 | if (new_fd) |
| 1823 | put_files_struct(new_fd); |
| 1824 | |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1825 | bad_unshare_cleanup_fs: |
| 1826 | if (new_fs) |
Al Viro | 498052b | 2009-03-30 07:20:30 -0400 | [diff] [blame] | 1827 | free_fs_struct(new_fs); |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1828 | |
JANAK DESAI | cf2e340 | 2006-02-07 12:58:58 -0800 | [diff] [blame] | 1829 | bad_unshare_out: |
| 1830 | return err; |
| 1831 | } |
Al Viro | 3b12538 | 2008-04-22 05:31:30 -0400 | [diff] [blame] | 1832 | |
| 1833 | /* |
| 1834 | * Helper to unshare the files of the current task. |
| 1835 | * We don't want to expose copy_files internals to |
| 1836 | * the exec layer of the kernel. |
| 1837 | */ |
| 1838 | |
| 1839 | int unshare_files(struct files_struct **displaced) |
| 1840 | { |
| 1841 | struct task_struct *task = current; |
Al Viro | 5070451 | 2008-04-26 05:25:00 +0100 | [diff] [blame] | 1842 | struct files_struct *copy = NULL; |
Al Viro | 3b12538 | 2008-04-22 05:31:30 -0400 | [diff] [blame] | 1843 | int error; |
| 1844 | |
| 1845 | error = unshare_fd(CLONE_FILES, ©); |
| 1846 | if (error || !copy) { |
| 1847 | *displaced = NULL; |
| 1848 | return error; |
| 1849 | } |
| 1850 | *displaced = task->files; |
| 1851 | task_lock(task); |
| 1852 | task->files = copy; |
| 1853 | task_unlock(task); |
| 1854 | return 0; |
| 1855 | } |