blob: 1cd7d581b3b2da32e07c57a8342d597ee7ceb73f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/fork.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7/*
8 * 'fork.c' contains the help-routines for the 'fork' system call
9 * (see also entry.S and others).
10 * Fork is rather simple, once you get the hang of it, but the memory
11 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
12 */
13
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/slab.h>
15#include <linux/init.h>
16#include <linux/unistd.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/module.h>
18#include <linux/vmalloc.h>
19#include <linux/completion.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/personality.h>
21#include <linux/mempolicy.h>
22#include <linux/sem.h>
23#include <linux/file.h>
Al Viro9f3acc32008-04-24 07:44:08 -040024#include <linux/fdtable.h>
Jens Axboeda9cbc82008-06-30 20:42:08 +020025#include <linux/iocontext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/key.h>
27#include <linux/binfmts.h>
28#include <linux/mman.h>
Andrea Arcangelicddb8a52008-07-28 15:46:29 -070029#include <linux/mmu_notifier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/fs.h>
Serge E. Hallynab516012006-10-02 02:18:06 -070031#include <linux/nsproxy.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080032#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <linux/cpu.h>
Paul Menageb4f48b62007-10-18 23:39:33 -070034#include <linux/cgroup.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/security.h>
Mel Gormana1e78772008-07-23 21:27:23 -070036#include <linux/hugetlb.h>
Will Drewrye2cfabdf2012-04-12 16:47:57 -050037#include <linux/seccomp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#include <linux/swap.h>
39#include <linux/syscalls.h>
40#include <linux/jiffies.h>
41#include <linux/futex.h>
Linus Torvalds8141c7f2008-11-15 10:20:36 -080042#include <linux/compat.h>
Eric Dumazet207205a2011-03-22 16:30:44 -070043#include <linux/kthread.h>
Andrew Morton7c3ab7382006-12-10 02:19:19 -080044#include <linux/task_io_accounting_ops.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070045#include <linux/rcupdate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070046#include <linux/ptrace.h>
47#include <linux/mount.h>
48#include <linux/audit.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080049#include <linux/memcontrol.h>
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +010050#include <linux/ftrace.h>
Mike Galbraith5e2bf012012-05-10 13:01:45 -070051#include <linux/proc_fs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070052#include <linux/profile.h>
53#include <linux/rmap.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070054#include <linux/ksm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/acct.h>
Jay Lan8f0ab512006-09-30 23:28:59 -070056#include <linux/tsacct_kern.h>
Matt Helsley9f460802005-11-07 00:59:16 -080057#include <linux/cn_proc.h>
Rafael J. Wysockiba96a0c2007-05-23 13:57:25 -070058#include <linux/freezer.h>
Shailabh Nagarca74e922006-07-14 00:24:36 -070059#include <linux/delayacct.h>
Shailabh Nagarad4ecbc2006-07-14 00:24:44 -070060#include <linux/taskstats_kern.h>
Arjan van de Ven0a4254052006-09-26 10:52:38 +020061#include <linux/random.h>
Miloslav Trmac522ed772007-07-15 23:40:56 -070062#include <linux/tty.h>
Jens Axboefd0928d2008-01-24 08:52:45 +010063#include <linux/blkdev.h>
Al Viro5ad4e532009-03-29 19:50:06 -040064#include <linux/fs_struct.h>
Eric Sandeen7c9f8862008-04-22 16:38:23 -050065#include <linux/magic.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020066#include <linux/perf_event.h>
Stanislaw Gruszka42c4ab42009-07-29 12:15:26 +020067#include <linux/posix-timers.h>
Avi Kivity8e7cac72009-11-29 16:34:48 +020068#include <linux/user-return-notifier.h>
Ying Han3d5992d2010-10-26 14:21:23 -070069#include <linux/oom.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080070#include <linux/khugepaged.h>
Oleg Nesterovd80e7312012-02-24 20:07:11 +010071#include <linux/signalfd.h>
Srikar Dronamraju0326f5a92012-03-13 23:30:11 +053072#include <linux/uprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
74#include <asm/pgtable.h>
75#include <asm/pgalloc.h>
76#include <asm/uaccess.h>
77#include <asm/mmu_context.h>
78#include <asm/cacheflush.h>
79#include <asm/tlbflush.h>
80
Steven Rostedtad8d75f2009-04-14 19:39:12 -040081#include <trace/events/sched.h>
82
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -080083#define CREATE_TRACE_POINTS
84#include <trace/events/task.h>
85
Linus Torvalds1da177e2005-04-16 15:20:36 -070086/*
87 * Protected counters by write_lock_irq(&tasklist_lock)
88 */
89unsigned long total_forks; /* Handle normal Linux uptimes. */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -070090int nr_threads; /* The idle threads do not count.. */
Linus Torvalds1da177e2005-04-16 15:20:36 -070091
92int max_threads; /* tunable limit on nr_threads */
93
94DEFINE_PER_CPU(unsigned long, process_counts) = 0;
95
Christoph Hellwigc59923a2006-07-10 04:45:40 -070096__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
Paul E. McKenneydb1466b2010-03-03 07:46:56 -080097
98#ifdef CONFIG_PROVE_RCU
99int lockdep_tasklist_lock_is_held(void)
100{
101 return lockdep_is_held(&tasklist_lock);
102}
103EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
104#endif /* #ifdef CONFIG_PROVE_RCU */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
106int nr_processes(void)
107{
108 int cpu;
109 int total = 0;
110
Ian Campbell1d510752009-11-03 10:11:14 +0000111 for_each_possible_cpu(cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 total += per_cpu(process_counts, cpu);
113
114 return total;
115}
116
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700117void __weak arch_release_task_struct(struct task_struct *tsk)
118{
119}
120
Thomas Gleixnerf5e10282012-05-05 15:05:48 +0000121#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
Christoph Lametere18b8902006-12-06 20:33:20 -0800122static struct kmem_cache *task_struct_cachep;
Thomas Gleixner41101802012-05-05 15:05:41 +0000123
124static inline struct task_struct *alloc_task_struct_node(int node)
125{
126 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
127}
128
Thomas Gleixner41101802012-05-05 15:05:41 +0000129static inline void free_task_struct(struct task_struct *tsk)
130{
Thomas Gleixner41101802012-05-05 15:05:41 +0000131 kmem_cache_free(task_struct_cachep, tsk);
132}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133#endif
134
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700135void __weak arch_release_thread_info(struct thread_info *ti)
136{
137}
138
Thomas Gleixnerf5e10282012-05-05 15:05:48 +0000139#ifndef CONFIG_ARCH_THREAD_INFO_ALLOCATOR
Thomas Gleixner41101802012-05-05 15:05:41 +0000140
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000141/*
142 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
143 * kmemcache based allocator.
144 */
145# if THREAD_SIZE >= PAGE_SIZE
Eric Dumazetb6a84012011-03-22 16:30:42 -0700146static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
147 int node)
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700148{
Thomas Gleixner2889f602012-05-05 15:05:41 +0000149 struct page *page = alloc_pages_node(node, THREADINFO_GFP,
150 THREAD_SIZE_ORDER);
Eric Dumazetb6a84012011-03-22 16:30:42 -0700151
152 return page ? page_address(page) : NULL;
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700153}
154
155static inline void free_thread_info(struct thread_info *ti)
156{
157 free_pages((unsigned long)ti, THREAD_SIZE_ORDER);
158}
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000159# else
160static struct kmem_cache *thread_info_cache;
161
162static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
163 int node)
164{
165 return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node);
166}
167
168static void free_thread_info(struct thread_info *ti)
169{
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000170 kmem_cache_free(thread_info_cache, ti);
171}
172
173void thread_info_cache_init(void)
174{
175 thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE,
176 THREAD_SIZE, 0, NULL);
177 BUG_ON(thread_info_cache == NULL);
178}
179# endif
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700180#endif
181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182/* SLAB cache for signal_struct structures (tsk->signal) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800183static struct kmem_cache *signal_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
185/* SLAB cache for sighand_struct structures (tsk->sighand) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800186struct kmem_cache *sighand_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187
188/* SLAB cache for files_struct structures (tsk->files) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800189struct kmem_cache *files_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
191/* SLAB cache for fs_struct structures (tsk->fs) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800192struct kmem_cache *fs_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194/* SLAB cache for vm_area_struct structures */
Christoph Lametere18b8902006-12-06 20:33:20 -0800195struct kmem_cache *vm_area_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
197/* SLAB cache for mm_struct structures (tsk->mm) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800198static struct kmem_cache *mm_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700200static void account_kernel_stack(struct thread_info *ti, int account)
201{
202 struct zone *zone = page_zone(virt_to_page(ti));
203
204 mod_zone_page_state(zone, NR_KERNEL_STACK, account);
205}
206
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207void free_task(struct task_struct *tsk)
208{
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700209 account_kernel_stack(tsk->stack, -1);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700210 arch_release_thread_info(tsk->stack);
Roman Zippelf7e42172007-05-09 02:35:17 -0700211 free_thread_info(tsk->stack);
Ingo Molnar23f78d4a2006-06-27 02:54:53 -0700212 rt_mutex_debug_task_free(tsk);
Frederic Weisbeckerfb526072008-11-25 21:07:04 +0100213 ftrace_graph_exit_task(tsk);
Will Drewrye2cfabdf2012-04-12 16:47:57 -0500214 put_seccomp_filter(tsk);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700215 arch_release_task_struct(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 free_task_struct(tsk);
217}
218EXPORT_SYMBOL(free_task);
219
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700220static inline void free_signal_struct(struct signal_struct *sig)
221{
Oleg Nesterov97101eb2010-05-26 14:43:20 -0700222 taskstats_tgid_free(sig);
Mike Galbraith1c5354d2011-01-05 11:16:04 +0100223 sched_autogroup_exit(sig);
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700224 kmem_cache_free(signal_cachep, sig);
225}
226
227static inline void put_signal_struct(struct signal_struct *sig)
228{
Mike Galbraith1c5354d2011-01-05 11:16:04 +0100229 if (atomic_dec_and_test(&sig->sigcnt))
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700230 free_signal_struct(sig);
231}
232
Andrew Morton158d9eb2006-03-31 02:31:34 -0800233void __put_task_struct(struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234{
Eugene Teo270f7222007-10-18 23:40:38 -0700235 WARN_ON(!tsk->exit_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 WARN_ON(atomic_read(&tsk->usage));
237 WARN_ON(tsk == current);
238
Kees Cook1a2a4d02011-12-21 12:17:03 -0800239 security_task_free(tsk);
David Howellse0e81732009-09-02 09:13:40 +0100240 exit_creds(tsk);
Shailabh Nagar35df17c2006-08-31 21:27:38 -0700241 delayacct_tsk_free(tsk);
Oleg Nesterovea6d2902010-05-26 14:43:16 -0700242 put_signal_struct(tsk->signal);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244 if (!profile_handoff_task(tsk))
245 free_task(tsk);
246}
Rik van Riel77c100c2011-02-01 09:51:46 -0500247EXPORT_SYMBOL_GPL(__put_task_struct);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248
Thomas Gleixner6c0a9fa2012-05-05 15:05:40 +0000249void __init __weak arch_task_cache_init(void) { }
Suresh Siddha61c46282008-03-10 15:28:04 -0700250
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251void __init fork_init(unsigned long mempages)
252{
Thomas Gleixnerf5e10282012-05-05 15:05:48 +0000253#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254#ifndef ARCH_MIN_TASKALIGN
255#define ARCH_MIN_TASKALIGN L1_CACHE_BYTES
256#endif
257 /* create a slab on which task_structs can be allocated */
258 task_struct_cachep =
259 kmem_cache_create("task_struct", sizeof(struct task_struct),
Vegard Nossum2dff4402008-05-31 15:56:17 +0200260 ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261#endif
262
Suresh Siddha61c46282008-03-10 15:28:04 -0700263 /* do the arch specific task caches init */
264 arch_task_cache_init();
265
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 /*
267 * The default maximum number of threads is set to a safe
268 * value: the thread structures can take up at most half
269 * of memory.
270 */
271 max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE);
272
273 /*
274 * we need to allow at least 20 threads to boot a system
275 */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700276 if (max_threads < 20)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 max_threads = 20;
278
279 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
280 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
281 init_task.signal->rlim[RLIMIT_SIGPENDING] =
282 init_task.signal->rlim[RLIMIT_NPROC];
283}
284
Suresh Siddha61c46282008-03-10 15:28:04 -0700285int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst,
286 struct task_struct *src)
287{
288 *dst = *src;
289 return 0;
290}
291
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292static struct task_struct *dup_task_struct(struct task_struct *orig)
293{
294 struct task_struct *tsk;
295 struct thread_info *ti;
Eric Sandeen7c9f8862008-04-22 16:38:23 -0500296 unsigned long *stackend;
Eric Dumazet207205a2011-03-22 16:30:44 -0700297 int node = tsk_fork_get_node(orig);
Peter Zijlstra3e26c142007-10-16 23:25:50 -0700298 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
Eric Dumazet504f52b2011-03-22 16:30:41 -0700300 tsk = alloc_task_struct_node(node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 if (!tsk)
302 return NULL;
303
Eric Dumazetb6a84012011-03-22 16:30:42 -0700304 ti = alloc_thread_info_node(tsk, node);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700305 if (!ti)
306 goto free_tsk;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700308 err = arch_dup_task_struct(tsk, orig);
Suresh Siddha61c46282008-03-10 15:28:04 -0700309 if (err)
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700310 goto free_ti;
Suresh Siddha61c46282008-03-10 15:28:04 -0700311
Andrew Morton87bec582012-07-30 14:42:31 -0700312 tsk->stack = ti;
313
314 setup_thread_stack(tsk, orig);
Avi Kivity8e7cac72009-11-29 16:34:48 +0200315 clear_user_return_notifier(tsk);
Mike Galbraithf26f9af2010-12-08 11:05:42 +0100316 clear_tsk_need_resched(tsk);
Eric Sandeen7c9f8862008-04-22 16:38:23 -0500317 stackend = end_of_stack(tsk);
318 *stackend = STACK_END_MAGIC; /* for overflow detection */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
Arjan van de Ven0a4254052006-09-26 10:52:38 +0200320#ifdef CONFIG_CC_STACKPROTECTOR
321 tsk->stack_canary = get_random_int();
322#endif
323
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700324 /*
325 * One for us, one for whoever does the "release_task()" (usually
326 * parent)
327 */
328 atomic_set(&tsk->usage, 2);
Alexey Dobriyan6c5c9342006-09-29 01:59:40 -0700329#ifdef CONFIG_BLK_DEV_IO_TRACE
Jens Axboe2056a782006-03-23 20:00:26 +0100330 tsk->btrace_seq = 0;
Alexey Dobriyan6c5c9342006-09-29 01:59:40 -0700331#endif
Jens Axboea0aa7f62006-04-20 13:05:33 +0200332 tsk->splice_pipe = NULL;
Eric Dumazet5640f762012-09-23 23:04:42 +0000333 tsk->task_frag.page = NULL;
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700334
335 account_kernel_stack(ti, 1);
336
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 return tsk;
Suresh Siddha61c46282008-03-10 15:28:04 -0700338
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700339free_ti:
Suresh Siddha61c46282008-03-10 15:28:04 -0700340 free_thread_info(ti);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700341free_tsk:
Suresh Siddha61c46282008-03-10 15:28:04 -0700342 free_task_struct(tsk);
343 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344}
345
346#ifdef CONFIG_MMU
Alexey Dobriyana39bc512007-10-18 23:41:10 -0700347static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348{
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700349 struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 struct rb_node **rb_link, *rb_parent;
351 int retval;
352 unsigned long charge;
353 struct mempolicy *pol;
354
355 down_write(&oldmm->mmap_sem);
Ralf Baechleec8c04462006-12-12 17:14:57 +0000356 flush_cache_dup_mm(oldmm);
Oleg Nesterovf8ac4ec2012-08-08 17:11:42 +0200357 uprobe_dup_mmap(oldmm, mm);
Ingo Molnarad339452006-07-03 00:25:15 -0700358 /*
359 * Not linked in yet - no deadlock potential:
360 */
361 down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
Hugh Dickins7ee78232005-10-29 18:16:08 -0700362
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 mm->locked_vm = 0;
364 mm->mmap = NULL;
365 mm->mmap_cache = NULL;
366 mm->free_area_cache = oldmm->mmap_base;
Wolfgang Wander1363c3c2005-06-21 17:14:49 -0700367 mm->cached_hole_size = ~0UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368 mm->map_count = 0;
Rusty Russell94894242009-03-30 22:05:12 -0600369 cpumask_clear(mm_cpumask(mm));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 mm->mm_rb = RB_ROOT;
371 rb_link = &mm->mm_rb.rb_node;
372 rb_parent = NULL;
373 pprev = &mm->mmap;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700374 retval = ksm_fork(mm, oldmm);
375 if (retval)
376 goto out;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800377 retval = khugepaged_fork(mm, oldmm);
378 if (retval)
379 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700381 prev = NULL;
Hugh Dickinsfd3e42f2005-10-29 18:16:06 -0700382 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 struct file *file;
384
385 if (mpnt->vm_flags & VM_DONTCOPY) {
Hugh Dickinsab50b8e2005-10-29 18:15:56 -0700386 vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
Huang Shijie44de9d02012-07-31 16:41:49 -0700387 -vma_pages(mpnt));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 continue;
389 }
390 charge = 0;
391 if (mpnt->vm_flags & VM_ACCOUNT) {
Huang Shijieb2412b72012-07-30 14:42:30 -0700392 unsigned long len = vma_pages(mpnt);
393
Al Viro191c5422012-02-13 03:58:52 +0000394 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 goto fail_nomem;
396 charge = len;
397 }
Christoph Lametere94b1762006-12-06 20:33:17 -0800398 tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 if (!tmp)
400 goto fail_nomem;
401 *tmp = *mpnt;
Rik van Riel5beb4932010-03-05 13:42:07 -0800402 INIT_LIST_HEAD(&tmp->anon_vma_chain);
Lee Schermerhorn846a16b2008-04-28 02:13:09 -0700403 pol = mpol_dup(vma_policy(mpnt));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 retval = PTR_ERR(pol);
405 if (IS_ERR(pol))
406 goto fail_nomem_policy;
407 vma_set_policy(tmp, pol);
Andrea Arcangelia247c3a2010-09-22 13:05:12 -0700408 tmp->vm_mm = mm;
Rik van Riel5beb4932010-03-05 13:42:07 -0800409 if (anon_vma_fork(tmp, mpnt))
410 goto fail_nomem_anon_vma_fork;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 tmp->vm_flags &= ~VM_LOCKED;
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700412 tmp->vm_next = tmp->vm_prev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 file = tmp->vm_file;
414 if (file) {
Josef "Jeff" Sipekf3a43f32006-12-08 02:36:43 -0800415 struct inode *inode = file->f_path.dentry->d_inode;
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000416 struct address_space *mapping = file->f_mapping;
417
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 get_file(file);
419 if (tmp->vm_flags & VM_DENYWRITE)
420 atomic_dec(&inode->i_writecount);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -0700421 mutex_lock(&mapping->i_mmap_mutex);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000422 if (tmp->vm_flags & VM_SHARED)
423 mapping->i_mmap_writable++;
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000424 flush_dcache_mmap_lock(mapping);
425 /* insert tmp into the share list, just after mpnt */
Michel Lespinasse9826a512012-10-08 16:31:35 -0700426 if (unlikely(tmp->vm_flags & VM_NONLINEAR))
427 vma_nonlinear_insert(tmp,
428 &mapping->i_mmap_nonlinear);
429 else
430 vma_interval_tree_insert_after(tmp, mpnt,
431 &mapping->i_mmap);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000432 flush_dcache_mmap_unlock(mapping);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -0700433 mutex_unlock(&mapping->i_mmap_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 }
435
436 /*
Mel Gormana1e78772008-07-23 21:27:23 -0700437 * Clear hugetlb-related page reserves for children. This only
438 * affects MAP_PRIVATE mappings. Faults generated by the child
439 * are not guaranteed to succeed, even if read-only
440 */
441 if (is_vm_hugetlb_page(tmp))
442 reset_vma_resv_huge_pages(tmp);
443
444 /*
Hugh Dickins7ee78232005-10-29 18:16:08 -0700445 * Link in the new vma and copy the page table entries.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 *pprev = tmp;
448 pprev = &tmp->vm_next;
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700449 tmp->vm_prev = prev;
450 prev = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451
452 __vma_link_rb(mm, tmp, rb_link, rb_parent);
453 rb_link = &tmp->vm_rb.rb_right;
454 rb_parent = &tmp->vm_rb;
455
456 mm->map_count++;
Hugh Dickins0b0db142005-11-21 21:32:20 -0800457 retval = copy_page_range(mm, oldmm, mpnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458
459 if (tmp->vm_ops && tmp->vm_ops->open)
460 tmp->vm_ops->open(tmp);
461
462 if (retval)
463 goto out;
464 }
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200465 /* a new mm has just been created */
466 arch_dup_mmap(oldmm, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 retval = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468out:
Hugh Dickins7ee78232005-10-29 18:16:08 -0700469 up_write(&mm->mmap_sem);
Hugh Dickinsfd3e42f2005-10-29 18:16:06 -0700470 flush_tlb_mm(oldmm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 up_write(&oldmm->mmap_sem);
472 return retval;
Rik van Riel5beb4932010-03-05 13:42:07 -0800473fail_nomem_anon_vma_fork:
474 mpol_put(pol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475fail_nomem_policy:
476 kmem_cache_free(vm_area_cachep, tmp);
477fail_nomem:
478 retval = -ENOMEM;
479 vm_unacct_memory(charge);
480 goto out;
481}
482
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700483static inline int mm_alloc_pgd(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484{
485 mm->pgd = pgd_alloc(mm);
486 if (unlikely(!mm->pgd))
487 return -ENOMEM;
488 return 0;
489}
490
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700491static inline void mm_free_pgd(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492{
Benjamin Herrenschmidt5e541972008-02-04 22:29:14 -0800493 pgd_free(mm, mm->pgd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494}
495#else
496#define dup_mmap(mm, oldmm) (0)
497#define mm_alloc_pgd(mm) (0)
498#define mm_free_pgd(mm)
499#endif /* CONFIG_MMU */
500
Daniel Walker23ff4442007-10-18 03:06:07 -0700501__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
Christoph Lametere94b1762006-12-06 20:33:17 -0800503#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
505
Hidehiro Kawai4cb0e112009-01-06 14:42:47 -0800506static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
507
508static int __init coredump_filter_setup(char *s)
509{
510 default_dump_filter =
511 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
512 MMF_DUMP_FILTER_MASK;
513 return 1;
514}
515
516__setup("coredump_filter=", coredump_filter_setup);
517
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518#include <linux/init_task.h>
519
Alexey Dobriyan858f0992009-09-23 15:57:32 -0700520static void mm_init_aio(struct mm_struct *mm)
521{
522#ifdef CONFIG_AIO
523 spin_lock_init(&mm->ioctx_lock);
524 INIT_HLIST_HEAD(&mm->ioctx_list);
525#endif
526}
527
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700528static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529{
530 atomic_set(&mm->mm_users, 1);
531 atomic_set(&mm->mm_count, 1);
532 init_rwsem(&mm->mmap_sem);
533 INIT_LIST_HEAD(&mm->mmlist);
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700534 mm->flags = (current->mm) ?
535 (current->mm->flags & MMF_INIT_MASK) : default_dump_filter;
Oleg Nesterov999d9fc2008-07-25 01:47:41 -0700536 mm->core_state = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 mm->nr_ptes = 0;
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -0800538 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 spin_lock_init(&mm->page_table_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540 mm->free_area_cache = TASK_UNMAPPED_BASE;
Wolfgang Wander1363c3c2005-06-21 17:14:49 -0700541 mm->cached_hole_size = ~0UL;
Alexey Dobriyan858f0992009-09-23 15:57:32 -0700542 mm_init_aio(mm);
Balbir Singhcf475ad2008-04-29 01:00:16 -0700543 mm_init_owner(mm, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544
545 if (likely(!mm_alloc_pgd(mm))) {
546 mm->def_flags = 0;
Andrea Arcangelicddb8a52008-07-28 15:46:29 -0700547 mmu_notifier_mm_init(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 return mm;
549 }
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800550
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 free_mm(mm);
552 return NULL;
553}
554
Konstantin Khlebnikovc3f03272012-03-21 16:33:48 -0700555static void check_mm(struct mm_struct *mm)
556{
557 int i;
558
559 for (i = 0; i < NR_MM_COUNTERS; i++) {
560 long x = atomic_long_read(&mm->rss_stat.count[i]);
561
562 if (unlikely(x))
563 printk(KERN_ALERT "BUG: Bad rss-counter state "
564 "mm:%p idx:%d val:%ld\n", mm, i, x);
565 }
566
567#ifdef CONFIG_TRANSPARENT_HUGEPAGE
568 VM_BUG_ON(mm->pmd_huge_pte);
569#endif
570}
571
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572/*
573 * Allocate and initialize an mm_struct.
574 */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700575struct mm_struct *mm_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576{
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700577 struct mm_struct *mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578
579 mm = allocate_mm();
KOSAKI Motohirode03c722011-05-24 17:12:15 -0700580 if (!mm)
581 return NULL;
582
583 memset(mm, 0, sizeof(*mm));
Linus Torvalds6345d242011-05-29 11:32:28 -0700584 mm_init_cpumask(mm);
585 return mm_init(mm, current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586}
587
588/*
589 * Called when the last reference to the mm
590 * is dropped: either by a lazy thread or by
591 * mmput. Free the page directory and the mm.
592 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800593void __mmdrop(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594{
595 BUG_ON(mm == &init_mm);
596 mm_free_pgd(mm);
597 destroy_context(mm);
Andrea Arcangelicddb8a52008-07-28 15:46:29 -0700598 mmu_notifier_mm_destroy(mm);
Konstantin Khlebnikovc3f03272012-03-21 16:33:48 -0700599 check_mm(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 free_mm(mm);
601}
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200602EXPORT_SYMBOL_GPL(__mmdrop);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603
604/*
605 * Decrement the use count and release all resources for an mm.
606 */
607void mmput(struct mm_struct *mm)
608{
Andrew Morton0ae26f12006-06-23 02:05:15 -0700609 might_sleep();
610
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 if (atomic_dec_and_test(&mm->mm_users)) {
Srikar Dronamrajud4b3b6382012-03-30 23:56:31 +0530612 uprobe_clear_state(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 exit_aio(mm);
Andrea Arcangeli1c2fb7a2009-09-21 17:02:22 -0700614 ksm_exit(mm);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800615 khugepaged_exit(mm); /* must run before exit_mmap */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 exit_mmap(mm);
Matt Helsley925d1c42008-04-29 01:01:36 -0700617 set_mm_exe_file(mm, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 if (!list_empty(&mm->mmlist)) {
619 spin_lock(&mmlist_lock);
620 list_del(&mm->mmlist);
621 spin_unlock(&mmlist_lock);
622 }
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -0700623 if (mm->binfmt)
624 module_put(mm->binfmt->module);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 mmdrop(mm);
626 }
627}
628EXPORT_SYMBOL_GPL(mmput);
629
Jiri Slaby38646012011-05-26 16:25:46 -0700630void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
631{
632 if (new_exe_file)
633 get_file(new_exe_file);
634 if (mm->exe_file)
635 fput(mm->exe_file);
636 mm->exe_file = new_exe_file;
Jiri Slaby38646012011-05-26 16:25:46 -0700637}
638
639struct file *get_mm_exe_file(struct mm_struct *mm)
640{
641 struct file *exe_file;
642
Konstantin Khlebnikov2dd8ad82012-10-08 16:28:51 -0700643 /* We need mmap_sem to protect against races with removal of exe_file */
Jiri Slaby38646012011-05-26 16:25:46 -0700644 down_read(&mm->mmap_sem);
645 exe_file = mm->exe_file;
646 if (exe_file)
647 get_file(exe_file);
648 up_read(&mm->mmap_sem);
649 return exe_file;
650}
651
652static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm)
653{
654 /* It's safe to write the exe_file pointer without exe_file_lock because
655 * this is called during fork when the task is not yet in /proc */
656 newmm->exe_file = get_mm_exe_file(oldmm);
657}
658
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659/**
660 * get_task_mm - acquire a reference to the task's mm
661 *
Oleg Nesterov246bb0b2008-07-25 01:47:38 -0700662 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 * this kernel workthread has transiently adopted a user mm with use_mm,
664 * to do its AIO) is not set and if so returns a reference to it, after
665 * bumping up the use count. User must release the mm via mmput()
666 * after use. Typically used by /proc and ptrace.
667 */
668struct mm_struct *get_task_mm(struct task_struct *task)
669{
670 struct mm_struct *mm;
671
672 task_lock(task);
673 mm = task->mm;
674 if (mm) {
Oleg Nesterov246bb0b2008-07-25 01:47:38 -0700675 if (task->flags & PF_KTHREAD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 mm = NULL;
677 else
678 atomic_inc(&mm->mm_users);
679 }
680 task_unlock(task);
681 return mm;
682}
683EXPORT_SYMBOL_GPL(get_task_mm);
684
Christopher Yeoh8cdb8782012-02-02 11:34:09 +1030685struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
686{
687 struct mm_struct *mm;
688 int err;
689
690 err = mutex_lock_killable(&task->signal->cred_guard_mutex);
691 if (err)
692 return ERR_PTR(err);
693
694 mm = get_task_mm(task);
695 if (mm && mm != current->mm &&
696 !ptrace_may_access(task, mode)) {
697 mmput(mm);
698 mm = ERR_PTR(-EACCES);
699 }
700 mutex_unlock(&task->signal->cred_guard_mutex);
701
702 return mm;
703}
704
Oleg Nesterov57b59c42012-03-05 14:59:13 -0800705static void complete_vfork_done(struct task_struct *tsk)
Oleg Nesterovc415c3b2012-03-05 14:59:13 -0800706{
Oleg Nesterovd68b46f2012-03-05 14:59:13 -0800707 struct completion *vfork;
Oleg Nesterovc415c3b2012-03-05 14:59:13 -0800708
Oleg Nesterovd68b46f2012-03-05 14:59:13 -0800709 task_lock(tsk);
710 vfork = tsk->vfork_done;
711 if (likely(vfork)) {
712 tsk->vfork_done = NULL;
713 complete(vfork);
714 }
715 task_unlock(tsk);
716}
717
718static int wait_for_vfork_done(struct task_struct *child,
719 struct completion *vfork)
720{
721 int killed;
722
723 freezer_do_not_count();
724 killed = wait_for_completion_killable(vfork);
725 freezer_count();
726
727 if (killed) {
728 task_lock(child);
729 child->vfork_done = NULL;
730 task_unlock(child);
731 }
732
733 put_task_struct(child);
734 return killed;
Oleg Nesterovc415c3b2012-03-05 14:59:13 -0800735}
736
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737/* Please note the differences between mmput and mm_release.
738 * mmput is called whenever we stop holding onto a mm_struct,
739 * error success whatever.
740 *
741 * mm_release is called after a mm_struct has been removed
742 * from the current process.
743 *
744 * This difference is important for error handling, when we
745 * only half set up a mm_struct for a new process and need to restore
746 * the old one. Because we mmput the new mm_struct before
747 * restoring the old one. . .
748 * Eric Biederman 10 January 1998
749 */
750void mm_release(struct task_struct *tsk, struct mm_struct *mm)
751{
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800752 /* Get rid of any futexes when releasing the mm */
753#ifdef CONFIG_FUTEX
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200754 if (unlikely(tsk->robust_list)) {
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800755 exit_robust_list(tsk);
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200756 tsk->robust_list = NULL;
757 }
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800758#ifdef CONFIG_COMPAT
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200759 if (unlikely(tsk->compat_robust_list)) {
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800760 compat_exit_robust_list(tsk);
Peter Zijlstrafc6b1772009-10-05 18:17:32 +0200761 tsk->compat_robust_list = NULL;
762 }
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800763#endif
Thomas Gleixner322a2c12009-10-05 18:18:03 +0200764 if (unlikely(!list_empty(&tsk->pi_state_list)))
765 exit_pi_state_list(tsk);
Linus Torvalds8141c7f2008-11-15 10:20:36 -0800766#endif
767
Srikar Dronamraju0326f5a92012-03-13 23:30:11 +0530768 uprobe_free_utask(tsk);
769
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 /* Get rid of any cached register state */
771 deactivate_mm(tsk, mm);
772
Roland McGrathfec1d012006-12-06 20:36:34 -0800773 /*
774 * If we're exiting normally, clear a user-space tid field if
775 * requested. We leave this alone when dying by signal, to leave
776 * the value intact in a core dump, and to save the unnecessary
Oleg Nesterovd68b46f2012-03-05 14:59:13 -0800777 * trouble, say, a killed vfork parent shouldn't touch this mm.
778 * Userland only wants this done for a sys_exit.
Roland McGrathfec1d012006-12-06 20:36:34 -0800779 */
Eric Dumazet9c8a8222009-08-06 15:09:28 -0700780 if (tsk->clear_child_tid) {
781 if (!(tsk->flags & PF_SIGNALED) &&
782 atomic_read(&mm->mm_users) > 1) {
783 /*
784 * We don't check the error code - if userspace has
785 * not set up a proper pointer then tough luck.
786 */
787 put_user(0, tsk->clear_child_tid);
788 sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
789 1, NULL, NULL, 0);
790 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 tsk->clear_child_tid = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 }
Konstantin Khlebnikovf7505d642012-05-31 16:26:21 -0700793
794 /*
795 * All done, finally we can wake up parent and return this mm to him.
796 * Also kthread_stop() uses this completion for synchronization.
797 */
798 if (tsk->vfork_done)
799 complete_vfork_done(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800}
801
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800802/*
803 * Allocate a new mm structure and copy contents from the
804 * mm structure of the passed in task structure.
805 */
Carsten Otte402b0862008-03-25 18:47:10 +0100806struct mm_struct *dup_mm(struct task_struct *tsk)
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800807{
808 struct mm_struct *mm, *oldmm = current->mm;
809 int err;
810
811 if (!oldmm)
812 return NULL;
813
814 mm = allocate_mm();
815 if (!mm)
816 goto fail_nomem;
817
818 memcpy(mm, oldmm, sizeof(*mm));
Linus Torvalds6345d242011-05-29 11:32:28 -0700819 mm_init_cpumask(mm);
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800820
Andrea Arcangelie7a00c42011-01-13 15:46:45 -0800821#ifdef CONFIG_TRANSPARENT_HUGEPAGE
822 mm->pmd_huge_pte = NULL;
823#endif
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800824 if (!mm_init(mm, tsk))
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800825 goto fail_nomem;
826
827 if (init_new_context(tsk, mm))
828 goto fail_nocontext;
829
Matt Helsley925d1c42008-04-29 01:01:36 -0700830 dup_mm_exe_file(oldmm, mm);
831
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800832 err = dup_mmap(mm, oldmm);
833 if (err)
834 goto free_pt;
835
836 mm->hiwater_rss = get_mm_rss(mm);
837 mm->hiwater_vm = mm->total_vm;
838
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -0700839 if (mm->binfmt && !try_module_get(mm->binfmt->module))
840 goto free_pt;
841
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800842 return mm;
843
844free_pt:
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -0700845 /* don't put binfmt in mmput, we haven't got module yet */
846 mm->binfmt = NULL;
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800847 mmput(mm);
848
849fail_nomem:
850 return NULL;
851
852fail_nocontext:
853 /*
854 * If init_new_context() failed, we cannot use mmput() to free the mm
855 * because it calls destroy_context()
856 */
857 mm_free_pgd(mm);
858 free_mm(mm);
859 return NULL;
860}
861
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700862static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863{
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700864 struct mm_struct *mm, *oldmm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865 int retval;
866
867 tsk->min_flt = tsk->maj_flt = 0;
868 tsk->nvcsw = tsk->nivcsw = 0;
Mandeep Singh Baines17406b82009-02-06 15:37:47 -0800869#ifdef CONFIG_DETECT_HUNG_TASK
870 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
871#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872
873 tsk->mm = NULL;
874 tsk->active_mm = NULL;
875
876 /*
877 * Are we cloning a kernel thread?
878 *
879 * We need to steal a active VM for that..
880 */
881 oldmm = current->mm;
882 if (!oldmm)
883 return 0;
884
885 if (clone_flags & CLONE_VM) {
886 atomic_inc(&oldmm->mm_users);
887 mm = oldmm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888 goto good_mm;
889 }
890
891 retval = -ENOMEM;
JANAK DESAIa0a7ec32006-02-07 12:59:01 -0800892 mm = dup_mm(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893 if (!mm)
894 goto fail_nomem;
895
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896good_mm:
897 tsk->mm = mm;
898 tsk->active_mm = mm;
899 return 0;
900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901fail_nomem:
902 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903}
904
Alexey Dobriyana39bc512007-10-18 23:41:10 -0700905static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906{
Al Viro498052b2009-03-30 07:20:30 -0400907 struct fs_struct *fs = current->fs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 if (clone_flags & CLONE_FS) {
Al Viro498052b2009-03-30 07:20:30 -0400909 /* tsk->fs is already what we want */
Nick Piggin2a4419b2010-08-18 04:37:33 +1000910 spin_lock(&fs->lock);
Al Viro498052b2009-03-30 07:20:30 -0400911 if (fs->in_exec) {
Nick Piggin2a4419b2010-08-18 04:37:33 +1000912 spin_unlock(&fs->lock);
Al Viro498052b2009-03-30 07:20:30 -0400913 return -EAGAIN;
914 }
915 fs->users++;
Nick Piggin2a4419b2010-08-18 04:37:33 +1000916 spin_unlock(&fs->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 return 0;
918 }
Al Viro498052b2009-03-30 07:20:30 -0400919 tsk->fs = copy_fs_struct(fs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 if (!tsk->fs)
921 return -ENOMEM;
922 return 0;
923}
924
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700925static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
JANAK DESAIa016f332006-02-07 12:59:02 -0800926{
927 struct files_struct *oldf, *newf;
928 int error = 0;
929
930 /*
931 * A background process may not have any files ...
932 */
933 oldf = current->files;
934 if (!oldf)
935 goto out;
936
937 if (clone_flags & CLONE_FILES) {
938 atomic_inc(&oldf->count);
939 goto out;
940 }
941
JANAK DESAIa016f332006-02-07 12:59:02 -0800942 newf = dup_fd(oldf, &error);
943 if (!newf)
944 goto out;
945
946 tsk->files = newf;
947 error = 0;
948out:
949 return error;
950}
951
Jens Axboefadad8782008-01-24 08:54:47 +0100952static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
Jens Axboefd0928d2008-01-24 08:52:45 +0100953{
954#ifdef CONFIG_BLOCK
955 struct io_context *ioc = current->io_context;
Tejun Heo6e736be2011-12-14 00:33:38 +0100956 struct io_context *new_ioc;
Jens Axboefd0928d2008-01-24 08:52:45 +0100957
958 if (!ioc)
959 return 0;
Jens Axboefadad8782008-01-24 08:54:47 +0100960 /*
961 * Share io context with parent, if CLONE_IO is set
962 */
963 if (clone_flags & CLONE_IO) {
Tejun Heo3d487492012-03-05 13:15:25 -0800964 ioc_task_link(ioc);
965 tsk->io_context = ioc;
Jens Axboefadad8782008-01-24 08:54:47 +0100966 } else if (ioprio_valid(ioc->ioprio)) {
Tejun Heo6e736be2011-12-14 00:33:38 +0100967 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
968 if (unlikely(!new_ioc))
Jens Axboefd0928d2008-01-24 08:52:45 +0100969 return -ENOMEM;
970
Tejun Heo6e736be2011-12-14 00:33:38 +0100971 new_ioc->ioprio = ioc->ioprio;
Tejun Heo11a31222012-02-07 07:51:30 +0100972 put_io_context(new_ioc);
Jens Axboefd0928d2008-01-24 08:52:45 +0100973 }
974#endif
975 return 0;
976}
977
Alexey Dobriyana39bc512007-10-18 23:41:10 -0700978static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979{
980 struct sighand_struct *sig;
981
Zhaolei60348802009-01-06 14:40:46 -0800982 if (clone_flags & CLONE_SIGHAND) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 atomic_inc(&current->sighand->count);
984 return 0;
985 }
986 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
Ingo Molnare56d0902006-01-08 01:01:37 -0800987 rcu_assign_pointer(tsk->sighand, sig);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 if (!sig)
989 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 atomic_set(&sig->count, 1);
991 memcpy(sig->action, current->sighand->action, sizeof(sig->action));
992 return 0;
993}
994
Oleg Nesterova7e53282006-03-28 16:11:27 -0800995void __cleanup_sighand(struct sighand_struct *sighand)
Oleg Nesterovc81addc2006-03-28 16:11:17 -0800996{
Oleg Nesterovd80e7312012-02-24 20:07:11 +0100997 if (atomic_dec_and_test(&sighand->count)) {
998 signalfd_cleanup(sighand);
Oleg Nesterovc81addc2006-03-28 16:11:17 -0800999 kmem_cache_free(sighand_cachep, sighand);
Oleg Nesterovd80e7312012-02-24 20:07:11 +01001000 }
Oleg Nesterovc81addc2006-03-28 16:11:17 -08001001}
1002
Frank Mayharf06febc2008-09-12 09:54:39 -07001003
1004/*
1005 * Initialize POSIX timer handling for a thread group.
1006 */
1007static void posix_cpu_timers_init_group(struct signal_struct *sig)
1008{
Jiri Slaby78d7d402010-03-05 13:42:54 -08001009 unsigned long cpu_limit;
1010
Frank Mayharf06febc2008-09-12 09:54:39 -07001011 /* Thread group counters. */
1012 thread_group_cputime_init(sig);
1013
Jiri Slaby78d7d402010-03-05 13:42:54 -08001014 cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1015 if (cpu_limit != RLIM_INFINITY) {
1016 sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001017 sig->cputimer.running = 1;
1018 }
1019
Frank Mayharf06febc2008-09-12 09:54:39 -07001020 /* The timer lists. */
1021 INIT_LIST_HEAD(&sig->cpu_timers[0]);
1022 INIT_LIST_HEAD(&sig->cpu_timers[1]);
1023 INIT_LIST_HEAD(&sig->cpu_timers[2]);
1024}
1025
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001026static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027{
1028 struct signal_struct *sig;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07001030 if (clone_flags & CLONE_THREAD)
Peter Zijlstra490dea42008-11-24 17:06:57 +01001031 return 0;
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001032
Veaceslav Falicoa56704e2010-03-10 15:23:01 -08001033 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034 tsk->signal = sig;
1035 if (!sig)
1036 return -ENOMEM;
1037
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001038 sig->nr_threads = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 atomic_set(&sig->live, 1);
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001040 atomic_set(&sig->sigcnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 init_waitqueue_head(&sig->wait_chldexit);
Sukadev Bhattiprolub3bfa0c2009-04-02 16:58:08 -07001042 if (clone_flags & CLONE_NEWPID)
1043 sig->flags |= SIGNAL_UNKILLABLE;
Oleg Nesterovdb51aec2008-04-30 00:52:52 -07001044 sig->curr_target = tsk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 init_sigpending(&sig->shared_pending);
1046 INIT_LIST_HEAD(&sig->posix_timers);
1047
Thomas Gleixnerc9cb2e32007-02-16 01:27:49 -08001048 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 sig->real_timer.function = it_real_fn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051 task_lock(current->group_leader);
1052 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1053 task_unlock(current->group_leader);
1054
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001055 posix_cpu_timers_init_group(sig);
1056
Miloslav Trmac522ed772007-07-15 23:40:56 -07001057 tty_audit_fork(sig);
Mike Galbraith5091faa2010-11-30 14:18:03 +01001058 sched_autogroup_fork(sig);
Miloslav Trmac522ed772007-07-15 23:40:56 -07001059
Ben Blum4714d1d2011-05-26 16:25:18 -07001060#ifdef CONFIG_CGROUPS
Tejun Heo257058a2011-12-12 18:12:21 -08001061 init_rwsem(&sig->group_rwsem);
Ben Blum4714d1d2011-05-26 16:25:18 -07001062#endif
1063
David Rientjesa63d83f2010-08-09 17:19:46 -07001064 sig->oom_score_adj = current->signal->oom_score_adj;
Mandeep Singh Bainesdabb16f632011-01-13 15:46:05 -08001065 sig->oom_score_adj_min = current->signal->oom_score_adj_min;
KOSAKI Motohiro28b83c52009-09-21 17:03:13 -07001066
Lennart Poetteringebec18a2012-03-23 15:01:54 -07001067 sig->has_child_subreaper = current->signal->has_child_subreaper ||
1068 current->signal->is_child_subreaper;
1069
KOSAKI Motohiro9b1bf122010-10-27 15:34:08 -07001070 mutex_init(&sig->cred_guard_mutex);
1071
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072 return 0;
1073}
1074
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001075static void copy_flags(unsigned long clone_flags, struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076{
1077 unsigned long new_flags = p->flags;
1078
Tejun Heo21aa9af2010-06-08 21:40:37 +02001079 new_flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080 new_flags |= PF_FORKNOEXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 p->flags = new_flags;
1082}
1083
Heiko Carstens17da2bd2009-01-14 14:14:10 +01001084SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085{
1086 current->clear_child_tid = tidptr;
1087
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001088 return task_pid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089}
1090
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001091static void rt_mutex_init_task(struct task_struct *p)
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001092{
Thomas Gleixner1d615482009-11-17 14:54:03 +01001093 raw_spin_lock_init(&p->pi_lock);
Zilvinas Valinskase29e1752007-03-16 13:38:34 -08001094#ifdef CONFIG_RT_MUTEXES
Dima Zavin732375c2011-07-07 17:27:59 -07001095 plist_head_init(&p->pi_waiters);
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001096 p->pi_blocked_on = NULL;
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001097#endif
1098}
1099
Balbir Singhcf475ad2008-04-29 01:00:16 -07001100#ifdef CONFIG_MM_OWNER
1101void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
1102{
1103 mm->owner = p;
1104}
1105#endif /* CONFIG_MM_OWNER */
1106
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107/*
Frank Mayharf06febc2008-09-12 09:54:39 -07001108 * Initialize POSIX timer handling for a single task.
1109 */
1110static void posix_cpu_timers_init(struct task_struct *tsk)
1111{
Martin Schwidefsky64861632011-12-15 14:56:09 +01001112 tsk->cputime_expires.prof_exp = 0;
1113 tsk->cputime_expires.virt_exp = 0;
Frank Mayharf06febc2008-09-12 09:54:39 -07001114 tsk->cputime_expires.sched_exp = 0;
1115 INIT_LIST_HEAD(&tsk->cpu_timers[0]);
1116 INIT_LIST_HEAD(&tsk->cpu_timers[1]);
1117 INIT_LIST_HEAD(&tsk->cpu_timers[2]);
1118}
1119
1120/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 * This creates a new process as a copy of the old one,
1122 * but does not actually start it yet.
1123 *
1124 * It copies the registers, and all the appropriate
1125 * parts of the process environment (as per the clone
1126 * flags). The actual kick-off is left to the caller.
1127 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07001128static struct task_struct *copy_process(unsigned long clone_flags,
1129 unsigned long stack_start,
1130 struct pt_regs *regs,
1131 unsigned long stack_size,
Ingo Molnar36c8b582006-07-03 00:25:41 -07001132 int __user *child_tidptr,
Roland McGrath09a05392008-07-25 19:45:47 -07001133 struct pid *pid,
1134 int trace)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135{
1136 int retval;
Mariusz Kozlowskia24efe62007-10-18 23:41:09 -07001137 struct task_struct *p;
Paul Menageb4f48b62007-10-18 23:39:33 -07001138 int cgroup_callbacks_done = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139
1140 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
1141 return ERR_PTR(-EINVAL);
1142
1143 /*
1144 * Thread groups must share signals as well, and detached threads
1145 * can only be started up within the thread group.
1146 */
1147 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
1148 return ERR_PTR(-EINVAL);
1149
1150 /*
1151 * Shared signal handlers imply shared VM. By way of the above,
1152 * thread groups also imply shared VM. Blocking this case allows
1153 * for various simplifications in other code.
1154 */
1155 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
1156 return ERR_PTR(-EINVAL);
1157
Sukadev Bhattiprolu123be072009-09-23 15:57:20 -07001158 /*
1159 * Siblings of global init remain as zombies on exit since they are
1160 * not reaped by their parent (swapper). To solve this and to avoid
1161 * multi-rooted process trees, prevent global and container-inits
1162 * from creating siblings.
1163 */
1164 if ((clone_flags & CLONE_PARENT) &&
1165 current->signal->flags & SIGNAL_UNKILLABLE)
1166 return ERR_PTR(-EINVAL);
1167
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168 retval = security_task_create(clone_flags);
1169 if (retval)
1170 goto fork_out;
1171
1172 retval = -ENOMEM;
1173 p = dup_task_struct(current);
1174 if (!p)
1175 goto fork_out;
1176
Steven Rostedtf7e8b612009-06-02 16:39:48 -04001177 ftrace_graph_init_task(p);
Will Drewrye2cfabdf2012-04-12 16:47:57 -05001178 get_seccomp_filter(p);
Steven Rostedtf7e8b612009-06-02 16:39:48 -04001179
Peter Zijlstrabea493a2006-10-17 00:10:33 -07001180 rt_mutex_init_task(p);
1181
Ingo Molnard12c1a32008-07-14 12:09:28 +02001182#ifdef CONFIG_PROVE_LOCKING
Ingo Molnarde30a2b2006-07-03 00:24:42 -07001183 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1184 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1185#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186 retval = -EAGAIN;
David Howells3b11a1d2008-11-14 10:39:26 +11001187 if (atomic_read(&p->real_cred->user->processes) >=
Jiri Slaby78d7d402010-03-05 13:42:54 -08001188 task_rlimit(p, RLIMIT_NPROC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
Serge Hallyn18b6e042008-10-15 16:38:45 -05001190 p->real_cred->user != INIT_USER)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191 goto bad_fork_free;
1192 }
Vasiliy Kulikov72fa5992011-08-08 19:02:04 +04001193 current->flags &= ~PF_NPROC_EXCEEDED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194
David Howellsf1752ee2008-11-14 10:39:17 +11001195 retval = copy_creds(p, clone_flags);
1196 if (retval < 0)
1197 goto bad_fork_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198
1199 /*
1200 * If multiple threads are within copy_process(), then this check
1201 * triggers too late. This doesn't hurt, the check is only there
1202 * to stop root fork bombs.
1203 */
Li Zefan04ec93f2009-02-06 08:17:19 +00001204 retval = -EAGAIN;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205 if (nr_threads >= max_threads)
1206 goto bad_fork_cleanup_count;
1207
Al Viroa1261f542005-11-13 16:06:55 -08001208 if (!try_module_get(task_thread_info(p)->exec_domain->module))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209 goto bad_fork_cleanup_count;
1210
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 p->did_exec = 0;
Shailabh Nagarca74e922006-07-14 00:24:36 -07001212 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 copy_flags(clone_flags, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 INIT_LIST_HEAD(&p->children);
1215 INIT_LIST_HEAD(&p->sibling);
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07001216 rcu_copy_process(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217 p->vfork_done = NULL;
1218 spin_lock_init(&p->alloc_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 init_sigpending(&p->pending);
1221
Martin Schwidefsky64861632011-12-15 14:56:09 +01001222 p->utime = p->stime = p->gtime = 0;
1223 p->utimescaled = p->stimescaled = 0;
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09001224#ifndef CONFIG_VIRT_CPU_ACCOUNTING
Martin Schwidefsky64861632011-12-15 14:56:09 +01001225 p->prev_utime = p->prev_stime = 0;
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09001226#endif
KAMEZAWA Hiroyukia3a2e762010-04-06 14:34:42 -07001227#if defined(SPLIT_RSS_COUNTING)
1228 memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1229#endif
Balbir Singh172ba842007-07-09 18:52:00 +02001230
Arjan van de Ven69766752008-09-01 15:52:40 -07001231 p->default_timer_slack_ns = current->timer_slack_ns;
1232
Andrea Righi59954772008-07-27 17:29:15 +02001233 task_io_accounting_init(&p->ioac);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234 acct_clear_integrals(p);
1235
Frank Mayharf06febc2008-09-12 09:54:39 -07001236 posix_cpu_timers_init(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238 do_posix_clock_monotonic_gettime(&p->start_time);
Tomas Janousek924b42d2007-07-15 23:39:42 -07001239 p->real_start_time = p->start_time;
1240 monotonic_to_bootbased(&p->real_start_time);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 p->io_context = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 p->audit_context = NULL;
Ben Blum4714d1d2011-05-26 16:25:18 -07001243 if (clone_flags & CLONE_THREAD)
Tejun Heo257058a2011-12-12 18:12:21 -08001244 threadgroup_change_begin(current);
Paul Menageb4f48b62007-10-18 23:39:33 -07001245 cgroup_fork(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246#ifdef CONFIG_NUMA
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07001247 p->mempolicy = mpol_dup(p->mempolicy);
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001248 if (IS_ERR(p->mempolicy)) {
1249 retval = PTR_ERR(p->mempolicy);
1250 p->mempolicy = NULL;
1251 goto bad_fork_cleanup_cgroup;
1252 }
Paul Jacksonc61afb12006-03-24 03:16:08 -08001253 mpol_fix_fork_child_flag(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254#endif
Michal Hocko778d3b02011-07-26 16:08:30 -07001255#ifdef CONFIG_CPUSETS
1256 p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
1257 p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
Mel Gormancc9a6c82012-03-21 16:34:11 -07001258 seqcount_init(&p->mems_allowed_seq);
Michal Hocko778d3b02011-07-26 16:08:30 -07001259#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -07001260#ifdef CONFIG_TRACE_IRQFLAGS
1261 p->irq_events = 0;
1262 p->hardirqs_enabled = 0;
1263 p->hardirq_enable_ip = 0;
1264 p->hardirq_enable_event = 0;
1265 p->hardirq_disable_ip = _THIS_IP_;
1266 p->hardirq_disable_event = 0;
1267 p->softirqs_enabled = 1;
1268 p->softirq_enable_ip = _THIS_IP_;
1269 p->softirq_enable_event = 0;
1270 p->softirq_disable_ip = 0;
1271 p->softirq_disable_event = 0;
1272 p->hardirq_context = 0;
1273 p->softirq_context = 0;
1274#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07001275#ifdef CONFIG_LOCKDEP
1276 p->lockdep_depth = 0; /* no locks held yet */
1277 p->curr_chain_key = 0;
1278 p->lockdep_recursion = 0;
1279#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280
Ingo Molnar408894e2006-01-09 15:59:20 -08001281#ifdef CONFIG_DEBUG_MUTEXES
1282 p->blocked_on = NULL; /* not blocked yet */
1283#endif
Andrew Mortonc255a452012-07-31 16:43:02 -07001284#ifdef CONFIG_MEMCG
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08001285 p->memcg_batch.do_batch = 0;
1286 p->memcg_batch.memcg = NULL;
1287#endif
Markus Metzger0f481402009-04-03 16:43:48 +02001288
Srivatsa Vaddagiri3c90e6e2007-11-09 22:39:39 +01001289 /* Perform scheduler related setup. Assign this task to a CPU. */
Samir Bellabes3e51e3e2011-05-11 18:18:05 +02001290 sched_fork(p);
Peter Zijlstra6ab423e2009-05-25 14:45:27 +02001291
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001292 retval = perf_event_init_task(p);
Peter Zijlstra6ab423e2009-05-25 14:45:27 +02001293 if (retval)
1294 goto bad_fork_cleanup_policy;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001295 retval = audit_alloc(p);
1296 if (retval)
David Howellsf1752ee2008-11-14 10:39:17 +11001297 goto bad_fork_cleanup_policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001298 /* copy all the process information */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001299 retval = copy_semundo(clone_flags, p);
1300 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301 goto bad_fork_cleanup_audit;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001302 retval = copy_files(clone_flags, p);
1303 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304 goto bad_fork_cleanup_semundo;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001305 retval = copy_fs(clone_flags, p);
1306 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307 goto bad_fork_cleanup_files;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001308 retval = copy_sighand(clone_flags, p);
1309 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 goto bad_fork_cleanup_fs;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001311 retval = copy_signal(clone_flags, p);
1312 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 goto bad_fork_cleanup_sighand;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001314 retval = copy_mm(clone_flags, p);
1315 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316 goto bad_fork_cleanup_signal;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001317 retval = copy_namespaces(clone_flags, p);
1318 if (retval)
David Howellsd84f4f92008-11-14 10:39:23 +11001319 goto bad_fork_cleanup_mm;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001320 retval = copy_io(clone_flags, p);
1321 if (retval)
Jens Axboefd0928d2008-01-24 08:52:45 +01001322 goto bad_fork_cleanup_namespaces;
Alexey Dobriyan6f2c55b2009-04-02 16:56:59 -07001323 retval = copy_thread(clone_flags, stack_start, stack_size, p, regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 if (retval)
Jens Axboefd0928d2008-01-24 08:52:45 +01001325 goto bad_fork_cleanup_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001327 if (pid != &init_struct_pid) {
1328 retval = -ENOMEM;
Eric W. Biederman61bce0f2009-01-07 18:08:49 -08001329 pid = alloc_pid(p->nsproxy->pid_ns);
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001330 if (!pid)
Jens Axboefd0928d2008-01-24 08:52:45 +01001331 goto bad_fork_cleanup_io;
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001332 }
1333
1334 p->pid = pid_nr(pid);
1335 p->tgid = p->pid;
1336 if (clone_flags & CLONE_THREAD)
1337 p->tgid = current->tgid;
1338
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
1340 /*
1341 * Clear TID on mm_release()?
1342 */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001343 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
Jens Axboe73c10102011-03-08 13:19:51 +01001344#ifdef CONFIG_BLOCK
1345 p->plug = NULL;
1346#endif
Alexey Dobriyan42b2dd02007-10-16 23:27:30 -07001347#ifdef CONFIG_FUTEX
Ingo Molnar8f17d3a2006-03-27 01:16:27 -08001348 p->robust_list = NULL;
1349#ifdef CONFIG_COMPAT
1350 p->compat_robust_list = NULL;
1351#endif
Ingo Molnarc87e2832006-06-27 02:54:58 -07001352 INIT_LIST_HEAD(&p->pi_state_list);
1353 p->pi_state_cache = NULL;
Alexey Dobriyan42b2dd02007-10-16 23:27:30 -07001354#endif
Srikar Dronamraju0326f5a92012-03-13 23:30:11 +05301355 uprobe_copy_process(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001356 /*
GOTO Masanorif9a38792006-03-13 21:20:44 -08001357 * sigaltstack should be cleared when sharing the same VM
1358 */
1359 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
1360 p->sas_ss_sp = p->sas_ss_size = 0;
1361
1362 /*
Oleg Nesterov65808072009-12-15 16:47:16 -08001363 * Syscall tracing and stepping should be turned off in the
1364 * child regardless of CLONE_PTRACE.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 */
Oleg Nesterov65808072009-12-15 16:47:16 -08001366 user_disable_single_step(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
Laurent Viviered75e8d2005-09-03 15:57:18 -07001368#ifdef TIF_SYSCALL_EMU
1369 clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
1370#endif
Arjan van de Ven97455122008-01-25 21:08:34 +01001371 clear_all_latency_tracing(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001372
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 /* ok, now we should be set up.. */
Oleg Nesterov5f8aadd2012-03-14 19:55:38 +01001374 if (clone_flags & CLONE_THREAD)
1375 p->exit_signal = -1;
1376 else if (clone_flags & CLONE_PARENT)
1377 p->exit_signal = current->group_leader->exit_signal;
1378 else
1379 p->exit_signal = (clone_flags & CSIGNAL);
1380
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381 p->pdeath_signal = 0;
1382 p->exit_state = 0;
1383
Wu Fengguang9d823e82011-06-11 18:10:12 -06001384 p->nr_dirtied = 0;
1385 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
Wu Fengguang83712352011-06-11 19:25:42 -06001386 p->dirty_paused_when = 0;
Wu Fengguang9d823e82011-06-11 18:10:12 -06001387
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388 /*
1389 * Ok, make it visible to the rest of the system.
1390 * We dont wake it up yet.
1391 */
1392 p->group_leader = p;
Oleg Nesterov47e65322006-03-28 16:11:25 -08001393 INIT_LIST_HEAD(&p->thread_group);
Al Viro158e1642012-06-27 09:24:13 +04001394 p->task_works = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395
Paul Menageb4f48b62007-10-18 23:39:33 -07001396 /* Now that the task is set up, run cgroup callbacks if
1397 * necessary. We need to run them before the task is visible
1398 * on the tasklist. */
1399 cgroup_fork_callbacks(p);
1400 cgroup_callbacks_done = 1;
1401
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402 /* Need tasklist lock for parent etc handling! */
1403 write_lock_irq(&tasklist_lock);
1404
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405 /* CLONE_PARENT re-uses the old parent */
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01001406 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 p->real_parent = current->real_parent;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01001408 p->parent_exec_id = current->parent_exec_id;
1409 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 p->real_parent = current;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01001411 p->parent_exec_id = current->self_exec_id;
1412 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413
Oleg Nesterov3f17da62006-02-15 22:13:24 +03001414 spin_lock(&current->sighand->siglock);
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08001415
1416 /*
1417 * Process group and session signals need to be delivered to just the
1418 * parent before the fork or both the parent and the child after the
1419 * fork. Restart if a signal comes in before we add the new process to
1420 * it's process group.
1421 * A fatal signal pending means that current will exit, so the new
1422 * thread can't slip out of an OOM kill (or normal SIGKILL).
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001423 */
Daniel Walker23ff4442007-10-18 03:06:07 -07001424 recalc_sigpending();
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08001425 if (signal_pending(current)) {
1426 spin_unlock(&current->sighand->siglock);
1427 write_unlock_irq(&tasklist_lock);
1428 retval = -ERESTARTNOINTR;
Steven Rostedtf7e8b612009-06-02 16:39:48 -04001429 goto bad_fork_free_pid;
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08001430 }
1431
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432 if (clone_flags & CLONE_THREAD) {
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001433 current->signal->nr_threads++;
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07001434 atomic_inc(&current->signal->live);
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001435 atomic_inc(&current->signal->sigcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 p->group_leader = current->group_leader;
Oleg Nesterov47e65322006-03-28 16:11:25 -08001437 list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438 }
1439
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001440 if (likely(p->pid)) {
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001441 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001443 if (thread_group_leader(p)) {
Eric W. Biederman45a68622011-03-23 16:43:12 -07001444 if (is_child_reaper(pid))
Pavel Emelyanov30e49c22007-10-18 23:40:10 -07001445 p->nsproxy->pid_ns->child_reaper = p;
Oleg Nesterovc97d9892006-03-28 16:11:06 -08001446
Oleg Nesterovfea9d172008-02-08 04:19:19 -08001447 p->signal->leader_pid = pid;
Alan Cox9c9f4de2008-10-13 10:37:26 +01001448 p->signal->tty = tty_kref_get(current->signal->tty);
Eric W. Biederman5cd17562007-12-04 23:45:04 -08001449 attach_pid(p, PIDTYPE_PGID, task_pgrp(current));
1450 attach_pid(p, PIDTYPE_SID, task_session(current));
Oleg Nesterov9cd80bb2009-12-17 15:27:15 -08001451 list_add_tail(&p->sibling, &p->real_parent->children);
Eric W. Biederman5e85d4a2006-04-18 22:20:16 -07001452 list_add_tail_rcu(&p->tasks, &init_task.tasks);
Christoph Lameter909ea962010-12-08 16:22:55 +01001453 __this_cpu_inc(process_counts);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001454 }
Sukadev Bhattiprolu85868992007-05-10 22:23:03 -07001455 attach_pid(p, PIDTYPE_PID, pid);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001456 nr_threads++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457 }
1458
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459 total_forks++;
Oleg Nesterov3f17da62006-02-15 22:13:24 +03001460 spin_unlock(&current->sighand->siglock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461 write_unlock_irq(&tasklist_lock);
Andrew Mortonc13cf852005-11-28 13:43:48 -08001462 proc_fork_connector(p);
Paul Menage817929e2007-10-18 23:39:36 -07001463 cgroup_post_fork(p);
Ben Blum4714d1d2011-05-26 16:25:18 -07001464 if (clone_flags & CLONE_THREAD)
Tejun Heo257058a2011-12-12 18:12:21 -08001465 threadgroup_change_end(current);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001466 perf_event_fork(p);
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -08001467
1468 trace_task_newtask(p, clone_flags);
1469
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470 return p;
1471
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07001472bad_fork_free_pid:
1473 if (pid != &init_struct_pid)
1474 free_pid(pid);
Jens Axboefd0928d2008-01-24 08:52:45 +01001475bad_fork_cleanup_io:
Louis Rillingb69f2292009-12-04 14:52:42 +01001476 if (p->io_context)
1477 exit_io_context(p);
Serge E. Hallynab516012006-10-02 02:18:06 -07001478bad_fork_cleanup_namespaces:
Mike Galbraith5e2bf012012-05-10 13:01:45 -07001479 if (unlikely(clone_flags & CLONE_NEWPID))
1480 pid_ns_release_proc(p->nsproxy->pid_ns);
Linus Torvalds444f3782007-01-30 13:35:18 -08001481 exit_task_namespaces(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001482bad_fork_cleanup_mm:
David Rientjesc9f01242011-10-31 17:07:15 -07001483 if (p->mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 mmput(p->mm);
1485bad_fork_cleanup_signal:
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07001486 if (!(clone_flags & CLONE_THREAD))
Mike Galbraith1c5354d2011-01-05 11:16:04 +01001487 free_signal_struct(p->signal);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488bad_fork_cleanup_sighand:
Oleg Nesterova7e53282006-03-28 16:11:27 -08001489 __cleanup_sighand(p->sighand);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490bad_fork_cleanup_fs:
1491 exit_fs(p); /* blocking */
1492bad_fork_cleanup_files:
1493 exit_files(p); /* blocking */
1494bad_fork_cleanup_semundo:
1495 exit_sem(p);
1496bad_fork_cleanup_audit:
1497 audit_free(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498bad_fork_cleanup_policy:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001499 perf_event_free_task(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500#ifdef CONFIG_NUMA
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07001501 mpol_put(p->mempolicy);
Paul Menageb4f48b62007-10-18 23:39:33 -07001502bad_fork_cleanup_cgroup:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503#endif
Ben Blum4714d1d2011-05-26 16:25:18 -07001504 if (clone_flags & CLONE_THREAD)
Tejun Heo257058a2011-12-12 18:12:21 -08001505 threadgroup_change_end(current);
Paul Menageb4f48b62007-10-18 23:39:33 -07001506 cgroup_exit(p, cgroup_callbacks_done);
Shailabh Nagar35df17c2006-08-31 21:27:38 -07001507 delayacct_tsk_free(p);
Al Viroa1261f542005-11-13 16:06:55 -08001508 module_put(task_thread_info(p)->exec_domain->module);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509bad_fork_cleanup_count:
David Howellsd84f4f92008-11-14 10:39:23 +11001510 atomic_dec(&p->cred->user->processes);
David Howellse0e81732009-09-02 09:13:40 +01001511 exit_creds(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512bad_fork_free:
1513 free_task(p);
Oleg Nesterovfe7d37d2006-01-08 01:04:02 -08001514fork_out:
1515 return ERR_PTR(retval);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516}
1517
Adrian Bunk6b2fb3c2008-02-06 01:37:55 -08001518noinline struct pt_regs * __cpuinit __attribute__((weak)) idle_regs(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519{
1520 memset(regs, 0, sizeof(struct pt_regs));
1521 return regs;
1522}
1523
Oleg Nesterovf106eee2010-05-26 14:44:11 -07001524static inline void init_idle_pids(struct pid_link *links)
1525{
1526 enum pid_type type;
1527
1528 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
1529 INIT_HLIST_NODE(&links[type].node); /* not really needed */
1530 links[type].pid = &init_struct_pid;
1531 }
1532}
1533
Al Viro9abcf402007-02-01 13:52:48 +00001534struct task_struct * __cpuinit fork_idle(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535{
Ingo Molnar36c8b582006-07-03 00:25:41 -07001536 struct task_struct *task;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537 struct pt_regs regs;
1538
Pavel Emelyanov30e49c22007-10-18 23:40:10 -07001539 task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL,
Roland McGrath09a05392008-07-25 19:45:47 -07001540 &init_struct_pid, 0);
Oleg Nesterovf106eee2010-05-26 14:44:11 -07001541 if (!IS_ERR(task)) {
1542 init_idle_pids(task->pids);
Akinobu Mita753ca4f2006-11-25 11:09:34 -08001543 init_idle(task, cpu);
Oleg Nesterovf106eee2010-05-26 14:44:11 -07001544 }
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08001545
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 return task;
1547}
1548
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549/*
1550 * Ok, this is the main fork-routine.
1551 *
1552 * It copies the process, and if successful kick-starts
1553 * it and waits for it to finish using the VM if required.
1554 */
1555long do_fork(unsigned long clone_flags,
1556 unsigned long stack_start,
1557 struct pt_regs *regs,
1558 unsigned long stack_size,
1559 int __user *parent_tidptr,
1560 int __user *child_tidptr)
1561{
1562 struct task_struct *p;
1563 int trace = 0;
Eric W. Biederman92476d72006-03-31 02:31:42 -08001564 long nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565
Andrew Mortonbdff7462008-02-04 22:27:22 -08001566 /*
Serge Hallyn18b6e042008-10-15 16:38:45 -05001567 * Do some preliminary argument and permissions checking before we
1568 * actually start allocating stuff
1569 */
1570 if (clone_flags & CLONE_NEWUSER) {
1571 if (clone_flags & CLONE_THREAD)
1572 return -EINVAL;
1573 /* hopefully this check will go away when userns support is
1574 * complete
1575 */
Serge E. Hallyn7657d902008-12-03 13:17:33 -06001576 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SETUID) ||
1577 !capable(CAP_SETGID))
Serge Hallyn18b6e042008-10-15 16:38:45 -05001578 return -EPERM;
1579 }
1580
1581 /*
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001582 * Determine whether and which event to report to ptracer. When
1583 * called from kernel_thread or CLONE_UNTRACED is explicitly
1584 * requested, no event is reported; otherwise, report if the event
1585 * for the type of forking is enabled.
Roland McGrath09a05392008-07-25 19:45:47 -07001586 */
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001587 if (likely(user_mode(regs)) && !(clone_flags & CLONE_UNTRACED)) {
1588 if (clone_flags & CLONE_VFORK)
1589 trace = PTRACE_EVENT_VFORK;
1590 else if ((clone_flags & CSIGNAL) != SIGCHLD)
1591 trace = PTRACE_EVENT_CLONE;
1592 else
1593 trace = PTRACE_EVENT_FORK;
1594
1595 if (likely(!ptrace_event_enabled(current, trace)))
1596 trace = 0;
1597 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598
Sukadev Bhattiprolua6f5e062007-10-18 23:39:53 -07001599 p = copy_process(clone_flags, stack_start, regs, stack_size,
Roland McGrath09a05392008-07-25 19:45:47 -07001600 child_tidptr, NULL, trace);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 /*
1602 * Do this prior waking up the new thread - the thread pointer
1603 * might get invalid after that point, if the thread exits quickly.
1604 */
1605 if (!IS_ERR(p)) {
1606 struct completion vfork;
1607
Mathieu Desnoyers0a16b602008-07-18 12:16:17 -04001608 trace_sched_process_fork(current, p);
1609
Pavel Emelyanov6c5f3e72008-02-08 04:19:20 -08001610 nr = task_pid_vnr(p);
Pavel Emelyanov30e49c22007-10-18 23:40:10 -07001611
1612 if (clone_flags & CLONE_PARENT_SETTID)
1613 put_user(nr, parent_tidptr);
Sukadev Bhattiprolua6f5e062007-10-18 23:39:53 -07001614
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615 if (clone_flags & CLONE_VFORK) {
1616 p->vfork_done = &vfork;
1617 init_completion(&vfork);
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001618 get_task_struct(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619 }
1620
Samir Bellabes3e51e3e2011-05-11 18:18:05 +02001621 wake_up_new_task(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622
Tejun Heo4b9d33e2011-06-17 16:50:38 +02001623 /* forking complete and child started to run, tell ptracer */
1624 if (unlikely(trace))
1625 ptrace_event(trace, nr);
Roland McGrath09a05392008-07-25 19:45:47 -07001626
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627 if (clone_flags & CLONE_VFORK) {
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001628 if (!wait_for_vfork_done(p, &vfork))
1629 ptrace_event(PTRACE_EVENT_VFORK_DONE, nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 }
1631 } else {
Eric W. Biederman92476d72006-03-31 02:31:42 -08001632 nr = PTR_ERR(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 }
Eric W. Biederman92476d72006-03-31 02:31:42 -08001634 return nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635}
1636
Ravikiran G Thirumalai5fd63b32006-01-11 22:46:15 +01001637#ifndef ARCH_MIN_MMSTRUCT_ALIGN
1638#define ARCH_MIN_MMSTRUCT_ALIGN 0
1639#endif
1640
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07001641static void sighand_ctor(void *data)
Oleg Nesterovaa1757f2006-03-28 16:11:12 -08001642{
1643 struct sighand_struct *sighand = data;
1644
Christoph Lametera35afb82007-05-16 22:10:57 -07001645 spin_lock_init(&sighand->siglock);
Davide Libenzib8fceee2007-09-20 12:40:16 -07001646 init_waitqueue_head(&sighand->signalfd_wqh);
Oleg Nesterovaa1757f2006-03-28 16:11:12 -08001647}
1648
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649void __init proc_caches_init(void)
1650{
1651 sighand_cachep = kmem_cache_create("sighand_cache",
1652 sizeof(struct sighand_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001653 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
1654 SLAB_NOTRACK, sighand_ctor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655 signal_cachep = kmem_cache_create("signal_cache",
1656 sizeof(struct signal_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001657 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
Paul Mundt20c2df82007-07-20 10:11:58 +09001658 files_cachep = kmem_cache_create("files_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659 sizeof(struct files_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001660 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
Paul Mundt20c2df82007-07-20 10:11:58 +09001661 fs_cachep = kmem_cache_create("fs_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662 sizeof(struct fs_struct), 0,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001663 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
Linus Torvalds6345d242011-05-29 11:32:28 -07001664 /*
1665 * FIXME! The "sizeof(struct mm_struct)" currently includes the
1666 * whole struct cpumask for the OFFSTACK case. We could change
1667 * this to *only* allocate as much of it as required by the
1668 * maximum number of CPU's we can ever have. The cpumask_allocation
1669 * is at the end of the structure, exactly for that reason.
1670 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671 mm_cachep = kmem_cache_create("mm_struct",
Ravikiran G Thirumalai5fd63b32006-01-11 22:46:15 +01001672 sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
Vegard Nossum2dff4402008-05-31 15:56:17 +02001673 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
David Howells33e5d7692009-04-02 16:56:32 -07001674 vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC);
David Howells8feae132009-01-08 12:04:47 +00001675 mmap_init();
Al Viro66577192011-06-28 15:41:10 -04001676 nsproxy_cache_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677}
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001678
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001679/*
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001680 * Check constraints on flags passed to the unshare system call.
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001681 */
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001682static int check_unshare_flags(unsigned long unshare_flags)
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001683{
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001684 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
1685 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
1686 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET))
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001687 return -EINVAL;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001688 /*
1689 * Not implemented, but pretend it works if there is nothing to
1690 * unshare. Note that unsharing CLONE_THREAD or CLONE_SIGHAND
1691 * needs to unshare vm.
1692 */
1693 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
1694 /* FIXME: get_task_mm() increments ->mm_users */
1695 if (atomic_read(&current->mm->mm_users) > 1)
1696 return -EINVAL;
1697 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001698
1699 return 0;
1700}
1701
1702/*
JANAK DESAI99d14192006-02-07 12:58:59 -08001703 * Unshare the filesystem structure if it is being shared
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001704 */
1705static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
1706{
1707 struct fs_struct *fs = current->fs;
1708
Al Viro498052b2009-03-30 07:20:30 -04001709 if (!(unshare_flags & CLONE_FS) || !fs)
1710 return 0;
1711
1712 /* don't need lock here; in the worst case we'll do useless copy */
1713 if (fs->users == 1)
1714 return 0;
1715
1716 *new_fsp = copy_fs_struct(fs);
1717 if (!*new_fsp)
1718 return -ENOMEM;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001719
1720 return 0;
1721}
1722
1723/*
JANAK DESAIa016f332006-02-07 12:59:02 -08001724 * Unshare file descriptor table if it is being shared
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001725 */
1726static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
1727{
1728 struct files_struct *fd = current->files;
JANAK DESAIa016f332006-02-07 12:59:02 -08001729 int error = 0;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001730
1731 if ((unshare_flags & CLONE_FILES) &&
JANAK DESAIa016f332006-02-07 12:59:02 -08001732 (fd && atomic_read(&fd->count) > 1)) {
1733 *new_fdp = dup_fd(fd, &error);
1734 if (!*new_fdp)
1735 return error;
1736 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001737
1738 return 0;
1739}
1740
1741/*
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001742 * unshare allows a process to 'unshare' part of the process
1743 * context which was originally shared using clone. copy_*
1744 * functions used by do_fork() cannot be used here directly
1745 * because they modify an inactive task_struct that is being
1746 * constructed. Here we are modifying the current, active,
1747 * task_struct.
1748 */
Heiko Carstens6559eed82009-01-14 14:14:32 +01001749SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001750{
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001751 struct fs_struct *fs, *new_fs = NULL;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001752 struct files_struct *fd, *new_fd = NULL;
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07001753 struct nsproxy *new_nsproxy = NULL;
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001754 int do_sysvsem = 0;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001755 int err;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001756
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001757 err = check_unshare_flags(unshare_flags);
1758 if (err)
Eric W. Biederman06f9d4f2006-03-22 00:07:40 -08001759 goto bad_unshare_out;
1760
Manfred Spraul6013f672008-04-29 01:00:59 -07001761 /*
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001762 * If unsharing namespace, must also unshare filesystem information.
1763 */
1764 if (unshare_flags & CLONE_NEWNS)
1765 unshare_flags |= CLONE_FS;
1766 /*
Manfred Spraul6013f672008-04-29 01:00:59 -07001767 * CLONE_NEWIPC must also detach from the undolist: after switching
1768 * to a new ipc namespace, the semaphore arrays from the old
1769 * namespace are unreachable.
1770 */
1771 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001772 do_sysvsem = 1;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001773 err = unshare_fs(unshare_flags, &new_fs);
1774 if (err)
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001775 goto bad_unshare_out;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001776 err = unshare_fd(unshare_flags, &new_fd);
1777 if (err)
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001778 goto bad_unshare_cleanup_fs;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001779 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, new_fs);
1780 if (err)
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001781 goto bad_unshare_cleanup_fd;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001782
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07001783 if (new_fs || new_fd || do_sysvsem || new_nsproxy) {
Manfred Spraul9edff4a2008-04-29 01:00:57 -07001784 if (do_sysvsem) {
1785 /*
1786 * CLONE_SYSVSEM is equivalent to sys_exit().
1787 */
1788 exit_sem(current);
1789 }
Serge E. Hallynab516012006-10-02 02:18:06 -07001790
Serge Hallync0b2fc32006-10-02 02:18:18 -07001791 if (new_nsproxy) {
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07001792 switch_task_namespaces(current, new_nsproxy);
1793 new_nsproxy = NULL;
Serge Hallync0b2fc32006-10-02 02:18:18 -07001794 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001795
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07001796 task_lock(current);
1797
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001798 if (new_fs) {
1799 fs = current->fs;
Nick Piggin2a4419b2010-08-18 04:37:33 +10001800 spin_lock(&fs->lock);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001801 current->fs = new_fs;
Al Viro498052b2009-03-30 07:20:30 -04001802 if (--fs->users)
1803 new_fs = NULL;
1804 else
1805 new_fs = fs;
Nick Piggin2a4419b2010-08-18 04:37:33 +10001806 spin_unlock(&fs->lock);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001807 }
1808
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001809 if (new_fd) {
1810 fd = current->files;
1811 current->files = new_fd;
1812 new_fd = fd;
1813 }
1814
1815 task_unlock(current);
1816 }
1817
Serge Hallync0b2fc32006-10-02 02:18:18 -07001818 if (new_nsproxy)
Linus Torvalds444f3782007-01-30 13:35:18 -08001819 put_nsproxy(new_nsproxy);
Serge Hallync0b2fc32006-10-02 02:18:18 -07001820
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001821bad_unshare_cleanup_fd:
1822 if (new_fd)
1823 put_files_struct(new_fd);
1824
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001825bad_unshare_cleanup_fs:
1826 if (new_fs)
Al Viro498052b2009-03-30 07:20:30 -04001827 free_fs_struct(new_fs);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001828
JANAK DESAIcf2e3402006-02-07 12:58:58 -08001829bad_unshare_out:
1830 return err;
1831}
Al Viro3b125382008-04-22 05:31:30 -04001832
1833/*
1834 * Helper to unshare the files of the current task.
1835 * We don't want to expose copy_files internals to
1836 * the exec layer of the kernel.
1837 */
1838
1839int unshare_files(struct files_struct **displaced)
1840{
1841 struct task_struct *task = current;
Al Viro50704512008-04-26 05:25:00 +01001842 struct files_struct *copy = NULL;
Al Viro3b125382008-04-22 05:31:30 -04001843 int error;
1844
1845 error = unshare_fd(CLONE_FILES, &copy);
1846 if (error || !copy) {
1847 *displaced = NULL;
1848 return error;
1849 }
1850 *displaced = task->files;
1851 task_lock(task);
1852 task->files = copy;
1853 task_unlock(task);
1854 return 0;
1855}